diff --git a/.github/workflows/release_biosimulators.yml b/.github/workflows/release_biosimulators.yml new file mode 100644 index 0000000000..2f0479e3cd --- /dev/null +++ b/.github/workflows/release_biosimulators.yml @@ -0,0 +1,32 @@ +name: Release to BioSimulators + +on: + release: + types: + - published + +jobs: + updateCliAndDockerImage: + name: Build and release downstream command-line interface and Docker image + runs-on: ubuntu-latest + env: + # Owner/repository-id for the GitHub repository for the downstream command-line interface and Docker image + DOWNSTREAM_REPOSITORY: biosimulators/Biosimulators_AMICI + + # Username/token to use the GitHub API to trigger an action on the GitHub repository for the downstream + # command-line interface and Docker image. Tokens can be generated at https://github.com/settings/tokens. + # The token should have the scope `repo` + GH_ISSUE_USERNAME: ${{ secrets.BIOSIMULATORS_USERNAME }} + GH_ISSUE_TOKEN: ${{ secrets.BIOSIMULATORS_TOKEN }} + steps: + - name: Trigger GitHub action that will build and release the downstream command-line interface and Docker image + run: | + PACKAGE_VERSION="${GITHUB_REF/refs\/tags\/v/}" + WORKFLOW_FILE=ci.yml + + curl \ + -X POST \ + -u ${GH_ISSUE_USERNAME}:${GH_ISSUE_TOKEN} \ + -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos/${DOWNSTREAM_REPOSITORY}/actions/workflows/${WORKFLOW_FILE}/dispatches \ + -d "{\"inputs\": {\"simulatorVersion\": \"${PACKAGE_VERSION}\", \"simulatorVersionLatest\": \"true\"}}" diff --git a/.github/workflows/test_performance.yml b/.github/workflows/test_performance.yml index 4e78e6edea..a2f463b2f4 100644 --- a/.github/workflows/test_performance.yml +++ b/.github/workflows/test_performance.yml @@ -4,7 +4,7 @@ on: branches: - develop - master - - speedup_reaction_ids + - compile_without_optimization pull_request: branches: @@ -50,12 +50,7 @@ jobs: # import test model - name: Import test model run: | - cd CS_Signalling_ERBB_RAS_AKT \ - && check_time.sh \ - petab_import amici_import_petab -v \ - -n 'CS_Signalling_ERBB_RAS_AKT_petab' \ - -y 'FroehlichKes2018/PEtab/FroehlichKes2018.yaml' \ - --no-compile + check_time.sh petab_import python tests/performance/test.py import - name: "Upload artifact: CS_Signalling_ERBB_RAS_AKT_petab" uses: actions/upload-artifact@v1 @@ -65,29 +60,60 @@ jobs: # install model package - name: Install test model - run: | - check_time.sh install_model \ - python3 CS_Signalling_ERBB_RAS_AKT/CS_Signalling_ERBB_RAS_AKT_petab/setup.py install --user + run: > + check_time.sh install_model tests/performance/test.py compile; + for opt in O0 O1 O2; + do + check_time.sh install_model_${opt} tests/performance/test.py compile_${opt}; + done # run simulations - name: forward_simulation - run: | - check_time.sh forward_simulation tests/performance/test.py forward_simulation + run: > + check_time.sh forward_simulation tests/performance/test.py forward_simulation; + for opt in O0 O1 O2; + do + check_time.sh forward_simulation tests/performance/test.py forward_simulation_${opt}; + done - name: forward_sensitivities - run: | - check_time.sh forward_sensitivities tests/performance/test.py forward_sensitivities + run: > + check_time.sh forward_sensitivities tests/performance/test.py forward_sensitivities; + for opt in O0 O1 O2; + do + check_time.sh forward_sensitivities tests/performance/test.py forward_sensitivities_${opt}; + done - name: adjoint_sensitivities - run: | - check_time.sh adjoint_sensitivities tests/performance/test.py adjoint_sensitivities + run: > + check_time.sh adjoint_sensitivities tests/performance/test.py adjoint_sensitivities; + for opt in O0 O1 O2; + do + check_time.sh adjoint_sensitivities tests/performance/test.py adjoint_sensitivities_${opt}; + done - name: forward_simulation_non_optimal_parameters run: | - check_time.sh forward_simulation_non_optimal_parameters tests/performance/test.py forward_simulation_non_optimal_parameters + check_time.sh forward_simulation_non_optimal_parameters tests/performance/test.py forward_simulation_non_optimal_parameters; + for opt in O0 O1 O2; + do + check_time.sh forward_simulation_non_optimal_parameters tests/performance/test.py forward_simulation_non_optimal_parameters_${opt}; + done - name: adjoint_sensitivities_non_optimal_parameters run: | - check_time.sh adjoint_sensitivities_non_optimal_parameters tests/performance/test.py adjoint_sensitivities_non_optimal_parameters + check_time.sh adjoint_sensitivities_non_optimal_parameters tests/performance/test.py adjoint_sensitivities_non_optimal_parameters; + for opt in O0 O1 O2; + do + check_time.sh adjoint_sensitivities_non_optimal_parameters tests/performance/test.py adjoint_sensitivities_non_optimal_parameters_${opt}; + done - name: forward_steadystate_sensitivities_non_optimal_parameters run: | - check_time.sh forward_steadystate_sensitivities_non_optimal_parameters tests/performance/test.py forward_steadystate_sensitivities_non_optimal_parameters + check_time.sh forward_steadystate_sensitivities_non_optimal_parameters tests/performance/test.py forward_steadystate_sensitivities_non_optimal_parameters; + for opt in O0 O1 O2; + do + check_time.sh forward_steadystate_sensitivities_non_optimal_parameters tests/performance/test.py forward_steadystate_sensitivities_non_optimal_parameters_${opt}; + done - name: adjoint_steadystate_sensitivities_non_optimal_parameters run: | - check_time.sh adjoint_steadystate_sensitivities_non_optimal_parameters tests/performance/test.py adjoint_steadystate_sensitivities_non_optimal_parameters + check_time.sh adjoint_steadystate_sensitivities_non_optimal_parameters tests/performance/test.py adjoint_steadystate_sensitivities_non_optimal_parameters; + for opt in O0 O1 O2; + do + check_time.sh adjoint_steadystate_sensitivities_non_optimal_parameters tests/performance/test.py adjoint_steadystate_sensitivities_non_optimal_parameters_${opt}; + done diff --git a/.github/workflows/test_sbml_semantic_test_suite.yml b/.github/workflows/test_sbml_semantic_test_suite.yml index d47a58631a..7dad35396d 100644 --- a/.github/workflows/test_sbml_semantic_test_suite.yml +++ b/.github/workflows/test_sbml_semantic_test_suite.yml @@ -4,6 +4,7 @@ on: branches: - develop - master + - release** pull_request: paths: - .github/workflows/test_sbml_semantic_test_suite.yml diff --git a/.gitignore b/.gitignore index 6871f44482..44053581ec 100644 --- a/.gitignore +++ b/.gitignore @@ -193,3 +193,4 @@ coverage_SBMLSuite.xml Benchmark-Models-PEtab/ +CS_Signalling_ERBB_RAS_AKT/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 875c9603a8..f3e981e425 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # Changelog -## v0.11.14 (2021-03-16) +## v0.X Series + +### v0.11.15 (2021-03-31) + +Fixes: +* Fixed initial state sensitivities in adjoint preequilibration (#1468) +* Fixed various model import / parameter mapping issues (#1469, #1473, #1475) + +New: +* New AMICI releases will automatically trigger releases at + https://biosimulators.org/simulators/amici/latest +* Transparent logo + +### v0.11.14 (2021-03-16) New features: * **Python import now supports SBML Events** (#1443) @@ -33,7 +46,7 @@ Misc: * Micro-optimize SUNMatrixWrapper::transpose (#1439) * Remove constant triggers from roots in Heaviside (#1440) -## v0.11.13 (2021-02-20) +### v0.11.13 (2021-02-20) Breaking changes: * AMICI requires Python>=3.7 @@ -69,7 +82,7 @@ Other: * Various minor CI improvements * ... -## v0.11.12 (2021-01-26) +### v0.11.12 (2021-01-26) Features: * Add expression IDs and names to generated models (#1374) @@ -83,9 +96,9 @@ Docs: * Update how-to-cite (#1378) -## v0.11.11 (2020-12-15) +### v0.11.11 (2020-12-15) -### Python +#### Python * Restore support for species references (#1358) * Add support for noise models in pysb (#1360) * Proper handling of discontinuities in the ODE rhs (#1352) @@ -93,42 +106,42 @@ Docs: * Extend mathml function support, particularly for numerical arguments (#1357) * Restore support for sympy 1.6 (#1356) -### C++ +#### C++ * Fix some compiler related warnings (#1349, #1362 ) * Fix a rare segfault for adjoint sensitivities (#1351) -### CI +#### CI * Move windows tests to GHA (#1354) * Pin breathe to 4.24.1 -### Docker +#### Docker * Update ubuntu to 20.04 -## v0.11.10 (2020-11-30) +### v0.11.10 (2020-11-30) Bugfix release that restores compatibility with sympy 1.7 -## v0.11.9 (2020-11-29) +### v0.11.9 (2020-11-29) -### Python +#### Python * General improvements to SBML import (#1312, #1316, #1315, #1322 , #1324 #1333, #1329) * Small bugfixes and improvements (#1321 ) * Improve derivative computation for instances of `power` (#1320 ) -### C++ +#### C++ * Fix FIM and residual computation for models with parameter dependent sigma. (#1338) * Disable chi2/residual/FIM computation for non-gaussian objective functions. (#1338) * Bugfix for integration failure during adjoint solve (#1327) -### Doc +#### Doc * Update references (#1331, #1336) -### CI +#### CI * Update OpenBLAS for windows (#1334) -## v0.11.8 (2020-10-21) +### v0.11.8 (2020-10-21) -### Python +#### Python * Fix pysb-petab support (#1288) * Fix ExpData constructor overloading (#1299) * Fix support for positivity enforcement (#1306) @@ -136,21 +149,21 @@ Bugfix release that restores compatibility with sympy 1.7 * Improve model generation for models with many parameters (#1300) * Add support for PEtab based synthetic data generation (#1283) -### C++ +#### C++ * Make HDF5 an optional dependency (#1285) -### Doc +#### Doc * General Improvements to Documentation (#1289, #1291, #1292, #1293, #1294, #1286, #1277, #1281) -### CI +#### CI * Add python 3.9 support test (#1282) * Allow manual triggering of GitHub actions (#1287) * Remove appveyor config (#1295) * Update GHA env and path management (#1302) -## v0.11.7 (2020-09-22) +### v0.11.7 (2020-09-22) -### Python +#### Python * Improve and extend available objective functions (#1235) * Fix processing of compartment definitions (#1223) * Fix replacement of reserved symbols (#1265) @@ -158,101 +171,101 @@ Bugfix release that restores compatibility with sympy 1.7 * Fix duplicate running of swig (#1216) * Overload python interface functions for amici.{Model,Solver,ExpData} and amici.{Model,Solver,ExpData}Ptr (#1271) -### C++ +#### C++ * Fix and extend use of sparse matrix operations (#1230, #1240, #1244, #1247, #1271) * **Fix application of maximal number of steps**, MaxNumStep parameter now limit total number of steps, not number of steps between output times. (#1267) -### Doc +#### Doc * Move all Documentation to RTD (#1229, #1241) * General Improvements to Documentation (#1225, #1227, #1219, #1228, #1232, #1233, #1234, #1237, #1238, #1239, #1243, #1253, #1255, #1262) -### CI +#### CI * Better check for doc building (#1226) * Add more gradient checks (#1213) * Update GHA to Ubuntu 20.04 (#1268) -## v0.11.6 (2020-08-20) +### v0.11.6 (2020-08-20) -### Python +#### Python * Bugfix for piecewise functions (#1199) * Refactor swigging - generate one single wrapper (#1213) -### C++ +#### C++ * Fix warnings: account for zero indexing in nan/inf error (#1112) -### Doc +#### Doc * Update Windows build instructions (#1200, #1202) * Update README: Projects using AMICI (#1209) * Add CODE_OF_CONDUCT.md (#1210) * Update documentation for Python interface (#1208) -### CI +#### CI * Create sdist on GHA using swig4.0.1 (#1204) (Fixing broken pypi package) * Fix links after repository move * Speed-up swig build: disable all languages except python (#1211) * Fix doc generation on readthedocs (#1196) -## v0.11.5 (2020-08-07) +### v0.11.5 (2020-08-07) -### General +#### General * Move repo to new organization (#1193) * Update Bibliography -### Python +#### Python * Fix bug for energyPySB models (#1191) -### CI +#### CI * Fix release deployment (#1189) -## v0.11.4 (2020-08-06) +### v0.11.4 (2020-08-06) -### Python +#### Python * Skip unnecessary expressions in pysb models (#1185) * MSVC compiler support (this time for real... #1152) -### CI +#### CI * Implement MSVC tests (#1152) * Rename and group GitHub actions (#1186) * Fix release deployment (#1186) -## v0.11.3 (2020-08-06) +### v0.11.3 (2020-08-06) -### Python +#### Python * Fix simplification for pysb models (#1168) * Pass verbosity flags to pysb network generation (#1173) * Enable experimental pysb-petab support (#1175) * Add installation instructions for Fedora (#1177) * Implement support for SBML rate-references (#1180) -### C++ +#### C++ * Refactoring (#1162, #1163) -### CI +#### CI * Move majority of tests to Github Actions (#1166, #1160) * Improve reporting of skipped tests in SBML testsuite (#1183) -## v0.11.2 (2020-07-17) +### v0.11.2 (2020-07-17) -### Python +#### Python * Speed up model import, compilation (#1123, #1112) * Improve/Add steady-state solver documentation (#1102) * Improve extension import (#1141) * Bugfixes SBML import (#1135, #1134, #1145, #1154) * Fixed issue that prevented simplification (#1158) -### C++ +#### C++ * Bugfixes (#1121, #1125, #1131, #1132, #1136) * Enable openMP by default (#1118) * Improve memoy footprint for simulations with replicates (#1153) * Improve steady-state solver and add option to to adjoint-steadystate hybrid (#1143, #1099, #1129, #1146) -### CI +#### CI * Store build artifacts from github actions (#1138) -## v0.11.1 (2020-06-05) +### v0.11.1 (2020-06-05) -### Python +#### Python * Upgrade to sympy 1.6.0, which is now required minimum version (#1098, #1103) * Speed up model import * Speed-up computation of sx0, reduce file size (#1109) @@ -265,21 +278,21 @@ Bugfix release that restores compatibility with sympy 1.7 * Fix MathML conversion (#1086) * Fix deepcopy of SymPy objects (#1091) -### Matlab +#### Matlab * handle empty rdata->{pre|post}eq_numlinsteps (Closes #1113), which previously made the matlab interface unusable * Fix generation of compileMexFile.m for matlab compilation of python code (#1115) -### C++ +#### C++ * Reduce memory requirements and speedup compilation of large models (#1105) * Place generated code into own namespace (#937) (#1112) * Fix several msvc compiler warnings (#1116) (Note that MSVC support is still experimental) **breaking change for users of C++ interface** * Fix swig warning: ensure base class ContextManager is known before use (Fixes #1092) (#1101) -### CI +#### CI * Don't install/run valgrind on travis CI (done with github actions… (#1111) -## v0.11.0 (2020-05-10) +### v0.11.0 (2020-05-10) Python: @@ -304,7 +317,7 @@ CI: - Move from Codacy to Sonarcloud (#1065) - Run SBML Testsuite when appropriate (#1058) -## v0.10.21 (2020-04-04) +### v0.10.21 (2020-04-04) Library: * Fix: Handle paths with blanks in build scripts @@ -317,7 +330,7 @@ CI: * Fix: benchmark problem test should fail on missing files (Closes #1015) -## v0.10.20 (2020-03-18) +### v0.10.20 (2020-03-18) * Fixed (re)initialization of sensitivities if ExpData::fixedParametersPreequilibration is set (#994) * Fixed sensitivities for parameters in sigma expressions for Python/SBML in case provided expression was not just a single parameter ID @@ -342,7 +355,7 @@ CI: * Removed mention of 'mex' in warning/error ids (#968) * More informative errors on SWIG interface import failures (#959) -## v0.10.19 (2020-02-13) +### v0.10.19 (2020-02-13) Python: * Fix logo display on pypi @@ -351,7 +364,7 @@ Python: Matlab: * Fix compilation errors due to switch to C++14 -## v0.10.18 (2020-02-11) +### v0.10.18 (2020-02-11) General: * AMICI now comes with a logo @@ -378,7 +391,7 @@ CI: ... and various minor fixes/updates -## v0.10.17 (2020-01-15) +### v0.10.17 (2020-01-15) - **added python 3.8 support, dropped python 3.6 support** (#898) - Added logging functionality (#900) @@ -388,7 +401,7 @@ CI: - Improved petab support (#886, #888, #891) - CI related fixes (#865, #896) -## v0.10.16 (2019-12-11) +### v0.10.16 (2019-12-11) * **Sparsify dwdp to reduce computation time for adjoints (#858)** * Fix(matlab) update example name example_dae_events->example_calvetti (Closes #866) @@ -397,13 +410,13 @@ CI: * Fix pysb_import (fixes #878) -## v0.10.15 (2019-12-03) +### v0.10.15 (2019-12-03) Bugfix release due to incorrect sensitivities w.r.t. sigmas introduced in 0.10.14. No other changes. -## v0.10.14 (2019-12-02) +### v0.10.14 (2019-12-02) **NOTE: For Python-imported SBML-models this release may compute incorrect sensitivities w.r.t. sigma. Bug introduced in 0.10.14, fixed in 0.10.15.** @@ -436,7 +449,7 @@ Misc: * Update documentation and FAQ for CBLAS requirement and others * Update reference list -## v0.10.13 (2019-10-09) +### v0.10.13 (2019-10-09) * BREAKING CHANGE: Renaming {get|set}tNewtonPreequilibration to {get|set}Preequilibration (Closes #720) * Make wurlitzer non-optional requirement for AMICI python package (Fixes missing AMICI errors when running from jupyter notebooks) @@ -450,7 +463,7 @@ Misc: * Update workflow figure to include PySB (Closes #799) * Fix compiler warnings -## v0.10.12 (2019-09-28) +### v0.10.12 (2019-09-28) * Fix handling of species specified in PEtab condition table (#813) * Fix some Visual C++ issues, update cppcheck handling, cleanup (VisualC++ still not fully supported) @@ -458,7 +471,7 @@ Misc: * Create SBML test suite result files for upload to http://sbml.org/Facilities/Database/ (#798) -## v0.10.11 (2019-08-31) +### v0.10.11 (2019-08-31) * Fixed setting initial conditions for preequilibration (#784) * Fixed species->parameter conversion during PEtab import (#782) @@ -467,7 +480,7 @@ Misc: * Fix various SBML import issues * Run SBML test suite using github actions instead of travisCI (#789) -## v0.10.10 (2019-08-07) +### v0.10.10 (2019-08-07) * Simplify/fix AMICI installation * If available use environment modules to detect dependencies @@ -504,11 +517,11 @@ Detaills: * Allow overriding cmake executable with environment variables in build scripts (Closes #738) -## v0.10.9 (2019-07-24) +### v0.10.9 (2019-07-24) Fixup for missing version bump in v0.10.8 release. No code changes compared to v0.10.8. -## v0.10.8 (2019-07-24) +### v0.10.8 (2019-07-24) Changes in this release: @@ -526,12 +539,12 @@ Python: ... and various other minor fixes/improvements -## v0.10.7 (2019-05-01) +### v0.10.7 (2019-05-01) Python * fix unset noise distribution when automatically generating observables in case None are passed (#691) -## v0.10.6 (2019-04-19) +### v0.10.6 (2019-04-19) C++ - Add SuperLUMT support (#681) @@ -544,7 +557,7 @@ Python - Fix dynamic override in PETab -## v0.10.5 (2019-04-08) +### v0.10.5 (2019-04-08) Bugfix release @@ -563,7 +576,7 @@ Python - Updated PEtab import to allow for different noise models -## v0.10.4 (2019-03-21) +### v0.10.4 (2019-03-21) Features / improvements: @@ -575,7 +588,7 @@ Bugfixes: + Speedup and fix travis build -## v0.10.3 (2019-03-13) +### v0.10.3 (2019-03-13) Features / improvements: @@ -588,7 +601,7 @@ Bugfixes: - fixed symbolic processing for unreleased pysb features -## v0.10.2 (2019-03-07) +### v0.10.2 (2019-03-07) Features / improvements: @@ -598,13 +611,13 @@ Bugfixes: - fixed output values of `ReturnData::x_ss` and `ReturnData::sx_ss` -## v0.10.1 (2019-03-04) +### v0.10.1 (2019-03-04) * travis-ci.com migration * fix problem with has{variable} functions * allow to import sbml model from string, not only file -## v0.10.0 (2019-03-01) +### v0.10.0 (2019-03-01) Features / improvements: @@ -616,7 +629,7 @@ Bugfixes: - fixed return value of `rz` when no data is provided. -## v0.9.5 (2019-02-26) +### v0.9.5 (2019-02-26) Features / improvements: @@ -634,13 +647,13 @@ Maintenance: - use newer CI images -## v0.9.4 (2019-02-11) +### v0.9.4 (2019-02-11) Minor fixes only: - fix(core) Get solver diagnostics for first(last) timepoint (#588) (Closes #586) - fix(ci) Fix autodeploy (Closes #589) -## v0.9.3 (2019-02-07) +### v0.9.3 (2019-02-07) **CRITICAL FIXES** - **fix(python) fix symbolic computations for adjoint (#583)** @@ -657,7 +670,7 @@ Minor fixes only: -## v0.9.2 (2019-01-30) +### v0.9.2 (2019-01-30) Bugfixes: @@ -669,7 +682,7 @@ Bugfixes: - #559 -## v0.9.1 (2019-01-21) +### v0.9.1 (2019-01-21) Features / improvements: @@ -686,7 +699,7 @@ Maintenance: - attempt to fix automatic deploy to pypi via travis -## v0.9.0 (2019-01-18) +### v0.9.0 (2019-01-18) Features / improvements: @@ -703,7 +716,7 @@ Maintenance: - Reenable run of SBML testsuite -## v0.8.2 (2019-01-07) +### v0.8.2 (2019-01-07) Features / improvements: * Speedup symbolic processing for ODE generation in python @@ -724,7 +737,7 @@ Maintenance: * Update documentation and tests * Add python version check and raise required version to 3.6 to prevent cryptic error messages when encountering f-strings -## v0.8.1 (2018-11-25) +### v0.8.1 (2018-11-25) - [all] **critical** Fix long standing bugs in solving steadystate problems (including preequilibration) (#471) - [all] Fix AmiVector constructor from std::vector (#471) @@ -732,24 +745,24 @@ Maintenance: - Update documentation -## v0.8.0 (2018-11-25) +### v0.8.0 (2018-11-25) - replaced symengine by sympy for symbolic processing in python *which fixes several critical bugs* that were due to bugs in symengine (#467) -## v0.7.13 (2018-11-18) +### v0.7.13 (2018-11-18) - fixes a critical bug in objective function computation for models compiled using `sbml2amici` and `pysb2amici` that was introduced in v0.7.12 - fixes a critical bug in sensitivity computation when`model.reinitializeFixedParameterInitialStates` was set to true - readds the python interface to the ExpData copy constructor that was inadvertently removed in 0.7.12 and streamlines the respective convenience wrapper to provide access to the full range of constructors. -## v0.7.12 (2018-11-17) +### v0.7.12 (2018-11-17) - fixed a critical bug in `amici.constructEdataFromDataFrame` - enabled multithreaded simulation of multiple experiments (requires compilation with openMP) - modularized sbml import and added pysb import -## v0.7.11 (2018-10-15) +### v0.7.11 (2018-10-15) - [python] Added numpy and python wrappers that provide a more user friendly python API - [python] Enable import of SBML models with non-float assignment rules @@ -758,27 +771,27 @@ Maintenance: - [core] Provide an API for more fine-grained control over sensitivity tolerances and steady-state tolerances - [core] Provide an API to specify non-negative state variables (this feature is still preliminary) -## v0.7.10 (2018-08-29) +### v0.7.10 (2018-08-29) - Fixed python SBML import `log()` issues (#412) -## v0.7.9 (2018-08-24) +### v0.7.9 (2018-08-24) - fixes MATLAB compilation of models - adds option to perform steady state sensitivity analysis via FSA - condition dependent intitial conditions are now newly set after preequilibration is done -## v0.7.8 (2018-08-19) +### v0.7.8 (2018-08-19) - bugfixes for the ExpData interface - created build configuration that enables debugging of c++ extensions on os x - fixed python sbml import when stoichiometry is empty -## v0.7.7 (2018-08-17) +### v0.7.7 (2018-08-17) Fixes a couple of bugs just introduced in v0.7.6 -## v0.7.6 (2018-08-13) +### v0.7.6 (2018-08-13) Important: **Use AMICI v0.7.7 due to https://github.com/ICB-DCM/AMICI/pull/403/commits/3a495d3db2fdbba70c2b0d52a3d4655c33c817a2** @@ -791,7 +804,7 @@ Bug fixes: Breaking C++ API changes: - Revised ExpData interface (#388) -## v0.7.5 (2018-07-30) +### v0.7.5 (2018-07-30) Features/enhancements: - Add computation of residuals, residuals sensitivity, Fisher information matrix (#223) @@ -802,7 +815,7 @@ Minor fixes: - Condition parameters in ExpData now only temporarily override Model parameters (#371) - Ensure non-negative states for Newton solver (#378) -## v0.7.4 (2018-07-27) +### v0.7.4 (2018-07-27) Features/enhancements: - Check SBML model validity (#343) @@ -816,7 +829,7 @@ Minor fixes: - Fix compiler warnings (#353) - Plotting, SBML example mode, ... -## v0.7.3 (2018-07-13) +### v0.7.3 (2018-07-13) Features: - Added symbol names to python-wrapped models and make available via Model.getParameterNames(), model.getStateNames(), ... @@ -824,7 +837,7 @@ Features: Python package available via pypi: https://pypi.org/project/amici/0.7.3/ -## v0.7.2 (2018-07-03) +### v0.7.2 (2018-07-03) Features: - Python package: more flexible HDF5 library localization @@ -838,7 +851,7 @@ Minor fixes: - Various fixes for mingw compilation of python source distribution - Cmake compatibility with < 3.8 restored -## v0.7.1 (2018-06-12) +### v0.7.1 (2018-06-12) Features: - Allow specifying sigma-parameters from Python interface @@ -846,7 +859,7 @@ Features: Major bugfixes: - Fix dsigma_y/dp and downstream sensitivity errors -## v0.7.0 (2018-06-09) +### v0.7.0 (2018-06-09) - Major revision of documentation - Improved Python interface @@ -857,7 +870,7 @@ Major bugfixes: WARNING: - For models with sigma-parameters and dsigma_y/dp != 0, dsigma_y/dp was computed incorrectly. This propagates to all dependent sensitivities. This applies also to some older releases and has been fixed in v0.7.1. -## v0.6.0 (2018-05-22) +### v0.6.0 (2018-05-22) Implement experimental support for python via swig. Python interface is now usable, but API will still receive some updates in the future. @@ -867,7 +880,7 @@ WARNING: - Matlab C++ compilation will fail due to undefined M_PI -> Please use v0.7.0 -## v0.5.0 (2018-03-15) +### v0.5.0 (2018-03-15) Main new features are: @@ -882,14 +895,14 @@ Main new features are: - Rewrote large parts of the code as proper c++11 code to allow easier code maintanance - Substantially extended testing in continuous integration to assure code quality -## v0.4.0 (2017-05-15) +### v0.4.0 (2017-05-15) * First citable version of AMICI (via zenodo integration). * Better support for standalone compilation * Improved SBML import scripts * General Code Cleanup -## v0.3.0 (2016-09-05) +### v0.3.0 (2016-09-05) This update comes with many improvements, bug fixes and several new features. Most notably: @@ -899,12 +912,12 @@ This update comes with many improvements, bug fixes and several new features. Mo 4) AMICI now supports more SBML, SBML v2 and rate rules -## 0.2.1 (2016-05-09) +### 0.2.1 (2016-05-09) Bugfix release. This release also includes some changes that should improve the performance on the new R2016a release of MATLAB. -## v0.2 (2016-03-17) +### v0.2 (2016-03-17) This update comes with many improvements to the computation time for both compilation and simulation. Moreover several new features were included: @@ -912,6 +925,6 @@ This update comes with many improvements to the computation time for both compil 2) Correct treatment of parameter/state dependent discontinuities for both forward and adjoint sensitivities -## v0.1 (2015-11-05) +### v0.1 (2015-11-05) This is the initial release of the toolbox diff --git a/docker/README.md b/docker/README.md index ec7ecc6771..97c0722545 100644 --- a/docker/README.md +++ b/docker/README.md @@ -12,4 +12,4 @@ cd docker && docker build -t $USER/amici:latest . ## Published images AMICI docker images are regularly published to -https://hub.docker.com/layers/dweindl/amici/. +https://hub.docker.com/r/dweindl/amici. diff --git a/documentation/gfx/banner.png b/documentation/gfx/banner.png index a88c2da489..40016b4da0 100644 Binary files a/documentation/gfx/banner.png and b/documentation/gfx/banner.png differ diff --git a/documentation/gfx/logo.png b/documentation/gfx/logo.png index 3f22c92a86..7810c1e7ce 100644 Binary files a/documentation/gfx/logo.png and b/documentation/gfx/logo.png differ diff --git a/documentation/gfx/logo_text.png b/documentation/gfx/logo_text.png index 22354e62b1..c72388e941 100644 Binary files a/documentation/gfx/logo_text.png and b/documentation/gfx/logo_text.png differ diff --git a/documentation/rtd_requirements.txt b/documentation/rtd_requirements.txt index 58ccbff6d3..154f12a9dc 100644 --- a/documentation/rtd_requirements.txt +++ b/documentation/rtd_requirements.txt @@ -21,4 +21,4 @@ git+https://github.com/dweindl/exhale@ea77a313777c1382a7830ce9ee6c405ce47f5f3b#e -e git+https://github.com/mithro/sphinx-contrib-mithro#egg=sphinx-contrib-exhale-multiproject&subdirectory=sphinx-contrib-exhale-multiproject sphinxcontrib-matlabdomain sphinxcontrib-napoleon -pygments==2.4.1 +pygments==2.7.4 diff --git a/include/amici/rdata.h b/include/amici/rdata.h index 1637f67ede..c0c42b78a5 100644 --- a/include/amici/rdata.h +++ b/include/amici/rdata.h @@ -618,10 +618,12 @@ class ReturnData: public ModelDimensions { * if preequilibration was run in adjoint mode * @param model model that was used for forward/backward simulation * @param preeq SteadystateProblem for preequilibration + * @param llhS0 contribution to likelihood for initial state sensitivities + * of preequilibration * @param xQB vector with quadratures from adjoint computation */ void handleSx0Backward(const Model &model, SteadystateProblem const &preeq, - AmiVector &xQB) const; + std::vector &llhS0, AmiVector &xQB) const; /** * @brief Updates contribution to likelihood for initial state sensitivities diff --git a/include/amici/serialization.h b/include/amici/serialization.h index f816d56633..7906a2cc48 100644 --- a/include/amici/serialization.h +++ b/include/amici/serialization.h @@ -290,16 +290,19 @@ char *serializeToChar(T const& data, int *size) { template T deserializeFromChar(const char *buffer, int size) { + namespace ba = ::boost::archive; + namespace bio = ::boost::iostreams; + + bio::basic_array_source device(buffer, size); + bio::stream> s(device); + + T data; + try { - ::boost::iostreams::basic_array_source device(buffer, size); - ::boost::iostreams::stream<::boost::iostreams::basic_array_source> s( - device); - ::boost::archive::binary_iarchive iar(s); - T data; + ba::binary_iarchive iar(s); iar >> data; - return data; - } catch(::boost::archive::archive_exception const& e) { + } catch(ba::archive_exception const& e) { throw AmiException("Deserialization from char failed: %s", e.what()); } } @@ -314,19 +317,20 @@ T deserializeFromChar(const char *buffer, int size) { template std::string serializeToString(T const& data) { + namespace ba = ::boost::archive; + namespace bio = ::boost::iostreams; + + std::string serialized; + bio::back_insert_device inserter(serialized); + bio::stream> os(inserter); + try { - std::string serialized; - ::boost::iostreams::back_insert_device inserter(serialized); - ::boost::iostreams::stream< - ::boost::iostreams::back_insert_device> - s(inserter); - ::boost::archive::binary_oarchive oar(s); + ba::binary_oarchive oar(os); oar << data; - s.flush(); return serialized; - } catch(::boost::archive::archive_exception const& e) { + } catch(ba::archive_exception const& e) { throw AmiException("Serialization to string failed: %s", e.what()); } } @@ -341,21 +345,22 @@ std::string serializeToString(T const& data) { template std::vector serializeToStdVec(T const& data) { + namespace ba = ::boost::archive; + namespace bio = ::boost::iostreams; + + std::vector buffer; + bio::stream< + bio::back_insert_device< + std::vector>> os(buffer); + try{ - std::string serialized; - ::boost::iostreams::back_insert_device inserter(serialized); - ::boost::iostreams::stream<::boost::iostreams::back_insert_device> - s(inserter); - ::boost::archive::binary_oarchive oar(s); + ba::binary_oarchive oar(os); oar << data; - s.flush(); - - std::vector buf(serialized.begin(), serialized.end()); - return buf; - } catch(::boost::archive::archive_exception const& e) { - throw AmiException("Serialization to StdVec failed: %s", e.what()); + return buffer; + } catch(ba::archive_exception const& e) { + throw AmiException("Serialization to std::vector failed: %s", e.what()); } } @@ -369,19 +374,22 @@ std::vector serializeToStdVec(T const& data) { template T deserializeFromString(std::string const& serialized) { + namespace ba = ::boost::archive; + namespace bio = ::boost::iostreams; + + bio::basic_array_source device(serialized.data(), serialized.size()); + bio::stream> os(device); + T deserialized; + try{ - ::boost::iostreams::basic_array_source device(serialized.data(), - serialized.size()); - ::boost::iostreams::stream<::boost::iostreams::basic_array_source> s( - device); - ::boost::archive::binary_iarchive iar(s); - T deserialized; + ba::binary_iarchive iar(os); iar >> deserialized; return deserialized; - } catch(::boost::archive::archive_exception const& e) { - throw AmiException("Deserialization from StdVec failed: %s", e.what()); + } catch(ba::archive_exception const& e) { + throw AmiException("Deserialization from std::string failed: %s", + e.what()); } } diff --git a/include/amici/steadystateproblem.h b/include/amici/steadystateproblem.h index 3629a4702a..bd7efe47c4 100644 --- a/include/amici/steadystateproblem.h +++ b/include/amici/steadystateproblem.h @@ -248,7 +248,6 @@ class SteadystateProblem { return x_; }; - /** * @brief Returns state sensitivity at steadystate * @return sx @@ -321,6 +320,12 @@ class SteadystateProblem { */ void getAdjointUpdates(Model &model, const ExpData &edata); + /** + * @brief Return the adjoint state + * @return xB adjoint state + */ + AmiVector const& getAdjointState() const { return xB_; } + /** * @brief Accessor for xQB * @return xQB @@ -372,7 +377,7 @@ class SteadystateProblem { AmiVector xQ_; /** quadrature state vector */ AmiVector xQB_; - /** quadrature state vector */ + /** time-derivative of quadrature state vector */ AmiVector xQBdot_; /** maximum number of steps for Newton solver for allocating numlinsteps */ diff --git a/python/amici/petab_import.py b/python/amici/petab_import.py index 68fb53e3d5..bfbbf744ad 100644 --- a/python/amici/petab_import.py +++ b/python/amici/petab_import.py @@ -459,6 +459,21 @@ def import_model_sbml( sbml_importer = amici.SbmlImporter(sbml_model) sbml_model = sbml_importer.sbml + allow_n_noise_pars = \ + not petab.lint.observable_table_has_nontrivial_noise_formula( + observable_df + ) + if measurement_table is not None and \ + petab.lint.measurement_table_has_timepoint_specific_mappings( + measurement_table, + allow_scalar_numeric_noise_parameters=allow_n_noise_pars + ): + raise ValueError( + 'AMICI does not support importing models with timepoint specific ' + 'mappings for noise or observable parameters. Please flatten ' + 'the problem and try again.' + ) + if observable_df is not None: observables, noise_distrs, sigmas = \ get_observation_model(observable_df) @@ -574,10 +589,10 @@ def import_model_sbml( import_model = import_model_sbml -def get_observation_model(observable_df: pd.DataFrame - ) -> Tuple[Dict[str, Dict[str, str]], - Dict[str, str], - Dict[str, Union[str, float]]]: +def get_observation_model( + observable_df: pd.DataFrame, +) -> Tuple[Dict[str, Dict[str, str]], Dict[str, str], + Dict[str, Union[str, float]]]: """ Get observables, sigmas, and noise distributions from PEtab observation table in a format suitable for @@ -596,13 +611,13 @@ def get_observation_model(observable_df: pd.DataFrame observables = {} sigmas = {} + nan_pat = r'^[nN]a[nN]$' for _, observable in observable_df.iterrows(): - oid = observable.name + oid = str(observable.name) # need to sanitize due to https://github.com/PEtab-dev/PEtab/issues/447 - pat = r'^[nN]a[nN]$' - name = re.sub(pat, '', str(observable.get(OBSERVABLE_NAME, ''))) - formula_obs = re.sub(pat, '', str(observable[OBSERVABLE_FORMULA])) - formula_noise = re.sub(pat, '', str(observable[NOISE_FORMULA])) + name = re.sub(nan_pat, '', str(observable.get(OBSERVABLE_NAME, ''))) + formula_obs = re.sub(nan_pat, '', str(observable[OBSERVABLE_FORMULA])) + formula_noise = re.sub(nan_pat, '', str(observable[NOISE_FORMULA])) observables[oid] = {'name': name, 'formula': formula_obs} sigmas[oid] = formula_noise @@ -617,7 +632,8 @@ def get_observation_model(observable_df: pd.DataFrame return observables, noise_distrs, sigmas -def petab_noise_distributions_to_amici(observable_df: pd.DataFrame) -> Dict: +def petab_noise_distributions_to_amici(observable_df: pd.DataFrame + ) -> Dict[str, str]: """ Map from the petab to the amici format of noise distribution identifiers. @@ -689,6 +705,10 @@ def parse_cli_args(): parser.add_argument('--no-compile', action='store_false', dest='compile', help='Only generate model code, do not compile') + parser.add_argument('--flatten', dest='flatten', default=False, + action='store_true', + help='Flatten measurement specific overrides of ' + 'observable and noise parameters') # Call with set of files parser.add_argument('-s', '--sbml', dest='sbml_file_name', @@ -740,6 +760,9 @@ def main(): # First check for valid PEtab petab.lint_problem(pp) + if args.flatten: + petab.flatten_timepoint_specific_output_overrides(pp) + import_model(model_name=args.model_name, sbml_model=pp.sbml_model, condition_table=pp.condition_df, diff --git a/python/amici/petab_import_pysb.py b/python/amici/petab_import_pysb.py index f5aa22dd5d..3d33906e5b 100644 --- a/python/amici/petab_import_pysb.py +++ b/python/amici/petab_import_pysb.py @@ -47,8 +47,10 @@ def __init__(self, pysb_model: 'pysb.Model' = None, *args, **kwargs): :param args: See :meth:`petab.Problem.__init__` :param kwargs: See :meth:`petab.Problem.__init__` """ - + flatten = kwargs.pop('flatten', False) super().__init__(*args, **kwargs) + if flatten: + petab.flatten_timepoint_specific_output_overrides(self) self.pysb_model: 'pysb.Model' = pysb_model self._add_observation_model() @@ -108,17 +110,33 @@ def from_files(condition_file: str = None, visualization_files: Union[str, Iterable[str]] = None, observable_files: Union[str, Iterable[str]] = None, pysb_model_file: str = None, - ) -> 'PysbPetabProblem': + flatten: bool = False) -> 'PysbPetabProblem': """ Factory method to load model and tables from files. - Arguments: - condition_file: PEtab condition table - measurement_file: PEtab measurement table - parameter_file: PEtab parameter table - visualization_files: PEtab visualization tables - observable_files: PEtab observables tables - pysb_model_file: PySB model file + :param condition_file: + PEtab condition table + + :param measurement_file: + PEtab measurement table + + :param parameter_file: + PEtab parameter table + + :param visualization_files: + PEtab visualization tables + + :param observable_files: + PEtab observables tables + + :param pysb_model_file: + PySB model file + + :param flatten: + Flatten the petab problem + + :return: + Petab Problem """ condition_df = measurement_df = parameter_df = visualization_df = None @@ -152,18 +170,27 @@ def from_files(condition_file: str = None, measurement_df=measurement_df, parameter_df=parameter_df, observable_df=observable_df, - visualization_df=visualization_df) + visualization_df=visualization_df, + flatten=flatten + ) @staticmethod - def from_yaml(yaml_config: Union[Dict, str]) -> 'PysbPetabProblem': + def from_yaml(yaml_config: Union[Dict, str], + flatten: bool = False) -> 'PysbPetabProblem': """ Factory method to load model and tables as specified by YAML file. NOTE: The PySB model is currently expected in the YAML file under ``sbml_files``. - Arguments: - yaml_config: PEtab configuration as dictionary or YAML file name + :param yaml_config: + PEtab configuration as dictionary or YAML file name + + :param flatten: + Flatten the petab problem + + :return: + Petab Problem """ from petab.yaml import (load_yaml, is_composite_problem, assert_single_condition_and_sbml_file) @@ -210,7 +237,8 @@ def from_yaml(yaml_config: Union[Dict, str]) -> 'PysbPetabProblem': for f in problem0.get(VISUALIZATION_FILES, [])], observable_files=[ os.path.join(path_prefix, f) - for f in problem0.get(OBSERVABLE_FILES, [])] + for f in problem0.get(OBSERVABLE_FILES, [])], + flatten=flatten ) diff --git a/python/amici/petab_objective.py b/python/amici/petab_objective.py index 7b4d4fcf6b..2039d68033 100644 --- a/python/amici/petab_objective.py +++ b/python/amici/petab_objective.py @@ -262,7 +262,12 @@ def create_parameter_mapping( prelim_parameter_mapping = \ petab_problem.get_optimization_to_simulation_parameter_mapping( - warn_unmapped=False, scaled_parameters=scaled_parameters) + warn_unmapped=False, scaled_parameters=scaled_parameters, + allow_timepoint_specific_numeric_noise_parameters= + not petab.lint.observable_table_has_nontrivial_noise_formula( + petab_problem.observable_df + ) + ) parameter_mapping = ParameterMapping() for (_, condition), prelim_mapping_for_condition in \ diff --git a/python/sdist/setup.cfg b/python/sdist/setup.cfg index 735b68e748..8e8fdbaeaf 100644 --- a/python/sdist/setup.cfg +++ b/python/sdist/setup.cfg @@ -41,7 +41,7 @@ include_package_data = True zip_safe = False [options.extras_require] -petab = petab>=0.1.11 +petab = petab>=0.1.17 pysb = pysb>=1.11.0 [options.package_data] diff --git a/src/hdf5.cpp b/src/hdf5.cpp index 12ef25cf7a..2dc974bb8d 100644 --- a/src/hdf5.cpp +++ b/src/hdf5.cpp @@ -1,3 +1,12 @@ +/** + * Functions for HDF5 I/O + * + * NOTE: Use only `const char*` versions of any HDF5 functions, not the + * `std::string` version. On many systems, HDF5 libraries are still not compiled + * with C++11 support, but use the old C++03 ABI, which will lead to linking + * issues. + */ + #include "amici/hdf5.h" #include "amici/amici.h" @@ -97,7 +106,7 @@ void createGroup(H5::H5File const& file, std::unique_ptr readSimulationExpData(std::string const& hdf5Filename, std::string const& hdf5Root, Model const& model) { - H5::H5File file(hdf5Filename, H5F_ACC_RDONLY); + H5::H5File file(hdf5Filename.c_str(), H5F_ACC_RDONLY); hsize_t m, n; @@ -495,7 +504,7 @@ void createAndWriteInt1DDataset(H5::H5File const& file, gsl::span buffer) { hsize_t size = buffer.size(); H5::DataSpace dataspace(1, &size); - auto dataset = file.createDataSet(datasetName, H5::PredType::NATIVE_INT, + auto dataset = file.createDataSet(datasetName.c_str(), H5::PredType::NATIVE_INT, dataspace); dataset.write(buffer.data(), H5::PredType::NATIVE_INT); } @@ -505,7 +514,7 @@ void createAndWriteDouble1DDataset(const H5::H5File &file, gsl::span buffer) { hsize_t size = buffer.size(); H5::DataSpace dataspace(1, &size); - auto dataset = file.createDataSet(datasetName, H5::PredType::NATIVE_DOUBLE, + auto dataset = file.createDataSet(datasetName.c_str(), H5::PredType::NATIVE_DOUBLE, dataspace); dataset.write(buffer.data(), H5::PredType::NATIVE_DOUBLE); } @@ -516,7 +525,7 @@ void createAndWriteDouble2DDataset(const H5::H5File &file, hsize_t n) { const hsize_t adims[] {m, n}; H5::DataSpace dataspace(2, adims); - auto dataset = file.createDataSet(datasetName, H5::PredType::NATIVE_DOUBLE, + auto dataset = file.createDataSet(datasetName.c_str(), H5::PredType::NATIVE_DOUBLE, dataspace); dataset.write(buffer.data(), H5::PredType::NATIVE_DOUBLE); } @@ -527,7 +536,7 @@ void createAndWriteInt2DDataset(H5::H5File const& file, hsize_t n) { const hsize_t adims[] {m, n}; H5::DataSpace dataspace(2, adims); - auto dataset = file.createDataSet(datasetName, H5::PredType::NATIVE_INT, + auto dataset = file.createDataSet(datasetName.c_str(), H5::PredType::NATIVE_INT, dataspace); dataset.write(buffer.data(), H5::PredType::NATIVE_INT); } @@ -538,7 +547,7 @@ void createAndWriteDouble3DDataset(H5::H5File const& file, hsize_t n, hsize_t o) { const hsize_t adims[] {m, n, o}; H5::DataSpace dataspace(3, adims); - auto dataset = file.createDataSet(datasetName, H5::PredType::NATIVE_DOUBLE, + auto dataset = file.createDataSet(datasetName.c_str(), H5::PredType::NATIVE_DOUBLE, dataspace); dataset.write(buffer.data(), H5::PredType::NATIVE_DOUBLE); } @@ -871,14 +880,14 @@ void readSolverSettingsFromHDF5(H5::H5File const& file, Solver &solver, void readSolverSettingsFromHDF5(const std::string &hdffile, Solver &solver, const std::string &datasetPath) { - H5::H5File file(hdffile, H5F_ACC_RDONLY, H5P_DEFAULT); + H5::H5File file(hdffile.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); readSolverSettingsFromHDF5(file, solver, datasetPath); } void readModelDataFromHDF5(const std::string &hdffile, Model &model, const std::string &datasetPath) { - H5::H5File file(hdffile, H5F_ACC_RDONLY, H5P_DEFAULT); + H5::H5File file(hdffile.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); readModelDataFromHDF5(file, model, datasetPath); } @@ -958,12 +967,12 @@ H5::H5File createOrOpenForWriting(const std::string &hdf5filename) { AMICI_H5_SAVE_ERROR_HANDLER; try { - H5::H5File file(hdf5filename, H5F_ACC_RDWR); + H5::H5File file(hdf5filename.c_str(), H5F_ACC_RDWR); AMICI_H5_RESTORE_ERROR_HANDLER; return file; } catch(...) { AMICI_H5_RESTORE_ERROR_HANDLER; - return H5::H5File(hdf5filename, H5F_ACC_EXCL); + return H5::H5File(hdf5filename.c_str(), H5F_ACC_EXCL); } } @@ -977,13 +986,13 @@ bool locationExists(const H5::H5File &file, const std::string &location) bool locationExists(const std::string &filename, const std::string &location) { - H5::H5File file(filename, H5F_ACC_RDONLY); + H5::H5File file(filename.c_str(), H5F_ACC_RDONLY); return locationExists(file, location); } std::vector getIntDataset1D(const H5::H5File &file, std::string const& name) { - auto dataset = file.openDataSet(name); + auto dataset = file.openDataSet(name.c_str()); auto dataspace = dataset.getSpace(); int rank = dataspace.getSimpleExtentNdims(); @@ -1002,7 +1011,7 @@ std::vector getIntDataset1D(const H5::H5File &file, std::vector getDoubleDataset1D(const H5::H5File &file, const std::string &name) { - auto dataset = file.openDataSet(name); + auto dataset = file.openDataSet(name.c_str()); auto dataspace = dataset.getSpace(); int rank = dataspace.getSimpleExtentNdims(); @@ -1025,7 +1034,7 @@ std::vector getDoubleDataset2D(const H5::H5File &file, { m = n = 0; - auto dataset = file.openDataSet(name); + auto dataset = file.openDataSet(name.c_str()); auto dataspace = dataset.getSpace(); int rank = dataspace.getSimpleExtentNdims(); @@ -1050,7 +1059,7 @@ std::vector getDoubleDataset3D(const H5::H5File &file, { m = n = o = 0; - auto dataset = file.openDataSet(name); + auto dataset = file.openDataSet(name.c_str()); auto dataspace = dataset.getSpace(); int rank = dataspace.getSimpleExtentNdims(); diff --git a/src/rdata.cpp b/src/rdata.cpp index 9b7f944611..6e78a4b83d 100644 --- a/src/rdata.cpp +++ b/src/rdata.cpp @@ -414,7 +414,7 @@ void ReturnData::processBackwardProblem(ForwardProblem const &fwd, auto xQB = bwd.getAdjointQuadrature(); if (preeq && preeq->hasQuadrature()) { - handleSx0Backward(model, *preeq, xQB); + handleSx0Backward(model, *preeq, llhS0, xQB); } else { handleSx0Forward(model, llhS0, xB); } @@ -433,6 +433,7 @@ void ReturnData::processBackwardProblem(ForwardProblem const &fwd, void ReturnData::handleSx0Backward(const Model &model, SteadystateProblem const &preeq, + std::vector &llhS0, AmiVector &xQB) const { /* If preequilibration is run in adjoint mode, the scalar product of sx0 with its adjoint counterpart (see handleSx0Forward()) is not necessary: @@ -441,9 +442,23 @@ void ReturnData::handleSx0Backward(const Model &model, and so is the scalar product. Instead of the scalar product, the quadratures xQB from preequilibration contribute to the gradient (see example notebook on equilibration for further documentation). */ - auto xQBpreeq = preeq.getAdjointQuadrature(); + const auto &xQBpreeq = preeq.getAdjointQuadrature(); for (int ip = 0; ip < model.nplist(); ++ip) - xQB[ip] += xQBpreeq[ip]; + xQB[ip] += xQBpreeq.at(ip); + + /* We really need references here, as sx0 can be large... */ + const auto& sx0preeq = preeq.getStateSensitivity(); + const auto& xBpreeq = preeq.getAdjointState(); + + /* Add the contribution for sx0 from preequilibration. If backward + * preequilibration was done by simulation due to a singular Jacobian, + * xB is not necessarily 0 and we may get a non-zero contribution here. */ + for (int ip = 0; ip < model.nplist(); ++ip) { + llhS0[ip] = 0.0; + for (int ix = 0; ix < model.nxtrue_solver; ++ix) { + llhS0[ip] += xBpreeq.at(ix) * sx0preeq.at(ix, ip); + } + } } void ReturnData::handleSx0Forward(const Model &model, diff --git a/src/steadystateproblem.cpp b/src/steadystateproblem.cpp index 6d75c469fd..969c5723f8 100644 --- a/src/steadystateproblem.cpp +++ b/src/steadystateproblem.cpp @@ -101,9 +101,6 @@ void SteadystateProblem::workSteadyStateBackwardProblem(Solver *solver, clock_t starttime = clock(); computeSteadyStateQuadrature(newtonSolver.get(), solver, model); cpu_timeB_ = (double)((clock() - starttime) * 1000) / CLOCKS_PER_SEC; - - /* Finalize by setting adjoint state to zero (its steady state) */ - xB_.zero(); } void SteadystateProblem::findSteadyState(Solver *solver, @@ -285,6 +282,9 @@ void SteadystateProblem::getQuadratureByLinSolve(NewtonSolver *newtonSolver, computeQBfromQ(model, xQ_, xQB_); /* set flag that quadratures is available (for processing in rdata) */ hasQuadrature_ = true; + + /* Finalize by setting adjoint state to zero (its steady state) */ + xB_.zero(); } catch (NewtonFailure const &) { hasQuadrature_ = false; } diff --git a/tests/benchmark-models/benchmark_models.yaml b/tests/benchmark-models/benchmark_models.yaml index a84afe34f3..d290171e82 100644 --- a/tests/benchmark-models/benchmark_models.yaml +++ b/tests/benchmark-models/benchmark_models.yaml @@ -5,7 +5,9 @@ Bachmann_MSB2011: Becker_Science2010: llh: -364.118614198023 -#Beer_MolBioSystems2014 None +Beer_MolBioSystems2014: + llh: 58622.9145631413 + note: benchmark collection reference parameters do not match petab, but reference llh has been confirmed for parameters reported there up to sign Boehm_JProteomeRes2014: llh: -138.22199693517703 @@ -16,8 +18,7 @@ Borghans_BiophysChem1997: note: benchmark collection reference value matches up to sign when applying log10-correction +sum(log(meas*log(10)) / 2 Brannmark_JBC2010: - llh: 283.778227541074 - note: unchecked + llh: -141.889113770537 #Bruno_JExpBio2016: None @@ -34,8 +35,7 @@ Elowitz_Nature2000: note: benchmark collection reference value matches up to sign when applying log10-correction +sum(log(meas*log(10))) / 2 Fiedler_BMC2016: - llh: -117.16780323362 - note: unchecked + llh: 58.58390161681 Fujita_SciSignal2010: llh: 53.08749124997969 diff --git a/tests/benchmark-models/test_benchmark_collection.sh b/tests/benchmark-models/test_benchmark_collection.sh index ea3dbbfbed..0422dd3e23 100755 --- a/tests/benchmark-models/test_benchmark_collection.sh +++ b/tests/benchmark-models/test_benchmark_collection.sh @@ -7,15 +7,18 @@ # Confirmed to be working models=" +Beer_MolBioSystems2014 Boehm_JProteomeRes2014 Borghans_BiophysChem1997 +Brannmark_JBC2010 +Crauste_CellSystems2017 Elowitz_Nature2000 -Schwen_PONE2014 +Fiedler_BMC2016 Fujita_SciSignal2010 +Schwen_PONE2014 Sneyd_PNAS2002 -Zheng_PNAS2012 Weber_BMC2015 -Crauste_CellSystems2017" +Zheng_PNAS2012" # # Not matching reference for unclear reasons @@ -29,19 +32,12 @@ Crauste_CellSystems2017" # # no reference value: # Alkan_SciSignal2018 -# Beer_MolBioSystems2014 # Blasi_CellSystems2016 # Hass_PONE2017 # Korkut_eLIFE2015 # Perelson_Science1996 # Bruno_JExpBio2016 # -# Timepoint-specific parameter overrides -# Fiedler_BMC2016 -# Brannmark_JBC2010 -# Isensee_JCB2018 -# Sobotta_Frontiers2017 -# # yaml missing: # Casaletto_PNAS2019 # @@ -59,6 +55,9 @@ Crauste_CellSystems2017" # # Evaluation is known to be inconsistent: # Chen_MSB2009 +# +# Integration Failure: +# Isensee_JCB2018 set -e @@ -91,7 +90,7 @@ for model in $models; do yaml="${model_dir}"/"${model}"/"${model}".yaml amici_model_dir=test_bmc/"${model}" mkdir -p "$amici_model_dir" - cmd_import="amici_import_petab --verbose -y ${yaml} -o ${amici_model_dir} -n ${model}" + cmd_import="amici_import_petab --verbose -y ${yaml} -o ${amici_model_dir} -n ${model} --flatten" cmd_run="$script_path/test_petab_model.py --verbose -y ${yaml} -d ${amici_model_dir} -m ${model} -c" printf '=%.0s' {1..40} diff --git a/tests/benchmark-models/test_petab_model.py b/tests/benchmark-models/test_petab_model.py index 370b38df6b..692a7aadd6 100755 --- a/tests/benchmark-models/test_petab_model.py +++ b/tests/benchmark-models/test_petab_model.py @@ -75,6 +75,7 @@ def main(): # load PEtab files problem = petab.Problem.from_yaml(args.yaml_file_name) + petab.flatten_timepoint_specific_output_overrides(problem) # load model if args.model_directory: diff --git a/tests/cpputest/testfunctions.cpp b/tests/cpputest/testfunctions.cpp index b24fdd1bee..382e7fa43d 100644 --- a/tests/cpputest/testfunctions.cpp +++ b/tests/cpputest/testfunctions.cpp @@ -73,7 +73,7 @@ void simulateVerifyWrite(const std::string& hdffileOptions, const std::string& h // write // delete destination group - H5::H5File in(hdffileOptions, H5F_ACC_RDONLY); + H5::H5File in(hdffileOptions.c_str(), H5F_ACC_RDONLY); auto out = amici::hdf5::createOrOpenForWriting(hdffilewrite); if(hdf5::locationExists(out, path)) H5Ldelete(out.getId(), path.c_str(), H5P_DEFAULT); @@ -169,7 +169,7 @@ void verifyReturnData(std::string const& hdffile, std::string const& resultPath, } // compare to saved data in hdf file - H5::H5File file(hdffile, H5F_ACC_RDONLY); + H5::H5File file(hdffile.c_str(), H5F_ACC_RDONLY); hsize_t m, n; diff --git a/tests/performance/reference.yml b/tests/performance/reference.yml index b641ac92a5..085363b6ae 100644 --- a/tests/performance/reference.yml +++ b/tests/performance/reference.yml @@ -2,7 +2,10 @@ create_sdist: 5 install_sdist: 150 petab_import: 2100 -install_model: 350 +install_model: 120 +install_model_O0: 40 +install_model_O1: 90 +install_model_O2: 120 forward_simulation: 2 forward_sensitivities: 2 adjoint_sensitivities: 2 diff --git a/tests/performance/test.py b/tests/performance/test.py index 0f853c4a66..cf58c116b8 100755 --- a/tests/performance/test.py +++ b/tests/performance/test.py @@ -5,47 +5,73 @@ import petab import subprocess import os +import re +import shutil from amici.petab_import import import_model -def main(): +def parse_args(): arg = sys.argv[1] + if '_' in arg and re.match(r'O[0-2]', arg.split("_")[-1]): + optim = arg.split("_")[-1] + os.environ['AMICI_CXXFLAGS'] = f'-{optim}' + suffix = f'_{optim}' + arg = '_'.join(arg.split("_")[:-1]) + else: + suffix = '' - if arg == 'compilation': - git_dir = os.path.join(os.curdir, 'CS_Signalling_ERBB_RAS_AKT') - if not os.path.exists(git_dir): - subprocess.run([ - 'git', 'clone', '--depth', '1', - 'https://github.com/ICB-DCM/CS_Signalling_ERBB_RAS_AKT'] - ) - os.chdir(os.path.join(os.curdir, 'CS_Signalling_ERBB_RAS_AKT')) - - pp = petab.Problem.from_yaml('FroehlichKes2018/PEtab/FroehlichKes2018.yaml') - petab.lint_problem(pp) - os.chdir(os.path.dirname(os.path.abspath(os.curdir))) - import_model(model_name='CS_Signalling_ERBB_RAS_AKT_petab', - sbml_model=pp.sbml_model, - condition_table=pp.condition_df, - observable_table=pp.observable_df, - measurement_table=pp.measurement_df, - compile=False, - verbose=True) - os.chdir(os.path.join(os.curdir, 'CS_Signalling_ERBB_RAS_AKT_petab')) - - subprocess.run(['python', 'setup.py', 'install']) + return arg, suffix - return - else: - import CS_Signalling_ERBB_RAS_AKT_petab as model_module - model = model_module.getModel() - solver = model.getSolver() - # TODO - edata = amici.ExpData(model) - edata.setTimepoints([1e8]) - edata.setObservedData([1.0]) - edata.setObservedDataStdDev([1.0]) +def check_results(rdata): + diagnostics = ['numsteps', 'numstepsB', 'numrhsevals', 'numrhsevalsB', + 'numerrtestfails', 'numerrtestfailsB', + 'numnonlinsolvconvfails', 'numnonlinsolvconvfailsB', + 'preeq_cpu_time', 'preeq_cpu_timeB', + 'cpu_time', 'cpu_timeB', + 'posteq_cpu_time', 'posteq_cpu_timeB'] + for d in diagnostics: + print(d, rdata[d]) + assert rdata['status'] == amici.AMICI_SUCCESS + + +def run_import(model_name): + git_dir = os.path.join(os.curdir, 'CS_Signalling_ERBB_RAS_AKT') + if not os.path.exists(git_dir): + subprocess.run([ + 'git', 'clone', '--depth', '1', + 'https://github.com/ICB-DCM/CS_Signalling_ERBB_RAS_AKT'] + ) + os.chdir(os.path.join(os.curdir, 'CS_Signalling_ERBB_RAS_AKT')) + + pp = petab.Problem.from_yaml( + 'FroehlichKes2018/PEtab/FroehlichKes2018.yaml' + ) + petab.lint_problem(pp) + import_model(model_name=model_name, + sbml_model=pp.sbml_model, + condition_table=pp.condition_df, + observable_table=pp.observable_df, + measurement_table=pp.measurement_df, + compile=False, + verbose=True) + + +def compile_model(model_name, model_dir): + if model_name != os.path.basename(model_dir): + shutil.copytree( + os.path.join(os.curdir, 'CS_Signalling_ERBB_RAS_AKT', + model_name), + model_dir + ) + + subprocess.run(['python', 'setup.py', + 'build_ext', f'--build-lib=.', '--force'], + cwd=model_dir) + + +def prepare_simulation(arg, model, solver, edata): if arg == 'forward_simulation': solver.setSensitivityMethod(amici.SensitivityMethod.none) solver.setSensitivityOrder(amici.SensitivityOrder.none) @@ -57,41 +83,60 @@ def main(): solver.setSensitivityMethod(amici.SensitivityMethod.adjoint) solver.setSensitivityOrder(amici.SensitivityOrder.first) elif arg == 'forward_simulation_non_optimal_parameters': - tmpPar = model.getParameters() - model.setParameters([0.1 for _ in tmpPar]) + tmp_par = model.getParameters() + model.setParameters([0.1 for _ in tmp_par]) solver.setSensitivityMethod(amici.SensitivityMethod.none) solver.setSensitivityOrder(amici.SensitivityOrder.none) elif arg == 'adjoint_sensitivities_non_optimal_parameters': - tmpPar = model.getParameters() - model.setParameters([0.1 for _ in tmpPar]) + tmp_par = model.getParameters() + model.setParameters([0.1 for _ in tmp_par]) solver.setSensitivityMethod(amici.SensitivityMethod.adjoint) solver.setSensitivityOrder(amici.SensitivityOrder.first) elif arg == 'forward_steadystate_sensitivities_non_optimal_parameters': - tmpPar = model.getParameters() - model.setParameters([0.1 for _ in tmpPar]) + tmp_par = model.getParameters() + model.setParameters([0.1 for _ in tmp_par]) solver.setSensitivityMethod(amici.SensitivityMethod.forward) solver.setSensitivityOrder(amici.SensitivityOrder.first) edata.setTimepoints([float('inf')]) elif arg == 'adjoint_steadystate_sensitivities_non_optimal_parameters': - tmpPar = model.getParameters() - model.setParameters([0.1 for _ in tmpPar]) + tmp_par = model.getParameters() + model.setParameters([0.1 for _ in tmp_par]) solver.setSensitivityMethod(amici.SensitivityMethod.adjoint) solver.setSensitivityOrder(amici.SensitivityOrder.first) edata.setTimepoints([float('inf')]) else: print("Unknown argument:", arg) sys.exit(1) + + +def main(): + arg, suffix = parse_args() + + model_dir = os.path.join(os.curdir, 'CS_Signalling_ERBB_RAS_AKT', + 'CS_Signalling_ERBB_RAS_AKT_petab' + suffix) + model_name = 'CS_Signalling_ERBB_RAS_AKT_petab' + + if arg == 'import': + run_import(model_name) + return + elif arg == 'compile': + compile_model(model_name, model_dir) + return + else: + model_module = amici.import_model_module(model_name, model_dir) + model = model_module.getModel() + solver = model.getSolver() + # TODO + edata = amici.ExpData(model) + edata.setTimepoints([1e8]) + edata.setObservedData([1.0]) + edata.setObservedDataStdDev([1.0]) + + prepare_simulation(arg, model, solver, edata) + rdata = amici.runAmiciSimulation(model, solver, edata) - diagnostics = ['numsteps', 'numstepsB', 'numrhsevals', 'numrhsevalsB', - 'numerrtestfails', 'numerrtestfailsB', - 'numnonlinsolvconvfails', 'numnonlinsolvconvfailsB', - 'preeq_cpu_time', 'preeq_cpu_timeB', - 'cpu_time', 'cpu_timeB', - 'posteq_cpu_time', 'posteq_cpu_timeB'] - for d in diagnostics: - print(d, rdata[d]) - assert rdata['status'] == amici.AMICI_SUCCESS + check_results(rdata) if __name__ == '__main__': diff --git a/tests/petab_test_suite/test_petab_suite.py b/tests/petab_test_suite/test_petab_suite.py index f799c28080..73c7a58db3 100755 --- a/tests/petab_test_suite/test_petab_suite.py +++ b/tests/petab_test_suite/test_petab_suite.py @@ -57,11 +57,14 @@ def _test_case(case, model_type): case_dir = os.path.join(petabtests.PYSB_DIR, case) # import petab problem yaml_file = os.path.join(case_dir, petabtests.problem_yaml_name(case)) - problem = PysbPetabProblem.from_yaml(yaml_file) + problem = PysbPetabProblem.from_yaml(yaml_file, + flatten=case.startswith('0006')) else: raise ValueError(f"Unsupported model_type: {model_type}") # compile amici model + if case.startswith('0006') and model_type != "pysb": + petab.flatten_timepoint_specific_output_overrides(problem) model_output_dir = f'amici_models/model_{case}' model = import_petab_problem( problem, model_output_dir=model_output_dir, @@ -83,6 +86,10 @@ def _test_case(case, model_type): gt_chi2 = solution[petabtests.CHI2] gt_llh = solution[petabtests.LLH] gt_simulation_dfs = solution[petabtests.SIMULATION_DFS] + if case.startswith('0006'): + # account for flattening + gt_simulation_dfs[0].loc[:, petab.OBSERVABLE_ID] = ('obs_a__10__c0', + 'obs_a__15__c0') tol_chi2 = solution[petabtests.TOL_CHI2] tol_llh = solution[petabtests.TOL_LLH] tol_simulations = solution[petabtests.TOL_SIMULATIONS] diff --git a/version.txt b/version.txt index a95c45d4fb..4aa0906934 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.11.14 +0.11.15