%0 Journal Article %1 9d6e0d55315b46308f4f6a378e6c11fa %A Maddu, Suryanarayana %A Sturm, Dominik %A Mueller, Christian L. %A Sbalzarini, Ivo F. %D 2022 %I IOP Publishing Ltd. %J Machine learning: science and technology %K topic_lifescience ALGORITHM FIS_scads active catastrophic flow forgetting, gradient modeling, multi-objective multi-scale networks, neural physics-informed regularization, training, turbulence, %N 1 %R 10.1088/2632-2153/ac3712 %T Inverse Dirichlet weighting enables reliable training of physics informed neural networks %V 3 %X We characterize and remedy a failure mode that may arise from multi-scale dynamics with scale imbalances during training of deep neural networks, such as physics informed neural networks (PINNs). PINNs are popular machine-learning templates that allow for seamless integration of physical equation models with data. Their training amounts to solving an optimization problem over a weighted sum of data-fidelity and equation-fidelity objectives. Conflicts between objectives can arise from scale imbalances, heteroscedasticity in the data, stiffness of the physical equation, or from catastrophic interference during sequential training. We explain the training pathology arising from this and propose a simple yet effective inverse Dirichlet weighting strategy to alleviate the issue. We compare with Sobolev training of neural networks, providing the baseline of analytically epsilon-optimal training. We demonstrate the effectiveness of inverse Dirichlet weighting in various applications, including a multi-scale model of active turbulence, where we show orders of magnitude improvement in accuracy and convergence over conventional PINN training. For inverse modeling using sequential training, we find that inverse Dirichlet weighting protects a PINN against catastrophic forgetting.

@article{9d6e0d55315b46308f4f6a378e6c11fa, abstract = {We characterize and remedy a failure mode that may arise from multi-scale dynamics with scale imbalances during training of deep neural networks, such as physics informed neural networks (PINNs). PINNs are popular machine-learning templates that allow for seamless integration of physical equation models with data. Their training amounts to solving an optimization problem over a weighted sum of data-fidelity and equation-fidelity objectives. Conflicts between objectives can arise from scale imbalances, heteroscedasticity in the data, stiffness of the physical equation, or from catastrophic interference during sequential training. We explain the training pathology arising from this and propose a simple yet effective inverse Dirichlet weighting strategy to alleviate the issue. We compare with Sobolev training of neural networks, providing the baseline of analytically epsilon-optimal training. We demonstrate the effectiveness of inverse Dirichlet weighting in various applications, including a multi-scale model of active turbulence, where we show orders of magnitude improvement in accuracy and convergence over conventional PINN training. For inverse modeling using sequential training, we find that inverse Dirichlet weighting protects a PINN against catastrophic forgetting.}, added-at = {2024-11-28T16:27:18.000+0100}, author = {Maddu, Suryanarayana and Sturm, Dominik and Mueller, {Christian L.} and Sbalzarini, {Ivo F.}}, biburl = {https://puma.scadsai.uni-leipzig.de/bibtex/2674d63c941f832d62e0046b80229a54f/scadsfct}, day = 15, doi = {10.1088/2632-2153/ac3712}, interhash = {cdf44028ac9cdcadd517e8c1aa70a7f8}, intrahash = {674d63c941f832d62e0046b80229a54f}, issn = {2632-2153}, journal = {Machine learning: science and technology}, keywords = {topic_lifescience ALGORITHM FIS_scads active catastrophic flow forgetting, gradient modeling, multi-objective multi-scale networks, neural physics-informed regularization, training, turbulence,}, language = {English}, month = feb, note = {Funding Information: This work was supported by the German Research Foundation (DFG) – EXC-2068, Cluster of Excellence {\textquoteleft}Physics of Life{\textquoteright}, and by the Center for Scalable Data Analytics and Artificial Intelligence (ScaDS.AI) Dresden/Leipzig, funded by the Federal Ministry of Education and Research (BMBF). The present work was also partly funded by the Center for Advanced Systems Understanding (CASUS), which is financed by Germany{\textquoteright}s Federal Ministry of Education and Research (BMBF) and by the Saxon Ministry for Science, Culture and Tourism (SMWK) with tax funds on the basis of the budget approved by the Saxon State Parliament. Publisher Copyright: {\textcopyright} 2022 The Author(s). Published by IOP Publishing Ltd.}, number = 1, publisher = {IOP Publishing Ltd.}, timestamp = {2024-11-28T17:41:28.000+0100}, title = {Inverse Dirichlet weighting enables reliable training of physics informed neural networks}, volume = 3, year = 2022 }