[PATCH v2] libstdc++: fix UB when ellint phi argument is infinity

[Kito Cheng]

When phi is +/-infinity, the periodicity-reduction step computes
std::floor(infinity) which returns infinity, then casts it to int.
That cast is undefined behavior per [conv.fpint] because infinity is
outside the range of int.  A conforming compiler is free to exploit
this UB; in practice, Clang eliminates the loop-exit check and
produces an infinite loop.  GCC currently does not exploit this
particular UB, but the cast is still undefined and should be fixed.

Fix by rejecting non-finite phi before the floor+cast and throwing
std::domain_error.  NaN phi is already handled by the __isnan check
above, so the new guard only needs __builtin_isinf(phi); std::isfinite
is unavailable in the C++98 build of this header.  Boost guards against
this by an overflow check (phi >= max_value<T>()), where -infinity is
first negated to +infinity by the sign-handling block[1].

[1] https://www.boost.org/doc/libs/1_85_0/boost/math/special_functions/ellint_1.hpp

Changes in v2:
- Use __builtin_isinf(phi) instead of std::isfinite(phi).
- Clean up testcases.

libstdc++-v3/ChangeLog:

	* include/tr1/ell_integral.tcc (__ellint_1): Throw domain_error
	when phi is not finite.
	(__ellint_2): Likewise.
	(__ellint_3): Likewise.
	* testsuite/special_functions/11_ellint_1/check_inf.cc: New test.
	* testsuite/special_functions/12_ellint_2/check_inf.cc: New test.
	* testsuite/special_functions/13_ellint_3/check_inf.cc: New test.
---
 libstdc++-v3/include/tr1/ell_integral.tcc     |  6 +-
 .../11_ellint_1/check_inf.cc                  | 70 +++++++++++++++++++
 .../12_ellint_2/check_inf.cc                  | 69 ++++++++++++++++++
 .../13_ellint_3/check_inf.cc                  | 69 ++++++++++++++++++
 4 files changed, 211 insertions(+), 3 deletions(-)
 create mode 100644 libstdc++-v3/testsuite/special_functions/11_ellint_1/check_inf.cc
 create mode 100644 libstdc++-v3/testsuite/special_functions/12_ellint_2/check_inf.cc
 create mode 100644 libstdc++-v3/testsuite/special_functions/13_ellint_3/check_inf.cc

diff --git a/libstdc++-v3/include/tr1/ell_integral.tcc b/libstdc++-v3/include/tr1/ell_integral.tcc
index e78d4e538a6..8e66857bc7b 100644
--- a/libstdc++-v3/include/tr1/ell_integral.tcc
+++ b/libstdc++-v3/include/tr1/ell_integral.tcc
@@ -223,7 +223,7 @@ namespace tr1
 
       if (__isnan(__k) || __isnan(__phi))
         return std::numeric_limits<_Tp>::quiet_NaN();
-      else if (std::abs(__k) > _Tp(1))
+      else if ((std::abs(__k) > _Tp(1)) || __builtin_isinf(__phi))
         std::__throw_domain_error(__N("Bad argument in __ellint_1."));
       else
         {
@@ -437,7 +437,7 @@ namespace tr1
 
       if (__isnan(__k) || __isnan(__phi))
         return std::numeric_limits<_Tp>::quiet_NaN();
-      else if (std::abs(__k) > _Tp(1))
+      else if ((std::abs(__k) > _Tp(1)) || __builtin_isinf(__phi))
         std::__throw_domain_error(__N("Bad argument in __ellint_2."));
       else
         {
@@ -705,7 +705,7 @@ namespace tr1
 
       if (__isnan(__k) || __isnan(__nu) || __isnan(__phi))
         return std::numeric_limits<_Tp>::quiet_NaN();
-      else if (std::abs(__k) > _Tp(1))
+      else if ((std::abs(__k) > _Tp(1)) || __builtin_isinf(__phi))
         std::__throw_domain_error(__N("Bad argument in __ellint_3."));
       else
         {
diff --git a/libstdc++-v3/testsuite/special_functions/11_ellint_1/check_inf.cc b/libstdc++-v3/testsuite/special_functions/11_ellint_1/check_inf.cc
new file mode 100644
index 00000000000..0b18d27e280
--- /dev/null
+++ b/libstdc++-v3/testsuite/special_functions/11_ellint_1/check_inf.cc
@@ -0,0 +1,70 @@
+// { dg-do run { target c++17 } }
+// { dg-options "-D__STDCPP_WANT_MATH_SPEC_FUNCS__" }
+
+// 8.1.11 ellint_1 - non-finite phi must not produce UB
+
+#include <cmath>
+#include <limits>
+#include <stdexcept>
+#include <testsuite_hooks.h>
+
+void
+test01()
+{
+  // +infinity phi should throw domain_error, not loop forever.
+  bool caught = false;
+  try
+    {
+      volatile float r = std::ellint_1f(0.5F,
+                               std::numeric_limits<float>::infinity());
+      (void) r;
+    }
+  catch (const std::domain_error&)
+    {
+      caught = true;
+    }
+  VERIFY(caught);
+}
+
+void
+test02()
+{
+  bool caught = false;
+  try
+    {
+      volatile double r = std::ellint_1(0.5,
+                               std::numeric_limits<double>::infinity());
+      (void) r;
+    }
+  catch (const std::domain_error&)
+    {
+      caught = true;
+    }
+  VERIFY(caught);
+}
+
+void
+test03()
+{
+  bool caught = false;
+  try
+    {
+      volatile long double r = std::ellint_1l(0.5L,
+                       std::numeric_limits<long double>::infinity());
+      (void) r;
+    }
+  catch (const std::domain_error&)
+    {
+      caught = true;
+    }
+  VERIFY(caught);
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+  return 0;
+}
diff --git a/libstdc++-v3/testsuite/special_functions/12_ellint_2/check_inf.cc b/libstdc++-v3/testsuite/special_functions/12_ellint_2/check_inf.cc
new file mode 100644
index 00000000000..2d2457bef2d
--- /dev/null
+++ b/libstdc++-v3/testsuite/special_functions/12_ellint_2/check_inf.cc
@@ -0,0 +1,69 @@
+// { dg-do run { target c++17 } }
+// { dg-options "-D__STDCPP_WANT_MATH_SPEC_FUNCS__" }
+
+// 8.1.12 ellint_2 - non-finite phi must not produce UB
+
+#include <cmath>
+#include <limits>
+#include <stdexcept>
+#include <testsuite_hooks.h>
+
+void
+test01()
+{
+  bool caught = false;
+  try
+    {
+      volatile float r = std::ellint_2f(0.5F,
+                               std::numeric_limits<float>::infinity());
+      (void) r;
+    }
+  catch (const std::domain_error&)
+    {
+      caught = true;
+    }
+  VERIFY(caught);
+}
+
+void
+test02()
+{
+  bool caught = false;
+  try
+    {
+      volatile double r = std::ellint_2(0.5,
+                               std::numeric_limits<double>::infinity());
+      (void) r;
+    }
+  catch (const std::domain_error&)
+    {
+      caught = true;
+    }
+  VERIFY(caught);
+}
+
+void
+test03()
+{
+  bool caught = false;
+  try
+    {
+      volatile long double r = std::ellint_2l(0.5L,
+                       std::numeric_limits<long double>::infinity());
+      (void) r;
+    }
+  catch (const std::domain_error&)
+    {
+      caught = true;
+    }
+  VERIFY(caught);
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+  return 0;
+}
diff --git a/libstdc++-v3/testsuite/special_functions/13_ellint_3/check_inf.cc b/libstdc++-v3/testsuite/special_functions/13_ellint_3/check_inf.cc
new file mode 100644
index 00000000000..a21143d0d50
--- /dev/null
+++ b/libstdc++-v3/testsuite/special_functions/13_ellint_3/check_inf.cc
@@ -0,0 +1,69 @@
+// { dg-do run { target c++17 } }
+// { dg-options "-D__STDCPP_WANT_MATH_SPEC_FUNCS__" }
+
+// 8.1.13 ellint_3 - non-finite phi must not produce UB
+
+#include <cmath>
+#include <limits>
+#include <stdexcept>
+#include <testsuite_hooks.h>
+
+void
+test01()
+{
+  bool caught = false;
+  try
+    {
+      volatile float r = std::ellint_3f(0.5F, 0.5F,
+                               std::numeric_limits<float>::infinity());
+      (void) r;
+    }
+  catch (const std::domain_error&)
+    {
+      caught = true;
+    }
+  VERIFY(caught);
+}
+
+void
+test02()
+{
+  bool caught = false;
+  try
+    {
+      volatile double r = std::ellint_3(0.5, 0.5,
+                               std::numeric_limits<double>::infinity());
+      (void) r;
+    }
+  catch (const std::domain_error&)
+    {
+      caught = true;
+    }
+  VERIFY(caught);
+}
+
+void
+test03()
+{
+  bool caught = false;
+  try
+    {
+      volatile long double r = std::ellint_3l(0.5L, 0.5L,
+                       std::numeric_limits<long double>::infinity());
+      (void) r;
+    }
+  catch (const std::domain_error&)
+    {
+      caught = true;
+    }
+  VERIFY(caught);
+}
+
+int
+main()
+{
+  test01();
+  test02();
+  test03();
+  return 0;
+}
-- 
2.54.0