From 46a6c14e9d6c45289080059779b35ce6288458d0 Mon Sep 17 00:00:00 2001 From: Reinier Heeres Date: Wed, 22 Apr 2009 21:54:57 +0000 Subject: Fix precision issue --- diff --git a/functions.py b/functions.py index 7f8a552..ac45610 100644 --- a/functions.py +++ b/functions.py @@ -80,7 +80,7 @@ def _d(val): d = _Decimal(val) return d.normalize() elif type(val) is types.FloatType or hasattr(val, '__float__'): - s = '%.10e' % float(val) + s = '%.18e' % float(val) d = _Decimal(s) return d.normalize() else: diff --git a/mathlib.py b/mathlib.py index 2cc2956..610bb07 100644 --- a/mathlib.py +++ b/mathlib.py @@ -90,7 +90,7 @@ class MathLib: d = Decimal(val) return d.normalize() elif type(val) is types.FloatType or hasattr(val, '__float__'): - s = '%.10e' % float(val) + s = '%.18e' % float(val) d = Decimal(s) return d.normalize() else: @@ -197,3 +197,14 @@ class MathLib: (sign, d, e) = n.normalize().as_tuple() return e >= 0 + +if __name__ == "__main__": + ml = MathLib() + val = 0.99999999999999878 + print 'is_int(%.18e): %s' % (val, ml.is_int(val)) + # Beyond float precision + val = 0.999999999999999999 + print 'is_int(%.18e): %s' % (val, ml.is_int(val)) + val = ml.d(0.99999999999999878)**2 + print 'is_int(%s): %s' % (val, ml.is_int(val)) + -- cgit v0.9.1