diff options
author | Reinier Heeres <reinier@heeres.eu> | 2009-04-22 21:54:57 (GMT) |
---|---|---|
committer | Reinier Heeres <reinier@heeres.eu> | 2009-04-22 21:54:57 (GMT) |
commit | 46a6c14e9d6c45289080059779b35ce6288458d0 (patch) | |
tree | 24f2141dd4210af8318d285832c896a2b019e0a2 | |
parent | 067ba4d139545d271beb255f8856219be33cbcd6 (diff) |
Fix precision issue
-rw-r--r-- | functions.py | 2 | ||||
-rw-r--r-- | mathlib.py | 13 |
2 files changed, 13 insertions, 2 deletions
diff --git a/functions.py b/functions.py index 7f8a552..ac45610 100644 --- a/functions.py +++ b/functions.py @@ -80,7 +80,7 @@ def _d(val): d = _Decimal(val) return d.normalize() elif type(val) is types.FloatType or hasattr(val, '__float__'): - s = '%.10e' % float(val) + s = '%.18e' % float(val) d = _Decimal(s) return d.normalize() else: @@ -90,7 +90,7 @@ class MathLib: d = Decimal(val) return d.normalize() elif type(val) is types.FloatType or hasattr(val, '__float__'): - s = '%.10e' % float(val) + s = '%.18e' % float(val) d = Decimal(s) return d.normalize() else: @@ -197,3 +197,14 @@ class MathLib: (sign, d, e) = n.normalize().as_tuple() return e >= 0 + +if __name__ == "__main__": + ml = MathLib() + val = 0.99999999999999878 + print 'is_int(%.18e): %s' % (val, ml.is_int(val)) + # Beyond float precision + val = 0.999999999999999999 + print 'is_int(%.18e): %s' % (val, ml.is_int(val)) + val = ml.d(0.99999999999999878)**2 + print 'is_int(%s): %s' % (val, ml.is_int(val)) + |