Web   ·   Wiki   ·   Activities   ·   Blog   ·   Lists   ·   Chat   ·   Meeting   ·   Bugs   ·   Git   ·   Translate   ·   Archive   ·   People   ·   Donate
summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorReinier Heeres <reinier@heeres.eu>2009-04-22 21:54:57 (GMT)
committer Reinier Heeres <reinier@heeres.eu>2009-04-22 21:54:57 (GMT)
commit46a6c14e9d6c45289080059779b35ce6288458d0 (patch)
tree24f2141dd4210af8318d285832c896a2b019e0a2
parent067ba4d139545d271beb255f8856219be33cbcd6 (diff)
Fix precision issue
-rw-r--r--functions.py2
-rw-r--r--mathlib.py13
2 files changed, 13 insertions, 2 deletions
diff --git a/functions.py b/functions.py
index 7f8a552..ac45610 100644
--- a/functions.py
+++ b/functions.py
@@ -80,7 +80,7 @@ def _d(val):
d = _Decimal(val)
return d.normalize()
elif type(val) is types.FloatType or hasattr(val, '__float__'):
- s = '%.10e' % float(val)
+ s = '%.18e' % float(val)
d = _Decimal(s)
return d.normalize()
else:
diff --git a/mathlib.py b/mathlib.py
index 2cc2956..610bb07 100644
--- a/mathlib.py
+++ b/mathlib.py
@@ -90,7 +90,7 @@ class MathLib:
d = Decimal(val)
return d.normalize()
elif type(val) is types.FloatType or hasattr(val, '__float__'):
- s = '%.10e' % float(val)
+ s = '%.18e' % float(val)
d = Decimal(s)
return d.normalize()
else:
@@ -197,3 +197,14 @@ class MathLib:
(sign, d, e) = n.normalize().as_tuple()
return e >= 0
+
+if __name__ == "__main__":
+ ml = MathLib()
+ val = 0.99999999999999878
+ print 'is_int(%.18e): %s' % (val, ml.is_int(val))
+ # Beyond float precision
+ val = 0.999999999999999999
+ print 'is_int(%.18e): %s' % (val, ml.is_int(val))
+ val = ml.d(0.99999999999999878)**2
+ print 'is_int(%s): %s' % (val, ml.is_int(val))
+