Skip to content

Commit

Permalink
Merge pull request #173 from typesafehub/wip/havocp-out-of-range-bytes
Browse files Browse the repository at this point in the history
Throw an exception if size-in-bytes values are out of Long range
  • Loading branch information
havocp committed Dec 27, 2014
2 parents 94dda5f + 02867c8 commit 197be59
Show file tree
Hide file tree
Showing 3 changed files with 70 additions and 16 deletions.
7 changes: 7 additions & 0 deletions HOCON.md
Original file line number Diff line number Diff line change
Expand Up @@ -1265,6 +1265,13 @@ spec copies that. You can certainly find examples of mapping these
to powers of ten, though. If you don't like ambiguity, don't use
the single-letter abbreviations.

Note: any value in zetta/zebi or yotta/yobi will overflow a 64-bit
integer, and of course large-enough values in any of the units may
overflow. Most real-world APIs and apps will not support byte
counts that overflow a 64-bit integer. The huge units are provided
just to be complete but probably aren't useful in practice. At
least not in 2014.

### Config object merging and file merging

It may be useful to offer a method to merge two objects. If such a
Expand Down
23 changes: 13 additions & 10 deletions config/src/main/java/com/typesafe/config/impl/SimpleConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@

import java.io.ObjectStreamException;
import java.io.Serializable;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.HashMap;
Expand Down Expand Up @@ -575,19 +577,13 @@ private static enum MemoryUnit {
final String prefix;
final int powerOf;
final int power;
final long bytes;
final BigInteger bytes;

MemoryUnit(String prefix, int powerOf, int power) {
this.prefix = prefix;
this.powerOf = powerOf;
this.power = power;
int i = power;
long bytes = 1;
while (i > 0) {
bytes *= powerOf;
--i;
}
this.bytes = bytes;
this.bytes = BigInteger.valueOf(powerOf).pow(power);
}

private static Map<String, MemoryUnit> makeUnitsMap() {
Expand Down Expand Up @@ -667,13 +663,20 @@ public static long parseBytes(String input, ConfigOrigin originForException,
}

try {
BigInteger result;
// if the string is purely digits, parse as an integer to avoid
// possible precision loss; otherwise as a double.
if (numberString.matches("[0-9]+")) {
return Long.parseLong(numberString) * units.bytes;
result = units.bytes.multiply(new BigInteger(numberString));
} else {
return (long) (Double.parseDouble(numberString) * units.bytes);
BigDecimal resultDecimal = (new BigDecimal(units.bytes)).multiply(new BigDecimal(numberString));
result = resultDecimal.toBigInteger();
}
if (result.bitLength() < 64)
return result.longValue();
else
throw new ConfigException.BadValue(originForException, pathForException,
"size-in-bytes value is out of range for a 64-bit long: '" + input + "'");
} catch (NumberFormatException e) {
throw new ConfigException.BadValue(originForException, pathForException,
"Could not parse size-in-bytes number '" + numberString + "'");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,16 +50,19 @@ class UnitParserTest extends TestUtils {

val conf = parseConfig("foo = 1d")
assertEquals("could get 1d from conf as days",
1L, conf.getDuration("foo", TimeUnit.DAYS))
1L, conf.getDuration("foo", TimeUnit.DAYS))
assertEquals("could get 1d from conf as nanos",
dayInNanos, conf.getNanoseconds("foo"))
dayInNanos, conf.getNanoseconds("foo"))
assertEquals("could get 1d from conf as millis",
TimeUnit.DAYS.toMillis(1), conf.getMilliseconds("foo"))
TimeUnit.DAYS.toMillis(1), conf.getMilliseconds("foo"))
}

@Test
def parseMemorySizeInBytes(): Unit = {
def parseMem(s: String) = SimpleConfig.parseBytes(s, fakeOrigin(), "test")
def parseMem(s: String): Long = SimpleConfig.parseBytes(s, fakeOrigin(), "test")

assertEquals(Long.MaxValue, parseMem(s"${Long.MaxValue} bytes"))
assertEquals(Long.MinValue, parseMem(s"${Long.MinValue} bytes"))

val oneMebis = List("1048576", "1048576b", "1048576bytes", "1048576byte",
"1048576 b", "1048576 bytes",
Expand Down Expand Up @@ -88,7 +91,7 @@ class UnitParserTest extends TestUtils {
}

var result = 1024L * 1024 * 1024
for (unit <- Seq("tebi", "pebi", "exbi", "zebi", "yobi")) {
for (unit <- Seq("tebi", "pebi", "exbi")) {
val first = unit.substring(0, 1).toUpperCase()
result = result * 1024;
assertEquals(result, parseMem("1" + first))
Expand All @@ -99,7 +102,7 @@ class UnitParserTest extends TestUtils {
}

result = 1000L * 1000 * 1000
for (unit <- Seq("tera", "peta", "exa", "zetta", "yotta")) {
for (unit <- Seq("tera", "peta", "exa")) {
val first = unit.substring(0, 1).toUpperCase()
result = result * 1000;
assertEquals(result, parseMem("1" + first + "B"))
Expand All @@ -119,4 +122,45 @@ class UnitParserTest extends TestUtils {
}
assertTrue(e2.getMessage().contains("size-in-bytes number"))
}

// later on we'll want to check this with BigInteger version of getBytes
@Test
def parseHugeMemorySizes(): Unit = {
def parseMem(s: String): Long = SimpleConfig.parseBytes(s, fakeOrigin(), "test")
def assertOutOfRange(s: String) = {
val fail = intercept[ConfigException.BadValue] {
parseMem(s)
}
assertTrue("number was too big", fail.getMessage.contains("out of range"))
}

import java.math.BigInteger
assertOutOfRange(s"${BigInteger.valueOf(Long.MaxValue).add(BigInteger.valueOf(1)).toString} bytes")
assertOutOfRange(s"${BigInteger.valueOf(Long.MinValue).subtract(BigInteger.valueOf(1)).toString} bytes")

var result = 1024L * 1024 * 1024
for (unit <- Seq("zebi", "yobi")) {
val first = unit.substring(0, 1).toUpperCase()
assertOutOfRange("1" + first)
assertOutOfRange("1" + first + "i")
assertOutOfRange("1" + first + "iB")
assertOutOfRange("1" + unit + "byte")
assertOutOfRange("1" + unit + "bytes")
assertOutOfRange("1.1" + first)
assertOutOfRange("-1" + first)
}

result = 1000L * 1000 * 1000
for (unit <- Seq("zetta", "yotta")) {
val first = unit.substring(0, 1).toUpperCase()
assertOutOfRange("1" + first + "B")
assertOutOfRange("1" + unit + "byte")
assertOutOfRange("1" + unit + "bytes")
assertOutOfRange("1.1" + first + "B")
assertOutOfRange("-1" + first + "B")
}

assertOutOfRange("1000 exabytes")
assertOutOfRange("10000000 petabytes")
}
}

0 comments on commit 197be59

Please sign in to comment.