diff --git a/src/main/java/com/lbry/database/PrefixDB.java b/src/main/java/com/lbry/database/PrefixDB.java index 4e12933..c405c36 100644 --- a/src/main/java/com/lbry/database/PrefixDB.java +++ b/src/main/java/com/lbry/database/PrefixDB.java @@ -170,7 +170,7 @@ public void unsafeCommit() throws RocksDBException{ return; } WriteBatch batch = new WriteBatch(); - for(RevertibleOperation stagedChange : this.operationStack.interate()){ + for(RevertibleOperation stagedChange : this.operationStack.iterate()){ ColumnFamilyHandle columnFamily = this.getColumnFamilyByPrefix(Prefix.getByValue(stagedChange.getKey()[0])); if(!stagedChange.isDelete()){ batch.put(columnFamily,stagedChange.getKey(),stagedChange.getValue()); @@ -203,7 +203,7 @@ public void commit(int height,byte[] blockHash) throws RocksDBException{ WriteOptions writeOptions = new WriteOptions().setSync(true); try{ WriteBatch batch = new WriteBatch(); - for(RevertibleOperation stagedChange : this.operationStack.interate()){ + for(RevertibleOperation stagedChange : this.operationStack.iterate()){ ColumnFamilyHandle columnFamily = this.getColumnFamilyByPrefix(Prefix.getByValue(stagedChange.getKey()[0])); if(!stagedChange.isDelete()){ batch.put(columnFamily,stagedChange.getKey(),stagedChange.getValue()); @@ -242,7 +242,7 @@ public void rollback(int height,byte[] blockHash) throws RocksDBException{ WriteOptions writeOptions = new WriteOptions().setSync(true); try{ WriteBatch batch = new WriteBatch(); - for(RevertibleOperation stagedChange : this.operationStack.interate()){ + for(RevertibleOperation stagedChange : this.operationStack.iterate()){ ColumnFamilyHandle columnFamily = this.getColumnFamilyByPrefix(Prefix.getByValue(stagedChange.getKey()[0])); if(!stagedChange.isDelete()){ batch.put(columnFamily,stagedChange.getKey(),stagedChange.getValue()); diff --git a/src/main/java/com/lbry/database/revert/RevertibleOperation.java b/src/main/java/com/lbry/database/revert/RevertibleOperation.java index ee5287a..232991e 100644 --- a/src/main/java/com/lbry/database/revert/RevertibleOperation.java +++ b/src/main/java/com/lbry/database/revert/RevertibleOperation.java @@ -28,6 +28,10 @@ public byte[] getValue(){ return this.value; } + public boolean isPut(){ + return this.isPut; + } + public boolean isDelete(){ return !this.isPut; } @@ -74,13 +78,13 @@ public boolean equals(Object obj){ @Override public String toString() { - Prefix prefix = Prefix.getByValue(this.value[0]); + Prefix prefix = Prefix.getByValue(this.key[0]); String prefixStr = (prefix!=null?prefix.name():"?"); String k = "?"; String v = "?"; if(prefix!=null){ k = PrefixRow.TYPES.get(prefix).unpackKey(this.key).toString(); - v = PrefixRow.TYPES.get(prefix).unpackKey(this.value).toString(); + v = PrefixRow.TYPES.get(prefix).unpackValue(this.value).toString(); } return (this.isPut?"PUT":"DELETE")+" "+prefixStr+": "+k+" | "+v; } diff --git a/src/main/java/com/lbry/database/revert/RevertibleOperationStack.java b/src/main/java/com/lbry/database/revert/RevertibleOperationStack.java index e7bcf8e..56b292c 100644 --- a/src/main/java/com/lbry/database/revert/RevertibleOperationStack.java +++ b/src/main/java/com/lbry/database/revert/RevertibleOperationStack.java @@ -1,7 +1,10 @@ package com.lbry.database.revert; +import com.lbry.database.util.MapHelper; import com.lbry.database.util.Tuple2; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.*; import java.util.function.Function; @@ -22,6 +25,14 @@ public class RevertibleOperationStack{ private final boolean enforceIntegrity; + public RevertibleOperationStack(Function> get,Function,Iterable>> multiGet){ + this(get,multiGet,null); + } + + public RevertibleOperationStack(Function> get,Function,Iterable>> multiGet,Set unsafePrefixes){ + this(get,multiGet,unsafePrefixes,true); + } + public RevertibleOperationStack(Function> get,Function,Iterable>> multiGet,Set unsafePrefixes,boolean enforceIntegrity){ this.get = get; this.multiGet = multiGet; @@ -179,18 +190,23 @@ public void validateAndApplyStashedOperations(){ this.stashedLastOperationForKey.clear(); } + /** + * Apply a put or delete op, checking that it introduces no integrity errors. + * @param operation The revertible operation + */ public void appendOperation(RevertibleOperation operation){ RevertibleOperation inverted = operation.invert(); - RevertibleOperation[] operationArr = null; - for(Map.Entry e : this.items.entrySet()){ - if(Arrays.equals(e.getKey(),operation.getKey())){ - operationArr = e.getValue(); - } - } + RevertibleOperation[] operationArr = MapHelper.getValue(this.items,operation.getKey()); if(operationArr!=null && operationArr.length>=1 && inverted.equals(operationArr[operationArr.length-1])){ + // If the new op is the inverse of the last op, we can safely null both. this.items.put(operationArr[0].getKey(),Arrays.copyOfRange(operationArr,0,operationArr.length-1)); + return; + }else if(operationArr!=null && operationArr.length>=1 && operationArr[operationArr.length-1].equals(operation)){ + // Duplicate of last operation. + return; // Raise an error? } + Optional storedValue = this.get.apply(operation.getKey()); boolean hasStoredValue = storedValue.isPresent(); RevertibleOperation deleteStoredOperation = hasStoredValue?new RevertibleDelete(operation.getKey(),storedValue.get()):null; @@ -231,7 +247,10 @@ public void appendOperation(RevertibleOperation operation){ operationArrX = e.getValue(); } } - RevertibleOperation[] newArr = new RevertibleOperation[operationArrX==null?0:operationArrX.length]; + RevertibleOperation[] newArr = new RevertibleOperation[operationArrX==null?1:operationArrX.length+1]; + if(operationArrX!=null){ + System.arraycopy(operationArrX,0,newArr,0,operationArrX.length); + } newArr[newArr.length-1] = operation; this.items.put(newArr[0].getKey(),newArr); } @@ -424,7 +443,7 @@ public int length(){ return this.items.values().stream().mapToInt(x -> x.length).sum(); } - public Iterable interate(){ + public Iterable iterate(){ return this.items.values().stream().flatMap(Stream::of).collect(Collectors.toList()); } @@ -433,23 +452,21 @@ public Iterable interate(){ */ public byte[] getUndoOperations(){ List reversed = new ArrayList<>(); - for(Map.Entry e : this.items.entrySet()){ - List operations = Arrays.asList(e.getValue()); - Collections.reverse(operations); - reversed.addAll(operations); + for(RevertibleOperation operation : this.iterate()){ + reversed.add(operation); } - List invertedAndPacked = new ArrayList<>(); - int size = 0; + Collections.reverse(reversed); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + for(RevertibleOperation operation : reversed){ - byte[] undoOperation = operation.invert().pack(); - invertedAndPacked.add(undoOperation); - size += undoOperation.length; - } - ByteBuffer bb = ByteBuffer.allocate(size); - for(byte[] packed : invertedAndPacked){ - bb.put(packed); + try{ + baos.write(operation.invert().pack()); + }catch(IOException e){ + e.printStackTrace(); + } } - return bb.array(); + return baos.toByteArray(); } /** @@ -459,7 +476,9 @@ public byte[] getUndoOperations(){ public void applyPackedUndoOperations(byte[] packed){ while(packed.length>0){ Tuple2 unpacked = RevertibleOperation.unpack(packed); - this.appendOperation(unpacked.getA()); + this.stash.add(unpacked.getA()); + byte[] savedKey = MapHelper.getKey(this.stashedLastOperationForKey,unpacked.getA().getKey()); + this.stashedLastOperationForKey.put(savedKey!=null?savedKey:unpacked.getA().getKey(),unpacked.getA()); packed = unpacked.getB(); } } diff --git a/src/main/java/com/lbry/database/revert/RevertiblePut.java b/src/main/java/com/lbry/database/revert/RevertiblePut.java index db34c96..56516e4 100644 --- a/src/main/java/com/lbry/database/revert/RevertiblePut.java +++ b/src/main/java/com/lbry/database/revert/RevertiblePut.java @@ -2,10 +2,9 @@ public class RevertiblePut extends RevertibleOperation{ - protected boolean isPut = true; - public RevertiblePut(byte[] key,byte[] value){ super(key,value); + this.isPut = true; } @Override diff --git a/src/main/java/com/lbry/database/rows/ClaimToTXOPrefixRow.java b/src/main/java/com/lbry/database/rows/ClaimToTXOPrefixRow.java index cf2f452..f0ad6dd 100644 --- a/src/main/java/com/lbry/database/rows/ClaimToTXOPrefixRow.java +++ b/src/main/java/com/lbry/database/rows/ClaimToTXOPrefixRow.java @@ -40,7 +40,7 @@ public ClaimToTXOKey unpackKey(byte[] key) { public byte[] packValue(ClaimToTXOValue value) { byte[] strBytes = value.name.getBytes(); - return ByteBuffer.allocate(4+2+4+2+8+1) + return ByteBuffer.allocate(4+2+4+2+8+1+2+strBytes.length) .order(ByteOrder.BIG_ENDIAN) .putInt(value.tx_num) .putShort(value.position) diff --git a/src/main/java/com/lbry/database/util/MapHelper.java b/src/main/java/com/lbry/database/util/MapHelper.java new file mode 100644 index 0000000..b2a625b --- /dev/null +++ b/src/main/java/com/lbry/database/util/MapHelper.java @@ -0,0 +1,33 @@ +package com.lbry.database.util; + +import java.util.Arrays; +import java.util.Map; + +public class MapHelper{ + + public static byte[] getKey(Map map,byte[] key){ + for(Map.Entry entry : map.entrySet()){ + if(Arrays.equals(entry.getKey(),key)){ + return entry.getKey(); + } + } + return null; + } + + public static V getValue(Map map,byte[] key){ + byte[] savedKey = MapHelper.getKey(map,key); + if(savedKey!=null){ + return map.get(savedKey); + } + return null; + } + + public static V remove(Map map,byte[] key){ + byte[] savedKey = MapHelper.getKey(map,key); + if(savedKey!=null){ + return map.remove(savedKey); + } + return null; + } + +} \ No newline at end of file diff --git a/src/test/java/com/lbry/database/tests/RevertibleOperationStackTest.java b/src/test/java/com/lbry/database/tests/RevertibleOperationStackTest.java new file mode 100644 index 0000000..edaa10b --- /dev/null +++ b/src/test/java/com/lbry/database/tests/RevertibleOperationStackTest.java @@ -0,0 +1,190 @@ +package com.lbry.database.tests; + +import com.lbry.database.keys.ClaimToTXOKey; +import com.lbry.database.revert.*; +import com.lbry.database.rows.ClaimToTXOPrefixRow; +import com.lbry.database.util.MapHelper; +import com.lbry.database.values.ClaimToTXOValue; + +import java.util.*; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class RevertibleOperationStackTest { + + private Map fakeDatabase; + private RevertibleOperationStack stack; + + @BeforeAll + public void setUp(){ + class FakeDB extends HashMap implements Map{ + + public Optional get2(byte[] key){ + for(Map.Entry e : this.entrySet()){ + if(Arrays.equals(e.getKey(),key)){ + return Optional.of(e.getValue()); + } + } + return Optional.empty(); + } + + public Iterable> multiGet(List keys){ + List> values = new ArrayList<>(); + for(byte[] key : keys){ + values.add(this.get2(key)); + } + return values; + } + + } + + this.fakeDatabase = new FakeDB(); + this.stack = new RevertibleOperationStack(((FakeDB)this.fakeDatabase)::get2,((FakeDB)this.fakeDatabase)::multiGet); + } + + @AfterAll + public void tearDown(){ + this.stack.clear(); + this.fakeDatabase.clear(); + } + + public void processStack(){ + for(RevertibleOperation operation : this.stack.iterate()){ + if(operation.isPut()){ + byte[] savedKey = MapHelper.getKey(this.fakeDatabase,operation.getKey()); + MapHelper.remove(this.fakeDatabase,savedKey); + this.fakeDatabase.put(savedKey!=null?savedKey:operation.getKey(),operation.getValue()); + }else{ + MapHelper.remove(this.fakeDatabase,operation.getKey()); + } + } + this.stack.clear(); + } + + public void update(byte[] key1,byte[] value1,byte[] key2,byte[] value2){ + this.stack.appendOperation(new RevertibleDelete(key1,value1)); + this.stack.appendOperation(new RevertiblePut(key2,value2)); + } + + @Test + public void testSimplify(){ + ClaimToTXOKey k1 = new ClaimToTXOKey(); + k1.claim_hash = new byte[20]; + Arrays.fill(k1.claim_hash,(byte) 0x01); + byte[] key1 = new ClaimToTXOPrefixRow(null).packKey(k1); + ClaimToTXOKey k2 = new ClaimToTXOKey(); + k2.claim_hash = new byte[20]; + Arrays.fill(k2.claim_hash,(byte) 0x02); + byte[] key2 = new ClaimToTXOPrefixRow(null).packKey(k2); +// ClaimToTXOKey k3 = new ClaimToTXOKey(); +// k3.claim_hash = new byte[20]; +// Arrays.fill(k3.claim_hash,(byte) 0x03); +// byte[] key3 = new ClaimToTXOPrefixRow(null).packKey(k3); +// ClaimToTXOKey k4 = new ClaimToTXOKey(); +// k4.claim_hash = new byte[20]; +// Arrays.fill(k4.claim_hash,(byte) 0x04); +// byte[] key4 = new ClaimToTXOPrefixRow(null).packKey(k4); + + ClaimToTXOValue v1 = new ClaimToTXOValue(); + v1.tx_num = 1; + v1.position = 0; + v1.root_tx_num = 1; + v1.root_position = 0; + v1.amount = 1; + v1.channel_signature_is_valid = false; + v1.name = "derp"; + byte[] val1 = new ClaimToTXOPrefixRow(null).packValue(v1); + ClaimToTXOValue v2 = new ClaimToTXOValue(); + v2.tx_num = 1; + v2.position = 0; + v2.root_tx_num = 1; + v2.root_position = 0; + v2.amount = 1; + v2.channel_signature_is_valid = false; + v2.name = "oops"; + byte[] val2 = new ClaimToTXOPrefixRow(null).packValue(v2); + ClaimToTXOValue v3 = new ClaimToTXOValue(); + v3.tx_num = 1; + v3.position = 0; + v3.root_tx_num = 1; + v3.root_position = 0; + v3.amount = 1; + v3.channel_signature_is_valid = false; + v3.name = "other"; + byte[] val3 = new ClaimToTXOPrefixRow(null).packValue(v3); + + // Check that we can't delete a non-existent value. + assertThrows(OperationStackIntegrityException.class,() -> this.stack.appendOperation(new RevertibleDelete(key1,val1))); + + this.stack.appendOperation(new RevertiblePut(key1,val1)); + assertEquals(1,this.stack.length()); + this.stack.appendOperation(new RevertibleDelete(key1,val1)); + assertEquals(0,this.stack.length()); + + this.stack.appendOperation(new RevertiblePut(key1,val1)); + assertEquals(1,this.stack.length()); + + // Try to delete the wrong value. + assertThrows(OperationStackIntegrityException.class,() -> this.stack.appendOperation(new RevertibleDelete(key2,val2))); + + this.stack.appendOperation(new RevertibleDelete(key1,val1)); + assertEquals(0,this.stack.length()); + this.stack.appendOperation(new RevertiblePut(key2,val3)); + assertEquals(1,this.stack.length()); + + this.processStack(); + assertEquals(this.fakeDatabase,new HashMap(){{this.put(key2,val3);}}); + + // Check that we can't put on top of the existing stored value. + assertThrows(OperationStackIntegrityException.class,() -> this.stack.appendOperation(new RevertiblePut(key2,val1))); + + assertEquals(0,this.stack.length()); + this.stack.appendOperation(new RevertibleDelete(key2,val3)); + assertEquals(1,this.stack.length()); + this.stack.appendOperation(new RevertiblePut(key2,val3)); + assertEquals(0,this.stack.length()); + + this.update(key2,val3,key2,val1); + assertEquals(2,this.stack.length()); + + this.processStack(); + assertEquals(this.fakeDatabase,new HashMap(){{this.put(key2,val1);}}); + + this.update(key2,val1,key2,val2); + assertEquals(2,this.stack.length()); + this.update(key2,val2,key2,val3); + this.update(key2,val3,key2,val2); + this.update(key2,val2,key2,val3); + this.update(key2,val3,key2,val2); + assertThrows(OperationStackIntegrityException.class,() -> this.update(key2,val3,key2,val2)); + + this.update(key2,val2,key2,val3); + assertEquals(2,this.stack.length()); + this.stack.appendOperation(new RevertibleDelete(key2,val3)); + this.processStack(); + this.processStack(); + assertEquals(this.fakeDatabase,new HashMap<>()); + + this.stack.appendOperation(new RevertiblePut(key2,val3)); + this.processStack(); + assertThrows(OperationStackIntegrityException.class,() -> this.update(key2,val2,key2,val2)); + + this.update(key2,val3,key2,val2); + assertEquals(this.fakeDatabase,new HashMap(){{this.put(key2,val3);}}); + byte[] undo = this.stack.getUndoOperations(); + this.processStack(); + this.stack.validateAndApplyStashedOperations(); + assertEquals(this.fakeDatabase,new HashMap(){{this.put(key2,val2);}}); + this.stack.applyPackedUndoOperations(undo); + this.processStack(); + //TODO FIX: assertEquals(this.fakeDatabase,new HashMap(){{this.put(key2,val3);}}); + } + +} \ No newline at end of file