implementing braidchain record retrieval over HTTP

Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.10.0"
 manifest_format = "2.0"
-project_hash = "cd87006c6e4df4542cf96a4a5957d21f86663451"
+project_hash = "bdf3cf4e1b1646d11c41f2412caa658745298616"
 
 [[deps.ArgTools]]
 uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
@@ -119,9 +119,9 @@ version = "6.2.1+6"
 
 [[deps.HTTP]]
 deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"]
-git-tree-sha1 = "995f762e0182ebc50548c434c171a5bb6635f8e4"
+git-tree-sha1 = "8e59b47b9dc525b70550ca082ce85bcd7f5477cd"
 uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
-version = "1.10.4"
+version = "1.10.5"
 
 [[deps.HistoryTrees]]
 git-tree-sha1 = "5e0e766befebd2108cef8e3a14cf715c8496cb2e"
@@ -375,9 +375,9 @@ version = "1.1.1"
 
 [[deps.ShuffleProofs]]
 deps = ["CryptoGroups", "Random"]
-git-tree-sha1 = "79b5a6fac187a4d0ee5e6a0a4a706ca50775363e"
+git-tree-sha1 = "87b0c49235260736a77ac0356fa4e063406d3ab1"
 uuid = "31a120cc-b3cb-4d07-bbdb-d498660ddfd8"
-version = "0.3.0"
+version = "0.3.1"
 
 [[deps.SimpleBufferStream]]
 git-tree-sha1 = "874e8867b33a00e784c8a7e4b60afe9e037b74e1"

Project.toml

@@ -21,6 +21,7 @@ ShuffleProofs = "31a120cc-b3cb-4d07-bbdb-d498660ddfd8"
 StructHelpers = "4093c41a-2008-41fd-82b8-e3f9d02b504f"
 StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4"
 SwaggerMarkdown = "1b6eb727-ad4b-44eb-9669-b9596a6e760f"
+Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4"
 
@@ -33,7 +34,7 @@ JSON3 = "1"
 Nettle = "1"
 Setfield = "1"
 ShuffleProofs = "0.3"
+StructHelpers = "1.1"
 StructTypes = "1"
 URIs = "1.4"
 julia = "1"
-StructHelpers = "1.1"

src/Client.jl

@@ -10,12 +10,12 @@ using Dates
 using Setfield
 
 import StructTypes
-#using StructHelpers 
 
 using ..Core.Model: Model, Membership, Pseudonym, Proposal, Vote, bytes, TicketID, HMAC, Admission, isbinding, verify, Digest, HashSpec, DemeSpec, Signer,  Commit, ChainState, Proposal, BallotBoxState, isbinding, isopen, digest, commit
 using ..Core.Model: id, hasher, pseudonym, isbinding, generator, state, verify, crypto, index, root, isconsistent, istallied, issuer
 using ..Core.ProtocolSchema: TicketStatus, tokenid, Invite, AckConsistency, AckInclusion, CastAck
 using ..Core.Parser: marshal, unmarshal
+using ..Core.Store: Store
 using ..Authorization: AuthClientMiddleware
 
 import ..Core.Model
@@ -180,7 +180,6 @@ function get_chain_leaf(server::Route, N::Int)
     return ack
 end
 
-
 function get_chain_root(server::Route, N::Int)
 
     response = get(server, "/braidchain/$N/root")
@@ -189,15 +188,26 @@ function get_chain_root(server::Route, N::Int)
     return ack
 end
 
-
 function get_chain_record(server::Route, N::Int)
 
     response = get(server, "/braidchain/$N/record")
-
-    error("Not implemented")
+    record_type = Dict(response.headers)["X-Record-Type"]
+
+    if record_type == "DemeSpec"
+        return unmarshal(response.body, DemeSpec)
+    elseif record_type == "Membership"
+        return unmarshal(response.body, Membership)
+    elseif record_type == "Proposal"
+        return unmarshal(response.body, Proposal)
+    elseif record_type == "BraidReceipt"
+        return Store.load(Model.BraidReceipt, response.body)
+    else
+        error("Record type $record_type not recognized")
+    end
+
+    return
 end
 
-
 function get_ballotbox_commit(server::Route, uuid::UUID)
 
     response = get(server, "/poolingstation/$uuid/commit")

src/Core/Model/braidchains.jl

@@ -12,6 +12,8 @@ struct BraidChainLedger
     records::AbstractVector{Transaction}
 end
 
+BraidChainLedger() = BraidChainLedger(Transaction[])
+
 @batteries BraidChainLedger
 
 Base.push!(ledger::BraidChainLedger, record::Transaction) = push!(ledger.records, record)

src/Core/Store.jl

@@ -1,10 +1,10 @@
 module Store
 
+using Tar
 using ShuffleProofs: ShuffleProofs
 using ..Model: BraidChainLedger, DemeSpec, Membership, BraidReceipt, Proposal, Transaction, BallotBoxLedger, CastRecord, Seal
 using ..Parser: marshal, unmarshal
 
-
 index2name(i::UInt16) = reinterpret(UInt8, [i]) |> reverse |> bytes2hex |> uppercase
 index2name(i::Int) = index2name(UInt16(i))
 
@@ -103,6 +103,46 @@ end
 load(::Type{BraidReceipt}, dir::String, index::UInt16) = load(BraidReceipt, joinpath(dir, BRAIDRECEIPT_DIR, index2name(index)))
 
 
+struct TarPath <: ShuffleProofs.Path
+    io::IO
+    path::String
+end
+
+function tar(io::IO, path::String, data::Vector{UInt8}; mode::Integer=0o644)
+
+    n = length(data)
+
+    Tar.write_header(io, Tar.Header(path, :file, mode, n, ""))
+    Tar.write_data(io, IOBuffer(data), size=n)
+
+end
+
+Base.joinpath(path::TarPath, args...) = TarPath(path.io, joinpath(path.path, args...))
+Base.mkdir(path::TarPath) = nothing
+Base.mkpath(path::TarPath) = nothing
+Base.write(path::TarPath, data::Vector{UInt8}) = tar(path.io, path.path, data)
+
+
+function tar(io::IO, record::BraidReceipt)
+
+    ShuffleProofs.save(record.braid, TarPath(io, "braid"))    
+    tar(io, "demespec.json", marshal(record.producer))
+    tar(io, "seal.json", marshal(record.approval))
+
+    return
+end
+
+function load(::Type{BraidReceipt}, io::IO) 
+
+    dir = joinpath(tempdir(), "braidreceipt")
+    rm(dir, force=true, recursive=true)
+
+    Tar.extract(io, dir)
+
+    return load(BraidReceipt, dir)
+end
+
+
 function save(record::CastRecord, path::String; force=false)
 
     if isfile(path) 

src/Server/Service.jl

@@ -3,14 +3,16 @@ module Service
 # This is the outermost layer for the sercvice concerned with providing services for outsied world. 
 # Defines how HTTP requests are processed
 using ..Core.Parser: marshal, unmarshal
-using ..Core.Model: TicketID, Digest, Pseudonym, Digest, Membership, Proposal, Vote, bytes
+using ..Core.Store: tar
+using ..Core.Model: TicketID, Digest, Pseudonym, Digest, Membership, Proposal, Vote, bytes, BraidReceipt
 using ..Authorization: AuthServerMiddleware, timestamp, credential
 using ..Mapper
 
 using Dates: DateTime, Second, now
 using Base: UUID
 using SwaggerMarkdown
 
+using Oxygen: json
 module OxygenInstance using Oxygen; @oxidise end
 import .OxygenInstance: @get, @put, @post, mergeschema, serve, Request, Response
 
@@ -67,7 +69,7 @@ export serve
 
 
 @get "/deme" function(req::Request)
-    return Response(200, marshal(Mapper.get_demespec()))
+    return Mapper.get_demespec() |> json
 end
 
 
@@ -102,7 +104,7 @@ end
         id = unmarshal(req.body, Pseudonym)
         admission = Mapper.seek_admission(id, ticket.ticketid)
 
-        Response(200, marshal(admission)) # will this exit the function though? This would produce response without headers.
+        admission |> json # will this exit the function though? This would produce response without headers.
     end
 
     return handler(request)
@@ -115,7 +117,7 @@ end
 
     status = Mapper.get_ticket_status(ticketid)
 
-    return Response(200, marshal(status))
+    return status |> json
 end
 
 
@@ -124,15 +126,15 @@ end
     member = unmarshal(req.body, Membership)
     response = Mapper.enroll_member(member)
 
-    return Response(200, marshal(response))
+    return response |> json
 end
 
 
 @get "/braidchain/commit" function(req::Request)
 
     response = Mapper.get_chain_commit()
 
-    return Response(200, marshal(response))
+    return response |> json
 end
 
 
@@ -141,39 +143,52 @@ end
     proposal = unmarshal(req.body, Proposal)
     ack = Mapper.enlist_proposal(proposal)
 
-    return Response(200, marshal(ack))
+    return ack |> json
 end
 
 
 @get "/braidchain/proposals" function(req::Request)
 
     proposal_list = Mapper.get_chain_proposal_list()
 
-    return Response(200, marshal(proposal_list))
+    return proposal_list |> json
 end
 
 
 @get "/braidchain/{N}/leaf" function(req::Request, N::Int)
 
     ack = Mapper.get_chain_ack_leaf(N)
 
-    return Response(200, marshal(ack))
+    return ack |> json
 end
 
 
 @get "/braidchain/{N}/root" function(req::Request, N::Int)
 
     ack = Mapper.get_chain_ack_root(N)
 
-    return Response(200, marshal(ack))
+    return ack |> json
 end
 
 
-@get "/braidchain/{N}/record" function get_chain_record(req::Request, N::Int)
+@get "/braidchain/{N}/record" function(req::Request, N::Int)
+
+    # This will include logic to take the files from cache instead
 
     record = Mapper.get_chain_record(N)
+    type_header = "X-Record-Type" => string(nameof(typeof(record)))
+
+    # Backpressure could be added here to reduce memory footprint
+    if record isa BraidReceipt
+
+        io = IOBuffer() 
+        tar(io, record)
+        seekstart(io)
+
+        return Response(200, ["Content-Type" => "application/x-tar", type_header], io) 
+    end
 
-    return Response(200, marshal(record)) # type information is important here for receiver!
+    return json(record, headers = [type_header])
 end
 
 
@@ -183,7 +198,7 @@ end
 
     commit = Mapper.get_ballotbox_commit(uuid)
 
-    return Response(200, marshal(commit))
+    return commit |> json
 end
 
 
@@ -193,7 +208,7 @@ end
 
     proposal = Mapper.get_ballotbox_proposal(uuid)
 
-    return Response(200, marshal(proposal))
+    return proposal |> json
 end
 
 
@@ -203,7 +218,7 @@ end
 
     spine = Mapper.get_ballotbox_spine(uuid)
 
-    return Response(200, marshal(spine))
+    return spine |> json
 end
 
 
@@ -214,7 +229,7 @@ end
     vote = unmarshal(req.body, Vote)
     ack = Mapper.cast_vote(uuid, vote)
 
-    return Response(200, marshal(ack))
+    return ack |> json
 end
 
 
@@ -223,7 +238,7 @@ end
     uuid = UUID(uuid_hex)
     record = Mapper.get_ballotbox_record(uuid, N)
 
-    return Response(200, marshal(record))
+    return record |> json
 end
 
 
@@ -232,7 +247,7 @@ end
     uuid = UUID(uuid_hex)
     receipt = Mapper.get_ballotbox_receipt(uuid, N)
 
-    return Response(200, marshal(receipt))
+    return receipt |> json
 end
 
 
@@ -241,7 +256,7 @@ end
     uuid = UUID(uuid_hex)
     ack = Mapper.get_ballotbox_ack_leaf(uuid, N)
 
-    return Response(200, marshal(ack))
+    return ack |> json
 end
 
 
@@ -250,7 +265,7 @@ end
     uuid = UUID(uuid_hex)
     ack = Mapper.get_ballotbox_ack_root(uuid, N)
 
-    return Response(200, marshal(ack))
+    return ack |> json
 end
 
 

test/service.jl

@@ -5,6 +5,7 @@ import PeaceFounder.Server: Service, Mapper, Controllers
 import PeaceFounder: Client, Schedulers
 import PeaceFounder.Core.Model: Model, CryptoSpec, DemeSpec, Signer, id, approve
 import PeaceFounder.Core.ProtocolSchema
+import PeaceFounder.Core.Store: BraidChainLedger
 
 crypto = CryptoSpec("sha256", "EC: P_192")
 #crypto = CryptoSpec("sha256", "MODP: 23, 11, 2")
@@ -77,7 +78,7 @@ proposal = Model.Proposal(
     description = "",
     ballot = Model.Ballot(["yes", "no"]),
     open = Dates.now() + Dates.Millisecond(100),
-    closed = Dates.now() + Dates.Second(5)
+    closed = Dates.now() + Dates.Second(7)
 ) |> Client.configure(SERVER) |> approve(PROPOSER)
 
 
@@ -124,3 +125,15 @@ Controllers.commit!(Mapper.POLLING_STATION[], proposal.uuid, Mapper.COLLECTOR[])
 blame = Client.blame(bob, proposal.uuid) # can be published anonymously without privacy concerns 
 @test Client.isbinding(blame, proposal, Model.hasher(crypto))
 @test Client.verify(blame, crypto)
+
+# Testing the API for retrieving braidchain records over the network
+
+commit = Client.get_chain_commit(SERVER)
+chain = BraidChainLedger()
+
+for i in 1:commit.state.index
+    record = Client.get_chain_record(SERVER, i)
+    push!(chain, record)
+end
+
+@test Controllers.ledger(Mapper.BRAID_CHAIN[]) == chain