From 6b927f300f1928a6b6cd530f9d8ba5fee63e9c41 Mon Sep 17 00:00:00 2001 From: Janis Erdmanis Date: Tue, 26 Mar 2024 17:06:01 +0200 Subject: [PATCH] implementing braidchain record retrieval over HTTP --- Manifest.toml | 10 +++---- Project.toml | 3 +- src/Client.jl | 22 ++++++++++---- src/Core/Model/braidchains.jl | 2 ++ src/Core/Store.jl | 42 +++++++++++++++++++++++++- src/Server/Service.jl | 55 ++++++++++++++++++++++------------- test/service.jl | 15 +++++++++- test/store.jl | 14 +++++++-- 8 files changed, 127 insertions(+), 36 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index c93bfdf..534486b 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.10.0" manifest_format = "2.0" -project_hash = "cd87006c6e4df4542cf96a4a5957d21f86663451" +project_hash = "bdf3cf4e1b1646d11c41f2412caa658745298616" [[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" @@ -119,9 +119,9 @@ version = "6.2.1+6" [[deps.HTTP]] deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] -git-tree-sha1 = "995f762e0182ebc50548c434c171a5bb6635f8e4" +git-tree-sha1 = "8e59b47b9dc525b70550ca082ce85bcd7f5477cd" uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" -version = "1.10.4" +version = "1.10.5" [[deps.HistoryTrees]] git-tree-sha1 = "5e0e766befebd2108cef8e3a14cf715c8496cb2e" @@ -375,9 +375,9 @@ version = "1.1.1" [[deps.ShuffleProofs]] deps = ["CryptoGroups", "Random"] -git-tree-sha1 = "79b5a6fac187a4d0ee5e6a0a4a706ca50775363e" +git-tree-sha1 = "87b0c49235260736a77ac0356fa4e063406d3ab1" uuid = "31a120cc-b3cb-4d07-bbdb-d498660ddfd8" -version = "0.3.0" +version = "0.3.1" [[deps.SimpleBufferStream]] git-tree-sha1 = "874e8867b33a00e784c8a7e4b60afe9e037b74e1" diff --git a/Project.toml b/Project.toml index c75a18a..f775c8e 100644 --- a/Project.toml +++ b/Project.toml @@ -21,6 +21,7 @@ ShuffleProofs = "31a120cc-b3cb-4d07-bbdb-d498660ddfd8" StructHelpers = "4093c41a-2008-41fd-82b8-e3f9d02b504f" StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" SwaggerMarkdown = "1b6eb727-ad4b-44eb-9669-b9596a6e760f" +Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" @@ -33,7 +34,7 @@ JSON3 = "1" Nettle = "1" Setfield = "1" ShuffleProofs = "0.3" +StructHelpers = "1.1" StructTypes = "1" URIs = "1.4" julia = "1" -StructHelpers = "1.1" \ No newline at end of file diff --git a/src/Client.jl b/src/Client.jl index 5488843..484b9bf 100644 --- a/src/Client.jl +++ b/src/Client.jl @@ -10,12 +10,12 @@ using Dates using Setfield import StructTypes -#using StructHelpers using ..Core.Model: Model, Membership, Pseudonym, Proposal, Vote, bytes, TicketID, HMAC, Admission, isbinding, verify, Digest, HashSpec, DemeSpec, Signer, Commit, ChainState, Proposal, BallotBoxState, isbinding, isopen, digest, commit using ..Core.Model: id, hasher, pseudonym, isbinding, generator, state, verify, crypto, index, root, isconsistent, istallied, issuer using ..Core.ProtocolSchema: TicketStatus, tokenid, Invite, AckConsistency, AckInclusion, CastAck using ..Core.Parser: marshal, unmarshal +using ..Core.Store: Store using ..Authorization: AuthClientMiddleware import ..Core.Model @@ -180,7 +180,6 @@ function get_chain_leaf(server::Route, N::Int) return ack end - function get_chain_root(server::Route, N::Int) response = get(server, "/braidchain/$N/root") @@ -189,15 +188,26 @@ function get_chain_root(server::Route, N::Int) return ack end - function get_chain_record(server::Route, N::Int) response = get(server, "/braidchain/$N/record") - - error("Not implemented") + record_type = Dict(response.headers)["X-Record-Type"] + + if record_type == "DemeSpec" + return unmarshal(response.body, DemeSpec) + elseif record_type == "Membership" + return unmarshal(response.body, Membership) + elseif record_type == "Proposal" + return unmarshal(response.body, Proposal) + elseif record_type == "BraidReceipt" + return Store.load(Model.BraidReceipt, response.body) + else + error("Record type $record_type not recognized") + end + + return end - function get_ballotbox_commit(server::Route, uuid::UUID) response = get(server, "/poolingstation/$uuid/commit") diff --git a/src/Core/Model/braidchains.jl b/src/Core/Model/braidchains.jl index 4d82de4..b9e9457 100644 --- a/src/Core/Model/braidchains.jl +++ b/src/Core/Model/braidchains.jl @@ -12,6 +12,8 @@ struct BraidChainLedger records::AbstractVector{Transaction} end +BraidChainLedger() = BraidChainLedger(Transaction[]) + @batteries BraidChainLedger Base.push!(ledger::BraidChainLedger, record::Transaction) = push!(ledger.records, record) diff --git a/src/Core/Store.jl b/src/Core/Store.jl index edf3337..40375eb 100644 --- a/src/Core/Store.jl +++ b/src/Core/Store.jl @@ -1,10 +1,10 @@ module Store +using Tar using ShuffleProofs: ShuffleProofs using ..Model: BraidChainLedger, DemeSpec, Membership, BraidReceipt, Proposal, Transaction, BallotBoxLedger, CastRecord, Seal using ..Parser: marshal, unmarshal - index2name(i::UInt16) = reinterpret(UInt8, [i]) |> reverse |> bytes2hex |> uppercase index2name(i::Int) = index2name(UInt16(i)) @@ -103,6 +103,46 @@ end load(::Type{BraidReceipt}, dir::String, index::UInt16) = load(BraidReceipt, joinpath(dir, BRAIDRECEIPT_DIR, index2name(index))) +struct TarPath <: ShuffleProofs.Path + io::IO + path::String +end + +function tar(io::IO, path::String, data::Vector{UInt8}; mode::Integer=0o644) + + n = length(data) + + Tar.write_header(io, Tar.Header(path, :file, mode, n, "")) + Tar.write_data(io, IOBuffer(data), size=n) + +end + +Base.joinpath(path::TarPath, args...) = TarPath(path.io, joinpath(path.path, args...)) +Base.mkdir(path::TarPath) = nothing +Base.mkpath(path::TarPath) = nothing +Base.write(path::TarPath, data::Vector{UInt8}) = tar(path.io, path.path, data) + + +function tar(io::IO, record::BraidReceipt) + + ShuffleProofs.save(record.braid, TarPath(io, "braid")) + tar(io, "demespec.json", marshal(record.producer)) + tar(io, "seal.json", marshal(record.approval)) + + return +end + +function load(::Type{BraidReceipt}, io::IO) + + dir = joinpath(tempdir(), "braidreceipt") + rm(dir, force=true, recursive=true) + + Tar.extract(io, dir) + + return load(BraidReceipt, dir) +end + + function save(record::CastRecord, path::String; force=false) if isfile(path) diff --git a/src/Server/Service.jl b/src/Server/Service.jl index 2be7c77..ee61030 100644 --- a/src/Server/Service.jl +++ b/src/Server/Service.jl @@ -3,7 +3,8 @@ module Service # This is the outermost layer for the sercvice concerned with providing services for outsied world. # Defines how HTTP requests are processed using ..Core.Parser: marshal, unmarshal -using ..Core.Model: TicketID, Digest, Pseudonym, Digest, Membership, Proposal, Vote, bytes +using ..Core.Store: tar +using ..Core.Model: TicketID, Digest, Pseudonym, Digest, Membership, Proposal, Vote, bytes, BraidReceipt using ..Authorization: AuthServerMiddleware, timestamp, credential using ..Mapper @@ -11,6 +12,7 @@ using Dates: DateTime, Second, now using Base: UUID using SwaggerMarkdown +using Oxygen: json module OxygenInstance using Oxygen; @oxidise end import .OxygenInstance: @get, @put, @post, mergeschema, serve, Request, Response @@ -67,7 +69,7 @@ export serve @get "/deme" function(req::Request) - return Response(200, marshal(Mapper.get_demespec())) + return Mapper.get_demespec() |> json end @@ -102,7 +104,7 @@ end id = unmarshal(req.body, Pseudonym) admission = Mapper.seek_admission(id, ticket.ticketid) - Response(200, marshal(admission)) # will this exit the function though? This would produce response without headers. + admission |> json # will this exit the function though? This would produce response without headers. end return handler(request) @@ -115,7 +117,7 @@ end status = Mapper.get_ticket_status(ticketid) - return Response(200, marshal(status)) + return status |> json end @@ -124,7 +126,7 @@ end member = unmarshal(req.body, Membership) response = Mapper.enroll_member(member) - return Response(200, marshal(response)) + return response |> json end @@ -132,7 +134,7 @@ end response = Mapper.get_chain_commit() - return Response(200, marshal(response)) + return response |> json end @@ -141,7 +143,7 @@ end proposal = unmarshal(req.body, Proposal) ack = Mapper.enlist_proposal(proposal) - return Response(200, marshal(ack)) + return ack |> json end @@ -149,7 +151,7 @@ end proposal_list = Mapper.get_chain_proposal_list() - return Response(200, marshal(proposal_list)) + return proposal_list |> json end @@ -157,7 +159,7 @@ end ack = Mapper.get_chain_ack_leaf(N) - return Response(200, marshal(ack)) + return ack |> json end @@ -165,15 +167,28 @@ end ack = Mapper.get_chain_ack_root(N) - return Response(200, marshal(ack)) + return ack |> json end -@get "/braidchain/{N}/record" function get_chain_record(req::Request, N::Int) +@get "/braidchain/{N}/record" function(req::Request, N::Int) + + # This will include logic to take the files from cache instead record = Mapper.get_chain_record(N) + type_header = "X-Record-Type" => string(nameof(typeof(record))) + + # Backpressure could be added here to reduce memory footprint + if record isa BraidReceipt + + io = IOBuffer() + tar(io, record) + seekstart(io) + + return Response(200, ["Content-Type" => "application/x-tar", type_header], io) + end - return Response(200, marshal(record)) # type information is important here for receiver! + return json(record, headers = [type_header]) end @@ -183,7 +198,7 @@ end commit = Mapper.get_ballotbox_commit(uuid) - return Response(200, marshal(commit)) + return commit |> json end @@ -193,7 +208,7 @@ end proposal = Mapper.get_ballotbox_proposal(uuid) - return Response(200, marshal(proposal)) + return proposal |> json end @@ -203,7 +218,7 @@ end spine = Mapper.get_ballotbox_spine(uuid) - return Response(200, marshal(spine)) + return spine |> json end @@ -214,7 +229,7 @@ end vote = unmarshal(req.body, Vote) ack = Mapper.cast_vote(uuid, vote) - return Response(200, marshal(ack)) + return ack |> json end @@ -223,7 +238,7 @@ end uuid = UUID(uuid_hex) record = Mapper.get_ballotbox_record(uuid, N) - return Response(200, marshal(record)) + return record |> json end @@ -232,7 +247,7 @@ end uuid = UUID(uuid_hex) receipt = Mapper.get_ballotbox_receipt(uuid, N) - return Response(200, marshal(receipt)) + return receipt |> json end @@ -241,7 +256,7 @@ end uuid = UUID(uuid_hex) ack = Mapper.get_ballotbox_ack_leaf(uuid, N) - return Response(200, marshal(ack)) + return ack |> json end @@ -250,7 +265,7 @@ end uuid = UUID(uuid_hex) ack = Mapper.get_ballotbox_ack_root(uuid, N) - return Response(200, marshal(ack)) + return ack |> json end diff --git a/test/service.jl b/test/service.jl index 8cec007..bc4b6b6 100644 --- a/test/service.jl +++ b/test/service.jl @@ -5,6 +5,7 @@ import PeaceFounder.Server: Service, Mapper, Controllers import PeaceFounder: Client, Schedulers import PeaceFounder.Core.Model: Model, CryptoSpec, DemeSpec, Signer, id, approve import PeaceFounder.Core.ProtocolSchema +import PeaceFounder.Core.Store: BraidChainLedger crypto = CryptoSpec("sha256", "EC: P_192") #crypto = CryptoSpec("sha256", "MODP: 23, 11, 2") @@ -77,7 +78,7 @@ proposal = Model.Proposal( description = "", ballot = Model.Ballot(["yes", "no"]), open = Dates.now() + Dates.Millisecond(100), - closed = Dates.now() + Dates.Second(5) + closed = Dates.now() + Dates.Second(7) ) |> Client.configure(SERVER) |> approve(PROPOSER) @@ -124,3 +125,15 @@ Controllers.commit!(Mapper.POLLING_STATION[], proposal.uuid, Mapper.COLLECTOR[]) blame = Client.blame(bob, proposal.uuid) # can be published anonymously without privacy concerns @test Client.isbinding(blame, proposal, Model.hasher(crypto)) @test Client.verify(blame, crypto) + +# Testing the API for retrieving braidchain records over the network + +commit = Client.get_chain_commit(SERVER) +chain = BraidChainLedger() + +for i in 1:commit.state.index + record = Client.get_chain_record(SERVER, i) + push!(chain, record) +end + +@test Controllers.ledger(Mapper.BRAID_CHAIN[]) == chain diff --git a/test/store.jl b/test/store.jl index 6089975..c4dbe8e 100644 --- a/test/store.jl +++ b/test/store.jl @@ -1,4 +1,4 @@ -using PeaceFounder.Core.Store: save, load +using PeaceFounder.Core.Store: save, load, tar using Test @@ -6,7 +6,7 @@ using Dates using PeaceFounder.Core: Model using PeaceFounder.Server: Controllers -import .Model: CryptoSpec, pseudonym, TicketID, id, commit, verify, generator, Membership, approve, isbinding, Proposal, vote, Ballot, Selection, uuid, tally, seed, hasher, HMAC, DemeSpec, generate, Signer, key, braid, Model, select, digest, voters, members, root +import .Model: CryptoSpec, pseudonym, TicketID, id, commit, verify, generator, Membership, approve, isbinding, Proposal, vote, Ballot, Selection, uuid, tally, seed, hasher, HMAC, DemeSpec, generate, Signer, key, braid, Model, select, digest, voters, members, root, BraidReceipt import .Controllers: Registrar, admit!, enlist!, set_demehash!, Ticket, tokenid import .Controllers: record!, commit!, ack_leaf @@ -152,3 +152,13 @@ loaded_bbox_ledger = load(BBOX_DIR) @test loaded_bbox_ledger.records == bbox.records @test loaded_bbox_ledger.proposal == bbox.proposal @test loaded_bbox_ledger.spec == bbox.spec + +# testing tar on braid + +io = IOBuffer() +tar(io, braidwork) +seekstart(io) + +loaded_braidwork = load(BraidReceipt, io) + +@test loaded_braidwork == braidwork