From 6dc64876dca5a170dd798d1ee9e62c6d8aeab063 Mon Sep 17 00:00:00 2001 From: Pino Toscano Date: Mon, 5 Aug 2024 08:04:30 +0200 Subject: [PATCH 1/5] feat: cockpit: switch from .last-upload.results to .lastupload The file /etc/insights-client/.last-upload.results is watched to determine when was the last upload done. A couple of important notes about it: 1) even though it is read (and parsed), its content is actually never used currently 2) the file is written by insights-client (better: insights-core) only when "legacy_upload" is true (which is the default as of this writing) While (1) is not much an issue, (2) means that the displaying of the last upload (timestamp) to Insights is not available when switching insights-client to non-legacy-upload. To avoid both the issues, switch from .last-upload.results to /etc/insights-client/.lastupload: - .lastupload is always written, no matter the mode, and its modification timestamp is indeed accurate - stop reading the file at all, saving a bit of resources There should be no behaviour changes. (cherry picked from commit 2ac616ffdf0d25afc10d72dc2a32b16032512dc8 in the subscription-manager-cockpit repository) --- cockpit/src/insights.jsx | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cockpit/src/insights.jsx b/cockpit/src/insights.jsx index 39022c3d01..3e6685b833 100644 --- a/cockpit/src/insights.jsx +++ b/cockpit/src/insights.jsx @@ -368,17 +368,15 @@ function jump_to_timer() { function monitor_last_upload() { let self = { timestamp: 0, - results: null, close: close }; cockpit.event_target(self); - let results_file = cockpit.file("/etc/insights-client/.last-upload.results", { syntax: JSON }); - results_file.watch(data => { - self.results = data; - cockpit.spawn([ "stat", "-c", "%Y", "/etc/insights-client/.last-upload.results" ], { err: "message" }) + let results_file = cockpit.file("/etc/insights-client/.lastupload"); + results_file.watch(() => { + cockpit.spawn([ "stat", "-c", "%Y", "/etc/insights-client/.lastupload" ], { err: "message" }) .then(ts => { self.timestamp = parseInt(ts); self.dispatchEvent("changed"); @@ -387,7 +385,7 @@ function monitor_last_upload() { self.timestamp = 0; self.dispatchEvent("changed"); }); - }); + }, { read: false }); function close() { results_file.close(); From c564bfccae8fa5f9f9903a4c187560b4f66d123a Mon Sep 17 00:00:00 2001 From: Pino Toscano Date: Mon, 5 Aug 2024 08:10:40 +0200 Subject: [PATCH 2/5] test: cockpit: mock-insights: refactor of two platform endpoints Tweak their implementations a bit to prepare them for more substantial changes; in particular: - properly name the ID we get: the "insights_id" query parameter as such, and the UUID as Inventory ID - use an helper variable to which add/set all the bits to the replies, dumping them as JSON only in one place This is only a refactor with no behaviour change. (cherry picked from commit 2cf70497eb9a1e8b87a0eb5d9910a3675a811929 in the subscription-manager-cockpit repository) --- test/files/mock-insights | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/files/mock-insights b/test/files/mock-insights index 2bb5e97c61..29d0baa747 100755 --- a/test/files/mock-insights +++ b/test/files/mock-insights @@ -70,21 +70,25 @@ class handler(BaseHTTPRequestHandler): m = self.match("/r/insights/platform/inventory/v1/hosts\\?insights_id=(.*)") if m: - machine_id = m[1] + insights_id = m[1] self.send_response(200) self.end_headers() - if machine_id in systems: - self.wfile.write(b'{ "total": 1, "results": [ { "id": "123-nice-id" } ] }\n') - else: - self.wfile.write(b'{ "total": 0, "results": [ ] }\n') + res = { + "total": 0, + "results": [], + } + if insights_id in systems: + res["total"] += 1 + res["results"].append({"id": "123-nice-id"}) + self.wfile.write(json.dumps(res).encode("utf-8") + b"\n") return m = self.match("/r/insights/platform/insights/v1/system/([^/]+)/reports/") if m: - platform_id = m[1] + inventory_id = m[1] self.send_response(200) self.end_headers() - if platform_id == "123-nice-id": + if inventory_id == "123-nice-id": self.wfile.write(b'[ { "rule": { "total_risk": 3 } }, { "rule": { "total_risk": 2 } }, { "rule": { "total_risk": 1 } }]\n') else: self.wfile.write(b'[ ]\n') From 117cb097a52f8469f518170a513d9533af5e16ef Mon Sep 17 00:00:00 2001 From: Pino Toscano Date: Mon, 5 Aug 2024 07:45:36 +0200 Subject: [PATCH 3/5] test: cockpit: mock-insights: fix/improve handling of IDs Currently, the fake systems (or better, only one) is kept in the helper "systems" dictionary by the machine ID; while this seems to work fine, in practice it will not work for upcoming changes, and it does not match what Inventory actually does. Change the ID handling to represent better what Inventory does: - assign "id" as Inventory ID for each newly registered system; in practice we have only one, and keep hardcoding "123-nice-id" for now (the "testInsights" test checks for it) - use the "id" as key in the "systems" dictionary, rather than the "machine_id" - when registering a new system, copy "machine_id" as "insights_id"; this will help later on when implementing the non-legacy API endpoints - adapt endpoints to search for the ID they need Even with all the changes, there should be no behaviour changes. (cherry picked from commit 40c42862eaf15b6f17b9a9f4e70a762574541b80 in the subscription-manager-cockpit repository) --- test/files/mock-insights | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/test/files/mock-insights b/test/files/mock-insights index 29d0baa747..883215a1ec 100755 --- a/test/files/mock-insights +++ b/test/files/mock-insights @@ -52,13 +52,14 @@ class handler(BaseHTTPRequestHandler): m = self.match("/r/insights/v1/systems/([^/]+)") if m: machine_id = m[1] - if machine_id not in systems: - self.send_response(404) - self.end_headers() - return - self.send_response(200) + for system in systems.values(): + if system["machine_id"] == machine_id: + self.send_response(200) + self.end_headers() + self.wfile.write(json.dumps(system).encode("utf-8") + b"\n") + return + self.send_response(404) self.end_headers() - self.wfile.write(json.dumps(systems[machine_id]).encode("utf-8") + b"\n") return m = self.match("/r/insights/v1/branch_info") @@ -77,9 +78,10 @@ class handler(BaseHTTPRequestHandler): "total": 0, "results": [], } - if insights_id in systems: - res["total"] += 1 - res["results"].append({"id": "123-nice-id"}) + for system in systems.values(): + if system["insights_id"] == insights_id: + res["total"] += 1 + res["results"].append(system) self.wfile.write(json.dumps(res).encode("utf-8") + b"\n") return @@ -88,7 +90,7 @@ class handler(BaseHTTPRequestHandler): inventory_id = m[1] self.send_response(200) self.end_headers() - if inventory_id == "123-nice-id": + if inventory_id in systems: self.wfile.write(b'[ { "rule": { "total_risk": 3 } }, { "rule": { "total_risk": 2 } }, { "rule": { "total_risk": 1 } }]\n') else: self.wfile.write(b'[ ]\n') @@ -106,8 +108,10 @@ class handler(BaseHTTPRequestHandler): s = json.loads(data) s["unregistered_at"] = None s["account_number"] = "123456" + s["insights_id"] = s["machine_id"] + s["id"] = "123-nice-id" print(s) - systems[s["machine_id"]] = s + systems[s["id"]] = s self.send_response(200) self.end_headers() self.wfile.write(data) From 52fb598a5e5866c5a19179bdbd198f10bcf6813b Mon Sep 17 00:00:00 2001 From: Pino Toscano Date: Mon, 5 Aug 2024 07:58:33 +0200 Subject: [PATCH 4/5] test: cockpit: mock-insights: implement upload and delete platform endpoints Implement a couple of missing platform endpoints needed to make insights-client work in non-legacy-upload mode: - the upload endpoint, which needs to get the system metadata from the MIME data sent with POST: because of this, the implementation needs a bit more work; because of testing reasons, the Inventory ID is still hardcoded as "123-nice-id" - the delete endpoint, needed to unregister a system (cherry picked from commit 5bafd3772f88ec2d3d8422f2aac6f53828ffe761 in the subscription-manager-cockpit repository) --- test/files/mock-insights | 58 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/test/files/mock-insights b/test/files/mock-insights index 883215a1ec..29b5ada38b 100755 --- a/test/files/mock-insights +++ b/test/files/mock-insights @@ -17,6 +17,7 @@ # username=admin # password=foobar +import email from http.server import * import json import os @@ -124,6 +125,51 @@ class handler(BaseHTTPRequestHandler): self.wfile.write(b'{ "reports": [ "foo", "bar" ] }\n') return + m = self.match("/r/insights/platform/ingress/v1/upload") + if m: + # The metadata of the system is in the multipart MIME data + # sent by the system for the upload; to pick it and use it + # we need to unpack the multipart MIME data. + + # First, create prologue to the data, so it can be recognized + # as multipart MIME. + multipart_data = ( + b"""MIME-Version: 1.0 +Content-Type: """ + + self.headers.get("content-type").encode("utf-8") + + b""" + +""" + ) + multipart_data += data + + # Parse the multipart MIME data, and then look for a "metadata" + # file part which contains the system metadata as JSON. + message = email.message_from_bytes(multipart_data) + for part in message.walk(): + if part.get_filename() == "metadata": + s = json.loads(part.get_payload()) + s["id"] = "123-nice-id" + print(s) + systems[s["id"]] = s + + self.send_response(202) + self.end_headers() + self.wfile.write( + b"{" + b' "request_id": "some-upload", ' + b' "upload": { ' + b' "account": "123456", ' + b' "org_id": "123456" ' + b" }" + b"}\n" + ) + return + + self.send_response(400) + self.end_headers() + return + self.send_response(404) self.end_headers() @@ -138,6 +184,18 @@ class handler(BaseHTTPRequestHandler): del systems[machine_id] return + m = self.match("/r/insights/platform/inventory/v1/hosts/([^/]+)") + if m: + inventory_id = m[1] + if inventory_id in systems: + del systems[inventory_id] + self.send_response(200) + self.end_headers() + return + self.send_response(404) + self.end_headers() + return + self.send_response(404) self.end_headers() From 9ed2e5db24645e733b444f2874252cab27060ee1 Mon Sep 17 00:00:00 2001 From: Pino Toscano Date: Mon, 5 Aug 2024 11:48:57 +0200 Subject: [PATCH 5/5] test: cockpit: switch to CERT auth for insights-client Switch the authentication method of insights-client to CERT-based, which uses the consumer certificate & key of subscription-manager. This means using mTLS for authentication, still without verifying them, just like the previous fake credentials; this is still acceptable in the testing environment of a transient guest. (cherry picked from commit e682d429aab936718bb902380d56849ec568a141 in the subscription-manager-cockpit repository) --- integration-tests/check-subscriptions | 3 +-- test/files/mock-insights | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/integration-tests/check-subscriptions b/integration-tests/check-subscriptions index 1673a6a30d..e1dd193e3c 100755 --- a/integration-tests/check-subscriptions +++ b/integration-tests/check-subscriptions @@ -159,8 +159,7 @@ auto_config=False auto_update=False base_url={hostname}:8443/r/insights cert_verify=/var/lib/insights/mock-certs/ca.crt -username=admin -password=foobar +authmethod=CERT """) m.upload(["files/mock-insights"], "/var/tmp") diff --git a/test/files/mock-insights b/test/files/mock-insights index 29b5ada38b..e6f4fff29b 100755 --- a/test/files/mock-insights +++ b/test/files/mock-insights @@ -14,8 +14,7 @@ # auto_update=False # base_url=127.0.0.1:8443/r/insights # cert_verify=/var/lib/insights/mock-certs/ca.crt -# username=admin -# password=foobar +# authmethod=CERT import email from http.server import *