From 6c38055de276f6495101970ec030fc7dfa98ddc3 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 6 Dec 2024 12:21:46 -0700 Subject: [PATCH 1/5] fix: truncate sync_assets version to 32 and url to 4096 --- src/Services/Metadata/AbstractMetadataService.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Services/Metadata/AbstractMetadataService.php b/src/Services/Metadata/AbstractMetadataService.php index 7877749..adf664e 100644 --- a/src/Services/Metadata/AbstractMetadataService.php +++ b/src/Services/Metadata/AbstractMetadataService.php @@ -63,8 +63,8 @@ protected function saveOpen(array $metadata): void $this->connection()->insert('sync_assets', [ 'id' => Uuid::uuid7()->toString(), 'sync_id' => $id, - 'version' => $version, - 'url' => $url, + 'version' => mb_substr((string) $version, 0, 32), + 'url' => mb_substr($url, 0, 4096), 'created' => time(), ]); } From 340696a86d14738081a80e2ef2233212d52f6342 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 6 Dec 2024 12:26:03 -0700 Subject: [PATCH 2/5] fix: add try/catch around onResponse/onError handlers --- .../Sync/Meta/AbstractMetaFetchCommand.php | 85 ++++++++++--------- 1 file changed, 45 insertions(+), 40 deletions(-) diff --git a/src/Commands/Sync/Meta/AbstractMetaFetchCommand.php b/src/Commands/Sync/Meta/AbstractMetaFetchCommand.php index 8949a82..8f02247 100644 --- a/src/Commands/Sync/Meta/AbstractMetaFetchCommand.php +++ b/src/Commands/Sync/Meta/AbstractMetaFetchCommand.php @@ -122,11 +122,12 @@ protected function generateRequests(iterable $slugs): Generator protected function onResponse(Response $saloonResponse): void { - $response = $saloonResponse->getPsrResponse(); - $request = $saloonResponse->getRequest(); - $slug = $request->slug ?? throw new Exception('Missing slug in request'); - + $slug = null; try { + $response = $saloonResponse->getPsrResponse(); + $request = $saloonResponse->getRequest(); + $slug = $request->slug ?? throw new Exception('Missing slug in request'); + $metadata = json_decode($response->getBody()->getContents(), assoc: true); $metadata = [ 'slug' => $slug, @@ -134,51 +135,55 @@ protected function onResponse(Response $saloonResponse): void 'status' => 'open', ...$metadata, ]; + if (!empty($metadata['versions'])) { + $this->log->info("$slug ... [" . count($metadata['versions']) . ' versions]'); + } elseif (isset($metadata['version'])) { + $this->log->info("$slug ... [1 version]"); + } elseif (isset($metadata['skipped'])) { + $this->log->info((string) $metadata['skipped']); + } else { + $this->log->info("$slug ... No versions found"); + } + $this->meta->save($metadata); } catch (Exception $e) { $this->log->error("$slug ... ERROR: {$e->getMessage()}"); return; } - - if (!empty($metadata['versions'])) { - $this->log->info("$slug ... [" . count($metadata['versions']) . ' versions]'); - } elseif (isset($metadata['version'])) { - $this->log->info("$slug ... [1 version]"); - } elseif (isset($metadata['skipped'])) { - $this->log->info((string) $metadata['skipped']); - } else { - $this->log->info("$slug ... No versions found"); - } - - $this->meta->save($metadata); } protected function onError(Exception $exception): void { - if (!$exception instanceof RequestException) { - $this->log->error($exception->getMessage()); + $slug = null; + try { + if (!$exception instanceof RequestException) { + $this->log->error($exception->getMessage()); + return; + } + $saloonResponse = $exception->getResponse(); + $response = $saloonResponse->getPsrResponse(); + $request = $saloonResponse->getRequest(); + $slug = $request->slug ?? throw new Exception('Missing slug in request'); + $code = $response->getStatusCode(); + $reason = $response->getReasonPhrase(); + + $metadata = json_decode($response->getBody()->getContents(), assoc: true); + $error = $metadata['error'] ?? null; + + $status = match ($code) { + 404 => $error === 'closed' ? 'closed' : 'not-found', + default => 'error', + }; + + if ($status === 'closed') { + $this->log->info("$slug ... [closed]"); + } else { + $this->log->error("$slug ... $code $reason"); + } + + $this->meta->save(['slug' => $slug, 'name' => $slug, 'status' => $status, ...$metadata]); + } catch (Exception $e) { + $this->log->error("$slug ... ERROR: {$e->getMessage()}"); return; } - $saloonResponse = $exception->getResponse(); - $response = $saloonResponse->getPsrResponse(); - $request = $saloonResponse->getRequest(); - $slug = $request->slug ?? throw new Exception('Missing slug in request'); - $code = $response->getStatusCode(); - $reason = $response->getReasonPhrase(); - - $metadata = json_decode($response->getBody()->getContents(), assoc: true); - $error = $metadata['error'] ?? null; - - $status = match ($code) { - 404 => $error === 'closed' ? 'closed' : 'not-found', - default => 'error', - }; - - if ($status === 'closed') { - $this->log->info("$slug ... [closed]"); - } else { - $this->log->error("$slug ... $code $reason"); - } - - $this->meta->save(['slug' => $slug, 'name' => $slug, 'status' => $status, ...$metadata]); } } From f47410fdc20700019fdc7ca4e78787c5371e6833 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 6 Dec 2024 12:50:47 -0700 Subject: [PATCH 3/5] fix: s/created/added/ --- src/Services/List/AbstractListService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Services/List/AbstractListService.php b/src/Services/List/AbstractListService.php index 15a7180..e1e4776 100644 --- a/src/Services/List/AbstractListService.php +++ b/src/Services/List/AbstractListService.php @@ -116,7 +116,7 @@ private function loadLatestRevisions(): void foreach ($this->em->getConnection()->fetchAllAssociative($sql) as $revision) { $this->revisionData[$revision['action']] = [ 'revision' => $revision['revision'], - 'added' => $revision['created'], + 'added' => $revision['added'], ]; } } From 7ff3ff369dd9e1235b8a2778560caabbaf6c7437 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 6 Dec 2024 13:11:34 -0700 Subject: [PATCH 4/5] chore: fix then re-break getOpenVersions for download --- src/Services/List/AbstractListService.php | 12 +++++------- src/Services/Metadata/AbstractMetadataService.php | 5 ++--- src/Services/Metadata/MetadataServiceInterface.php | 2 +- tests/Stubs/MetadataServiceStub.php | 2 +- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/Services/List/AbstractListService.php b/src/Services/List/AbstractListService.php index e1e4776..f105a24 100644 --- a/src/Services/List/AbstractListService.php +++ b/src/Services/List/AbstractListService.php @@ -37,11 +37,9 @@ public function getItems(): array /** @return array */ public function getUpdatedItems(): array { - $revision = $this->getRevisionDate(); - if ($revision) { - $revision = \Safe\date('Y-m-d', \Safe\strtotime($revision)); - } - return $this->meta->getOpenVersions($revision); + // HACK: return everything until meta and download versions of ListService get different names + return $this->meta->getOpenVersions(-1); // FIXME: make getRevisionTime() work again + // return $this->meta->getOpenVersions($this->getRevisionTime()); } public function preserveRevision(): string @@ -101,9 +99,9 @@ public function getRevision(): ?string return $this->revisionData[$this->name]['revision'] ?? null; } - public function getRevisionDate(): ?string + public function getRevisionTime(): int { - return $this->revisionData[$this->name]['added'] ?? null; + return $this->revisionData[$this->name]['added'] ?? 1; } private function loadLatestRevisions(): void diff --git a/src/Services/Metadata/AbstractMetadataService.php b/src/Services/Metadata/AbstractMetadataService.php index adf664e..6c5e807 100644 --- a/src/Services/Metadata/AbstractMetadataService.php +++ b/src/Services/Metadata/AbstractMetadataService.php @@ -120,9 +120,8 @@ public function getDownloadUrl(string $slug, string $version): ?string } /** @return array [slug => [versions]] */ - public function getOpenVersions(string $revDate = '1900-01-01'): array + public function getOpenVersions(int $timestamp = 1): array { - $stamp = strtotime($revDate); $sql = <<connection()->fetchAllAssociative($sql, ['stamp' => $stamp, ...$this->stdArgs()]); + $result = $this->connection()->fetchAllAssociative($sql, ['stamp' => $timestamp, ...$this->stdArgs()]); $out = []; foreach ($result as $row) { diff --git a/src/Services/Metadata/MetadataServiceInterface.php b/src/Services/Metadata/MetadataServiceInterface.php index 3b6436d..3701362 100644 --- a/src/Services/Metadata/MetadataServiceInterface.php +++ b/src/Services/Metadata/MetadataServiceInterface.php @@ -22,7 +22,7 @@ public function exportAllMetadata(): Generator; public function save(array $metadata): void; /** @return array */ - public function getOpenVersions(string $revDate = '1900-01-01'): array; + public function getOpenVersions(int $timestamp = 1): array; public function markProcessed(string $slug, string $version): void; diff --git a/tests/Stubs/MetadataServiceStub.php b/tests/Stubs/MetadataServiceStub.php index 0d6d973..0719d1a 100644 --- a/tests/Stubs/MetadataServiceStub.php +++ b/tests/Stubs/MetadataServiceStub.php @@ -34,7 +34,7 @@ public function getStatus(string $slug): ?string return 'open'; } - public function getOpenVersions(string $revDate = '1900-01-01'): array + public function getOpenVersions(?int $timestamp = 1): array { return []; } From 785be338a3607f1c674b15ad2eff02c385d8ec91 Mon Sep 17 00:00:00 2001 From: Chuck Adams Date: Fri, 6 Dec 2024 13:35:10 -0700 Subject: [PATCH 5/5] fix: allow any exception type in download error handler --- src/Services/Download/AbstractDownloadService.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Services/Download/AbstractDownloadService.php b/src/Services/Download/AbstractDownloadService.php index 1df7f36..876a750 100644 --- a/src/Services/Download/AbstractDownloadService.php +++ b/src/Services/Download/AbstractDownloadService.php @@ -70,8 +70,13 @@ private function onResponse(Response $saloonResponse): void $this->filesystem->write($localPath, $contents); } - protected function onError(RequestException $exception): void + protected function onError(Exception $exception): void { + if (!($exception instanceof RequestException)) { + $this->log->error("ERROR: " . $exception->getMessage()); + return; + } + $saloonResponse = $exception->getResponse(); $response = $saloonResponse->getPsrResponse(); $request = $saloonResponse->getRequest();