Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: truncate sync_assets version to 32 and url to 4096 #40

Merged
merged 5 commits into from
Dec 7, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 45 additions & 40 deletions src/Commands/Sync/Meta/AbstractMetaFetchCommand.php
Original file line number Diff line number Diff line change
Expand Up @@ -122,63 +122,68 @@ protected function generateRequests(iterable $slugs): Generator

protected function onResponse(Response $saloonResponse): void
{
$response = $saloonResponse->getPsrResponse();
$request = $saloonResponse->getRequest();
$slug = $request->slug ?? throw new Exception('Missing slug in request');

$slug = null;
try {
$response = $saloonResponse->getPsrResponse();
$request = $saloonResponse->getRequest();
$slug = $request->slug ?? throw new Exception('Missing slug in request');

$metadata = json_decode($response->getBody()->getContents(), assoc: true);
$metadata = [
'slug' => $slug,
'name' => $slug,
'status' => 'open',
...$metadata,
];
if (!empty($metadata['versions'])) {
$this->log->info("$slug ... [" . count($metadata['versions']) . ' versions]');
} elseif (isset($metadata['version'])) {
$this->log->info("$slug ... [1 version]");
} elseif (isset($metadata['skipped'])) {
$this->log->info((string) $metadata['skipped']);
} else {
$this->log->info("$slug ... No versions found");
}
$this->meta->save($metadata);
} catch (Exception $e) {
$this->log->error("$slug ... ERROR: {$e->getMessage()}");
return;
}

if (!empty($metadata['versions'])) {
$this->log->info("$slug ... [" . count($metadata['versions']) . ' versions]');
} elseif (isset($metadata['version'])) {
$this->log->info("$slug ... [1 version]");
} elseif (isset($metadata['skipped'])) {
$this->log->info((string) $metadata['skipped']);
} else {
$this->log->info("$slug ... No versions found");
}

$this->meta->save($metadata);
}

protected function onError(Exception $exception): void
{
if (!$exception instanceof RequestException) {
$this->log->error($exception->getMessage());
$slug = null;
try {
if (!$exception instanceof RequestException) {
$this->log->error($exception->getMessage());
return;
}
$saloonResponse = $exception->getResponse();
$response = $saloonResponse->getPsrResponse();
$request = $saloonResponse->getRequest();
$slug = $request->slug ?? throw new Exception('Missing slug in request');
$code = $response->getStatusCode();
$reason = $response->getReasonPhrase();

$metadata = json_decode($response->getBody()->getContents(), assoc: true);
$error = $metadata['error'] ?? null;

$status = match ($code) {
404 => $error === 'closed' ? 'closed' : 'not-found',
default => 'error',
};

if ($status === 'closed') {
$this->log->info("$slug ... [closed]");
} else {
$this->log->error("$slug ... $code $reason");
}

$this->meta->save(['slug' => $slug, 'name' => $slug, 'status' => $status, ...$metadata]);
} catch (Exception $e) {
$this->log->error("$slug ... ERROR: {$e->getMessage()}");
return;
}
$saloonResponse = $exception->getResponse();
$response = $saloonResponse->getPsrResponse();
$request = $saloonResponse->getRequest();
$slug = $request->slug ?? throw new Exception('Missing slug in request');
$code = $response->getStatusCode();
$reason = $response->getReasonPhrase();

$metadata = json_decode($response->getBody()->getContents(), assoc: true);
$error = $metadata['error'] ?? null;

$status = match ($code) {
404 => $error === 'closed' ? 'closed' : 'not-found',
default => 'error',
};

if ($status === 'closed') {
$this->log->info("$slug ... [closed]");
} else {
$this->log->error("$slug ... $code $reason");
}

$this->meta->save(['slug' => $slug, 'name' => $slug, 'status' => $status, ...$metadata]);
}
}
7 changes: 6 additions & 1 deletion src/Services/Download/AbstractDownloadService.php
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,13 @@ private function onResponse(Response $saloonResponse): void
$this->filesystem->write($localPath, $contents);
}

protected function onError(RequestException $exception): void
protected function onError(Exception $exception): void
{
if (!($exception instanceof RequestException)) {
$this->log->error("ERROR: " . $exception->getMessage());
return;
}

$saloonResponse = $exception->getResponse();
$response = $saloonResponse->getPsrResponse();
$request = $saloonResponse->getRequest();
Expand Down
14 changes: 6 additions & 8 deletions src/Services/List/AbstractListService.php
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,9 @@ public function getItems(): array
/** @return array<string|int, array{}> */
public function getUpdatedItems(): array
{
$revision = $this->getRevisionDate();
if ($revision) {
$revision = \Safe\date('Y-m-d', \Safe\strtotime($revision));
}
return $this->meta->getOpenVersions($revision);
// HACK: return everything until meta and download versions of ListService get different names
return $this->meta->getOpenVersions(-1); // FIXME: make getRevisionTime() work again
// return $this->meta->getOpenVersions($this->getRevisionTime());
}

public function preserveRevision(): string
Expand Down Expand Up @@ -101,9 +99,9 @@ public function getRevision(): ?string
return $this->revisionData[$this->name]['revision'] ?? null;
}

public function getRevisionDate(): ?string
public function getRevisionTime(): int
{
return $this->revisionData[$this->name]['added'] ?? null;
return $this->revisionData[$this->name]['added'] ?? 1;
}

private function loadLatestRevisions(): void
Expand All @@ -116,7 +114,7 @@ private function loadLatestRevisions(): void
foreach ($this->em->getConnection()->fetchAllAssociative($sql) as $revision) {
$this->revisionData[$revision['action']] = [
'revision' => $revision['revision'],
'added' => $revision['created'],
'added' => $revision['added'],
];
}
}
Expand Down
9 changes: 4 additions & 5 deletions src/Services/Metadata/AbstractMetadataService.php
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ protected function saveOpen(array $metadata): void
$this->connection()->insert('sync_assets', [
'id' => Uuid::uuid7()->toString(),
'sync_id' => $id,
'version' => $version,
'url' => $url,
'version' => mb_substr((string) $version, 0, 32),
'url' => mb_substr($url, 0, 4096),
'created' => time(),
]);
}
Expand Down Expand Up @@ -120,9 +120,8 @@ public function getDownloadUrl(string $slug, string $version): ?string
}

/** @return array<string, string[]> [slug => [versions]] */
public function getOpenVersions(string $revDate = '1900-01-01'): array
public function getOpenVersions(int $timestamp = 1): array
{
$stamp = strtotime($revDate);
$sql = <<<SQL
SELECT slug, sync_assets.version
FROM sync_assets
Expand All @@ -132,7 +131,7 @@ public function getOpenVersions(string $revDate = '1900-01-01'): array
AND sync.type = :type
AND sync.origin = :origin
SQL;
$result = $this->connection()->fetchAllAssociative($sql, ['stamp' => $stamp, ...$this->stdArgs()]);
$result = $this->connection()->fetchAllAssociative($sql, ['stamp' => $timestamp, ...$this->stdArgs()]);

$out = [];
foreach ($result as $row) {
Expand Down
2 changes: 1 addition & 1 deletion src/Services/Metadata/MetadataServiceInterface.php
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ public function exportAllMetadata(): Generator;
public function save(array $metadata): void;

/** @return array<string|int, string[]> */
public function getOpenVersions(string $revDate = '1900-01-01'): array;
public function getOpenVersions(int $timestamp = 1): array;

public function markProcessed(string $slug, string $version): void;

Expand Down
2 changes: 1 addition & 1 deletion tests/Stubs/MetadataServiceStub.php
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public function getStatus(string $slug): ?string
return 'open';
}

public function getOpenVersions(string $revDate = '1900-01-01'): array
public function getOpenVersions(?int $timestamp = 1): array
{
return [];
}
Expand Down
Loading