From f4ba41550a44c9737fc6ff50c34643fda591c810 Mon Sep 17 00:00:00 2001 From: micbar Date: Thu, 25 Jul 2019 11:30:42 +0200 Subject: [PATCH 1/4] New data format Fix Importer Unit Tests --- lib/Exporter.php | 14 +- lib/Extractor/MetadataExtractor.php | 10 +- .../FilesMetadataExtractor.php | 34 +- .../MetadataExtractor/SharesExtractor.php | 152 ++++-- lib/Importer.php | 9 +- lib/Importer/FilesImporter.php | 102 ++-- .../MetadataImporter/ShareImporter.php | 271 ++++++----- lib/Model/File.php | 24 + lib/Model/Instance.php | 6 + lib/Model/Metadata.php | 25 +- lib/Model/{User => }/Share.php | 19 +- lib/Model/User.php | 25 +- lib/Model/User/Preference.php | 6 + lib/Utilities/StreamHelper.php | 127 +++++ .../data/simpleExport/userfoo/files.jsonl | 4 + .../data/simpleExport/userfoo/metadata.json | 54 --- .../data/simpleExport/userfoo/user.json | 27 ++ .../bootstrap/DataExporterContext.php | 33 +- .../integration/MetadataImportExportTest.php | 141 ++++-- tests/unit/ExporterTest.php | 2 +- tests/unit/Extractor/FilesExtractorTest.php | 6 + .../FilesMetadataExtractorTest.php | 61 ++- .../MetadataExtractor/SharesExtractorTest.php | 443 ++++++++++-------- .../unit/Extractor/MetadataExtractorTest.php | 25 +- tests/unit/Importer/FilesImporterTest.php | 116 ++++- .../MetadataImporter/ShareImporterTest.php | 105 ++++- tests/unit/Utilities/StreamHelperTest.php | 166 +++++++ 27 files changed, 1405 insertions(+), 602 deletions(-) rename lib/Model/{User => }/Share.php (93%) create mode 100644 lib/Utilities/StreamHelper.php create mode 100644 tests/acceptance/data/simpleExport/userfoo/files.jsonl delete mode 100644 tests/acceptance/data/simpleExport/userfoo/metadata.json create mode 100644 tests/acceptance/data/simpleExport/userfoo/user.json create mode 100644 tests/unit/Utilities/StreamHelperTest.php diff --git a/lib/Exporter.php b/lib/Exporter.php index c13da77..8afdbd0 100644 --- a/lib/Exporter.php +++ b/lib/Exporter.php @@ -44,11 +44,21 @@ public function __construct(Serializer $serializer, MetadataExtractor $metadataE $this->filesystem = $filesystem; } + /** + * @param string $uid + * @param string $exportDirectoryPath + * + * @throws \OCP\Files\NotFoundException + * @throws \OCP\Files\NotPermittedException + * @throws \Exception + * + * @return void + */ public function export($uid, $exportDirectoryPath) { $exportPath = "$exportDirectoryPath/$uid"; - $metaData = $this->metadataExtractor->extract($uid); + $metaData = $this->metadataExtractor->extract($uid, $exportPath); $this->filesystem->dumpFile( - "$exportPath/metadata.json", + "$exportPath/user.json", $this->serializer->serialize($metaData) ); diff --git a/lib/Extractor/MetadataExtractor.php b/lib/Extractor/MetadataExtractor.php index 0793ecb..191c596 100644 --- a/lib/Extractor/MetadataExtractor.php +++ b/lib/Extractor/MetadataExtractor.php @@ -80,17 +80,17 @@ public function __construct( * @throws \Exception * @throws \RuntimeException if user can not be read */ - public function extract($uid) { + public function extract($uid, $exportPath) { $user = $this->userExtractor->extract($uid); - $user->setPreferences($this->preferencesExtractor->extract($uid)) - ->setShares($this->sharesExtractor->extract($uid)); - + $user->setPreferences($this->preferencesExtractor->extract($uid)); $metadata = new Metadata(); $metadata->setDate(new \DateTimeImmutable()) ->setUser($user) - ->setFiles($this->filesMetadataExtractor->extract($uid)) ->setOriginServer($this->urlGenerator->getAbsoluteURL('/')); + $this->filesMetadataExtractor->extract($uid, $exportPath); + $this->sharesExtractor->extract($uid, $exportPath); + return $metadata; } } diff --git a/lib/Extractor/MetadataExtractor/FilesMetadataExtractor.php b/lib/Extractor/MetadataExtractor/FilesMetadataExtractor.php index fecb849..b513c5f 100644 --- a/lib/Extractor/MetadataExtractor/FilesMetadataExtractor.php +++ b/lib/Extractor/MetadataExtractor/FilesMetadataExtractor.php @@ -23,27 +23,43 @@ */ namespace OCA\DataExporter\Extractor\MetadataExtractor; +use OC\User\NoUserException; use OCA\DataExporter\Utilities\Iterators\Nodes\RecursiveNodeIteratorFactory; use OCA\DataExporter\Model\File; +use OCA\DataExporter\Utilities\StreamHelper; use OCP\Files\Node; class FilesMetadataExtractor { + const FILE_NAME = 'files.jsonl'; /** @var RecursiveNodeIteratorFactory */ private $iteratorFactory; + /** + * @var StreamHelper + */ + private $streamHelper; + /** + * @var resource + */ + private $streamFile; - public function __construct(RecursiveNodeIteratorFactory $iteratorFactory) { + public function __construct(RecursiveNodeIteratorFactory $iteratorFactory, StreamHelper $streamHelper) { $this->iteratorFactory = $iteratorFactory; + $this->streamHelper = $streamHelper; } /** * @param string $userId - * @return File[] - * @throws \OCP\Files\InvalidPathException - * @throws \OCP\Files\NotFoundException + * @param string $exportPath + * + * @return void + * + * @throws NoUserException */ - public function extract($userId) { + public function extract($userId, $exportPath) { list($iterator, $baseFolder) = $this->iteratorFactory->getUserFolderParentRecursiveIterator($userId); - $files = []; + + $filename = $exportPath . '/' . $this::FILE_NAME; + $this->streamFile = $this->streamHelper->initStream($filename, 'ab', true); foreach ($iterator as $node) { $nodePath = $node->getPath(); @@ -53,6 +69,7 @@ public function extract($userId) { $file->setPath($relativePath); $file->setETag($node->getEtag()); + $file->setMtime($node->getMTime()); $file->setPermissions($node->getPermissions()); if ($node->getType() === Node::TYPE_FILE) { @@ -61,9 +78,8 @@ public function extract($userId) { $file->setType(File::TYPE_FOLDER); } - $files[] = $file; + $this->streamHelper->writelnToStream($this->streamFile, $file); } - - return $files; + $this->streamHelper->closeStream($this->streamFile); } } diff --git a/lib/Extractor/MetadataExtractor/SharesExtractor.php b/lib/Extractor/MetadataExtractor/SharesExtractor.php index be98f06..5017e1b 100644 --- a/lib/Extractor/MetadataExtractor/SharesExtractor.php +++ b/lib/Extractor/MetadataExtractor/SharesExtractor.php @@ -20,58 +20,93 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. * */ + namespace OCA\DataExporter\Extractor\MetadataExtractor; -use OCA\DataExporter\Model\User\Share; +use OCA\DataExporter\Model\Share; +use OCA\DataExporter\Utilities\StreamHelper; +use OCP\Files\IRootFolder; +use OCP\Files\NotFoundException; use OCP\IConfig; -use OCP\Share\IManager; use OCP\Share as ShareConstants; -use OCP\Files\IRootFolder; +use OCP\Share\IManager; class SharesExtractor { - /** @var IManager */ + const FILE_NAME = 'shares.jsonl'; + /** + * @var IManager + */ private $manager; - /** @var IRootFolder */ + /** + * @var IRootFolder + */ private $rootFolder; /** * @var IConfig */ private $config; + /** + * @var StreamHelper + */ + private $streamHelper; + /** + * @var resource $streamFile + */ + private $streamFile; - public function __construct(IManager $manager, IRootFolder $rootFolder, IConfig $config) { + /** + * SharesExtractor constructor. + * + * @param IManager $manager + * @param IRootFolder $rootFolder + * @param IConfig $config + * @param StreamHelper $streamHelper + */ + public function __construct( + IManager $manager, + IRootFolder $rootFolder, + IConfig $config, + StreamHelper $streamHelper + ) { $this->manager = $manager; $this->rootFolder = $rootFolder; $this->config = $config; + $this->streamHelper = $streamHelper; } /** * @param string $userId the id of the user to extract the info from - * @return Share[] + * @param string $exportPath + * + * @throws NotFoundException + * + * @return void */ - public function extract($userId) { + public function extract($userId, $exportPath) { $ocVersion = $this->config->getSystemValue('version', ''); + $filename = $exportPath . '/' . $this::FILE_NAME; + $this->streamFile = $this->streamHelper + ->initStream($filename, 'ab', true); + + $this->getUserShares($userId); + $this->getGroupShares($userId); if (\version_compare($ocVersion, '10', '<')) { - return \array_merge( - $this->getUserShares($userId), - $this->getGroupShares($userId), - $this->getLinkShares9($userId), - $this->getRemoteShares($userId) - ); + $this->getLinkShares9($userId); + } else { + $this->getLinkShares($userId); } - return \array_merge( - $this->getUserShares($userId), - $this->getGroupShares($userId), - $this->getLinkShares($userId), - $this->getRemoteShares($userId) - ); + $this->getRemoteShares($userId); + $this->streamHelper->closeStream($this->streamFile); } /** * @param string $userId the user id to get the shares from - * @return Share[] the list of matching share models + * + * @throws NotFoundException + * + * @return void */ private function getUserShares($userId) { - $shareModels = []; $limit = 50; $offset = 0; $userFolder = $this->rootFolder->getUserFolder($userId); @@ -91,24 +126,28 @@ private function getUserShares($userId) { $shareModel = new Share(); $shareModel->setPath($userFolder->getRelativePath($share->getNode()->getPath())) ->setShareType(Share::SHARETYPE_USER) + ->setType($share->getNodeType()) ->setOwner($share->getShareOwner()) ->setSharedBy($share->getSharedBy()) ->setSharedWith($share->getSharedWith()) ->setPermissions($share->getPermissions()); // the rest of the model attributes doesn't make sense with local shares - $shareModels[] = $shareModel; + $this->streamHelper->writelnToStream( + $this->streamFile, + $shareModel + ); } } while (\count($shares) >= $limit); - - return $shareModels; } /** * @param string $userId the user id to get the shares from - * @return Share[] the list of matching share models + * + * @throws NotFoundException + * + * @return void */ private function getGroupShares($userId) { - $shareModels = []; $limit = 50; $offset = 0; $userFolder = $this->rootFolder->getUserFolder($userId); @@ -128,24 +167,28 @@ private function getGroupShares($userId) { $shareModel = new Share(); $shareModel->setPath($userFolder->getRelativePath($share->getNode()->getPath())) ->setShareType(Share::SHARETYPE_GROUP) + ->setType($share->getNodeType()) ->setOwner($share->getShareOwner()) ->setSharedBy($share->getSharedBy()) ->setSharedWith($share->getSharedWith()) ->setPermissions($share->getPermissions()); // the rest of the model attributes doesn't make sense with local shares - $shareModels[] = $shareModel; + $this->streamHelper->writelnToStream( + $this->streamFile, + $shareModel + ); } } while (\count($shares) >= $limit); - - return $shareModels; } /** * @param string $userId the user id to get the shares from - * @return Share[] the list of matching share models + * + * @throws NotFoundException + * + * @return void */ - private function getLinkShares($userId) { - $shareModels = []; + private function getLinkShares9($userId) { $limit = 50; $offset = 0; $userFolder = $this->rootFolder->getUserFolder($userId); @@ -165,10 +208,11 @@ private function getLinkShares($userId) { $shareModel = new Share(); $shareModel->setPath($userFolder->getRelativePath($share->getNode()->getPath())) ->setShareType(Share::SHARETYPE_LINK) + ->setType($share->getNodeType()) ->setOwner($share->getShareOwner()) ->setSharedBy($share->getSharedBy()) ->setPermissions($share->getPermissions()) - ->setName($share->getName()) + ->setName('') ->setToken($share->getToken()); $expiration = $share->getExpirationDate(); @@ -181,19 +225,22 @@ private function getLinkShares($userId) { $shareModel->setPassword($password); } // the rest of the model attributes doesn't make sense with link shares - $shareModels[] = $shareModel; + $this->streamHelper->writelnToStream( + $this->streamFile, + $shareModel + ); } } while (\count($shares) >= $limit); - - return $shareModels; } /** * @param string $userId the user id to get the shares from - * @return Share[] the list of matching share models + * + * @throws NotFoundException + * + * @return void */ - private function getLinkShares9($userId) { - $shareModels = []; + private function getLinkShares($userId) { $limit = 50; $offset = 0; $userFolder = $this->rootFolder->getUserFolder($userId); @@ -213,10 +260,11 @@ private function getLinkShares9($userId) { $shareModel = new Share(); $shareModel->setPath($userFolder->getRelativePath($share->getNode()->getPath())) ->setShareType(Share::SHARETYPE_LINK) + ->setType($share->getNodeType()) ->setOwner($share->getShareOwner()) ->setSharedBy($share->getSharedBy()) ->setPermissions($share->getPermissions()) - ->setName('') + ->setName($share->getName()) ->setToken($share->getToken()); $expiration = $share->getExpirationDate(); @@ -229,19 +277,22 @@ private function getLinkShares9($userId) { $shareModel->setPassword($password); } // the rest of the model attributes doesn't make sense with link shares - $shareModels[] = $shareModel; + $this->streamHelper->writelnToStream( + $this->streamFile, + $shareModel + ); } } while (\count($shares) >= $limit); - - return $shareModels; } /** * @param string $userId the user id to get the shares from - * @return Share[] the list of matching share models + * + * @throws NotFoundException + * + * @return void */ private function getRemoteShares($userId) { - $shareModels = []; $limit = 50; $offset = 0; $userFolder = $this->rootFolder->getUserFolder($userId); @@ -255,21 +306,24 @@ private function getRemoteShares($userId) { $limit, $offset ); + $offset += $limit; foreach ($shares as $share) { $shareModel = new Share(); $shareModel->setPath($userFolder->getRelativePath($share->getNode()->getPath())) ->setShareType(Share::SHARETYPE_REMOTE) + ->setType($share->getNodeType()) ->setOwner($share->getShareOwner()) ->setSharedBy($share->getSharedBy()) ->setSharedWith($share->getSharedWith()) ->setPermissions($share->getPermissions()); // the rest of the model attributes doesn't make sense with remote shares - $shareModels[] = $shareModel; + $this->streamHelper->writelnToStream( + $this->streamFile, + $shareModel + ); } } while (\count($shares) >= $limit); - - return $shareModels; } } diff --git a/lib/Importer.php b/lib/Importer.php index dd107e7..00666b5 100644 --- a/lib/Importer.php +++ b/lib/Importer.php @@ -67,10 +67,10 @@ public function __construct( * @throws \OCP\PreConditionNotMetException */ public function import($pathToExportDir, $alias = null) { - $metaDataPath = "$pathToExportDir/metadata.json"; + $metaDataPath = "$pathToExportDir/user.json"; if (!$this->filesystem->exists($metaDataPath)) { - throw new ImportException("metadata.json not found in '$metaDataPath'"); + throw new ImportException("user.json not found in '$metaDataPath'"); } /** @var Metadata $metadata */ @@ -86,12 +86,11 @@ public function import($pathToExportDir, $alias = null) { $this->metadataImporter->import($metadata); $this->filesImporter->import( $metadata->getUser()->getUserId(), - $metadata->getFiles(), - "$pathToExportDir/files" + $pathToExportDir ); $this->shareImporter->import( $metadata->getUser()->getUserId(), - $metadata->getUser()->getShares(), + $pathToExportDir, $metadata->getOriginServer() ); } diff --git a/lib/Importer/FilesImporter.php b/lib/Importer/FilesImporter.php index ae8c1e6..9cca9e8 100644 --- a/lib/Importer/FilesImporter.php +++ b/lib/Importer/FilesImporter.php @@ -20,66 +20,108 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. * */ + namespace OCA\DataExporter\Importer; use OCA\DataExporter\Model\File; +use OCA\DataExporter\Utilities\StreamHelper; use OCP\Files\IRootFolder; use Symfony\Component\Filesystem\Filesystem; class FilesImporter { - - /** @var Filesystem */ + const FILE_NAME = 'files.jsonl'; + /** @var Filesystem */ private $filesystem; - /** @var IRootFolder */ + /** @var IRootFolder */ private $rootFolder; + /** + * @var StreamHelper + */ + private $streamHelper; + /** + * @var resource + */ + private $streamFile; - public function __construct(Filesystem $filesystem, IRootFolder $rootFolder) { + /** + * @var int + */ + private $currentLine; + + public function __construct(Filesystem $filesystem, IRootFolder $rootFolder, StreamHelper $streamHelper) { $this->filesystem = $filesystem; $this->rootFolder = $rootFolder; + $this->streamHelper = $streamHelper; } /** * @param string $userId - * @param array $filesMetadata - * @param string $exportRootFilesPath + * @param string $exportPath + * * @throws \OCP\Files\InvalidPathException * @throws \OCP\Files\NotFoundException * @throws \OCP\Files\NotPermittedException * @throws \OCP\Files\StorageNotAvailableException */ - public function import(string $userId, array $filesMetadata, string $exportRootFilesPath) { + public function import($userId, $exportPath) { // Trigger creation of user-folder $this->rootFolder->getUserFolder($userId); - /** @var \OCP\Files\Folder $userFolder */ + /** + * @var \OCP\Files\Folder $userFolder + */ + $filename = $exportPath . '/' . $this::FILE_NAME; + $exportRootFilesPath = $exportPath . '/files'; + $userFolder = $this->rootFolder->getUserFolder($userId)->getParent(); + $this->streamFile = $this + ->streamHelper + ->initStream($filename, 'rb'); + $this->currentLine = 1; - /** @var File $fileMetadata */ - foreach ($filesMetadata as $fileMetadata) { - $fileCachePath = $fileMetadata->getPath(); - $pathToFileInExport = "$exportRootFilesPath/$fileCachePath"; + try { + while (( + /** + * @var File $fileMetadata + */ + $fileMetadata = $this->streamHelper->readlnFromStream( + $this->streamFile, + File::class + )) + !== false + ) { + $fileCachePath = $fileMetadata->getPath(); + $pathToFileInExport = "$exportRootFilesPath/$fileCachePath"; - if (!$this->filesystem->exists($pathToFileInExport)) { - throw new ImportException("File '$pathToFileInExport' not found in export but exists in metadata.json"); - } + if (!$this->filesystem->exists($pathToFileInExport)) { + throw new ImportException("File '$pathToFileInExport' not found in export but exists in metadata.json"); + } - if ($fileMetadata->getType() === File::TYPE_FILE) { - $file = $userFolder->newFile($fileCachePath); - $file->putContent(\file_get_contents($pathToFileInExport)); - $file->getStorage()->getCache()->update($file->getId(), [ - 'etag' => $fileMetadata->getETag(), - 'permissions' => $fileMetadata->getPermissions() - ]); + if ($fileMetadata->getType() === File::TYPE_FILE) { + $file = $userFolder->newFile($fileCachePath); + $file->putContent(\file_get_contents($pathToFileInExport)); + $file->getStorage()->getCache()->update($file->getId(), [ + 'etag' => $fileMetadata->getETag(), + 'permissions' => $fileMetadata->getPermissions() + ]); - continue; - } + continue; + } - if ($fileMetadata->getType() === File::TYPE_FOLDER) { - $folder = $userFolder->newFolder($fileCachePath); - $folder->getStorage()->getCache()->update($folder->getId(), [ - 'etag' => $fileMetadata->getETag(), - 'permissions' => $fileMetadata->getPermissions() - ]); + if ($fileMetadata->getType() === File::TYPE_FOLDER) { + $folder = $userFolder->newFolder($fileCachePath); + $folder->getStorage()->getCache()->update($folder->getId(), [ + 'etag' => $fileMetadata->getETag(), + 'permissions' => $fileMetadata->getPermissions() + ]); + } + $this->currentLine++; } + } catch (ImportException $exception) { + $message = $exception->getMessage(); + $line = $this->currentLine + 1; + throw new ImportException("Import failed on $filename on line $line: $message", 0, $exception); } + $this->streamHelper->checkEndOfStream($this->streamFile); + $this->streamHelper->closeStream($this->streamFile); } } diff --git a/lib/Importer/MetadataImporter/ShareImporter.php b/lib/Importer/MetadataImporter/ShareImporter.php index 29d5a77..709805f 100644 --- a/lib/Importer/MetadataImporter/ShareImporter.php +++ b/lib/Importer/MetadataImporter/ShareImporter.php @@ -20,19 +20,23 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. * */ + namespace OCA\DataExporter\Importer\MetadataImporter; -use OCP\Share\IManager; +use OCA\DataExporter\Importer\ImportException; +use OCA\DataExporter\Model\Share; +use OCA\DataExporter\Utilities\ShareConverter; +use OCA\DataExporter\Utilities\StreamHelper; use OCP\Files\IRootFolder; -use OCP\IUserManager; use OCP\IGroupManager; -use OCP\Share as ShareConstants; -use OCP\IURLGenerator; use OCP\ILogger; -use OCA\DataExporter\Model\User\Share; -use OCA\DataExporter\Utilities\ShareConverter; +use OCP\IURLGenerator; +use OCP\IUserManager; +use OCP\Share as ShareConstants; +use OCP\Share\IManager; class ShareImporter { + const FILE_NAME = 'shares.jsonl'; /** @var IManager */ private $shareManager; /** @var IRootFolder */ @@ -47,7 +51,31 @@ class ShareImporter { private $urlGenerator; /** @var ILogger */ private $logger; + /** + * @var StreamHelper + */ + private $streamHelper; + /** + * @var resource + */ + private $streamFile; + /** + * @var int + */ + private $currentLine; + /** + * ShareImporter constructor. + * + * @param IManager $shareManager + * @param IRootFolder $rootFolder + * @param IUserManager $userManager + * @param IGroupManager $groupManager + * @param ShareConverter $shareConverter + * @param IURLGenerator $urlGenerator + * @param ILogger $logger + * @param StreamHelper $streamHelper + */ public function __construct( IManager $shareManager, IRootFolder $rootFolder, @@ -55,7 +83,8 @@ public function __construct( IGroupManager $groupManager, ShareConverter $shareConverter, IURLGenerator $urlGenerator, - ILogger $logger + ILogger $logger, + StreamHelper $streamHelper ) { $this->shareManager = $shareManager; $this->rootFolder = $rootFolder; @@ -64,6 +93,7 @@ public function __construct( $this->shareConverter = $shareConverter; $this->urlGenerator = $urlGenerator; $this->logger = $logger; + $this->streamHelper = $streamHelper; } /** @@ -72,88 +102,140 @@ public function __construct( * either created when importing the share owner or handled with migrate:share command * - remote share targeting the importing server, but missing "shared with" user (might has been deleted after the share creation) * this shouldn't happen, it will likely crash. + * * @param string $userId the importing user Id - * @param Share[] $shareModels the share models to be imported + * @param string $exportPath the share models to be imported * @param string $targetRemoteHost the exporting host where the shares come from. * This is the origin of the shares, in case we need to convert some share * to remote shares. + * * @throws \OCP\Files\NotFoundException if there is no node matching the path in any * share model. Note that this will abort the process and might leave the importing account in a bad state. + * + * @return void */ - public function import(string $userId, array $shareModels, string $targetRemoteHost) { + public function import($userId, $exportPath, $targetRemoteHost) { $currentHostUrl = \rtrim($this->urlGenerator->getAbsoluteURL('/'), '/'); - foreach ($shareModels as $shareModel) { - switch ($shareModel->getShareType()) { - case Share::SHARETYPE_USER: - if ($shareModel->getOwner() === $userId) { - $this->createRemoteUserShareFromLocalUserShare($shareModel, $targetRemoteHost); - if ($this->userManager->userExists($shareModel->getSharedWith())) { - $this->createLocalUserShareFromLocalUserShare($shareModel); + $filename = $exportPath . '/' . $this::FILE_NAME; + $this->streamFile = $this + ->streamHelper + ->initStream($filename, 'rb'); + $this->currentLine = 0; + + try { + while (( + /** + * @var Share $shareModel + */ + $shareModel = $this->streamHelper->readlnFromStream( + $this->streamFile, Share::class + ) + ) !== false + ) { + switch ($shareModel->getShareType()) { + case Share::SHARETYPE_USER: + if ($shareModel->getOwner() === $userId) { + $this->createRemoteUserShareFromLocalUserShare($shareModel, $targetRemoteHost); + if ($this->userManager->userExists($shareModel->getSharedWith())) { + $this->createLocalUserShareFromLocalUserShare($shareModel); + } + } else { + $this->logger->info('Ignoring user share {path} . The owner {owner} does not match {user}', [ + 'app' => 'data_exporter', + 'path' => $shareModel->getPath(), + 'owner' => $shareModel->getOwner(), + 'user' => $userId + ]); + } + break; + case Share::SHARETYPE_GROUP: + // check if the group exists before trying to create the share + if ($shareModel->getOwner() === $userId) { + if ($this->groupManager->groupExists($shareModel->getSharedWith())) { + $this->createLocalGroupShareFromLocalGroupShare($shareModel); + } else { + $this->logger->info('Ignoring group share {path} . Target group {group} missing', [ + 'app' => 'data_exporter', + 'path' => $shareModel->getPath(), + 'group' => $shareModel->getSharedWith() + ]); + } + } else { + $this->logger->info('Ignoring group share {path} . The owner {owner} does not match {user}', [ + 'app' => 'data_exporter', + 'path' => $shareModel->getPath(), + 'owner' => $shareModel->getOwner(), + 'user' => $userId + ]); } - } else { - $this->logger->info('Ignoring user share {path} . The owner {owner} does not match {user}', [ - 'app' => 'data_exporter', - 'path' => $shareModel->getPath(), - 'owner' => $shareModel->getOwner(), - 'user' => $userId - ]); - } - break; - case Share::SHARETYPE_GROUP: - // check if the group exists before trying to create the share - if ($shareModel->getOwner() === $userId) { - if ($this->groupManager->groupExists($shareModel->getSharedWith())) { - $this->createLocalGroupShareFromLocalGroupShare($shareModel); + break; + case Share::SHARETYPE_LINK: + if ($shareModel->getOwner() === $userId) { + $this->createLinkShareFromLinkShare($shareModel); } else { - $this->logger->info('Ignoring group share {path} . Target group {group} missing', [ + $this->logger->info('Ignoring link share {path} . The owner {owner} does not match {user}', [ 'app' => 'data_exporter', 'path' => $shareModel->getPath(), - 'group' => $shareModel->getSharedWith() + 'owner' => $shareModel->getOwner(), + 'user' => $userId ]); } - } else { - $this->logger->info('Ignoring group share {path} . The owner {owner} does not match {user}', [ - 'app' => 'data_exporter', - 'path' => $shareModel->getPath(), - 'owner' => $shareModel->getOwner(), - 'user' => $userId - ]); - } - break; - case Share::SHARETYPE_LINK: - if ($shareModel->getOwner() === $userId) { - $this->createLinkShareFromLinkShare($shareModel); - } else { - $this->logger->info('Ignoring link share {path} . The owner {owner} does not match {user}', [ - 'app' => 'data_exporter', - 'path' => $shareModel->getPath(), - 'owner' => $shareModel->getOwner(), - 'user' => $userId - ]); - } - break; - case Share::SHARETYPE_REMOTE: - if ($shareModel->getOwner() === $userId) { - $pieces = \explode('@', $shareModel->getSharedWith()); - $sharedWithRemoteHost = $pieces[\count($pieces) - 1]; - if ($sharedWithRemoteHost === $currentHostUrl) { - $this->createLocalUserShareFromRemoteUserShare($shareModel); + break; + case Share::SHARETYPE_REMOTE: + if ($shareModel->getOwner() === $userId) { + $pieces = \explode('@', $shareModel->getSharedWith()); + $sharedWithRemoteHost = $pieces[\count($pieces) - 1]; + if ($sharedWithRemoteHost === $currentHostUrl) { + $this->createLocalUserShareFromRemoteUserShare($shareModel); + } else { + $this->createRemoteUserShareFromRemoteUserShare($shareModel); + } } else { - $this->createRemoteUserShareFromRemoteUserShare($shareModel); + $this->logger->info('Ignoring remote share {path} . The owner {owner} does not match {user}', [ + 'app' => 'data_exporter', + 'path' => $shareModel->getPath(), + 'owner' => $shareModel->getOwner(), + 'user' => $userId + ]); } - } else { - $this->logger->info('Ignoring remote share {path} . The owner {owner} does not match {user}', [ - 'app' => 'data_exporter', - 'path' => $shareModel->getPath(), - 'owner' => $shareModel->getOwner(), - 'user' => $userId - ]); - } - break; + break; + } + $this->currentLine++; } + } catch (ImportException $exception) { + $message = $exception->getMessage(); + $line = $this->currentLine + 1; + throw new ImportException("Import failed on $filename on line $line: $message"); } // the userId exists as it has been imported (we shouldn't reach here otherwise) $this->shareConverter->convertRemoteUserShareToLocalUserShare($userId, $targetRemoteHost); + $this->streamHelper->checkEndOfStream($this->streamFile); + $this->streamHelper->closeStream($this->streamFile); + } + + /** + * @param Share $shareModel the model to be used in order to create the share + * @param string $targetRemoteHost the remote ownCloud server to be used for the remote share, + * for example, "https://my.server:8080/owncloud" + * @throws \OCP\Files\NotFoundException if there is no node matching the path in the share model. + */ + private function createRemoteUserShareFromLocalUserShare(Share $shareModel, string $targetRemoteHost) { + // find the node id of the share + $userId = $shareModel->getOwner(); + $userFolder = $this->rootFolder->getUserFolder($userId); + // userFolder should exists because the user information should have been imported at this point + $node = $userFolder->get($shareModel->getPath()); // this might throw an OCP\Files\NotFoundException + + $remoteUser = \rtrim($shareModel->getSharedWith() . "@{$targetRemoteHost}", '/'); + $share = $this->shareManager->newShare(); + $share->setNode($node) + ->setShareType(ShareConstants::SHARE_TYPE_REMOTE) + ->setSharedWith($remoteUser)// sharedWith existence was checked before + ->setPermissions($shareModel->getPermissions()) + ->setSharedBy($shareModel->getOwner())// Override the initiator with the owner to avoid problems + ->setShareOwner($shareModel->getOwner()); + + $this->shareManager->createShare($share); } /** @@ -170,9 +252,9 @@ private function createLocalUserShareFromLocalUserShare(Share $shareModel) { $share = $this->shareManager->newShare(); $share->setNode($node) ->setShareType(ShareConstants::SHARE_TYPE_USER) - ->setSharedWith($shareModel->getSharedWith()) // sharedWith existence was checked before + ->setSharedWith($shareModel->getSharedWith())// sharedWith existence was checked before ->setPermissions($shareModel->getPermissions()) - ->setSharedBy($shareModel->getOwner()) // Override the initiator with the owner to avoid problems + ->setSharedBy($shareModel->getOwner())// Override the initiator with the owner to avoid problems ->setShareOwner($shareModel->getOwner()); try { @@ -194,24 +276,22 @@ private function createLocalUserShareFromLocalUserShare(Share $shareModel) { /** * @param Share $shareModel the model to be used in order to create the share - * @param string $targetRemoteHost the remote ownCloud server to be used for the remote share, - * for example, "https://my.server:8080/owncloud" * @throws \OCP\Files\NotFoundException if there is no node matching the path in the share model. */ - private function createRemoteUserShareFromLocalUserShare(Share $shareModel, string $targetRemoteHost) { + private function createLocalGroupShareFromLocalGroupShare(Share $shareModel) { + // we assume the target group exists in the new server // find the node id of the share $userId = $shareModel->getOwner(); $userFolder = $this->rootFolder->getUserFolder($userId); // userFolder should exists because the user information should have been imported at this point $node = $userFolder->get($shareModel->getPath()); // this might throw an OCP\Files\NotFoundException - $remoteUser = \rtrim($shareModel->getSharedWith() . "@{$targetRemoteHost}", '/'); $share = $this->shareManager->newShare(); $share->setNode($node) - ->setShareType(ShareConstants::SHARE_TYPE_REMOTE) - ->setSharedWith($remoteUser) // sharedWith existence was checked before + ->setShareType(ShareConstants::SHARE_TYPE_GROUP) + ->setSharedWith($shareModel->getSharedWith())// sharedWith existence was checked before ->setPermissions($shareModel->getPermissions()) - ->setSharedBy($shareModel->getOwner()) // Override the initiator with the owner to avoid problems + ->setSharedBy($shareModel->getOwner())// Override the initiator with the owner to avoid problems ->setShareOwner($shareModel->getOwner()); $this->shareManager->createShare($share); @@ -233,7 +313,7 @@ private function createLinkShareFromLinkShare(Share $shareModel) { $share->setNode($node) ->setShareType(ShareConstants::SHARE_TYPE_LINK) ->setPermissions($shareModel->getPermissions()) - ->setSharedBy($shareModel->getOwner()) // Override the initiator with the owner to avoid problems + ->setSharedBy($shareModel->getOwner())// Override the initiator with the owner to avoid problems ->setShareOwner($shareModel->getOwner()) ->setName($shareModel->getName()) ->setPassword($shareModel->getPassword()); @@ -250,28 +330,6 @@ private function createLinkShareFromLinkShare(Share $shareModel) { $this->shareManager->updateShare($share); } - /** - * @param Share $shareModel the model to be used in order to create the share - * @throws \OCP\Files\NotFoundException if there is no node matching the path in the share model. - */ - private function createRemoteUserShareFromRemoteUserShare(Share $shareModel) { - // find the node id of the share - $userId = $shareModel->getOwner(); - $userFolder = $this->rootFolder->getUserFolder($userId); - // userFolder should exists because the user information should have been imported at this point - $node = $userFolder->get($shareModel->getPath()); // this might throw an OCP\Files\NotFoundException - - $share = $this->shareManager->newShare(); - $share->setNode($node) - ->setShareType(ShareConstants::SHARE_TYPE_REMOTE) - ->setSharedWith($shareModel->getSharedWith()) - ->setPermissions($shareModel->getPermissions()) - ->setSharedBy($shareModel->getOwner()) // Override the initiator with the owner to avoid problems - ->setShareOwner($shareModel->getOwner()); - - $this->shareManager->createShare($share); - } - /** * @param Share $shareModel the model to be used in order to create the share * @throws \OCP\Files\NotFoundException if there is no node matching the path in the share model. @@ -291,7 +349,7 @@ private function createLocalUserShareFromRemoteUserShare(Share $shareModel) { ->setShareType(ShareConstants::SHARE_TYPE_USER) ->setSharedWith($localUser) ->setPermissions($shareModel->getPermissions()) - ->setSharedBy($shareModel->getOwner()) // Override the initiator with the owner to avoid problems + ->setSharedBy($shareModel->getOwner())// Override the initiator with the owner to avoid problems ->setShareOwner($shareModel->getOwner()); try { @@ -315,8 +373,7 @@ private function createLocalUserShareFromRemoteUserShare(Share $shareModel) { * @param Share $shareModel the model to be used in order to create the share * @throws \OCP\Files\NotFoundException if there is no node matching the path in the share model. */ - private function createLocalGroupShareFromLocalGroupShare(Share $shareModel) { - // we assume the target group exists in the new server + private function createRemoteUserShareFromRemoteUserShare(Share $shareModel) { // find the node id of the share $userId = $shareModel->getOwner(); $userFolder = $this->rootFolder->getUserFolder($userId); @@ -325,10 +382,10 @@ private function createLocalGroupShareFromLocalGroupShare(Share $shareModel) { $share = $this->shareManager->newShare(); $share->setNode($node) - ->setShareType(ShareConstants::SHARE_TYPE_GROUP) - ->setSharedWith($shareModel->getSharedWith()) // sharedWith existence was checked before + ->setShareType(ShareConstants::SHARE_TYPE_REMOTE) + ->setSharedWith($shareModel->getSharedWith()) ->setPermissions($shareModel->getPermissions()) - ->setSharedBy($shareModel->getOwner()) // Override the initiator with the owner to avoid problems + ->setSharedBy($shareModel->getOwner())// Override the initiator with the owner to avoid problems ->setShareOwner($shareModel->getOwner()); $this->shareManager->createShare($share); diff --git a/lib/Model/File.php b/lib/Model/File.php index 17c608a..ee20de7 100644 --- a/lib/Model/File.php +++ b/lib/Model/File.php @@ -23,6 +23,12 @@ */ namespace OCA\DataExporter\Model; +/** + * Represents the Files Metadata export format + * + * @package OCA\DataExporter\Model + * @codeCoverageIgnore + */ class File { const TYPE_FOLDER = 'folder'; const TYPE_FILE = 'file'; @@ -34,6 +40,8 @@ class File { private $eTag; /** @var int */ private $permissions; + /** @var int */ + private $mtime; /** * @return string @@ -98,4 +106,20 @@ public function setPermissions($permissions) { $this->permissions = $permissions; return $this; } + + /** + * @return int + */ + public function getMtime() { + return $this->mtime; + } + + /** + * @param int $mtime + * @return File + */ + public function setMtime($mtime) { + $this->mtime = $mtime; + return $this; + } } diff --git a/lib/Model/Instance.php b/lib/Model/Instance.php index 43166f7..41ef827 100644 --- a/lib/Model/Instance.php +++ b/lib/Model/Instance.php @@ -23,6 +23,12 @@ namespace OCA\DataExporter\Model; +/** + * Represents the Instance Data export format + * + * @package OCA\DataExporter\Model + * @codeCoverageIgnore + */ class Instance { /** * @var \DateTimeImmutable diff --git a/lib/Model/Metadata.php b/lib/Model/Metadata.php index 3785035..153f10f 100644 --- a/lib/Model/Metadata.php +++ b/lib/Model/Metadata.php @@ -23,6 +23,12 @@ */ namespace OCA\DataExporter\Model; +/** + * Represents the Metadata export format + * + * @package OCA\DataExporter\Model + * @codeCoverageIgnore + */ class Metadata { /** @var \DateTimeImmutable */ @@ -31,8 +37,6 @@ class Metadata { private $originServer; /** @var User */ private $user; - /** @var File[] */ - private $files = []; /** * @return \DateTimeImmutable @@ -82,21 +86,4 @@ public function setUser(User $user) { $this->user = $user; return $this; } - - /** - * @return File[] - */ - public function getFiles() { - return $this->files; - } - - /** - * @param File[] $files - * - * @return Metadata - */ - public function setFiles(array $files) { - $this->files = $files; - return $this; - } } diff --git a/lib/Model/User/Share.php b/lib/Model/Share.php similarity index 93% rename from lib/Model/User/Share.php rename to lib/Model/Share.php index 17cde9f..4183c93 100644 --- a/lib/Model/User/Share.php +++ b/lib/Model/Share.php @@ -20,8 +20,14 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. * */ -namespace OCA\DataExporter\Model\User; +namespace OCA\DataExporter\Model; +/** + * Represents the Share Data export format + * + * @package OCA\DataExporter\Model + * @codeCoverageIgnore + */ class Share { const SHARETYPE_USER = 'user'; const SHARETYPE_GROUP = 'group'; @@ -33,6 +39,8 @@ class Share { /** @var string */ private $shareType; /** @var string */ + private $type; + /** @var string */ private $owner; /** @var string */ private $initiator; @@ -81,6 +89,15 @@ public function setShareType($shareType) { return $this; } + /** + * @param string $type + * @return Share + */ + public function setType($type) { + $this->type = $type; + return $this; + } + /** * @return string the owner of the share */ diff --git a/lib/Model/User.php b/lib/Model/User.php index 54f6810..fb966ca 100644 --- a/lib/Model/User.php +++ b/lib/Model/User.php @@ -24,8 +24,13 @@ namespace OCA\DataExporter\Model; use OCA\DataExporter\Model\User\Preference; -use OCA\DataExporter\Model\User\Share; +/** + * Represents the User Data export format + * + * @package OCA\DataExporter\Model + * @codeCoverageIgnore + */ class User { /** @var string */ @@ -44,8 +49,6 @@ class User { private $groups = []; /** @var Preference[] */ private $preferences = []; - /** @var Share[] */ - private $shares = []; /** * @return string @@ -174,20 +177,4 @@ public function setPreferences(array $preferences) { $this->preferences = $preferences; return $this; } - - /** - * @return Share[] - */ - public function getShares() { - return $this->shares; - } - - /** - * @param Share[] $shares - * @return User - */ - public function setShares(array $shares) { - $this->shares = $shares; - return $this; - } } diff --git a/lib/Model/User/Preference.php b/lib/Model/User/Preference.php index 3ea4566..2b4982f 100644 --- a/lib/Model/User/Preference.php +++ b/lib/Model/User/Preference.php @@ -22,6 +22,12 @@ */ namespace OCA\DataExporter\Model\User; +/** + * Represents the User Preferences Export format + * + * @package OCA\DataExporter\Model\User + * @codeCoverageIgnore + */ class Preference { /** @var string */ private $appId; diff --git a/lib/Utilities/StreamHelper.php b/lib/Utilities/StreamHelper.php new file mode 100644 index 0000000..494ee2d --- /dev/null +++ b/lib/Utilities/StreamHelper.php @@ -0,0 +1,127 @@ + + * + * @copyright Copyright (c) 2019, ownCloud GmbH + * @license GPL-2.0 + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + */ +namespace OCA\DataExporter\Utilities; + +use OCA\DataExporter\Importer\ImportException; +use OCA\DataExporter\Model\File; +use OCA\DataExporter\Model\Share; +use OCA\DataExporter\Serializer; +use Symfony\Component\Filesystem\Filesystem; +use Symfony\Component\Serializer\Exception\UnexpectedValueException; + +class StreamHelper { + /** + * @var Filesystem + */ + private $filesystem; + /** + * @var Serializer + */ + private $serializer; + + /** + * StreamHelper constructor. + * + * @param Filesystem $filesystem + * @param Serializer $serializer + */ + public function __construct(Filesystem $filesystem, Serializer $serializer) { + $this->filesystem = $filesystem; + $this->serializer = $serializer; + } + + /** + * SetUp the Stream for the export + * Create empty file and open stream + * + * @param string $file + * @param string $mode + * @param bool $createFile + * + * @return resource + */ + public function initStream($file, $mode, $createFile = false) { + if ($createFile) { + $this->filesystem->dumpFile($file, ''); + } + return \fopen($file, $mode); + } + + /** + * close the Stream for the export + * + * @param resource $resource + * + * @return void + */ + public function closeStream($resource) { + \fclose($resource); + } + + /** + * Write line to the Stream for the export + * Create empty file and open stream + * + * @param resource $resource + * @param Share | File $entity + * + * @return void + */ + public function writelnToStream($resource, $entity) { + $data = $this->serializer->serialize($entity); + \fwrite($resource, $data . PHP_EOL); + } + + /** + * Read single line from stream resource + * + * @param resource $resource + * @param string $type + * + * @return Share | File | object | bool + */ + public function readlnFromStream($resource, $type) { + try { + return $this->serializer->deserialize( + \fgets($resource), + $type + ); + } catch (UnexpectedValueException $exception) { + if (\feof($resource)) { + return false; + } + throw new ImportException('Invalid Data.'); + } + } + + /** + * @param resource $resource + * + * @return void + */ + public function checkEndOfStream($resource) { + if (!\feof($resource)) { + throw new ImportException("Error: Failure while reading Stream"); + } + } +} diff --git a/tests/acceptance/data/simpleExport/userfoo/files.jsonl b/tests/acceptance/data/simpleExport/userfoo/files.jsonl new file mode 100644 index 0000000..79ea006 --- /dev/null +++ b/tests/acceptance/data/simpleExport/userfoo/files.jsonl @@ -0,0 +1,4 @@ +{"type":"folder","path":"\/files","eTag":"5bc8867cc2375","permissions":31,"mtime":1565124588} +{"type":"folder","path":"\/files\/AFolder","eTag":"5bc8867cc2375","permissions":31,"mtime":1565124588} +{"type":"file","path":"\/files\/AFolder\/afile.txt","eTag":"533c8d4b4c45b62e68cc09e810db7a23","permissions":27,"mtime":1565124588} +{"type":"file","path":"\/files\/welcome.txt","eTag":"84131779d95429f06405840e136babc2","permissions":27,"mtime":1565124588} diff --git a/tests/acceptance/data/simpleExport/userfoo/metadata.json b/tests/acceptance/data/simpleExport/userfoo/metadata.json deleted file mode 100644 index ba4e681..0000000 --- a/tests/acceptance/data/simpleExport/userfoo/metadata.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "date": "2018-10-18T13:12:05+00:00", - "originServer": "http:\/\/localhost:8000\/", - "user": { - "userId": "userfoo", - "displayName": "User Foo", - "email": "foo@example.com", - "quota": "default", - "backend": "Database", - "enabled": true, - "groups": [ - "admin" - ], - "preferences": [ - { - "appId": "core", - "configKey": "lang", - "configValue": "en" - }, - { - "appId": "core", - "configKey": "timezone", - "configValue": "Europe\/Berlin" - } - ], - "shares": [] - }, - "files": [ - { - "type": "folder", - "path": "\/files", - "eTag": "5bc8867cc2375", - "permissions": 31 - }, - { - "type": "folder", - "path": "\/files\/AFolder", - "eTag": "5bc8867cc2375", - "permissions": 31 - }, - { - "type": "file", - "path": "\/files\/AFolder\/afile.txt", - "eTag": "533c8d4b4c45b62e68cc09e810db7a23", - "permissions": 27 - }, - { - "type": "file", - "path": "\/files\/welcome.txt", - "eTag": "84131779d95429f06405840e136babc2", - "permissions": 27 - } - ] -} \ No newline at end of file diff --git a/tests/acceptance/data/simpleExport/userfoo/user.json b/tests/acceptance/data/simpleExport/userfoo/user.json new file mode 100644 index 0000000..17f24fb --- /dev/null +++ b/tests/acceptance/data/simpleExport/userfoo/user.json @@ -0,0 +1,27 @@ +{ + "date": "2018-10-18T13:12:05+00:00", + "originServer": "http:\/\/localhost:8000\/", + "user": { + "userId": "userfoo", + "displayName": "User Foo", + "email": "foo@example.com", + "quota": "default", + "backend": "Database", + "enabled": true, + "groups": [ + "admin" + ], + "preferences": [ + { + "appId": "core", + "configKey": "lang", + "configValue": "en" + }, + { + "appId": "core", + "configKey": "timezone", + "configValue": "Europe\/Berlin" + } + ] + } +} \ No newline at end of file diff --git a/tests/acceptance/features/bootstrap/DataExporterContext.php b/tests/acceptance/features/bootstrap/DataExporterContext.php index 26d8125..b0a6c19 100644 --- a/tests/acceptance/features/bootstrap/DataExporterContext.php +++ b/tests/acceptance/features/bootstrap/DataExporterContext.php @@ -138,7 +138,7 @@ public function exportUserUsingTheOccCommand($user, $path) { $this->lastExportBasePath = $internalPath; $this->lastExportPath = self::path("{$this->lastExportBasePath}/$user/"); $this->lastExportUser = $user; - $this->lastExportMetadataPath = "{$this->lastExportPath}metadata.json"; + $this->lastExportMetadataPath = "{$this->lastExportPath}files.jsonl"; } /** @@ -204,8 +204,13 @@ private static function assertPathContainsExport($path) { ); \PHPUnit\Framework\Assert::assertFileExists( - self::path("$path/metadata.json"), - "No metadata.json found inside export $path" + self::path("$path/user.json"), + "No user.json found inside export $path" + ); + + \PHPUnit\Framework\Assert::assertFileExists( + self::path("$path/files.jsonl"), + "No files.jsonl found inside export $path" ); } @@ -259,19 +264,23 @@ private function readFileFromServerRoot($path) { * @return void */ private function assertFileExistsInLastExportMetadata($filename) { - $metadata = \json_decode( - $this->readFileFromServerRoot($this->lastExportMetadataPath), - true - ); + $fileContent = $this->readFileFromServerRoot($this->lastExportMetadataPath); + $fileContents = \explode(PHP_EOL, $fileContent); - if (!isset($metadata['files']) || empty($metadata['files'])) { - \PHPUnit\Framework\Assert::fail('File not found in metadata'); + if (!isset($fileContents) || empty($fileContents)) { + \PHPUnit\Framework\Assert::fail('Not a valid metadata file'); } $isFileFoundInExport = false; - foreach ($metadata['files'] as $file) { - if (isset($file['path']) && $file['path'] === self::path("/$filename")) { - $isFileFoundInExport = true; + foreach ($fileContents as $file) { + if (!empty($file)) { + $fileMetadata = \json_decode( + $file, + true + ); + if (isset($fileMetadata['path']) && $fileMetadata['path'] === self::path("/$filename")) { + $isFileFoundInExport = true; + } } } diff --git a/tests/integration/MetadataImportExportTest.php b/tests/integration/MetadataImportExportTest.php index e065034..50b9be0 100644 --- a/tests/integration/MetadataImportExportTest.php +++ b/tests/integration/MetadataImportExportTest.php @@ -22,12 +22,17 @@ */ namespace OCA\DataExporter\Tests\Integration; +use OCA\DataExporter\Extractor\FilesExtractor; use OCA\DataExporter\Extractor\MetadataExtractor; +use OCA\DataExporter\Importer\FilesImporter; use OCA\DataExporter\Importer\MetadataImporter; use OCA\DataExporter\Importer\MetadataImporter\UserImporter; +use OCA\DataExporter\Model\File; +use OCA\DataExporter\Model\Metadata; use OCA\DataExporter\Serializer; -use OCP\IUser; -use OCP\IUserManager; +use OCP\IGroup; +use OCP\IGroupManager; +use org\bovigo\vfs\vfsStream; use Test\Traits\UserTrait; /** @@ -36,21 +41,29 @@ class MetadataImportExportTest extends \Test\TestCase { use UserTrait; - /** @var IUser */ - private $testUser; + /** @var IGroup $testGroup */ + private $testGroup; + const FILES_CONTENT = <<< JSONL +{"type":"folder","path":"\/files","eTag":"5d481eb6273f6","permissions":31} +{"type":"file","path":"\/files\/someFile.txt","eTag":"0f7c0b5154f85e7aa74a54d361feb7d5","permissions":27} +{"type":"folder","path":"\/files\/someFolder","eTag":"5d481eb6273f6","permissions":31} + +JSONL; + const SHARES_CONTENT = <<< JSONL +{"path":"\/someFile.txt","shareType":"remote","type":"file","owner":"testuser","sharedBy":"testuser","sharedWith":"testuser2@http://localhost","permissions":19,"expirationDate":null,"password":null,"name":null,"token":null} +{"path":"\/someFolder","shareType":"group","type":"folder","owner":"testuser","sharedBy":"testuser","sharedWith":"people","permissions":31,"expirationDate":null,"password":null,"name":null,"token":null} + +JSONL; + const USER_CONTENT = <<< JSONL +{"date":"2019-08-05T12:21:14+00:00","originServer":"http:\/\/localhost\/","user":{"userId":"testuser","displayName":"someUser","email":"test@owncloud.com","quota":"default","backend":"Database","enabled":true,"groups":["admin","people"],"preferences":[{"appId":"core","configKey":"lang","configValue":"de"},{"appId":"core","configKey":"timezone","configValue":"Europe\/Berlin"}]}} +JSONL; public function setUp() { parent::setUp(); - /** @var IUserManager $userManager */ - $userManager = \OC::$server->getUserManager(); - $testId = \bin2hex(\random_bytes(4)); + /** @var IGroupManager $groupManager */ + $groupManager = \OC::$server->getGroupManager(); - $this->testUser = $userManager->createUser("importexportintest-$testId", '123'); - $this->testUser->setEMailAddress("$testId@example.com"); - - $config = \OC::$server->getConfig(); - $config->setUserValue($this->testUser->getUID(), 'core', 'status', 'ok'); - $config->setUserValue($this->testUser->getUID(), 'core', 'value', 'key'); + $this->testGroup = $groupManager->createGroup('people'); // need to adjust the UserImporter to allow to create users in the Dummy backend $userImporter = \OC::$server->query(UserImporter::class); @@ -58,42 +71,86 @@ public function setUp() { } public function testExtractThenImportThenExtractGeneratesSameMetadata() { - /** @var MetadataExtractor $extractor */ - $extractor = \OC::$server->query(MetadataExtractor::class); - /** @var MetadataImporter $importer */ - $importer = \OC::$server->query(MetadataImporter::class); - - $export = $extractor->extract($this->testUser->getUID()); - // Don't test files for now - $export->setFiles([]); - - $this->testUser->delete(); - - $importer->import($export); - - $reExport = $extractor->extract($this->testUser->getUID()); - // Don't test files for now - $reExport->setFiles([]); - + /** @var MetadataExtractor $metadataExtractor */ + $metadataExtractor = \OC::$server->query(MetadataExtractor::class); + /** @var MetadataImporter $metadataImporter */ + $metadataImporter = \OC::$server->query(MetadataImporter::class); + /** @var FilesExtractor $filesExtractor */ + $filesExtractor = \OC::$server->query(FilesExtractor::class); + /** @var MetadataExtractor\FilesMetadataExtractor $filesMetadataExtractor */ + $filesMetadataExtractor = \OC::$server->query(MetadataExtractor\FilesMetadataExtractor::class); + /** @var FilesImporter $filesImporter*/ + $filesImporter = \OC::$server->query(FilesImporter::class); + /** @var Serializer $serializer */ + $serializer = \OC::$server->query(Serializer::class); + + $directoryTree = [ + 'testimport' => [ + 'testuser' => [ + 'shares.jsonl' => $this::SHARES_CONTENT, + 'files.jsonl' => $this::FILES_CONTENT, + 'user.json' => $this::USER_CONTENT, + 'files' => [ + 'files' => [ + 'someFile.txt' => 'This is a test!', + 'someFolder' => [] + ] + ] + ] + ], + 'testexport' => [ + 'testuser' => [] + ] + ]; + $fileSystem = vfsStream::setup('testroot', '644', $directoryTree); $date = new \DateTimeImmutable('now'); - - // Set same date on both exports for testing - $export->setDate($date); - $reExport->setDate($date); - - $ser = new Serializer(); - $export = $ser->serialize($export); - $reExport = $ser->serialize($reExport); - + /** @var Metadata $metadata */ + $metadata = $serializer->deserialize( + \file_get_contents($fileSystem->url() . '/testimport/testuser/user.json'), + Metadata::class + ); + $metadata->setDate($date); + $metadataImporter->import($metadata); + $filesImporter->import('testuser', $fileSystem->url() . '/testimport/testuser'); + // @todo Shares do not import / export cleanly because of conversion + // to federated + + $metadataExported = $metadataExtractor->extract('testuser', $fileSystem->url() . '/testexport'); + $metadataExported->setDate($date); + $metadataExported->getUser()->setBackend('Database'); + $user = $metadataExported->getUser(); + $groups = $user->getGroups(); + \sort($groups); + $metadataExported->getUser()->setGroups($groups); $this->assertEquals( - $export, - $reExport, + $metadata, + $metadataExported, 'Export metadata does not match after import/export cycle' ); + $filesMetadataExtractor->extract('testuser', $fileSystem->url() . '/testexport/testuser'); + $filesExtractor->export('testuser', $fileSystem->url() . '/testexport/testuser'); + $filesMetadata = \explode( + PHP_EOL, + \file_get_contents($fileSystem->url() . '/testexport/testuser/files.jsonl') + + ); + $expectedFilesMetadata = \explode( + PHP_EOL, + \file_get_contents($fileSystem->url() . '/testexport/testuser/files.jsonl') + ); + foreach ($expectedFilesMetadata as $key => $expected) { + if (!empty($expected)) { + $expectedObject = $serializer->deserialize($expected, File::class); + $actualObject = $serializer->deserialize($filesMetadata[$key], File::class); + // @todo Etag of parent doesn't match + $actualObject->setEtag($expectedObject->getEtag()); + $this->assertEquals($expectedObject, $actualObject); + } + } } public function tearDown() { - $this->testUser->delete(); + $this->testGroup->delete(); return parent::tearDown(); } } diff --git a/tests/unit/ExporterTest.php b/tests/unit/ExporterTest.php index 267022e..8350423 100644 --- a/tests/unit/ExporterTest.php +++ b/tests/unit/ExporterTest.php @@ -79,7 +79,7 @@ public function testExporter() { $this->filesystem ->expects($this->once()) ->method('dumpFile') - ->with('/tmp/testuser/metadata.json'); + ->with('/tmp/testuser/user.json'); $exporter = new Exporter( $this->serializer, $this->metadataExtractor, diff --git a/tests/unit/Extractor/FilesExtractorTest.php b/tests/unit/Extractor/FilesExtractorTest.php index cf52c50..a0510da 100644 --- a/tests/unit/Extractor/FilesExtractorTest.php +++ b/tests/unit/Extractor/FilesExtractorTest.php @@ -50,6 +50,7 @@ public function testExportFile() { $mockFile = $this->createMock(File::class); $mockFile->method('getPath')->willReturn('/usertest/files/foo/bar.txt'); $mockFile->method('getContent')->willReturn('weeee eee eeeeeeeee!'); + $mockFile->method('getMTime')->willReturn(1565074220); $userFolderParent = $this->createMock(Folder::class); $userFolderParent->method('getRelativePath') @@ -74,6 +75,7 @@ public function testExportFile() { public function testExportFolder() { $mockFolder = $this->createMock(Folder::class); $mockFolder->method('getPath')->willReturn('/usertest/files/foo/courses'); + $mockFolder->method('getMTime')->willReturn(1565074220); $userFolderParent = $this->createMock(Folder::class); $userFolderParent->method('getRelativePath') @@ -98,17 +100,21 @@ public function testExportFolder() { public function testExportFileAndFolder() { $mockFolder1 = $this->createMock(Folder::class); $mockFolder1->method('getPath')->willReturn('/usertest/files/foo'); + $mockFolder1->method('getMTime')->willReturn(1565074232); $mockFolder2 = $this->createMock(Folder::class); $mockFolder2->method('getPath')->willReturn('/usertest/files/foo/courses'); + $mockFolder2->method('getMTime')->willReturn(1565074200); $mockFile1 = $this->createMock(File::class); $mockFile1->method('getPath')->willReturn('/usertest/files/foo/courses/awesome qwerty'); $mockFile1->method('getContent')->willReturn('qwerty!!'); + $mockFile1->method('getMTime')->willReturn(1565074520); $mockFile2 = $this->createMock(File::class); $mockFile2->method('getPath')->willReturn('/usertest/files/foo/bar.txt'); $mockFile2->method('getContent')->willReturn('weeee eee eeeeeeeee!'); + $mockFile2->method('getMTime')->willReturn(1565074221); $userFolderParent = $this->createMock(Folder::class); $userFolderParent->method('getRelativePath') diff --git a/tests/unit/Extractor/MetadataExtractor/FilesMetadataExtractorTest.php b/tests/unit/Extractor/MetadataExtractor/FilesMetadataExtractorTest.php index 550d2f5..981691c 100644 --- a/tests/unit/Extractor/MetadataExtractor/FilesMetadataExtractorTest.php +++ b/tests/unit/Extractor/MetadataExtractor/FilesMetadataExtractorTest.php @@ -23,47 +23,70 @@ namespace OCA\DataExporter\Tests\Unit\Extractor\MetadataExtractor; use OCA\DataExporter\Extractor\MetadataExtractor\FilesMetadataExtractor; -use OCA\DataExporter\Utilities\Iterators\Nodes\RecursiveNodeIteratorFactory; use OCA\DataExporter\Model\File; -use OCP\Files\Node; +use OCA\DataExporter\Utilities\Iterators\Nodes\RecursiveNodeIteratorFactory; +use OCA\DataExporter\Utilities\StreamHelper; use OCP\Files\Folder; +use OCP\Files\Node; +use org\bovigo\vfs\vfsStream; use Test\TestCase; class FilesMetadataExtractorTest extends TestCase { - /** @var RecursiveNodeIteratorFactory */ + /** + * @var RecursiveNodeIteratorFactory | \PHPUnit_Framework_MockObject_MockObject + */ private $iteratorFactory; - - /** @var FilesMetadataExtractor */ + /** + * @var FilesMetadataExtractor | \PHPUnit_Framework_MockObject_MockObject + */ private $filesMetadataExtractor; + /** + * @var StreamHelper | \PHPUnit_Framework_MockObject_MockObject + */ + private $streamHelper; protected function setUp() { $this->iteratorFactory = $this->createMock(RecursiveNodeIteratorFactory::class); + $this->streamHelper = $this->createMock(StreamHelper::class); - $this->filesMetadataExtractor = new FilesMetadataExtractor($this->iteratorFactory); + $this->filesMetadataExtractor = new FilesMetadataExtractor($this->iteratorFactory, $this->streamHelper); } public function testExtract() { + $directoryTree = [ + 'testexport' => [ + 'usertest' => [ + 'files.jsonl' => '' + ] + ] + ]; + $fileSystem = vfsStream::setup('testroot', '644', $directoryTree); + $resource = \fopen($fileSystem->url() . '/testexport/usertest/files.jsonl', 'ab'); $mockFolder1 = $this->createMock(Node::class); $mockFolder1->method('getPath')->willReturn('/usertest/files/foo'); $mockFolder1->method('getEtag')->willReturn('123qweasdzxc'); + $mockFolder1->method('getMTime')->willReturn(1565074220); $mockFolder1->method('getPermissions')->willReturn(31); $mockFolder1->method('getType')->willReturn(Node::TYPE_FOLDER); $mockFolder2 = $this->createMock(Node::class); $mockFolder2->method('getPath')->willReturn('/usertest/files/foo/courses'); $mockFolder2->method('getEtag')->willReturn('zaqxswcde'); + $mockFolder2->method('getMTime')->willReturn(1565074223); $mockFolder2->method('getPermissions')->willReturn(31); $mockFolder2->method('getType')->willReturn(Node::TYPE_FOLDER); $mockFile1 = $this->createMock(Node::class); $mockFile1->method('getPath')->willReturn('/usertest/files/foo/courses/awesome qwerty'); $mockFile1->method('getEtag')->willReturn('poiulkjhmnbv'); + $mockFile1->method('getMTime')->willReturn(1565074221); $mockFile1->method('getPermissions')->willReturn(1); $mockFile1->method('getType')->willReturn(Node::TYPE_FILE); $mockFile2 = $this->createMock(Node::class); $mockFile2->method('getPath')->willReturn('/usertest/files/foo/bar.txt'); $mockFile2->method('getEtag')->willReturn('123456789'); + $mockFile2->method('getMTime')->willReturn(1565074120); $mockFile2->method('getPermissions')->willReturn(9); $mockFile2->method('getType')->willReturn(Node::TYPE_FILE); @@ -84,29 +107,49 @@ public function testExtract() { $expectedFolder1 = new File(); $expectedFolder1->setPath('/files/foo') ->setEtag('123qweasdzxc') + ->setMtime(1565074220) ->setPermissions(31) ->setType(File::TYPE_FOLDER); $expectedFolder2 = new File(); $expectedFolder2->setPath('/files/foo/courses') ->setEtag('zaqxswcde') + ->setMtime(1565074223) ->setPermissions(31) ->setType(File::TYPE_FOLDER); $expectedFile1 = new File(); $expectedFile1->setPath('/files/foo/courses/awesome qwerty') ->setEtag('poiulkjhmnbv') + ->setMtime(1565074221) ->setPermissions(1) ->setType(File::TYPE_FILE); $expectedFile2 = new File(); $expectedFile2->setPath('/files/foo/bar.txt') ->setEtag('123456789') + ->setMtime(1565074120) ->setPermissions(9) ->setType(File::TYPE_FILE); - $expectedFileModels = [$expectedFolder1, $expectedFolder2, $expectedFile1, $expectedFile2]; - $currentFileModels = $this->filesMetadataExtractor->extract('usertest'); - $this->assertEquals($expectedFileModels, $currentFileModels); + $this->streamHelper + ->expects($this->exactly(4)) + ->method('writelnToStream') + ->withConsecutive( + [$resource, $expectedFolder1], + [$resource, $expectedFolder2], + [$resource, $expectedFile1], + [$resource, $expectedFile2] + ); + $this->streamHelper + ->expects($this->once()) + ->method('initStream') + ->willReturn($resource); + + $this->streamHelper + ->expects($this->once()) + ->method('closeStream'); + + $this->filesMetadataExtractor->extract('usertest', '/usertest'); } } diff --git a/tests/unit/Extractor/MetadataExtractor/SharesExtractorTest.php b/tests/unit/Extractor/MetadataExtractor/SharesExtractorTest.php index 3f3d5d3..0e7165b 100644 --- a/tests/unit/Extractor/MetadataExtractor/SharesExtractorTest.php +++ b/tests/unit/Extractor/MetadataExtractor/SharesExtractorTest.php @@ -23,26 +23,43 @@ namespace OCA\DataExporter\Tests\Unit\Extractor\MetadataExtractor; use OCA\DataExporter\Extractor\MetadataExtractor\SharesExtractor; -use OCA\DataExporter\Model\User\Share; +use OCA\DataExporter\Model\Share; +use OCA\DataExporter\Utilities\StreamHelper; +use OCP\Files\Folder; +use OCP\Files\IRootFolder; +use OCP\Files\Node; use OCP\IConfig; -use OCP\Share\IShare; use OCP\Share as ShareConstants; use OCP\Share\IManager; -use OCP\Files\IRootFolder; -use OCP\Files\Node; -use OCP\Files\Folder; +use OCP\Share\IShare; +use org\bovigo\vfs\vfsStream; use Test\TestCase; class SharesExtractorTest extends TestCase { - /** @var SharesExtractor */ + /** + * @var SharesExtractor + */ private $sharesExtractor; - - /** @var IManager | \PHPUnit\Framework\MockObject\MockObject */ + /** + * @var IManager | \PHPUnit\Framework\MockObject\MockObject + */ private $manager; - /** @var IRootFolder | \PHPUnit\Framework\MockObject\MockObject */ + /** + * @var IRootFolder | \PHPUnit\Framework\MockObject\MockObject + */ private $rootFolder; - /** @var IConfig | \PHPUnit\Framework\MockObject\MockObject */ + /** + * @var IConfig | \PHPUnit\Framework\MockObject\MockObject + */ private $config; + /** + * @var StreamHelper | \PHPUnit\Framework\MockObject\MockObject + */ + private $streamHelper; + /** + * @var resource + */ + private $resource; protected function setUp() { parent::setUp(); @@ -50,12 +67,22 @@ protected function setUp() { $this->manager = $this->createMock(IManager::class); $this->rootFolder = $this->createMock(IRootFolder::class); $this->config = $this->createMock(IConfig::class); + $this->streamHelper = $this->createMock(StreamHelper::class); $this->config ->method('getSystemValue') ->with('version', '') ->willReturn('10.1.0'); - $this->sharesExtractor = new SharesExtractor($this->manager, $this->rootFolder, $this->config); + $this->sharesExtractor = new SharesExtractor($this->manager, $this->rootFolder, $this->config, $this->streamHelper); + $directoryTree = [ + 'testexport' => [ + 'usertest' => [ + 'files.jsonl' => '' + ] + ] + ]; + $fileSystem = vfsStream::setup('testroot', '644', $directoryTree); + $this->resource = \fopen($fileSystem->url() . '/testexport/usertest/shares.jsonl', 'ab'); } private function getFakeShare(string $path, string $owner, string $sharedBy, string $sharedWith, int $permissions) { @@ -64,6 +91,7 @@ private function getFakeShare(string $path, string $owner, string $sharedBy, str $share = $this->createMock(IShare::class); $share->method('getNode')->willReturn($node); + $share->method('getNodeType')->willReturn('file'); $share->method('getShareOwner')->willReturn($owner); $share->method('getSharedBy')->willReturn($sharedBy); $share->method('getSharedWith')->willReturn($sharedWith); @@ -77,6 +105,7 @@ private function getFakeLinkShare(string $path, string $owner, string $sharedBy, $share = $this->createMock(IShare::class); $share->method('getNode')->willReturn($node); + $share->method('getNodeType')->willReturn('file'); $share->method('getShareOwner')->willReturn($owner); $share->method('getSharedBy')->willReturn($sharedBy); $share->method('getPermissions')->willReturn($permissions); @@ -123,79 +152,96 @@ public function testExtract() { return $node; })); - $realModels = $this->sharesExtractor->extract($user); - - $expectedShareModels = [ - (new Share()) - ->setPath('/path/to/file') - ->setShareType(SHARE::SHARETYPE_USER) - ->setOwner('usertest') - ->setSharedBy('usertest') - ->setSharedWith('usertest2') - ->setPermissions(1), - (new Share()) - ->setPath('/path/to/file') - ->setShareType(SHARE::SHARETYPE_USER) - ->setOwner('usertest') - ->setSharedBy('initiator') - ->setSharedWith('usertest2') - ->setPermissions(1), - (new Share()) - ->setPath('/path/to/file') - ->setShareType(SHARE::SHARETYPE_GROUP) - ->setOwner('usertest') - ->setSharedBy('initiator') - ->setSharedWith('group') - ->setPermissions(31), - (new Share()) - ->setPath('/path/to/file') - ->setShareType(SHARE::SHARETYPE_LINK) - ->setOwner('usertest') - ->setSharedBy('initiator') - ->setPermissions(31) - ->setToken('#token') - ->setName('my link name') - ->setExpirationDate(1556150400) - ->setPassword('password'), - (new Share()) - ->setPath('/path/to/file') - ->setShareType(SHARE::SHARETYPE_REMOTE) - ->setOwner('usertest') - ->setSharedBy('initiator') - ->setSharedWith('user@remote') - ->setPermissions(1), - ]; - - $this->assertEquals(\count($expectedShareModels), \count($realModels)); - - foreach ($expectedShareModels as $key => $expectedShareModel) { - $this->assertEquals($expectedShareModel->getPath(), $realModels[$key]->getPath()); - $this->assertEquals($expectedShareModel->getShareType(), $realModels[$key]->getShareType()); - $this->assertEquals($expectedShareModel->getOwner(), $realModels[$key]->getOwner()); - $this->assertEquals($expectedShareModel->getSharedBy(), $realModels[$key]->getSharedBy()); - $this->assertEquals($expectedShareModel->getSharedWith(), $realModels[$key]->getSharedWith()); - $this->assertEquals($expectedShareModel->getPermissions(), $realModels[$key]->getPermissions()); - $this->assertEquals($expectedShareModel->getToken(), $realModels[$key]->getToken()); - $this->assertEquals($expectedShareModel->getExpirationDate(), $realModels[$key]->getExpirationDate()); - $this->assertEquals($expectedShareModel->getPassword(), $realModels[$key]->getPassword()); - $this->assertEquals($expectedShareModel->getName(), $realModels[$key]->getName()); - } + $userShareModel1 = (new Share()) + ->setPath('/path/to/file') + ->setShareType(SHARE::SHARETYPE_USER) + ->setType('file') + ->setOwner('usertest') + ->setSharedBy('usertest') + ->setSharedWith('usertest2') + ->setPermissions(1); + $userShareModel2 = (new Share()) + ->setPath('/path/to/file') + ->setShareType(SHARE::SHARETYPE_USER) + ->setType('file') + ->setOwner('usertest') + ->setSharedBy('initiator') + ->setSharedWith('usertest2') + ->setPermissions(1); + $groupShareModel1 = (new Share()) + ->setPath('/path/to/file') + ->setShareType(SHARE::SHARETYPE_GROUP) + ->setType('file') + ->setOwner('usertest') + ->setSharedBy('initiator') + ->setSharedWith('group') + ->setPermissions(31); + $linksShareModel1 = (new Share()) + ->setPath('/path/to/file') + ->setShareType(SHARE::SHARETYPE_LINK) + ->setType('file') + ->setOwner('usertest') + ->setSharedBy('initiator') + ->setPermissions(31) + ->setToken('#token') + ->setName('my link name') + ->setExpirationDate(1556150400) + ->setPassword('password'); + $remoteShareModel1 = (new Share()) + ->setPath('/path/to/file') + ->setShareType(SHARE::SHARETYPE_REMOTE) + ->setType('file') + ->setOwner('usertest') + ->setSharedBy('initiator') + ->setSharedWith('user@remote') + ->setPermissions(1); + $this->streamHelper + ->expects($this->exactly(5)) + ->method('writelnToStream') + ->withConsecutive( + [ $this->resource, $userShareModel1], + [ $this->resource, $userShareModel2], + [ $this->resource, $groupShareModel1], + [ $this->resource, $linksShareModel1], + [ $this->resource, $remoteShareModel1] + ); + $this->streamHelper + ->expects($this->once()) + ->method('initStream') + ->willReturn($this->resource); + $this->streamHelper + ->expects($this->once()) + ->method('closeStream') + ->with($this->resource); + + $this->sharesExtractor->extract($user, '/usertest'); } public function testExtractNoData() { $user = 'usertest'; $this->manager->method('getSharesBy') - ->will($this->returnValueMap([ - [$user, ShareConstants::SHARE_TYPE_USER, null, true, 50, 0, []], - [$user, ShareConstants::SHARE_TYPE_GROUP, null, true, 50, 0, []], - [$user, ShareConstants::SHARE_TYPE_LINK, null, true, 50, 0, []], - [$user, ShareConstants::SHARE_TYPE_REMOTE, null, true, 50, 0, []], - ])); - - $realModels = $this->sharesExtractor->extract($user); - - $this->assertEmpty($realModels); + ->will( + $this->returnValueMap( + [ + [$user, ShareConstants::SHARE_TYPE_USER, null, true, 50, 0, []], + [$user, ShareConstants::SHARE_TYPE_GROUP, null, true, 50, 0, []], + [$user, ShareConstants::SHARE_TYPE_LINK, null, true, 50, 0, []], + [$user, ShareConstants::SHARE_TYPE_REMOTE, null, true, 50, 0, []], + ] + ) + ); + + $this->streamHelper + ->expects($this->once()) + ->method('initStream') + ->willReturn($this->resource); + $this->streamHelper + ->expects($this->once()) + ->method('closeStream') + ->with($this->resource); + + $this->sharesExtractor->extract($user, '/usertest'); } public function testExtractLinksWithExpirationAndPassword() { @@ -230,61 +276,65 @@ public function testExtractLinksWithExpirationAndPassword() { return $node; })); - $realModels = $this->sharesExtractor->extract($user); - - $expectedShareModels = [ - (new Share()) - ->setPath('/path/to/file') - ->setShareType(SHARE::SHARETYPE_LINK) - ->setOwner('usertest') - ->setSharedBy('initiator') - ->setPermissions(31) - ->setToken('#token-1') - ->setName('my link name'), - (new Share()) - ->setPath('/path/to/file') - ->setShareType(SHARE::SHARETYPE_LINK) - ->setOwner('usertest') - ->setSharedBy('initiator') - ->setPermissions(31) - ->setToken('#token-2') - ->setName('my link name') - ->setExpirationDate(12345678), - (new Share()) - ->setPath('/path/to/file') - ->setShareType(SHARE::SHARETYPE_LINK) - ->setOwner('usertest') - ->setSharedBy('initiator') - ->setPermissions(31) - ->setToken('#token-3') - ->setName('my link name') - ->setPassword('hashed#Password'), - (new Share()) - ->setPath('/path/to/file') - ->setShareType(SHARE::SHARETYPE_LINK) - ->setOwner('usertest') - ->setSharedBy('initiator') - ->setPermissions(31) - ->setToken('#token-4') - ->setName('my link name') - ->setExpirationDate(12345678) - ->setPassword('hashed#Password'), - ]; - - $this->assertEquals(\count($expectedShareModels), \count($realModels)); - - foreach ($expectedShareModels as $key => $expectedShareModel) { - $this->assertEquals($expectedShareModel->getPath(), $realModels[$key]->getPath()); - $this->assertEquals($expectedShareModel->getShareType(), $realModels[$key]->getShareType()); - $this->assertEquals($expectedShareModel->getOwner(), $realModels[$key]->getOwner()); - $this->assertEquals($expectedShareModel->getSharedBy(), $realModels[$key]->getSharedBy()); - $this->assertEquals($expectedShareModel->getSharedWith(), $realModels[$key]->getSharedWith()); - $this->assertEquals($expectedShareModel->getPermissions(), $realModels[$key]->getPermissions()); - $this->assertEquals($expectedShareModel->getToken(), $realModels[$key]->getToken()); - $this->assertEquals($expectedShareModel->getExpirationDate(), $realModels[$key]->getExpirationDate()); - $this->assertEquals($expectedShareModel->getPassword(), $realModels[$key]->getPassword()); - $this->assertEquals($expectedShareModel->getName(), $realModels[$key]->getName()); - } + $linkShareModel1 = (new Share()) + ->setPath('/path/to/file') + ->setShareType(SHARE::SHARETYPE_LINK) + ->setType('file') + ->setOwner('usertest') + ->setSharedBy('initiator') + ->setPermissions(31) + ->setToken('#token-1') + ->setName('my link name'); + $linkShareModel2 = (new Share()) + ->setPath('/path/to/file') + ->setShareType(SHARE::SHARETYPE_LINK) + ->setType('file') + ->setOwner('usertest') + ->setSharedBy('initiator') + ->setPermissions(31) + ->setToken('#token-2') + ->setName('my link name') + ->setExpirationDate(12345678); + $linkShareModel3 = (new Share()) + ->setPath('/path/to/file') + ->setShareType(SHARE::SHARETYPE_LINK) + ->setType('file') + ->setOwner('usertest') + ->setSharedBy('initiator') + ->setPermissions(31) + ->setToken('#token-3') + ->setName('my link name') + ->setPassword('hashed#Password'); + $linkShareModel4 = (new Share()) + ->setPath('/path/to/file') + ->setShareType(SHARE::SHARETYPE_LINK) + ->setType('file') + ->setOwner('usertest') + ->setSharedBy('initiator') + ->setPermissions(31) + ->setToken('#token-4') + ->setName('my link name') + ->setExpirationDate(12345678) + ->setPassword('hashed#Password'); + $this->streamHelper + ->expects($this->exactly(4)) + ->method('writelnToStream') + ->withConsecutive( + [ $this->resource, $linkShareModel1], + [ $this->resource, $linkShareModel2], + [ $this->resource, $linkShareModel3], + [ $this->resource, $linkShareModel4] + ); + $this->streamHelper + ->expects($this->once()) + ->method('initStream') + ->willReturn($this->resource); + $this->streamHelper + ->expects($this->once()) + ->method('closeStream') + ->with($this->resource); + + $this->sharesExtractor->extract($user, '/usertest'); } public function testExtractOC9() { @@ -294,7 +344,12 @@ public function testExtractOC9() { ->with('version', '') ->willReturn('9.0.1'); - $this->sharesExtractor = new SharesExtractor($this->manager, $this->rootFolder, $this->config); + $this->sharesExtractor = new SharesExtractor( + $this->manager, + $this->rootFolder, + $this->config, + $this->streamHelper + ); $user = 'usertest'; $expiration = new \DateTime(); @@ -327,62 +382,68 @@ public function testExtractOC9() { return $node; })); - $realModels = $this->sharesExtractor->extract($user); - - $expectedShareModels = [ - (new Share()) - ->setPath('/path/to/file') - ->setShareType(SHARE::SHARETYPE_USER) - ->setOwner('usertest') - ->setSharedBy('usertest') - ->setSharedWith('usertest2') - ->setPermissions(1), - (new Share()) - ->setPath('/path/to/file') - ->setShareType(SHARE::SHARETYPE_USER) - ->setOwner('usertest') - ->setSharedBy('initiator') - ->setSharedWith('usertest2') - ->setPermissions(1), - (new Share()) - ->setPath('/path/to/file') - ->setShareType(SHARE::SHARETYPE_GROUP) - ->setOwner('usertest') - ->setSharedBy('initiator') - ->setSharedWith('group') - ->setPermissions(31), - (new Share()) - ->setPath('/path/to/file') - ->setShareType(SHARE::SHARETYPE_LINK) - ->setOwner('usertest') - ->setSharedBy('initiator') - ->setPermissions(31) - ->setToken('#token') - ->setName('') - ->setExpirationDate(1556150400) - ->setPassword('password'), - (new Share()) - ->setPath('/path/to/file') - ->setShareType(SHARE::SHARETYPE_REMOTE) - ->setOwner('usertest') - ->setSharedBy('initiator') - ->setSharedWith('user@remote') - ->setPermissions(1), - ]; - - $this->assertEquals(\count($expectedShareModels), \count($realModels)); - - foreach ($expectedShareModels as $key => $expectedShareModel) { - $this->assertEquals($expectedShareModel->getPath(), $realModels[$key]->getPath()); - $this->assertEquals($expectedShareModel->getShareType(), $realModels[$key]->getShareType()); - $this->assertEquals($expectedShareModel->getOwner(), $realModels[$key]->getOwner()); - $this->assertEquals($expectedShareModel->getSharedBy(), $realModels[$key]->getSharedBy()); - $this->assertEquals($expectedShareModel->getSharedWith(), $realModels[$key]->getSharedWith()); - $this->assertEquals($expectedShareModel->getPermissions(), $realModels[$key]->getPermissions()); - $this->assertEquals($expectedShareModel->getToken(), $realModels[$key]->getToken()); - $this->assertEquals($expectedShareModel->getExpirationDate(), $realModels[$key]->getExpirationDate()); - $this->assertEquals($expectedShareModel->getPassword(), $realModels[$key]->getPassword()); - $this->assertEquals($expectedShareModel->getName(), $realModels[$key]->getName()); - } + $userShareModel1 = (new Share()) + ->setPath('/path/to/file') + ->setShareType(SHARE::SHARETYPE_USER) + ->setType('file') + ->setOwner('usertest') + ->setSharedBy('usertest') + ->setSharedWith('usertest2') + ->setPermissions(1); + $userShareModel2 = (new Share()) + ->setPath('/path/to/file') + ->setShareType(SHARE::SHARETYPE_USER) + ->setType('file') + ->setOwner('usertest') + ->setSharedBy('initiator') + ->setSharedWith('usertest2') + ->setPermissions(1); + $groupShareModel1 = (new Share()) + ->setPath('/path/to/file') + ->setShareType(SHARE::SHARETYPE_GROUP) + ->setType('file') + ->setOwner('usertest') + ->setSharedBy('initiator') + ->setSharedWith('group') + ->setPermissions(31); + $linksShareModel1 = (new Share()) + ->setPath('/path/to/file') + ->setShareType(SHARE::SHARETYPE_LINK) + ->setType('file') + ->setOwner('usertest') + ->setSharedBy('initiator') + ->setPermissions(31) + ->setToken('#token') + ->setName('') + ->setExpirationDate(1556150400) + ->setPassword('password'); + $remoteShareModel1 = (new Share()) + ->setPath('/path/to/file') + ->setShareType(SHARE::SHARETYPE_REMOTE) + ->setType('file') + ->setOwner('usertest') + ->setSharedBy('initiator') + ->setSharedWith('user@remote') + ->setPermissions(1); + $this->streamHelper + ->expects($this->exactly(5)) + ->method('writelnToStream') + ->withConsecutive( + [ $this->resource, $userShareModel1], + [ $this->resource, $userShareModel2], + [ $this->resource, $groupShareModel1], + [ $this->resource, $linksShareModel1], + [ $this->resource, $remoteShareModel1] + ); + $this->streamHelper + ->expects($this->once()) + ->method('initStream') + ->willReturn($this->resource); + $this->streamHelper + ->expects($this->once()) + ->method('closeStream') + ->with($this->resource); + + $this->sharesExtractor->extract($user, '/usertest'); } } diff --git a/tests/unit/Extractor/MetadataExtractorTest.php b/tests/unit/Extractor/MetadataExtractorTest.php index 351d84d..dc5dd1a 100644 --- a/tests/unit/Extractor/MetadataExtractorTest.php +++ b/tests/unit/Extractor/MetadataExtractorTest.php @@ -31,7 +31,7 @@ use OCA\DataExporter\Extractor\MetadataExtractor\UserExtractor; use OCA\DataExporter\Model\User; use OCA\DataExporter\Model\User\Preference; -use OCA\DataExporter\Model\User\Share; +use OCA\DataExporter\Model\Share; use OCP\IURLGenerator; use Test\TestCase; @@ -89,32 +89,11 @@ public function testMetadataExtractor() { ->method('extract') ->with('testuser') ->willReturn([$preference]); - $share = new Share(); - $share - ->setShareType(Share::SHARETYPE_LINK) - ->setPath('path/to/file') - ->setName('test') - ->setOwner('testuser') - ->setSharedBy('initiator') - ->setPermissions(31) - ->setToken('token') - ->setPassword('password') - ->setExpirationDate(1556150400); - $this->sharesExtractor - ->method('extract') - ->with('testuser') - ->willReturn([$share]); - $this->filesMetadataExtractor - ->method('extract') - ->with('testuser') - ->willReturn([]); - $metadata = $this->metadataExtractor->extract('testuser'); + $metadata = $this->metadataExtractor->extract('testuser', '/testuser'); self::assertEquals($user->isEnabled(), $metadata->getUser()->isEnabled()); self::assertEquals($user->getUserId(), $metadata->getUser()->getUserId()); self::assertEquals($user->getEmail(), $metadata->getUser()->getEmail()); self::assertEquals($preference, $metadata->getUser()->getPreferences()[0]); - self::assertEquals($share, $metadata->getUser()->getShares()[0]); - self::assertEquals([], $metadata->getFiles()); } } diff --git a/tests/unit/Importer/FilesImporterTest.php b/tests/unit/Importer/FilesImporterTest.php index 0a01e71..44d3c82 100644 --- a/tests/unit/Importer/FilesImporterTest.php +++ b/tests/unit/Importer/FilesImporterTest.php @@ -29,6 +29,7 @@ use OC\Files\Storage\Storage; use OCA\DataExporter\Importer\FilesImporter; use OCA\DataExporter\Model\File; +use OCA\DataExporter\Utilities\StreamHelper; use OCP\Files\IRootFolder; use OCP\Files\Node; use OC\Files\Node\File as FileNode; @@ -54,6 +55,10 @@ class FilesImporterTest extends TestCase { * @var IRootFolder | \PHPUnit_Framework_MockObject_MockObject */ private $rootFolder; + /** + * @var StreamHelper | \PHPUnit_Framework_MockObject_MockObject + */ + private $streamHelper; /** * Set up the Tests @@ -64,6 +69,7 @@ public function setUp() { parent::setUp(); $this->filesystem = $this->createMock(Filesystem::class); $this->rootFolder = $this->createMock(IRootFolder::class); + $this->streamHelper = $this->createMock(StreamHelper::class); } /** @@ -129,7 +135,7 @@ public function testFilesImporter() { $mockFile1->method('putContent')->willReturn(true); vfsStreamWrapper::register(); $root = vfsStreamWrapper::setRoot(new vfsStreamDirectory('tmp')); - $vfile = new vfsStreamFile('testuser/files/AFolder/afile.txt'); + $vfile = new vfsStreamFile('testuser/files/files/AFolder/afile.txt'); $vfile->setContent('test'); $root->addChild($vfile); @@ -145,13 +151,109 @@ public function testFilesImporter() { ->method('exists') ->willReturnMap( [ - ['vfs://tmp/testuser/files/AFolder/afile.txt', true], - ['vfs://tmp/testuser/files/AFolder', true] + ['vfs://tmp/testuser/files/files/AFolder/afile.txt', true], + ['vfs://tmp/testuser/files/files/AFolder', true] ] ); $filesImporter = new FilesImporter( $this->filesystem, - $this->rootFolder + $this->rootFolder, + $this->streamHelper + ); + $fileMetadata = new File(); + $fileMetadata + ->setPath('files/AFolder/afile.txt') + ->setType(File::TYPE_FILE) + ->setETag('533c8d4b4c45b62e68cc09e810db7a23') + ->setPermissions(27); + $folderMetadata = new File(); + $folderMetadata + ->setPath('files/AFolder') + ->setType(File::TYPE_FOLDER) + ->setETag('5bc8867cc2375') + ->setPermissions(31); + + $this->streamHelper->expects($this->exactly(3)) + ->method('readlnFromStream') + ->willReturnOnConsecutiveCalls($folderMetadata, $fileMetadata, false); + + $filesImporter->import( + 'testuser', + 'vfs://tmp/testuser' + ); + vfsStreamWrapper::unregister(); + } + + /** + * @expectedException \OCA\DataExporter\Importer\ImportException + * @expectedExceptionMessage Import failed on vfs://tmp/testuser/files.jsonl on line 2: File 'vfs://tmp/testuser/files/files/AFolder + */ + public function testFilesImporterError() { + // User Folder + $mockFolder1 = $this->createMock(Folder::class); + $mockFolder1->method('getPath')->willReturn('/testuser/files'); + $mockFolder1->method('getEtag')->willReturn('123qweasdzxc'); + $mockFolder1->method('getPermissions')->willReturn(31); + $mockFolder1->method('getType')->willReturn(Node::TYPE_FOLDER); + + // User Folder Parent + $mockFolder2 = $this->createMock(Folder::class); + $mockFolder2->method('getPath')->willReturn('/testuser'); + $mockFolder2->method('getEtag')->willReturn('123qweasdzxc'); + $mockFolder2->method('getPermissions')->willReturn(31); + $mockFolder2->method('getType')->willReturn(Node::TYPE_FOLDER); + + // Test Folder + $mockFolder3 = $this->createMock(Folder::class); + $mockFolder3->method('getPath')->willReturn('files/AFolder'); + $mockFolder3->method('getId')->willReturn(1); + $mockFolder3->method('getEtag')->willReturn('5bc8867cc2375'); + $mockFolder3->method('getPermissions')->willReturn(31); + $mockFolder3->method('getType')->willReturn(Node::TYPE_FOLDER); + $storage3 = $this->createMock(Storage::class); + $cache = $this->createMock(Cache::class); + // Test that the File Cache is updated for this folder + $cache->method('update') + ->with(1, ['etag' => '5bc8867cc2375', 'permissions' => 31]); + $storage3->method('getCache')->willReturn($cache); + $mockFolder3->method('getStorage')->willReturn($storage3); + + // Test File + $mockFile1 = $this->createMock(FileNode::class); + $mockFile1->method('getPath')->willReturn('files/AFolder/afile.txt'); + $mockFile1->method('getId')->willReturn(2); + $mockFile1->method('getEtag')->willReturn('533c8d4b4c45b62e68cc09e810db7a23'); + $mockFile1->method('getPermissions')->willReturn(27); + $mockFile1->method('getType')->willReturn(Node::TYPE_FILE); + $storageFile = $this->createMock(Storage::class); + $cacheFile = $this->createMock(Cache::class); + // Test that the File Cache is updated for this file + $cacheFile->method('update') + ->with(2, ['etag' => '533c8d4b4c45b62e68cc09e810db7a23', 'permissions' => 27]); + $storageFile->method('getCache')->willReturn($cacheFile); + $mockFile1->method('getStorage')->willReturn($storageFile); + $mockFile1->method('putContent')->willReturn(true); + vfsStreamWrapper::register(); + $root = vfsStreamWrapper::setRoot(new vfsStreamDirectory('tmp')); + $vfile = new vfsStreamFile('testuser/files/files/AFolder/afile.txt'); + $vfile->setContent('test'); + $root->addChild($vfile); + + $mockFolder2->method('newFolder') + ->willReturn($mockFolder3); + $mockFolder2->method('newFile') + ->willReturn($mockFile1); + + $mockFolder1->method('getParent')->willReturn($mockFolder2); + $this->rootFolder->method('getUserFolder')->willReturn($mockFolder1); + + $this->filesystem + ->method('exists') + ->willReturn(false); + $filesImporter = new FilesImporter( + $this->filesystem, + $this->rootFolder, + $this->streamHelper ); $fileMetadata = new File(); $fileMetadata @@ -165,11 +267,13 @@ public function testFilesImporter() { ->setType(File::TYPE_FOLDER) ->setETag('5bc8867cc2375') ->setPermissions(31); - $filesMetadata = [$folderMetadata, $fileMetadata]; + + $this->streamHelper->expects($this->exactly(1)) + ->method('readlnFromStream') + ->willReturnOnConsecutiveCalls($folderMetadata, $fileMetadata, false); $filesImporter->import( 'testuser', - $filesMetadata, 'vfs://tmp/testuser' ); vfsStreamWrapper::unregister(); diff --git a/tests/unit/Importer/MetadataImporter/ShareImporterTest.php b/tests/unit/Importer/MetadataImporter/ShareImporterTest.php index 441e549..555083e 100644 --- a/tests/unit/Importer/MetadataImporter/ShareImporterTest.php +++ b/tests/unit/Importer/MetadataImporter/ShareImporterTest.php @@ -24,7 +24,8 @@ use OC\Share20\Share as PrivateShare; use OCA\DataExporter\Importer\MetadataImporter\ShareImporter; -use OCA\DataExporter\Model\User\Share; +use OCA\DataExporter\Model\Share; +use OCA\DataExporter\Utilities\StreamHelper; use OCP\Share\IManager; use OCP\Share\IShare; use OCP\Files\IRootFolder; @@ -55,6 +56,8 @@ class ShareImporterTest extends TestCase { private $shareImporter; /** @var ILogger | \PHPUnit\Framework\MockObject\MockObject */ private $logger; + /** @var StreamHelper | \PHPUnit\Framework\MockObject\MockObject*/ + private $streamHelper; protected function setUp() { $this->shareManager = $this->createMock(IManager::class); @@ -64,6 +67,7 @@ protected function setUp() { $this->shareConverter = $this->createMock(ShareConverter::class); $this->urlGenerator = $this->createMock(IURLGenerator::class); $this->logger = $this->createMock(ILogger::class); + $this->streamHelper = $this->createMock(StreamHelper::class); $this->shareImporter = new ShareImporter( $this->shareManager, @@ -72,7 +76,8 @@ protected function setUp() { $this->groupManager, $this->shareConverter, $this->urlGenerator, - $this->logger + $this->logger, + $this->streamHelper ); } @@ -167,7 +172,11 @@ public function testImportLocalUserShare() { return $this->isSameShare($share, $shareData1) || $this->isSameShare($share, $shareData2); })); - $this->shareImporter->import('usertest', [$shareModel], 'https://random.host/oc'); + $this->streamHelper + ->expects($this->exactly(2)) + ->method('readlnFromStream') + ->willReturnOnConsecutiveCalls($shareModel, false); + $this->shareImporter->import('usertest', '/tmp/usertest', 'https://random.host/oc'); } /** @@ -205,7 +214,11 @@ public function testImportRemoteUserShareFromLocalShare() { return $this->isSameShare($share, $shareData); })); - $this->shareImporter->import('usertest', [$shareModel], 'https://random.host/oc'); + $this->streamHelper + ->expects($this->exactly(2)) + ->method('readlnFromStream') + ->willReturnOnConsecutiveCalls($shareModel, false); + $this->shareImporter->import('usertest', '/tmp/testuser', 'https://random.host/oc'); } /** @@ -234,7 +247,11 @@ public function testImportLocalUserShareSharedWithTargetsUser() { $this->shareManager->expects($this->never()) ->method('createShare'); - $this->shareImporter->import('usertest', [$shareModel], 'https://random.host/oc'); + $this->streamHelper + ->expects($this->exactly(2)) + ->method('readlnFromStream') + ->willReturnOnConsecutiveCalls($shareModel, false); + $this->shareImporter->import('usertest', '/tmp/testuser', 'https://random.host/oc'); } /** @@ -261,7 +278,11 @@ public function testImportLocalUserShareWeirdData() { $this->shareManager->expects($this->never()) ->method('createShare'); - $this->shareImporter->import('usertest', [$shareModel], 'https://random.host/oc'); + $this->streamHelper + ->expects($this->exactly(2)) + ->method('readlnFromStream') + ->willReturnOnConsecutiveCalls($shareModel, false); + $this->shareImporter->import('usertest', '/tmp/testuser', 'https://random.host/oc'); } /** @@ -298,7 +319,11 @@ public function testImportGroupShare() { return $this->isSameShare($share, $shareData); })); - $this->shareImporter->import('usertest', [$shareModel], 'https://random.host/oc'); + $this->streamHelper + ->expects($this->exactly(2)) + ->method('readlnFromStream') + ->willReturnOnConsecutiveCalls($shareModel, false); + $this->shareImporter->import('usertest', '/tmp/usertest', 'https://random.host/oc'); } /** @@ -324,7 +349,11 @@ public function testImportGroupShareWrongImportingUser() { $this->shareManager->expects($this->never()) ->method('createShare'); - $this->shareImporter->import('usertest', [$shareModel], 'https://random.host/oc'); + $this->streamHelper + ->expects($this->exactly(2)) + ->method('readlnFromStream') + ->willReturnOnConsecutiveCalls($shareModel, false); + $this->shareImporter->import('usertest', '/tmp/testuser', 'https://random.host/oc'); } /** @@ -350,7 +379,11 @@ public function testImportGroupShareMissingGroup() { $this->shareManager->expects($this->never()) ->method('createShare'); - $this->shareImporter->import('usertest', [$shareModel], 'https://random.host/oc'); + $this->streamHelper + ->expects($this->exactly(2)) + ->method('readlnFromStream') + ->willReturnOnConsecutiveCalls($shareModel, false); + $this->shareImporter->import('usertest', '/tmp/testuser', 'https://random.host/oc'); } /** @@ -407,7 +440,11 @@ public function testImportLinkShare() { return $this->isSameLinkShare($share, $shareData); })); - $this->shareImporter->import('usertest', [$shareModel], 'https://random.host/oc'); + $this->streamHelper + ->expects($this->exactly(2)) + ->method('readlnFromStream') + ->willReturnOnConsecutiveCalls($shareModel, false); + $this->shareImporter->import('usertest', '/tmp/usertest', 'https://random.host/oc'); } public function testImportLinkShareWithPassword() { @@ -462,7 +499,11 @@ public function testImportLinkShareWithPassword() { return $this->isSameLinkShare($share, $shareData); })); - $this->shareImporter->import('usertest', [$shareModel], 'https://random.host/oc'); + $this->streamHelper + ->expects($this->exactly(2)) + ->method('readlnFromStream') + ->willReturnOnConsecutiveCalls($shareModel, false); + $this->shareImporter->import('usertest', '/tmp/usertest', 'https://random.host/oc'); } public function testImportLinkShareWithExpiration() { @@ -519,7 +560,11 @@ public function testImportLinkShareWithExpiration() { return $this->isSameLinkShare($share, $shareData); })); - $this->shareImporter->import('usertest', [$shareModel], 'https://random.host/oc'); + $this->streamHelper + ->expects($this->exactly(2)) + ->method('readlnFromStream') + ->willReturnOnConsecutiveCalls($shareModel, false); + $this->shareImporter->import('usertest', '/tmp/usertest', 'https://random.host/oc'); } public function testImportLinkShareWithPasswordAndExpiration() { @@ -577,7 +622,11 @@ public function testImportLinkShareWithPasswordAndExpiration() { return $this->isSameLinkShare($share, $shareData); })); - $this->shareImporter->import('usertest', [$shareModel], 'https://random.host/oc'); + $this->streamHelper + ->expects($this->exactly(2)) + ->method('readlnFromStream') + ->willReturnOnConsecutiveCalls($shareModel, false); + $this->shareImporter->import('usertest', '/tmp/usertest', 'https://random.host/oc'); } /** @@ -601,7 +650,11 @@ public function testImportLinkShareWrongImportingUser() { $this->shareManager->expects($this->never()) ->method('createShare'); - $this->shareImporter->import('usertest', [$shareModel], 'https://random.host/oc'); + $this->streamHelper + ->expects($this->exactly(2)) + ->method('readlnFromStream') + ->willReturnOnConsecutiveCalls($shareModel, false); + $this->shareImporter->import('usertest', '/tmp/usertest', 'https://random.host/oc'); } /** @@ -638,7 +691,11 @@ public function testImportRemoteShare() { return $this->isSameShare($share, $shareData); })); - $this->shareImporter->import('usertest', [$shareModel], 'https://random.host/oc'); + $this->streamHelper + ->expects($this->exactly(2)) + ->method('readlnFromStream') + ->willReturnOnConsecutiveCalls($shareModel, false); + $this->shareImporter->import('usertest', '/tmp/usertest', 'https://random.host/oc'); } /** @@ -675,7 +732,11 @@ public function testImportRemoteShareTargetingSource() { return $this->isSameShare($share, $shareData); })); - $this->shareImporter->import('usertest', [$shareModel], 'https://random.host/oc'); + $this->streamHelper + ->expects($this->exactly(2)) + ->method('readlnFromStream') + ->willReturnOnConsecutiveCalls($shareModel, false); + $this->shareImporter->import('usertest', '/tmp/usertest', 'https://random.host/oc'); } /** @@ -712,7 +773,11 @@ public function testImportRemoteShareTargetingCurrent() { return $this->isSameShare($share, $shareData); })); - $this->shareImporter->import('usertest', [$shareModel], 'https://random.host/oc'); + $this->streamHelper + ->expects($this->exactly(2)) + ->method('readlnFromStream') + ->willReturnOnConsecutiveCalls($shareModel, false); + $this->shareImporter->import('usertest', '/tmp/usertest', 'https://random.host/oc'); } /** @@ -738,6 +803,10 @@ public function testImportRemoteShareWrongImportingUser() { $this->shareManager->expects($this->never()) ->method('createShare'); - $this->shareImporter->import('usertest2', [$shareModel], 'https://random.host/oc'); + $this->streamHelper + ->expects($this->exactly(2)) + ->method('readlnFromStream') + ->willReturnOnConsecutiveCalls($shareModel, false); + $this->shareImporter->import('usertest2', '/tmp/usertest', 'https://random.host/oc'); } } diff --git a/tests/unit/Utilities/StreamHelperTest.php b/tests/unit/Utilities/StreamHelperTest.php new file mode 100644 index 0000000..11b88d4 --- /dev/null +++ b/tests/unit/Utilities/StreamHelperTest.php @@ -0,0 +1,166 @@ + + * + * @copyright Copyright (c) 2018, ownCloud GmbH + * @license GPL-2.0 + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + */ +namespace OCA\DataExporter\Tests\Unit\Utilities; + +use OCA\DataExporter\Importer\ImportException; +use OCA\DataExporter\Model\File; +use OCA\DataExporter\Serializer; +use OCA\DataExporter\Utilities\StreamHelper; +use org\bovigo\vfs\vfsStream; +use Symfony\Component\Filesystem\Filesystem; +use Test\TestCase; + +class StreamHelperTest extends TestCase { + /** + * @var Filesystem + */ + private $filesystem; + /** + * @var Serializer + */ + private $serializer; + /** + * @var StreamHelper + */ + private $streamHelper; + + /** + * Setup Tests + * + * @return void + */ + public function setUp() { + parent::setUp(); + $this->filesystem = new Filesystem(); + $this->serializer = new Serializer(); + $this->streamHelper = new StreamHelper( + $this->filesystem, + $this->serializer + ); + } + + public function testWrite() { + $directory = [ + 'testfolder' => [ + 'files.jsonl' => '' + ] + ]; + $virtualFilesystem = vfsStream::setup('testroot', '644', $directory); + $fileModel = new File(); + $fileModel->setPath('files/welcome.txt'); + $fileModel->setType('file'); + $fileModel->setPermissions(31); + $fileModel->setETag('234s34ser234'); + $fileModel->setMtime(1565267598); + + $fileModel2 = new File(); + $fileModel2->setPath('files/someFolder'); + $fileModel2->setType('folder'); + $fileModel2->setPermissions(31); + $fileModel2->setETag('234s34ser234'); + $fileModel2->setMtime(1565267598); + $resource = $this->streamHelper->initStream($virtualFilesystem->url() . '/testfolder/files.jsonl', 'a'); + $this->streamHelper->writelnToStream($resource, $fileModel); + $this->streamHelper->writelnToStream($resource, $fileModel2); + $this->streamHelper->closeStream($resource); + $writtenData = \file_get_contents($virtualFilesystem->url() . '/testfolder/files.jsonl'); + $expectedData = <<< JSONL +{"type":"file","path":"files\/welcome.txt","eTag":"234s34ser234","permissions":31,"mtime":1565267598} +{"type":"folder","path":"files\/someFolder","eTag":"234s34ser234","permissions":31,"mtime":1565267598} + +JSONL; + $this->assertEquals($expectedData, $writtenData); + } + + public function testRead() { + $fileContent = <<< JSONL +{"type":"file","path":"files\/welcome.txt","eTag":"234s34ser234","permissions":31,"mtime":1565267598} +{"type":"folder","path":"files\/someFolder","eTag":"234s34ser234","permissions":31,"mtime":1565267598} + +JSONL; + $directory = [ + 'testfolder' => [ + 'files.jsonl' => $fileContent + ] + ]; + $expectedFileModel1 = new File(); + $expectedFileModel1->setPath('files/welcome.txt'); + $expectedFileModel1->setType('file'); + $expectedFileModel1->setPermissions(31); + $expectedFileModel1->setETag('234s34ser234'); + $expectedFileModel1->setMtime(1565267598); + + $expectedFileModel2 = new File(); + $expectedFileModel2->setPath('files/someFolder'); + $expectedFileModel2->setType('folder'); + $expectedFileModel2->setPermissions(31); + $expectedFileModel2->setETag('234s34ser234'); + $expectedFileModel2->setMtime(1565267598); + + $virtualFilesystem = vfsStream::setup('testroot', '644', $directory); + $resource = $this->streamHelper->initStream($virtualFilesystem->url(). '/testfolder/files.jsonl', 'r'); + + $fileModel1 = $this->streamHelper->readlnFromStream($resource, File::class); + $fileModel2 = $this->streamHelper->readlnFromStream($resource, File::class); + + $this->assertEquals($expectedFileModel1, $fileModel1); + $this->assertEquals($expectedFileModel2, $fileModel2); + $this->assertFalse($this->streamHelper->readlnFromStream($resource, File::class)); + $this->streamHelper->closeStream($resource); + } + + /** + * @expectedException \OCA\DataExporter\Importer\ImportException + * + * @return void + */ + public function testReadCorruptedFile() { + $fileContent = <<< JSONL +{"type":"file","path":"files\/welcome.txt","eTag":"234s34ser234","permissions":31,"mtime":1565267598} +{"type":"folder","path:"files\/someFolder","eTag":"234s34ser234","permissions":31,"mtime":1565267598} +{"type":"folder","path":"files\/someFolder","eTag":"234s34ser234","permissions":31,"mtime":1565267598} + +JSONL; + $directory = [ + 'testfolder' => [ + 'files.jsonl' => $fileContent + ] + ]; + $expectedFileModel1 = new File(); + $expectedFileModel1->setPath('files/welcome.txt'); + $expectedFileModel1->setType('file'); + $expectedFileModel1->setPermissions(31); + $expectedFileModel1->setETag('234s34ser234'); + $expectedFileModel1->setMtime(1565267598); + + $virtualFilesystem = vfsStream::setup('testroot', '644', $directory); + $resource = $this->streamHelper->initStream( + $virtualFilesystem->url() . '/testfolder/files.jsonl', + 'r' + ); + + $fileModel1 = $this->streamHelper->readlnFromStream($resource, File::class); + $this->assertEquals($expectedFileModel1, $fileModel1); + $this->streamHelper->readlnFromStream($resource, File::class); + } +} From 7a0e0d15419ec05f6747a4f4d8c4e63b03a762e2 Mon Sep 17 00:00:00 2001 From: Ilja Neumann Date: Wed, 28 Aug 2019 14:33:05 +0200 Subject: [PATCH 2/4] Remove stable10 test jobs --- .drone.yml | 67 ++++++++---------------------------------------------- 1 file changed, 10 insertions(+), 57 deletions(-) diff --git a/.drone.yml b/.drone.yml index 7a2e2e4..a3b3536 100644 --- a/.drone.yml +++ b/.drone.yml @@ -20,7 +20,7 @@ pipeline: install-server: image: owncloudci/core pull: true - version: daily-stable10-qa + version: daily-master-qa db_type: ${DB_TYPE} db_name: ${DB_NAME} db_host: ${DB_HOST} @@ -32,7 +32,7 @@ pipeline: image: owncloudci/php:${PHP_VERSION} pull: true commands: - - git clone -b stable10 --depth=1 https://github.com/owncloud/core.git /var/www/owncloud/testrunner + - git clone -b master --depth=1 https://github.com/owncloud/core.git /var/www/owncloud/testrunner - cp -r /var/www/owncloud/apps/data_exporter /var/www/owncloud/testrunner/apps/ - cd /var/www/owncloud/testrunner/apps/data_exporter && make - cd /var/www/owncloud/testrunner @@ -258,25 +258,25 @@ matrix: include: # unit tests - PHP_VERSION: 7.0 - OC_VERSION: daily-stable10-qa + OC_VERSION: daily-master-qa DB_TYPE: sqlite REBUILT_CACHE: true FLUSH_CACHE: true TEST_SUITE: phpunit - PHP_VERSION: 7.1 - OC_VERSION: daily-stable10-qa + OC_VERSION: daily-master-qa DB_TYPE: sqlite COVERAGE: true TEST_SUITE: phpunit - PHP_VERSION: 7.2 - OC_VERSION: daily-stable10-qa + OC_VERSION: daily-master-qa DB_TYPE: sqlite TEST_SUITE: phpunit - PHP_VERSION: 7.1 - OC_VERSION: daily-stable10-qa + OC_VERSION: daily-master-qa DB_TYPE: pgsql DB_HOST: pgsql DB_NAME: oc_db @@ -285,7 +285,7 @@ matrix: TEST_SUITE: phpunit - PHP_VERSION: 7.1 - OC_VERSION: daily-stable10-qa + OC_VERSION: daily-master-qa DB_TYPE: oci DB_HOST: oci DB_NAME: XE @@ -293,7 +293,7 @@ matrix: TEST_SUITE: phpunit - PHP_VERSION: 7.1 - OC_VERSION: daily-stable10-qa + OC_VERSION: daily-master-qa DB_TYPE: mysql DB_HOST: mysqlmb4 DB_NAME: oc_db @@ -302,7 +302,7 @@ matrix: TEST_SUITE: phpunit - PHP_VERSION: 7.1 - OC_VERSION: daily-stable10-qa + OC_VERSION: daily-master-qa DB_TYPE: mysql DB_HOST: mysql DB_NAME: oc_db @@ -313,54 +313,7 @@ matrix: # acceptance tests - PHP_VERSION: 7.0 - OC_VERSION: daily-stable10-qa - TEST_SUITE: cli-acceptance - BEHAT_SUITE: cliDataExporter - DB_TYPE: mysql - DB_HOST: mysql - DB_NAME: oc_db - DB_USERNAME: admin - DB_PASSWORD: secret - NEED_SERVER: true - NEED_INSTALL_APP: true - - - PHP_VERSION: 7.1 - OC_VERSION: daily-stable10-qa - TEST_SUITE: cli-acceptance - BEHAT_SUITE: cliDataExporter - DB_TYPE: mysql - DB_HOST: mysql - DB_NAME: oc_db - DB_USERNAME: admin - DB_PASSWORD: secret - NEED_SERVER: true - NEED_INSTALL_APP: true - - - PHP_VERSION: 7.1 - OC_VERSION: daily-stable10-qa - TEST_SUITE: cli-acceptance - BEHAT_SUITE: cliDataExporter - DB_HOST: oci - DB_TYPE: oci - DB_NAME: XE - DB_USERNAME: autotest - NEED_SERVER: true - NEED_INSTALL_APP: true - - - PHP_VERSION: 7.1 - OC_VERSION: daily-stable10-qa - TEST_SUITE: cli-acceptance - BEHAT_SUITE: cliDataExporter - DB_TYPE: pgsql - DB_HOST: pgsql - DB_NAME: oc_db - DB_USERNAME: admin - DB_PASSWORD: secret - NEED_SERVER: true - NEED_INSTALL_APP: true - - - PHP_VERSION: 7.2 - OC_VERSION: daily-stable10-qa + OC_VERSION: daily-master-qa TEST_SUITE: cli-acceptance BEHAT_SUITE: cliDataExporter DB_TYPE: mysql From 75da2784c4565071ba7a35fa9876932b9d75b4a9 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Mon, 12 Aug 2019 12:20:05 +0545 Subject: [PATCH 3/4] do not have multiple when in steps --- .drone.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index a3b3536..dc6c70b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -219,7 +219,6 @@ services: - MYSQL_ROOT_PASSWORD=secret when: matrix: - DB_TYPE: mysql DB_HOST: mysql mysqlmb4: @@ -231,7 +230,6 @@ services: - MYSQL_ROOT_PASSWORD=secret when: matrix: - DB_TYPE: mysql DB_HOST: mysqlmb4 pgsql: From 7eee03a00de252e39629af93b113cd4d45e493e7 Mon Sep 17 00:00:00 2001 From: micbar Date: Tue, 27 Aug 2019 15:35:54 +0200 Subject: [PATCH 4/4] Modify drone pipeline --- .drone.yml | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/.drone.yml b/.drone.yml index dc6c70b..e689f4b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -83,7 +83,7 @@ pipeline: - make test-php-style when: matrix: - TEST_SUITE: phpunit + TEST_SUITE: owncloud-coding-standard lint-test: image: owncloudci/php:${PHP_VERSION} @@ -94,7 +94,7 @@ pipeline: - make test-php-lint when: matrix: - TEST_SUITE: phpunit + TEST_SUITE: lint-test phpstan: # phpstan requires php7.1 @@ -106,7 +106,7 @@ pipeline: - make test-php-phpstan when: matrix: - TEST_SUITE: phpunit + TEST_SUITE: php-stan phan: # phan requires a recent php-ast extension @@ -118,7 +118,7 @@ pipeline: - make test-php-phan when: matrix: - TEST_SUITE: phpunit + TEST_SUITE: phan phpunit-unit-tests: image: owncloudci/php:${PHP_VERSION} @@ -129,7 +129,6 @@ pipeline: matrix: TEST_SUITE: phpunit - phpunit-integration-tests: image: owncloudci/php:${PHP_VERSION} pull: true @@ -254,7 +253,26 @@ services: matrix: include: - # unit tests + # owncloud-coding-standard + - PHP_VERSION: 7.2 + TEST_SUITE: owncloud-coding-standard + + # owncloud-coding-standard + - PHP_VERSION: 7.2 + TEST_SUITE: lint-test + + # php-stan + - PHP_VERSION: 7.1 + TEST_SUITE: php-stan + NEED_SERVER: true + NEED_INSTALL_APP: true + + # phan + - PHP_VERSION: 7.2 + TEST_SUITE: phan + NEED_SERVER: true + + # unit tests - PHP_VERSION: 7.0 OC_VERSION: daily-master-qa DB_TYPE: sqlite