From a9c9e8e0c9df1de1511ba847772959aa18bd60ce Mon Sep 17 00:00:00 2001 From: Oliver Kurth Date: Mon, 19 Apr 2021 16:30:58 -0700 Subject: [PATCH 01/17] move special metalink handling out of TDNFDownloadFile() to caller --- client/prototypes.h | 4 +--- client/remoterepo.c | 14 ++------------ client/repo.c | 17 ++++++++--------- client/rpmtrans.c | 2 +- plugins/repogpgcheck/repogpgcheck.c | 4 +--- 5 files changed, 13 insertions(+), 28 deletions(-) diff --git a/client/prototypes.h b/client/prototypes.h index c36482a9..77a0c3a2 100644 --- a/client/prototypes.h +++ b/client/prototypes.h @@ -245,9 +245,7 @@ TDNFDownloadFile( const char *pszRepo, const char *pszFileUrl, const char *pszFile, - const char *pszProgressData, - int metalink, - TDNF_METALINK_FILE **ml_file + const char *pszProgressData ); uint32_t diff --git a/client/remoterepo.c b/client/remoterepo.c index 407af865..303cd9c4 100644 --- a/client/remoterepo.c +++ b/client/remoterepo.c @@ -707,9 +707,7 @@ TDNFDownloadFile( const char *pszRepo, const char *pszFileUrl, const char *pszFile, - const char *pszProgressData, - int is_metalink, - TDNF_METALINK_FILE **ml_file + const char *pszProgressData ) { uint32_t dwError = 0; @@ -838,12 +836,6 @@ TDNFDownloadFile( { dwError = rename(pszFileTmp, pszFile); BAIL_ON_TDNF_ERROR(dwError); - - if (is_metalink && ml_file) - { - dwError = TDNFParseAndGetURLFromMetalink(pTdnf, pszRepo, pszFile, ml_file); - BAIL_ON_TDNF_ERROR(dwError); - } } cleanup: TDNF_SAFE_FREE_MEMORY(pszUserPass); @@ -928,9 +920,7 @@ TDNFDownloadPackage( pszRepoName, pszPackageUrl, pszPackageFile, - pszPkgName, - 0, - NULL); + pszPkgName); BAIL_ON_TDNF_ERROR(dwError); pr_info("\n"); diff --git a/client/repo.c b/client/repo.c index 6ec72677..fbd7ba28 100644 --- a/client/repo.c +++ b/client/repo.c @@ -599,7 +599,7 @@ TDNFDownloadUsingMetalinkResources( BAIL_ON_TDNF_ERROR(dwError); } dwError = TDNFDownloadFile(pTdnf, pszRepo, urls->url, pszFile, - pszProgressData, 0, NULL); + pszProgressData); if (dwError) { urls = urls->next; @@ -836,8 +836,11 @@ TDNFGetRepoMD( TDNF_REPO_BASEURL_FILE_NAME); BAIL_ON_TDNF_ERROR(dwError); dwError = TDNFDownloadFile(pTdnf, pRepoData->pszId, pszRepoMetalink, - pszTmpRepoMetalinkFile, pRepoData->pszId, - metalink, &ml_file); + pszTmpRepoMetalinkFile, pRepoData->pszId); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFParseAndGetURLFromMetalink(pTdnf, + pRepoData->pszId, pszTmpRepoMetalinkFile, &ml_file); BAIL_ON_TDNF_ERROR(dwError); nReplaceRepoMD = 1; @@ -901,9 +904,7 @@ TDNFGetRepoMD( pRepoData->pszId, pszRepoMDUrl, pszTmpRepoMDFile, - pRepoData->pszId, - 0, - NULL); + pRepoData->pszId); BAIL_ON_TDNF_ERROR(dwError); nReplaceRepoMD = 1; if (pszCookie[0]) @@ -1051,9 +1052,7 @@ TDNFDownloadRepoMDPart( pszRepo, pszTempUrl, pszDestPath, - pszRepo, - 0, - NULL); + pszRepo); BAIL_ON_TDNF_ERROR(dwError); } diff --git a/client/rpmtrans.c b/client/rpmtrans.c index dccd9a7e..9ee0d999 100644 --- a/client/rpmtrans.c +++ b/client/rpmtrans.c @@ -576,7 +576,7 @@ TDNFFetchRemoteGPGKey( } dwError = TDNFDownloadFile(pTdnf, pszRepoName, pszUrlGPGKey, pszFilePath, - basename(pszFilePath), 0, NULL); + basename(pszFilePath)); BAIL_ON_TDNF_ERROR(dwError); *ppszKeyLocation = pszNormalPath; diff --git a/plugins/repogpgcheck/repogpgcheck.c b/plugins/repogpgcheck/repogpgcheck.c index 2e5ea3c6..eb9a9e13 100644 --- a/plugins/repogpgcheck/repogpgcheck.c +++ b/plugins/repogpgcheck/repogpgcheck.c @@ -201,9 +201,7 @@ TDNFVerifySignature( pcszRepoId, pszRepoMDSigUrl, pszRepoMDSigFile, - pcszRepoId, - 0, - NULL); + pcszRepoId); BAIL_ON_TDNF_ERROR(dwError); dwError = TDNFVerifyRepoMDSignature(pHandle, pcszRepoMDFile, pszRepoMDSigFile); From 6abe7df8297e5a047bc8c6bf3cca3a40fc078ae0 Mon Sep 17 00:00:00 2001 From: Oliver Kurth Date: Wed, 28 Apr 2021 13:41:18 -0700 Subject: [PATCH 02/17] add TDNFGetFileSize() --- client/prototypes.h | 6 ++++++ client/utils.c | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/client/prototypes.h b/client/prototypes.h index 77a0c3a2..07d03da7 100644 --- a/client/prototypes.h +++ b/client/prototypes.h @@ -902,6 +902,12 @@ TDNFIsFileOrSymlink( int* pnPathIsFile ); +uint32_t +TDNFGetFileSize( + const char* pszPath, + int *pnSize + ); + uint32_t TDNFIsDir( const char* pszPath, diff --git a/client/utils.c b/client/utils.c index 0b220d9a..c1d9993f 100644 --- a/client/utils.c +++ b/client/utils.c @@ -290,6 +290,46 @@ TDNFIsFileOrSymlink( goto cleanup; } +uint32_t +TDNFGetFileSize( + const char* pszPath, + int *pnSize + ) +{ + uint32_t dwError = 0; + struct stat stStat = {0}; + + if(!pnSize || IsNullOrEmptyString(pszPath)) + { + dwError = ERROR_TDNF_INVALID_PARAMETER; + BAIL_ON_TDNF_ERROR(dwError); + } + + if(stat(pszPath, &stStat)) + { + { + dwError = errno; + BAIL_ON_TDNF_SYSTEM_ERROR(dwError); + } + } + else + { + if (S_ISREG(stStat.st_mode)) + { + *pnSize = stStat.st_size; + } + } +cleanup: + return dwError; + +error: + if(pnSize) + { + *pnSize = 0; + } + goto cleanup; +} + uint32_t TDNFIsDir( const char* pszPath, From 52e5ade2f667834d853f01fdea19ba7d1cc9dbf6 Mon Sep 17 00:00:00 2001 From: Oliver Kurth Date: Fri, 9 Apr 2021 17:12:18 -0700 Subject: [PATCH 03/17] reposync: basics, metadata, metadata/download-path and option handling --- client/api.c | 138 +++++++++++++++++++++++++++ client/packageutils.c | 94 ++++++++++++++++++ client/prototypes.h | 22 +++++ client/repo.c | 152 ++++++++++++++++++++++++++++++ client/structs.h | 1 + common/structs.h | 1 + include/tdnf.h | 7 ++ include/tdnfcli.h | 12 +++ include/tdnfclitypes.h | 7 ++ include/tdnftypes.h | 14 +++ tools/cli/lib/CMakeLists.txt | 1 + tools/cli/lib/api.c | 27 ++++++ tools/cli/lib/parseargs.c | 23 ++++- tools/cli/lib/parsereposyncargs.c | 85 +++++++++++++++++ tools/cli/main.c | 12 +++ tools/cli/prototypes.h | 13 +++ 16 files changed, 608 insertions(+), 1 deletion(-) create mode 100644 tools/cli/lib/parsereposyncargs.c diff --git a/client/api.c b/client/api.c index 143b14d3..e191c4fb 100644 --- a/client/api.c +++ b/client/api.c @@ -980,6 +980,144 @@ TDNFRepoList( goto cleanup; } +uint32_t +TDNFRepoSync( + PTDNF pTdnf, + PTDNF_REPOSYNC_ARGS pReposyncArgs + ) +{ + uint32_t dwError = 0; + PTDNF_PKG_INFO pPkgInfos = NULL; + PTDNF_PKG_INFO pPkgInfo = NULL; + PTDNF_REPO_DATA_INTERNAL pRepo = NULL; + char *pszRepoDir = NULL; + PSolvQuery pQuery = NULL; + PSolvPackageList pPkgList = NULL; + char *pszRootPath = NULL; + char *pszDir = NULL; + char *pszFilePath; + uint32_t dwCount = 0; + + if(!pTdnf || !pTdnf->pSack) + { + dwError = ERROR_TDNF_INVALID_PARAMETER; + BAIL_ON_TDNF_ERROR(dwError); + } + + dwError = TDNFRefresh(pTdnf); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = SolvCreateQuery(pTdnf->pSack, &pQuery); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFApplyScopeFilter(pQuery, SCOPE_ALL); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = SolvApplyListQuery(pQuery); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = SolvGetQueryResult(pQuery, &pPkgList); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFPopulatePkgInfoForRepoSync(pTdnf->pSack, pPkgList, &pPkgInfos); + BAIL_ON_TDNF_ERROR(dwError); + + if (pReposyncArgs->pszDownloadPath == NULL) + { + pszRootPath = getcwd(NULL, 0); + if (!pszRootPath) + { + BAIL_ON_TDNF_SYSTEM_ERROR(errno); + } + } + else + { + dwError = TDNFAllocateString(pReposyncArgs->pszDownloadPath, &pszRootPath); + BAIL_ON_TDNF_ERROR(dwError); + } + + for (pPkgInfo = pPkgInfos; pPkgInfo; pPkgInfo = pPkgInfo->pNext) + { + dwCount++; + if (strcmp(pPkgInfo->pszRepoName, SYSTEM_REPO_NAME) == 0) + { + continue; + } + + dwError = TDNFAllocateStringPrintf(&pszDir, "%s/%s", + pszRootPath, pPkgInfo->pszRepoName); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFUtilsMakeDir(pszDir); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFDownloadPackageToDirectory(pTdnf, + pPkgInfo->pszLocation, pPkgInfo->pszName, + pPkgInfo->pszRepoName, pszDir, + &pszFilePath); + BAIL_ON_TDNF_ERROR(dwError); + + TDNF_SAFE_FREE_MEMORY(pszDir); + TDNF_SAFE_FREE_MEMORY(pszFilePath); + } + + if (pReposyncArgs->nDownloadMetadata) + { + for (pRepo = pTdnf->pRepos; pRepo; pRepo = pRepo->pNext) + { + + if ((strcmp(pRepo->pszName, "@cmdline") == 0) || + (!pRepo->nEnabled)) + { + continue; + } + + if (pReposyncArgs->pszMetaDataPath == NULL) + { + dwError = TDNFAllocateStringPrintf(&pszRepoDir, "%s/%s", + pszRootPath, pRepo->pszId); + BAIL_ON_TDNF_ERROR(dwError); + } + else + { + dwError = TDNFAllocateStringPrintf(&pszRepoDir, "%s/%s", + pReposyncArgs->pszMetaDataPath, pRepo->pszId); + BAIL_ON_TDNF_ERROR(dwError); + } + + + dwError = TDNFUtilsMakeDir(pszRepoDir); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFDownloadMetadata(pTdnf, pRepo, pszRepoDir); + BAIL_ON_TDNF_ERROR(dwError); + + TDNF_SAFE_FREE_MEMORY(pszRepoDir); + } + } + +cleanup: + if(pQuery) + { + SolvFreeQuery(pQuery); + } + if(pPkgList) + { + SolvFreePackageList(pPkgList); + } + TDNF_SAFE_FREE_MEMORY(pszDir); + TDNF_SAFE_FREE_MEMORY(pszRepoDir); + TDNF_SAFE_FREE_MEMORY(pszRootPath); + TDNFFreePackageInfoArray(pPkgInfos, dwCount); + return dwError; +error: + if(dwError == ERROR_TDNF_NO_MATCH) + { + dwError = ERROR_TDNF_NO_DATA; + } + goto cleanup; +} + //Resolve alter command before presenting //the goal steps to user for approval uint32_t diff --git a/client/packageutils.c b/client/packageutils.c index 66aca2a4..07099a39 100644 --- a/client/packageutils.c +++ b/client/packageutils.c @@ -207,6 +207,100 @@ TDNFPopulatePkgInfoArray( goto cleanup; } +uint32_t +TDNFPopulatePkgInfoForRepoSync( + PSolvSack pSack, + PSolvPackageList pPkgList, + PTDNF_PKG_INFO* ppPkgInfo + ) +{ + uint32_t dwError = 0; + uint32_t dwCount = 0; + int dwPkgIndex = 0; + Id dwPkgId = 0; + PTDNF_PKG_INFO pPkgInfos = NULL; + PTDNF_PKG_INFO pPkgInfo = NULL; + + if(!ppPkgInfo || !pSack || !pPkgList) + { + dwError = ERROR_TDNF_INVALID_PARAMETER; + BAIL_ON_TDNF_ERROR(dwError); + } + + dwError = SolvGetPackageListSize(pPkgList, &dwCount); + BAIL_ON_TDNF_ERROR(dwError); + + if(dwCount == 0) + { + dwError = ERROR_TDNF_NO_MATCH; + BAIL_ON_TDNF_ERROR(dwError); + } + + dwError = TDNFAllocateMemory( + dwCount, + sizeof(TDNF_PKG_INFO), + (void**)&pPkgInfos); + BAIL_ON_TDNF_ERROR(dwError); + + for (dwPkgIndex = 0; (uint32_t)dwPkgIndex < dwCount; dwPkgIndex++) + { + pPkgInfo = &pPkgInfos[dwPkgIndex]; + if ((uint32_t)dwPkgIndex < dwCount-1) + { + pPkgInfo->pNext = &pPkgInfos[dwPkgIndex+1]; + } + + dwError = SolvGetPackageId(pPkgList, dwPkgIndex, &dwPkgId); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = SolvGetPkgNameFromId(pSack, dwPkgId, &pPkgInfo->pszName); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = SolvGetPkgArchFromId(pSack, dwPkgId, &pPkgInfo->pszArch); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = SolvGetPkgVersionFromId( + pSack, + dwPkgId, + &pPkgInfo->pszVersion); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = SolvGetPkgReleaseFromId( + pSack, + dwPkgId, + &pPkgInfo->pszRelease); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = SolvGetPkgRepoNameFromId( + pSack, + dwPkgId, + &pPkgInfo->pszRepoName); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = SolvGetPkgLocationFromId( + pSack, + dwPkgId, + &pPkgInfo->pszLocation); + BAIL_ON_TDNF_ERROR(dwError); + } + + *ppPkgInfo = pPkgInfos; + +cleanup: + return dwError; + +error: + if(ppPkgInfo) + { + *ppPkgInfo = NULL; + } + if(pPkgInfos) + { + TDNFFreePackageInfoArray(pPkgInfos, dwCount); + } + goto cleanup; +} + uint32_t TDNFAppendPackages( PTDNF_PKG_INFO* ppDest, diff --git a/client/prototypes.h b/client/prototypes.h index 07d03da7..4540f353 100644 --- a/client/prototypes.h +++ b/client/prototypes.h @@ -291,6 +291,13 @@ TDNFPopulatePkgInfos( PTDNF_PKG_INFO* ppPkgInfo ); +uint32_t +TDNFPopulatePkgInfoForRepoSync( + PSolvSack pSack, + PSolvPackageList pPkgList, + PTDNF_PKG_INFO* ppPkgInfo + ); + uint32_t TDNFPopulatePkgInfoArray( PSolvSack pSack, @@ -642,6 +649,21 @@ TDNFReplaceFile( const char *pszDstFile ); +uint32_t +TDNFDownloadMetadata( + PTDNF pTdnf, + PTDNF_REPO_DATA_INTERNAL pRepo, + const char *pszRepoDir + ); + +uint32_t +TDNFDownloadRepoMDParts( + PTDNF pTdnf, + Repo *pSolvRepo, + PTDNF_REPO_DATA_INTERNAL pRepo, + const char *pszDir + ); + //repolist.c uint32_t TDNFLoadReposFromFile( diff --git a/client/repo.c b/client/repo.c index fbd7ba28..8211b044 100644 --- a/client/repo.c +++ b/client/repo.c @@ -1327,3 +1327,155 @@ TDNFReplaceFile( error: goto cleanup; } + +uint32_t +TDNFDownloadMetadata( + PTDNF pTdnf, + PTDNF_REPO_DATA_INTERNAL pRepo, + const char *pszRepoDir + ) +{ + uint32_t dwError = 0; + char *pszRepoMDUrl = NULL; + char *pszRepoMDPath = NULL; + char *pszRepoDataDir = NULL; + Repo *pSolvRepo = NULL; + Pool *pPool = NULL; + FILE *fp = NULL; + + dwError = TDNFUtilsMakeDir(pszRepoDir); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFAllocateStringPrintf(&pszRepoDataDir, "%s/repodata", + pszRepoDir); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFUtilsMakeDir(pszRepoDataDir); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFAllocateStringPrintf(&pszRepoMDPath, "%s/%s", + pszRepoDataDir, TDNF_REPO_METADATA_FILE_NAME); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFAllocateStringPrintf(&pszRepoMDUrl, + "%s/%s", + pRepo->pszBaseUrl, + TDNF_REPO_METADATA_FILE_PATH); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFDownloadFile(pTdnf, pRepo->pszId, pszRepoMDUrl, pszRepoMDPath, pRepo->pszId); + BAIL_ON_TDNF_ERROR(dwError); + + pPool = pool_create(); + pSolvRepo = repo_create(pPool, "md_parse_temp"); + + fp = fopen(pszRepoMDPath, "r"); + if(!fp) + { + dwError = errno; + BAIL_ON_TDNF_SYSTEM_ERROR(dwError); + } + + dwError = repo_add_repomdxml(pSolvRepo, fp, 0); + if(dwError) + { + pr_crit("Error(%u) parsing repomd: %s\n", + dwError, + pool_errstr(pPool)); + } + + dwError = TDNFDownloadRepoMDParts(pTdnf, pSolvRepo, pRepo, pszRepoDir); + BAIL_ON_TDNF_ERROR(dwError); + +cleanup: + if (fp) + { + fclose(fp); + } + if (pSolvRepo) + { + repo_free(pSolvRepo, 0); + } + if (pPool) + { + pool_free(pPool); + } + TDNF_SAFE_FREE_MEMORY(pszRepoMDPath); + TDNF_SAFE_FREE_MEMORY(pszRepoMDUrl); + TDNF_SAFE_FREE_MEMORY(pszRepoDataDir); + return dwError; +error: + goto cleanup; +} + +uint32_t +TDNFDownloadRepoMDParts( + PTDNF pTdnf, + Repo *pSolvRepo, + PTDNF_REPO_DATA_INTERNAL pRepo, + const char *pszDir + ) +{ + uint32_t dwError = 0; + Pool *pPool = NULL; + Dataiterator di = {0}; + const char *pszPartFile = NULL; + char *pszPartUrl = NULL; + char *pszPartPath = NULL; + + if(!pSolvRepo || + !pSolvRepo->pool || + !pRepo) + { + dwError = ERROR_TDNF_INVALID_PARAMETER; + BAIL_ON_TDNF_ERROR(dwError); + } + + pPool = pSolvRepo->pool; + + dwError = dataiterator_init( + &di, + pPool, + pSolvRepo, + SOLVID_META, + REPOSITORY_REPOMD_TYPE, + 0, + SEARCH_STRING); + BAIL_ON_TDNF_ERROR(dwError); + + dataiterator_prepend_keyname(&di, REPOSITORY_REPOMD); + + while (dataiterator_step(&di)) + { + dataiterator_setpos_parent(&di); + pszPartFile = pool_lookup_str( + pPool, + SOLVID_POS, + REPOSITORY_REPOMD_LOCATION); + + dwError = TDNFAllocateStringPrintf(&pszPartUrl, + "%s/%s", + pRepo->pszBaseUrl, + pszPartFile); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFAllocateStringPrintf(&pszPartPath, + "%s/%s", + pszDir, + pszPartFile); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFDownloadFile(pTdnf, pRepo->pszId, pszPartUrl, pszPartPath, pRepo->pszId); + BAIL_ON_TDNF_ERROR(dwError); + + TDNF_SAFE_FREE_MEMORY(pszPartUrl); + TDNF_SAFE_FREE_MEMORY(pszPartPath); + } + +cleanup: + dataiterator_free(&di); + return dwError; +error: + goto cleanup; +} + diff --git a/client/structs.h b/client/structs.h index 57dbefe7..74bd3ac5 100644 --- a/client/structs.h +++ b/client/structs.h @@ -118,3 +118,4 @@ typedef struct _TDNF_EVENT_DATA_ const char *pcszName; struct _TDNF_EVENT_DATA_ *pNext; }TDNF_EVENT_DATA, *PTDNF_EVENT_DATA; + diff --git a/common/structs.h b/common/structs.h index a100bb37..caec9b7a 100644 --- a/common/structs.h +++ b/common/structs.h @@ -32,3 +32,4 @@ typedef uint32_t const char *pszKey, const char *pszValue ); + diff --git a/include/tdnf.h b/include/tdnf.h index 6af03db2..13c97b0b 100644 --- a/include/tdnf.h +++ b/include/tdnf.h @@ -113,6 +113,13 @@ TDNFProvides( PTDNF_PKG_INFO* ppPkgInfo ); +//query repo +uint32_t +TDNFRepoSync( + PTDNF pTdnf, + PTDNF_REPOSYNC_ARGS pReposyncArgs + ); + //Show update info for specified scope uint32_t TDNFUpdateInfo( diff --git a/include/tdnfcli.h b/include/tdnfcli.h index 219cf237..f45c589b 100644 --- a/include/tdnfcli.h +++ b/include/tdnfcli.h @@ -71,6 +71,12 @@ TDNFCliParseRepoListArgs( TDNF_REPOLISTFILTER* pnFilter ); +uint32_t +TDNFCliParseRepoSyncArgs( + PTDNF_CMD_ARGS pCmdArgs, + PTDNF_REPOSYNC_ARGS* ppReposyncArgs + ); + uint32_t TDNFCliParseMode( const char* pszOutMode, @@ -189,6 +195,12 @@ TDNFCliProvidesCommand( PTDNF_CMD_ARGS pCmdArgs ); +uint32_t +TDNFCliRepoSyncCommand( + PTDNF_CLI_CONTEXT pContext, + PTDNF_CMD_ARGS pCmdArgs + ); + uint32_t TDNFCliUpdateInfoCommand( PTDNF_CLI_CONTEXT pContext, diff --git a/include/tdnfclitypes.h b/include/tdnfclitypes.h index 65de3952..a0522f04 100644 --- a/include/tdnfclitypes.h +++ b/include/tdnfclitypes.h @@ -107,6 +107,12 @@ typedef uint32_t PTDNF_REPO_DATA * ); +typedef uint32_t +(*PFN_TDNF_REPOSYNC)( + PTDNF_CLI_CONTEXT, + PTDNF_REPOSYNC_ARGS + ); + typedef uint32_t (*PFN_TDNF_RESOLVE)( PTDNF_CLI_CONTEXT, @@ -148,6 +154,7 @@ typedef struct _TDNF_CLI_CONTEXT_ PFN_TDNF_LIST pFnList; PFN_TDNF_PROVIDES pFnProvides; PFN_TDNF_REPOLIST pFnRepoList; + PFN_TDNF_REPOSYNC pFnRepoSync; PFN_TDNF_RESOLVE pFnResolve; PFN_TDNF_SEARCH pFnSearch; PFN_TDNF_UPDATEINFO pFnUpdateInfo; diff --git a/include/tdnftypes.h b/include/tdnftypes.h index 16f84b90..28b5586d 100644 --- a/include/tdnftypes.h +++ b/include/tdnftypes.h @@ -333,6 +333,20 @@ typedef struct _TDNF_UPDATEINFO_SUMMARY int nType; }TDNF_UPDATEINFO_SUMMARY, *PTDNF_UPDATEINFO_SUMMARY; +typedef struct _TDNF_REPOSYNC_ARGS +{ + int nDelete; + int nDownloadMetadata; + int nGPGCheck; + int nNewestOnly; + int nPrintUrlsOnly; + char *pszDownloadPath; + char *pszMetaDataPath; + char **pszArchs; +}TDNF_REPOSYNC_ARGS, *PTDNF_REPOSYNC_ARGS; + + + #ifdef __cplusplus } #endif diff --git a/tools/cli/lib/CMakeLists.txt b/tools/cli/lib/CMakeLists.txt index 0a85d529..d70fb8bd 100644 --- a/tools/cli/lib/CMakeLists.txt +++ b/tools/cli/lib/CMakeLists.txt @@ -22,6 +22,7 @@ add_library(${LIB_TDNF_CLI} SHARED parsecleanargs.c parselistargs.c parserepolistargs.c + parsereposyncargs.c parseupdateinfo.c updateinfocmd.c ) diff --git a/tools/cli/lib/api.c b/tools/cli/lib/api.c index 2d7a94a4..d9d9d4b3 100644 --- a/tools/cli/lib/api.c +++ b/tools/cli/lib/api.c @@ -456,6 +456,33 @@ TDNFCliProvidesCommand( goto cleanup; } +uint32_t +TDNFCliRepoSyncCommand( + PTDNF_CLI_CONTEXT pContext, + PTDNF_CMD_ARGS pCmdArgs + ) +{ + uint32_t dwError = 0; + PTDNF_REPOSYNC_ARGS pReposyncArgs; + + if(!pContext || !pContext->hTdnf || !pCmdArgs || !pContext->pFnRepoSync) + { + dwError = ERROR_TDNF_CLI_INVALID_ARGUMENT; + BAIL_ON_CLI_ERROR(dwError); + } + + dwError = TDNFCliParseRepoSyncArgs(pCmdArgs, &pReposyncArgs); + BAIL_ON_CLI_ERROR(dwError); + + dwError = pContext->pFnRepoSync(pContext, pReposyncArgs); + BAIL_ON_CLI_ERROR(dwError); +cleanup: + return dwError; + +error: + goto cleanup; +} + uint32_t TDNFCliCheckUpdateCommand( PTDNF_CLI_CONTEXT pContext, diff --git a/tools/cli/lib/parseargs.c b/tools/cli/lib/parseargs.c index 0e792d93..c088f01c 100644 --- a/tools/cli/lib/parseargs.c +++ b/tools/cli/lib/parseargs.c @@ -34,7 +34,10 @@ static SetOptArgs OptValTable[] = { {CMDOPT_DISABLEREPO, "disablerepo", NULL}, {CMDOPT_ENABLEPLUGIN, "enableplugin", NULL}, {CMDOPT_DISABLEPLUGIN, "disableplugin", NULL}, - {CMDOPT_KEYVALUE, "skipconflicts;skipobsoletes;skipsignature;skipdigest;noplugins;reboot-required;security", "1"} + {CMDOPT_KEYVALUE, "skipconflicts;skipobsoletes;skipsignature;skipdigest;" + "noplugins;reboot-required;security;" + "delete;download-metadata;gpgcheck;newest-only;urls", + "1"} }; static TDNF_CMD_ARGS _opt = {0}; @@ -80,6 +83,15 @@ static struct option pstOptions[] = {"disableexcludes", no_argument, &_opt.nDisableExcludes, 1}, //--disableexcludes {"downloadonly", no_argument, &_opt.nDownloadOnly, 1}, //--downloadonly {"downloaddir", required_argument, 0, 0}, //--downloaddir + // reposync options + {"arch", required_argument, 0, 'a'}, + {"delete", no_argument, 0, 0}, + {"download-metadata", no_argument, 0, 0}, + {"gpgcheck", no_argument, 0, 'g'}, + {"metadata-path", required_argument, 0, 0}, + {"newest-only", no_argument, 0, 'n'}, + {"download-path", required_argument, 0, 0}, + {"urls", no_argument, 0, 'u'}, {0, 0, 0, 0} }; @@ -315,6 +327,15 @@ ParseOption( { dwError = TDNFAllocateString(optarg, &pCmdArgs->pszReleaseVer); } + else if ((!strcasecmp(pszName, "metadata-path")) || + (!strcasecmp(pszName, "download-path"))) + { + dwError = AddSetOptWithValues(pCmdArgs, + CMDOPT_KEYVALUE, + pszName, + optarg); + BAIL_ON_CLI_ERROR(dwError); + } else if (!strcasecmp(pszName, "setopt")) { if (!optarg) diff --git a/tools/cli/lib/parsereposyncargs.c b/tools/cli/lib/parsereposyncargs.c new file mode 100644 index 00000000..11f05bc9 --- /dev/null +++ b/tools/cli/lib/parsereposyncargs.c @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2021 VMware, Inc. All Rights Reserved. + * + * Licensed under the GNU Lesser General Public License v2.1 (the "License"); + * you may not use this file except in compliance with the License. The terms + * of the License are located in the COPYING file of this distribution. + */ + +/* + * Module : parsereposyncargs.c + * + * Abstract : + * + * tdnf + * + * command line tools + */ + +#include "includes.h" + +uint32_t +TDNFCliParseRepoSyncArgs( + PTDNF_CMD_ARGS pArgs, + PTDNF_REPOSYNC_ARGS* ppReposyncArgs + ) +{ + uint32_t dwError = 0; + PTDNF_REPOSYNC_ARGS pReposyncArgs = NULL; + PTDNF_CMD_OPT pSetOpt = NULL; + + dwError = TDNFAllocateMemory( + 1, + sizeof(TDNF_REPOSYNC_ARGS), + (void**) &pReposyncArgs); + BAIL_ON_CLI_ERROR(dwError); + + for (pSetOpt = pArgs->pSetOpt; + pSetOpt; + pSetOpt = pSetOpt->pNext) + { + if(pSetOpt->nType == CMDOPT_KEYVALUE) + { + if (strcasecmp(pSetOpt->pszOptName, "delete") == 0) + { + pReposyncArgs->nDelete = 1; + } + else if (strcasecmp(pSetOpt->pszOptName, "download-metadata") == 0) + { + pReposyncArgs->nDownloadMetadata = 1; + } + else if (strcasecmp(pSetOpt->pszOptName, "gpgcheck") == 0) + { + pReposyncArgs->nGPGCheck = 1; + } + else if (strcasecmp(pSetOpt->pszOptName, "newest-only") == 0) + { + pReposyncArgs->nNewestOnly = 1; + } + else if (strcasecmp(pSetOpt->pszOptName, "urls") == 0) + { + pReposyncArgs->nPrintUrlsOnly = 1; + } + else if (strcasecmp(pSetOpt->pszOptName, "download-path") == 0) + { + dwError = TDNFAllocateString( + pSetOpt->pszOptValue, + &pReposyncArgs->pszDownloadPath); + BAIL_ON_CLI_ERROR(dwError); + } + else if (strcasecmp(pSetOpt->pszOptName, "metadata-path") == 0) + { + dwError = TDNFAllocateString( + pSetOpt->pszOptValue, + &pReposyncArgs->pszMetaDataPath); + BAIL_ON_CLI_ERROR(dwError); + } + } + } + *ppReposyncArgs = pReposyncArgs; +cleanup: + return dwError; +error: + goto cleanup; +} + diff --git a/tools/cli/main.c b/tools/cli/main.c index 7c20a565..e3731d39 100644 --- a/tools/cli/main.c +++ b/tools/cli/main.c @@ -55,6 +55,8 @@ int main(int argc, char* argv[]) {"reinstall", TDNFCliReinstallCommand}, {"remove", TDNFCliEraseCommand}, {"repolist", TDNFCliRepoListCommand}, + {"reposync", TDNFCliRepoSyncCommand}, + {"provides", TDNFCliProvidesCommand}, {"search", TDNFCliSearchCommand}, {"update", TDNFCliUpgradeCommand}, {"update-to", TDNFCliUpgradeCommand}, @@ -102,6 +104,7 @@ int main(int argc, char* argv[]) _context.pFnList = TDNFCliInvokeList; _context.pFnProvides = TDNFCliInvokeProvides; _context.pFnRepoList = TDNFCliInvokeRepoList; + _context.pFnRepoSync = TDNFCliInvokeRepoSync; /* * Alter and resolve will address commands like @@ -444,6 +447,15 @@ TDNFCliInvokeRepoList( return TDNFRepoList(pContext->hTdnf, nFilter, ppRepos); } +uint32_t +TDNFCliInvokeRepoSync( + PTDNF_CLI_CONTEXT pContext, + PTDNF_REPOSYNC_ARGS pRepoSyncArgs + ) +{ + return TDNFRepoSync(pContext->hTdnf, pRepoSyncArgs); +} + uint32_t TDNFCliInvokeResolve( PTDNF_CLI_CONTEXT pContext, diff --git a/tools/cli/prototypes.h b/tools/cli/prototypes.h index c7f41803..fd722239 100644 --- a/tools/cli/prototypes.h +++ b/tools/cli/prototypes.h @@ -91,6 +91,12 @@ TDNFCliInvokeRepoList( PTDNF_REPO_DATA *ppRepos ); +uint32_t +TDNFCliInvokeRepoSync( + PTDNF_CLI_CONTEXT pContext, + PTDNF_REPOSYNC_ARGS pReposyncArgs + ); + uint32_t TDNFCliInvokeResolve( PTDNF_CLI_CONTEXT pContext, @@ -301,6 +307,13 @@ TDNFCliParseRepoListArgs( TDNF_REPOLISTFILTER* pnFilter ); +//parsereposyncargs.c +uint32_t +TDNFCliParseRepoSyncArgs( + PTDNF_CMD_ARGS pCmdArgs, + PTDNF_REPOSYNC_ARGS* ppReposyncArgs + ); + //parseupdateinfo.c uint32_t ParseMode( From 9e403764c0e5066877870e82699bb252f32f26c4 Mon Sep 17 00:00:00 2001 From: Oliver Kurth Date: Wed, 28 Apr 2021 14:08:34 -0700 Subject: [PATCH 04/17] do not redownload packages --- client/remoterepo.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/client/remoterepo.c b/client/remoterepo.c index 303cd9c4..a29ec361 100644 --- a/client/remoterepo.c +++ b/client/remoterepo.c @@ -879,6 +879,7 @@ TDNFDownloadPackage( char *pszPackageUrl = NULL; char *pszPackageFile = NULL; char *pszCopyOfPackageLocation = NULL; + int nSize; if(!pTdnf || !pTdnf->pArgs || @@ -916,11 +917,21 @@ TDNFDownloadPackage( basename(pszCopyOfPackageLocation)); BAIL_ON_TDNF_ERROR(dwError); - dwError = TDNFDownloadFile(pTdnf, - pszRepoName, - pszPackageUrl, - pszPackageFile, - pszPkgName); + /* don't download if file is already there. Older versions may have left + size 0 files, so check for those too */ + dwError = TDNFGetFileSize(pszPackageFile, &nSize); + if ((dwError == ERROR_TDNF_FILE_NOT_FOUND) || (nSize == 0)) + { + dwError = TDNFDownloadFile(pTdnf, + pszRepoName, + pszPackageUrl, + pszPackageFile, + pszPkgName); + } + else + { + pr_info("%s package already downloaded", pszPkgName); + } BAIL_ON_TDNF_ERROR(dwError); pr_info("\n"); From 9225ddae7eefa0f23c297162414503d83620e8e9 Mon Sep 17 00:00:00 2001 From: Oliver Kurth Date: Wed, 28 Apr 2021 14:29:21 -0700 Subject: [PATCH 05/17] add TDNFCreatePackageUrl() --- client/prototypes.h | 8 +++++++ client/remoterepo.c | 53 ++++++++++++++++++++++++++++++++++++--------- 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/client/prototypes.h b/client/prototypes.h index 4540f353..a4793abc 100644 --- a/client/prototypes.h +++ b/client/prototypes.h @@ -248,6 +248,14 @@ TDNFDownloadFile( const char *pszProgressData ); +uint32_t +TDNFCreatePackageUrl( + PTDNF pTdnf, + const char* pszRepoName, + const char* pszPackageLocation, + char **ppszPackageUrl + ); + uint32_t TDNFDownloadPackage( PTDNF pTdnf, diff --git a/client/remoterepo.c b/client/remoterepo.c index a29ec361..b5e4052d 100644 --- a/client/remoterepo.c +++ b/client/remoterepo.c @@ -866,26 +866,22 @@ TDNFDownloadFile( } uint32_t -TDNFDownloadPackage( +TDNFCreatePackageUrl( PTDNF pTdnf, - const char* pszPackageLocation, - const char* pszPkgName, const char* pszRepoName, - const char* pszRpmCacheDir + const char* pszPackageLocation, + char **ppszPackageUrl ) { uint32_t dwError = 0; char* pszBaseUrl = NULL; char *pszPackageUrl = NULL; - char *pszPackageFile = NULL; - char *pszCopyOfPackageLocation = NULL; - int nSize; if(!pTdnf || !pTdnf->pArgs || IsNullOrEmptyString(pszPackageLocation) || - IsNullOrEmptyString(pszPkgName) || - IsNullOrEmptyString(pszRepoName)) + IsNullOrEmptyString(pszRepoName) || + !ppszPackageUrl) { dwError = ERROR_TDNF_INVALID_PARAMETER; BAIL_ON_TDNF_ERROR(dwError); @@ -906,6 +902,44 @@ TDNFDownloadPackage( dwError = TDNFAllocateString(pszPackageLocation, &pszPackageUrl); BAIL_ON_TDNF_ERROR(dwError); } + *ppszPackageUrl = pszPackageUrl; + +cleanup: + TDNF_SAFE_FREE_MEMORY(pszBaseUrl); + return dwError; + +error: + TDNF_SAFE_FREE_MEMORY(pszPackageUrl); + goto cleanup; +} + +uint32_t +TDNFDownloadPackage( + PTDNF pTdnf, + const char* pszPackageLocation, + const char* pszPkgName, + const char* pszRepoName, + const char* pszRpmCacheDir + ) +{ + uint32_t dwError = 0; + char *pszPackageUrl = NULL; + char *pszPackageFile = NULL; + char *pszCopyOfPackageLocation = NULL; + int nSize; + + if(!pTdnf || + !pTdnf->pArgs || + IsNullOrEmptyString(pszPackageLocation) || + IsNullOrEmptyString(pszPkgName) || + IsNullOrEmptyString(pszRepoName)) + { + dwError = ERROR_TDNF_INVALID_PARAMETER; + BAIL_ON_TDNF_ERROR(dwError); + } + + dwError = TDNFCreatePackageUrl(pTdnf, pszRepoName, pszPackageLocation, &pszPackageUrl); + BAIL_ON_TDNF_ERROR(dwError); dwError = TDNFAllocateString(pszPackageLocation, &pszCopyOfPackageLocation); @@ -940,7 +974,6 @@ TDNFDownloadPackage( TDNF_SAFE_FREE_MEMORY(pszPackageUrl); TDNF_SAFE_FREE_MEMORY(pszCopyOfPackageLocation); TDNF_SAFE_FREE_MEMORY(pszPackageFile); - TDNF_SAFE_FREE_MEMORY(pszBaseUrl); return dwError; error: From 8544b87e141474f3e0293b7b3a059a3701c5ae3a Mon Sep 17 00:00:00 2001 From: Oliver Kurth Date: Thu, 29 Apr 2021 10:38:49 -0700 Subject: [PATCH 06/17] reposync: urls only option, delete option, norepopath and check conflicting options --- client/api.c | 184 ++++++++++++++++++++++++++---- client/prototypes.h | 6 +- client/repo.c | 67 +++++++---- include/tdnftypes.h | 3 +- tools/cli/lib/parseargs.c | 11 +- tools/cli/lib/parsereposyncargs.c | 4 + 6 files changed, 225 insertions(+), 50 deletions(-) diff --git a/client/api.c b/client/api.c index e191c4fb..1e45e743 100644 --- a/client/api.c +++ b/client/api.c @@ -18,6 +18,7 @@ * Authors : Priyesh Padmavilasom (ppadmavilasom@vmware.com) */ +#include #include "includes.h" #include "config.h" @@ -980,6 +981,58 @@ TDNFRepoList( goto cleanup; } +static int +_rm_rpms( + const char *pszFilePath, + const struct stat *sbuf, + int type, + struct FTW *ftwb + ) +{ + uint32_t dwError = 0; + char *pszKeepFile = NULL; + struct stat statKeep = {0}; + + UNUSED(sbuf); + UNUSED(type); + UNUSED(ftwb); + + if (strcmp(&pszFilePath[strlen(pszFilePath)-4], ".rpm") == 0) + { + dwError = TDNFAllocateStringPrintf(&pszKeepFile, "%s.reposync-keep", pszFilePath); + BAIL_ON_TDNF_ERROR(dwError); + + if (stat(pszKeepFile, &statKeep)) + { + if (errno == ENOENT) + { + if(remove(pszFilePath) < 0) + { + pr_crit("unable to remove %s: %s\n", pszFilePath, strerror(errno)); + } + } + else + { + dwError = errno; + BAIL_ON_TDNF_SYSTEM_ERROR(dwError); + } + } + else + { + /* marker file can be removed now */ + if(remove(pszKeepFile) < 0) + { + pr_crit("unable to remove %s: %s\n", pszKeepFile, strerror(errno)); + } + } + } +cleanup: + TDNF_SAFE_FREE_MEMORY(pszKeepFile); + return dwError; +error: + goto cleanup; +} + uint32_t TDNFRepoSync( PTDNF pTdnf, @@ -994,9 +1047,12 @@ TDNFRepoSync( PSolvQuery pQuery = NULL; PSolvPackageList pPkgList = NULL; char *pszRootPath = NULL; + char *pszUrl = NULL; char *pszDir = NULL; - char *pszFilePath; + char *pszFilePath = NULL; + char *pszKeepFile = NULL; uint32_t dwCount = 0; + uint32_t dwRepoCount = 0; if(!pTdnf || !pTdnf->pSack) { @@ -1004,6 +1060,31 @@ TDNFRepoSync( BAIL_ON_TDNF_ERROR(dwError); } + for (pRepo = pTdnf->pRepos; pRepo; pRepo = pRepo->pNext) + { + if ((strcmp(pRepo->pszName, "@cmdline") == 0) || + (!pRepo->nEnabled)) + { + continue; + } + dwRepoCount++; + } + + if (dwRepoCount > 1 && pReposyncArgs->nNoRepoPath) + { + pr_crit("cannot use norepopath with multiple repos\n"); + dwError = ERROR_TDNF_INVALID_PARAMETER; + BAIL_ON_TDNF_ERROR(dwError); + } + + if (pReposyncArgs->nDelete && pReposyncArgs->nNoRepoPath) + { + /* prevent accidental deletion of packages */ + pr_crit("cannot use the delete option with norepopath\n"); + dwError = ERROR_TDNF_INVALID_PARAMETER; + BAIL_ON_TDNF_ERROR(dwError); + } + dwError = TDNFRefresh(pTdnf); BAIL_ON_TDNF_ERROR(dwError); @@ -1044,52 +1125,113 @@ TDNFRepoSync( continue; } - dwError = TDNFAllocateStringPrintf(&pszDir, "%s/%s", - pszRootPath, pPkgInfo->pszRepoName); - BAIL_ON_TDNF_ERROR(dwError); + if (!pReposyncArgs->nPrintUrlsOnly) + { + if (!pReposyncArgs->nNoRepoPath) + { + dwError = TDNFAllocateStringPrintf(&pszDir, "%s/%s", + pszRootPath, pPkgInfo->pszRepoName); + BAIL_ON_TDNF_ERROR(dwError); + } + else + { + dwError = TDNFAllocateString(pszRootPath, &pszDir); + BAIL_ON_TDNF_ERROR(dwError); + } - dwError = TDNFUtilsMakeDir(pszDir); - BAIL_ON_TDNF_ERROR(dwError); + dwError = TDNFUtilsMakeDir(pszDir); + BAIL_ON_TDNF_ERROR(dwError); - dwError = TDNFDownloadPackageToDirectory(pTdnf, - pPkgInfo->pszLocation, pPkgInfo->pszName, - pPkgInfo->pszRepoName, pszDir, - &pszFilePath); - BAIL_ON_TDNF_ERROR(dwError); + dwError = TDNFDownloadPackageToDirectory(pTdnf, + pPkgInfo->pszLocation, pPkgInfo->pszName, + pPkgInfo->pszRepoName, pszDir, + &pszFilePath); + BAIL_ON_TDNF_ERROR(dwError); + + if (pReposyncArgs->nDelete) + { + /* if deleting, create a marker file to protect + what we just downloaded. Later all *.rpm files that do not + have a marker file will be deleted */ + dwError = TDNFAllocateStringPrintf(&pszKeepFile, "%s.reposync-keep", pszFilePath); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFTouchFile(pszKeepFile); + BAIL_ON_TDNF_ERROR(dwError); + TDNF_SAFE_FREE_MEMORY(pszKeepFile); + } + + TDNF_SAFE_FREE_MEMORY(pszDir); + TDNF_SAFE_FREE_MEMORY(pszFilePath); + } + else + { + dwError = TDNFCreatePackageUrl(pTdnf, + pPkgInfo->pszRepoName, + pPkgInfo->pszLocation, + &pszUrl); + BAIL_ON_TDNF_ERROR(dwError); + + pr_info("%s\n", pszUrl); - TDNF_SAFE_FREE_MEMORY(pszDir); - TDNF_SAFE_FREE_MEMORY(pszFilePath); + TDNF_SAFE_FREE_MEMORY(pszUrl); + } } - if (pReposyncArgs->nDownloadMetadata) + if (pReposyncArgs->nDelete) { for (pRepo = pTdnf->pRepos; pRepo; pRepo = pRepo->pNext) { + if ((strcmp(pRepo->pszName, "@cmdline") == 0) || + (!pRepo->nEnabled)) + { + continue; + } + + dwError = TDNFAllocateStringPrintf(&pszRepoDir, "%s/%s", + pszRootPath, pRepo->pszId); + BAIL_ON_TDNF_ERROR(dwError); + if (nftw(pszRepoDir, _rm_rpms, 10, FTW_DEPTH|FTW_PHYS) < 0) + { + dwError = errno; + BAIL_ON_TDNF_SYSTEM_ERROR(dwError); + } + } + } + + if (pReposyncArgs->nDownloadMetadata) + { + for (pRepo = pTdnf->pRepos; pRepo; pRepo = pRepo->pNext) + { if ((strcmp(pRepo->pszName, "@cmdline") == 0) || (!pRepo->nEnabled)) { continue; } - if (pReposyncArgs->pszMetaDataPath == NULL) + if (!pReposyncArgs->nNoRepoPath) { dwError = TDNFAllocateStringPrintf(&pszRepoDir, "%s/%s", - pszRootPath, pRepo->pszId); + pReposyncArgs->pszMetaDataPath ? + pReposyncArgs->pszMetaDataPath : pszRootPath, + pRepo->pszId); BAIL_ON_TDNF_ERROR(dwError); } else { - dwError = TDNFAllocateStringPrintf(&pszRepoDir, "%s/%s", - pReposyncArgs->pszMetaDataPath, pRepo->pszId); + dwError = TDNFAllocateString( + pReposyncArgs->pszMetaDataPath ? + pReposyncArgs->pszMetaDataPath : pszRootPath, + &pszRepoDir); BAIL_ON_TDNF_ERROR(dwError); } - dwError = TDNFUtilsMakeDir(pszRepoDir); BAIL_ON_TDNF_ERROR(dwError); - dwError = TDNFDownloadMetadata(pTdnf, pRepo, pszRepoDir); + dwError = TDNFDownloadMetadata(pTdnf, pRepo, pszRepoDir, + pReposyncArgs->nPrintUrlsOnly); BAIL_ON_TDNF_ERROR(dwError); TDNF_SAFE_FREE_MEMORY(pszRepoDir); @@ -1108,6 +1250,8 @@ TDNFRepoSync( TDNF_SAFE_FREE_MEMORY(pszDir); TDNF_SAFE_FREE_MEMORY(pszRepoDir); TDNF_SAFE_FREE_MEMORY(pszRootPath); + TDNF_SAFE_FREE_MEMORY(pszKeepFile); + TDNF_SAFE_FREE_MEMORY(pszFilePath); TDNFFreePackageInfoArray(pPkgInfos, dwCount); return dwError; error: diff --git a/client/prototypes.h b/client/prototypes.h index a4793abc..a58eea67 100644 --- a/client/prototypes.h +++ b/client/prototypes.h @@ -661,7 +661,8 @@ uint32_t TDNFDownloadMetadata( PTDNF pTdnf, PTDNF_REPO_DATA_INTERNAL pRepo, - const char *pszRepoDir + const char *pszRepoDir, + int nPrintOnly ); uint32_t @@ -669,7 +670,8 @@ TDNFDownloadRepoMDParts( PTDNF pTdnf, Repo *pSolvRepo, PTDNF_REPO_DATA_INTERNAL pRepo, - const char *pszDir + const char *pszDir, + int nPrintOnly ); //repolist.c diff --git a/client/repo.c b/client/repo.c index 8211b044..0f1213da 100644 --- a/client/repo.c +++ b/client/repo.c @@ -1332,7 +1332,8 @@ uint32_t TDNFDownloadMetadata( PTDNF pTdnf, PTDNF_REPO_DATA_INTERNAL pRepo, - const char *pszRepoDir + const char *pszRepoDir, + int nPrintOnly ) { uint32_t dwError = 0; @@ -1343,28 +1344,44 @@ TDNFDownloadMetadata( Pool *pPool = NULL; FILE *fp = NULL; - dwError = TDNFUtilsMakeDir(pszRepoDir); - BAIL_ON_TDNF_ERROR(dwError); - - dwError = TDNFAllocateStringPrintf(&pszRepoDataDir, "%s/repodata", - pszRepoDir); - BAIL_ON_TDNF_ERROR(dwError); - - dwError = TDNFUtilsMakeDir(pszRepoDataDir); - BAIL_ON_TDNF_ERROR(dwError); - - dwError = TDNFAllocateStringPrintf(&pszRepoMDPath, "%s/%s", - pszRepoDataDir, TDNF_REPO_METADATA_FILE_NAME); - BAIL_ON_TDNF_ERROR(dwError); - dwError = TDNFAllocateStringPrintf(&pszRepoMDUrl, "%s/%s", pRepo->pszBaseUrl, TDNF_REPO_METADATA_FILE_PATH); BAIL_ON_TDNF_ERROR(dwError); - dwError = TDNFDownloadFile(pTdnf, pRepo->pszId, pszRepoMDUrl, pszRepoMDPath, pRepo->pszId); - BAIL_ON_TDNF_ERROR(dwError); + if (!nPrintOnly) + { + dwError = TDNFUtilsMakeDir(pszRepoDir); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFAllocateStringPrintf(&pszRepoDataDir, "%s/repodata", + pszRepoDir); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFUtilsMakeDir(pszRepoDataDir); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFAllocateStringPrintf(&pszRepoMDPath, "%s/%s", + pszRepoDataDir, TDNF_REPO_METADATA_FILE_NAME); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFDownloadFile(pTdnf, pRepo->pszId, pszRepoMDUrl, pszRepoMDPath, pRepo->pszId); + BAIL_ON_TDNF_ERROR(dwError); + } + else + { + /* if printing only we use the already downloaded repomd.xml */ + dwError = TDNFAllocateStringPrintf( + &pszRepoMDPath, + "%s/%s/%s", + pTdnf->pConf->pszCacheDir, + pRepo->pszId, + TDNF_REPO_METADATA_FILE_PATH); + BAIL_ON_TDNF_ERROR(dwError); + + pr_info("%s\n", pszRepoMDUrl); + } pPool = pool_create(); pSolvRepo = repo_create(pPool, "md_parse_temp"); @@ -1384,7 +1401,7 @@ TDNFDownloadMetadata( pool_errstr(pPool)); } - dwError = TDNFDownloadRepoMDParts(pTdnf, pSolvRepo, pRepo, pszRepoDir); + dwError = TDNFDownloadRepoMDParts(pTdnf, pSolvRepo, pRepo, pszRepoDir, nPrintOnly); BAIL_ON_TDNF_ERROR(dwError); cleanup: @@ -1413,7 +1430,8 @@ TDNFDownloadRepoMDParts( PTDNF pTdnf, Repo *pSolvRepo, PTDNF_REPO_DATA_INTERNAL pRepo, - const char *pszDir + const char *pszDir, + int nPrintOnly ) { uint32_t dwError = 0; @@ -1465,8 +1483,15 @@ TDNFDownloadRepoMDParts( pszPartFile); BAIL_ON_TDNF_ERROR(dwError); - dwError = TDNFDownloadFile(pTdnf, pRepo->pszId, pszPartUrl, pszPartPath, pRepo->pszId); - BAIL_ON_TDNF_ERROR(dwError); + if (!nPrintOnly) + { + dwError = TDNFDownloadFile(pTdnf, pRepo->pszId, pszPartUrl, pszPartPath, pRepo->pszId); + BAIL_ON_TDNF_ERROR(dwError); + } + else + { + pr_info("%s\n", pszPartUrl); + } TDNF_SAFE_FREE_MEMORY(pszPartUrl); TDNF_SAFE_FREE_MEMORY(pszPartPath); diff --git a/include/tdnftypes.h b/include/tdnftypes.h index 28b5586d..c4f3725d 100644 --- a/include/tdnftypes.h +++ b/include/tdnftypes.h @@ -340,13 +340,12 @@ typedef struct _TDNF_REPOSYNC_ARGS int nGPGCheck; int nNewestOnly; int nPrintUrlsOnly; + int nNoRepoPath; char *pszDownloadPath; char *pszMetaDataPath; char **pszArchs; }TDNF_REPOSYNC_ARGS, *PTDNF_REPOSYNC_ARGS; - - #ifdef __cplusplus } #endif diff --git a/tools/cli/lib/parseargs.c b/tools/cli/lib/parseargs.c index c088f01c..9d894b6a 100644 --- a/tools/cli/lib/parseargs.c +++ b/tools/cli/lib/parseargs.c @@ -36,7 +36,7 @@ static SetOptArgs OptValTable[] = { {CMDOPT_DISABLEPLUGIN, "disableplugin", NULL}, {CMDOPT_KEYVALUE, "skipconflicts;skipobsoletes;skipsignature;skipdigest;" "noplugins;reboot-required;security;" - "delete;download-metadata;gpgcheck;newest-only;urls", + "delete;download-metadata;gpgcheck;newest-only;norepopath;urls", "1"} }; @@ -84,14 +84,15 @@ static struct option pstOptions[] = {"downloadonly", no_argument, &_opt.nDownloadOnly, 1}, //--downloadonly {"downloaddir", required_argument, 0, 0}, //--downloaddir // reposync options - {"arch", required_argument, 0, 'a'}, + {"arch", required_argument, 0, 0}, {"delete", no_argument, 0, 0}, {"download-metadata", no_argument, 0, 0}, - {"gpgcheck", no_argument, 0, 'g'}, + {"gpgcheck", no_argument, 0, 0}, {"metadata-path", required_argument, 0, 0}, - {"newest-only", no_argument, 0, 'n'}, + {"newest-only", no_argument, 0, 0}, + {"norepopath", no_argument, 0, 0}, {"download-path", required_argument, 0, 0}, - {"urls", no_argument, 0, 'u'}, + {"urls", no_argument, 0, 0}, {0, 0, 0, 0} }; diff --git a/tools/cli/lib/parsereposyncargs.c b/tools/cli/lib/parsereposyncargs.c index 11f05bc9..b28d6024 100644 --- a/tools/cli/lib/parsereposyncargs.c +++ b/tools/cli/lib/parsereposyncargs.c @@ -56,6 +56,10 @@ TDNFCliParseRepoSyncArgs( { pReposyncArgs->nNewestOnly = 1; } + else if (strcasecmp(pSetOpt->pszOptName, "norepopath") == 0) + { + pReposyncArgs->nNoRepoPath = 1; + } else if (strcasecmp(pSetOpt->pszOptName, "urls") == 0) { pReposyncArgs->nPrintUrlsOnly = 1; From 070c5be6dcef20b0563648807c3115a15ffe7cde Mon Sep 17 00:00:00 2001 From: Oliver Kurth Date: Fri, 30 Apr 2021 14:56:35 -0700 Subject: [PATCH 07/17] move gpg package check to new TDNFGPGCheckPackage() function --- client/gpgcheck.c | 264 ++++++++++++++++++++++++++++++++++++++++++++ client/prototypes.h | 17 +++ client/rpmtrans.c | 231 +------------------------------------- 3 files changed, 284 insertions(+), 228 deletions(-) diff --git a/client/gpgcheck.c b/client/gpgcheck.c index 0f76b203..c7b41c42 100644 --- a/client/gpgcheck.c +++ b/client/gpgcheck.c @@ -389,3 +389,267 @@ TDNFImportGPGKeyFile( goto cleanup; } +uint32_t +TDNFGPGCheckPackage( + PTDNFRPMTS pTS, + PTDNF pTdnf, + const char* pszRepoName, + const char* pszFilePath, + Header *pRpmHeader + ) +{ + uint32_t dwError = 0; + Header rpmHeader = NULL; + rpmKeyring pKeyring = NULL; + int nGPGSigCheck = 0; + FD_t fp = NULL; + char** ppszUrlGPGKeys = NULL; + char* pszLocalGPGKey = NULL; + int nAnswer = 0; + int nRemote = 0; + int i; + int nMatched = 0; + + dwError = TDNFGetGPGSignatureCheck(pTdnf, pszRepoName, &nGPGSigCheck, NULL); + BAIL_ON_TDNF_ERROR(dwError); + + fp = Fopen (pszFilePath, "r.ufdio"); + if(!fp) + { + dwError = errno; + BAIL_ON_TDNF_SYSTEM_ERROR(dwError); + } + + dwError = rpmReadPackageFile( + pTS->pTS, + fp, + pszFilePath, + &rpmHeader); + + Fclose(fp); + fp = NULL; + + if (dwError != RPMRC_NOTTRUSTED && dwError != RPMRC_NOKEY) + { + BAIL_ON_TDNF_RPM_ERROR(dwError); + } + else if(nGPGSigCheck) + { + dwError = TDNFGetGPGSignatureCheck(pTdnf, pszRepoName, &nGPGSigCheck, &ppszUrlGPGKeys); + BAIL_ON_TDNF_ERROR(dwError); + + for (i = 0; ppszUrlGPGKeys[i]; i++) { + pr_info("importing key from %s\n", ppszUrlGPGKeys[i]); + dwError = TDNFYesOrNo(pTdnf->pArgs, "Is this ok [y/N]: ", &nAnswer); + BAIL_ON_TDNF_ERROR(dwError); + + if(!nAnswer) + { + dwError = ERROR_TDNF_OPERATION_ABORTED; + BAIL_ON_TDNF_ERROR(dwError); + } + + dwError = TDNFUriIsRemote(ppszUrlGPGKeys[i], &nRemote); + if (dwError == ERROR_TDNF_URL_INVALID) + { + dwError = ERROR_TDNF_KEYURL_INVALID; + } + BAIL_ON_TDNF_ERROR(dwError); + + if (nRemote) + { + dwError = TDNFFetchRemoteGPGKey(pTdnf, pszRepoName, ppszUrlGPGKeys[i], &pszLocalGPGKey); + BAIL_ON_TDNF_ERROR(dwError); + } + else + { + dwError = TDNFPathFromUri(ppszUrlGPGKeys[i], &pszLocalGPGKey); + if (dwError == ERROR_TDNF_URL_INVALID) + { + dwError = ERROR_TDNF_KEYURL_INVALID; + } + BAIL_ON_TDNF_ERROR(dwError); + } + dwError = TDNFImportGPGKeyFile(pTS->pTS, pszLocalGPGKey); + BAIL_ON_TDNF_ERROR(dwError); + + pKeyring = rpmtsGetKeyring(pTS->pTS, 0); + dwError = TDNFGPGCheck(pKeyring, pszLocalGPGKey, pszFilePath); + if (dwError == 0) + { + nMatched++; + } + else if (dwError == ERROR_TDNF_RPM_GPG_NO_MATCH) + { + dwError = 0; + } + BAIL_ON_TDNF_ERROR(dwError); + + if (nRemote) + { + if (unlink(pszLocalGPGKey)) + { + dwError = errno; + BAIL_ON_TDNF_SYSTEM_ERROR(dwError); + } + } + } + + if (nMatched == 0) + { + dwError = ERROR_TDNF_RPM_GPG_NO_MATCH; + BAIL_ON_TDNF_ERROR(dwError); + } + fp = Fopen (pszFilePath, "r.ufdio"); + if(!fp) + { + dwError = errno; + BAIL_ON_TDNF_SYSTEM_ERROR(dwError); + } + + dwError = rpmReadPackageFile( + pTS->pTS, + fp, + pszFilePath, + &rpmHeader); + + BAIL_ON_TDNF_RPM_ERROR(dwError); + + Fclose(fp); + fp = NULL; + } + else + { + dwError = 0; + } + + /* optional output parameter */ + if (pRpmHeader != NULL) + { + *pRpmHeader = rpmHeader; + } + else if(rpmHeader) + { + headerFree(rpmHeader); + } + +cleanup: + TDNF_SAFE_FREE_STRINGARRAY(ppszUrlGPGKeys); + TDNF_SAFE_FREE_MEMORY(pszLocalGPGKey); + if(fp) + { + Fclose(fp); + } + return dwError; + +error: + if(rpmHeader) + { + headerFree(rpmHeader); + } + goto cleanup; +} + +uint32_t +TDNFFetchRemoteGPGKey( + PTDNF pTdnf, + const char* pszRepoName, + const char* pszUrlGPGKey, + char** ppszKeyLocation + ) +{ + uint32_t dwError = 0; + char* pszFilePath = NULL; + char* pszNormalPath = NULL; + char* pszFilePathCopy = NULL; + char* pszTopKeyCacheDir = NULL; + char* pszRealTopKeyCacheDir = NULL; + char* pszDownloadCacheDir = NULL; + char* pszKeyLocation = NULL; + + if(!pTdnf || IsNullOrEmptyString(pszRepoName || IsNullOrEmptyString(pszUrlGPGKey))) + { + dwError = ERROR_TDNF_INVALID_PARAMETER; + BAIL_ON_TDNF_ERROR(dwError); + } + + dwError = TDNFPathFromUri(pszUrlGPGKey, &pszKeyLocation); + if (dwError == ERROR_TDNF_URL_INVALID) + { + dwError = ERROR_TDNF_KEYURL_INVALID; + } + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFAllocateStringPrintf( + &pszTopKeyCacheDir, + "%s/%s/keys", + pTdnf->pConf->pszCacheDir, + pszRepoName); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFNormalizePath(pszTopKeyCacheDir, + &pszRealTopKeyCacheDir); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFAllocateStringPrintf( + &pszFilePath, + "%s/%s", + pszRealTopKeyCacheDir, + pszKeyLocation); + BAIL_ON_TDNF_ERROR(dwError); + + dwError = TDNFNormalizePath( + pszFilePath, + &pszNormalPath); + BAIL_ON_TDNF_ERROR(dwError); + + if (strncmp(pszRealTopKeyCacheDir, pszNormalPath, strlen(pszRealTopKeyCacheDir))) + { + dwError = ERROR_TDNF_KEYURL_INVALID; + BAIL_ON_TDNF_ERROR(dwError); + } + + // dirname() may modify the contents of path, so it may be desirable to + // pass a copy when calling this function. + dwError = TDNFAllocateString(pszNormalPath, &pszFilePathCopy); + BAIL_ON_TDNF_ERROR(dwError); + pszDownloadCacheDir = dirname(pszFilePathCopy); + if(!pszDownloadCacheDir) + { + dwError = ENOENT; + BAIL_ON_TDNF_SYSTEM_ERROR(dwError); + } + + if(access(pszDownloadCacheDir, F_OK)) + { + if(errno != ENOENT) + { + dwError = errno; + } + BAIL_ON_TDNF_SYSTEM_ERROR(dwError); + + dwError = TDNFUtilsMakeDirs(pszDownloadCacheDir); + BAIL_ON_TDNF_ERROR(dwError); + } + + dwError = TDNFDownloadFile(pTdnf, pszRepoName, pszUrlGPGKey, pszFilePath, + basename(pszFilePath)); + BAIL_ON_TDNF_ERROR(dwError); + + *ppszKeyLocation = pszNormalPath; + +cleanup: + TDNF_SAFE_FREE_MEMORY(pszFilePath); + TDNF_SAFE_FREE_MEMORY(pszRealTopKeyCacheDir); + TDNF_SAFE_FREE_MEMORY(pszTopKeyCacheDir); + TDNF_SAFE_FREE_MEMORY(pszFilePathCopy); + TDNF_SAFE_FREE_MEMORY(pszKeyLocation); + return dwError; + +error: + pr_err("Error processing key: %s\n", pszUrlGPGKey); + TDNF_SAFE_FREE_MEMORY(pszNormalPath); + *ppszKeyLocation = NULL; + goto cleanup; +} + diff --git a/client/prototypes.h b/client/prototypes.h index a58eea67..82497893 100644 --- a/client/prototypes.h +++ b/client/prototypes.h @@ -92,6 +92,23 @@ TDNFImportGPGKeyFile( const char* pszFile ); +uint32_t +TDNFGPGCheckPackage( + PTDNFRPMTS pTS, + PTDNF pTdnf, + const char* pszRepoName, + const char* pszFilePath, + Header *pRpmHeader + ); + +uint32_t +TDNFFetchRemoteGPGKey( + PTDNF pTdnf, + const char* pszRepoName, + const char* pszUrlGPGKey, + char** ppszKeyLocation + ); + //init.c uint32_t TDNFCloneCmdArgs( diff --git a/client/rpmtrans.c b/client/rpmtrans.c index 9ee0d999..8fa055a5 100644 --- a/client/rpmtrans.c +++ b/client/rpmtrans.c @@ -493,109 +493,6 @@ TDNFTransAddReInstallPkgs( return TDNFTransAddInstallPkgs(pTS, pTdnf, pInfo); } -uint32_t -TDNFFetchRemoteGPGKey( - PTDNF pTdnf, - const char* pszRepoName, - const char* pszUrlGPGKey, - char** ppszKeyLocation - ) -{ - uint32_t dwError = 0; - char* pszFilePath = NULL; - char* pszNormalPath = NULL; - char* pszFilePathCopy = NULL; - char* pszTopKeyCacheDir = NULL; - char* pszRealTopKeyCacheDir = NULL; - char* pszDownloadCacheDir = NULL; - char* pszKeyLocation = NULL; - - if(!pTdnf || IsNullOrEmptyString(pszRepoName || IsNullOrEmptyString(pszUrlGPGKey))) - { - dwError = ERROR_TDNF_INVALID_PARAMETER; - BAIL_ON_TDNF_ERROR(dwError); - } - - dwError = TDNFPathFromUri(pszUrlGPGKey, &pszKeyLocation); - if (dwError == ERROR_TDNF_URL_INVALID) - { - dwError = ERROR_TDNF_KEYURL_INVALID; - } - BAIL_ON_TDNF_ERROR(dwError); - - dwError = TDNFAllocateStringPrintf( - &pszTopKeyCacheDir, - "%s/%s/keys", - pTdnf->pConf->pszCacheDir, - pszRepoName); - BAIL_ON_TDNF_ERROR(dwError); - - dwError = TDNFNormalizePath(pszTopKeyCacheDir, - &pszRealTopKeyCacheDir); - BAIL_ON_TDNF_ERROR(dwError); - - dwError = TDNFAllocateStringPrintf( - &pszFilePath, - "%s/%s", - pszRealTopKeyCacheDir, - pszKeyLocation); - BAIL_ON_TDNF_ERROR(dwError); - - dwError = TDNFNormalizePath( - pszFilePath, - &pszNormalPath); - BAIL_ON_TDNF_ERROR(dwError); - - if (strncmp(pszRealTopKeyCacheDir, pszNormalPath, strlen(pszRealTopKeyCacheDir))) - { - dwError = ERROR_TDNF_KEYURL_INVALID; - BAIL_ON_TDNF_ERROR(dwError); - } - - // dirname() may modify the contents of path, so it may be desirable to - // pass a copy when calling this function. - dwError = TDNFAllocateString(pszNormalPath, &pszFilePathCopy); - BAIL_ON_TDNF_ERROR(dwError); - pszDownloadCacheDir = dirname(pszFilePathCopy); - if(!pszDownloadCacheDir) - { - dwError = ENOENT; - BAIL_ON_TDNF_SYSTEM_ERROR(dwError); - } - - if(access(pszDownloadCacheDir, F_OK)) - { - if(errno != ENOENT) - { - dwError = errno; - } - BAIL_ON_TDNF_SYSTEM_ERROR(dwError); - - dwError = TDNFUtilsMakeDirs(pszDownloadCacheDir); - BAIL_ON_TDNF_ERROR(dwError); - } - - dwError = TDNFDownloadFile(pTdnf, pszRepoName, pszUrlGPGKey, pszFilePath, - basename(pszFilePath)); - BAIL_ON_TDNF_ERROR(dwError); - - *ppszKeyLocation = pszNormalPath; - -cleanup: - TDNF_SAFE_FREE_MEMORY(pszFilePath); - TDNF_SAFE_FREE_MEMORY(pszRealTopKeyCacheDir); - TDNF_SAFE_FREE_MEMORY(pszTopKeyCacheDir); - TDNF_SAFE_FREE_MEMORY(pszFilePathCopy); - TDNF_SAFE_FREE_MEMORY(pszKeyLocation); - return dwError; - -error: - pr_err("Error processing key: %s\n", pszUrlGPGKey); - TDNF_SAFE_FREE_MEMORY(pszNormalPath); - *ppszKeyLocation = NULL; - goto cleanup; -} - uint32_t TDNFTransAddInstallPkg( PTDNFRPMTS pTS, @@ -608,18 +505,9 @@ TDNFTransAddInstallPkg( { uint32_t dwError = 0; int nGPGCheck = 0; - int nGPGSigCheck = 0; char* pszFilePath = NULL; Header rpmHeader = NULL; - FD_t fp = NULL; - char** ppszUrlGPGKeys = NULL; - char* pszLocalGPGKey = NULL; PTDNF_CACHED_RPM_ENTRY pRpmCache = NULL; - rpmKeyring pKeyring = NULL; - int nAnswer = 0; - int nRemote = 0; - int i; - int nMatched = 0; if (pszPackageLocation[0] == '/') { @@ -664,117 +552,9 @@ TDNFTransAddInstallPkg( BAIL_ON_TDNF_SYSTEM_ERROR(dwError); } - dwError = TDNFGetGPGSignatureCheck(pTdnf, pszRepoName, &nGPGSigCheck, NULL); + dwError = TDNFGPGCheckPackage(pTS, pTdnf, pszRepoName, pszFilePath, &rpmHeader); BAIL_ON_TDNF_ERROR(dwError); - fp = Fopen (pszFilePath, "r.ufdio"); - if(!fp) - { - dwError = errno; - BAIL_ON_TDNF_SYSTEM_ERROR(dwError); - } - - dwError = rpmReadPackageFile( - pTS->pTS, - fp, - pszFilePath, - &rpmHeader); - - Fclose(fp); - fp = NULL; - - if (dwError != RPMRC_NOTTRUSTED && dwError != RPMRC_NOKEY) - { - BAIL_ON_TDNF_RPM_ERROR(dwError); - } - else if(nGPGSigCheck) - { - dwError = TDNFGetGPGSignatureCheck(pTdnf, pszRepoName, &nGPGSigCheck, &ppszUrlGPGKeys); - BAIL_ON_TDNF_ERROR(dwError); - - for (i = 0; ppszUrlGPGKeys[i]; i++) { - pr_info("importing key from %s\n", ppszUrlGPGKeys[i]); - dwError = TDNFYesOrNo(pTdnf->pArgs, "Is this ok [y/N]: ", &nAnswer); - BAIL_ON_TDNF_ERROR(dwError); - - if(!nAnswer) - { - dwError = ERROR_TDNF_OPERATION_ABORTED; - BAIL_ON_TDNF_ERROR(dwError); - } - - dwError = TDNFUriIsRemote(ppszUrlGPGKeys[i], &nRemote); - if (dwError == ERROR_TDNF_URL_INVALID) - { - dwError = ERROR_TDNF_KEYURL_INVALID; - } - BAIL_ON_TDNF_ERROR(dwError); - - if (nRemote) - { - dwError = TDNFFetchRemoteGPGKey(pTdnf, pszRepoName, ppszUrlGPGKeys[i], &pszLocalGPGKey); - BAIL_ON_TDNF_ERROR(dwError); - } - else - { - dwError = TDNFPathFromUri(ppszUrlGPGKeys[i], &pszLocalGPGKey); - if (dwError == ERROR_TDNF_URL_INVALID) - { - dwError = ERROR_TDNF_KEYURL_INVALID; - } - BAIL_ON_TDNF_ERROR(dwError); - } - - dwError = TDNFImportGPGKeyFile(pTS->pTS, pszLocalGPGKey); - BAIL_ON_TDNF_ERROR(dwError); - - pKeyring = rpmtsGetKeyring(pTS->pTS, 0); - dwError = TDNFGPGCheck(pKeyring, pszLocalGPGKey, pszFilePath); - if (dwError == 0) - { - nMatched++; - } - else if (dwError == ERROR_TDNF_RPM_GPG_NO_MATCH) - { - dwError = 0; - } - BAIL_ON_TDNF_ERROR(dwError); - - if (nRemote) - { - if (unlink(pszLocalGPGKey)) - { - dwError = errno; - BAIL_ON_TDNF_SYSTEM_ERROR(dwError); - } - } - } - - if (nMatched == 0) - { - dwError = ERROR_TDNF_RPM_GPG_NO_MATCH; - BAIL_ON_TDNF_ERROR(dwError); - } - - fp = Fopen (pszFilePath, "r.ufdio"); - if(!fp) - { - dwError = errno; - BAIL_ON_TDNF_SYSTEM_ERROR(dwError); - } - - dwError = rpmReadPackageFile( - pTS->pTS, - fp, - pszFilePath, - &rpmHeader); - - BAIL_ON_TDNF_RPM_ERROR(dwError); - - Fclose(fp); - fp = NULL; - } - dwError = TDNFGetGPGCheck(pTdnf, pszRepoName, &nGPGCheck); BAIL_ON_TDNF_ERROR(dwError); if (!nGPGCheck) @@ -791,7 +571,7 @@ TDNFTransAddInstallPkg( NULL); BAIL_ON_TDNF_RPM_ERROR(dwError); - /* add to cached array only when file is actully in cache dir */ + /* add to cached array only when file is actually in cache dir */ if(pTS->pCachedRpmsArray && !strncmp(pszFilePath, pTdnf->pConf->pszCacheDir, strlen(pTdnf->pConf->pszCacheDir))) @@ -805,13 +585,8 @@ TDNFTransAddInstallPkg( pRpmCache->pNext = pTS->pCachedRpmsArray->pHead; pTS->pCachedRpmsArray->pHead = pRpmCache; } + cleanup: - TDNF_SAFE_FREE_STRINGARRAY(ppszUrlGPGKeys); - TDNF_SAFE_FREE_MEMORY(pszLocalGPGKey); - if(fp) - { - Fclose(fp); - } if(rpmHeader) { headerFree(rpmHeader); From eb590a3fe28818300951b6695f97ba873e69f6ed Mon Sep 17 00:00:00 2001 From: Oliver Kurth Date: Fri, 30 Apr 2021 17:05:57 -0700 Subject: [PATCH 08/17] reposync: implement gpgcheck option --- client/api.c | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/client/api.c b/client/api.c index 1e45e743..278119ae 100644 --- a/client/api.c +++ b/client/api.c @@ -1006,6 +1006,7 @@ _rm_rpms( { if (errno == ENOENT) { + pr_info("deleting %s\n", pszFilePath); if(remove(pszFilePath) < 0) { pr_crit("unable to remove %s: %s\n", pszFilePath, strerror(errno)); @@ -1053,6 +1054,7 @@ TDNFRepoSync( char *pszKeepFile = NULL; uint32_t dwCount = 0; uint32_t dwRepoCount = 0; + TDNFRPMTS ts = {0}; if(!pTdnf || !pTdnf->pSack) { @@ -1103,6 +1105,16 @@ TDNFRepoSync( dwError = TDNFPopulatePkgInfoForRepoSync(pTdnf->pSack, pPkgList, &pPkgInfos); BAIL_ON_TDNF_ERROR(dwError); + if (pReposyncArgs->nGPGCheck) + { + ts.pTS = rpmtsCreate(); + if(!ts.pTS) + { + dwError = ERROR_TDNF_RPMTS_CREATE_FAILED; + BAIL_ON_TDNF_ERROR(dwError); + } + } + if (pReposyncArgs->pszDownloadPath == NULL) { pszRootPath = getcwd(NULL, 0); @@ -1148,9 +1160,23 @@ TDNFRepoSync( &pszFilePath); BAIL_ON_TDNF_ERROR(dwError); - if (pReposyncArgs->nDelete) + if (pReposyncArgs->nGPGCheck) + { + dwError = TDNFGPGCheckPackage(&ts, pTdnf, pPkgInfo->pszRepoName, pszFilePath, NULL); + if (dwError) + { + pr_crit("checking package %s failed: %d, deleting\n", pszFilePath, dwError); + if(remove(pszFilePath) < 0) + { + pr_crit("unable to remove %s: %s\n", pszFilePath, strerror(errno)); + } + } + } + + /* dwError==0 means TDNFGPGCheckPackage() succeeded or wasn't called */ + if (pReposyncArgs->nDelete && dwError == 0) { - /* if deleting, create a marker file to protect + /* if "delete" option is given, create a marker file to protect what we just downloaded. Later all *.rpm files that do not have a marker file will be deleted */ dwError = TDNFAllocateStringPrintf(&pszKeepFile, "%s.reposync-keep", pszFilePath); @@ -1160,6 +1186,7 @@ TDNFRepoSync( BAIL_ON_TDNF_ERROR(dwError); TDNF_SAFE_FREE_MEMORY(pszKeepFile); } + dwError = 0; TDNF_SAFE_FREE_MEMORY(pszDir); TDNF_SAFE_FREE_MEMORY(pszFilePath); @@ -1247,6 +1274,11 @@ TDNFRepoSync( { SolvFreePackageList(pPkgList); } + if(ts.pTS) + { + rpmtsCloseDB(ts.pTS); + rpmtsFree(ts.pTS); + } TDNF_SAFE_FREE_MEMORY(pszDir); TDNF_SAFE_FREE_MEMORY(pszRepoDir); TDNF_SAFE_FREE_MEMORY(pszRootPath); From 69574a072784355da7dec1ad35d4423a69556c92 Mon Sep 17 00:00:00 2001 From: Oliver Kurth Date: Mon, 3 May 2021 15:26:42 -0700 Subject: [PATCH 09/17] fix indentation in SolvApplyListQuery() --- solv/tdnfquery.c | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/solv/tdnfquery.c b/solv/tdnfquery.c index cc5d645c..33a0c9cb 100644 --- a/solv/tdnfquery.c +++ b/solv/tdnfquery.c @@ -674,36 +674,36 @@ SolvApplyListQuery( Pool *pool = pQuery->pSack->pPool; if (how == SOLVER_SOLVABLE_ALL) { - FOR_POOL_SOLVABLES(p) - { - if(is_pseudo_package(pool, &pool->solvables[p])) - continue; - queue_push(&queueTmp, p); - } + FOR_POOL_SOLVABLES(p) + { + if(is_pseudo_package(pool, &pool->solvables[p])) + continue; + queue_push(&queueTmp, p); + } } else if (how == SOLVER_SOLVABLE_REPO) { - Repo *repo = pool_id2repo(pool, what); - if (repo) - { - Solvable *s = NULL; - - FOR_REPO_SOLVABLES(repo, p, s) - { - if (is_pseudo_package(pool, &pool->solvables[p])) - continue; - queue_push(&queueTmp, p); - } - } + Repo *repo = pool_id2repo(pool, what); + if (repo) + { + Solvable *s = NULL; + + FOR_REPO_SOLVABLES(repo, p, s) + { + if (is_pseudo_package(pool, &pool->solvables[p])) + continue; + queue_push(&queueTmp, p); + } + } } else { - FOR_JOB_SELECT(p, pp, how, what) - { - if (is_pseudo_package(pool, &pool->solvables[p])) - continue; - queue_push(&queueTmp, p); - } + FOR_JOB_SELECT(p, pp, how, what) + { + if (is_pseudo_package(pool, &pool->solvables[p])) + continue; + queue_push(&queueTmp, p); + } } queue_insertn(&pQuery->queueResult, pQuery->queueResult.count, From 9089cce07161f5678da776bf1fc4f30f3ca90a0c Mon Sep 17 00:00:00 2001 From: Oliver Kurth Date: Fri, 7 May 2021 14:27:26 -0700 Subject: [PATCH 10/17] reposync: fix various nits --- client/api.c | 8 ++++---- client/utils.c | 6 ++---- pytests/conftest.py | 9 +++++---- tools/cli/lib/parsereposyncargs.c | 6 ++++++ tools/cli/main.c | 1 - 5 files changed, 17 insertions(+), 13 deletions(-) diff --git a/client/api.c b/client/api.c index 278119ae..d086e1b8 100644 --- a/client/api.c +++ b/client/api.c @@ -1056,7 +1056,7 @@ TDNFRepoSync( uint32_t dwRepoCount = 0; TDNFRPMTS ts = {0}; - if(!pTdnf || !pTdnf->pSack) + if(!pTdnf || !pTdnf->pSack || !pReposyncArgs) { dwError = ERROR_TDNF_INVALID_PARAMETER; BAIL_ON_TDNF_ERROR(dwError); @@ -1194,9 +1194,9 @@ TDNFRepoSync( else { dwError = TDNFCreatePackageUrl(pTdnf, - pPkgInfo->pszRepoName, - pPkgInfo->pszLocation, - &pszUrl); + pPkgInfo->pszRepoName, + pPkgInfo->pszLocation, + &pszUrl); BAIL_ON_TDNF_ERROR(dwError); pr_info("%s\n", pszUrl); diff --git a/client/utils.c b/client/utils.c index c1d9993f..8b84f40c 100644 --- a/client/utils.c +++ b/client/utils.c @@ -307,10 +307,8 @@ TDNFGetFileSize( if(stat(pszPath, &stStat)) { - { - dwError = errno; - BAIL_ON_TDNF_SYSTEM_ERROR(dwError); - } + dwError = errno; + BAIL_ON_TDNF_SYSTEM_ERROR(dwError); } else { diff --git a/pytests/conftest.py b/pytests/conftest.py index c7736485..fd539065 100644 --- a/pytests/conftest.py +++ b/pytests/conftest.py @@ -241,16 +241,17 @@ def run_memcheck(self, cmd): '--error-exitcode=1'] return self._run(memcheck_cmd + cmd, retvalonly=True) - def run(self, cmd): + def run(self, cmd, cwd=None): self._decorate_tdnf_cmd_for_test(cmd) - return self._run(cmd) + return self._run(cmd, cwd=cwd) - def _run(self, cmd, retvalonly=False): + def _run(self, cmd, retvalonly=False, cwd=None): use_shell = not isinstance(cmd, list) print(cmd) process = subprocess.Popen(cmd, shell=use_shell, #nosec stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + cwd=cwd) #process.wait() out, err = process.communicate() if retvalonly: diff --git a/tools/cli/lib/parsereposyncargs.c b/tools/cli/lib/parsereposyncargs.c index b28d6024..6095d381 100644 --- a/tools/cli/lib/parsereposyncargs.c +++ b/tools/cli/lib/parsereposyncargs.c @@ -28,6 +28,12 @@ TDNFCliParseRepoSyncArgs( PTDNF_REPOSYNC_ARGS pReposyncArgs = NULL; PTDNF_CMD_OPT pSetOpt = NULL; + if (!pArgs || !ppReposyncArgs) + { + dwError = ERROR_TDNF_CLI_INVALID_ARGUMENT; + BAIL_ON_CLI_ERROR(dwError); + } + dwError = TDNFAllocateMemory( 1, sizeof(TDNF_REPOSYNC_ARGS), diff --git a/tools/cli/main.c b/tools/cli/main.c index e3731d39..add5f158 100644 --- a/tools/cli/main.c +++ b/tools/cli/main.c @@ -56,7 +56,6 @@ int main(int argc, char* argv[]) {"remove", TDNFCliEraseCommand}, {"repolist", TDNFCliRepoListCommand}, {"reposync", TDNFCliRepoSyncCommand}, - {"provides", TDNFCliProvidesCommand}, {"search", TDNFCliSearchCommand}, {"update", TDNFCliUpgradeCommand}, {"update-to", TDNFCliUpgradeCommand}, From 3edb3e0f3f17d34009ddf348d8d5277e03dad14e Mon Sep 17 00:00:00 2001 From: Oliver Kurth Date: Wed, 12 May 2021 10:16:21 -0700 Subject: [PATCH 11/17] reposync: add TDNFDownloadPackageToTree() function --- client/api.c | 4 +-- client/prototypes.h | 10 +++++++ client/remoterepo.c | 69 +++++++++++++++++++++++++++++++++++++++------ 3 files changed, 73 insertions(+), 10 deletions(-) diff --git a/client/api.c b/client/api.c index d086e1b8..c86738f3 100644 --- a/client/api.c +++ b/client/api.c @@ -1125,7 +1125,7 @@ TDNFRepoSync( } else { - dwError = TDNFAllocateString(pReposyncArgs->pszDownloadPath, &pszRootPath); + dwError = TDNFNormalizePath(pReposyncArgs->pszDownloadPath, &pszRootPath); BAIL_ON_TDNF_ERROR(dwError); } @@ -1154,7 +1154,7 @@ TDNFRepoSync( dwError = TDNFUtilsMakeDir(pszDir); BAIL_ON_TDNF_ERROR(dwError); - dwError = TDNFDownloadPackageToDirectory(pTdnf, + dwError = TDNFDownloadPackageToTree(pTdnf, pPkgInfo->pszLocation, pPkgInfo->pszName, pPkgInfo->pszRepoName, pszDir, &pszFilePath); diff --git a/client/prototypes.h b/client/prototypes.h index 82497893..0451f490 100644 --- a/client/prototypes.h +++ b/client/prototypes.h @@ -291,6 +291,16 @@ TDNFDownloadPackageToCache( char** ppszFilePath ); +uint32_t +TDNFDownloadPackageToTree( + PTDNF pTdnf, + const char* pszPackageLocation, + const char* pszPkgName, + const char* pszRepoName, + char* pszNormalRpmCacheDir, + char** ppszFilePath + ); + uint32_t TDNFDownloadPackageToDirectory( PTDNF pTdnf, diff --git a/client/remoterepo.c b/client/remoterepo.c index b5e4052d..a504cd3b 100644 --- a/client/remoterepo.c +++ b/client/remoterepo.c @@ -992,11 +992,6 @@ TDNFDownloadPackageToCache( uint32_t dwError = 0; char* pszRpmCacheDir = NULL; char* pszNormalRpmCacheDir = NULL; - char* pszFilePath = NULL; - char* pszNormalPath = NULL; - char* pszFilePathCopy = NULL; - char* pszDownloadCacheDir = NULL; - char* pszRemotePath = NULL; if(!pTdnf || IsNullOrEmptyString(pszPackageLocation) || @@ -1020,6 +1015,58 @@ TDNFDownloadPackageToCache( &pszNormalRpmCacheDir); BAIL_ON_TDNF_ERROR(dwError); + dwError = TDNFDownloadPackageToTree(pTdnf, + pszPackageLocation, + pszPkgName, + pszRepoName, + pszNormalRpmCacheDir, + ppszFilePath); + BAIL_ON_TDNF_ERROR(dwError); +cleanup: + TDNF_SAFE_FREE_MEMORY(pszNormalRpmCacheDir); + TDNF_SAFE_FREE_MEMORY(pszRpmCacheDir); + return dwError; +error: + goto cleanup; +} + +/* + * TDNFDownloadPackageToTree() + * + * Download a package while preserving the directory path. For example, + * if pszPackageLocation is "RPMS/x86_64/foo-1.2-3.rpm, the destination will + * be downloaded under the destination directory in RPMS/x86_64/foo-1.2-3.rpm + * (so 'RPMS/x86_64/' will be preserved). +*/ + +uint32_t +TDNFDownloadPackageToTree( + PTDNF pTdnf, + const char* pszPackageLocation, + const char* pszPkgName, + const char* pszRepoName, + char* pszNormalRpmCacheDir, + char** ppszFilePath + ) +{ + uint32_t dwError = 0; + char* pszFilePath = NULL; + char* pszNormalPath = NULL; + char* pszFilePathCopy = NULL; + char* pszDownloadCacheDir = NULL; + char* pszRemotePath = NULL; + + if(!pTdnf || + IsNullOrEmptyString(pszPackageLocation) || + IsNullOrEmptyString(pszPkgName) || + IsNullOrEmptyString(pszRepoName) || + IsNullOrEmptyString(pszNormalRpmCacheDir) || + !ppszFilePath) + { + dwError = ERROR_TDNF_INVALID_PARAMETER; + BAIL_ON_TDNF_ERROR(dwError); + } + dwError = TDNFPathFromUri(pszPackageLocation, &pszRemotePath); if (dwError == ERROR_TDNF_URL_INVALID) { @@ -1030,7 +1077,7 @@ TDNFDownloadPackageToCache( dwError = TDNFAllocateStringPrintf( &pszFilePath, "%s/%s", - pszRpmCacheDir, + pszNormalRpmCacheDir, pszRemotePath); BAIL_ON_TDNF_ERROR(dwError); @@ -1085,8 +1132,6 @@ TDNFDownloadPackageToCache( cleanup: TDNF_SAFE_FREE_MEMORY(pszFilePath); TDNF_SAFE_FREE_MEMORY(pszFilePathCopy); - TDNF_SAFE_FREE_MEMORY(pszRpmCacheDir); - TDNF_SAFE_FREE_MEMORY(pszNormalRpmCacheDir); TDNF_SAFE_FREE_MEMORY(pszRemotePath); return dwError; @@ -1096,6 +1141,14 @@ TDNFDownloadPackageToCache( } +/* + * TDNFDownloadPackageToDirectory() + * + * Download a package withou preserving the directory path. For example, + * if pszPackageLocation is "RPMS/x86_64/foo-1.2-3.rpm, the destination will + * be downloaded under the destination directory (pszDirectory) as foo-1.2-3.rpm + * (so RPMS/x86_64/ will be stripped). +*/ uint32_t TDNFDownloadPackageToDirectory( From a9eee721f37ac735c1ab6c74655d983e58a2d425 Mon Sep 17 00:00:00 2001 From: Oliver Kurth Date: Wed, 19 May 2021 11:19:16 -0700 Subject: [PATCH 12/17] add TDNFStringMatchesOneOf() function --- common/prototypes.h | 6 ++++++ common/utils.c | 28 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/common/prototypes.h b/common/prototypes.h index 05fed621..272b614b 100644 --- a/common/prototypes.h +++ b/common/prototypes.h @@ -232,6 +232,12 @@ TDNFRecursivelyRemoveDir( const char *pszPath ); +uint32_t +TDNFStringMatchesOneOf( + const char *pszSearch, + char **ppszList, + int *pRet); + //setopt.c uint32_t AddSetOpt( diff --git a/common/utils.c b/common/utils.c index 70a061d8..2a6d2f36 100644 --- a/common/utils.c +++ b/common/utils.c @@ -731,3 +731,31 @@ TDNFRecursivelyRemoveDir(const char *pszPath) error: goto cleanup; } + +/* search pszSearch in the string ppszList, result will be in pRet */ +uint32_t +TDNFStringMatchesOneOf(const char *pszSearch, char **ppszList, int *pRet) +{ + int i; + uint32_t dwError = 0; + + if (IsNullOrEmptyString(pszSearch) || !ppszList || !pRet) + { + dwError = ERROR_TDNF_INVALID_PARAMETER; + BAIL_ON_TDNF_ERROR(dwError); + } + + *pRet = 0; + for(i = 0; ppszList[i]; i++) + { + if (strcmp(pszSearch, ppszList[i]) == 0) + { + *pRet = 1; + goto cleanup; + } + } +cleanup: + return dwError; +error: + goto cleanup; +} From cede8c89185eadf150c7cccd04a9b5b6791a3906 Mon Sep 17 00:00:00 2001 From: Oliver Kurth Date: Thu, 20 May 2021 10:52:34 -0700 Subject: [PATCH 13/17] reposync: arch and source options, TDNFCliFreeRepoSyncArgs() --- client/api.c | 33 ++++++++++++++++++++++++ include/tdnfcli.h | 5 ++++ include/tdnftypes.h | 5 +++- tools/cli/lib/api.c | 2 ++ tools/cli/lib/parseargs.c | 6 +++-- tools/cli/lib/parsereposyncargs.c | 42 ++++++++++++++++++++++++++++++- 6 files changed, 89 insertions(+), 4 deletions(-) diff --git a/client/api.c b/client/api.c index c86738f3..c934d1b7 100644 --- a/client/api.c +++ b/client/api.c @@ -1062,6 +1062,7 @@ TDNFRepoSync( BAIL_ON_TDNF_ERROR(dwError); } + /* count enabled repos */ for (pRepo = pTdnf->pRepos; pRepo; pRepo = pRepo->pNext) { if ((strcmp(pRepo->pszName, "@cmdline") == 0) || @@ -1087,9 +1088,18 @@ TDNFRepoSync( BAIL_ON_TDNF_ERROR(dwError); } + if (pReposyncArgs->nSourceOnly && pReposyncArgs->ppszArchs) + { + pr_crit("cannot use the source option with arch\n"); + dwError = ERROR_TDNF_INVALID_PARAMETER; + BAIL_ON_TDNF_ERROR(dwError); + } + dwError = TDNFRefresh(pTdnf); BAIL_ON_TDNF_ERROR(dwError); + /* generate list of packages, result will be + in pPkgInfos */ dwError = SolvCreateQuery(pTdnf->pSack, &pQuery); BAIL_ON_TDNF_ERROR(dwError); @@ -1129,6 +1139,7 @@ TDNFRepoSync( BAIL_ON_TDNF_ERROR(dwError); } + /* iterate through all packages */ for (pPkgInfo = pPkgInfos; pPkgInfo; pPkgInfo = pPkgInfo->pNext) { dwCount++; @@ -1136,6 +1147,22 @@ TDNFRepoSync( { continue; } + if (pReposyncArgs->ppszArchs) + { + int result = 0; + TDNFStringMatchesOneOf(pPkgInfo->pszArch, pReposyncArgs->ppszArchs, &result); + if (result == 0) + { + continue; + } + } + else if (pReposyncArgs->nSourceOnly) + { + if (strcmp(pPkgInfo->pszArch, "src") != 0) + { + continue; + } + } if (!pReposyncArgs->nPrintUrlsOnly) { @@ -1160,6 +1187,8 @@ TDNFRepoSync( &pszFilePath); BAIL_ON_TDNF_ERROR(dwError); + /* if gpgcheck option is given, check for a valid signature. If that fails, + delete the package */ if (pReposyncArgs->nGPGCheck) { dwError = TDNFGPGCheckPackage(&ts, pTdnf, pPkgInfo->pszRepoName, pszFilePath, NULL); @@ -1193,6 +1222,7 @@ TDNFRepoSync( } else { + /* print URLs only */ dwError = TDNFCreatePackageUrl(pTdnf, pPkgInfo->pszRepoName, pPkgInfo->pszLocation, @@ -1207,6 +1237,9 @@ TDNFRepoSync( if (pReposyncArgs->nDelete) { + /* go through all packages in the destination directory, + delete those that were not just downloaded as indicated by the + marker file */ for (pRepo = pTdnf->pRepos; pRepo; pRepo = pRepo->pNext) { if ((strcmp(pRepo->pszName, "@cmdline") == 0) || diff --git a/include/tdnfcli.h b/include/tdnfcli.h index f45c589b..31c502cb 100644 --- a/include/tdnfcli.h +++ b/include/tdnfcli.h @@ -122,6 +122,11 @@ TDNFCliRefresh( PTDNF_CLI_CONTEXT pContext ); +void +TDNFCliFreeRepoSyncArgs( + PTDNF_REPOSYNC_ARGS pReposyncArgs + ); + //Commands uint32_t TDNFCliAutoEraseCommand( diff --git a/include/tdnftypes.h b/include/tdnftypes.h index c4f3725d..5fdca1fc 100644 --- a/include/tdnftypes.h +++ b/include/tdnftypes.h @@ -333,6 +333,8 @@ typedef struct _TDNF_UPDATEINFO_SUMMARY int nType; }TDNF_UPDATEINFO_SUMMARY, *PTDNF_UPDATEINFO_SUMMARY; +#define TDNF_REPOSYNC_MAXARCHS 10 + typedef struct _TDNF_REPOSYNC_ARGS { int nDelete; @@ -341,9 +343,10 @@ typedef struct _TDNF_REPOSYNC_ARGS int nNewestOnly; int nPrintUrlsOnly; int nNoRepoPath; + int nSourceOnly; char *pszDownloadPath; char *pszMetaDataPath; - char **pszArchs; + char **ppszArchs; }TDNF_REPOSYNC_ARGS, *PTDNF_REPOSYNC_ARGS; #ifdef __cplusplus diff --git a/tools/cli/lib/api.c b/tools/cli/lib/api.c index d9d9d4b3..ec8a2a04 100644 --- a/tools/cli/lib/api.c +++ b/tools/cli/lib/api.c @@ -476,7 +476,9 @@ TDNFCliRepoSyncCommand( dwError = pContext->pFnRepoSync(pContext, pReposyncArgs); BAIL_ON_CLI_ERROR(dwError); + cleanup: + TDNFCliFreeRepoSyncArgs(pReposyncArgs); return dwError; error: diff --git a/tools/cli/lib/parseargs.c b/tools/cli/lib/parseargs.c index 9d894b6a..43392cfe 100644 --- a/tools/cli/lib/parseargs.c +++ b/tools/cli/lib/parseargs.c @@ -36,7 +36,7 @@ static SetOptArgs OptValTable[] = { {CMDOPT_DISABLEPLUGIN, "disableplugin", NULL}, {CMDOPT_KEYVALUE, "skipconflicts;skipobsoletes;skipsignature;skipdigest;" "noplugins;reboot-required;security;" - "delete;download-metadata;gpgcheck;newest-only;norepopath;urls", + "delete;download-metadata;gpgcheck;newest-only;norepopath;source;urls", "1"} }; @@ -92,6 +92,7 @@ static struct option pstOptions[] = {"newest-only", no_argument, 0, 0}, {"norepopath", no_argument, 0, 0}, {"download-path", required_argument, 0, 0}, + {"source", no_argument, 0, 0}, {"urls", no_argument, 0, 0}, {0, 0, 0, 0} }; @@ -329,7 +330,8 @@ ParseOption( dwError = TDNFAllocateString(optarg, &pCmdArgs->pszReleaseVer); } else if ((!strcasecmp(pszName, "metadata-path")) || - (!strcasecmp(pszName, "download-path"))) + (!strcasecmp(pszName, "download-path")) || + (!strcasecmp(pszName, "arch"))) { dwError = AddSetOptWithValues(pCmdArgs, CMDOPT_KEYVALUE, diff --git a/tools/cli/lib/parsereposyncargs.c b/tools/cli/lib/parsereposyncargs.c index 6095d381..42fdde7a 100644 --- a/tools/cli/lib/parsereposyncargs.c +++ b/tools/cli/lib/parsereposyncargs.c @@ -27,6 +27,7 @@ TDNFCliParseRepoSyncArgs( uint32_t dwError = 0; PTDNF_REPOSYNC_ARGS pReposyncArgs = NULL; PTDNF_CMD_OPT pSetOpt = NULL; + int i; if (!pArgs || !ppReposyncArgs) { @@ -46,7 +47,24 @@ TDNFCliParseRepoSyncArgs( { if(pSetOpt->nType == CMDOPT_KEYVALUE) { - if (strcasecmp(pSetOpt->pszOptName, "delete") == 0) + if (strcasecmp(pSetOpt->pszOptName, "arch") == 0) + { + if (pReposyncArgs->ppszArchs == NULL) + { + TDNFAllocateMemory(TDNF_REPOSYNC_MAXARCHS+1, sizeof(char *), + (void **)&pReposyncArgs->ppszArchs); + BAIL_ON_CLI_ERROR(dwError); + } + for (i = 0; pReposyncArgs->ppszArchs[i] && i < TDNF_REPOSYNC_MAXARCHS; i++); + if (i < TDNF_REPOSYNC_MAXARCHS) + { + dwError = TDNFAllocateString( + pSetOpt->pszOptValue, + &(pReposyncArgs->ppszArchs[i])); + BAIL_ON_CLI_ERROR(dwError); + } + } + else if (strcasecmp(pSetOpt->pszOptName, "delete") == 0) { pReposyncArgs->nDelete = 1; } @@ -66,6 +84,10 @@ TDNFCliParseRepoSyncArgs( { pReposyncArgs->nNoRepoPath = 1; } + else if (strcasecmp(pSetOpt->pszOptName, "source") == 0) + { + pReposyncArgs->nSourceOnly = 1; + } else if (strcasecmp(pSetOpt->pszOptName, "urls") == 0) { pReposyncArgs->nPrintUrlsOnly = 1; @@ -90,6 +112,24 @@ TDNFCliParseRepoSyncArgs( cleanup: return dwError; error: + if (pReposyncArgs) + { + TDNFCliFreeRepoSyncArgs(pReposyncArgs); + } goto cleanup; } +void +TDNFCliFreeRepoSyncArgs( + PTDNF_REPOSYNC_ARGS pReposyncArgs + ) +{ + if(pReposyncArgs) + { + TDNF_CLI_SAFE_FREE_STRINGARRAY(pReposyncArgs->ppszArchs); + TDNF_SAFE_FREE_MEMORY(pReposyncArgs->pszDownloadPath); + TDNF_SAFE_FREE_MEMORY(pReposyncArgs->pszMetaDataPath); + TDNFFreeMemory(pReposyncArgs); + } +} + From 68be8d9bd58df8a19c6ac7ff40ada8958c1d46cc Mon Sep 17 00:00:00 2001 From: Oliver Kurth Date: Mon, 24 May 2021 12:45:02 -0700 Subject: [PATCH 14/17] reposync: implement --newest-only option --- client/api.c | 5 ++ client/packageutils.c | 112 ++++++++++++++++++++++++++++++++++++------ client/prototypes.h | 6 +++ common/utils.c | 1 + include/tdnftypes.h | 1 + solv/prototypes.h | 3 +- solv/tdnfpackage.c | 16 +++++- 7 files changed, 127 insertions(+), 17 deletions(-) diff --git a/client/api.c b/client/api.c index c934d1b7..4d8cf190 100644 --- a/client/api.c +++ b/client/api.c @@ -1139,6 +1139,11 @@ TDNFRepoSync( BAIL_ON_TDNF_ERROR(dwError); } + if (pReposyncArgs->nNewestOnly) + { + TDNFPkgInfoFilterNewest(pTdnf->pSack, pPkgInfos); + } + /* iterate through all packages */ for (pPkgInfo = pPkgInfos; pPkgInfo; pPkgInfo = pPkgInfo->pNext) { diff --git a/client/packageutils.c b/client/packageutils.c index 07099a39..813823da 100644 --- a/client/packageutils.c +++ b/client/packageutils.c @@ -18,6 +18,7 @@ * Authors : Priyesh Padmavilasom (ppadmavilasom@vmware.com) */ +#define _GNU_SOURCE 1 #include "includes.h" uint32_t @@ -253,22 +254,15 @@ TDNFPopulatePkgInfoForRepoSync( dwError = SolvGetPackageId(pPkgList, dwPkgIndex, &dwPkgId); BAIL_ON_TDNF_ERROR(dwError); - dwError = SolvGetPkgNameFromId(pSack, dwPkgId, &pPkgInfo->pszName); - BAIL_ON_TDNF_ERROR(dwError); - - dwError = SolvGetPkgArchFromId(pSack, dwPkgId, &pPkgInfo->pszArch); - BAIL_ON_TDNF_ERROR(dwError); - - dwError = SolvGetPkgVersionFromId( - pSack, - dwPkgId, - &pPkgInfo->pszVersion); - BAIL_ON_TDNF_ERROR(dwError); - - dwError = SolvGetPkgReleaseFromId( + dwError = SolvGetNevraFromId( pSack, dwPkgId, - &pPkgInfo->pszRelease); + &pPkgInfo->dwEpoch, + &pPkgInfo->pszName, + &pPkgInfo->pszVersion, + &pPkgInfo->pszRelease, + &pPkgInfo->pszArch, + &pPkgInfo->pszEVR); BAIL_ON_TDNF_ERROR(dwError); dwError = SolvGetPkgRepoNameFromId( @@ -301,6 +295,93 @@ TDNFPopulatePkgInfoForRepoSync( goto cleanup; } +static +int _pkginfo_compare( + const void *ptr1, + const void *ptr2, + void *data + ) +{ + const PTDNF_PKG_INFO* ppPkgInfo1 = (PTDNF_PKG_INFO*)ptr1; + const PTDNF_PKG_INFO* ppPkgInfo2 = (PTDNF_PKG_INFO*)ptr2; + Pool *pPool = (Pool *)data; + int ret; + + /* sort by repo name first, then name, then version */ + ret = strcmp((*ppPkgInfo1)->pszRepoName, (*ppPkgInfo2)->pszRepoName); + if (ret != 0) + { + return ret; + } + ret = strcmp((*ppPkgInfo1)->pszName, (*ppPkgInfo2)->pszName); + if (ret != 0) + { + return ret; + } + + /* we want newest version first, so reverse it by using the negated value */ + ret = - pool_evrcmp_str(pPool, + (*ppPkgInfo1)->pszEVR, (*ppPkgInfo2)->pszEVR, + EVRCMP_COMPARE); + return ret; +} + +uint32_t +TDNFPkgInfoFilterNewest( + PSolvSack pSack, + PTDNF_PKG_INFO pPkgInfos +) +{ + uint32_t dwError = 0; + uint32_t dwCount, i; + PTDNF_PKG_INFO* ppPkgInfos = NULL; + PTDNF_PKG_INFO pPkgInfo = NULL; + + dwCount = 0; + for (pPkgInfo = pPkgInfos; pPkgInfo; pPkgInfo = pPkgInfo->pNext) + { + dwCount++; + } + + dwError = TDNFAllocateMemory( + dwCount, + sizeof(PTDNF_PKG_INFO), + (void**)&ppPkgInfos); + BAIL_ON_TDNF_ERROR(dwError); + + i = 0; + for (pPkgInfo = pPkgInfos; pPkgInfo; pPkgInfo = pPkgInfo->pNext) + { + ppPkgInfos[i++] = pPkgInfo; + } + + qsort_r(ppPkgInfos, dwCount, + sizeof(PTDNF_PKG_INFO), _pkginfo_compare, (void *)pSack->pPool); + + /* Loop though pointer array, use the linked list to skip over + older versions of the same packages. The linked list will only + touch the newest (first) version of a package. + The same package in different repos will be handled as two different + packages. */ + pPkgInfo = ppPkgInfos[0]; + for (i = 1; i < dwCount; i++) + { + if ((strcmp(ppPkgInfos[i]->pszRepoName, pPkgInfo->pszRepoName) != 0) || + (strcmp(ppPkgInfos[i]->pszName, pPkgInfo->pszName) != 0)) + { + pPkgInfo->pNext = ppPkgInfos[i]; + pPkgInfo = ppPkgInfos[i]; + } + } + +cleanup: + TDNF_SAFE_FREE_MEMORY(ppPkgInfos); + return dwError; + +error: + goto cleanup; +} + uint32_t TDNFAppendPackages( PTDNF_PKG_INFO* ppDest, @@ -864,7 +945,8 @@ TDNFPopulatePkgInfos( &pPkgInfo->pszName, &pPkgInfo->pszVersion, &pPkgInfo->pszRelease, - &pPkgInfo->pszArch); + &pPkgInfo->pszArch, + NULL); BAIL_ON_TDNF_ERROR(dwError); dwError = SolvGetPkgRepoNameFromId( diff --git a/client/prototypes.h b/client/prototypes.h index 0451f490..2408b3ea 100644 --- a/client/prototypes.h +++ b/client/prototypes.h @@ -333,6 +333,12 @@ TDNFPopulatePkgInfoForRepoSync( PTDNF_PKG_INFO* ppPkgInfo ); +uint32_t +TDNFPkgInfoFilterNewest( + PSolvSack pSack, + PTDNF_PKG_INFO pPkgInfos +); + uint32_t TDNFPopulatePkgInfoArray( PSolvSack pSack, diff --git a/common/utils.c b/common/utils.c index 2a6d2f36..ab2532b7 100644 --- a/common/utils.c +++ b/common/utils.c @@ -220,6 +220,7 @@ TDNFFreePackageInfoContents( TDNF_SAFE_FREE_MEMORY(pPkgInfo->pszRepoName); TDNF_SAFE_FREE_MEMORY(pPkgInfo->pszVersion); TDNF_SAFE_FREE_MEMORY(pPkgInfo->pszArch); + TDNF_SAFE_FREE_MEMORY(pPkgInfo->pszEVR); TDNF_SAFE_FREE_MEMORY(pPkgInfo->pszSummary); TDNF_SAFE_FREE_MEMORY(pPkgInfo->pszURL); TDNF_SAFE_FREE_MEMORY(pPkgInfo->pszLicense); diff --git a/include/tdnftypes.h b/include/tdnftypes.h index 5fdca1fc..66d020f7 100644 --- a/include/tdnftypes.h +++ b/include/tdnftypes.h @@ -160,6 +160,7 @@ typedef struct _TDNF_PKG_INFO char* pszRepoName; char* pszVersion; char* pszArch; + char* pszEVR; char* pszSummary; char* pszURL; char* pszLicense; diff --git a/solv/prototypes.h b/solv/prototypes.h index c414dcab..4abba7ed 100644 --- a/solv/prototypes.h +++ b/solv/prototypes.h @@ -262,7 +262,8 @@ SolvGetNevraFromId( char **ppszName, char **ppszVersion, char **ppszRelease, - char **ppszArch + char **ppszArch, + char **ppszEVR ); // tdnfpool.c diff --git a/solv/tdnfpackage.c b/solv/tdnfpackage.c index ae8eab7d..9f8902bb 100644 --- a/solv/tdnfpackage.c +++ b/solv/tdnfpackage.c @@ -1804,7 +1804,8 @@ SolvGetNevraFromId( char **ppszName, char **ppszVersion, char **ppszRelease, - char **ppszArch + char **ppszArch, + char **ppszEVR ) { uint32_t dwError = 0; @@ -1816,6 +1817,7 @@ SolvGetNevraFromId( char *pszVersion = NULL; char *pszRelease = NULL; char *pszArch = NULL; + char *pszEVR = NULL; if(!pSack || !ppszName || @@ -1862,6 +1864,9 @@ SolvGetNevraFromId( BAIL_ON_TDNF_ERROR(dwError); } + dwError = TDNFAllocateString(pszTmp, &pszEVR); + BAIL_ON_TDNF_ERROR(dwError); + dwError = SolvSplitEvr(pSack, pszTmp, &pszEpoch, @@ -1879,6 +1884,10 @@ SolvGetNevraFromId( *ppszVersion = pszVersion; *ppszRelease = pszRelease; *ppszArch = pszArch; + if (ppszEVR) + { + *ppszEVR = pszEVR; + } cleanup: TDNF_SAFE_FREE_MEMORY(pszEpoch); return dwError; @@ -1904,9 +1913,14 @@ SolvGetNevraFromId( { *ppszArch = NULL; } + if(ppszEVR) + { + *ppszEVR = NULL; + } TDNF_SAFE_FREE_MEMORY(pszName); TDNF_SAFE_FREE_MEMORY(pszVersion); TDNF_SAFE_FREE_MEMORY(pszRelease); TDNF_SAFE_FREE_MEMORY(pszArch); + TDNF_SAFE_FREE_MEMORY(pszEVR); goto cleanup; } From 2770ae91aff29631eadd6cd6cb974de111c38090 Mon Sep 17 00:00:00 2001 From: Oliver Kurth Date: Thu, 27 May 2021 10:54:02 -0700 Subject: [PATCH 15/17] reposync: improve error handling --- client/api.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/client/api.c b/client/api.c index 4d8cf190..304019a9 100644 --- a/client/api.c +++ b/client/api.c @@ -1029,7 +1029,7 @@ _rm_rpms( } cleanup: TDNF_SAFE_FREE_MEMORY(pszKeepFile); - return dwError; + return (int)dwError; error: goto cleanup; } @@ -1041,6 +1041,7 @@ TDNFRepoSync( ) { uint32_t dwError = 0; + int ret; PTDNF_PKG_INFO pPkgInfos = NULL; PTDNF_PKG_INFO pPkgInfo = NULL; PTDNF_REPO_DATA_INTERNAL pRepo = NULL; @@ -1197,7 +1198,11 @@ TDNFRepoSync( if (pReposyncArgs->nGPGCheck) { dwError = TDNFGPGCheckPackage(&ts, pTdnf, pPkgInfo->pszRepoName, pszFilePath, NULL); - if (dwError) + if (dwError != RPMRC_NOTTRUSTED && dwError != RPMRC_NOKEY) + { + BAIL_ON_TDNF_ERROR(dwError); + } + else if (dwError) { pr_crit("checking package %s failed: %d, deleting\n", pszFilePath, dwError); if(remove(pszFilePath) < 0) @@ -1257,11 +1262,17 @@ TDNFRepoSync( pszRootPath, pRepo->pszId); BAIL_ON_TDNF_ERROR(dwError); - if (nftw(pszRepoDir, _rm_rpms, 10, FTW_DEPTH|FTW_PHYS) < 0) + ret = nftw(pszRepoDir, _rm_rpms, 10, FTW_DEPTH|FTW_PHYS); + if (ret < 0) { dwError = errno; BAIL_ON_TDNF_SYSTEM_ERROR(dwError); } + else + { + dwError = ret; + BAIL_ON_TDNF_ERROR(dwError); + } } } From 2aad58fd7503324e310b965170bff8129f00ec1f Mon Sep 17 00:00:00 2001 From: Oliver Kurth Date: Wed, 5 May 2021 12:44:11 -0700 Subject: [PATCH 16/17] reposync: add tests --- pytests/repo/setup-repo.sh | 1 + pytests/tests/test_downloadonly.py | 2 +- pytests/tests/test_reposync.py | 407 +++++++++++++++++++++++++++++ 3 files changed, 409 insertions(+), 1 deletion(-) create mode 100644 pytests/tests/test_reposync.py diff --git a/pytests/repo/setup-repo.sh b/pytests/repo/setup-repo.sh index cc34d69b..0e38fc04 100755 --- a/pytests/repo/setup-repo.sh +++ b/pytests/repo/setup-repo.sh @@ -25,6 +25,7 @@ PUBLISH_PATH=${TEST_REPO_DIR}/photon-test mkdir -p ${BUILD_PATH}/BUILD \ ${BUILD_PATH}/SRPMS \ ${BUILD_PATH}/RPMS/x86_64 \ + ${BUILD_PATH}/RPMS/noarch \ ${PUBLISH_PATH} \ ${TEST_REPO_DIR}/yum.repos.d diff --git a/pytests/tests/test_downloadonly.py b/pytests/tests/test_downloadonly.py index c8077b04..f1046c89 100644 --- a/pytests/tests/test_downloadonly.py +++ b/pytests/tests/test_downloadonly.py @@ -39,7 +39,7 @@ def test_install_download_only(utils): # be in specified directory def test_install_download_only_to_directory(utils): pkgname = utils.config["sglversion_pkgname"] - os.makedirs(DOWNLOADDIR) + os.makedirs(DOWNLOADDIR, exist_ok=True) ret = utils.run([ 'tdnf', 'install', '-y', '--downloadonly', '--downloaddir', DOWNLOADDIR, pkgname]) print (ret) assert(ret['retval'] == 0) diff --git a/pytests/tests/test_reposync.py b/pytests/tests/test_reposync.py new file mode 100644 index 00000000..3e9d02c1 --- /dev/null +++ b/pytests/tests/test_reposync.py @@ -0,0 +1,407 @@ +# +# Copyright (C) 2019 VMware, Inc. All Rights Reserved. +# +# Licensed under the GNU General Public License v2 (the "License"); +# you may not use this file except in compliance with the License. The terms +# of the License are located in the COPYING file of this distribution. +# +# Author: Oliver Kurth + +import os +import shutil +import errno +import pytest + +DOWNLOADDIR='/root/reposync/download' +METADATADIR='/root/reposync/metadata' +WORKDIR='/root/reposync/workdir' +REPOFILENAME='reposync.repo' + +@pytest.fixture(scope='function', autouse=True) +def setup_test(utils): + yield + teardown_test(utils) + +def teardown_test(utils): + pkgname = utils.config["mulversion_pkgname"] + utils.erase_package(pkgname) + + if os.path.isdir(DOWNLOADDIR): + shutil.rmtree(DOWNLOADDIR) + if os.path.isdir(WORKDIR): + shutil.rmtree(WORKDIR) + if os.path.isdir(METADATADIR): + shutil.rmtree(METADATADIR) + filename = os.path.join(utils.config['repo_path'], "yum.repos.d", REPOFILENAME) + if os.path.isfile(filename): + os.remove(filename) + +# helper to create directory tree without complains when it exists: +def makedirs(d): + try: + os.makedirs(d) + except OSError as e: + if e.errno != errno.EEXIST: + raise + +# helper to check a synced repo - +# uses the local repository and compares the list of RPMs +def check_synced_repo(utils, reponame, synced_dir): + + local_dir = os.path.join(utils.config['repo_path'], reponame, 'RPMS', 'x86_64') + + local_rpms = \ + list(filter(lambda f: os.path.isfile(os.path.join(local_dir, f)) and f.endswith('.rpm'), \ + os.listdir(local_dir))) + + # make sure we aren't confused which directory to check + assert(len(local_rpms) > 0) + + for rpm in local_rpms: + assert(os.path.isfile(os.path.join(synced_dir, 'RPMS', 'x86_64', rpm))) + +def create_repoconf(filename, baseurl, name): + templ = """ +[{name}] +name=Test Repo +baseurl={baseurl} +enabled=1 +gpgcheck=0 +metadata_expire=86400 +ui_repoid_vars=basearch +""" + with open(filename, "w") as f: + f.write(templ.format(name=name, baseurl=baseurl)) + +# reposync with no options - sync to local directory +def test_reposync(utils): + reponame = 'photon-test' + workdir = WORKDIR + makedirs(workdir) + + ret = utils.run(['tdnf', + '--disablerepo=*', '--enablerepo={}'.format(reponame), + 'reposync'], + cwd=workdir) + assert(ret['retval'] == 0) + synced_dir = os.path.join(workdir, reponame) + assert(os.path.isdir(synced_dir)) + + check_synced_repo(utils, reponame, synced_dir) + + shutil.rmtree(synced_dir) + +# reposync with download directory +def test_reposync_download_path(utils): + reponame = 'photon-test' + downloaddir = DOWNLOADDIR + makedirs(downloaddir) + assert(os.path.isdir(downloaddir)) + + ret = utils.run(['tdnf', '--disablerepo=*', '--enablerepo={}'.format(reponame), + '--download-path={}'.format(downloaddir), + 'reposync']) + assert(ret['retval'] == 0) + assert(os.path.isdir(os.path.join(downloaddir, reponame))) + + check_synced_repo(utils, reponame, os.path.join(downloaddir, reponame)) + +# reposync with download directory with an ending slash +# (There was a bug about this) +def test_reposync_download_path_slash(utils): + reponame = 'photon-test' + downloaddir = DOWNLOADDIR + '/' + makedirs(downloaddir) + assert(os.path.isdir(downloaddir)) + + ret = utils.run(['tdnf', '--disablerepo=*', '--enablerepo={}'.format(reponame), + '--download-path={}'.format(downloaddir), + 'reposync']) + assert(ret['retval'] == 0) + assert(os.path.isdir(os.path.join(downloaddir, reponame))) + + check_synced_repo(utils, reponame, os.path.join(downloaddir, reponame)) + +# reposync excluding the repo name from path +def test_reposync_download_path_norepopath(utils): + reponame = 'photon-test' + downloaddir = DOWNLOADDIR + ret = utils.run(['tdnf', + '--disablerepo=*', '--enablerepo={}'.format(reponame), + '--download-path={}'.format(downloaddir), + '--norepopath', + 'reposync']) + assert(ret['retval'] == 0) + assert(os.path.isdir(downloaddir)) + + check_synced_repo(utils, reponame, downloaddir) + +# reposync excluding the repo name and delete option is incompatible +def test_reposync_download_path_norepopath_delete(utils): + reponame = 'photon-test' + downloaddir = DOWNLOADDIR + ret = utils.run(['tdnf', + '--disablerepo=*', '--enablerepo={}'.format(reponame), + '--download-path={}'.format(downloaddir), + '--norepopath', + '--delete', + 'reposync']) + assert(ret['retval'] == 1622) + +# reposync excluding the repo name and multiple repos is incompatible +def xxxtest_reposync_download_path_norepopath_multiple_repos(utils): + reponame = 'photon-test' + downloaddir = DOWNLOADDIR + ret = utils.run(['tdnf', + '--download-path={}'.format(downloaddir), + '--norepopath', + 'reposync']) + assert(ret['retval'] == 1622) + +# reposync with metadata +def test_reposync_metadata(utils): + reponame = 'photon-test' + workdir = WORKDIR + makedirs(workdir) + + ret = utils.run(['tdnf', '--disablerepo=*', '--enablerepo={}'.format(reponame), + '--download-metadata', + 'reposync'], + cwd=workdir) + assert(ret['retval'] == 0) + synced_dir = os.path.join(workdir, reponame) + assert(os.path.isdir(synced_dir)) + assert(os.path.isdir(os.path.join(synced_dir, 'repodata'))) + assert(os.path.isfile(os.path.join(synced_dir, 'repodata', 'repomd.xml'))) + + check_synced_repo(utils, reponame, synced_dir) + + shutil.rmtree(synced_dir) + +# reposync with metadata +def test_reposync_metadata_path(utils): + reponame = 'photon-test' + workdir = WORKDIR + makedirs(workdir) + mdatadir = METADATADIR + makedirs(mdatadir) + + ret = utils.run(['tdnf', '--disablerepo=*', '--enablerepo={}'.format(reponame), + '--download-metadata', + '--metadata-path={}'.format(mdatadir), + 'reposync'], + cwd=workdir) + assert(ret['retval'] == 0) + synced_dir = os.path.join(workdir, reponame) + assert(os.path.isdir(synced_dir)) + assert(os.path.isfile(os.path.join(mdatadir, reponame, 'repodata', 'repomd.xml'))) + + check_synced_repo(utils, reponame, synced_dir) + + shutil.rmtree(synced_dir) + +# test --delete option +def test_reposync_delete(utils): + reponame = 'photon-test' + workdir = WORKDIR + makedirs(workdir) + + synced_dir = os.path.join(workdir, reponame) + makedirs(synced_dir) + + faked_rpm = os.path.join((synced_dir), 'faked-0.1.2.rpm') + with open(faked_rpm, 'w') as f: + f.write('fake package') + + assert(os.path.isfile(faked_rpm)) + + ret = utils.run(['tdnf', + '--disablerepo=*', '--enablerepo={}'.format(reponame), + '--delete', + 'reposync'], + cwd=workdir) + assert(ret['retval'] == 0) + assert(os.path.isdir(synced_dir)) + + check_synced_repo(utils, reponame, synced_dir) + + # file should be gone + assert(not os.path.isfile(faked_rpm)) + + shutil.rmtree(synced_dir) + +# test no --delete option (we should not delete files if not asked to) +def test_reposync_no_delete(utils): + reponame = 'photon-test' + workdir = WORKDIR + makedirs(workdir) + + synced_dir = os.path.join(workdir, reponame) + makedirs(synced_dir) + + faked_rpm = os.path.join((synced_dir), 'faked-0.1.2.rpm') + with open(faked_rpm, 'w') as f: + f.write('fake package') + + assert(os.path.isfile(faked_rpm)) + + ret = utils.run(['tdnf', + '--disablerepo=*', '--enablerepo={}'.format(reponame), + 'reposync'], + cwd=workdir) + assert(ret['retval'] == 0) + assert(os.path.isdir(synced_dir)) + + check_synced_repo(utils, reponame, synced_dir) + + # file should still be there + assert(os.path.isfile(faked_rpm)) + + shutil.rmtree(synced_dir) + +# reposync with gpgcheck +def test_reposync_gpgcheck(utils): + reponame = 'photon-test' + workdir = WORKDIR + makedirs(workdir) + + ret = utils.run(['tdnf', + '--disablerepo=*', '--enablerepo={}'.format(reponame), + '--gpgcheck', + 'reposync'], + cwd=workdir) + assert(ret['retval'] == 0) + synced_dir = os.path.join(workdir, reponame) + assert(os.path.isdir(synced_dir)) + + check_synced_repo(utils, reponame, synced_dir) + + shutil.rmtree(synced_dir) + +# reposync with --urls option (print only) +def test_reposync_urls(utils): + reponame = 'photon-test' + workdir = WORKDIR + makedirs(workdir) + + ret = utils.run(['tdnf', + '--disablerepo=*', '--enablerepo={}'.format(reponame), + '--urls', + 'reposync'], + cwd=workdir) + assert(ret['retval'] == 0) + synced_dir = os.path.join(workdir, reponame) + assert(not os.path.isdir(synced_dir)) + +# reposync a repo and install from it +def test_reposync_create_repo(utils): + reponame = 'photon-test' + workdir = WORKDIR + makedirs(workdir) + + ret = utils.run(['tdnf', '--disablerepo=*', '--enablerepo={}'.format(reponame), + '--download-metadata', + 'reposync'], + cwd=workdir) + assert(ret['retval'] == 0) + synced_dir = os.path.join(workdir, reponame) + assert(os.path.isdir(synced_dir)) + assert(os.path.isdir(os.path.join(synced_dir, 'repodata'))) + assert(os.path.isfile(os.path.join(synced_dir, 'repodata', 'repomd.xml'))) + + check_synced_repo(utils, reponame, synced_dir) + + filename = os.path.join(utils.config['repo_path'], "yum.repos.d", REPOFILENAME) + baseurl = "file://{}".format(synced_dir) + + create_repoconf(filename, baseurl, "synced-repo") + + ret = utils.run(['tdnf', + '--disablerepo=*', '--enablerepo=synced-repo', + 'makecache'], + cwd=workdir) + assert(ret['retval'] == 0) + + pkgname = utils.config["mulversion_pkgname"] + utils.erase_package(pkgname) + ret = utils.run(['tdnf', + '-y', '--nogpgcheck', + '--disablerepo=*', '--enablerepo=synced-repo', + 'install', pkgname ], + cwd=workdir) + assert(utils.check_package(pkgname) == True) + +# reposync with arch option should not sync other archs +def test_reposync_arch_x86_64(utils): + reponame = 'photon-test' + workdir = WORKDIR + makedirs(workdir) + + synced_dir = os.path.join(workdir, reponame) + if os.path.isdir(synced_dir): + shutil.rmtree(synced_dir) + + ret = utils.run(['tdnf', + '--disablerepo=*', '--enablerepo={}'.format(reponame), + '--arch', 'x86_64', + 'reposync'], + cwd=workdir) + assert(ret['retval'] == 0) + assert(os.path.isdir(synced_dir)) + + check_synced_repo(utils, reponame, synced_dir) + + assert(not os.path.isdir(os.path.join(synced_dir, 'RPMS', 'noarch'))) + shutil.rmtree(synced_dir) + + +# reposync with arch option should work for multiple archs +def test_reposync_arch_x86_64_others(utils): + reponame = 'photon-test' + workdir = WORKDIR + makedirs(workdir) + + synced_dir = os.path.join(workdir, reponame) + if os.path.isdir(synced_dir): + shutil.rmtree(synced_dir) + + ret = utils.run(['tdnf', + '--disablerepo=*', '--enablerepo={}'.format(reponame), + '--arch', 'classic', + '--arch', 'baroque', + '--arch', 'modern', + '--arch', 'x86_64', + 'reposync'], + cwd=workdir) + assert(ret['retval'] == 0) + assert(os.path.isdir(synced_dir)) + + check_synced_repo(utils, reponame, synced_dir) + + assert(not os.path.isdir(os.path.join(synced_dir, 'RPMS', 'noarch'))) + shutil.rmtree(synced_dir) + +# reposync with --newest-only option - sync to local directory +def test_reposync_newest(utils): + reponame = 'photon-test' + workdir = WORKDIR + makedirs(workdir) + + ret = utils.run(['tdnf', + '--disablerepo=*', '--enablerepo={}'.format(reponame), + '--newest-only', + 'reposync'], + cwd=workdir) + assert(ret['retval'] == 0) + synced_dir = os.path.join(workdir, reponame) + assert(os.path.isdir(synced_dir)) + + mulversion_pkgname_found = False + for f in os.listdir(os.path.join(synced_dir, 'RPMS', 'x86_64')): + if f.startswith(utils.config['mulversion_pkgname']): + assert(not utils.config['mulversion_lower'] in f) + mulversion_pkgname_found = True + + assert(mulversion_pkgname_found) + + shutil.rmtree(synced_dir) From e47d2c65dd97a3bf99f97a25e70b09e84d83d9a1 Mon Sep 17 00:00:00 2001 From: Oliver Kurth Date: Tue, 25 May 2021 13:58:25 -0700 Subject: [PATCH 17/17] reposync: add help --- tools/cli/lib/help.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tools/cli/lib/help.c b/tools/cli/lib/help.c index b3ec72c9..612208e2 100644 --- a/tools/cli/lib/help.c +++ b/tools/cli/lib/help.c @@ -60,6 +60,19 @@ TDNFCliShowHelp( pr_crit(" [--disableexcludes]\n"); pr_crit(" [--downloadonly]\n"); pr_crit(" [--downloaddir=]\n"); + pr_crit("\n"); + pr_crit("reposync options:\n"); + pr_crit(" [--arch= [--arch= [..]]\n"); + pr_crit(" [--delete]\n"); + pr_crit(" [--download-path=]\n"); + pr_crit(" [--download-metadata]\n"); + pr_crit(" [--gpgcheck]\n"); + pr_crit(" [--metadata-path=]\n"); + pr_crit(" [--newest-only]\n"); + pr_crit(" [--norepopath]\n"); + pr_crit(" [--source]\n"); + pr_crit(" [--urls]\n"); + pr_crit("\n"); pr_crit("List of Main Commands\n"); pr_crit("\n"); @@ -79,6 +92,7 @@ TDNFCliShowHelp( pr_crit("remove Remove a package or packages from your system\n"); pr_crit("reinstall reinstall a package\n"); pr_crit("repolist Display the configured software repositories\n"); + pr_crit("reposync Download all packages from one or more repositories to a directory\n"); pr_crit("search Search package details for the given string\n"); pr_crit("update Upgrade a package or packages on your system (same as 'upgrade')\n"); pr_crit("updateinfo Display advisories about packages\n");