Add virtual repo snapshot feature to tdnf

vmware · Sep 20, 2024 · ad4dd2e · ad4dd2e
1 parent 066bd38
commit ad4dd2e
Show file tree

Hide file tree

Showing 14 changed files with 958 additions and 17 deletions.
diff --git a/client/config.c b/client/config.c
@@ -100,6 +100,10 @@ TDNFConfigFromCnfTree(PTDNF_CONF pConf, struct cnfnode *cn_top)
         {
             pConf->nInstallOnlyLimit = strtoi(cn->value);
         }
+        else if (strcmp(cn->name, TDNF_CONF_KEY_SNAPSHOT_TIME) == 0)
+        {
+            pConf->pszSnapshotTime = strdup(cn->value);
+        }
         else if (strcmp(cn->name, TDNF_CONF_KEY_CLEAN_REQ_ON_REMOVE) == 0)
         {
             pConf->nCleanRequirementsOnRemove = isTrue(cn->value);
@@ -255,6 +259,7 @@ TDNFReadConfig(
     pConf->nOpenMax = TDNF_CONF_DEFAULT_OPENMAX;
     pConf->nInstallOnlyLimit = TDNF_CONF_DEFAULT_INSTALLONLY_LIMIT;
     pConf->nSSLVerify = TDNF_CONF_DEFAULT_SSLVERIFY;
+    pConf->pszSnapshotTime = NULL;
 
     register_ini(NULL);
     mod_ini = find_cnfmodule("ini");

diff --git a/client/prototypes.h b/client/prototypes.h
@@ -578,7 +578,8 @@ uint32_t
 TDNFInitRepoFromMetadata(
     Repo *pRepo,
     const char* pszRepoName,
-    PTDNF_REPO_METADATA pRepoMD
+    PTDNF_REPO_METADATA pRepoMD,
+    char * pszSnapshotTime
     );
 
 uint32_t
@@ -736,6 +737,12 @@ TDNFAlterRepoState(
     const char* pszId
     );
 
+uint32_t
+TDNFExcludeFromSnapshot(
+    PTDNF_REPO_DATA pRepos,
+    const char* pszId
+    );
+
 uint32_t
 TDNFCloneRepo(
     PTDNF_REPO_DATA pRepoIn,

diff --git a/client/repo.c b/client/repo.c
@@ -24,13 +24,30 @@ TDNFInitRepo(
     Pool* pPool = NULL;
     int nUseMetaDataCache = 0;
     PSOLV_REPO_INFO_INTERNAL pSolvRepoInfo = NULL;
+    PTDNF_CMD_OPT pSetOpt = NULL;
+    char * pszSnapshotTime = NULL;
 
     if (!pTdnf || !pRepoData || !pSack || !pSack->pPool)
     {
         dwError = ERROR_TDNF_INVALID_PARAMETER;
         BAIL_ON_TDNF_ERROR(dwError);
     }
 
+    // set local POSIX limit if conf or cmd line opt is present
+    if (pTdnf->pConf != NULL && pTdnf->pConf->pszSnapshotTime!= NULL) 
+    {
+        pszSnapshotTime = pTdnf->pConf->pszSnapshotTime;
+    }
+
+    // take command line over config if both are present
+    for (pSetOpt = pTdnf->pArgs->pSetOpt; pSetOpt; pSetOpt = pSetOpt->pNext) 
+    {
+        if(strncmp(pSetOpt->pszOptName, TDNF_CONF_KEY_SNAPSHOT_TIME, strlen(TDNF_CONF_KEY_SNAPSHOT_TIME)) == 0)
+        {
+            pszSnapshotTime = pSetOpt->pszOptValue;
+        }
+    }
+
     pPool = pSack->pPool;
 
     dwError = TDNFGetCachePath(pTdnf, pRepoData,
@@ -82,20 +99,27 @@ TDNFInitRepo(
     pRepo->appdata = pSolvRepoInfo;
 
     if (pRepoData->nHasMetaData) {
-        dwError = SolvCalculateCookieForFile(pRepoMD->pszRepoMD, pSolvRepoInfo->cookie);
-        BAIL_ON_TDNF_ERROR(dwError);
-        pSolvRepoInfo->nCookieSet = 1;
-
-        dwError = SolvUseMetaDataCache(pSack, pSolvRepoInfo, &nUseMetaDataCache);
-        BAIL_ON_TDNF_ERROR(dwError);
-
-        if (nUseMetaDataCache == 0) {
-            dwError = TDNFInitRepoFromMetadata(pRepo, pRepoData->pszId, pRepoMD);
+        if (!pRepoData->nExcludeSnapshot && pszSnapshotTime != NULL) {
+            dwError = TDNFInitRepoFromMetadata(pRepo, pRepoData->pszId, pRepoMD, pszSnapshotTime);
             BAIL_ON_TDNF_ERROR(dwError);
+        } else {
+            dwError = SolvCalculateCookieForFile(pRepoMD->pszRepoMD, pSolvRepoInfo->cookie);
+            BAIL_ON_TDNF_ERROR(dwError);
+            pSolvRepoInfo->nCookieSet = 1;
 
-            dwError = SolvCreateMetaDataCache(pSack, pSolvRepoInfo);
+            dwError = SolvUseMetaDataCache(pSack, pSolvRepoInfo, &nUseMetaDataCache);
             BAIL_ON_TDNF_ERROR(dwError);
+
+            //force load from repo if POSIX time limit is present
+            if (nUseMetaDataCache == 0) {
+                dwError = TDNFInitRepoFromMetadata(pRepo, pRepoData->pszId, pRepoMD, NULL);
+                BAIL_ON_TDNF_ERROR(dwError);
+
+                dwError = SolvCreateMetaDataCache(pSack, pSolvRepoInfo);
+                BAIL_ON_TDNF_ERROR(dwError);
+            }
         }
+
     } else {
         dwError = SolvReadRpmsFromDirectory(pRepo, pRepoData->ppszBaseUrls[0]);
         BAIL_ON_TDNF_ERROR(dwError);
@@ -136,7 +160,8 @@ uint32_t
 TDNFInitRepoFromMetadata(
     Repo *pRepo,
     const char* pszRepoName,
-    PTDNF_REPO_METADATA pRepoMD
+    PTDNF_REPO_METADATA pRepoMD,
+    char * pszSnapshotTime
     )
 {
     uint32_t dwError = 0;
@@ -153,7 +178,8 @@ TDNFInitRepoFromMetadata(
                   pRepoMD->pszPrimary,
                   pRepoMD->pszFileLists,
                   pRepoMD->pszUpdateInfo,
-                  pRepoMD->pszOther);
+                  pRepoMD->pszOther,
+                  pszSnapshotTime);
 cleanup:
     return dwError;
 

diff --git a/client/repolist.c b/client/repolist.c
@@ -380,6 +380,7 @@ TDNFCreateRepo(
     BAIL_ON_TDNF_ERROR(dwError);
 
     pRepo->nEnabled = TDNF_REPO_DEFAULT_ENABLED;
+    pRepo->nExcludeSnapshot = 0;
     pRepo->nHasMetaData = 1;
     pRepo->nSkipIfUnavailable = TDNF_REPO_DEFAULT_SKIP;
     pRepo->nGPGCheck = TDNF_REPO_DEFAULT_GPGCHECK;
@@ -675,6 +676,7 @@ TDNFRepoListFinalize(
     PTDNF_CMD_OPT pSetOpt = NULL;
     PTDNF_REPO_DATA pRepo = NULL;
     int nRepoidSeen = 0;
+    char ** ppszRepos = NULL;
 
     if(!pTdnf || !pTdnf->pArgs || !pTdnf->pRepos)
     {
@@ -715,6 +717,22 @@ TDNFRepoListFinalize(
                           1,
                           pSetOpt->pszOptValue);
         }
+        else if (strcmp(pSetOpt->pszOptName, "snapshotexcluderepos") == 0)
+        {
+            ppszRepos = NULL;
+            int i = 0;
+            dwError = TDNFSplitStringToArray(pSetOpt->pszOptValue, ",", &ppszRepos);
+            BAIL_ON_TDNF_ERROR(dwError);
+
+            while (ppszRepos && ppszRepos[i]){
+                dwError = TDNFExcludeFromSnapshot(
+                pTdnf->pRepos,
+                ppszRepos[i]);
+                BAIL_ON_TDNF_ERROR(dwError);
+                i++;
+            }
+
+        }
         BAIL_ON_TDNF_ERROR(dwError);
     }
 
@@ -775,6 +793,7 @@ TDNFRepoListFinalize(
         BAIL_ON_TDNF_ERROR(dwError);
     }
 cleanup:
+    TDNFFreeStringArray(ppszRepos);
     return dwError;
 error:
     goto cleanup;
@@ -826,6 +845,53 @@ TDNFAlterRepoState(
     goto cleanup;
 }
 
+uint32_t
+TDNFExcludeFromSnapshot(
+    PTDNF_REPO_DATA pRepos,
+    const char* pszId
+    )
+{
+    uint32_t dwError = 0;
+    int nIsGlob = 0;
+    if(!pRepos || IsNullOrEmptyString(pszId))
+    {
+        dwError = ERROR_TDNF_INVALID_PARAMETER;
+        BAIL_ON_TDNF_ERROR(dwError);
+    }
+
+    nIsGlob = TDNFIsGlob(pszId);
+
+    for (int nMatch = 0; pRepos; pRepos = pRepos->pNext)
+    {
+        if(nIsGlob)
+        {
+            if(!fnmatch(pszId, pRepos->pszId, 0))
+            {
+                nMatch = 1;
+            }
+        }
+        else if(!strcmp(pRepos->pszId, pszId))
+        {
+            nMatch = 1;
+        }
+
+        if(nMatch)
+        {
+            pRepos->nExcludeSnapshot = 1;
+            if(!nIsGlob)
+            {
+                break;
+            }
+        }
+    }
+
+cleanup:
+    return dwError;
+
+error:
+    goto cleanup;
+}
+
 uint32_t
 TDNFCloneRepo(
     PTDNF_REPO_DATA pRepoIn,

diff --git a/common/config.h b/common/config.h
@@ -23,6 +23,7 @@
 #define TDNF_CONF_KEY_GPGCHECK            "gpgcheck"
 #define TDNF_CONF_KEY_INSTALLONLY_LIMIT   "installonly_limit"
 #define TDNF_CONF_KEY_INSTALLONLYPKGS     "installonlypkgs"
+#define TDNF_CONF_KEY_SNAPSHOT_TIME       "snapshottime"
 #define TDNF_CONF_KEY_CLEAN_REQ_ON_REMOVE "clean_requirements_on_remove"
 #define TDNF_CONF_KEY_REPODIR             "repodir" // typo, keep for back compatibility
 #define TDNF_CONF_KEY_REPOSDIR            "reposdir"

diff --git a/etc/bash_completion.d/tdnf-completion.bash b/etc/bash_completion.d/tdnf-completion.bash
@@ -26,6 +26,11 @@ _tdnf__process_if_prev_is_option()
             COMPREPLY=( $(compgen -W "$opts" -- $cur) )
             return 0
             ;;
+        --snapshotexcluderepos)
+            opts=`tdnf repolist enabled | awk '{if (NR > 1) print $1}'`
+            COMPREPLY=( $(compgen -W "$opts" -- $cur) )
+            return 0
+            ;;
         --installroot)
             COMPREPLY=( $(compgen -d -- $cur) )
             return 0
@@ -92,7 +97,7 @@ _tdnf()
 {
     local c=0 cur __opts __cmds
     COMPREPLY=()
-    __opts="--assumeno --assumeyes --cacheonly --debugsolver --disableexcludes --disableplugin --disablerepo --downloaddir --downloadonly --enablerepo --enableplugin --exclude --installroot --noautoremove --nogpgcheck --noplugins --quiet --reboot --refresh --releasever --repo --repofrompath --repoid --rpmverbosity --security --sec --setopt --skip --skipconflicts --skipdigest --skipsignature --skipobsoletes --testonly --version --available --duplicates --extras --file --installed --whatdepends --whatrequires --whatenhances --whatobsoletes --whatprovides --whatrecommends --whatrequires --whatsuggests --whatsupplements --depends --enhances --list --obsoletes --provides --recommends --requires --requires --suggests --source --supplements --arch --delete --download --download --gpgcheck --metadata --newest --norepopath --source --urls"
+    __opts="--assumeno --assumeyes --cacheonly --debugsolver --disableexcludes --disableplugin --disablerepo --downloaddir --downloadonly --enablerepo --enableplugin --exclude --installroot --noautoremove --nogpgcheck --noplugins --quiet --reboot --refresh --releasever --repo --repofrompath --repoid --rpmverbosity --security --sec --setopt --skip --skipconflicts --skipdigest --skipsignature --skipobsoletes --snapshotexcluderepos --snapshottime --testonly --version --available --duplicates --extras --file --installed --whatdepends --whatrequires --whatenhances --whatobsoletes --whatprovides --whatrecommends --whatrequires --whatsuggests --whatsupplements --depends --enhances --list --obsoletes --provides --recommends --requires --requires --suggests --source --supplements --arch --delete --download --download --gpgcheck --metadata --newest --norepopath --source --urls"
     __cmds="autoerase autoremove check check-local check-update clean distro-sync downgrade erase help history info install list makecache mark provides whatprovides reinstall remove repolist repoquery reposync search update update-to updateinfo upgrade upgrade-to"
     cur="${COMP_WORDS[COMP_CWORD]}"
     _tdnf__process_if_prev_is_option && return 0

diff --git a/include/tdnferror.h b/include/tdnferror.h
@@ -187,6 +187,15 @@ extern "C" {
 #define ERROR_TDNF_HISTORY_ERROR 1801
 #define ERROR_TDNF_HISTORY_NODB 1802
 
+#define ERROR_TDNF_TIME_FILTER_BASE 1900
+// filter MEMORY
+#define ERROR_TDNF_TIME_FILTER_MEMORY        (ERROR_TDNF_TIME_FILTER_BASE + 1)
+// filter parsing error
+#define ERROR_TDNF_TIME_FILTER_PARSE         (ERROR_TDNF_TIME_FILTER_BASE + 2)
+// filter IO error
+#define ERROR_TDNF_TIME_FILTER_IO            (ERROR_TDNF_TIME_FILTER_BASE + 3)
+// filter general error
+# define ERROR_TDNF_TIME_FILTER_GENERAL      (ERROR_TDNF_TIME_FILTER_BASE + 4)
 
 #define ERROR_TDNF_PLUGIN_BASE          2000
 

diff --git a/include/tdnftypes.h b/include/tdnftypes.h
@@ -9,6 +9,7 @@
 #pragma once
 
 #include <curl/curl.h>
+#include <time.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -259,6 +260,7 @@ typedef struct _TDNF_CONF
     int nDistroSyncReinstallChanged;
     int nPluginsEnabled;
     char* pszRepoDir;
+    char* pszSnapshotTime;
     char* pszCacheDir;
     char* pszPersistDir;
     char* pszProxy;
@@ -283,6 +285,7 @@ typedef struct _TDNF_CONF
 typedef struct _TDNF_REPO_DATA
 {
     int nEnabled;
+    int nExcludeSnapshot;
     int nSkipIfUnavailable;
     int nGPGCheck;
     int nHasMetaData;

diff --git a/solv/defines.h b/solv/defines.h
@@ -18,4 +18,33 @@
         }                                                          \
     } while(0)
 
+typedef struct {
+    // frequently changed values
+    char * pszElementBuffer;
+    int nBufferLen;
+    int nInPackage;
+    int nPrintPackage;
+    int nTimeFound;
+
+    // managed values
+    int nBufferMaxLen;
+    int nDepth;
+    int nPrevElement; // enum 0 -> start, 1 -> data, 2 -> end
+
+    //set and forget on creation
+    time_t nSearchTime;
+    FILE * pbOutfile;
+} TDNFFilterData;
+
+#define TDNF_MAX_FILTER_INPUT_THRESHOLD 500000000
+#define TDNF_DEFAULT_TIME_FILTER_BUFF_SIZE 16000
+
+#define BAIL_ON_TDNF_TIME_FILTER_ERROR(dwError) \
+    do {                                                           \
+        if (dwError)                                               \
+        {                                                          \
+            goto error;                                            \
+        }                                                          \
+    } while(0)
+
 #endif /* __SOLV_DEFINES_H__ */
diff --git a/solv/includes.h b/solv/includes.h
@@ -10,7 +10,6 @@
 #include <unistd.h>
 #include <stdbool.h>
 #include <sys/types.h>
-#include <sys/stat.h>
 #include <dirent.h>
 
 // libsolv
@@ -44,4 +43,9 @@
 #include "../history/history.h"
 #include "prototypes.h"
 
+#include <expat.h>
+#include <stdlib.h>
+#include <time.h>
+#include <string.h>
+
 #endif /* __SOLV_INCLUDES_H__ */
diff --git a/solv/prototypes.h b/solv/prototypes.h
@@ -520,7 +520,8 @@ SolvReadYumRepo(
     const char *pszPrimary,
     const char *pszFilelists,
     const char *pszUpdateinfo,
-    const char *pszOther
+    const char *pszOther,
+    const char *pszSnapshotTime
     );
 
 uint32_t