From 43b1440df8d2cad94fd9f56a00b2b4e4e150e3e5 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Tue, 29 Oct 2024 21:52:27 +0000 Subject: [PATCH 01/64] key manager --- config/init_server_creds.go | 27 +---- config/key_manager.go | 218 ++++++++++++++++++++++++++++++++++++ 2 files changed, 223 insertions(+), 22 deletions(-) create mode 100644 config/key_manager.go diff --git a/config/init_server_creds.go b/config/init_server_creds.go index 2f9c97e8f..b1c373228 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -36,7 +36,6 @@ import ( "sync/atomic" "time" - "github.com/lestrrat-go/jwx/v2/jwa" "github.com/lestrrat-go/jwx/v2/jwk" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -506,30 +505,14 @@ func GenerateCert() error { // Helper function to load the issuer/server's private key to sign tokens it issues. // Only intended to be called internally func loadIssuerPrivateJWK(issuerKeyFile string) (jwk.Key, error) { - // Check to see if we already had an IssuerKey or generate one - if err := GeneratePrivateKey(issuerKeyFile, elliptic.P256(), false); err != nil { - return nil, errors.Wrap(err, "Failed to generate new private key") - } - contents, err := os.ReadFile(issuerKeyFile) - if err != nil { - return nil, errors.Wrap(err, "Failed to read issuer key file") - } - key, err := jwk.ParseKey(contents, jwk.WithPEM(true)) - if err != nil { - return nil, errors.Wrapf(err, "Failed to parse issuer key file %v", issuerKeyFile) - } - - // Add the algorithm to the key, needed for verifying tokens elsewhere - err = key.Set(jwk.AlgorithmKey, jwa.ES256) - if err != nil { - return nil, errors.Wrap(err, "Failed to add alg specification to key header") + km := GetKeyManager() + if err := km.Initialize(); err != nil { + return nil, errors.Wrap(err, "Failed to initialize key manager") } - // Assign key id to the private key so that the public key obtainer thereafter - // has the same kid - err = jwk.AssignKeyID(key) + key, err := km.GetActivePrivateKey() if err != nil { - return nil, errors.Wrap(err, "Failed to assign key ID to private key") + return nil, errors.Wrap(err, "Failed to get active key") } // Store the key in the in-memory cache diff --git a/config/key_manager.go b/config/key_manager.go new file mode 100644 index 000000000..d7d563203 --- /dev/null +++ b/config/key_manager.go @@ -0,0 +1,218 @@ +package config + +import ( + "crypto/elliptic" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/lestrrat-go/jwx/v2/jwa" + "github.com/lestrrat-go/jwx/v2/jwk" + "github.com/pelicanplatform/pelican/param" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +const ( + defaultPrivateKeyDir = "/etc/pelican/private-keys" // TODO: change it to a param +) + +type KeyManager struct { + keys map[string]jwk.Key // Map of key ID to jwk.Key + keyDir string + keyMutex sync.RWMutex +} + +var ( + globalKeyManager *KeyManager + managerOnce sync.Once +) + +// GetKeyManager returns the singleton KeyManager instance +func GetKeyManager() *KeyManager { + managerOnce.Do(func() { + globalKeyManager = &KeyManager{ + keys: make(map[string]jwk.Key), + keyDir: defaultPrivateKeyDir, + } + }) + return globalKeyManager +} + +// Initialize sets up the key directory and loads existing keys +func (km *KeyManager) Initialize() error { + gid, err := GetDaemonGID() + if err != nil { + return err + } + // Create key directory if it doesn't exist + if err := MkdirAll(km.keyDir, 0750, -1, gid); err != nil { + return errors.Wrap(err, "Failed to create key directory") + } + + // Try to migrate legacy key if it exists + if err := km.migrateLegacyPrivateKey(); err != nil { + log.Warnf("Failed to migrate legacy key: %v", err) + // Continue execution - we'll create a new key if no key found + } + + // Load all keys from directory + if err := km.loadPrivateKeysFromDirectory(); err != nil { + log.Warnf("Failed to load keys from directory: %v", err) + // Continue execution - we'll create a new key if no key found + } + + // Check if we have any keys after migration and loading + km.keyMutex.RLock() + keyCount := len(km.keys) + km.keyMutex.RUnlock() + + if keyCount == 0 { + log.Info("No existing keys found. Generating new initial key...") + _, err := km.generateNewPrivateKey() + if err != nil { + return errors.Wrap(err, "Failed to generate initial key") + } + log.Info("Successfully generated initial key") + } + + return nil +} + +// migrateLegacyPrivateKey moves the old single key file to the new directory structure +func (km *KeyManager) migrateLegacyPrivateKey() error { + legacyPrivateKeyFile := param.IssuerKey.GetString() + + if _, err := os.Stat(legacyPrivateKeyFile); os.IsNotExist(err) { + return nil // No legacy key exists + } + + contents, err := os.ReadFile(legacyPrivateKeyFile) + if err != nil { + return errors.Wrap(err, "Failed to read legacy key file") + } + + // Parse the key to get its key ID + key, err := jwk.ParseKey(contents, jwk.WithPEM(true)) + if err != nil { + return errors.Wrap(err, "Failed to parse legacy key") + } + + if err := jwk.AssignKeyID(key); err != nil { + return errors.Wrap(err, "Failed to assign key ID to legacy key") + } + + kid := key.KeyID() + if kid == "" { + kid = "legacy-key" + } + + // Write to new location + newPath := filepath.Join(km.keyDir, fmt.Sprintf("%s.pem", kid)) + if err := os.WriteFile(newPath, contents, 0400); err != nil { + return errors.Wrap(err, "Failed to write legacy key to new location") + } + + return nil +} + +// loadPrivateKeysFromDirectory loads all .pem files from the key directory +func (km *KeyManager) loadPrivateKeysFromDirectory() error { + km.keyMutex.Lock() + defer km.keyMutex.Unlock() + + files, err := os.ReadDir(km.keyDir) + if err != nil { + return errors.Wrap(err, "Failed to read key directory") + } + + for _, file := range files { + if filepath.Ext(file.Name()) != ".pem" { + continue + } + + keyPath := filepath.Join(km.keyDir, file.Name()) + key, err := km.loadSinglePrivateKey(keyPath) + if err != nil { + log.Warnf("Failed to load key %s: %v", keyPath, err) + continue + } + + km.keys[key.KeyID()] = key + } + + return nil +} + +// loadSinglePrivateKey loads and prepares a single key file +func (km *KeyManager) loadSinglePrivateKey(path string) (jwk.Key, error) { + contents, err := os.ReadFile(path) + if err != nil { + return nil, errors.Wrap(err, "Failed to read key file") + } + + key, err := jwk.ParseKey(contents, jwk.WithPEM(true)) + if err != nil { + return nil, errors.Wrap(err, "Failed to parse key") + } + + // Add the algorithm to the key + if err := key.Set(jwk.AlgorithmKey, jwa.ES256); err != nil { + return nil, errors.Wrap(err, "Failed to set algorithm") + } + + // Ensure key has an ID + if err := jwk.AssignKeyID(key); err != nil { + return nil, errors.Wrap(err, "Failed to assign key ID") + } + + return key, nil +} + +// GetActivePrivateKey returns the current active key for signing +func (km *KeyManager) GetActivePrivateKey() (jwk.Key, error) { + km.keyMutex.RLock() + defer km.keyMutex.RUnlock() + + // If no keys exist, generate one + if len(km.keys) == 0 { + return km.generateNewPrivateKey() + } + + // For now, return the first key (TODO: implement more sophisticated key selection) + for _, key := range km.keys { + return key, nil + } + + return nil, errors.New("No active key available") +} + +// generateNewPrivateKey creates a new key and adds it to the manager +func (km *KeyManager) generateNewPrivateKey() (jwk.Key, error) { + km.keyMutex.Lock() + defer km.keyMutex.Unlock() + + keyPath := filepath.Join(km.keyDir, fmt.Sprintf("key-%d.pem", time.Now().Unix())) + if err := GeneratePrivateKey(keyPath, elliptic.P256(), false); err != nil { + return nil, errors.Wrap(err, "Failed to generate new private key") + } + + key, err := km.loadSinglePrivateKey(keyPath) + if err != nil { + return nil, err + } + + km.keys[key.KeyID()] = key + return key, nil +} + +// GetKeyByID returns a specific key by its ID +func (km *KeyManager) GetPrivateKeyByID(kid string) (jwk.Key, bool) { + km.keyMutex.RLock() + defer km.keyMutex.RUnlock() + + key, exists := km.keys[kid] + return key, exists +} From 5f4532f62cb67ddd003ba1f75d1b0312b0ad0777 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Wed, 30 Oct 2024 20:56:23 +0000 Subject: [PATCH 02/64] check private keys dir every 10 mins --- config/key_manager.go | 31 +++++++++++++++++++++++++++++++ launchers/origin_serve.go | 3 +++ 2 files changed, 34 insertions(+) diff --git a/config/key_manager.go b/config/key_manager.go index d7d563203..0418d4ec0 100644 --- a/config/key_manager.go +++ b/config/key_manager.go @@ -1,6 +1,7 @@ package config import ( + "context" "crypto/elliptic" "fmt" "os" @@ -13,6 +14,7 @@ import ( "github.com/pelicanplatform/pelican/param" "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" ) const ( @@ -140,7 +142,13 @@ func (km *KeyManager) loadPrivateKeysFromDirectory() error { continue } + // Skip if the key is already loaded + if _, exists := km.keys[key.KeyID()]; exists { + continue + } + km.keys[key.KeyID()] = key + log.Debugf("Loaded the private key in %s into the key manager", file.Name()) } return nil @@ -216,3 +224,26 @@ func (km *KeyManager) GetPrivateKeyByID(kid string) (jwk.Key, bool) { key, exists := km.keys[kid] return key, exists } + +// LaunchPrivateKeysDirRefresh checks the directory for new .pem files every 10 minutes +// and loads new private keys if a new file is found. +func LaunchPrivateKeysDirRefresh(ctx context.Context, egrp *errgroup.Group) { + egrp.Go(func() error { + ticker := time.NewTicker(10 * time.Minute) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + log.Debugln("Stopping periodic check for private keys directory.") + return nil + case <-ticker.C: + if err := globalKeyManager.loadPrivateKeysFromDirectory(); err != nil { + log.Errorf("Error loading private keys: %v", err) + } else { + log.Debugln("All private keys loaded successfully.") + } + } + } + }) +} diff --git a/launchers/origin_serve.go b/launchers/origin_serve.go index c07f0ea40..a69577ff8 100644 --- a/launchers/origin_serve.go +++ b/launchers/origin_serve.go @@ -33,6 +33,7 @@ import ( "github.com/spf13/viper" "golang.org/x/sync/errgroup" + "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/launcher_utils" "github.com/pelicanplatform/pelican/metrics" "github.com/pelicanplatform/pelican/oa4mp" @@ -102,6 +103,8 @@ func OriginServe(ctx context.Context, engine *gin.Engine, egrp *errgroup.Group, return nil, err } + config.LaunchPrivateKeysDirRefresh(ctx, egrp) + if param.Origin_SelfTest.GetBool() { egrp.Go(func() error { return origin.PeriodicSelfTest(ctx) }) } From da110aba87074ce3b61ff2742aa9336c0406b3df Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Thu, 31 Oct 2024 22:50:49 +0000 Subject: [PATCH 03/64] Use the latest private key --- config/init_server_creds.go | 4 ++ config/key_manager.go | 97 ++++++++++++++++++++++++------------- 2 files changed, 66 insertions(+), 35 deletions(-) diff --git a/config/init_server_creds.go b/config/init_server_creds.go index b1c373228..7d297cc1b 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -49,6 +49,10 @@ var ( issuerPrivateJWK atomic.Pointer[jwk.Key] ) +func UpdateIssuerJWKPtr(newKey jwk.Key) { + issuerPrivateJWK.Store(&newKey) +} + // Reset the atomic pointer to issuer private jwk func ResetIssuerJWKPtr() { issuerPrivateJWK.Store(nil) diff --git a/config/key_manager.go b/config/key_manager.go index 0418d4ec0..f2b3c0cfc 100644 --- a/config/key_manager.go +++ b/config/key_manager.go @@ -6,6 +6,8 @@ import ( "fmt" "os" "path/filepath" + "strconv" + "strings" "sync" "time" @@ -17,12 +19,8 @@ import ( "golang.org/x/sync/errgroup" ) -const ( - defaultPrivateKeyDir = "/etc/pelican/private-keys" // TODO: change it to a param -) - type KeyManager struct { - keys map[string]jwk.Key // Map of key ID to jwk.Key + keys map[string]jwk.Key // Map filename (e.g. `1730394365.pem`, also used as timestamp) to jwk.Key keyDir string keyMutex sync.RWMutex } @@ -34,6 +32,7 @@ var ( // GetKeyManager returns the singleton KeyManager instance func GetKeyManager() *KeyManager { + defaultPrivateKeyDir := strings.Replace(param.IssuerKey.GetString(), "issuer.jwk", "private-keys", 1) managerOnce.Do(func() { globalKeyManager = &KeyManager{ keys: make(map[string]jwk.Key), @@ -106,13 +105,8 @@ func (km *KeyManager) migrateLegacyPrivateKey() error { return errors.Wrap(err, "Failed to assign key ID to legacy key") } - kid := key.KeyID() - if kid == "" { - kid = "legacy-key" - } - // Write to new location - newPath := filepath.Join(km.keyDir, fmt.Sprintf("%s.pem", kid)) + newPath := filepath.Join(km.keyDir, fmt.Sprintf("%d.pem", time.Now().Unix())) if err := os.WriteFile(newPath, contents, 0400); err != nil { return errors.Wrap(err, "Failed to write legacy key to new location") } @@ -135,6 +129,11 @@ func (km *KeyManager) loadPrivateKeysFromDirectory() error { continue } + // Skip if the key is already loaded + if _, exists := km.keys[file.Name()]; exists { + continue + } + keyPath := filepath.Join(km.keyDir, file.Name()) key, err := km.loadSinglePrivateKey(keyPath) if err != nil { @@ -142,12 +141,7 @@ func (km *KeyManager) loadPrivateKeysFromDirectory() error { continue } - // Skip if the key is already loaded - if _, exists := km.keys[key.KeyID()]; exists { - continue - } - - km.keys[key.KeyID()] = key + km.keys[file.Name()] = key log.Debugf("Loaded the private key in %s into the key manager", file.Name()) } @@ -179,7 +173,7 @@ func (km *KeyManager) loadSinglePrivateKey(path string) (jwk.Key, error) { return key, nil } -// GetActivePrivateKey returns the current active key for signing +// GetActivePrivateKey returns the latest created key for signing func (km *KeyManager) GetActivePrivateKey() (jwk.Key, error) { km.keyMutex.RLock() defer km.keyMutex.RUnlock() @@ -189,12 +183,45 @@ func (km *KeyManager) GetActivePrivateKey() (jwk.Key, error) { return km.generateNewPrivateKey() } - // For now, return the first key (TODO: implement more sophisticated key selection) - for _, key := range km.keys { - return key, nil + var ( + latestKey jwk.Key + latestKeyCreatedTime int64 = 0 + validKeyFound bool = false + ) + + // For now, return the most recent created key (future TODO: implement key selection by the admin) + for filename, key := range km.keys { + // Parse the .pem file creation time from filename + var keyCreatedTimeStr string + pos := strings.Index(filename, ".pem") + if pos != -1 { + keyCreatedTimeStr = filename[:pos] + } else { + log.Warnf("Skipped file %s because it doesn't match the naming pattern", filename) + continue + } + keyCreatedTime, err := strconv.ParseInt(keyCreatedTimeStr, 10, 64) + if err != nil { + log.Warnf("Cannot convert %s to number", keyCreatedTimeStr) + continue + } + // Compare the timestamp of all keys to get the most recent one + if !validKeyFound || keyCreatedTime > latestKeyCreatedTime { + latestKey = key + latestKeyCreatedTime = keyCreatedTime + validKeyFound = true + } } - return nil, errors.New("No active key available") + if !validKeyFound { + return nil, errors.New("No active key available") + } + log.Debugln("Current private keys in the memory:") + for filename, _ := range km.keys { + log.Debugln(filename) + } + log.Debugf("Current private keys in use: %d.pem", latestKeyCreatedTime) + return latestKey, nil } // generateNewPrivateKey creates a new key and adds it to the manager @@ -202,7 +229,8 @@ func (km *KeyManager) generateNewPrivateKey() (jwk.Key, error) { km.keyMutex.Lock() defer km.keyMutex.Unlock() - keyPath := filepath.Join(km.keyDir, fmt.Sprintf("key-%d.pem", time.Now().Unix())) + filename := fmt.Sprintf("%d.pem", time.Now().Unix()) + keyPath := filepath.Join(km.keyDir, filename) if err := GeneratePrivateKey(keyPath, elliptic.P256(), false); err != nil { return nil, errors.Wrap(err, "Failed to generate new private key") } @@ -212,24 +240,15 @@ func (km *KeyManager) generateNewPrivateKey() (jwk.Key, error) { return nil, err } - km.keys[key.KeyID()] = key + km.keys[filename] = key return key, nil } -// GetKeyByID returns a specific key by its ID -func (km *KeyManager) GetPrivateKeyByID(kid string) (jwk.Key, bool) { - km.keyMutex.RLock() - defer km.keyMutex.RUnlock() - - key, exists := km.keys[kid] - return key, exists -} - // LaunchPrivateKeysDirRefresh checks the directory for new .pem files every 10 minutes // and loads new private keys if a new file is found. func LaunchPrivateKeysDirRefresh(ctx context.Context, egrp *errgroup.Group) { egrp.Go(func() error { - ticker := time.NewTicker(10 * time.Minute) + ticker := time.NewTicker(1 * time.Minute) defer ticker.Stop() for { @@ -241,8 +260,16 @@ func LaunchPrivateKeysDirRefresh(ctx context.Context, egrp *errgroup.Group) { if err := globalKeyManager.loadPrivateKeysFromDirectory(); err != nil { log.Errorf("Error loading private keys: %v", err) } else { - log.Debugln("All private keys loaded successfully.") + log.Debugln("Private keys directory refreshed successfully.") } + + key, err := globalKeyManager.GetActivePrivateKey() + if err != nil { + log.Errorf("Failed to get private key in use") + } + UpdateIssuerJWKPtr(key) + log.Debugf("Successfully update the private key in use in memory, kid: %s", key.KeyID()) + } } }) From 3fc8d52a087892abbd45bf8489744f352abc1e07 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 1 Nov 2024 23:07:28 +0000 Subject: [PATCH 04/64] enbale concurrent access to the issuer private keys in memory, by adopting sync.Map --- config/init_server_creds.go | 145 ++++++++++++++++++- config/key_manager.go | 276 ------------------------------------ launchers/origin_serve.go | 3 - 3 files changed, 140 insertions(+), 284 deletions(-) delete mode 100644 config/key_manager.go diff --git a/config/init_server_creds.go b/config/init_server_creds.go index 7d297cc1b..3e01196fe 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -33,9 +33,13 @@ import ( "os/exec" "path/filepath" "runtime" + "strconv" + "strings" + "sync" "sync/atomic" "time" + "github.com/lestrrat-go/jwx/v2/jwa" "github.com/lestrrat-go/jwx/v2/jwk" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -47,6 +51,9 @@ var ( // This is the private JWK for the server to sign tokens. This key remains // the same if the IssuerKey is unchanged issuerPrivateJWK atomic.Pointer[jwk.Key] + + // Representing all private keys (.pem files) in the directory + issuerPrivateKeys sync.Map ) func UpdateIssuerJWKPtr(newKey jwk.Key) { @@ -506,17 +513,145 @@ func GenerateCert() error { return nil } +// Helper function to load one .pem file from specified filename +func loadSinglePEM(path string) (jwk.Key, error) { + contents, err := os.ReadFile(path) + if err != nil { + return nil, errors.Wrap(err, "Failed to read key file") + } + + key, err := jwk.ParseKey(contents, jwk.WithPEM(true)) + if err != nil { + return nil, errors.Wrap(err, "Failed to parse key") + } + + // Add the algorithm to the key, needed for verifying tokens elsewhere + if err := key.Set(jwk.AlgorithmKey, jwa.ES256); err != nil { + return nil, errors.Wrap(err, "Failed to set algorithm") + } + + // Ensure key has an ID + if err := jwk.AssignKeyID(key); err != nil { + return nil, errors.Wrap(err, "Failed to assign key ID") + } + + return key, nil +} + +// Helper function to load all .pem files from specified directory +func loadPEMFiles(dir string) error { + files, err := os.ReadDir(dir) + if err != nil { + return errors.Wrap(err, "Failed to read directory that stores private keys") + } + + for _, file := range files { + if filepath.Ext(file.Name()) != ".pem" { + continue + } + + // Skip if the key is already loaded + if _, ok := issuerPrivateKeys.Load(file.Name()); ok { + continue + } + + keyPath := filepath.Join(dir, file.Name()) + key, err := loadSinglePEM(keyPath) + if err != nil { + log.Warnf("Failed to load key %s: %v", keyPath, err) + continue + } + + issuerPrivateKeys.Store(file.Name(), key) + log.Debugf("Loaded the private key in %s into issuerPrivateKeys", file.Name()) + } + + return nil +} + +// Helper function to check if a sync.Map is empty +func isSyncMapEmpty(m *sync.Map) bool { + isEmpty := true + m.Range(func(key, value interface{}) bool { + isEmpty = false + return false + }) + return isEmpty +} + +// Helper function to create a new .pem file and add its key to issuerPrivateKeys +func generateAndAddNewPrivateKey(dir string) (jwk.Key, error) { + filename := fmt.Sprintf("%d.pem", time.Now().Unix()) + keyPath := filepath.Join(dir, filename) + if err := GeneratePrivateKey(keyPath, elliptic.P256(), false); err != nil { + return nil, errors.Wrap(err, "Failed to generate new private key") + } + + key, err := loadSinglePEM(keyPath) + if err != nil { + log.Warnf("Failed to load key %s: %v", keyPath, err) + } + + issuerPrivateKeys.Store(filename, key) + log.Debugf("Loaded the private key in %s into issuerPrivateKeys", filename) + return key, nil +} + +// Helper function to get the most recent private key +func getLatestPrivateKey() (jwk.Key, error) { + var ( + latestKey jwk.Key + latestKeyCreatedTime int64 = 0 + validKeyFound bool = false + ) + + issuerPrivateKeys.Range(func(key, value interface{}) bool { + filename := key.(string) + // Parse the .pem file creation time from filename + var keyCreatedTimeStr string + pos := strings.Index(filename, ".pem") + if pos != -1 { + keyCreatedTimeStr = filename[:pos] + } else { + log.Warnf("Skipped file %s because it doesn't match the naming pattern", filename) + } + keyCreatedTime, err := strconv.ParseInt(keyCreatedTimeStr, 10, 64) + if err != nil { + log.Warnf("Cannot convert %s to number", keyCreatedTimeStr) + } + // Compare the timestamp of all keys to get the most recent one + if !validKeyFound || keyCreatedTime > latestKeyCreatedTime { + latestKey = value.(jwk.Key) + latestKeyCreatedTime = keyCreatedTime + validKeyFound = true + } + return true // return true to continue iterating + }) + + if !validKeyFound { + return nil, errors.New("No active key available") + } + log.Debugf("Current private key id: %s, PEM file in use: %d.pem", latestKey.KeyID(), latestKeyCreatedTime) + + return latestKey, nil +} + // Helper function to load the issuer/server's private key to sign tokens it issues. // Only intended to be called internally func loadIssuerPrivateJWK(issuerKeyFile string) (jwk.Key, error) { - km := GetKeyManager() - if err := km.Initialize(); err != nil { - return nil, errors.Wrap(err, "Failed to initialize key manager") + defaultPrivateKeyDir := strings.Replace(param.IssuerKey.GetString(), "issuer.jwk", "private-keys", 1) + err := loadPEMFiles(defaultPrivateKeyDir) + if err != nil { + return nil, errors.Wrap(err, "Failed to load .pem files") + } + + if isSyncMapEmpty(&issuerPrivateKeys) { + generateAndAddNewPrivateKey(defaultPrivateKeyDir) } - key, err := km.GetActivePrivateKey() + key, err := getLatestPrivateKey() if err != nil { - return nil, errors.Wrap(err, "Failed to get active key") + return nil, errors.Wrap(err, "Failed to get the latest private key") } // Store the key in the in-memory cache diff --git a/config/key_manager.go b/config/key_manager.go deleted file mode 100644 index f2b3c0cfc..000000000 --- a/config/key_manager.go +++ /dev/null @@ -1,276 +0,0 @@ -package config - -import ( - "context" - "crypto/elliptic" - "fmt" - "os" - "path/filepath" - "strconv" - "strings" - "sync" - "time" - - "github.com/lestrrat-go/jwx/v2/jwa" - "github.com/lestrrat-go/jwx/v2/jwk" - "github.com/pelicanplatform/pelican/param" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "golang.org/x/sync/errgroup" -) - -type KeyManager struct { - keys map[string]jwk.Key // Map filename (e.g. `1730394365.pem`, also used as timestamp) to jwk.Key - keyDir string - keyMutex sync.RWMutex -} - -var ( - globalKeyManager *KeyManager - managerOnce sync.Once -) - -// GetKeyManager returns the singleton KeyManager instance -func GetKeyManager() *KeyManager { - defaultPrivateKeyDir := strings.Replace(param.IssuerKey.GetString(), "issuer.jwk", "private-keys", 1) - managerOnce.Do(func() { - globalKeyManager = &KeyManager{ - keys: make(map[string]jwk.Key), - keyDir: defaultPrivateKeyDir, - } - }) - return globalKeyManager -} - -// Initialize sets up the key directory and loads existing keys -func (km *KeyManager) Initialize() error { - gid, err := GetDaemonGID() - if err != nil { - return err - } - // Create key directory if it doesn't exist - if err := MkdirAll(km.keyDir, 0750, -1, gid); err != nil { - return errors.Wrap(err, "Failed to create key directory") - } - - // Try to migrate legacy key if it exists - if err := km.migrateLegacyPrivateKey(); err != nil { - log.Warnf("Failed to migrate legacy key: %v", err) - // Continue execution - we'll create a new key if no key found - } - - // Load all keys from directory - if err := km.loadPrivateKeysFromDirectory(); err != nil { - log.Warnf("Failed to load keys from directory: %v", err) - // Continue execution - we'll create a new key if no key found - } - - // Check if we have any keys after migration and loading - km.keyMutex.RLock() - keyCount := len(km.keys) - km.keyMutex.RUnlock() - - if keyCount == 0 { - log.Info("No existing keys found. Generating new initial key...") - _, err := km.generateNewPrivateKey() - if err != nil { - return errors.Wrap(err, "Failed to generate initial key") - } - log.Info("Successfully generated initial key") - } - - return nil -} - -// migrateLegacyPrivateKey moves the old single key file to the new directory structure -func (km *KeyManager) migrateLegacyPrivateKey() error { - legacyPrivateKeyFile := param.IssuerKey.GetString() - - if _, err := os.Stat(legacyPrivateKeyFile); os.IsNotExist(err) { - return nil // No legacy key exists - } - - contents, err := os.ReadFile(legacyPrivateKeyFile) - if err != nil { - return errors.Wrap(err, "Failed to read legacy key file") - } - - // Parse the key to get its key ID - key, err := jwk.ParseKey(contents, jwk.WithPEM(true)) - if err != nil { - return errors.Wrap(err, "Failed to parse legacy key") - } - - if err := jwk.AssignKeyID(key); err != nil { - return errors.Wrap(err, "Failed to assign key ID to legacy key") - } - - // Write to new location - newPath := filepath.Join(km.keyDir, fmt.Sprintf("%d.pem", time.Now().Unix())) - if err := os.WriteFile(newPath, contents, 0400); err != nil { - return errors.Wrap(err, "Failed to write legacy key to new location") - } - - return nil -} - -// loadPrivateKeysFromDirectory loads all .pem files from the key directory -func (km *KeyManager) loadPrivateKeysFromDirectory() error { - km.keyMutex.Lock() - defer km.keyMutex.Unlock() - - files, err := os.ReadDir(km.keyDir) - if err != nil { - return errors.Wrap(err, "Failed to read key directory") - } - - for _, file := range files { - if filepath.Ext(file.Name()) != ".pem" { - continue - } - - // Skip if the key is already loaded - if _, exists := km.keys[file.Name()]; exists { - continue - } - - keyPath := filepath.Join(km.keyDir, file.Name()) - key, err := km.loadSinglePrivateKey(keyPath) - if err != nil { - log.Warnf("Failed to load key %s: %v", keyPath, err) - continue - } - - km.keys[file.Name()] = key - log.Debugf("Loaded the private key in %s into the key manager", file.Name()) - } - - return nil -} - -// loadSinglePrivateKey loads and prepares a single key file -func (km *KeyManager) loadSinglePrivateKey(path string) (jwk.Key, error) { - contents, err := os.ReadFile(path) - if err != nil { - return nil, errors.Wrap(err, "Failed to read key file") - } - - key, err := jwk.ParseKey(contents, jwk.WithPEM(true)) - if err != nil { - return nil, errors.Wrap(err, "Failed to parse key") - } - - // Add the algorithm to the key - if err := key.Set(jwk.AlgorithmKey, jwa.ES256); err != nil { - return nil, errors.Wrap(err, "Failed to set algorithm") - } - - // Ensure key has an ID - if err := jwk.AssignKeyID(key); err != nil { - return nil, errors.Wrap(err, "Failed to assign key ID") - } - - return key, nil -} - -// GetActivePrivateKey returns the latest created key for signing -func (km *KeyManager) GetActivePrivateKey() (jwk.Key, error) { - km.keyMutex.RLock() - defer km.keyMutex.RUnlock() - - // If no keys exist, generate one - if len(km.keys) == 0 { - return km.generateNewPrivateKey() - } - - var ( - latestKey jwk.Key - latestKeyCreatedTime int64 = 0 - validKeyFound bool = false - ) - - // For now, return the most recent created key (future TODO: implement key selection by the admin) - for filename, key := range km.keys { - // Parse the .pem file creation time from filename - var keyCreatedTimeStr string - pos := strings.Index(filename, ".pem") - if pos != -1 { - keyCreatedTimeStr = filename[:pos] - } else { - log.Warnf("Skipped file %s because it doesn't match the naming pattern", filename) - continue - } - keyCreatedTime, err := strconv.ParseInt(keyCreatedTimeStr, 10, 64) - if err != nil { - log.Warnf("Cannot convert %s to number", keyCreatedTimeStr) - continue - } - // Compare the timestamp of all keys to get the most recent one - if !validKeyFound || keyCreatedTime > latestKeyCreatedTime { - latestKey = key - latestKeyCreatedTime = keyCreatedTime - validKeyFound = true - } - } - - if !validKeyFound { - return nil, errors.New("No active key available") - } - log.Debugln("Current private keys in the memory:") - for filename, _ := range km.keys { - log.Debugln(filename) - } - log.Debugf("Current private keys in use: %d.pem", latestKeyCreatedTime) - return latestKey, nil -} - -// generateNewPrivateKey creates a new key and adds it to the manager -func (km *KeyManager) generateNewPrivateKey() (jwk.Key, error) { - km.keyMutex.Lock() - defer km.keyMutex.Unlock() - - filename := fmt.Sprintf("%d.pem", time.Now().Unix()) - keyPath := filepath.Join(km.keyDir, filename) - if err := GeneratePrivateKey(keyPath, elliptic.P256(), false); err != nil { - return nil, errors.Wrap(err, "Failed to generate new private key") - } - - key, err := km.loadSinglePrivateKey(keyPath) - if err != nil { - return nil, err - } - - km.keys[filename] = key - return key, nil -} - -// LaunchPrivateKeysDirRefresh checks the directory for new .pem files every 10 minutes -// and loads new private keys if a new file is found. -func LaunchPrivateKeysDirRefresh(ctx context.Context, egrp *errgroup.Group) { - egrp.Go(func() error { - ticker := time.NewTicker(1 * time.Minute) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - log.Debugln("Stopping periodic check for private keys directory.") - return nil - case <-ticker.C: - if err := globalKeyManager.loadPrivateKeysFromDirectory(); err != nil { - log.Errorf("Error loading private keys: %v", err) - } else { - log.Debugln("Private keys directory refreshed successfully.") - } - - key, err := globalKeyManager.GetActivePrivateKey() - if err != nil { - log.Errorf("Failed to get private key in use") - } - UpdateIssuerJWKPtr(key) - log.Debugf("Successfully update the private key in use in memory, kid: %s", key.KeyID()) - - } - } - }) -} diff --git a/launchers/origin_serve.go b/launchers/origin_serve.go index a69577ff8..c07f0ea40 100644 --- a/launchers/origin_serve.go +++ b/launchers/origin_serve.go @@ -33,7 +33,6 @@ import ( "github.com/spf13/viper" "golang.org/x/sync/errgroup" - "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/launcher_utils" "github.com/pelicanplatform/pelican/metrics" "github.com/pelicanplatform/pelican/oa4mp" @@ -103,8 +102,6 @@ func OriginServe(ctx context.Context, engine *gin.Engine, egrp *errgroup.Group, return nil, err } - config.LaunchPrivateKeysDirRefresh(ctx, egrp) - if param.Origin_SelfTest.GetBool() { egrp.Go(func() error { return origin.PeriodicSelfTest(ctx) }) } From 46380ef851d19940dc0fbbb4d98ee84d23409625 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Mon, 4 Nov 2024 21:36:38 +0000 Subject: [PATCH 05/64] Checks the directory containing .pem files every 5 minutes and loads new private key(s) if new file(s) are detected --- config/config.go | 23 +++++++++++++++++++++++ launchers/registry_serve.go | 3 +++ 2 files changed, 26 insertions(+) diff --git a/config/config.go b/config/config.go index 245c2edba..3e790ad54 100644 --- a/config/config.go +++ b/config/config.go @@ -665,6 +665,29 @@ func checkWatermark(wmStr string) (bool, int64, error) { } } +// Checks the directory containing .pem files every 5 minutes and loads new private key(s) if new file(s) are detected +func LaunchIssuerKeysDirRefresh(ctx context.Context, egrp *errgroup.Group) { + egrp.Go(func() error { + ticker := time.NewTicker(5 * time.Minute) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + log.Debugln("Stopping periodic check for private keys directory.") + return nil + case <-ticker.C: + if key, err := loadIssuerPrivateJWK(param.IssuerKey.GetName()); err != nil { + log.Errorf("Error loading private keys: %v", err) + } else { + log.Debugln("Private keys directory refreshed successfully. The latest private key in use is", key.KeyID()) + } + + } + } + }) +} + func setupTranslation() error { err := en_translations.RegisterDefaultTranslations(validate, GetEnTranslator()) if err != nil { diff --git a/launchers/registry_serve.go b/launchers/registry_serve.go index f5f9d2291..a59f820b1 100644 --- a/launchers/registry_serve.go +++ b/launchers/registry_serve.go @@ -75,6 +75,9 @@ func RegistryServe(ctx context.Context, engine *gin.Engine, egrp *errgroup.Group // Launch registry prometheus metrics registry.LaunchRegistryMetrics(ctx, egrp) + // Launch + config.LaunchIssuerKeysDirRefresh(ctx, egrp) + egrp.Go(func() error { <-ctx.Done() return registry.ShutdownRegistryDB() From 05eb5f663a2f013a6258516a3d478376f0e78920 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Thu, 7 Nov 2024 05:59:49 +0000 Subject: [PATCH 06/64] backward compatibility: migrate existing issuer key, patches and tweaks that fix all problems happened in the unit tests --- cmd/generate_keygen_test.go | 9 +-- config/config.go | 2 +- config/init_server_creds.go | 113 +++++++++++++++++++++++++++++++----- registry/registry.go | 28 ++++++--- web_ui/prometheus_test.go | 2 +- 5 files changed, 121 insertions(+), 33 deletions(-) diff --git a/cmd/generate_keygen_test.go b/cmd/generate_keygen_test.go index 3eb111509..36abd4427 100644 --- a/cmd/generate_keygen_test.go +++ b/cmd/generate_keygen_test.go @@ -50,8 +50,8 @@ func setupTestRun(t *testing.T) string { return tmpDir } -func checkKeys(t *testing.T, privateKey, publicKey string) { - _, err := config.LoadPrivateKey(privateKey, false) +func checkKeys(t *testing.T, publicKey string) { + _, err := config.GetIssuerPrivateJWK() require.NoError(t, err) jwks, err := jwk.ReadFile(publicKey) @@ -81,7 +81,6 @@ func TestKeygenMain(t *testing.T) { checkKeys( t, - filepath.Join(tempDir, "issuer.jwk"), filepath.Join(tempDir, "issuer-pub.jwks"), ) }) @@ -97,7 +96,6 @@ func TestKeygenMain(t *testing.T) { checkKeys( t, - privateKeyPath, filepath.Join(tempWd, "issuer-pub.jwks"), ) }) @@ -113,7 +111,6 @@ func TestKeygenMain(t *testing.T) { checkKeys( t, - filepath.Join(tempWd, "issuer.jwk"), publicKeyPath, ) }) @@ -130,7 +127,6 @@ func TestKeygenMain(t *testing.T) { checkKeys( t, - privateKeyPath, filepath.Join(tempWd, "issuer-pub.jwks"), ) }) @@ -147,7 +143,6 @@ func TestKeygenMain(t *testing.T) { checkKeys( t, - filepath.Join(tempWd, "issuer.jwk"), publicKeyPath, ) }) diff --git a/config/config.go b/config/config.go index 3e790ad54..d7fab4af7 100644 --- a/config/config.go +++ b/config/config.go @@ -677,7 +677,7 @@ func LaunchIssuerKeysDirRefresh(ctx context.Context, egrp *errgroup.Group) { log.Debugln("Stopping periodic check for private keys directory.") return nil case <-ticker.C: - if key, err := loadIssuerPrivateJWK(param.IssuerKey.GetName()); err != nil { + if key, err := loadIssuerPrivateKey(param.IssuerKey.GetName()); err != nil { log.Errorf("Error loading private keys: %v", err) } else { log.Debugln("Private keys directory refreshed successfully. The latest private key in use is", key.KeyID()) diff --git a/config/init_server_creds.go b/config/init_server_creds.go index 3e01196fe..8f4c29d8c 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -54,17 +54,39 @@ var ( // Representing all private keys (.pem files) in the directory issuerPrivateKeys sync.Map -) -func UpdateIssuerJWKPtr(newKey jwk.Key) { - issuerPrivateJWK.Store(&newKey) -} + // Record the value of "issuerKey" param in case it changes during runtime + currentIssuerKeysDir atomic.Value + + // Used to ensure `migratePrivateKey` is only called once + migrateOnce sync.Once +) // Reset the atomic pointer to issuer private jwk func ResetIssuerJWKPtr() { issuerPrivateJWK.Store(nil) } +// Clear all entries from the issuerPrivateKeys sync.Map +func ResetIssuerPrivateKeys() { + // delete each key from the map + issuerPrivateKeys.Range(func(key, value interface{}) bool { + issuerPrivateKeys.Delete(key) + return true // continue iterating + }) +} + +func getCurrentIssuerKeysDir() string { + if val := currentIssuerKeysDir.Load(); val != nil { + return val.(string) + } + return "" +} + +func setCurrentIssuerKeysDir(dir string) { + currentIssuerKeysDir.Store(dir) +} + // Return a pointer to an ECDSA private key or RSA private key read from keyLocation. // // This can be used to load ECDSA or RSA private key for various purposes, @@ -513,6 +535,35 @@ func GenerateCert() error { return nil } +// Helper function to move the existing single key file to new directory +func migratePrivateKey(newDir string) error { + legacyPrivateKeyFile := param.IssuerKey.GetString() + if _, err := os.Stat(legacyPrivateKeyFile); os.IsNotExist(err) { + return nil // No existing key exists + } + + // Create the destination directory if it doesn't exist + if err := os.MkdirAll(newDir, 0750); err != nil { + return errors.Wrapf(err, "Failed to create destination directory %s", newDir) + } + + // Rename the existing private key file using the naming pattern in new directory + fileName := fmt.Sprintf("%d.pem", time.Now().Unix()) + destPath := filepath.Join(newDir, fileName) + + // Check if a file with the same name already exists in the destination + if _, err := os.Stat(destPath); err == nil { + return errors.Errorf("Destination file already exists: %s", destPath) + } + + // Move the file + if err := os.Rename(legacyPrivateKeyFile, destPath); err != nil { + return errors.Wrapf(err, "Failed to move %s to %s", legacyPrivateKeyFile, destPath) + } + + return nil +} + // Helper function to load one .pem file from specified filename func loadSinglePEM(path string) (jwk.Key, error) { contents, err := os.ReadFile(path) @@ -542,6 +593,11 @@ func loadSinglePEM(path string) (jwk.Key, error) { func loadPEMFiles(dir string) error { files, err := os.ReadDir(dir) if err != nil { + // It's fine if error is "directory not found". We'll create it using generatePEM func + if os.IsNotExist(err) { + generatePEM(dir) + return nil + } return errors.Wrap(err, "Failed to read directory that stores private keys") } @@ -580,7 +636,8 @@ func isSyncMapEmpty(m *sync.Map) bool { } // Helper function to create a new .pem file and add its key to issuerPrivateKeys -func generateAndAddNewPrivateKey(dir string) (jwk.Key, error) { +// In other words, it combines GeneratePrivateKey and LoadPrivateKey functions +func generatePEM(dir string) (jwk.Key, error) { filename := fmt.Sprintf("%d.pem", time.Now().Unix()) keyPath := filepath.Join(dir, filename) if err := GeneratePrivateKey(keyPath, elliptic.P256(), false); err != nil { @@ -638,15 +695,41 @@ func getLatestPrivateKey() (jwk.Key, error) { // Helper function to load the issuer/server's private key to sign tokens it issues. // Only intended to be called internally -func loadIssuerPrivateJWK(issuerKeyFile string) (jwk.Key, error) { - defaultPrivateKeyDir := strings.Replace(param.IssuerKey.GetString(), "issuer.jwk", "private-keys", 1) +func loadIssuerPrivateKey(issuerKeysDir string) (jwk.Key, error) { + // Handles runtime changes to the issuer keys directory (configured via "IssuerKey" parameter). + // When the directory path changes: + // 1. Resets the active private key + // 2. Clears the in-memory cache of private keys (sync.Map) + // Note: This does not affect the actual key files on disk. + // Primarily used in the "invalid-token-sig-key" test scenario. + currentIssuerKeysDir := getCurrentIssuerKeysDir() + if currentIssuerKeysDir != issuerKeysDir { + log.Infof("Issuer key file has changed from %s to %s", currentIssuerKeysDir, issuerKeysDir) + ResetIssuerJWKPtr() + ResetIssuerPrivateKeys() + setCurrentIssuerKeysDir(issuerKeysDir) + } + + // Replace the leaf directory or file name with a new dir /issuer-keys + parentDir := filepath.Dir(issuerKeysDir) + defaultPrivateKeyDir := filepath.Join(parentDir, "issuer-keys") + + // Ensure migratePrivateKey is only called once across the program’s runtime + var migrateErr error + migrateOnce.Do(func() { + migrateErr = migratePrivateKey(defaultPrivateKeyDir) + }) + if migrateErr != nil { + return nil, errors.Wrap(migrateErr, "Failed to migrate existing private key file") + } + err := loadPEMFiles(defaultPrivateKeyDir) if err != nil { return nil, errors.Wrap(err, "Failed to load .pem files") } if isSyncMapEmpty(&issuerPrivateKeys) { - generateAndAddNewPrivateKey(defaultPrivateKeyDir) + generatePEM(defaultPrivateKeyDir) } key, err := getLatestPrivateKey() @@ -662,7 +745,7 @@ func loadIssuerPrivateJWK(issuerKeyFile string) (jwk.Key, error) { // Helper function to load the issuer/server's public key for other servers // to verify the token signed by this server. Only intended to be called internally -func loadIssuerPublicJWKS(existingJWKS string, issuerKeyFile string) (jwk.Set, error) { +func loadIssuerPublicJWKS(existingJWKS string, issuerKeysDir string) (jwk.Set, error) { jwks := jwk.NewSet() if existingJWKS != "" { var err error @@ -675,7 +758,7 @@ func loadIssuerPublicJWKS(existingJWKS string, issuerKeyFile string) (jwk.Set, e if key == nil { // This returns issuerPrivateJWK if it's non-nil, or find and parse private JWK // located at IssuerKey if there is one, or generate a new private key - loadedKey, err := loadIssuerPrivateJWK(issuerKeyFile) + loadedKey, err := loadIssuerPrivateKey(issuerKeysDir) if err != nil { return nil, errors.Wrap(err, "Failed to load issuer private JWK") } @@ -684,7 +767,7 @@ func loadIssuerPublicJWKS(existingJWKS string, issuerKeyFile string) (jwk.Set, e pkey, err := jwk.PublicKeyOf(*key) if err != nil { - return nil, errors.Wrapf(err, "Failed to generate public key from file %v", issuerKeyFile) + return nil, errors.Wrapf(err, "Failed to generate public key from file %v", issuerKeysDir) } if err = jwks.AddKey(pkey); err != nil { @@ -697,8 +780,8 @@ func loadIssuerPublicJWKS(existingJWKS string, issuerKeyFile string) (jwk.Set, e func GetIssuerPrivateJWK() (jwk.Key, error) { key := issuerPrivateJWK.Load() if key == nil { - issuerKeyFile := param.IssuerKey.GetString() - newKey, err := loadIssuerPrivateJWK(issuerKeyFile) + issuerKeysDir := param.IssuerKey.GetString() + newKey, err := loadIssuerPrivateKey(issuerKeysDir) if err != nil { return nil, errors.Wrap(err, "Failed to load issuer private key") } @@ -717,8 +800,8 @@ func GetIssuerPrivateJWK() (jwk.Key, error) { // i.e. "/.well-known/issuer.jwks" func GetIssuerPublicJWKS() (jwk.Set, error) { existingJWKS := param.Server_IssuerJwks.GetString() - issuerKeyFile := param.IssuerKey.GetString() - return loadIssuerPublicJWKS(existingJWKS, issuerKeyFile) + issuerKeysDir := param.IssuerKey.GetString() + return loadIssuerPublicJWKS(existingJWKS, issuerKeysDir) } // Check if there is a session secret exists at param.Server_SessionSecretFile and is not empty if there is one. diff --git a/registry/registry.go b/registry/registry.go index 077ba917e..2a084bded 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -173,16 +173,26 @@ func loadServerKeys() (*ecdsa.PrivateKey, error) { // Note: go 1.21 introduces `OnceValues` which automates this procedure. // TODO: Reimplement the function once we switch to a minimum of 1.21 serverCredsLoad.Do(func() { - issuerFileName := param.IssuerKey.GetString() - var privateKey crypto.PrivateKey - privateKey, serverCredsErr = config.LoadPrivateKey(issuerFileName, false) - - switch key := privateKey.(type) { - case *ecdsa.PrivateKey: - serverCredsPrivKey = key - default: - serverCredsErr = errors.Errorf("unsupported key type for server issuer key: %T", key) + var privateKey jwk.Key + privateKey, serverCredsErr = config.GetIssuerPrivateJWK() + if serverCredsErr != nil { + return + } + + // Extract the underlying ECDSA private key + var rawKey interface{} + if err := privateKey.Raw(&rawKey); err != nil { + serverCredsErr = errors.Errorf("failed to extract raw key: %v", err) + return } + + ecdsaKey, ok := rawKey.(*ecdsa.PrivateKey) + if !ok { + serverCredsErr = errors.Errorf("unsupported key type for server issuer key: %T", rawKey) + return + } + + serverCredsPrivKey = ecdsaKey }) return serverCredsPrivKey, serverCredsErr diff --git a/web_ui/prometheus_test.go b/web_ui/prometheus_test.go index 22197aade..c1833ea18 100644 --- a/web_ui/prometheus_test.go +++ b/web_ui/prometheus_test.go @@ -237,7 +237,7 @@ func TestPrometheusProtectionOriginHeaderScope(t *testing.T) { } // Create a new private key by re-initializing config to point at a new temp dir - k2file := filepath.Join(tDir, "testKey2") + k2file := filepath.Join(tDir, "whatever", "testKey2") viper.Set("IssuerKey", k2file) err = config.InitServer(ctx, server_structs.OriginType) require.NoError(t, err) From ab4da0cbcd56fc00cc2829d74b6cbe8f8ca10c56 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Thu, 7 Nov 2024 23:02:56 +0000 Subject: [PATCH 07/64] use atomic pointer for the in-memory private keys map --- config/config.go | 2 +- config/init_server_creds.go | 117 +++++++++++++++++++++--------------- 2 files changed, 70 insertions(+), 49 deletions(-) diff --git a/config/config.go b/config/config.go index d7fab4af7..d91a21f40 100644 --- a/config/config.go +++ b/config/config.go @@ -677,7 +677,7 @@ func LaunchIssuerKeysDirRefresh(ctx context.Context, egrp *errgroup.Group) { log.Debugln("Stopping periodic check for private keys directory.") return nil case <-ticker.C: - if key, err := loadIssuerPrivateKey(param.IssuerKey.GetName()); err != nil { + if key, err := loadIssuerPrivateKey(param.IssuerKey.GetString()); err != nil { log.Errorf("Error loading private keys: %v", err) } else { log.Debugln("Private keys directory refreshed successfully. The latest private key in use is", key.KeyID()) diff --git a/config/init_server_creds.go b/config/init_server_creds.go index 8f4c29d8c..5a77d22fa 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -53,13 +53,13 @@ var ( issuerPrivateJWK atomic.Pointer[jwk.Key] // Representing all private keys (.pem files) in the directory - issuerPrivateKeys sync.Map + issuerPrivateKeys atomic.Pointer[map[string]jwk.Key] // Record the value of "issuerKey" param in case it changes during runtime currentIssuerKeysDir atomic.Value - // Used to ensure `migratePrivateKey` is only called once - migrateOnce sync.Once + // Used to ensure initialization func init() is only called once + initOnce sync.Once ) // Reset the atomic pointer to issuer private jwk @@ -67,13 +67,22 @@ func ResetIssuerJWKPtr() { issuerPrivateJWK.Store(nil) } -// Clear all entries from the issuerPrivateKeys sync.Map +// Clear all entries from the issuerPrivateKeys func ResetIssuerPrivateKeys() { - // delete each key from the map - issuerPrivateKeys.Range(func(key, value interface{}) bool { - issuerPrivateKeys.Delete(key) - return true // continue iterating - }) + emptyMap := make(map[string]jwk.Key) + issuerPrivateKeys.Store(&emptyMap) +} + +// Safely load the current map and create a copy for modification +func getIssuerPrivateKeysCopy() map[string]jwk.Key { + currentKeys := issuerPrivateKeys.Load() + newMap := make(map[string]jwk.Key) + if currentKeys != nil { + for k, v := range *currentKeys { + newMap[k] = v + } + } + return newMap } func getCurrentIssuerKeysDir() string { @@ -187,7 +196,7 @@ func GeneratePrivateKey(keyLocation string, curve elliptic.Curve, allowRSA bool) } // If we're generating a new key, log a warning in case the user intended to pass an existing key (maybe they made a typo) - log.Warningf("IssuerKey is set to %v but the file does not exist. Will generate a new private key", param.IssuerKey.GetString()) + log.Warningf("Issuer private keys directory is set to %v but there is no file inside. Will generate a new private key", keyLocation) keyDir := filepath.Dir(keyLocation) if err := MkdirAll(keyDir, 0750, -1, gid); err != nil { @@ -564,6 +573,18 @@ func migratePrivateKey(newDir string) error { return nil } +// Helper function to initialize the in-memory map to save all private keys and migrate existing private key file +func initKeyManager(privateKeysDir string) error { + initialMap := make(map[string]jwk.Key) + issuerPrivateKeys.Store(&initialMap) + + migrateErr := migratePrivateKey(privateKeysDir) + if migrateErr != nil { + return errors.Wrap(migrateErr, "Failed to migrate existing private key file") + } + return nil +} + // Helper function to load one .pem file from specified filename func loadSinglePEM(path string) (jwk.Key, error) { contents, err := os.ReadFile(path) @@ -593,7 +614,7 @@ func loadSinglePEM(path string) (jwk.Key, error) { func loadPEMFiles(dir string) error { files, err := os.ReadDir(dir) if err != nil { - // It's fine if error is "directory not found". We'll create it using generatePEM func + // It's fine if error is "directory not found". We'll create this dir using generatePEM func if os.IsNotExist(err) { generatePEM(dir) return nil @@ -601,14 +622,16 @@ func loadPEMFiles(dir string) error { return errors.Wrap(err, "Failed to read directory that stores private keys") } + currentKeys := issuerPrivateKeys.Load() for _, file := range files { if filepath.Ext(file.Name()) != ".pem" { continue } - // Skip if the key is already loaded - if _, ok := issuerPrivateKeys.Load(file.Name()); ok { - continue + if currentKeys != nil { + if _, exists := (*currentKeys)[file.Name()]; exists { + continue // Skip if key already loaded + } } keyPath := filepath.Join(dir, file.Name()) @@ -618,21 +641,21 @@ func loadPEMFiles(dir string) error { continue } - issuerPrivateKeys.Store(file.Name(), key) + newKeys := getIssuerPrivateKeysCopy() + newKeys[file.Name()] = key + issuerPrivateKeys.Store(&newKeys) log.Debugf("Loaded the private key in %s into issuerPrivateKeys", file.Name()) } return nil } -// Helper function to check if a sync.Map is empty -func isSyncMapEmpty(m *sync.Map) bool { - isEmpty := true - m.Range(func(key, value interface{}) bool { - isEmpty = false - return false - }) - return isEmpty +func isIssuerPrivateKeysEmpty() bool { + currentKeys := issuerPrivateKeys.Load() + if currentKeys == nil { + return true // Map has not been initialized, so it's considered empty + } + return len(*currentKeys) == 0 } // Helper function to create a new .pem file and add its key to issuerPrivateKeys @@ -649,41 +672,39 @@ func generatePEM(dir string) (jwk.Key, error) { log.Warnf("Failed to load key %s: %v", keyPath, err) } - issuerPrivateKeys.Store(filename, key) + newKeys := getIssuerPrivateKeysCopy() + newKeys[filename] = key + issuerPrivateKeys.Store(&newKeys) log.Debugf("Loaded the private key in %s into issuerPrivateKeys", filename) return key, nil } // Helper function to get the most recent private key func getLatestPrivateKey() (jwk.Key, error) { + currentKeys := issuerPrivateKeys.Load() + if currentKeys == nil { + return nil, errors.New("No keys loaded in issuerPrivateKeys") + } var ( latestKey jwk.Key latestKeyCreatedTime int64 = 0 validKeyFound bool = false ) - issuerPrivateKeys.Range(func(key, value interface{}) bool { - filename := key.(string) + for filename, key := range *currentKeys { // Parse the .pem file creation time from filename - var keyCreatedTimeStr string - pos := strings.Index(filename, ".pem") - if pos != -1 { - keyCreatedTimeStr = filename[:pos] - } else { - log.Warnf("Skipped file %s because it doesn't match the naming pattern", filename) - } - keyCreatedTime, err := strconv.ParseInt(keyCreatedTimeStr, 10, 64) + keyCreatedTime, err := strconv.ParseInt(strings.TrimSuffix(filename, ".pem"), 10, 64) if err != nil { - log.Warnf("Cannot convert %s to number", keyCreatedTimeStr) + log.Warnf("Cannot convert %s to timestamp: %v", filename, err) + continue } // Compare the timestamp of all keys to get the most recent one if !validKeyFound || keyCreatedTime > latestKeyCreatedTime { - latestKey = value.(jwk.Key) + latestKey = key latestKeyCreatedTime = keyCreatedTime validKeyFound = true } - return true // return true to continue iterating - }) + } if !validKeyFound { return nil, errors.New("No active key available") @@ -699,12 +720,12 @@ func loadIssuerPrivateKey(issuerKeysDir string) (jwk.Key, error) { // Handles runtime changes to the issuer keys directory (configured via "IssuerKey" parameter). // When the directory path changes: // 1. Resets the active private key - // 2. Clears the in-memory cache of private keys (sync.Map) + // 2. Clears the in-memory cache of all private keys // Note: This does not affect the actual key files on disk. // Primarily used in the "invalid-token-sig-key" test scenario. currentIssuerKeysDir := getCurrentIssuerKeysDir() if currentIssuerKeysDir != issuerKeysDir { - log.Infof("Issuer key file has changed from %s to %s", currentIssuerKeysDir, issuerKeysDir) + log.Debugf("The private keys dir generated by IssuerKey param has changed from '%s' to '%s'", currentIssuerKeysDir, issuerKeysDir) ResetIssuerJWKPtr() ResetIssuerPrivateKeys() setCurrentIssuerKeysDir(issuerKeysDir) @@ -714,13 +735,13 @@ func loadIssuerPrivateKey(issuerKeysDir string) (jwk.Key, error) { parentDir := filepath.Dir(issuerKeysDir) defaultPrivateKeyDir := filepath.Join(parentDir, "issuer-keys") - // Ensure migratePrivateKey is only called once across the program’s runtime - var migrateErr error - migrateOnce.Do(func() { - migrateErr = migratePrivateKey(defaultPrivateKeyDir) + // Ensure initKeyManager is only called once across the program’s runtime + var initErr error + initOnce.Do(func() { + initErr = initKeyManager(defaultPrivateKeyDir) }) - if migrateErr != nil { - return nil, errors.Wrap(migrateErr, "Failed to migrate existing private key file") + if initErr != nil { + return nil, errors.Wrap(initErr, "Failed to initialize and/or migrate existing private key file") } err := loadPEMFiles(defaultPrivateKeyDir) @@ -728,7 +749,7 @@ func loadIssuerPrivateKey(issuerKeysDir string) (jwk.Key, error) { return nil, errors.Wrap(err, "Failed to load .pem files") } - if isSyncMapEmpty(&issuerPrivateKeys) { + if isIssuerPrivateKeysEmpty() { generatePEM(defaultPrivateKeyDir) } @@ -737,7 +758,7 @@ func loadIssuerPrivateKey(issuerKeysDir string) (jwk.Key, error) { return nil, errors.Wrap(err, "Failed to get the latest private key") } - // Store the key in the in-memory cache + // Store the active key in the in-memory cache issuerPrivateJWK.Store(&key) return key, nil From e6a72f21572a3c7a0728171d715849b9b6ef47f4 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Mon, 11 Nov 2024 19:42:05 +0000 Subject: [PATCH 08/64] newIssuerKey API endpoint on origin, and unit tests --- config/init_server_creds.go | 81 +++++++++++++++++--------------- config/init_server_creds_test.go | 38 +++++++++++++++ origin/origin_ui.go | 29 ++++++++++++ 3 files changed, 111 insertions(+), 37 deletions(-) diff --git a/config/init_server_creds.go b/config/init_server_creds.go index 5a77d22fa..8601af057 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -56,7 +56,7 @@ var ( issuerPrivateKeys atomic.Pointer[map[string]jwk.Key] // Record the value of "issuerKey" param in case it changes during runtime - currentIssuerKeysDir atomic.Value + currentIssuerKeyDir atomic.Value // Used to ensure initialization func init() is only called once initOnce sync.Once @@ -85,15 +85,15 @@ func getIssuerPrivateKeysCopy() map[string]jwk.Key { return newMap } -func getCurrentIssuerKeysDir() string { - if val := currentIssuerKeysDir.Load(); val != nil { +func getCurrentIssuerKeyDir() string { + if val := currentIssuerKeyDir.Load(); val != nil { return val.(string) } return "" } -func setCurrentIssuerKeysDir(dir string) { - currentIssuerKeysDir.Store(dir) +func setCurrentIssuerKeyDir(dir string) { + currentIssuerKeyDir.Store(dir) } // Return a pointer to an ECDSA private key or RSA private key read from keyLocation. @@ -196,7 +196,7 @@ func GeneratePrivateKey(keyLocation string, curve elliptic.Curve, allowRSA bool) } // If we're generating a new key, log a warning in case the user intended to pass an existing key (maybe they made a typo) - log.Warningf("Issuer private keys directory is set to %v but there is no file inside. Will generate a new private key", keyLocation) + log.Warningf("Will generate a new private key at location: %v", keyLocation) keyDir := filepath.Dir(keyLocation) if err := MkdirAll(keyDir, 0750, -1, gid); err != nil { @@ -574,7 +574,7 @@ func migratePrivateKey(newDir string) error { } // Helper function to initialize the in-memory map to save all private keys and migrate existing private key file -func initKeyManager(privateKeysDir string) error { +func initKeysMap(privateKeysDir string) error { initialMap := make(map[string]jwk.Key) issuerPrivateKeys.Store(&initialMap) @@ -614,9 +614,9 @@ func loadSinglePEM(path string) (jwk.Key, error) { func loadPEMFiles(dir string) error { files, err := os.ReadDir(dir) if err != nil { - // It's fine if error is "directory not found". We'll create this dir using generatePEM func + // It's fine if error is "directory not found". We'll create this dir using GeneratePEM func if os.IsNotExist(err) { - generatePEM(dir) + GeneratePEM(dir) return nil } return errors.Wrap(err, "Failed to read directory that stores private keys") @@ -658,9 +658,10 @@ func isIssuerPrivateKeysEmpty() bool { return len(*currentKeys) == 0 } -// Helper function to create a new .pem file and add its key to issuerPrivateKeys +// Create a new .pem file and add its key to issuerPrivateKeys // In other words, it combines GeneratePrivateKey and LoadPrivateKey functions -func generatePEM(dir string) (jwk.Key, error) { +// This function is used in origin API to let admin generate new private key +func GeneratePEM(dir string) (jwk.Key, error) { filename := fmt.Sprintf("%d.pem", time.Now().Unix()) keyPath := filepath.Join(dir, filename) if err := GeneratePrivateKey(keyPath, elliptic.P256(), false); err != nil { @@ -714,59 +715,65 @@ func getLatestPrivateKey() (jwk.Key, error) { return latestKey, nil } +// Refresh to get the latest private key and save it to issuerPrivateJWK +func RefreshActivePrivateKey() (jwk.Key, error) { + key, err := getLatestPrivateKey() + if err != nil { + return nil, errors.Wrap(err, "Failed to get the latest private key") + } + + // Store the active key in the in-memory cache + issuerPrivateJWK.Store(&key) + return key, nil +} + // Helper function to load the issuer/server's private key to sign tokens it issues. // Only intended to be called internally -func loadIssuerPrivateKey(issuerKeysDir string) (jwk.Key, error) { +func loadIssuerPrivateKey(issuerKeyDir string) (jwk.Key, error) { // Handles runtime changes to the issuer keys directory (configured via "IssuerKey" parameter). // When the directory path changes: // 1. Resets the active private key // 2. Clears the in-memory cache of all private keys // Note: This does not affect the actual key files on disk. // Primarily used in the "invalid-token-sig-key" test scenario. - currentIssuerKeysDir := getCurrentIssuerKeysDir() - if currentIssuerKeysDir != issuerKeysDir { - log.Debugf("The private keys dir generated by IssuerKey param has changed from '%s' to '%s'", currentIssuerKeysDir, issuerKeysDir) + currentIssuerKeyDir := getCurrentIssuerKeyDir() + if currentIssuerKeyDir != issuerKeyDir { + log.Debugf("The private keys dir generated by IssuerKey param has changed from '%s' to '%s'", currentIssuerKeyDir, issuerKeyDir) ResetIssuerJWKPtr() ResetIssuerPrivateKeys() - setCurrentIssuerKeysDir(issuerKeysDir) + setCurrentIssuerKeyDir(issuerKeyDir) } // Replace the leaf directory or file name with a new dir /issuer-keys - parentDir := filepath.Dir(issuerKeysDir) - defaultPrivateKeyDir := filepath.Join(parentDir, "issuer-keys") + parentDir := filepath.Dir(issuerKeyDir) + defaultPrivateKeysDir := filepath.Join(parentDir, "issuer-keys") - // Ensure initKeyManager is only called once across the program’s runtime + // Ensure initKeysMap is only called once across the program’s runtime var initErr error initOnce.Do(func() { - initErr = initKeyManager(defaultPrivateKeyDir) + initErr = initKeysMap(defaultPrivateKeysDir) }) if initErr != nil { return nil, errors.Wrap(initErr, "Failed to initialize and/or migrate existing private key file") } - err := loadPEMFiles(defaultPrivateKeyDir) + err := loadPEMFiles(defaultPrivateKeysDir) if err != nil { return nil, errors.Wrap(err, "Failed to load .pem files") } if isIssuerPrivateKeysEmpty() { - generatePEM(defaultPrivateKeyDir) + GeneratePEM(defaultPrivateKeysDir) } - key, err := getLatestPrivateKey() - if err != nil { - return nil, errors.Wrap(err, "Failed to get the latest private key") - } + key, err := RefreshActivePrivateKey() - // Store the active key in the in-memory cache - issuerPrivateJWK.Store(&key) - - return key, nil + return key, err } // Helper function to load the issuer/server's public key for other servers // to verify the token signed by this server. Only intended to be called internally -func loadIssuerPublicJWKS(existingJWKS string, issuerKeysDir string) (jwk.Set, error) { +func loadIssuerPublicJWKS(existingJWKS string, issuerKeyDir string) (jwk.Set, error) { jwks := jwk.NewSet() if existingJWKS != "" { var err error @@ -779,7 +786,7 @@ func loadIssuerPublicJWKS(existingJWKS string, issuerKeysDir string) (jwk.Set, e if key == nil { // This returns issuerPrivateJWK if it's non-nil, or find and parse private JWK // located at IssuerKey if there is one, or generate a new private key - loadedKey, err := loadIssuerPrivateKey(issuerKeysDir) + loadedKey, err := loadIssuerPrivateKey(issuerKeyDir) if err != nil { return nil, errors.Wrap(err, "Failed to load issuer private JWK") } @@ -788,7 +795,7 @@ func loadIssuerPublicJWKS(existingJWKS string, issuerKeysDir string) (jwk.Set, e pkey, err := jwk.PublicKeyOf(*key) if err != nil { - return nil, errors.Wrapf(err, "Failed to generate public key from file %v", issuerKeysDir) + return nil, errors.Wrapf(err, "Failed to generate public key from file %v", issuerKeyDir) } if err = jwks.AddKey(pkey); err != nil { @@ -801,8 +808,8 @@ func loadIssuerPublicJWKS(existingJWKS string, issuerKeysDir string) (jwk.Set, e func GetIssuerPrivateJWK() (jwk.Key, error) { key := issuerPrivateJWK.Load() if key == nil { - issuerKeysDir := param.IssuerKey.GetString() - newKey, err := loadIssuerPrivateKey(issuerKeysDir) + issuerKeyDir := param.IssuerKey.GetString() + newKey, err := loadIssuerPrivateKey(issuerKeyDir) if err != nil { return nil, errors.Wrap(err, "Failed to load issuer private key") } @@ -821,8 +828,8 @@ func GetIssuerPrivateJWK() (jwk.Key, error) { // i.e. "/.well-known/issuer.jwks" func GetIssuerPublicJWKS() (jwk.Set, error) { existingJWKS := param.Server_IssuerJwks.GetString() - issuerKeysDir := param.IssuerKey.GetString() - return loadIssuerPublicJWKS(existingJWKS, issuerKeysDir) + issuerKeyDir := param.IssuerKey.GetString() + return loadIssuerPublicJWKS(existingJWKS, issuerKeyDir) } // Check if there is a session secret exists at param.Server_SessionSecretFile and is not empty if there is one. diff --git a/config/init_server_creds_test.go b/config/init_server_creds_test.go index e29cc2f9d..ddea13719 100644 --- a/config/init_server_creds_test.go +++ b/config/init_server_creds_test.go @@ -28,7 +28,9 @@ import ( "os" "path/filepath" "testing" + "time" + "github.com/pelicanplatform/pelican/param" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -150,3 +152,39 @@ func TestLoadPrivateKey(t *testing.T) { require.Nil(t, privateKey) }) } + +func TestLoadIssuerPrivateKey(t *testing.T) { + tempDir := t.TempDir() + issuerKeyDir := filepath.Join(tempDir, param.IssuerKey.GetString()) + + key, err := loadIssuerPrivateKey(issuerKeyDir) + require.NoError(t, err) + require.NotNil(t, key) +} + +// This test also imitates the origin API endpoint "/newIssuerKey" +func TestSecondPrivateKey(t *testing.T) { + tempDir := t.TempDir() + issuerKeyDir := filepath.Join(tempDir, param.IssuerKey.GetString()) + + key, err := loadIssuerPrivateKey(issuerKeyDir) + require.NoError(t, err) + require.NotNil(t, key) + + // Wait for 1 second to avoid duplicated private key filenames + // because they are named after unix epoch timestamp + time.Sleep(1 * time.Second) + + // Create another private key + parentDir := filepath.Dir(issuerKeyDir) + defaultPrivateKeysDir := filepath.Join(parentDir, "issuer-keys") + secondKey, err := GeneratePEM(defaultPrivateKeysDir) + require.NoError(t, err) + require.NotNil(t, secondKey) + assert.NotEqual(t, key.KeyID(), secondKey.KeyID()) + + // Check if the active private key points to the lastest key + latestKey, err := RefreshActivePrivateKey() + require.NoError(t, err) + assert.Equal(t, secondKey.KeyID(), latestKey.KeyID()) +} diff --git a/origin/origin_ui.go b/origin/origin_ui.go index 07a16dc25..d7b7d27e2 100644 --- a/origin/origin_ui.go +++ b/origin/origin_ui.go @@ -21,6 +21,7 @@ package origin import ( "net/http" "net/url" + "path/filepath" "time" "github.com/gin-gonic/gin" @@ -161,10 +162,38 @@ func handleExports(ctx *gin.Context) { ctx.JSON(http.StatusOK, res) } +func createNewIssuerKey(ctx *gin.Context) { + issuerKeyDir := param.IssuerKey.GetString() + parentDir := filepath.Dir(issuerKeyDir) + defaultPrivateKeysDir := filepath.Join(parentDir, "issuer-keys") + + _, err := config.GeneratePEM(defaultPrivateKeysDir) + if err != nil { + log.Errorf("Error creating and loading a new private key in a new .pem file: %v", err) + ctx.JSON(http.StatusInternalServerError, server_structs.SimpleApiResp{ + Status: server_structs.RespFailed, + Msg: "Error creating and loading a new private key in a new .pem file"}) + } + _, err = config.RefreshActivePrivateKey() + if err != nil { + log.Errorf("Error retrieving latest private key: %v", err) + ctx.JSON(http.StatusInternalServerError, server_structs.SimpleApiResp{ + Status: server_structs.RespFailed, + Msg: "Error retrieving latest private key"}) + } + + ctx.JSON(http.StatusOK, + server_structs.SimpleApiResp{ + Status: server_structs.RespOK, + Msg: "success", + }) +} + func RegisterOriginWebAPI(engine *gin.Engine) error { originWebAPI := engine.Group("/api/v1.0/origin_ui") { originWebAPI.GET("/exports", web_ui.AuthHandler, web_ui.AdminAuthHandler, handleExports) + originWebAPI.GET("/newIssuerKey", web_ui.AuthHandler, web_ui.AdminAuthHandler, createNewIssuerKey) } // Globus backend specific. Config other origin routes above this line From 5ba6057e84e5a95b4b332a7f2f3233070e2bc7d6 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Tue, 12 Nov 2024 00:26:07 +0000 Subject: [PATCH 09/64] get the most recent modified private key from file dir, simply the code --- config/init_server_creds.go | 168 +++++++++++++++---------------- config/init_server_creds_test.go | 4 +- origin/origin_ui.go | 10 +- 3 files changed, 85 insertions(+), 97 deletions(-) diff --git a/config/init_server_creds.go b/config/init_server_creds.go index 8601af057..7bc3d1d71 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -33,8 +33,6 @@ import ( "os/exec" "path/filepath" "runtime" - "strconv" - "strings" "sync" "sync/atomic" "time" @@ -67,6 +65,11 @@ func ResetIssuerJWKPtr() { issuerPrivateJWK.Store(nil) } +// Set a private key as the active private key in use +func SetActiveKey(key jwk.Key) { + issuerPrivateJWK.Store(&key) +} + // Clear all entries from the issuerPrivateKeys func ResetIssuerPrivateKeys() { emptyMap := make(map[string]jwk.Key) @@ -553,21 +556,27 @@ func migratePrivateKey(newDir string) error { // Create the destination directory if it doesn't exist if err := os.MkdirAll(newDir, 0750); err != nil { - return errors.Wrapf(err, "Failed to create destination directory %s", newDir) + return errors.Wrapf(err, "failed to create destination directory %s", newDir) + } + + // Get the key id of the existing key + key, err := loadSinglePEM(legacyPrivateKeyFile) + if err != nil { + log.Warnf("Failed to load key %s: %v", key.KeyID(), err) } // Rename the existing private key file using the naming pattern in new directory - fileName := fmt.Sprintf("%d.pem", time.Now().Unix()) + fileName := fmt.Sprintf("%s.pem", key.KeyID()) destPath := filepath.Join(newDir, fileName) // Check if a file with the same name already exists in the destination if _, err := os.Stat(destPath); err == nil { - return errors.Errorf("Destination file already exists: %s", destPath) + return errors.Errorf("destination file already exists: %s", destPath) } // Move the file if err := os.Rename(legacyPrivateKeyFile, destPath); err != nil { - return errors.Wrapf(err, "Failed to move %s to %s", legacyPrivateKeyFile, destPath) + return errors.Wrapf(err, "failed to move %s to %s", legacyPrivateKeyFile, destPath) } return nil @@ -580,7 +589,7 @@ func initKeysMap(privateKeysDir string) error { migrateErr := migratePrivateKey(privateKeysDir) if migrateErr != nil { - return errors.Wrap(migrateErr, "Failed to migrate existing private key file") + return errors.Wrap(migrateErr, "failed to migrate existing private key file") } return nil } @@ -589,65 +598,84 @@ func initKeysMap(privateKeysDir string) error { func loadSinglePEM(path string) (jwk.Key, error) { contents, err := os.ReadFile(path) if err != nil { - return nil, errors.Wrap(err, "Failed to read key file") + return nil, errors.Wrap(err, "failed to read key file") } key, err := jwk.ParseKey(contents, jwk.WithPEM(true)) if err != nil { - return nil, errors.Wrap(err, "Failed to parse key") + return nil, errors.Wrap(err, "failed to parse key") } // Add the algorithm to the key, needed for verifying tokens elsewhere if err := key.Set(jwk.AlgorithmKey, jwa.ES256); err != nil { - return nil, errors.Wrap(err, "Failed to set algorithm") + return nil, errors.Wrap(err, "failed to set algorithm") } // Ensure key has an ID if err := jwk.AssignKeyID(key); err != nil { - return nil, errors.Wrap(err, "Failed to assign key ID") + return nil, errors.Wrap(err, "failed to assign key ID") } return key, nil } -// Helper function to load all .pem files from specified directory -func loadPEMFiles(dir string) error { +// Helper function to load/refresh all .pem files from specified directory and find the most recent modified private key +func loadPEMFiles(dir string) (jwk.Key, error) { + var mostRecentKey jwk.Key + var mostRecentModTime time.Time + files, err := os.ReadDir(dir) if err != nil { // It's fine if error is "directory not found". We'll create this dir using GeneratePEM func if os.IsNotExist(err) { - GeneratePEM(dir) - return nil + newKey, err := GeneratePEMandSetActiveKey(dir) + if err != nil { + return nil, errors.Wrapf(err, "failed to create a new .pem file to save private key") + } + return newKey, nil } - return errors.Wrap(err, "Failed to read directory that stores private keys") + return nil, errors.Wrap(err, "failed to read directory that stores private keys") } - currentKeys := issuerPrivateKeys.Load() + latestKeys := make(map[string]jwk.Key) for _, file := range files { if filepath.Ext(file.Name()) != ".pem" { continue } - - if currentKeys != nil { - if _, exists := (*currentKeys)[file.Name()]; exists { - continue // Skip if key already loaded - } - } - + // parse the private key in this file and add to the in-memory keys map keyPath := filepath.Join(dir, file.Name()) key, err := loadSinglePEM(keyPath) if err != nil { log.Warnf("Failed to load key %s: %v", keyPath, err) continue } + latestKeys[key.KeyID()] = key - newKeys := getIssuerPrivateKeysCopy() - newKeys[file.Name()] = key - issuerPrivateKeys.Store(&newKeys) - log.Debugf("Loaded the private key in %s into issuerPrivateKeys", file.Name()) + // find the most recent modified file + fileInfo, err := file.Info() + if err != nil { + continue + } + if fileInfo.ModTime().After(mostRecentModTime) { + mostRecentModTime = fileInfo.ModTime() + mostRecentKey = key + } + } + // The directory exists but contains no .pem files + if len(latestKeys) == 0 { + newKey, err := GeneratePEMandSetActiveKey(dir) + if err != nil { + return nil, errors.Wrapf(err, "failed to create a new .pem file to save private key") + } + return newKey, nil } - return nil + // Save up-to-date private keys and the active key in memory + issuerPrivateKeys.Store(&latestKeys) + SetActiveKey(mostRecentKey) + log.Debugf("Set private key %s as active", mostRecentKey.KeyID()) + + return mostRecentKey, nil } func isIssuerPrivateKeysEmpty() bool { @@ -660,12 +688,11 @@ func isIssuerPrivateKeysEmpty() bool { // Create a new .pem file and add its key to issuerPrivateKeys // In other words, it combines GeneratePrivateKey and LoadPrivateKey functions -// This function is used in origin API to let admin generate new private key func GeneratePEM(dir string) (jwk.Key, error) { filename := fmt.Sprintf("%d.pem", time.Now().Unix()) keyPath := filepath.Join(dir, filename) if err := GeneratePrivateKey(keyPath, elliptic.P256(), false); err != nil { - return nil, errors.Wrap(err, "Failed to generate new private key") + return nil, errors.Wrap(err, "failed to generate new private key") } key, err := loadSinglePEM(keyPath) @@ -673,58 +700,31 @@ func GeneratePEM(dir string) (jwk.Key, error) { log.Warnf("Failed to load key %s: %v", keyPath, err) } - newKeys := getIssuerPrivateKeysCopy() - newKeys[filename] = key - issuerPrivateKeys.Store(&newKeys) - log.Debugf("Loaded the private key in %s into issuerPrivateKeys", filename) - return key, nil -} - -// Helper function to get the most recent private key -func getLatestPrivateKey() (jwk.Key, error) { - currentKeys := issuerPrivateKeys.Load() - if currentKeys == nil { - return nil, errors.New("No keys loaded in issuerPrivateKeys") + // Rename the file using its key.KeyID() + newFilename := fmt.Sprintf("%s.pem", key.KeyID()) + newKeyPath := filepath.Join(dir, newFilename) + if err := os.Rename(keyPath, newKeyPath); err != nil { + return nil, errors.Wrap(err, "failed to rename private key file with KeyID") } - var ( - latestKey jwk.Key - latestKeyCreatedTime int64 = 0 - validKeyFound bool = false - ) - for filename, key := range *currentKeys { - // Parse the .pem file creation time from filename - keyCreatedTime, err := strconv.ParseInt(strings.TrimSuffix(filename, ".pem"), 10, 64) - if err != nil { - log.Warnf("Cannot convert %s to timestamp: %v", filename, err) - continue - } - // Compare the timestamp of all keys to get the most recent one - if !validKeyFound || keyCreatedTime > latestKeyCreatedTime { - latestKey = key - latestKeyCreatedTime = keyCreatedTime - validKeyFound = true - } - } - - if !validKeyFound { - return nil, errors.New("No active key available") - } - log.Debugf("Current private key id: %s, PEM file in use: %d.pem", latestKey.KeyID(), latestKeyCreatedTime) + // Save this new key in the in-memory map for all private keys + newKeys := getIssuerPrivateKeysCopy() + newKeys[key.KeyID()] = key + issuerPrivateKeys.Store(&newKeys) - return latestKey, nil + log.Debugf("Loaded the private key (key id: %s) into issuerPrivateKeys and set as active key", key.KeyID()) + return key, nil } -// Refresh to get the latest private key and save it to issuerPrivateJWK -func RefreshActivePrivateKey() (jwk.Key, error) { - key, err := getLatestPrivateKey() +// Generate a new .pem file and then set the private key it contains as the active one +// This function is also used in origin API to let admin generate new private key +func GeneratePEMandSetActiveKey(dir string) (jwk.Key, error) { + newKey, err := GeneratePEM(dir) if err != nil { - return nil, errors.Wrap(err, "Failed to get the latest private key") + return nil, errors.Wrapf(err, "failed to create a new .pem file to save private key") } - - // Store the active key in the in-memory cache - issuerPrivateJWK.Store(&key) - return key, nil + SetActiveKey(newKey) + return newKey, nil } // Helper function to load the issuer/server's private key to sign tokens it issues. @@ -754,21 +754,15 @@ func loadIssuerPrivateKey(issuerKeyDir string) (jwk.Key, error) { initErr = initKeysMap(defaultPrivateKeysDir) }) if initErr != nil { - return nil, errors.Wrap(initErr, "Failed to initialize and/or migrate existing private key file") + return nil, errors.Wrap(initErr, "failed to initialize and/or migrate existing private key file") } - err := loadPEMFiles(defaultPrivateKeysDir) + activeKey, err := loadPEMFiles(defaultPrivateKeysDir) if err != nil { - return nil, errors.Wrap(err, "Failed to load .pem files") + return nil, errors.Wrap(err, "failed to load .pem files and find the most recent private key") } - if isIssuerPrivateKeysEmpty() { - GeneratePEM(defaultPrivateKeysDir) - } - - key, err := RefreshActivePrivateKey() - - return key, err + return activeKey, err } // Helper function to load the issuer/server's public key for other servers @@ -795,7 +789,7 @@ func loadIssuerPublicJWKS(existingJWKS string, issuerKeyDir string) (jwk.Set, er pkey, err := jwk.PublicKeyOf(*key) if err != nil { - return nil, errors.Wrapf(err, "Failed to generate public key from file %v", issuerKeyDir) + return nil, errors.Wrapf(err, "failed to generate public key from file %v", issuerKeyDir) } if err = jwks.AddKey(pkey); err != nil { diff --git a/config/init_server_creds_test.go b/config/init_server_creds_test.go index ddea13719..35fd44603 100644 --- a/config/init_server_creds_test.go +++ b/config/init_server_creds_test.go @@ -178,13 +178,13 @@ func TestSecondPrivateKey(t *testing.T) { // Create another private key parentDir := filepath.Dir(issuerKeyDir) defaultPrivateKeysDir := filepath.Join(parentDir, "issuer-keys") - secondKey, err := GeneratePEM(defaultPrivateKeysDir) + secondKey, err := GeneratePEMandSetActiveKey(defaultPrivateKeysDir) require.NoError(t, err) require.NotNil(t, secondKey) assert.NotEqual(t, key.KeyID(), secondKey.KeyID()) // Check if the active private key points to the lastest key - latestKey, err := RefreshActivePrivateKey() + latestKey, err := GetIssuerPrivateJWK() require.NoError(t, err) assert.Equal(t, secondKey.KeyID(), latestKey.KeyID()) } diff --git a/origin/origin_ui.go b/origin/origin_ui.go index d7b7d27e2..b4f6314fb 100644 --- a/origin/origin_ui.go +++ b/origin/origin_ui.go @@ -162,25 +162,19 @@ func handleExports(ctx *gin.Context) { ctx.JSON(http.StatusOK, res) } +// Create a new private key in the given directory and set it as the active key func createNewIssuerKey(ctx *gin.Context) { issuerKeyDir := param.IssuerKey.GetString() parentDir := filepath.Dir(issuerKeyDir) defaultPrivateKeysDir := filepath.Join(parentDir, "issuer-keys") - _, err := config.GeneratePEM(defaultPrivateKeysDir) + _, err := config.GeneratePEMandSetActiveKey(defaultPrivateKeysDir) if err != nil { log.Errorf("Error creating and loading a new private key in a new .pem file: %v", err) ctx.JSON(http.StatusInternalServerError, server_structs.SimpleApiResp{ Status: server_structs.RespFailed, Msg: "Error creating and loading a new private key in a new .pem file"}) } - _, err = config.RefreshActivePrivateKey() - if err != nil { - log.Errorf("Error retrieving latest private key: %v", err) - ctx.JSON(http.StatusInternalServerError, server_structs.SimpleApiResp{ - Status: server_structs.RespFailed, - Msg: "Error retrieving latest private key"}) - } ctx.JSON(http.StatusOK, server_structs.SimpleApiResp{ From 04342d0d2ba084c8c29f597b875c967d268cf9b7 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 15 Nov 2024 21:47:29 +0000 Subject: [PATCH 10/64] move the LaunchIssuerKeysDirRefresh func to origin service --- launchers/origin_serve.go | 5 +++++ launchers/registry_serve.go | 3 --- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/launchers/origin_serve.go b/launchers/origin_serve.go index c07f0ea40..4e987768a 100644 --- a/launchers/origin_serve.go +++ b/launchers/origin_serve.go @@ -33,6 +33,7 @@ import ( "github.com/spf13/viper" "golang.org/x/sync/errgroup" + "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/launcher_utils" "github.com/pelicanplatform/pelican/metrics" "github.com/pelicanplatform/pelican/oa4mp" @@ -76,6 +77,10 @@ func OriginServe(ctx context.Context, engine *gin.Engine, egrp *errgroup.Group, origin.LaunchGlobusTokenRefresh(ctx, egrp) } + // Start a routine to periodically refresh the private key directory. + // This ensures that new or updated private keys are automatically loaded + config.LaunchIssuerKeysDirRefresh(ctx, egrp) + // Set up the APIs unrelated to UI, which only contains director-based health test reporting endpoint for now if err = origin.RegisterOriginAPI(engine, ctx, egrp); err != nil { return nil, err diff --git a/launchers/registry_serve.go b/launchers/registry_serve.go index a59f820b1..f5f9d2291 100644 --- a/launchers/registry_serve.go +++ b/launchers/registry_serve.go @@ -75,9 +75,6 @@ func RegistryServe(ctx context.Context, engine *gin.Engine, egrp *errgroup.Group // Launch registry prometheus metrics registry.LaunchRegistryMetrics(ctx, egrp) - // Launch - config.LaunchIssuerKeysDirRefresh(ctx, egrp) - egrp.Go(func() error { <-ctx.Done() return registry.ShutdownRegistryDB() From e94c3118174745762ed9da95b43ced47e898994e Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Mon, 18 Nov 2024 14:42:21 +0000 Subject: [PATCH 11/64] fix linting problem --- config/init_server_creds.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/config/init_server_creds.go b/config/init_server_creds.go index 7bc3d1d71..3bf88c357 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -678,14 +678,6 @@ func loadPEMFiles(dir string) (jwk.Key, error) { return mostRecentKey, nil } -func isIssuerPrivateKeysEmpty() bool { - currentKeys := issuerPrivateKeys.Load() - if currentKeys == nil { - return true // Map has not been initialized, so it's considered empty - } - return len(*currentKeys) == 0 -} - // Create a new .pem file and add its key to issuerPrivateKeys // In other words, it combines GeneratePrivateKey and LoadPrivateKey functions func GeneratePEM(dir string) (jwk.Key, error) { From 54f48f5959c17e035590c42a913c661e4e4e5fc2 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Tue, 19 Nov 2024 06:25:56 +0000 Subject: [PATCH 12/64] use new config param IssuerKeysDirectory to replace IssuerKey --- cmd/generate_keygen.go | 4 +- cmd/registry_client.go | 9 +--- config/config.go | 5 +- config/encrypted.go | 14 +++--- config/encrypted_test.go | 20 ++++---- config/init_server_creds.go | 81 ++++++++++++++------------------ config/init_server_creds_test.go | 70 ++++++++++++++------------- docs/parameters.yaml | 11 +++++ origin/origin_db_test.go | 4 +- origin/origin_ui.go | 7 +-- param/parameters.go | 1 + param/parameters_struct.go | 2 + web_ui/prometheus_test.go | 10 ++-- web_ui/ui_test.go | 6 +-- 14 files changed, 123 insertions(+), 121 deletions(-) diff --git a/cmd/generate_keygen.go b/cmd/generate_keygen.go index 8f0f03d4c..7db4b428b 100644 --- a/cmd/generate_keygen.go +++ b/cmd/generate_keygen.go @@ -39,7 +39,7 @@ func keygenMain(cmd *cobra.Command, args []string) error { return errors.Wrap(err, "failed to get the current working directory") } if privateKeyPath == "" { - privateKeyPath = filepath.Join(wd, "issuer.jwk") + privateKeyPath = filepath.Join(wd, "issuer-keys") } else { privateKeyPath = filepath.Clean(strings.TrimSpace(privateKeyPath)) } @@ -68,7 +68,7 @@ func keygenMain(cmd *cobra.Command, args []string) error { return fmt.Errorf("file exists for public key under %s", publicKeyPath) } - viper.Set(param.IssuerKey.GetName(), privateKeyPath) + viper.Set(param.IssuerKeysDirectory.GetName(), privateKeyPath) // GetIssuerPublicJWKS will generate the private key at IssuerKey if it does not exist // and parse the private key and generate the corresponding public key for us diff --git a/cmd/registry_client.go b/cmd/registry_client.go index 8bc03d910..710254eef 100644 --- a/cmd/registry_client.go +++ b/cmd/registry_client.go @@ -32,14 +32,12 @@ import ( "net/url" "os" - "github.com/lestrrat-go/jwx/v2/jwk" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/pelicanplatform/pelican/config" - "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/registry" ) @@ -109,16 +107,11 @@ func registerANamespace(cmd *cobra.Command, args []string) { os.Exit(1) } - privateKeyRaw, err := config.LoadPrivateKey(param.IssuerKey.GetString(), false) + privateKey, err := config.GetIssuerPrivateJWK() if err != nil { log.Error("Failed to load private key", err) os.Exit(1) } - privateKey, err := jwk.FromRaw(privateKeyRaw) - if err != nil { - log.Error("Failed to create JWK private key", err) - os.Exit(1) - } if withIdentity { // We haven't added support to pass sitename from CLI, so leave it empty diff --git a/config/config.go b/config/config.go index d91a21f40..0e9c7272d 100644 --- a/config/config.go +++ b/config/config.go @@ -677,7 +677,7 @@ func LaunchIssuerKeysDirRefresh(ctx context.Context, egrp *errgroup.Group) { log.Debugln("Stopping periodic check for private keys directory.") return nil case <-ticker.C: - if key, err := loadIssuerPrivateKey(param.IssuerKey.GetString()); err != nil { + if key, err := loadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()); err != nil { log.Errorf("Error loading private keys: %v", err) } else { log.Debugln("Private keys directory refreshed successfully. The latest private key in use is", key.KeyID()) @@ -959,6 +959,7 @@ func SetServerDefaults(v *viper.Viper) error { v.SetDefault(param.Xrootd_Authfile.GetName(), filepath.Join(configDir, "xrootd", "authfile")) v.SetDefault(param.Xrootd_MacaroonsKeyFile.GetName(), filepath.Join(configDir, "macaroons-secret")) v.SetDefault(param.IssuerKey.GetName(), filepath.Join(configDir, "issuer.jwk")) + v.SetDefault(param.IssuerKeysDirectory.GetName(), filepath.Join(configDir, "issuer-keys")) v.SetDefault(param.Server_UIPasswordFile.GetName(), filepath.Join(configDir, "server-web-passwd")) v.SetDefault(param.Server_UIActivationCodeFile.GetName(), filepath.Join(configDir, "server-web-activation-code")) v.SetDefault(param.OIDC_ClientIDFile.GetName(), filepath.Join(configDir, "oidc-client-id")) @@ -1424,6 +1425,8 @@ func SetClientDefaults(v *viper.Viper) error { configDir := v.GetString("ConfigDir") v.SetDefault(param.IssuerKey.GetName(), filepath.Join(configDir, "issuer.jwk")) + v.SetDefault(param.IssuerKeysDirectory.GetName(), filepath.Join(configDir, "issuer-keys")) + upperPrefix := GetPreferredPrefix() if upperPrefix == OsdfPrefix || upperPrefix == StashPrefix { v.SetDefault("Federation.TopologyNamespaceURL", "https://topology.opensciencegrid.org/osdf/namespaces") diff --git a/config/encrypted.go b/config/encrypted.go index 189edde28..155eb3786 100644 --- a/config/encrypted.go +++ b/config/encrypted.go @@ -22,7 +22,6 @@ import ( "crypto/aes" "crypto/cipher" "crypto/ed25519" - "crypto/elliptic" "crypto/rand" "crypto/sha256" "crypto/sha512" @@ -42,8 +41,6 @@ import ( "golang.org/x/crypto/nacl/box" "golang.org/x/term" "gopkg.in/yaml.v3" - - "github.com/pelicanplatform/pelican/param" ) // If we prompted the user for a new password while setting up the file, @@ -380,17 +377,18 @@ func SaveConfigContents_internal(config *OSDFConfig, forcePassword bool) error { // Take a hash, and use the hash's bytes as the secret. func GetSecret() (string, error) { // Use issuer private key as the source to generate the secret - issuerKeyFile := param.IssuerKey.GetString() - err := GeneratePrivateKey(issuerKeyFile, elliptic.P256(), false) + privateKey, err := GetIssuerPrivateJWK() if err != nil { return "", err } - privateKey, err := LoadPrivateKey(issuerKeyFile, false) - if err != nil { + + // Extract the underlying ECDSA private key in native Go crypto key type + var rawKey interface{} + if err := privateKey.Raw(&rawKey); err != nil { return "", err } - derPrivateKey, err := x509.MarshalPKCS8PrivateKey(privateKey) + derPrivateKey, err := x509.MarshalPKCS8PrivateKey(rawKey) if err != nil { return "", err diff --git a/config/encrypted_test.go b/config/encrypted_test.go index bd100ea64..fa6f9295c 100644 --- a/config/encrypted_test.go +++ b/config/encrypted_test.go @@ -37,8 +37,8 @@ func TestGetSecret(t *testing.T) { }) t.Run("generate-32B-hash", func(t *testing.T) { tmp := t.TempDir() - keyName := filepath.Join(tmp, "issuer.key") - viper.Set(param.IssuerKey.GetName(), keyName) + keyDir := filepath.Join(tmp, "issuer-keys") + viper.Set(param.IssuerKeysDirectory.GetName(), keyDir) get, err := GetSecret() require.NoError(t, err) @@ -55,8 +55,8 @@ func TestEncryptString(t *testing.T) { t.Run("encrypt-without-err", func(t *testing.T) { tmp := t.TempDir() - keyName := filepath.Join(tmp, "issuer.key") - viper.Set(param.IssuerKey.GetName(), keyName) + keyDir := filepath.Join(tmp, "issuer-keys") + viper.Set(param.IssuerKeysDirectory.GetName(), keyDir) get, err := EncryptString("Some secret to encrypt") require.NoError(t, err) @@ -72,8 +72,8 @@ func TestDecryptString(t *testing.T) { }) t.Run("decrypt-without-err", func(t *testing.T) { tmp := t.TempDir() - keyName := filepath.Join(tmp, "issuer.key") - viper.Set(param.IssuerKey.GetName(), keyName) + keyDir := filepath.Join(tmp, "issuer-keys") + viper.Set(param.IssuerKeysDirectory.GetName(), keyDir) secret := "Some secret to encrypt" @@ -88,8 +88,8 @@ func TestDecryptString(t *testing.T) { t.Run("diff-secrets-yield-diff-result", func(t *testing.T) { tmp := t.TempDir() - keyName := filepath.Join(tmp, "issuer.key") - viper.Set(param.IssuerKey.GetName(), keyName) + keyDir := filepath.Join(tmp, "issuer-keys") + viper.Set(param.IssuerKeysDirectory.GetName(), keyDir) secret := "Some secret to encrypt" @@ -97,8 +97,8 @@ func TestDecryptString(t *testing.T) { require.NoError(t, err) assert.NotEmpty(t, getEncrypt) - newKeyName := filepath.Join(tmp, "new-issuer.key") - viper.Set(param.IssuerKey.GetName(), newKeyName) + newKeyDir := filepath.Join(tmp, "new-issuer-keys") + viper.Set(param.IssuerKeysDirectory.GetName(), newKeyDir) getDecrypt, err := DecryptString(getEncrypt) require.NoError(t, err) diff --git a/config/init_server_creds.go b/config/init_server_creds.go index 3bf88c357..d43593c16 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -54,7 +54,7 @@ var ( issuerPrivateKeys atomic.Pointer[map[string]jwk.Key] // Record the value of "issuerKey" param in case it changes during runtime - currentIssuerKeyDir atomic.Value + currentIssuerKeysDir atomic.Value // Used to ensure initialization func init() is only called once initOnce sync.Once @@ -88,15 +88,15 @@ func getIssuerPrivateKeysCopy() map[string]jwk.Key { return newMap } -func getCurrentIssuerKeyDir() string { - if val := currentIssuerKeyDir.Load(); val != nil { +func getCurrentIssuerKeysDir() string { + if val := currentIssuerKeysDir.Load(); val != nil { return val.(string) } return "" } -func setCurrentIssuerKeyDir(dir string) { - currentIssuerKeyDir.Store(dir) +func setCurrentIssuerKeysDir(dir string) { + currentIssuerKeysDir.Store(dir) } // Return a pointer to an ECDSA private key or RSA private key read from keyLocation. @@ -565,8 +565,8 @@ func migratePrivateKey(newDir string) error { log.Warnf("Failed to load key %s: %v", key.KeyID(), err) } - // Rename the existing private key file using the naming pattern in new directory - fileName := fmt.Sprintf("%s.pem", key.KeyID()) + // Set destination path + fileName := filepath.Base(legacyPrivateKeyFile) destPath := filepath.Join(newDir, fileName) // Check if a file with the same name already exists in the destination @@ -681,7 +681,7 @@ func loadPEMFiles(dir string) (jwk.Key, error) { // Create a new .pem file and add its key to issuerPrivateKeys // In other words, it combines GeneratePrivateKey and LoadPrivateKey functions func GeneratePEM(dir string) (jwk.Key, error) { - filename := fmt.Sprintf("%d.pem", time.Now().Unix()) + filename := fmt.Sprintf("pelican_generated_%d.pem", time.Now().Unix()) keyPath := filepath.Join(dir, filename) if err := GeneratePrivateKey(keyPath, elliptic.P256(), false); err != nil { return nil, errors.Wrap(err, "failed to generate new private key") @@ -692,13 +692,6 @@ func GeneratePEM(dir string) (jwk.Key, error) { log.Warnf("Failed to load key %s: %v", keyPath, err) } - // Rename the file using its key.KeyID() - newFilename := fmt.Sprintf("%s.pem", key.KeyID()) - newKeyPath := filepath.Join(dir, newFilename) - if err := os.Rename(keyPath, newKeyPath); err != nil { - return nil, errors.Wrap(err, "failed to rename private key file with KeyID") - } - // Save this new key in the in-memory map for all private keys newKeys := getIssuerPrivateKeysCopy() newKeys[key.KeyID()] = key @@ -721,35 +714,17 @@ func GeneratePEMandSetActiveKey(dir string) (jwk.Key, error) { // Helper function to load the issuer/server's private key to sign tokens it issues. // Only intended to be called internally -func loadIssuerPrivateKey(issuerKeyDir string) (jwk.Key, error) { - // Handles runtime changes to the issuer keys directory (configured via "IssuerKey" parameter). - // When the directory path changes: - // 1. Resets the active private key - // 2. Clears the in-memory cache of all private keys - // Note: This does not affect the actual key files on disk. - // Primarily used in the "invalid-token-sig-key" test scenario. - currentIssuerKeyDir := getCurrentIssuerKeyDir() - if currentIssuerKeyDir != issuerKeyDir { - log.Debugf("The private keys dir generated by IssuerKey param has changed from '%s' to '%s'", currentIssuerKeyDir, issuerKeyDir) - ResetIssuerJWKPtr() - ResetIssuerPrivateKeys() - setCurrentIssuerKeyDir(issuerKeyDir) - } - - // Replace the leaf directory or file name with a new dir /issuer-keys - parentDir := filepath.Dir(issuerKeyDir) - defaultPrivateKeysDir := filepath.Join(parentDir, "issuer-keys") - +func loadIssuerPrivateKey(issuerKeysDir string) (jwk.Key, error) { // Ensure initKeysMap is only called once across the program’s runtime var initErr error initOnce.Do(func() { - initErr = initKeysMap(defaultPrivateKeysDir) + initErr = initKeysMap(issuerKeysDir) }) if initErr != nil { return nil, errors.Wrap(initErr, "failed to initialize and/or migrate existing private key file") } - activeKey, err := loadPEMFiles(defaultPrivateKeysDir) + activeKey, err := loadPEMFiles(issuerKeysDir) if err != nil { return nil, errors.Wrap(err, "failed to load .pem files and find the most recent private key") } @@ -759,7 +734,7 @@ func loadIssuerPrivateKey(issuerKeyDir string) (jwk.Key, error) { // Helper function to load the issuer/server's public key for other servers // to verify the token signed by this server. Only intended to be called internally -func loadIssuerPublicJWKS(existingJWKS string, issuerKeyDir string) (jwk.Set, error) { +func loadIssuerPublicJWKS(existingJWKS string, issuerKeysDir string) (jwk.Set, error) { jwks := jwk.NewSet() if existingJWKS != "" { var err error @@ -771,8 +746,8 @@ func loadIssuerPublicJWKS(existingJWKS string, issuerKeyDir string) (jwk.Set, er key := issuerPrivateJWK.Load() if key == nil { // This returns issuerPrivateJWK if it's non-nil, or find and parse private JWK - // located at IssuerKey if there is one, or generate a new private key - loadedKey, err := loadIssuerPrivateKey(issuerKeyDir) + // located at IssuerKeysDirectory if there is one, or generate a new private key + loadedKey, err := loadIssuerPrivateKey(issuerKeysDir) if err != nil { return nil, errors.Wrap(err, "Failed to load issuer private JWK") } @@ -781,7 +756,7 @@ func loadIssuerPublicJWKS(existingJWKS string, issuerKeyDir string) (jwk.Set, er pkey, err := jwk.PublicKeyOf(*key) if err != nil { - return nil, errors.Wrapf(err, "failed to generate public key from file %v", issuerKeyDir) + return nil, errors.Wrapf(err, "failed to generate public key from file %v", issuerKeysDir) } if err = jwks.AddKey(pkey); err != nil { @@ -793,9 +768,25 @@ func loadIssuerPublicJWKS(existingJWKS string, issuerKeyDir string) (jwk.Set, er // Return the private JWK for the server to sign tokens func GetIssuerPrivateJWK() (jwk.Key, error) { key := issuerPrivateJWK.Load() - if key == nil { - issuerKeyDir := param.IssuerKey.GetString() - newKey, err := loadIssuerPrivateKey(issuerKeyDir) + issuerKeysDir := param.IssuerKeysDirectory.GetString() + currentIssuerKeysDir := getCurrentIssuerKeysDir() + // Handles runtime changes to the issuer keys directory (configured via "IssuerKeysDirectory" parameter). + // When the directory path changes: + // 1. Resets the active private key + // 2. Clears the in-memory cache of all private keys + // Note: This does not affect the actual key files on disk. + // Primarily used in the "invalid-token-sig-key", "diff-secrets-yield-diff-result" test scenario. + var isDirChanged bool + if currentIssuerKeysDir != issuerKeysDir { + log.Debugf("The private keys dir generated by IssuerKeysDirectory param has changed from '%s' to '%s'", currentIssuerKeysDir, issuerKeysDir) + ResetIssuerJWKPtr() + ResetIssuerPrivateKeys() + setCurrentIssuerKeysDir(issuerKeysDir) + isDirChanged = true + } + + if key == nil || isDirChanged { + newKey, err := loadIssuerPrivateKey(issuerKeysDir) if err != nil { return nil, errors.Wrap(err, "Failed to load issuer private key") } @@ -814,8 +805,8 @@ func GetIssuerPrivateJWK() (jwk.Key, error) { // i.e. "/.well-known/issuer.jwks" func GetIssuerPublicJWKS() (jwk.Set, error) { existingJWKS := param.Server_IssuerJwks.GetString() - issuerKeyDir := param.IssuerKey.GetString() - return loadIssuerPublicJWKS(existingJWKS, issuerKeyDir) + issuerKeysDir := param.IssuerKeysDirectory.GetString() + return loadIssuerPublicJWKS(existingJWKS, issuerKeysDir) } // Check if there is a session secret exists at param.Server_SessionSecretFile and is not empty if there is one. diff --git a/config/init_server_creds_test.go b/config/init_server_creds_test.go index 35fd44603..9ab5d97bf 100644 --- a/config/init_server_creds_test.go +++ b/config/init_server_creds_test.go @@ -32,6 +32,7 @@ import ( "github.com/pelicanplatform/pelican/param" "github.com/pkg/errors" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -153,38 +154,43 @@ func TestLoadPrivateKey(t *testing.T) { }) } -func TestLoadIssuerPrivateKey(t *testing.T) { - tempDir := t.TempDir() - issuerKeyDir := filepath.Join(tempDir, param.IssuerKey.GetString()) +func TestMultiPrivateKey(t *testing.T) { + t.Run("generate-and-load-single-key", func(t *testing.T) { + ResetConfig() + defer ResetConfig() + tempDir := t.TempDir() + issuerKeysDir := filepath.Join(tempDir, "issuer-keys") - key, err := loadIssuerPrivateKey(issuerKeyDir) - require.NoError(t, err) - require.NotNil(t, key) -} + key, err := loadIssuerPrivateKey(issuerKeysDir) + require.NoError(t, err) + require.NotNil(t, key) + }) -// This test also imitates the origin API endpoint "/newIssuerKey" -func TestSecondPrivateKey(t *testing.T) { - tempDir := t.TempDir() - issuerKeyDir := filepath.Join(tempDir, param.IssuerKey.GetString()) - - key, err := loadIssuerPrivateKey(issuerKeyDir) - require.NoError(t, err) - require.NotNil(t, key) - - // Wait for 1 second to avoid duplicated private key filenames - // because they are named after unix epoch timestamp - time.Sleep(1 * time.Second) - - // Create another private key - parentDir := filepath.Dir(issuerKeyDir) - defaultPrivateKeysDir := filepath.Join(parentDir, "issuer-keys") - secondKey, err := GeneratePEMandSetActiveKey(defaultPrivateKeysDir) - require.NoError(t, err) - require.NotNil(t, secondKey) - assert.NotEqual(t, key.KeyID(), secondKey.KeyID()) - - // Check if the active private key points to the lastest key - latestKey, err := GetIssuerPrivateJWK() - require.NoError(t, err) - assert.Equal(t, secondKey.KeyID(), latestKey.KeyID()) + // This test also imitates the origin API endpoint "/newIssuerKey" + t.Run("second-private-key", func(t *testing.T) { + ResetConfig() + defer ResetConfig() + tempDir := t.TempDir() + issuerKeysDir := filepath.Join(tempDir, "issuer-keys") + + key, err := loadIssuerPrivateKey(issuerKeysDir) + require.NoError(t, err) + require.NotNil(t, key) + + // Wait for 1 second to avoid duplicated private key filenames + // because they are named after unix epoch timestamp + time.Sleep(1 * time.Second) + + // Create another private key + secondKey, err := GeneratePEMandSetActiveKey(issuerKeysDir) + require.NoError(t, err) + require.NotNil(t, secondKey) + assert.NotEqual(t, key.KeyID(), secondKey.KeyID()) + + // Check if the active private key points to the lastest key + viper.Set(param.IssuerKeysDirectory.GetName(), issuerKeysDir) + latestKey, err := GetIssuerPrivateJWK() + require.NoError(t, err) + assert.Equal(t, secondKey.KeyID(), latestKey.KeyID()) + }) } diff --git a/docs/parameters.yaml b/docs/parameters.yaml index 47e263d9c..8332eb042 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -73,6 +73,17 @@ root_default: /etc/pelican/issuer.jwk default: $ConfigBase/issuer.jwk components: ["client", "registry", "director"] --- +name: IssuerKeysDirectory +description: |+ + A filepath to the directory used for storing one or multiple PEM-encoded ecdsa private keys. The most recent modified + private key will be parsed into a JWK and serves as the active private key to sign various JWTs issued by this server. + + A public JWK will be derived from this private key and used as the key for token verification. +type: filename +root_default: /etc/pelican/issuer-keys +default: $ConfigBase/issuer-keys +components: ["*"] +--- name: Transport.DialerTimeout description: |+ Maximum time allowed for establishing a connection to target host. diff --git a/origin/origin_db_test.go b/origin/origin_db_test.go index 2e2a4d19f..f505a8143 100644 --- a/origin/origin_db_test.go +++ b/origin/origin_db_test.go @@ -62,8 +62,8 @@ func setupMockOriginDB(t *testing.T) { // Setup encryption tmp := t.TempDir() - keyName := filepath.Join(tmp, "issuer.key") - viper.Set(param.IssuerKey.GetName(), keyName) + keyDir := filepath.Join(tmp, "issuer-keys") + viper.Set(param.IssuerKeysDirectory.GetName(), keyDir) // Also update the refresh token to be encrpted for idx := range mockGC { diff --git a/origin/origin_ui.go b/origin/origin_ui.go index b4f6314fb..d2104772f 100644 --- a/origin/origin_ui.go +++ b/origin/origin_ui.go @@ -21,7 +21,6 @@ package origin import ( "net/http" "net/url" - "path/filepath" "time" "github.com/gin-gonic/gin" @@ -164,11 +163,9 @@ func handleExports(ctx *gin.Context) { // Create a new private key in the given directory and set it as the active key func createNewIssuerKey(ctx *gin.Context) { - issuerKeyDir := param.IssuerKey.GetString() - parentDir := filepath.Dir(issuerKeyDir) - defaultPrivateKeysDir := filepath.Join(parentDir, "issuer-keys") + issuerKeysDir := param.IssuerKeysDirectory.GetString() - _, err := config.GeneratePEMandSetActiveKey(defaultPrivateKeysDir) + _, err := config.GeneratePEMandSetActiveKey(issuerKeysDir) if err != nil { log.Errorf("Error creating and loading a new private key in a new .pem file: %v", err) ctx.JSON(http.StatusInternalServerError, server_structs.SimpleApiResp{ diff --git a/param/parameters.go b/param/parameters.go index f915935ef..53efb582f 100644 --- a/param/parameters.go +++ b/param/parameters.go @@ -163,6 +163,7 @@ var ( Federation_TopologyNamespaceUrl = StringParam{"Federation.TopologyNamespaceUrl"} Federation_TopologyUrl = StringParam{"Federation.TopologyUrl"} IssuerKey = StringParam{"IssuerKey"} + IssuerKeysDirectory = StringParam{"IssuerKeysDirectory"} Issuer_AuthenticationSource = StringParam{"Issuer.AuthenticationSource"} Issuer_GroupFile = StringParam{"Issuer.GroupFile"} Issuer_GroupSource = StringParam{"Issuer.GroupSource"} diff --git a/param/parameters_struct.go b/param/parameters_struct.go index 8f9165ca1..0009faa4c 100644 --- a/param/parameters_struct.go +++ b/param/parameters_struct.go @@ -117,6 +117,7 @@ type Config struct { UserStripDomain bool `mapstructure:"userstripdomain" yaml:"UserStripDomain"` } `mapstructure:"issuer" yaml:"Issuer"` IssuerKey string `mapstructure:"issuerkey" yaml:"IssuerKey"` + IssuerKeysDirectory string `mapstructure:"issuerkeysdirectory" yaml:"IssuerKeysDirectory"` LocalCache struct { DataLocation string `mapstructure:"datalocation" yaml:"DataLocation"` HighWaterMarkPercentage int `mapstructure:"highwatermarkpercentage" yaml:"HighWaterMarkPercentage"` @@ -421,6 +422,7 @@ type configWithType struct { UserStripDomain struct { Type string; Value bool } } IssuerKey struct { Type string; Value string } + IssuerKeysDirectory struct { Type string; Value string } LocalCache struct { DataLocation struct { Type string; Value string } HighWaterMarkPercentage struct { Type string; Value int } diff --git a/web_ui/prometheus_test.go b/web_ui/prometheus_test.go index c1833ea18..d92f226c0 100644 --- a/web_ui/prometheus_test.go +++ b/web_ui/prometheus_test.go @@ -170,10 +170,10 @@ func TestPrometheusProtectionOriginHeaderScope(t *testing.T) { // Create temp dir for the origin key file tDir := t.TempDir() - kfile := filepath.Join(tDir, "testKey") + keysDir := filepath.Join(tDir, "testKeys") //Setup a private key and a token - viper.Set("IssuerKey", kfile) + viper.Set("IssuerKeysDirectory", keysDir) // Setting the ConfigDir to t.TempDir() causes issues with this test on Windows because // the process tries to clean up the directory before the test is done with it. @@ -237,15 +237,15 @@ func TestPrometheusProtectionOriginHeaderScope(t *testing.T) { } // Create a new private key by re-initializing config to point at a new temp dir - k2file := filepath.Join(tDir, "whatever", "testKey2") - viper.Set("IssuerKey", k2file) + k2dir := filepath.Join(tDir, "whatever", "testDir2") + viper.Set("IssuerKeysDirectory", k2dir) err = config.InitServer(ctx, server_structs.OriginType) require.NoError(t, err) token := createToken("monitoring.query", issuerUrl) // Re-init the config again, this time pointing at the original key - viper.Set("IssuerKey", kfile) + viper.Set("IssuerKeysDirectory", keysDir) err = config.InitServer(ctx, server_structs.OriginType) require.NoError(t, err) diff --git a/web_ui/ui_test.go b/web_ui/ui_test.go index a57c8e1e2..d6855c94c 100644 --- a/web_ui/ui_test.go +++ b/web_ui/ui_test.go @@ -96,7 +96,7 @@ func TestMain(m *testing.M) { } //Override viper default for testing - viper.Set("IssuerKey", filepath.Join(tempJWKDir, "issuer.jwk")) + viper.Set(param.IssuerKeysDirectory.GetName(), filepath.Join(tempJWKDir, "issuer-keys")) // Ensure we load up the default configs. dirname, err := os.MkdirTemp("", "tmpDir") @@ -218,8 +218,8 @@ func TestHandleWebUIAuth(t *testing.T) { }) tmpDir := t.TempDir() - issuerFile := filepath.Join(tmpDir, "issuer.key") - viper.Set(param.IssuerKey.GetName(), issuerFile) + issuerFile := filepath.Join(tmpDir, "issuer-keys") + viper.Set(param.IssuerKeysDirectory.GetName(), issuerFile) viper.Set(param.Server_ExternalWebUrl.GetName(), "https://example.com") _, err := config.GetIssuerPrivateJWK() From 532cd05a2c68b7aeaab3a3d0ce53da656021b4b1 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Tue, 19 Nov 2024 14:39:29 +0000 Subject: [PATCH 13/64] fix linting problems --- config/init_server_creds_test.go | 3 ++- docs/parameters.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/config/init_server_creds_test.go b/config/init_server_creds_test.go index 9ab5d97bf..fedecc015 100644 --- a/config/init_server_creds_test.go +++ b/config/init_server_creds_test.go @@ -30,11 +30,12 @@ import ( "testing" "time" - "github.com/pelicanplatform/pelican/param" "github.com/pkg/errors" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/pelicanplatform/pelican/param" ) // encrypt should be ecdsa|rsa diff --git a/docs/parameters.yaml b/docs/parameters.yaml index 8332eb042..35b79efe8 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -75,7 +75,7 @@ components: ["client", "registry", "director"] --- name: IssuerKeysDirectory description: |+ - A filepath to the directory used for storing one or multiple PEM-encoded ecdsa private keys. The most recent modified + A filepath to the directory used for storing one or multiple PEM-encoded ecdsa private keys. The most recent modified private key will be parsed into a JWK and serves as the active private key to sign various JWTs issued by this server. A public JWK will be derived from this private key and used as the key for token verification. From 7b738d25e0d38c7c4decb01c041c970ceeb902e6 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Tue, 19 Nov 2024 16:03:49 +0000 Subject: [PATCH 14/64] in docs, correct the scope of components affected by IssuerKeysDirectory --- docs/parameters.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/parameters.yaml b/docs/parameters.yaml index 35b79efe8..accec16b0 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -71,7 +71,7 @@ description: |+ type: filename root_default: /etc/pelican/issuer.jwk default: $ConfigBase/issuer.jwk -components: ["client", "registry", "director"] +components: ["origin", "cache", "registry", "director"] --- name: IssuerKeysDirectory description: |+ @@ -82,7 +82,7 @@ description: |+ type: filename root_default: /etc/pelican/issuer-keys default: $ConfigBase/issuer-keys -components: ["*"] +components: ["origin", "cache", "registry", "director"] --- name: Transport.DialerTimeout description: |+ From f4fdbd8fd198844df0b446e1005673864f619b0b Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Tue, 19 Nov 2024 16:07:23 +0000 Subject: [PATCH 15/64] deprecate IssuerKey --- docs/parameters.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/parameters.yaml b/docs/parameters.yaml index accec16b0..6e3bd1742 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -64,11 +64,13 @@ components: ["origin", "registry", "director"] --- name: IssuerKey description: |+ - A filepath to the file containing a PEM-encoded ecdsa private key which later will be parsed + [Deprecated] A filepath to the file containing a PEM-encoded ecdsa private key which later will be parsed into a JWK and serves as the private key to sign various JWTs issued by this server. A public JWK will be derived from this private key and used as the key for token verification. type: filename +deprecated: true +replacedby: "IssuerKeysDirectory" root_default: /etc/pelican/issuer.jwk default: $ConfigBase/issuer.jwk components: ["origin", "cache", "registry", "director"] From 09d3586eba7f1f699af939c91c6fe567b96015a2 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Tue, 19 Nov 2024 16:34:03 +0000 Subject: [PATCH 16/64] improve the naming of migrated key --- config/init_server_creds.go | 4 ++-- param/parameters.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/config/init_server_creds.go b/config/init_server_creds.go index d43593c16..d16fd1f8e 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -565,8 +565,8 @@ func migratePrivateKey(newDir string) error { log.Warnf("Failed to load key %s: %v", key.KeyID(), err) } - // Set destination path - fileName := filepath.Base(legacyPrivateKeyFile) + // Rename the existing private key file and set destination path + fileName := "initial_issuer.pem" destPath := filepath.Join(newDir, fileName) // Check if a file with the same name already exists in the destination diff --git a/param/parameters.go b/param/parameters.go index 53efb582f..883dad5a3 100644 --- a/param/parameters.go +++ b/param/parameters.go @@ -55,6 +55,7 @@ func GetDeprecated() map[string][]string { "Director.EnableStat": {"Director.CheckOriginPresence"}, "DisableHttpProxy": {"Client.DisableHttpProxy"}, "DisableProxyFallback": {"Client.DisableProxyFallback"}, + "IssuerKey": {"IssuerKeysDirectory"}, "MinimumDownloadSpeed": {"Client.MinimumDownloadSpeed"}, "Origin.EnableDirListing": {"Origin.EnableListings"}, "Origin.EnableFallbackRead": {"Origin.EnableDirectReads"}, From a80f2720df3980310765749792f542578cae333b Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Thu, 21 Nov 2024 00:03:22 +0000 Subject: [PATCH 17/64] patch for the algorithm of new key regristration --- config/init_server_creds.go | 10 ++++- launcher_utils/register_namespace.go | 55 +++++++++++++++-------- launcher_utils/register_namespace_test.go | 2 + 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/config/init_server_creds.go b/config/init_server_creds.go index d16fd1f8e..c7a366317 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -50,10 +50,10 @@ var ( // the same if the IssuerKey is unchanged issuerPrivateJWK atomic.Pointer[jwk.Key] - // Representing all private keys (.pem files) in the directory + // Representing private keys (from all .pem files) in the directory cache issuerPrivateKeys atomic.Pointer[map[string]jwk.Key] - // Record the value of "issuerKey" param in case it changes during runtime + // Record the value of "IssuerKeysDirectory" param in case it changes during runtime currentIssuerKeysDir atomic.Value // Used to ensure initialization func init() is only called once @@ -88,6 +88,12 @@ func getIssuerPrivateKeysCopy() map[string]jwk.Key { return newMap } +// Read the current map +func GetIssuerPrivateKeys() map[string]jwk.Key { + keysPtr := issuerPrivateKeys.Load() + return *keysPtr +} + func getCurrentIssuerKeysDir() string { if val := currentIssuerKeysDir.Load(); val != nil { return val.(string) diff --git a/launcher_utils/register_namespace.go b/launcher_utils/register_namespace.go index 3159dac3b..670f562e0 100644 --- a/launcher_utils/register_namespace.go +++ b/launcher_utils/register_namespace.go @@ -226,32 +226,49 @@ func registerNamespacePrep(ctx context.Context, prefix string) (key jwk.Key, reg err = errors.Wrap(err, "Failed to construct registration endpoint URL: %v") return } - key, err = config.GetIssuerPrivateJWK() - if err != nil { - err = errors.Wrap(err, "failed to load the origin's JWK") + privateKeys := config.GetIssuerPrivateKeys() + if len(privateKeys) == 0 { + err = errors.Wrap(err, "failed to load the origin's private key(s)") return } - if key.KeyID() == "" { - if err = jwk.AssignKeyID(key); err != nil { - err = errors.Wrap(err, "Error when generating a key ID for registration") - return + // Traverse each private key provided by the origin to see if it is registered + var misMatchCount int + for _, privateKey := range privateKeys { + if privateKey.KeyID() == "" { + if err = jwk.AssignKeyID(privateKey); err != nil { + err = errors.Wrap(err, "Error when generating a key ID for registration") + return + } + } + + keyStatus, err := keyIsRegistered(privateKey, registrationUrl, prefix) + if err != nil { + err = errors.Wrap(err, "Failed to determine whether namespace is already registered") + break + } + switch keyStatus { + case keyMatch: + isRegistered = true + case keyMismatch: + misMatchCount += 1 + case noKeyPresent: + log.Infof("Namespace %v not registered; new registration will proceed\n", prefix) } } - keyStatus, err := keyIsRegistered(key, registrationUrl, prefix) - if err != nil { - err = errors.Wrap(err, "Failed to determine whether namespace is already registered") - return - } - switch keyStatus { - case keyMatch: - isRegistered = true - return - case keyMismatch: + // All private keys from the origin mismatch the registered public key in registry db, + // meaning this origin don't have the credentials to own the namespace they claim + if misMatchCount == len(privateKeys) { err = errors.Errorf("Namespace %v already registered under a different key", prefix) return - case noKeyPresent: - log.Infof("Namespace %v not registered; new registration will proceed\n", prefix) } + // Else, which means there is at least one private key from the origin match the registered public key, + // we will update the public key of the namespace in registry db with the active private key + // held by this origin + key, err = config.GetIssuerPrivateJWK() + if err != nil { + err = errors.Wrap(err, "Failed to obtain origin's active private key") + } + return } diff --git a/launcher_utils/register_namespace_test.go b/launcher_utils/register_namespace_test.go index f6e273b63..56a1f5217 100644 --- a/launcher_utils/register_namespace_test.go +++ b/launcher_utils/register_namespace_test.go @@ -59,6 +59,8 @@ func TestRegistration(t *testing.T) { server_utils.ResetTestState() viper.Set("ConfigDir", tempConfigDir) + keysDir := filepath.Join(tempConfigDir, "issuer-keys") + viper.Set(param.IssuerKeysDirectory.GetName(), keysDir) config.InitConfig() viper.Set("Registry.DbLocation", filepath.Join(tempConfigDir, "test.sql")) From a518afbe92ded7171f8b53f4e6a9c38b50689382 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 22 Nov 2024 01:48:48 +0000 Subject: [PATCH 18/64] register namespace with new key (with TODO left) --- config/config.go | 23 ---- config/init_server_creds.go | 24 ++-- config/init_server_creds_test.go | 4 +- launcher_utils/register_namespace.go | 10 +- launcher_utils/register_namespace_test.go | 151 ++++++++++++++++++++++ launchers/origin_serve.go | 33 ++++- origin/origin_ui.go | 2 +- registry/registry_db.go | 10 +- utils/utils.go | 11 ++ 9 files changed, 226 insertions(+), 42 deletions(-) diff --git a/config/config.go b/config/config.go index 0e9c7272d..0762d9abf 100644 --- a/config/config.go +++ b/config/config.go @@ -665,29 +665,6 @@ func checkWatermark(wmStr string) (bool, int64, error) { } } -// Checks the directory containing .pem files every 5 minutes and loads new private key(s) if new file(s) are detected -func LaunchIssuerKeysDirRefresh(ctx context.Context, egrp *errgroup.Group) { - egrp.Go(func() error { - ticker := time.NewTicker(5 * time.Minute) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - log.Debugln("Stopping periodic check for private keys directory.") - return nil - case <-ticker.C: - if key, err := loadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()); err != nil { - log.Errorf("Error loading private keys: %v", err) - } else { - log.Debugln("Private keys directory refreshed successfully. The latest private key in use is", key.KeyID()) - } - - } - } - }) -} - func setupTranslation() error { err := en_translations.RegisterDefaultTranslations(validate, GetEnTranslator()) if err != nil { diff --git a/config/init_server_creds.go b/config/init_server_creds.go index c7a366317..fe32b0074 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -43,6 +43,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/pelicanplatform/pelican/param" + "github.com/pelicanplatform/pelican/utils" ) var ( @@ -569,10 +570,11 @@ func migratePrivateKey(newDir string) error { key, err := loadSinglePEM(legacyPrivateKeyFile) if err != nil { log.Warnf("Failed to load key %s: %v", key.KeyID(), err) + return err } // Rename the existing private key file and set destination path - fileName := "initial_issuer.pem" + fileName := "migrated_key.pem" destPath := filepath.Join(newDir, fileName) // Check if a file with the same name already exists in the destination @@ -592,6 +594,7 @@ func migratePrivateKey(newDir string) error { func initKeysMap(privateKeysDir string) error { initialMap := make(map[string]jwk.Key) issuerPrivateKeys.Store(&initialMap) + setCurrentIssuerKeysDir(privateKeysDir) migrateErr := migratePrivateKey(privateKeysDir) if migrateErr != nil { @@ -643,7 +646,7 @@ func loadPEMFiles(dir string) (jwk.Key, error) { return nil, errors.Wrap(err, "failed to read directory that stores private keys") } - latestKeys := make(map[string]jwk.Key) + latestKeys := getIssuerPrivateKeysCopy() for _, file := range files { if filepath.Ext(file.Name()) != ".pem" { continue @@ -687,7 +690,10 @@ func loadPEMFiles(dir string) (jwk.Key, error) { // Create a new .pem file and add its key to issuerPrivateKeys // In other words, it combines GeneratePrivateKey and LoadPrivateKey functions func GeneratePEM(dir string) (jwk.Key, error) { - filename := fmt.Sprintf("pelican_generated_%d.pem", time.Now().Unix()) + filename := fmt.Sprintf("pelican_generated_%d_%s.pem", + time.Now().UnixNano(), + utils.CreateRandomString(4)) + keyPath := filepath.Join(dir, filename) if err := GeneratePrivateKey(keyPath, elliptic.P256(), false); err != nil { return nil, errors.Wrap(err, "failed to generate new private key") @@ -699,9 +705,9 @@ func GeneratePEM(dir string) (jwk.Key, error) { } // Save this new key in the in-memory map for all private keys - newKeys := getIssuerPrivateKeysCopy() - newKeys[key.KeyID()] = key - issuerPrivateKeys.Store(&newKeys) + keysCopy := getIssuerPrivateKeysCopy() + keysCopy[key.KeyID()] = key + issuerPrivateKeys.Store(&keysCopy) log.Debugf("Loaded the private key (key id: %s) into issuerPrivateKeys and set as active key", key.KeyID()) return key, nil @@ -720,7 +726,7 @@ func GeneratePEMandSetActiveKey(dir string) (jwk.Key, error) { // Helper function to load the issuer/server's private key to sign tokens it issues. // Only intended to be called internally -func loadIssuerPrivateKey(issuerKeysDir string) (jwk.Key, error) { +func LoadIssuerPrivateKey(issuerKeysDir string) (jwk.Key, error) { // Ensure initKeysMap is only called once across the program’s runtime var initErr error initOnce.Do(func() { @@ -753,7 +759,7 @@ func loadIssuerPublicJWKS(existingJWKS string, issuerKeysDir string) (jwk.Set, e if key == nil { // This returns issuerPrivateJWK if it's non-nil, or find and parse private JWK // located at IssuerKeysDirectory if there is one, or generate a new private key - loadedKey, err := loadIssuerPrivateKey(issuerKeysDir) + loadedKey, err := LoadIssuerPrivateKey(issuerKeysDir) if err != nil { return nil, errors.Wrap(err, "Failed to load issuer private JWK") } @@ -792,7 +798,7 @@ func GetIssuerPrivateJWK() (jwk.Key, error) { } if key == nil || isDirChanged { - newKey, err := loadIssuerPrivateKey(issuerKeysDir) + newKey, err := LoadIssuerPrivateKey(issuerKeysDir) if err != nil { return nil, errors.Wrap(err, "Failed to load issuer private key") } diff --git a/config/init_server_creds_test.go b/config/init_server_creds_test.go index fedecc015..f50b023ae 100644 --- a/config/init_server_creds_test.go +++ b/config/init_server_creds_test.go @@ -162,7 +162,7 @@ func TestMultiPrivateKey(t *testing.T) { tempDir := t.TempDir() issuerKeysDir := filepath.Join(tempDir, "issuer-keys") - key, err := loadIssuerPrivateKey(issuerKeysDir) + key, err := LoadIssuerPrivateKey(issuerKeysDir) require.NoError(t, err) require.NotNil(t, key) }) @@ -174,7 +174,7 @@ func TestMultiPrivateKey(t *testing.T) { tempDir := t.TempDir() issuerKeysDir := filepath.Join(tempDir, "issuer-keys") - key, err := loadIssuerPrivateKey(issuerKeysDir) + key, err := LoadIssuerPrivateKey(issuerKeysDir) require.NoError(t, err) require.NotNil(t, key) diff --git a/launcher_utils/register_namespace.go b/launcher_utils/register_namespace.go index 670f562e0..291087839 100644 --- a/launcher_utils/register_namespace.go +++ b/launcher_utils/register_namespace.go @@ -226,6 +226,8 @@ func registerNamespacePrep(ctx context.Context, prefix string) (key jwk.Key, reg err = errors.Wrap(err, "Failed to construct registration endpoint URL: %v") return } + + // Get in-memory private keys to check if any of them matches the registered public key privateKeys := config.GetIssuerPrivateKeys() if len(privateKeys) == 0 { err = errors.Wrap(err, "failed to load the origin's private key(s)") @@ -261,10 +263,10 @@ func registerNamespacePrep(ctx context.Context, prefix string) (key jwk.Key, reg err = errors.Errorf("Namespace %v already registered under a different key", prefix) return } - // Else, which means there is at least one private key from the origin match the registered public key, + // Else, there is at least one private key from the origin match the registered public key, // we will update the public key of the namespace in registry db with the active private key - // held by this origin - key, err = config.GetIssuerPrivateJWK() + // held by this origin; the verified new private key also get added to in-memory map in this process + key, err = config.LoadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()) if err != nil { err = errors.Wrap(err, "Failed to obtain origin's active private key") } @@ -300,7 +302,7 @@ func RegisterNamespaceWithRetry(ctx context.Context, egrp *errgroup.Group, prefi if err := origin.FetchAndSetRegStatus(prefix); err != nil { return errors.Wrapf(err, "failed to fetch registration status for the prefix %s", prefix) } - return nil + // continue to update the public key of this prefix } if err = registerNamespaceImpl(key, prefix, siteName, url); err == nil { diff --git a/launcher_utils/register_namespace_test.go b/launcher_utils/register_namespace_test.go index 56a1f5217..8e329bc2b 100644 --- a/launcher_utils/register_namespace_test.go +++ b/launcher_utils/register_namespace_test.go @@ -46,6 +46,11 @@ import ( ) func TestRegistration(t *testing.T) { + t.Cleanup(func() { + server_utils.ResetTestState() + config.ResetIssuerJWKPtr() + config.ResetIssuerPrivateKeys() + }) // Use a temp os directory to better control the deletion of the directory. // Fixes issue on Windows where we are trying to delete a file in use so this // better waits for the file/process to be shut down before deletion @@ -168,3 +173,149 @@ func TestRegistration(t *testing.T) { assert.NoError(t, err) assert.True(t, isRegistered) } + +func TestMultiKeysRegistration(t *testing.T) { + t.Cleanup(func() { + server_utils.ResetTestState() + config.ResetIssuerJWKPtr() + config.ResetIssuerPrivateKeys() + }) + // Use a temp os directory to better control the deletion of the directory. + // Fixes issue on Windows where we are trying to delete a file in use so this + // better waits for the file/process to be shut down before deletion + tempConfigDir, err := os.MkdirTemp("", "test") + assert.NoError(t, err) + defer os.RemoveAll(tempConfigDir) + + ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) + defer func() { require.NoError(t, egrp.Wait()) }() + defer cancel() + + server_utils.ResetTestState() + viper.Set("ConfigDir", tempConfigDir) + keysDir := filepath.Join(tempConfigDir, "issuer-keys") + viper.Set(param.IssuerKeysDirectory.GetName(), keysDir) + + config.InitConfig() + viper.Set("Registry.DbLocation", filepath.Join(tempConfigDir, "test.sql")) + err = config.InitServer(ctx, server_structs.OriginType) + require.NoError(t, err) + + err = registry.InitializeDB() + require.NoError(t, err) + defer func() { + err := registry.ShutdownRegistryDB() + assert.NoError(t, err) + }() + + gin.SetMode(gin.TestMode) + engine := gin.Default() + + // Ensure we have a issuer key + _, err = config.GetIssuerPublicJWKS() + require.NoError(t, err) + privKey, err := config.GetIssuerPrivateJWK() + require.NoError(t, err) + key, err := privKey.PublicKey() + require.NoError(t, err) + assert.NoError(t, jwk.AssignKeyID(key)) + keyId := key.KeyID() + require.NotEmpty(t, keyId) + keysMap := config.GetIssuerPrivateKeys() + require.Equal(t, 1, len(keysMap)) + + // Create a new issuer key and rotate out the old one + secondKey, err := config.GeneratePEMandSetActiveKey(keysDir) + require.NoError(t, err) + require.NotEqual(t, privKey.KeyID(), secondKey.KeyID()) + secondPubKey, err := secondKey.PublicKey() + require.NoError(t, err) + activeKey, err := config.GetIssuerPrivateJWK() + require.Equal(t, secondKey, activeKey) + keysMap = config.GetIssuerPrivateKeys() + require.Equal(t, secondKey, keysMap[secondKey.KeyID()]) + require.Equal(t, privKey, keysMap[key.KeyID()]) + require.Equal(t, 2, len(keysMap)) + secondKeyId := secondKey.KeyID() + require.NotEmpty(t, keyId) + + //Configure registry + registry.RegisterRegistryAPI(engine.Group("/")) + + //Create a test HTTP server that sends requests to gin + svr := httptest.NewServer(engine) + defer svr.CloseClientConnections() + defer svr.Close() + + viper.Set("Federation.RegistryUrl", svr.URL) + viper.Set("Origin.FederationPrefix", "/test123") + + // Re-run the InitServer to reflect the new RegistryUrl set above + require.NoError(t, config.InitServer(ctx, server_structs.OriginType)) + + // Test registration succeeds + prefix := param.Origin_FederationPrefix.GetString() + key, registerURL, isRegistered, err := registerNamespacePrep(ctx, prefix) + require.NoError(t, err) + assert.False(t, isRegistered) + assert.Equal(t, registerURL, svr.URL+"/api/v1.0/registry") + err = registerNamespaceImpl(key, prefix, "mock_site_name", registerURL) + require.NoError(t, err) + + // Test we can query for the new key + req, err := http.NewRequest("GET", svr.URL+"/api/v1.0/registry", nil) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + tr := config.GetTransport() + client := &http.Client{Transport: tr} + + resp, err := client.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + + // Test new key is the same one we registered. + entries := []server_structs.Namespace{} + err = json.Unmarshal(body, &entries) + require.NoError(t, err) + require.Equal(t, len(entries), 1) + assert.Equal(t, entries[0].Prefix, "/test123") + keySet, err := jwk.Parse([]byte(entries[0].Pubkey)) + require.NoError(t, err) + registryKey, isPresent := keySet.LookupKeyID(secondKeyId) + assert.True(t, isPresent) + assert.True(t, jwk.Equal(registryKey, secondPubKey)) + assert.Equal(t, "mock_site_name", entries[0].AdminMetadata.SiteName) + + // Test the functionality of the keyIsRegistered function + keyStatus, err := keyIsRegistered(key, svr.URL+"/api/v1.0/registry", "/test123") + assert.NoError(t, err) + require.Equal(t, keyStatus, keyMatch) + + // Generate a new key, test we get mismatch + privKeyAltRaw, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + privKeyAlt, err := jwk.FromRaw(privKeyAltRaw) + require.NoError(t, err) + keyAlt, err := privKeyAlt.PublicKey() + require.NoError(t, err) + assert.NoError(t, jwk.AssignKeyID(keyAlt)) + keyStatus, err = keyIsRegistered(keyAlt, svr.URL+"/api/v1.0/registry", "/test123") + assert.NoError(t, err) + assert.Equal(t, keyStatus, keyMismatch) + + // Verify that no key is present for an alternate prefix + keyStatus, err = keyIsRegistered(key, svr.URL, "test456") + assert.NoError(t, err) + assert.Equal(t, keyStatus, noKeyPresent) + + // Redo the namespace prep, ensure that isPresent is true + prefix = param.Origin_FederationPrefix.GetString() + _, registerURL, isRegistered, err = registerNamespacePrep(ctx, prefix) + assert.Equal(t, svr.URL+"/api/v1.0/registry", registerURL) + assert.NoError(t, err) + assert.True(t, isRegistered) +} diff --git a/launchers/origin_serve.go b/launchers/origin_serve.go index 4e987768a..4a0f753ae 100644 --- a/launchers/origin_serve.go +++ b/launchers/origin_serve.go @@ -44,6 +44,35 @@ import ( "github.com/pelicanplatform/pelican/xrootd" ) +// Check the directory containing .pem files every 5 minutes, load new private key(s) +// if new file(s) are detected, then register the new public key +func launchIssuerKeysDirRefresh(ctx context.Context, egrp *errgroup.Group) { + egrp.Go(func() error { + ticker := time.NewTicker(5 * time.Minute) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + log.Debugln("Stopping periodic check for private keys directory.") + return nil + case <-ticker.C: + extUrlStr := param.Server_ExternalWebUrl.GetString() + extUrl, _ := url.Parse(extUrlStr) + namespace := server_structs.GetOriginNs(extUrl.Host) + if err := launcher_utils.RegisterNamespaceWithRetry(ctx, egrp, namespace); err != nil { + log.Errorf("Error refreshing private keys: %v", err) + } else { + if key, err := config.GetIssuerPrivateJWK(); err != nil { + log.Debugln("Private keys directory refreshed successfully. The active (latest) private key is", key.KeyID()) + } + } + + } + } + }) +} + func OriginServe(ctx context.Context, engine *gin.Engine, egrp *errgroup.Group, modules server_structs.ServerType) (server_structs.XRootDServer, error) { metrics.SetComponentHealthStatus(metrics.OriginCache_XRootD, metrics.StatusWarning, "XRootD is initializing") metrics.SetComponentHealthStatus(metrics.OriginCache_CMSD, metrics.StatusWarning, "CMSD is initializting") @@ -78,8 +107,8 @@ func OriginServe(ctx context.Context, engine *gin.Engine, egrp *errgroup.Group, } // Start a routine to periodically refresh the private key directory. - // This ensures that new or updated private keys are automatically loaded - config.LaunchIssuerKeysDirRefresh(ctx, egrp) + // This ensures that new or updated private keys are automatically loaded and registered + launchIssuerKeysDirRefresh(ctx, egrp) // Set up the APIs unrelated to UI, which only contains director-based health test reporting endpoint for now if err = origin.RegisterOriginAPI(engine, ctx, egrp); err != nil { diff --git a/origin/origin_ui.go b/origin/origin_ui.go index d2104772f..804d00cd1 100644 --- a/origin/origin_ui.go +++ b/origin/origin_ui.go @@ -176,7 +176,7 @@ func createNewIssuerKey(ctx *gin.Context) { ctx.JSON(http.StatusOK, server_structs.SimpleApiResp{ Status: server_structs.RespOK, - Msg: "success", + Msg: "Created a new issuer key and set it as the active private key", }) } diff --git a/registry/registry_db.go b/registry/registry_db.go index 3d8afb9bc..b0e2777c7 100644 --- a/registry/registry_db.go +++ b/registry/registry_db.go @@ -382,7 +382,15 @@ func AddNamespace(ns *server_structs.Namespace) error { if ns.AdminMetadata.Status == "" { ns.AdminMetadata.Status = server_structs.RegPending } - + set, err := jwk.ParseString(ns.Pubkey) + if err != nil { + log.Debugln("can't parse pubkey in string", err.Error()) + } + key, ok := set.Key(0) + if !ok { + log.Debugln("can't get pubkey", err.Error()) + } + log.Debugln("this is the keyID i want to check: ", key.KeyID()) return db.Save(&ns).Error } diff --git a/utils/utils.go b/utils/utils.go index 3915f47f4..f108599ae 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -19,6 +19,7 @@ package utils import ( + "math/rand" "net" "net/url" "regexp" @@ -153,3 +154,13 @@ func MapToSlice[K comparable, V any](m map[K]V) []V { } return s } + +// Generate random string +func CreateRandomString(length int) string { + const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + b := make([]byte, length) + for i := range b { + b[i] = charset[rand.Intn(len(charset))] + } + return string(b) +} From dceede7a8d7a3c48b69570c6c870d0b06a428370 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 22 Nov 2024 21:19:37 +0000 Subject: [PATCH 19/64] update namespace pubKey in registry db --- registry/registry.go | 39 +++++++++++++++++++++++++++++++++++---- registry/registry_db.go | 20 +++++++++++--------- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/registry/registry.go b/registry/registry.go index 2a084bded..85a9f2ec4 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -292,11 +292,42 @@ func keySignChallengeCommit(ctx *gin.Context, data *registrationData) (bool, map return false, nil, errors.Wrap(err, "Server encountered an error checking if namespace already exists") } if exists { - returnMsg := map[string]interface{}{ - "message": fmt.Sprintf("The prefix %s is already registered -- nothing else to do!", data.Prefix), + // Update the namespace's public key with the latest one + // when origin provides a new key (the origin is authorized to do that in upstream code) + existingNs, err := getNamespaceByPrefix(data.Prefix) + if err != nil { + log.Errorf("Failed to get existing namespace to update: %v", err) + return false, nil, errors.Wrap(err, "Server encountered an error getting existing namespace to update") + } + log.Debugf("New public key %s is going to replace the old one: %s", string(data.Pubkey), existingNs.Pubkey) + + // Process origin's new public key + pubkeyData, err := json.Marshal(data.Pubkey) + if err != nil { + return false, nil, errors.Wrapf(err, "Failed to convert public key from json to string format for the prefix %s", data.Prefix) + } + pubkeyDbString := string(pubkeyData) + + // Perform the update action when origin provides a new key + if pubkeyDbString != existingNs.Pubkey { + err = updateNamespacePubKey(data.Prefix, pubkeyDbString) + if err != nil { + log.Errorf("Failed to update the public key of namespace %s: %v", data.Prefix, err) + return false, nil, errors.Wrap(err, "Server encountered an error updating the public key of an existing namespace") + } + // Skip the rest steps in namesapce initial registration + returnMsg := map[string]interface{}{ + "message": fmt.Sprintf("Updated the public key of namespace %s:", data.Prefix), + } + log.Infof("Updated the public key of namespace %s:", data.Prefix) + return false, returnMsg, nil + } else { + returnMsg := map[string]interface{}{ + "message": fmt.Sprintf("The prefix %s is already registered -- nothing else to do!", data.Prefix), + } + log.Infof("Skipping registration of prefix %s because it's already registered.", data.Prefix) + return false, returnMsg, nil } - log.Infof("Skipping registration of prefix %s because it's already registered.", data.Prefix) - return false, returnMsg, nil } reqPrefix, err := validatePrefix(data.Prefix) diff --git a/registry/registry_db.go b/registry/registry_db.go index b0e2777c7..61a8c8ff7 100644 --- a/registry/registry_db.go +++ b/registry/registry_db.go @@ -382,15 +382,6 @@ func AddNamespace(ns *server_structs.Namespace) error { if ns.AdminMetadata.Status == "" { ns.AdminMetadata.Status = server_structs.RegPending } - set, err := jwk.ParseString(ns.Pubkey) - if err != nil { - log.Debugln("can't parse pubkey in string", err.Error()) - } - key, ok := set.Key(0) - if !ok { - log.Debugln("can't get pubkey", err.Error()) - } - log.Debugln("this is the keyID i want to check: ", key.KeyID()) return db.Save(&ns).Error } @@ -447,6 +438,17 @@ func updateNamespaceStatusById(id int, status server_structs.RegistrationStatus, return db.Model(ns).Where("id = ?", id).Update("admin_metadata", string(adminMetadataByte)).Error } +func updateNamespacePubKey(prefix string, pubkeyDbString string) error { + if prefix == "" { + return errors.New("invalid prefix. Prefix must not be empty") + } + if pubkeyDbString == "" { + return errors.New("invalid pubkeyDbString. pubkeyDbString must not be empty") + } + ns := server_structs.Namespace{} + return db.Model(ns).Where("prefix = ? ", prefix).Update("pubkey", pubkeyDbString).Error +} + func deleteNamespaceByID(id int) error { return db.Delete(&server_structs.Namespace{}, id).Error } From 8cb80907d0d5f4aa5a09231b1a6e1b98bf876bae Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Wed, 27 Nov 2024 16:51:24 +0000 Subject: [PATCH 20/64] update pubkey of all origin exports; align new key API and manual add behavior; linter problems fix --- config/init_server_creds.go | 3 +- launcher_utils/register_namespace.go | 48 +++++++++++++++++++++-- launcher_utils/register_namespace_test.go | 1 + launchers/origin_serve.go | 32 +-------------- origin/origin_ui.go | 10 +++-- 5 files changed, 53 insertions(+), 41 deletions(-) diff --git a/config/init_server_creds.go b/config/init_server_creds.go index fe32b0074..aef240cf5 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -724,8 +724,7 @@ func GeneratePEMandSetActiveKey(dir string) (jwk.Key, error) { return newKey, nil } -// Helper function to load the issuer/server's private key to sign tokens it issues. -// Only intended to be called internally +// Re-scan the disk to load the issuer/server's private keys, return the active key to sign tokens it issues func LoadIssuerPrivateKey(issuerKeysDir string) (jwk.Key, error) { // Ensure initKeysMap is only called once across the program’s runtime var initErr error diff --git a/launcher_utils/register_namespace.go b/launcher_utils/register_namespace.go index 291087839..9bb0bf0f4 100644 --- a/launcher_utils/register_namespace.go +++ b/launcher_utils/register_namespace.go @@ -40,6 +40,7 @@ import ( "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/registry" "github.com/pelicanplatform/pelican/server_structs" + "github.com/pelicanplatform/pelican/server_utils" ) type ( @@ -243,12 +244,13 @@ func registerNamespacePrep(ctx context.Context, prefix string) (key jwk.Key, reg } } - keyStatus, err := keyIsRegistered(privateKey, registrationUrl, prefix) + var keyStatusVar keyStatus + keyStatusVar, err = keyIsRegistered(privateKey, registrationUrl, prefix) if err != nil { err = errors.Wrap(err, "Failed to determine whether namespace is already registered") - break + return } - switch keyStatus { + switch keyStatusVar { case keyMatch: isRegistered = true case keyMismatch: @@ -263,7 +265,7 @@ func registerNamespacePrep(ctx context.Context, prefix string) (key jwk.Key, reg err = errors.Errorf("Namespace %v already registered under a different key", prefix) return } - // Else, there is at least one private key from the origin match the registered public key, + // If there is at least one private key from the origin match the registered public key, // we will update the public key of the namespace in registry db with the active private key // held by this origin; the verified new private key also get added to in-memory map in this process key, err = config.LoadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()) @@ -332,3 +334,41 @@ func RegisterNamespaceWithRetry(ctx context.Context, egrp *errgroup.Group, prefi }) return nil } + +// Check the directory containing .pem files every 5 minutes, load new private key(s) +// if new file(s) are detected, then register the new public key +func LaunchIssuerKeysDirRefresh(ctx context.Context, egrp *errgroup.Group) { + egrp.Go(func() error { + ticker := time.NewTicker(5 * time.Minute) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + log.Debugln("Stopping periodic check for private keys directory.") + return nil + case <-ticker.C: + extUrlStr := param.Server_ExternalWebUrl.GetString() + extUrl, _ := url.Parse(extUrlStr) + namespace := server_structs.GetOriginNs(extUrl.Host) + if err := RegisterNamespaceWithRetry(ctx, egrp, namespace); err != nil { + log.Errorf("Error refreshing private keys: %v", err) + } + + originExports, err := server_utils.GetOriginExports() + if err != nil { + return err + } + for _, export := range originExports { + if err := RegisterNamespaceWithRetry(ctx, egrp, export.FederationPrefix); err != nil { + return err + } + } + + if key, err := config.GetIssuerPrivateJWK(); err != nil { + log.Debugln("Private keys directory refreshed successfully. The active (latest) private key is", key.KeyID()) + } + } + } + }) +} diff --git a/launcher_utils/register_namespace_test.go b/launcher_utils/register_namespace_test.go index 8e329bc2b..78bd5e8d4 100644 --- a/launcher_utils/register_namespace_test.go +++ b/launcher_utils/register_namespace_test.go @@ -231,6 +231,7 @@ func TestMultiKeysRegistration(t *testing.T) { secondPubKey, err := secondKey.PublicKey() require.NoError(t, err) activeKey, err := config.GetIssuerPrivateJWK() + require.NoError(t, err) require.Equal(t, secondKey, activeKey) keysMap = config.GetIssuerPrivateKeys() require.Equal(t, secondKey, keysMap[secondKey.KeyID()]) diff --git a/launchers/origin_serve.go b/launchers/origin_serve.go index 4a0f753ae..5672f2902 100644 --- a/launchers/origin_serve.go +++ b/launchers/origin_serve.go @@ -33,7 +33,6 @@ import ( "github.com/spf13/viper" "golang.org/x/sync/errgroup" - "github.com/pelicanplatform/pelican/config" "github.com/pelicanplatform/pelican/launcher_utils" "github.com/pelicanplatform/pelican/metrics" "github.com/pelicanplatform/pelican/oa4mp" @@ -44,35 +43,6 @@ import ( "github.com/pelicanplatform/pelican/xrootd" ) -// Check the directory containing .pem files every 5 minutes, load new private key(s) -// if new file(s) are detected, then register the new public key -func launchIssuerKeysDirRefresh(ctx context.Context, egrp *errgroup.Group) { - egrp.Go(func() error { - ticker := time.NewTicker(5 * time.Minute) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - log.Debugln("Stopping periodic check for private keys directory.") - return nil - case <-ticker.C: - extUrlStr := param.Server_ExternalWebUrl.GetString() - extUrl, _ := url.Parse(extUrlStr) - namespace := server_structs.GetOriginNs(extUrl.Host) - if err := launcher_utils.RegisterNamespaceWithRetry(ctx, egrp, namespace); err != nil { - log.Errorf("Error refreshing private keys: %v", err) - } else { - if key, err := config.GetIssuerPrivateJWK(); err != nil { - log.Debugln("Private keys directory refreshed successfully. The active (latest) private key is", key.KeyID()) - } - } - - } - } - }) -} - func OriginServe(ctx context.Context, engine *gin.Engine, egrp *errgroup.Group, modules server_structs.ServerType) (server_structs.XRootDServer, error) { metrics.SetComponentHealthStatus(metrics.OriginCache_XRootD, metrics.StatusWarning, "XRootD is initializing") metrics.SetComponentHealthStatus(metrics.OriginCache_CMSD, metrics.StatusWarning, "CMSD is initializting") @@ -108,7 +78,7 @@ func OriginServe(ctx context.Context, engine *gin.Engine, egrp *errgroup.Group, // Start a routine to periodically refresh the private key directory. // This ensures that new or updated private keys are automatically loaded and registered - launchIssuerKeysDirRefresh(ctx, egrp) + launcher_utils.LaunchIssuerKeysDirRefresh(ctx, egrp) // Set up the APIs unrelated to UI, which only contains director-based health test reporting endpoint for now if err = origin.RegisterOriginAPI(engine, ctx, egrp); err != nil { diff --git a/origin/origin_ui.go b/origin/origin_ui.go index 804d00cd1..931eeddce 100644 --- a/origin/origin_ui.go +++ b/origin/origin_ui.go @@ -161,16 +161,18 @@ func handleExports(ctx *gin.Context) { ctx.JSON(http.StatusOK, res) } -// Create a new private key in the given directory and set it as the active key +// Create a new private key in the given directory, but without loading it immediately +// This new key will be detected and loaded later by the ongoing goroutine `LaunchIssuerKeysDirRefresh`, +// which periodically refreshes the keys in the issuer keys directory func createNewIssuerKey(ctx *gin.Context) { issuerKeysDir := param.IssuerKeysDirectory.GetString() - _, err := config.GeneratePEMandSetActiveKey(issuerKeysDir) + _, err := config.GeneratePEM(issuerKeysDir) if err != nil { - log.Errorf("Error creating and loading a new private key in a new .pem file: %v", err) + log.Errorf("Error creating a new private key in a new .pem file: %v", err) ctx.JSON(http.StatusInternalServerError, server_structs.SimpleApiResp{ Status: server_structs.RespFailed, - Msg: "Error creating and loading a new private key in a new .pem file"}) + Msg: "Error creating a new private key in a new .pem file"}) } ctx.JSON(http.StatusOK, From 473c5f312c8dcd765aae9fa7e88f1851d1900ad0 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Mon, 2 Dec 2024 23:50:42 +0000 Subject: [PATCH 21/64] improve how the registry authorize origin to address the security concerns --- registry/client_commands.go | 19 ++++++++++- registry/registry.go | 63 +++++++++++++++++++++++++++++++++++-- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/registry/client_commands.go b/registry/client_commands.go index c56f481bf..2f82b17e2 100644 --- a/registry/client_commands.go +++ b/registry/client_commands.go @@ -169,9 +169,26 @@ func NamespaceRegister(privateKey jwk.Key, namespaceRegistryEndpoint string, acc return errors.Wrap(err, "failed to sign payload") } - // // Create data for the second POST request + // Send origin's all public keys in another key set + privateKeys := config.GetIssuerPrivateKeys() + if len(privateKeys) == 0 { + return errors.Wrap(err, "The server doesn't have any in-memory private key") + } + allKeysSet := jwk.NewSet() + for _, privKey := range privateKeys { + pubKey, err := privKey.PublicKey() + if err != nil { + return errors.Wrapf(err, "failed to get the public key of a private key") + } + if err = allKeysSet.AddKey(pubKey); err != nil { + return errors.Wrap(err, "failed to add public key to all keys JWKS") + } + } + + // Create data for the second POST request unidentifiedPayload := map[string]interface{}{ "pubkey": keySet, + "all_pubkeys": allKeysSet, "prefix": prefix, "site_name": siteName, "client_nonce": clientNonce, diff --git a/registry/registry.go b/registry/registry.go index 85a9f2ec4..8627a1b95 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -101,6 +101,7 @@ type registrationData struct { ServerSignature string `json:"server_signature"` Pubkey json.RawMessage `json:"pubkey"` + AllPubkeys json.RawMessage `json:"all_pubkeys"` Prefix string `json:"prefix"` SiteName string `json:"site_name"` AccessToken string `json:"access_token"` @@ -259,6 +260,7 @@ func keySignChallengeCommit(ctx *gin.Context, data *registrationData) (bool, map return false, nil, errors.Wrap(err, "failed to generate raw pubkey from jwks") } + // Verify client and server private keys' Proof of Possesion clientPayload := []byte(data.ClientNonce + data.ServerNonce) clientSignature, err := hex.DecodeString(data.ClientSignature) if err != nil { @@ -292,13 +294,70 @@ func keySignChallengeCommit(ctx *gin.Context, data *registrationData) (bool, map return false, nil, errors.Wrap(err, "Server encountered an error checking if namespace already exists") } if exists { - // Update the namespace's public key with the latest one - // when origin provides a new key (the origin is authorized to do that in upstream code) + // Update the namespace's public key with the latest one when authorized origin provides a new key existingNs, err := getNamespaceByPrefix(data.Prefix) if err != nil { log.Errorf("Failed to get existing namespace to update: %v", err) return false, nil, errors.Wrap(err, "Server encountered an error getting existing namespace to update") } + + // Check the origin is authorized to update (possessing the public key used for prefix initial registration) + // Parse all public keys of the sender into a JWKS + clientKeySet, err := jwk.Parse(data.AllPubkeys) + if err != nil { + log.Errorf("Failed to parse in-memory public keys of the client: %v", err) + return false, nil, errors.Wrapf(err, "Invalid in-memory public keys format of the client") + } + // Parse `existingNs.Pubkey` as a JWKS + existingKeySet := jwk.NewSet() + err = json.Unmarshal([]byte(existingNs.Pubkey), &existingKeySet) + if err != nil { + log.Errorf("Failed to parse existing namespace public key as JWKS: %v", err) + return false, nil, errors.Wrap(err, "Invalid existing namespace public key format") + } + + // Check if any key in `clientKeySet` matches a key in `existingKeySet` + ctx := context.Background() + existingKeysIter := existingKeySet.Keys(ctx) + clientKeysIter := clientKeySet.Keys(ctx) + matchFound := false + + for existingKeysIter.Next(ctx) { + existingKey := existingKeysIter.Pair().Value.(jwk.Key) + + existingKid, ok := existingKey.Get("kid") + if !ok { + log.Warnf("Skipping registry db existing key without 'kid'") + continue + } + + for clientKeysIter.Next(ctx) { + clientKey := clientKeysIter.Pair().Value.(jwk.Key) + + clientKid, ok := clientKey.Get("kid") + if !ok { + log.Warnf("Skipping client key without 'kid'") + continue + } + + // Compare the "kid" values (or other properties if needed) + if existingKid == clientKid { + matchFound = true + break + } + } + + if matchFound { + break + } + } + + if !matchFound { + return false, nil, permissionDeniedError{ + Message: fmt.Sprintf("The provided public keys do not match the existing namespace's public key for prefix: %s", data.Prefix), + } + } + log.Debugf("New public key %s is going to replace the old one: %s", string(data.Pubkey), existingNs.Pubkey) // Process origin's new public key From 007b7f18d5c5dead18cec48d4f81ad66942a4308 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 6 Dec 2024 22:49:43 +0000 Subject: [PATCH 22/64] =?UTF-8?q?Enhanced=20PoP=20using=20a=20"previous=20?= =?UTF-8?q?private=C2=A0key's=20signature";=20previous=20active=20private?= =?UTF-8?q?=C2=A0key=20var;=20Remove=20the=20redundant=20"isRegistered"=20?= =?UTF-8?q?logic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/init_server_creds.go | 23 +++++++- launcher_utils/register_namespace.go | 70 +++++------------------ launcher_utils/register_namespace_test.go | 12 ++-- registry/client_commands.go | 37 +++++++++--- registry/registry.go | 40 ++++++++++--- 5 files changed, 100 insertions(+), 82 deletions(-) diff --git a/config/init_server_creds.go b/config/init_server_creds.go index aef240cf5..50bf71cab 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -47,10 +47,12 @@ import ( ) var ( - // This is the private JWK for the server to sign tokens. This key remains - // the same if the IssuerKey is unchanged + // This is the private JWK for the server to sign tokens issuerPrivateJWK atomic.Pointer[jwk.Key] + // This is the previous private JWK for the server to sign tokens (before current one) + previousIssuerPrivateJWK atomic.Pointer[jwk.Key] + // Representing private keys (from all .pem files) in the directory cache issuerPrivateKeys atomic.Pointer[map[string]jwk.Key] @@ -66,6 +68,22 @@ func ResetIssuerJWKPtr() { issuerPrivateJWK.Store(nil) } +func GetPreviousIssuerPrivateJWK() jwk.Key { + previousKey := previousIssuerPrivateJWK.Load() + if previousKey == nil { + return nil + } + return *previousKey +} + +func UpdatePreviousIssuerPrivateJWK() { + // Save the current key to previousIssuerPrivateJWK + currentKey := issuerPrivateJWK.Load() + if currentKey != nil { + previousIssuerPrivateJWK.Store(currentKey) + } +} + // Set a private key as the active private key in use func SetActiveKey(key jwk.Key) { issuerPrivateJWK.Store(&key) @@ -796,6 +814,7 @@ func GetIssuerPrivateJWK() (jwk.Key, error) { isDirChanged = true } + // Re-scan the private keys dir when no active private key in memory or dir changes if key == nil || isDirChanged { newKey, err := LoadIssuerPrivateKey(issuerKeysDir) if err != nil { diff --git a/launcher_utils/register_namespace.go b/launcher_utils/register_namespace.go index 9bb0bf0f4..87b9d0c8d 100644 --- a/launcher_utils/register_namespace.go +++ b/launcher_utils/register_namespace.go @@ -200,7 +200,7 @@ func keyIsRegistered(privkey jwk.Key, registryUrlStr string, prefix string) (key } } -func registerNamespacePrep(ctx context.Context, prefix string) (key jwk.Key, registrationUrl string, isRegistered bool, err error) { +func registerNamespacePrep(ctx context.Context, prefix string) (key jwk.Key, registrationUrl string, err error) { // TODO: We eventually want to be able to export multiple prefixes; at that point, we'll // refactor to loop around all the namespaces if prefix == "" { @@ -228,47 +228,7 @@ func registerNamespacePrep(ctx context.Context, prefix string) (key jwk.Key, reg return } - // Get in-memory private keys to check if any of them matches the registered public key - privateKeys := config.GetIssuerPrivateKeys() - if len(privateKeys) == 0 { - err = errors.Wrap(err, "failed to load the origin's private key(s)") - return - } - // Traverse each private key provided by the origin to see if it is registered - var misMatchCount int - for _, privateKey := range privateKeys { - if privateKey.KeyID() == "" { - if err = jwk.AssignKeyID(privateKey); err != nil { - err = errors.Wrap(err, "Error when generating a key ID for registration") - return - } - } - - var keyStatusVar keyStatus - keyStatusVar, err = keyIsRegistered(privateKey, registrationUrl, prefix) - if err != nil { - err = errors.Wrap(err, "Failed to determine whether namespace is already registered") - return - } - switch keyStatusVar { - case keyMatch: - isRegistered = true - case keyMismatch: - misMatchCount += 1 - case noKeyPresent: - log.Infof("Namespace %v not registered; new registration will proceed\n", prefix) - } - } - // All private keys from the origin mismatch the registered public key in registry db, - // meaning this origin don't have the credentials to own the namespace they claim - if misMatchCount == len(privateKeys) { - err = errors.Errorf("Namespace %v already registered under a different key", prefix) - return - } - // If there is at least one private key from the origin match the registered public key, - // we will update the public key of the namespace in registry db with the active private key - // held by this origin; the verified new private key also get added to in-memory map in this process - key, err = config.LoadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()) + key, err = config.GetIssuerPrivateJWK() if err != nil { err = errors.Wrap(err, "Failed to obtain origin's active private key") } @@ -294,19 +254,10 @@ func RegisterNamespaceWithRetry(ctx context.Context, egrp *errgroup.Group, prefi } siteName := param.Xrootd_Sitename.GetString() - key, url, isRegistered, err := registerNamespacePrep(ctx, prefix) + key, url, err := registerNamespacePrep(ctx, prefix) if err != nil { return err } - if isRegistered { - metrics.SetComponentHealthStatus(metrics.OriginCache_Registry, metrics.StatusOK, "") - log.Debugf("Origin already has prefix %v registered\n", prefix) - if err := origin.FetchAndSetRegStatus(prefix); err != nil { - return errors.Wrapf(err, "failed to fetch registration status for the prefix %s", prefix) - } - // continue to update the public key of this prefix - } - if err = registerNamespaceImpl(key, prefix, siteName, url); err == nil { return nil } @@ -348,11 +299,21 @@ func LaunchIssuerKeysDirRefresh(ctx context.Context, egrp *errgroup.Group) { log.Debugln("Stopping periodic check for private keys directory.") return nil case <-ticker.C: + // Refresh the disk to pick up any new private key + config.UpdatePreviousIssuerPrivateJWK() + key, err := config.LoadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()) + if err != nil { + return errors.Wrap(err, "Failed to refresh the disk to pick up any new private key") + } + log.Debugln("Private keys directory refreshed successfully. The active (latest) private key is", key.KeyID()) + log.Debugln("Previous private key is", config.GetPreviousIssuerPrivateJWK().KeyID()) + + // Update public key in registry db with the new active private key extUrlStr := param.Server_ExternalWebUrl.GetString() extUrl, _ := url.Parse(extUrlStr) namespace := server_structs.GetOriginNs(extUrl.Host) if err := RegisterNamespaceWithRetry(ctx, egrp, namespace); err != nil { - log.Errorf("Error refreshing private keys: %v", err) + log.Errorf("Error registering updated private key: %v", err) } originExports, err := server_utils.GetOriginExports() @@ -365,9 +326,6 @@ func LaunchIssuerKeysDirRefresh(ctx context.Context, egrp *errgroup.Group) { } } - if key, err := config.GetIssuerPrivateJWK(); err != nil { - log.Debugln("Private keys directory refreshed successfully. The active (latest) private key is", key.KeyID()) - } } } }) diff --git a/launcher_utils/register_namespace_test.go b/launcher_utils/register_namespace_test.go index 78bd5e8d4..22c5a5015 100644 --- a/launcher_utils/register_namespace_test.go +++ b/launcher_utils/register_namespace_test.go @@ -109,9 +109,8 @@ func TestRegistration(t *testing.T) { // Test registration succeeds prefix := param.Origin_FederationPrefix.GetString() - key, registerURL, isRegistered, err := registerNamespacePrep(ctx, prefix) + key, registerURL, err := registerNamespacePrep(ctx, prefix) require.NoError(t, err) - assert.False(t, isRegistered) assert.Equal(t, registerURL, svr.URL+"/api/v1.0/registry") err = registerNamespaceImpl(key, prefix, "mock_site_name", registerURL) require.NoError(t, err) @@ -168,10 +167,9 @@ func TestRegistration(t *testing.T) { // Redo the namespace prep, ensure that isPresent is true prefix = param.Origin_FederationPrefix.GetString() - _, registerURL, isRegistered, err = registerNamespacePrep(ctx, prefix) + _, registerURL, err = registerNamespacePrep(ctx, prefix) assert.Equal(t, svr.URL+"/api/v1.0/registry", registerURL) assert.NoError(t, err) - assert.True(t, isRegistered) } func TestMultiKeysRegistration(t *testing.T) { @@ -256,9 +254,8 @@ func TestMultiKeysRegistration(t *testing.T) { // Test registration succeeds prefix := param.Origin_FederationPrefix.GetString() - key, registerURL, isRegistered, err := registerNamespacePrep(ctx, prefix) + key, registerURL, err := registerNamespacePrep(ctx, prefix) require.NoError(t, err) - assert.False(t, isRegistered) assert.Equal(t, registerURL, svr.URL+"/api/v1.0/registry") err = registerNamespaceImpl(key, prefix, "mock_site_name", registerURL) require.NoError(t, err) @@ -315,8 +312,7 @@ func TestMultiKeysRegistration(t *testing.T) { // Redo the namespace prep, ensure that isPresent is true prefix = param.Origin_FederationPrefix.GetString() - _, registerURL, isRegistered, err = registerNamespacePrep(ctx, prefix) + _, registerURL, err = registerNamespacePrep(ctx, prefix) assert.Equal(t, svr.URL+"/api/v1.0/registry", registerURL) assert.NoError(t, err) - assert.True(t, isRegistered) } diff --git a/registry/client_commands.go b/registry/client_commands.go index 2f82b17e2..3ad606eb9 100644 --- a/registry/client_commands.go +++ b/registry/client_commands.go @@ -168,6 +168,21 @@ func NamespaceRegister(privateKey jwk.Key, namespaceRegistryEndpoint string, acc if err != nil { return errors.Wrap(err, "failed to sign payload") } + // Also sign the payload with the previous private key + prevPrivateKey := config.GetPreviousIssuerPrivateJWK() + + var prevSignature []byte + // When Pelican starts at the first time or the admin never changes the key, there is no previous private key + if prevPrivateKey != nil { + prevPrivateKeyRaw := &ecdsa.PrivateKey{} + if err = prevPrivateKey.Raw(prevPrivateKeyRaw); err != nil { + return errors.Wrap(err, "failed to get previous raw private key in ECDSA") + } + prevSignature, err = signPayload([]byte(clientPayload), prevPrivateKeyRaw) + if err != nil { + return errors.Wrapf(err, "failed to sign payload with previous private key, previous signature is '%s'", prevSignature) + } + } // Send origin's all public keys in another key set privateKeys := config.GetIssuerPrivateKeys() @@ -187,14 +202,20 @@ func NamespaceRegister(privateKey jwk.Key, namespaceRegistryEndpoint string, acc // Create data for the second POST request unidentifiedPayload := map[string]interface{}{ - "pubkey": keySet, - "all_pubkeys": allKeysSet, - "prefix": prefix, - "site_name": siteName, - "client_nonce": clientNonce, - "server_nonce": respData.ServerNonce, - "client_payload": clientPayload, - "client_signature": hex.EncodeToString(signature), + "pubkey": keySet, + "all_pubkeys": allKeysSet, + "prefix": prefix, + "site_name": siteName, + "client_nonce": clientNonce, + "server_nonce": respData.ServerNonce, + "client_payload": clientPayload, + "client_signature": hex.EncodeToString(signature), + "client_prev_signature": func() string { + if prevSignature == nil { + return "" + } + return hex.EncodeToString(prevSignature) + }(), "server_payload": respData.ServerPayload, "server_signature": respData.ServerSignature, "access_token": accessToken, diff --git a/registry/registry.go b/registry/registry.go index 8627a1b95..3417dc0f5 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -92,9 +92,10 @@ type NamespaceConfig struct { Various auxiliary functions used for client-server security handshakes */ type registrationData struct { - ClientNonce string `json:"client_nonce"` - ClientPayload string `json:"client_payload"` - ClientSignature string `json:"client_signature"` + ClientNonce string `json:"client_nonce"` + ClientPayload string `json:"client_payload"` + ClientSignature string `json:"client_signature"` + ClientPrevSignature string `json:"client_prev_signature"` ServerNonce string `json:"server_nonce"` ServerPayload string `json:"server_payload"` @@ -260,9 +261,10 @@ func keySignChallengeCommit(ctx *gin.Context, data *registrationData) (bool, map return false, nil, errors.Wrap(err, "failed to generate raw pubkey from jwks") } - // Verify client and server private keys' Proof of Possesion + // Verify the Proof of Possession of the client and server's active private keys clientPayload := []byte(data.ClientNonce + data.ServerNonce) clientSignature, err := hex.DecodeString(data.ClientSignature) + log.Debugf("client's active signature: %s; active key: %s", string(clientSignature), key.KeyID()) if err != nil { return false, nil, errors.Wrap(err, "Failed to decode the client's signature") } @@ -340,10 +342,32 @@ func keySignChallengeCommit(ctx *gin.Context, data *registrationData) (bool, map continue } - // Compare the "kid" values (or other properties if needed) if existingKid == clientKid { - matchFound = true - break + // Verify the Proof of Possession of client's previous active private key + // Get client's previous public key recorded in db + var prevRawkey interface{} + if err := existingKey.Raw(&prevRawkey); err != nil { + return false, nil, errors.Wrap(err, "failed to generate raw pubkey from client's previous pubkey") + } + // Get client's previous signature from payload + var prevKeyVerified bool + if data.ClientPrevSignature == "" { + prevKeyVerified = true + log.Debugf("This is the initial namespace registration. Client's previous signature is an empty string") + } else { + clientPrevSignature, err := hex.DecodeString(data.ClientPrevSignature) + if err != nil { + return false, nil, errors.Wrap(err, "Failed to decode the client's previous signature") + } + prevKeyVerified = verifySignature(clientPayload, clientPrevSignature, (prevRawkey).(*ecdsa.PublicKey)) + } + + if prevKeyVerified { + matchFound = true + break + } else { + log.Debugf("Client cannot prove that it possesses the key it claims, key id: %s", existingKid) + } } } @@ -354,7 +378,7 @@ func keySignChallengeCommit(ctx *gin.Context, data *registrationData) (bool, map if !matchFound { return false, nil, permissionDeniedError{ - Message: fmt.Sprintf("The provided public keys do not match the existing namespace's public key for prefix: %s", data.Prefix), + Message: fmt.Sprintf("The client that tries to prefix '%s' cannot be authorized: either it doesn't contain any public key matching the existing namespace's public key in db, or it fails to pass the proof of possession verification", data.Prefix), } } From 00b24c854062604770da3f797306f2a9452fd276 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Tue, 10 Dec 2024 21:05:41 +0000 Subject: [PATCH 23/64] Avoid key file naming collision when running new and old codebase back and forth --- config/init_server_creds.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/init_server_creds.go b/config/init_server_creds.go index 50bf71cab..d89b2926d 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -592,7 +592,9 @@ func migratePrivateKey(newDir string) error { } // Rename the existing private key file and set destination path - fileName := "migrated_key.pem" + fileName := fmt.Sprintf("migrated_%d_%s.pem", + time.Now().UnixNano(), + utils.CreateRandomString(4)) destPath := filepath.Join(newDir, fileName) // Check if a file with the same name already exists in the destination From ac8eadf951e724b7060e1a6b0aa1fa0ec9643135 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 13 Dec 2024 21:03:42 +0000 Subject: [PATCH 24/64] handle old origin and a new registry (and vice versa); fix bug and add tests for new key creation endpoint --- config/config.go | 2 +- origin/origin_ui.go | 15 ++++++-- origin/origin_ui_test.go | 75 ++++++++++++++++++++++++++++++++++++++++ registry/registry.go | 7 +++- 4 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 origin/origin_ui_test.go diff --git a/config/config.go b/config/config.go index a5866e698..6075fdf01 100644 --- a/config/config.go +++ b/config/config.go @@ -1439,7 +1439,7 @@ func InitServer(ctx context.Context, currentServers server_structs.ServerType) e // As necessary, generate private keys, JWKS and corresponding certs - // Note: This function will generate a private key in the location stored by the viper var "IssuerKey" + // Note: This function will generate a private key in the location stored by the viper var "IssuerKeysDirectory" // iff there isn't any valid private key present in that location _, err = GetIssuerPublicJWKS() if err != nil { diff --git a/origin/origin_ui.go b/origin/origin_ui.go index 931eeddce..15d789dd4 100644 --- a/origin/origin_ui.go +++ b/origin/origin_ui.go @@ -24,6 +24,7 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/lestrrat-go/jwx/v2/jwk" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -164,15 +165,16 @@ func handleExports(ctx *gin.Context) { // Create a new private key in the given directory, but without loading it immediately // This new key will be detected and loaded later by the ongoing goroutine `LaunchIssuerKeysDirRefresh`, // which periodically refreshes the keys in the issuer keys directory -func createNewIssuerKey(ctx *gin.Context) { +func createNewIssuerKey(ctx *gin.Context, generatePEM func(string) (jwk.Key, error)) { issuerKeysDir := param.IssuerKeysDirectory.GetString() - _, err := config.GeneratePEM(issuerKeysDir) + _, err := generatePEM(issuerKeysDir) if err != nil { log.Errorf("Error creating a new private key in a new .pem file: %v", err) ctx.JSON(http.StatusInternalServerError, server_structs.SimpleApiResp{ Status: server_structs.RespFailed, Msg: "Error creating a new private key in a new .pem file"}) + return } ctx.JSON(http.StatusOK, @@ -186,7 +188,14 @@ func RegisterOriginWebAPI(engine *gin.Engine) error { originWebAPI := engine.Group("/api/v1.0/origin_ui") { originWebAPI.GET("/exports", web_ui.AuthHandler, web_ui.AdminAuthHandler, handleExports) - originWebAPI.GET("/newIssuerKey", web_ui.AuthHandler, web_ui.AdminAuthHandler, createNewIssuerKey) + + // For unit test: GeneratePEM will be replaced by a mockup func + defaultGeneratePEM := func(directory string) (jwk.Key, error) { + return config.GeneratePEM(directory) + } + originWebAPI.GET("/newIssuerKey", web_ui.AuthHandler, web_ui.AdminAuthHandler, func(ctx *gin.Context) { + createNewIssuerKey(ctx, defaultGeneratePEM) + }) } // Globus backend specific. Config other origin routes above this line diff --git a/origin/origin_ui_test.go b/origin/origin_ui_test.go new file mode 100644 index 000000000..b8287b71c --- /dev/null +++ b/origin/origin_ui_test.go @@ -0,0 +1,75 @@ +package origin + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/gin-gonic/gin" + "github.com/lestrrat-go/jwx/v2/jwk" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/pelicanplatform/pelican/config" + "github.com/pelicanplatform/pelican/server_structs" + "github.com/pelicanplatform/pelican/server_utils" +) + +func TestSuccessfulCreateNewIssuerKey(t *testing.T) { + server_utils.ResetTestState() + + tDir := t.TempDir() + viper.Set("IssuerKeysDirectory", filepath.Join(tDir, "test-issuer-keys")) + viper.Set("ConfigDir", t.TempDir()) + config.InitConfig() + + router := gin.Default() + router.GET("/api/v1.0/origin_ui/newIssuerKey", func(ctx *gin.Context) { + createNewIssuerKey(ctx, config.GeneratePEM) + }) + + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/api/v1.0/origin_ui/newIssuerKey", nil) + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var response server_structs.SimpleApiResp + err := json.Unmarshal(w.Body.Bytes(), &response) + require.NoError(t, err) + assert.Equal(t, server_structs.RespOK, response.Status) + assert.Equal(t, "Created a new issuer key and set it as the active private key", response.Msg) + +} + +func TestFailedCreateNewIssuerKey(t *testing.T) { + server_utils.ResetTestState() + + tDir := t.TempDir() + viper.Set("IssuerKeysDirectory", filepath.Join(tDir, "test-issuer-keys")) + viper.Set("ConfigDir", t.TempDir()) + config.InitConfig() + + router := gin.Default() + mockErrGeneratePEM := func(directory string) (jwk.Key, error) { + return nil, assert.AnError + } + router.GET("/api/v1.0/origin_ui/newIssuerKey", func(ctx *gin.Context) { + createNewIssuerKey(ctx, mockErrGeneratePEM) + }) + + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/api/v1.0/origin_ui/newIssuerKey", nil) + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusInternalServerError, w.Code) + + var response server_structs.SimpleApiResp + err := json.Unmarshal(w.Body.Bytes(), &response) + require.NoError(t, err) + assert.Equal(t, server_structs.RespFailed, response.Status) + assert.Equal(t, "Error creating a new private key in a new .pem file", response.Msg) +} diff --git a/registry/registry.go b/registry/registry.go index 3417dc0f5..c76685168 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -305,7 +305,12 @@ func keySignChallengeCommit(ctx *gin.Context, data *registrationData) (bool, map // Check the origin is authorized to update (possessing the public key used for prefix initial registration) // Parse all public keys of the sender into a JWKS - clientKeySet, err := jwk.Parse(data.AllPubkeys) + var clientKeySet jwk.Set + if data.AllPubkeys == nil { // backward compatibility - AllPubkeys only exists in the payload in Pelican 7.12 or later + clientKeySet, err = jwk.Parse(data.Pubkey) + } else { + clientKeySet, err = jwk.Parse(data.AllPubkeys) + } if err != nil { log.Errorf("Failed to parse in-memory public keys of the client: %v", err) return false, nil, errors.Wrapf(err, "Invalid in-memory public keys format of the client") From da8322540e21a0b746647e12b63285b2393c0d1b Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Mon, 16 Dec 2024 14:42:03 +0000 Subject: [PATCH 25/64] fix semantic issues in PR review --- config/init_server_creds.go | 20 +++++++++++++++----- config/init_server_creds_test.go | 2 +- launcher_utils/register_namespace.go | 4 ++-- registry/client_commands.go | 2 +- utils/utils.go | 11 ----------- web_ui/ui_test.go | 4 ++-- 6 files changed, 21 insertions(+), 22 deletions(-) diff --git a/config/init_server_creds.go b/config/init_server_creds.go index d89b2926d..9cd1695fd 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -29,6 +29,7 @@ import ( "encoding/pem" "fmt" "math/big" + mathrand "math/rand" "os" "os/exec" "path/filepath" @@ -43,7 +44,6 @@ import ( log "github.com/sirupsen/logrus" "github.com/pelicanplatform/pelican/param" - "github.com/pelicanplatform/pelican/utils" ) var ( @@ -124,6 +124,16 @@ func setCurrentIssuerKeysDir(dir string) { currentIssuerKeysDir.Store(dir) } +// Helper function to generate random string +func createRandomString(length int) string { + const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + b := make([]byte, length) + for i := range b { + b[i] = charset[mathrand.Intn(len(charset))] + } + return string(b) +} + // Return a pointer to an ECDSA private key or RSA private key read from keyLocation. // // This can be used to load ECDSA or RSA private key for various purposes, @@ -594,7 +604,7 @@ func migratePrivateKey(newDir string) error { // Rename the existing private key file and set destination path fileName := fmt.Sprintf("migrated_%d_%s.pem", time.Now().UnixNano(), - utils.CreateRandomString(4)) + createRandomString(4)) destPath := filepath.Join(newDir, fileName) // Check if a file with the same name already exists in the destination @@ -632,7 +642,7 @@ func loadSinglePEM(path string) (jwk.Key, error) { key, err := jwk.ParseKey(contents, jwk.WithPEM(true)) if err != nil { - return nil, errors.Wrap(err, "failed to parse key") + return nil, errors.Wrapf(err, "failed to parse issuer key file %v", path) } // Add the algorithm to the key, needed for verifying tokens elsewhere @@ -712,7 +722,7 @@ func loadPEMFiles(dir string) (jwk.Key, error) { func GeneratePEM(dir string) (jwk.Key, error) { filename := fmt.Sprintf("pelican_generated_%d_%s.pem", time.Now().UnixNano(), - utils.CreateRandomString(4)) + createRandomString(4)) keyPath := filepath.Join(dir, filename) if err := GeneratePrivateKey(keyPath, elliptic.P256(), false); err != nil { @@ -757,7 +767,7 @@ func LoadIssuerPrivateKey(issuerKeysDir string) (jwk.Key, error) { activeKey, err := loadPEMFiles(issuerKeysDir) if err != nil { - return nil, errors.Wrap(err, "failed to load .pem files and find the most recent private key") + return nil, errors.Wrapf(err, "failed to re-scan %s to load .pem files and set the key in the most recent modified file as active private key", issuerKeysDir) } return activeKey, err diff --git a/config/init_server_creds_test.go b/config/init_server_creds_test.go index f50b023ae..45dadf227 100644 --- a/config/init_server_creds_test.go +++ b/config/init_server_creds_test.go @@ -188,7 +188,7 @@ func TestMultiPrivateKey(t *testing.T) { require.NotNil(t, secondKey) assert.NotEqual(t, key.KeyID(), secondKey.KeyID()) - // Check if the active private key points to the lastest key + // Check if the active private key points to the latest key viper.Set(param.IssuerKeysDirectory.GetName(), issuerKeysDir) latestKey, err := GetIssuerPrivateJWK() require.NoError(t, err) diff --git a/launcher_utils/register_namespace.go b/launcher_utils/register_namespace.go index 87b9d0c8d..f2f0b9c6b 100644 --- a/launcher_utils/register_namespace.go +++ b/launcher_utils/register_namespace.go @@ -286,7 +286,7 @@ func RegisterNamespaceWithRetry(ctx context.Context, egrp *errgroup.Group, prefi return nil } -// Check the directory containing .pem files every 5 minutes, load new private key(s) +// Check the issuer key directory containing .pem files every 5 minutes, load new private key(s) // if new file(s) are detected, then register the new public key func LaunchIssuerKeysDirRefresh(ctx context.Context, egrp *errgroup.Group) { egrp.Go(func() error { @@ -303,7 +303,7 @@ func LaunchIssuerKeysDirRefresh(ctx context.Context, egrp *errgroup.Group) { config.UpdatePreviousIssuerPrivateJWK() key, err := config.LoadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()) if err != nil { - return errors.Wrap(err, "Failed to refresh the disk to pick up any new private key") + return err } log.Debugln("Private keys directory refreshed successfully. The active (latest) private key is", key.KeyID()) log.Debugln("Previous private key is", config.GetPreviousIssuerPrivateJWK().KeyID()) diff --git a/registry/client_commands.go b/registry/client_commands.go index 3ad606eb9..fd6d44a9e 100644 --- a/registry/client_commands.go +++ b/registry/client_commands.go @@ -184,7 +184,7 @@ func NamespaceRegister(privateKey jwk.Key, namespaceRegistryEndpoint string, acc } } - // Send origin's all public keys in another key set + // Send origins all public keys in another key set privateKeys := config.GetIssuerPrivateKeys() if len(privateKeys) == 0 { return errors.Wrap(err, "The server doesn't have any in-memory private key") diff --git a/utils/utils.go b/utils/utils.go index f108599ae..3915f47f4 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -19,7 +19,6 @@ package utils import ( - "math/rand" "net" "net/url" "regexp" @@ -154,13 +153,3 @@ func MapToSlice[K comparable, V any](m map[K]V) []V { } return s } - -// Generate random string -func CreateRandomString(length int) string { - const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" - b := make([]byte, length) - for i := range b { - b[i] = charset[rand.Intn(len(charset))] - } - return string(b) -} diff --git a/web_ui/ui_test.go b/web_ui/ui_test.go index d6855c94c..27ffe4526 100644 --- a/web_ui/ui_test.go +++ b/web_ui/ui_test.go @@ -218,8 +218,8 @@ func TestHandleWebUIAuth(t *testing.T) { }) tmpDir := t.TempDir() - issuerFile := filepath.Join(tmpDir, "issuer-keys") - viper.Set(param.IssuerKeysDirectory.GetName(), issuerFile) + issuerDirectory := filepath.Join(tmpDir, "issuer-keys") + viper.Set(param.IssuerKeysDirectory.GetName(), issuerDirectory) viper.Set(param.Server_ExternalWebUrl.GetName(), "https://example.com") _, err := config.GetIssuerPrivateJWK() From 6ae326027943d53f3a5dcec5bcff9c6a7bfc409c Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Tue, 17 Dec 2024 00:12:00 +0000 Subject: [PATCH 26/64] detach Namespaces PubKey Update from Namespace Registration workflow --- launcher_utils/register_namespace.go | 46 ---- launcher_utils/update_namespace_pubkey.go | 118 +++++++++ registry/client_commands.go | 147 +++++++++++ registry/registry.go | 1 + registry/registry_pubkey_update.go | 296 ++++++++++++++++++++++ 5 files changed, 562 insertions(+), 46 deletions(-) create mode 100644 launcher_utils/update_namespace_pubkey.go create mode 100644 registry/registry_pubkey_update.go diff --git a/launcher_utils/register_namespace.go b/launcher_utils/register_namespace.go index f2f0b9c6b..b62a3d1c4 100644 --- a/launcher_utils/register_namespace.go +++ b/launcher_utils/register_namespace.go @@ -40,7 +40,6 @@ import ( "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/registry" "github.com/pelicanplatform/pelican/server_structs" - "github.com/pelicanplatform/pelican/server_utils" ) type ( @@ -285,48 +284,3 @@ func RegisterNamespaceWithRetry(ctx context.Context, egrp *errgroup.Group, prefi }) return nil } - -// Check the issuer key directory containing .pem files every 5 minutes, load new private key(s) -// if new file(s) are detected, then register the new public key -func LaunchIssuerKeysDirRefresh(ctx context.Context, egrp *errgroup.Group) { - egrp.Go(func() error { - ticker := time.NewTicker(5 * time.Minute) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - log.Debugln("Stopping periodic check for private keys directory.") - return nil - case <-ticker.C: - // Refresh the disk to pick up any new private key - config.UpdatePreviousIssuerPrivateJWK() - key, err := config.LoadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()) - if err != nil { - return err - } - log.Debugln("Private keys directory refreshed successfully. The active (latest) private key is", key.KeyID()) - log.Debugln("Previous private key is", config.GetPreviousIssuerPrivateJWK().KeyID()) - - // Update public key in registry db with the new active private key - extUrlStr := param.Server_ExternalWebUrl.GetString() - extUrl, _ := url.Parse(extUrlStr) - namespace := server_structs.GetOriginNs(extUrl.Host) - if err := RegisterNamespaceWithRetry(ctx, egrp, namespace); err != nil { - log.Errorf("Error registering updated private key: %v", err) - } - - originExports, err := server_utils.GetOriginExports() - if err != nil { - return err - } - for _, export := range originExports { - if err := RegisterNamespaceWithRetry(ctx, egrp, export.FederationPrefix); err != nil { - return err - } - } - - } - } - }) -} diff --git a/launcher_utils/update_namespace_pubkey.go b/launcher_utils/update_namespace_pubkey.go new file mode 100644 index 000000000..5d6570c81 --- /dev/null +++ b/launcher_utils/update_namespace_pubkey.go @@ -0,0 +1,118 @@ +package launcher_utils + +import ( + "context" + "net/url" + "time" + + "github.com/lestrrat-go/jwx/v2/jwk" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" + + "github.com/pelicanplatform/pelican/config" + "github.com/pelicanplatform/pelican/param" + "github.com/pelicanplatform/pelican/registry" + "github.com/pelicanplatform/pelican/server_structs" + "github.com/pelicanplatform/pelican/server_utils" +) + +func updateNamespacesPubKeyPrep(ctx context.Context, prefixes []string) (jwk.Key, string, error) { + // Validate the namespace format + for _, prefix := range prefixes { + if prefix == "" { + err := errors.New("Invalid empty prefix for public key update") + return nil, "", err + } + if prefix[0] != '/' { + err := errors.New("Prefix specified for public key update must start with a '/'") + return nil, "", err + } + } + + // Generate the endpoint url that can update the public key of prefixes + fedInfo, err := config.GetFederation(ctx) + if err != nil { + return nil, "", err + } + registryEndpoint := fedInfo.RegistryEndpoint + if registryEndpoint == "" { + err = errors.New("No registry endpoint specified; try passing the `-f` flag specifying the federation name") + return nil, "", err + } + + prefixPubKeyUpdateUrl, err := url.JoinPath(registryEndpoint, "api", "v1.0", "registry", "updateNamespacesPubKey") + if err != nil { + err = errors.Wrap(err, "Failed to construct public key update endpoint URL: %v") + return nil, "", err + } + + // Obtain server's active private key + key, err := config.GetIssuerPrivateJWK() + if err != nil { + err = errors.Wrap(err, "Failed to obtain server's active private key") + return nil, "", err + } + + return key, prefixPubKeyUpdateUrl, nil +} + +func updateNamespacesPubKey(ctx context.Context, prefixes []string) error { + siteName := param.Xrootd_Sitename.GetString() + + key, url, err := updateNamespacesPubKeyPrep(ctx, prefixes) + if err != nil { + return err + } + if err = registry.NamespacesPubKeyUpdate(key, prefixes, siteName, url); err != nil { + return err + } + return nil +} + +// Check the issuer key directory containing .pem files every 5 minutes, load new private key(s) +// if new file(s) are detected, then register the new public key +func LaunchIssuerKeysDirRefresh(ctx context.Context, egrp *errgroup.Group) { + egrp.Go(func() error { + ticker := time.NewTicker(5 * time.Minute) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + log.Debugln("Stopping periodic check for private keys directory.") + return nil + case <-ticker.C: + // Refresh the disk to pick up any new private key + config.UpdatePreviousIssuerPrivateJWK() + key, err := config.LoadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()) + if err != nil { + return err + } + log.Debugln("Private keys directory refreshed successfully. The active (latest) private key is", key.KeyID()) + log.Debugln("Previous private key is", config.GetPreviousIssuerPrivateJWK().KeyID()) + + // Update public key in registry db with the new active private key + extUrlStr := param.Server_ExternalWebUrl.GetString() + extUrl, _ := url.Parse(extUrlStr) + namespace := server_structs.GetOriginNs(extUrl.Host) + if err := updateNamespacesPubKey(ctx, []string{namespace}); err != nil { + log.Errorf("Error updating the public key of the registered origin namespace %s: %v", namespace, err) + } + + originExports, err := server_utils.GetOriginExports() + if err != nil { + return err + } + originExportsNs := make([]string, len(originExports)) + for i, export := range originExports { + originExportsNs[i] = export.FederationPrefix + } + if err := updateNamespacesPubKey(ctx, originExportsNs); err != nil { + log.Errorf("Error updating the public key of origin-exported namespace(s): %v", err) + } + + } + } + }) +} diff --git a/registry/client_commands.go b/registry/client_commands.go index fd6d44a9e..ba0713771 100644 --- a/registry/client_commands.go +++ b/registry/client_commands.go @@ -26,6 +26,7 @@ import ( "encoding/json" "fmt" "os" + "strings" "time" "github.com/lestrrat-go/jwx/v2/jwk" @@ -319,3 +320,149 @@ func NamespaceDelete(endpoint string, prefix string) error { fmt.Println(string(respData)) return nil } + +func NamespacesPubKeyUpdate(privateKey jwk.Key, prefixes []string, siteName string, namespacePubKeyUpdateEndpoint string) error { + publicKey, err := privateKey.PublicKey() + if err != nil { + return errors.Wrapf(err, "failed to generate public key for namespace registration") + } + err = jwk.AssignKeyID(publicKey) + if err != nil { + return errors.Wrap(err, "failed to assign key ID to public key") + } + if err = publicKey.Set("alg", "ES256"); err != nil { + return errors.Wrap(err, "failed to assign signature algorithm to public key") + } + keySet := jwk.NewSet() + if err = keySet.AddKey(publicKey); err != nil { + return errors.Wrap(err, "failed to add public key to new JWKS") + } + + if log.IsLevelEnabled(log.DebugLevel) { + // Let's check that we can convert to JSON and get the right thing... + jsonbuf, err := json.Marshal(keySet) + if err != nil { + return errors.Wrap(err, "failed to marshal the public key into JWKS JSON") + } + log.Debugln("Constructed JWKS from loading public key:", string(jsonbuf)) + } + + clientNonce, err := generateNonce() + if err != nil { + return errors.Wrap(err, "failed to generate client nonce") + } + + data := map[string]interface{}{ + "client_nonce": clientNonce, + "pubkey": keySet, + } + + tr := config.GetTransport() + resp, err := utils.MakeRequest(context.Background(), tr, namespacePubKeyUpdateEndpoint, "POST", data, nil) + + var respData clientResponseData + // Handle case where there was an error encoded in the body + if err != nil { + if strings.Contains(err.Error(), "status code 404") { + log.Warnf("Registered namespace public key update endpoint returned 404 Not Found: %s. This endpoint is available in Pelican v7.12 or later.", namespacePubKeyUpdateEndpoint) + return nil + } + if unmarshalErr := json.Unmarshal(resp, &respData); unmarshalErr == nil { + return errors.Wrapf(err, "Server responded with an error: %s. %s", respData.Message, respData.Error) + } + return errors.Wrapf(err, "Server responded with an error and failed to parse JSON response from the server. Raw server response is %s", resp) + } + + // No error + if err = json.Unmarshal(resp, &respData); err != nil { + return errors.Wrapf(err, "Failure when parsing JSON response from the server with a success request. Raw server response is %s", resp) + } + + // Create client payload by concatenating client_nonce and server_nonce + clientPayload := clientNonce + respData.ServerNonce + + // Sign the payload + privateKeyRaw := &ecdsa.PrivateKey{} + if err = privateKey.Raw(privateKeyRaw); err != nil { + return errors.Wrap(err, "failed to get an ECDSA private key") + } + signature, err := signPayload([]byte(clientPayload), privateKeyRaw) + if err != nil { + return errors.Wrap(err, "failed to sign payload") + } + // Also sign the payload with the previous private key + prevPrivateKey := config.GetPreviousIssuerPrivateJWK() + + var prevSignature []byte + if prevPrivateKey != nil { + prevPrivateKeyRaw := &ecdsa.PrivateKey{} + if err = prevPrivateKey.Raw(prevPrivateKeyRaw); err != nil { + return errors.Wrap(err, "failed to get previous raw private key in ECDSA") + } + prevSignature, err = signPayload([]byte(clientPayload), prevPrivateKeyRaw) + if err != nil { + return errors.Wrapf(err, "failed to sign payload with previous private key, previous signature is '%s'", prevSignature) + } + } + + // Send origins all public keys in another key set + privateKeys := config.GetIssuerPrivateKeys() + if len(privateKeys) == 0 { + return errors.Wrap(err, "The server doesn't have any private key in memory") + } + allKeysSet := jwk.NewSet() + for _, privKey := range privateKeys { + pubKey, err := privKey.PublicKey() + if err != nil { + return errors.Wrapf(err, "failed to get the public key of a private key") + } + if err = allKeysSet.AddKey(pubKey); err != nil { + return errors.Wrap(err, "failed to add public key to all keys JWKS") + } + } + + // Create data for the second POST request + unidentifiedPayload := map[string]interface{}{ + "pubkey": keySet, + "all_pubkeys": allKeysSet, + "prefixes": prefixes, + "site_name": siteName, + "client_nonce": clientNonce, + "server_nonce": respData.ServerNonce, + "client_payload": clientPayload, + "client_signature": hex.EncodeToString(signature), + "client_prev_signature": func() string { + if prevSignature == nil { + return "" + } + return hex.EncodeToString(prevSignature) + }(), + "server_payload": respData.ServerPayload, + "server_signature": respData.ServerSignature, + } + + // Send the second POST request + resp, err = utils.MakeRequest(context.Background(), tr, namespacePubKeyUpdateEndpoint, "POST", unidentifiedPayload, nil) + + // Handle case where there was an error encoded in the body + if unmarshalErr := json.Unmarshal(resp, &respData); unmarshalErr == nil { + if err != nil { + if strings.Contains(err.Error(), "status code 404") { + log.Warnf("Registered namespace public key update endpoint returned 404 Not Found: %s. This endpoint is available in Pelican v7.12 or later.", namespacePubKeyUpdateEndpoint) + return nil + } + log.Errorf("Server responded with an error: %v. %s. %s", respData.Message, respData.Error, err) + return errors.Wrapf(err, "Server responded with an error: %s. %s", respData.Message, respData.Error) + } + if respData.Message != "" { + log.Debugf("Server responded to registration confirmation successfully with message: %s", respData.Message) + } + } else { // Error decoding JSON + if err != nil { + return errors.Wrapf(err, "Server responded with an error and failed to parse JSON response from the server. Raw response is %s", resp) + } + return errors.Wrapf(unmarshalErr, "Failure when parsing JSON response from the server with a success request. Raw server response is %s", resp) + } + + return nil +} diff --git a/registry/registry.go b/registry/registry.go index c76685168..96e8223cc 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -1297,6 +1297,7 @@ func RegisterRegistryAPI(router *gin.RouterGroup) { registryAPI.GET("/*wildcard", wildcardHandler) registryAPI.POST("/checkNamespaceExists", checkNamespaceExistsHandler) registryAPI.POST("/checkNamespaceStatus", checkApprovalHandler) + registryAPI.POST("/updateNamespacesPubKey", updateNamespacesPubKey) registryAPI.DELETE("/*wildcard", deleteNamespaceHandler) } diff --git a/registry/registry_pubkey_update.go b/registry/registry_pubkey_update.go new file mode 100644 index 000000000..83d0f05ac --- /dev/null +++ b/registry/registry_pubkey_update.go @@ -0,0 +1,296 @@ +package registry + +import ( + "crypto/ecdsa" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/lestrrat-go/jwx/v2/jwk" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + + "github.com/pelicanplatform/pelican/server_structs" +) + +type RegisteredPrefixUpdate struct { + ClientNonce string `json:"client_nonce"` + ClientPayload string `json:"client_payload"` + ClientSignature string `json:"client_signature"` + ClientPrevSignature string `json:"client_prev_signature"` + + ServerNonce string `json:"server_nonce"` + ServerPayload string `json:"server_payload"` + ServerSignature string `json:"server_signature"` + + Pubkey json.RawMessage `json:"pubkey"` + AllPubkeys json.RawMessage `json:"all_pubkeys"` + Prefixes []string `json:"prefixes"` +} + +// Generate server nonce for key-sign challenge when updating the public key of registered namespace(s) +func updateNsKeySignChallengeInit(data *RegisteredPrefixUpdate) (map[string]interface{}, error) { + serverNonce, err := generateNonce() + if err != nil { + return nil, errors.Wrap(err, "Failed to generate nonce for key-sign challenge") + } + + serverPayload := []byte(data.ClientNonce + data.ServerNonce) + + privateKey, err := loadServerKeys() + if err != nil { + return nil, errors.Wrap(err, "Server is unable to generate a key sign challenge: Failed to load the server's private key") + } + + serverSignature, err := signPayload(serverPayload, privateKey) + if err != nil { + return nil, errors.Wrap(err, "Failed to sign payload for key-sign challenge") + } + + res := map[string]interface{}{ + "server_nonce": serverNonce, + "client_nonce": data.ClientNonce, + "server_payload": hex.EncodeToString(serverPayload), + "server_signature": hex.EncodeToString(serverSignature), + } + return res, nil +} + +// Update the public key of registered prefix(es) if the http request passed client and server verification for nonce. +// It returns whether registration is created, the response data, and an error if any +func updateNsKeySignChallengeCommit(ctx *gin.Context, data *RegisteredPrefixUpdate) (bool, map[string]interface{}, error) { + // Validate the client's jwks as a set here + key, err := validateJwks(string(data.Pubkey)) + if err != nil { + return false, nil, badRequestError{Message: err.Error()} + } + var rawkey interface{} // This is the raw key, like *rsa.PrivateKey or *ecdsa.PrivateKey + if err := key.Raw(&rawkey); err != nil { + return false, nil, errors.Wrap(err, "failed to generate raw pubkey from jwks") + } + + // Verify the Proof of Possession of the client and server's active private keys + clientPayload := []byte(data.ClientNonce + data.ServerNonce) + clientSignature, err := hex.DecodeString(data.ClientSignature) + if err != nil { + return false, nil, errors.Wrap(err, "Failed to decode the client's signature") + } + clientVerified := verifySignature(clientPayload, clientSignature, (rawkey).(*ecdsa.PublicKey)) + serverPayload, err := hex.DecodeString(data.ServerPayload) + if err != nil { + return false, nil, errors.Wrap(err, "Failed to decode the server's payload") + } + + serverSignature, err := hex.DecodeString(data.ServerSignature) + if err != nil { + return false, nil, errors.Wrap(err, "Failed to decode the server's signature") + } + + serverPrivateKey, err := loadServerKeys() + if err != nil { + return false, nil, errors.Wrap(err, "Failed to decode the server's private key") + } + serverPubkey := serverPrivateKey.PublicKey + serverVerified := verifySignature(serverPayload, serverSignature, &serverPubkey) + + if clientVerified && serverVerified { + for _, prefix := range data.Prefixes { + log.Debug("Start updating namespace: ", prefix) + + // Check if prefix exists before doing anything else + exists, err := namespaceExistsByPrefix(prefix) + if err != nil { + log.Errorf("Failed to check if namespace already exists: %v", err) + return false, nil, errors.Wrap(err, "Server encountered an error checking if namespace already exists") + } + if exists { + // Update the namespace's public key with the latest one when authorized origin provides a new key + existingNs, err := getNamespaceByPrefix(prefix) + if err != nil { + log.Errorf("Failed to get existing namespace to update: %v", err) + return false, nil, errors.Wrap(err, "Server encountered an error getting existing namespace to update") + } + + // Check the origin is authorized to update (possessing the public key used for prefix initial registration) + // Parse all public keys of the sender into a JWKS + var clientKeySet jwk.Set + if data.AllPubkeys == nil { // backward compatibility - AllPubkeys only exists in the payload in Pelican 7.12 or later + clientKeySet, err = jwk.Parse(data.Pubkey) + } else { + clientKeySet, err = jwk.Parse(data.AllPubkeys) + } + if err != nil { + log.Errorf("Failed to parse in-memory public keys of the client: %v", err) + return false, nil, errors.Wrapf(err, "Invalid in-memory public keys format of the client") + } + // Parse `existingNs.Pubkey` as a JWKS + existingKeySet := jwk.NewSet() + err = json.Unmarshal([]byte(existingNs.Pubkey), &existingKeySet) + if err != nil { + log.Errorf("Failed to parse existing namespace public key as JWKS: %v", err) + return false, nil, errors.Wrap(err, "Invalid existing namespace public key format") + } + + // Check if any key in `clientKeySet` matches a key in `existingKeySet` + existingKeysIter := existingKeySet.Keys(ctx) + clientKeysIter := clientKeySet.Keys(ctx) + matchFound := false + + for existingKeysIter.Next(ctx) { + existingKey := existingKeysIter.Pair().Value.(jwk.Key) + + existingKid, ok := existingKey.Get("kid") + if !ok { + log.Warnf("Skipping registry db existing key without 'kid'") + continue + } + + for clientKeysIter.Next(ctx) { + clientKey := clientKeysIter.Pair().Value.(jwk.Key) + + clientKid, ok := clientKey.Get("kid") + if !ok { + log.Warnf("Skipping client key without 'kid'") + continue + } + + if existingKid == clientKid { + // Verify the Proof of Possession of client's previous active private key + // Get client's previous public key recorded in db + var prevRawkey interface{} + if err := existingKey.Raw(&prevRawkey); err != nil { + return false, nil, errors.Wrap(err, "failed to generate raw pubkey from client's previous pubkey") + } + // Get client's previous signature from payload + var prevKeyVerified bool + if data.ClientPrevSignature == "" { + prevKeyVerified = true + } else { + clientPrevSignature, err := hex.DecodeString(data.ClientPrevSignature) + if err != nil { + return false, nil, errors.Wrap(err, "Failed to decode the client's previous signature") + } + prevKeyVerified = verifySignature(clientPayload, clientPrevSignature, (prevRawkey).(*ecdsa.PublicKey)) + } + + if prevKeyVerified { + matchFound = true + break + } else { + log.Debugf("Client cannot prove that it possesses the key it claims, key id: %s", existingKid) + } + } + } + + if matchFound { + break + } + } + + if !matchFound { + return false, nil, permissionDeniedError{ + Message: fmt.Sprintf("The client that tries to prefix '%s' cannot be authorized: either it doesn't contain any public key matching the existing namespace's public key in db, or it fails to pass the proof of possession verification", prefix), + } + } + + log.Debugf("New public key %s is going to replace the old one: %s", string(data.Pubkey), existingNs.Pubkey) + + // Process origin's new public key + pubkeyData, err := json.Marshal(data.Pubkey) + if err != nil { + return false, nil, errors.Wrapf(err, "Failed to convert public key from json to string format for the prefix %s", prefix) + } + pubkeyDbString := string(pubkeyData) + + // Perform the update action when origin provides a new key + if pubkeyDbString != existingNs.Pubkey { + err = updateNamespacePubKey(prefix, pubkeyDbString) + if err != nil { + log.Errorf("Failed to update the public key of namespace %s: %v", prefix, err) + return false, nil, errors.Wrap(err, "Server encountered an error updating the public key of an existing namespace") + } + returnMsg := map[string]interface{}{ + "message": fmt.Sprintf("Updated the public key of namespace %s:", prefix), + } + log.Infof("Updated the public key of namespace %s:", prefix) + return false, returnMsg, nil + } else { + returnMsg := map[string]interface{}{ + "message": fmt.Sprintf("The public key of prefix %s hasn't changed -- nothing to update!", prefix), + } + log.Infof("The public key of prefix %s hasn't changed -- nothing to update!", prefix) + return false, returnMsg, nil + } + } + } + } + + return false, nil, errors.Errorf("Unable to verify the client's public key, or an encountered an error with its own: "+ + "server verified:%t, client verified:%t", serverVerified, clientVerified) + +} + +// Handle the registered namespace public key update with nonce generation and verifcation, regardless of +// using OIDC Authorization or not +func updateNsKeySignChallenge(ctx *gin.Context, data *RegisteredPrefixUpdate) (bool, map[string]interface{}, error) { + if data.ClientNonce != "" && data.ClientPayload != "" && data.ClientSignature != "" && + data.ServerNonce != "" && data.ServerPayload != "" && data.ServerSignature != "" { + created, res, err := updateNsKeySignChallengeCommit(ctx, data) + if err != nil { + return false, nil, err + } else { + return created, res, nil + } + } else if data.ClientNonce != "" { + res, err := updateNsKeySignChallengeInit(data) + if err != nil { + return false, nil, err + } else { + return false, res, nil + } + } else { + return false, nil, badRequestError{Message: "Key sign challenge is missing parameters"} + } +} + +func updateNamespacesPubKey(ctx *gin.Context) { + + var reqData RegisteredPrefixUpdate + if err := ctx.BindJSON(&reqData); err != nil { + log.Errorln("Bad request: ", err) + ctx.JSON(http.StatusBadRequest, server_structs.SimpleApiResp{ + Status: server_structs.RespFailed, + Msg: fmt.Sprint("Bad Request: ", err.Error())}) + return + } + + created, res, err := updateNsKeySignChallenge(ctx, &reqData) + if err != nil { + if errors.As(err, &permissionDeniedError{}) { + ctx.JSON(http.StatusForbidden, + server_structs.SimpleApiResp{ + Status: server_structs.RespFailed, + Msg: fmt.Sprintf("You don't have permission to update the registered public key of the prefix: %v", err), + }) + } else if errors.As(err, &badRequestError{}) { + ctx.JSON(http.StatusBadRequest, server_structs.SimpleApiResp{ + Status: server_structs.RespFailed, + Msg: fmt.Sprintf("Bad request for key-sign challenge: %v", err), + }) + } else { + ctx.JSON(http.StatusInternalServerError, server_structs.SimpleApiResp{ + Status: server_structs.RespFailed, + Msg: fmt.Sprintf("Server encountered an error during key-sign challenge: %v", err), + }) + log.Warningf("Failed to complete key sign challenge without identity requirement: %v", err) + } + } else { + if created { + ctx.JSON(http.StatusCreated, res) + } else { + ctx.JSON(http.StatusOK, res) + } + } +} From adcc5ee56ed4e389b93e5c3b7f1b2ae52105fe51 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Tue, 17 Dec 2024 01:21:05 +0000 Subject: [PATCH 27/64] reverts changes on namespace registration workflow in previous commits --- config/init_server_creds.go | 2 +- launcher_utils/register_namespace.go | 39 ++++++- launcher_utils/register_namespace_test.go | 20 ++-- registry/client_commands.go | 54 ++------- registry/registry.go | 133 ++-------------------- 5 files changed, 62 insertions(+), 186 deletions(-) diff --git a/config/init_server_creds.go b/config/init_server_creds.go index 9cd1695fd..fb0f97704 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -739,7 +739,7 @@ func GeneratePEM(dir string) (jwk.Key, error) { keysCopy[key.KeyID()] = key issuerPrivateKeys.Store(&keysCopy) - log.Debugf("Loaded the private key (key id: %s) into issuerPrivateKeys and set as active key", key.KeyID()) + log.Debugf("Generated private key %s", key.KeyID()) return key, nil } diff --git a/launcher_utils/register_namespace.go b/launcher_utils/register_namespace.go index b62a3d1c4..3159dac3b 100644 --- a/launcher_utils/register_namespace.go +++ b/launcher_utils/register_namespace.go @@ -199,7 +199,7 @@ func keyIsRegistered(privkey jwk.Key, registryUrlStr string, prefix string) (key } } -func registerNamespacePrep(ctx context.Context, prefix string) (key jwk.Key, registrationUrl string, err error) { +func registerNamespacePrep(ctx context.Context, prefix string) (key jwk.Key, registrationUrl string, isRegistered bool, err error) { // TODO: We eventually want to be able to export multiple prefixes; at that point, we'll // refactor to loop around all the namespaces if prefix == "" { @@ -226,12 +226,32 @@ func registerNamespacePrep(ctx context.Context, prefix string) (key jwk.Key, reg err = errors.Wrap(err, "Failed to construct registration endpoint URL: %v") return } - key, err = config.GetIssuerPrivateJWK() if err != nil { - err = errors.Wrap(err, "Failed to obtain origin's active private key") + err = errors.Wrap(err, "failed to load the origin's JWK") + return + } + if key.KeyID() == "" { + if err = jwk.AssignKeyID(key); err != nil { + err = errors.Wrap(err, "Error when generating a key ID for registration") + return + } + } + keyStatus, err := keyIsRegistered(key, registrationUrl, prefix) + if err != nil { + err = errors.Wrap(err, "Failed to determine whether namespace is already registered") + return + } + switch keyStatus { + case keyMatch: + isRegistered = true + return + case keyMismatch: + err = errors.Errorf("Namespace %v already registered under a different key", prefix) + return + case noKeyPresent: + log.Infof("Namespace %v not registered; new registration will proceed\n", prefix) } - return } @@ -253,10 +273,19 @@ func RegisterNamespaceWithRetry(ctx context.Context, egrp *errgroup.Group, prefi } siteName := param.Xrootd_Sitename.GetString() - key, url, err := registerNamespacePrep(ctx, prefix) + key, url, isRegistered, err := registerNamespacePrep(ctx, prefix) if err != nil { return err } + if isRegistered { + metrics.SetComponentHealthStatus(metrics.OriginCache_Registry, metrics.StatusOK, "") + log.Debugf("Origin already has prefix %v registered\n", prefix) + if err := origin.FetchAndSetRegStatus(prefix); err != nil { + return errors.Wrapf(err, "failed to fetch registration status for the prefix %s", prefix) + } + return nil + } + if err = registerNamespaceImpl(key, prefix, siteName, url); err == nil { return nil } diff --git a/launcher_utils/register_namespace_test.go b/launcher_utils/register_namespace_test.go index 22c5a5015..e60aaa931 100644 --- a/launcher_utils/register_namespace_test.go +++ b/launcher_utils/register_namespace_test.go @@ -109,8 +109,9 @@ func TestRegistration(t *testing.T) { // Test registration succeeds prefix := param.Origin_FederationPrefix.GetString() - key, registerURL, err := registerNamespacePrep(ctx, prefix) + key, registerURL, isRegistered, err := registerNamespacePrep(ctx, prefix) require.NoError(t, err) + assert.False(t, isRegistered) assert.Equal(t, registerURL, svr.URL+"/api/v1.0/registry") err = registerNamespaceImpl(key, prefix, "mock_site_name", registerURL) require.NoError(t, err) @@ -165,9 +166,10 @@ func TestRegistration(t *testing.T) { assert.NoError(t, err) assert.Equal(t, keyStatus, noKeyPresent) - // Redo the namespace prep, ensure that isPresent is true + // Redo the namespace prep, ensure that isRegistered is true prefix = param.Origin_FederationPrefix.GetString() - _, registerURL, err = registerNamespacePrep(ctx, prefix) + key, registerURL, isRegistered, err = registerNamespacePrep(ctx, prefix) + assert.True(t, isRegistered) assert.Equal(t, svr.URL+"/api/v1.0/registry", registerURL) assert.NoError(t, err) } @@ -254,8 +256,9 @@ func TestMultiKeysRegistration(t *testing.T) { // Test registration succeeds prefix := param.Origin_FederationPrefix.GetString() - key, registerURL, err := registerNamespacePrep(ctx, prefix) + key, registerURL, isRegistered, err := registerNamespacePrep(ctx, prefix) require.NoError(t, err) + assert.False(t, isRegistered) assert.Equal(t, registerURL, svr.URL+"/api/v1.0/registry") err = registerNamespaceImpl(key, prefix, "mock_site_name", registerURL) require.NoError(t, err) @@ -305,14 +308,15 @@ func TestMultiKeysRegistration(t *testing.T) { assert.NoError(t, err) assert.Equal(t, keyStatus, keyMismatch) - // Verify that no key is present for an alternate prefix + // Verify that no key is registered for an alternate prefix keyStatus, err = keyIsRegistered(key, svr.URL, "test456") assert.NoError(t, err) assert.Equal(t, keyStatus, noKeyPresent) - // Redo the namespace prep, ensure that isPresent is true + // Redo the namespace prep, ensure that isRegistered is true prefix = param.Origin_FederationPrefix.GetString() - _, registerURL, err = registerNamespacePrep(ctx, prefix) + key, registerURL, isRegistered, err = registerNamespacePrep(ctx, prefix) + require.NoError(t, err) + assert.True(t, isRegistered) assert.Equal(t, svr.URL+"/api/v1.0/registry", registerURL) - assert.NoError(t, err) } diff --git a/registry/client_commands.go b/registry/client_commands.go index ba0713771..79a1175b1 100644 --- a/registry/client_commands.go +++ b/registry/client_commands.go @@ -169,54 +169,16 @@ func NamespaceRegister(privateKey jwk.Key, namespaceRegistryEndpoint string, acc if err != nil { return errors.Wrap(err, "failed to sign payload") } - // Also sign the payload with the previous private key - prevPrivateKey := config.GetPreviousIssuerPrivateJWK() - - var prevSignature []byte - // When Pelican starts at the first time or the admin never changes the key, there is no previous private key - if prevPrivateKey != nil { - prevPrivateKeyRaw := &ecdsa.PrivateKey{} - if err = prevPrivateKey.Raw(prevPrivateKeyRaw); err != nil { - return errors.Wrap(err, "failed to get previous raw private key in ECDSA") - } - prevSignature, err = signPayload([]byte(clientPayload), prevPrivateKeyRaw) - if err != nil { - return errors.Wrapf(err, "failed to sign payload with previous private key, previous signature is '%s'", prevSignature) - } - } - - // Send origins all public keys in another key set - privateKeys := config.GetIssuerPrivateKeys() - if len(privateKeys) == 0 { - return errors.Wrap(err, "The server doesn't have any in-memory private key") - } - allKeysSet := jwk.NewSet() - for _, privKey := range privateKeys { - pubKey, err := privKey.PublicKey() - if err != nil { - return errors.Wrapf(err, "failed to get the public key of a private key") - } - if err = allKeysSet.AddKey(pubKey); err != nil { - return errors.Wrap(err, "failed to add public key to all keys JWKS") - } - } - // Create data for the second POST request + // // Create data for the second POST request unidentifiedPayload := map[string]interface{}{ - "pubkey": keySet, - "all_pubkeys": allKeysSet, - "prefix": prefix, - "site_name": siteName, - "client_nonce": clientNonce, - "server_nonce": respData.ServerNonce, - "client_payload": clientPayload, - "client_signature": hex.EncodeToString(signature), - "client_prev_signature": func() string { - if prevSignature == nil { - return "" - } - return hex.EncodeToString(prevSignature) - }(), + "pubkey": keySet, + "prefix": prefix, + "site_name": siteName, + "client_nonce": clientNonce, + "server_nonce": respData.ServerNonce, + "client_payload": clientPayload, + "client_signature": hex.EncodeToString(signature), "server_payload": respData.ServerPayload, "server_signature": respData.ServerSignature, "access_token": accessToken, diff --git a/registry/registry.go b/registry/registry.go index 96e8223cc..c63ca6492 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -92,17 +92,15 @@ type NamespaceConfig struct { Various auxiliary functions used for client-server security handshakes */ type registrationData struct { - ClientNonce string `json:"client_nonce"` - ClientPayload string `json:"client_payload"` - ClientSignature string `json:"client_signature"` - ClientPrevSignature string `json:"client_prev_signature"` + ClientNonce string `json:"client_nonce"` + ClientPayload string `json:"client_payload"` + ClientSignature string `json:"client_signature"` ServerNonce string `json:"server_nonce"` ServerPayload string `json:"server_payload"` ServerSignature string `json:"server_signature"` Pubkey json.RawMessage `json:"pubkey"` - AllPubkeys json.RawMessage `json:"all_pubkeys"` Prefix string `json:"prefix"` SiteName string `json:"site_name"` AccessToken string `json:"access_token"` @@ -261,10 +259,8 @@ func keySignChallengeCommit(ctx *gin.Context, data *registrationData) (bool, map return false, nil, errors.Wrap(err, "failed to generate raw pubkey from jwks") } - // Verify the Proof of Possession of the client and server's active private keys clientPayload := []byte(data.ClientNonce + data.ServerNonce) clientSignature, err := hex.DecodeString(data.ClientSignature) - log.Debugf("client's active signature: %s; active key: %s", string(clientSignature), key.KeyID()) if err != nil { return false, nil, errors.Wrap(err, "Failed to decode the client's signature") } @@ -296,126 +292,11 @@ func keySignChallengeCommit(ctx *gin.Context, data *registrationData) (bool, map return false, nil, errors.Wrap(err, "Server encountered an error checking if namespace already exists") } if exists { - // Update the namespace's public key with the latest one when authorized origin provides a new key - existingNs, err := getNamespaceByPrefix(data.Prefix) - if err != nil { - log.Errorf("Failed to get existing namespace to update: %v", err) - return false, nil, errors.Wrap(err, "Server encountered an error getting existing namespace to update") - } - - // Check the origin is authorized to update (possessing the public key used for prefix initial registration) - // Parse all public keys of the sender into a JWKS - var clientKeySet jwk.Set - if data.AllPubkeys == nil { // backward compatibility - AllPubkeys only exists in the payload in Pelican 7.12 or later - clientKeySet, err = jwk.Parse(data.Pubkey) - } else { - clientKeySet, err = jwk.Parse(data.AllPubkeys) - } - if err != nil { - log.Errorf("Failed to parse in-memory public keys of the client: %v", err) - return false, nil, errors.Wrapf(err, "Invalid in-memory public keys format of the client") - } - // Parse `existingNs.Pubkey` as a JWKS - existingKeySet := jwk.NewSet() - err = json.Unmarshal([]byte(existingNs.Pubkey), &existingKeySet) - if err != nil { - log.Errorf("Failed to parse existing namespace public key as JWKS: %v", err) - return false, nil, errors.Wrap(err, "Invalid existing namespace public key format") - } - - // Check if any key in `clientKeySet` matches a key in `existingKeySet` - ctx := context.Background() - existingKeysIter := existingKeySet.Keys(ctx) - clientKeysIter := clientKeySet.Keys(ctx) - matchFound := false - - for existingKeysIter.Next(ctx) { - existingKey := existingKeysIter.Pair().Value.(jwk.Key) - - existingKid, ok := existingKey.Get("kid") - if !ok { - log.Warnf("Skipping registry db existing key without 'kid'") - continue - } - - for clientKeysIter.Next(ctx) { - clientKey := clientKeysIter.Pair().Value.(jwk.Key) - - clientKid, ok := clientKey.Get("kid") - if !ok { - log.Warnf("Skipping client key without 'kid'") - continue - } - - if existingKid == clientKid { - // Verify the Proof of Possession of client's previous active private key - // Get client's previous public key recorded in db - var prevRawkey interface{} - if err := existingKey.Raw(&prevRawkey); err != nil { - return false, nil, errors.Wrap(err, "failed to generate raw pubkey from client's previous pubkey") - } - // Get client's previous signature from payload - var prevKeyVerified bool - if data.ClientPrevSignature == "" { - prevKeyVerified = true - log.Debugf("This is the initial namespace registration. Client's previous signature is an empty string") - } else { - clientPrevSignature, err := hex.DecodeString(data.ClientPrevSignature) - if err != nil { - return false, nil, errors.Wrap(err, "Failed to decode the client's previous signature") - } - prevKeyVerified = verifySignature(clientPayload, clientPrevSignature, (prevRawkey).(*ecdsa.PublicKey)) - } - - if prevKeyVerified { - matchFound = true - break - } else { - log.Debugf("Client cannot prove that it possesses the key it claims, key id: %s", existingKid) - } - } - } - - if matchFound { - break - } - } - - if !matchFound { - return false, nil, permissionDeniedError{ - Message: fmt.Sprintf("The client that tries to prefix '%s' cannot be authorized: either it doesn't contain any public key matching the existing namespace's public key in db, or it fails to pass the proof of possession verification", data.Prefix), - } - } - - log.Debugf("New public key %s is going to replace the old one: %s", string(data.Pubkey), existingNs.Pubkey) - - // Process origin's new public key - pubkeyData, err := json.Marshal(data.Pubkey) - if err != nil { - return false, nil, errors.Wrapf(err, "Failed to convert public key from json to string format for the prefix %s", data.Prefix) - } - pubkeyDbString := string(pubkeyData) - - // Perform the update action when origin provides a new key - if pubkeyDbString != existingNs.Pubkey { - err = updateNamespacePubKey(data.Prefix, pubkeyDbString) - if err != nil { - log.Errorf("Failed to update the public key of namespace %s: %v", data.Prefix, err) - return false, nil, errors.Wrap(err, "Server encountered an error updating the public key of an existing namespace") - } - // Skip the rest steps in namesapce initial registration - returnMsg := map[string]interface{}{ - "message": fmt.Sprintf("Updated the public key of namespace %s:", data.Prefix), - } - log.Infof("Updated the public key of namespace %s:", data.Prefix) - return false, returnMsg, nil - } else { - returnMsg := map[string]interface{}{ - "message": fmt.Sprintf("The prefix %s is already registered -- nothing else to do!", data.Prefix), - } - log.Infof("Skipping registration of prefix %s because it's already registered.", data.Prefix) - return false, returnMsg, nil + returnMsg := map[string]interface{}{ + "message": fmt.Sprintf("The prefix %s is already registered -- nothing else to do!", data.Prefix), } + log.Infof("Skipping registration of prefix %s because it's already registered.", data.Prefix) + return false, returnMsg, nil } reqPrefix, err := validatePrefix(data.Prefix) From 0ea7e4f844bc3c4203f3bfa89bd749c55824fba1 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Tue, 17 Dec 2024 15:05:10 +0000 Subject: [PATCH 28/64] fix linter issues --- config/init_server_creds.go | 14 +++++++------- launcher_utils/register_namespace_test.go | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/config/init_server_creds.go b/config/init_server_creds.go index fb0f97704..cdab29bbd 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -717,8 +717,7 @@ func loadPEMFiles(dir string) (jwk.Key, error) { return mostRecentKey, nil } -// Create a new .pem file and add its key to issuerPrivateKeys -// In other words, it combines GeneratePrivateKey and LoadPrivateKey functions +// Create a new .pem file (combining GeneratePrivateKey and LoadPrivateKey functions) func GeneratePEM(dir string) (jwk.Key, error) { filename := fmt.Sprintf("pelican_generated_%d_%s.pem", time.Now().UnixNano(), @@ -734,11 +733,6 @@ func GeneratePEM(dir string) (jwk.Key, error) { log.Warnf("Failed to load key %s: %v", keyPath, err) } - // Save this new key in the in-memory map for all private keys - keysCopy := getIssuerPrivateKeysCopy() - keysCopy[key.KeyID()] = key - issuerPrivateKeys.Store(&keysCopy) - log.Debugf("Generated private key %s", key.KeyID()) return key, nil } @@ -750,7 +744,13 @@ func GeneratePEMandSetActiveKey(dir string) (jwk.Key, error) { if err != nil { return nil, errors.Wrapf(err, "failed to create a new .pem file to save private key") } + + // Save this new key in the in-memory map for all private keys + keysCopy := getIssuerPrivateKeysCopy() + keysCopy[newKey.KeyID()] = newKey + issuerPrivateKeys.Store(&keysCopy) SetActiveKey(newKey) + return newKey, nil } diff --git a/launcher_utils/register_namespace_test.go b/launcher_utils/register_namespace_test.go index e60aaa931..19aead30d 100644 --- a/launcher_utils/register_namespace_test.go +++ b/launcher_utils/register_namespace_test.go @@ -168,7 +168,7 @@ func TestRegistration(t *testing.T) { // Redo the namespace prep, ensure that isRegistered is true prefix = param.Origin_FederationPrefix.GetString() - key, registerURL, isRegistered, err = registerNamespacePrep(ctx, prefix) + _, registerURL, isRegistered, err = registerNamespacePrep(ctx, prefix) assert.True(t, isRegistered) assert.Equal(t, svr.URL+"/api/v1.0/registry", registerURL) assert.NoError(t, err) @@ -315,7 +315,7 @@ func TestMultiKeysRegistration(t *testing.T) { // Redo the namespace prep, ensure that isRegistered is true prefix = param.Origin_FederationPrefix.GetString() - key, registerURL, isRegistered, err = registerNamespacePrep(ctx, prefix) + _, registerURL, isRegistered, err = registerNamespacePrep(ctx, prefix) require.NoError(t, err) assert.True(t, isRegistered) assert.Equal(t, svr.URL+"/api/v1.0/registry", registerURL) From bc07e32e0f7bdf9f621ea20d75e4fdcd6d0f0805 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Wed, 18 Dec 2024 05:38:44 +0000 Subject: [PATCH 29/64] add unit tests to ensure public key update failures occur as expected --- launcher_utils/register_namespace_test.go | 19 ++------- registry/client_commands_test.go | 48 +++++++++++++++++++++-- 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/launcher_utils/register_namespace_test.go b/launcher_utils/register_namespace_test.go index 19aead30d..39ae51701 100644 --- a/launcher_utils/register_namespace_test.go +++ b/launcher_utils/register_namespace_test.go @@ -27,7 +27,6 @@ import ( "io" "net/http" "net/http/httptest" - "os" "path/filepath" "testing" @@ -51,12 +50,7 @@ func TestRegistration(t *testing.T) { config.ResetIssuerJWKPtr() config.ResetIssuerPrivateKeys() }) - // Use a temp os directory to better control the deletion of the directory. - // Fixes issue on Windows where we are trying to delete a file in use so this - // better waits for the file/process to be shut down before deletion - tempConfigDir, err := os.MkdirTemp("", "test") - assert.NoError(t, err) - defer os.RemoveAll(tempConfigDir) + tempConfigDir := t.TempDir() ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) defer func() { require.NoError(t, egrp.Wait()) }() @@ -69,7 +63,7 @@ func TestRegistration(t *testing.T) { config.InitConfig() viper.Set("Registry.DbLocation", filepath.Join(tempConfigDir, "test.sql")) - err = config.InitServer(ctx, server_structs.OriginType) + err := config.InitServer(ctx, server_structs.OriginType) require.NoError(t, err) err = registry.InitializeDB() @@ -180,12 +174,7 @@ func TestMultiKeysRegistration(t *testing.T) { config.ResetIssuerJWKPtr() config.ResetIssuerPrivateKeys() }) - // Use a temp os directory to better control the deletion of the directory. - // Fixes issue on Windows where we are trying to delete a file in use so this - // better waits for the file/process to be shut down before deletion - tempConfigDir, err := os.MkdirTemp("", "test") - assert.NoError(t, err) - defer os.RemoveAll(tempConfigDir) + tempConfigDir := t.TempDir() ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) defer func() { require.NoError(t, egrp.Wait()) }() @@ -198,7 +187,7 @@ func TestMultiKeysRegistration(t *testing.T) { config.InitConfig() viper.Set("Registry.DbLocation", filepath.Join(tempConfigDir, "test.sql")) - err = config.InitServer(ctx, server_structs.OriginType) + err := config.InitServer(ctx, server_structs.OriginType) require.NoError(t, err) err = registry.InitializeDB() diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index e9d96fa95..75ab3acde 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -29,11 +29,13 @@ import ( "testing" "github.com/gin-gonic/gin" + "github.com/lestrrat-go/jwx/v2/jwk" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/pelicanplatform/pelican/config" + "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/server_structs" "github.com/pelicanplatform/pelican/server_utils" "github.com/pelicanplatform/pelican/test_utils" @@ -43,8 +45,8 @@ func registryMockup(ctx context.Context, t *testing.T, testName string) *httptes issuerTempDir := filepath.Join(t.TempDir(), testName) - ikey := filepath.Join(issuerTempDir, "issuer.jwk") - viper.Set("IssuerKey", ikey) + ikey := filepath.Join(issuerTempDir, "issuer-keys") + viper.Set("IssuerKeysDirectory", ikey) viper.Set("Registry.DbLocation", filepath.Join(issuerTempDir, "test.sql")) viper.Set("Server.WebPort", 8444) err := config.InitConfigDir(viper.GetViper()) @@ -90,6 +92,7 @@ func TestServeNamespaceRegistry(t *testing.T) { //Test functionality of registering a namespace (without identity) err = NamespaceRegister(privKey, svr.URL+"/api/v1.0/registry", "", "/foo/bar", "mock_site_name") require.NoError(t, err) + var privKey2 jwk.Key //Test we can list the namespace without an error t.Run("Test namespace list", func(t *testing.T) { @@ -123,6 +126,43 @@ func TestServeNamespaceRegistry(t *testing.T) { assert.Equal(t, ns.AdminMetadata.SiteName, "mock_site_name") }) + t.Run("test-registered-namespace-pubkey-update-with-new-active-key", func(t *testing.T) { + activeKey, err := config.GetIssuerPrivateJWK() + require.NoError(t, err) + require.Equal(t, privKey.KeyID(), activeKey.KeyID()) + + // Imitate LaunchIssuerKeysDirRefresh function + config.UpdatePreviousIssuerPrivateJWK() + config.GeneratePEM(param.IssuerKeysDirectory.GetString()) + privKey2, err = config.LoadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()) + require.NoError(t, err) + + err = NamespacesPubKeyUpdate(privKey2, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") + require.NoError(t, err) + }) + + t.Run("test-registered-namespace-pubkey-update-with-nonsense-key", func(t *testing.T) { + tempDir := filepath.Join(t.TempDir(), "in_the_middle_of_nowhere") + privKey3, err := config.GeneratePEM(tempDir) + require.NoError(t, err) + err = NamespacesPubKeyUpdate(privKey3, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") + require.ErrorContains(t, err, "it doesn't contain any public key matching the existing namespace's public key in db") + }) + + t.Run("test-registered-namespace-pubkey-update-with-imposter-key", func(t *testing.T) { + privKey4, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) + require.NoError(t, err) + config.UpdatePreviousIssuerPrivateJWK() + // Both active key and previous key are set to privKey4 + err = NamespacesPubKeyUpdate(privKey4, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") + require.ErrorContains(t, err, "it fails to pass the proof of possession verification") + + // Revert the active key changes happened in this subtest + config.SetActiveKey(privKey) + config.UpdatePreviousIssuerPrivateJWK() + config.SetActiveKey(privKey2) + }) + t.Run("Test namespace delete", func(t *testing.T) { //Test functionality of namespace delete err = NamespaceDelete(svr.URL+"/api/v1.0/registry/foo/bar", "/foo/bar") @@ -201,7 +241,7 @@ func TestRegistryKeyChainingOSDF(t *testing.T) { assert.Contains(t, err.Error(), "A superspace or subspace of this namespace /topo/foo already exists in the OSDF topology: /topo/foo. To register a Pelican equivalence, you need to present your identity.") // Now we create a new key and try to use it to register a super/sub space. These shouldn't succeed - viper.Set("IssuerKey", t.TempDir()+"/keychaining") + viper.Set("IssuerKeysDirectory", t.TempDir()+"/keychaining") viper.Set("ConfigDir", t.TempDir()) config.InitConfig() err = config.InitServer(ctx, server_structs.RegistryType) @@ -273,7 +313,7 @@ func TestRegistryKeyChaining(t *testing.T) { require.NoError(t, err) // Now we create a new key and try to use it to register a super/sub space. These shouldn't succeed - viper.Set("IssuerKey", t.TempDir()+"/keychaining") + viper.Set("IssuerKeysDirectory", t.TempDir()+"/keychaining") viper.Set("ConfigDir", t.TempDir()) config.InitConfig() err = config.InitServer(ctx, server_structs.RegistryType) From e3ca516bdf549954bbe30144a72023b8b1ed87c5 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Wed, 18 Dec 2024 18:26:15 +0000 Subject: [PATCH 30/64] fix issues in follow-up reviews --- docs/parameters.yaml | 6 ++-- launcher_utils/register_namespace_test.go | 19 +++++++--- launcher_utils/update_namespace_pubkey.go | 32 ++++++++++++++--- origin/origin_ui.go | 18 ++++------ origin/origin_ui_test.go | 44 +++++++++++++++++------ registry/client_commands.go | 16 ++++----- registry/client_commands_test.go | 4 +-- registry/registry_pubkey_update.go | 39 +++++++++++++++----- 8 files changed, 125 insertions(+), 53 deletions(-) diff --git a/docs/parameters.yaml b/docs/parameters.yaml index efa0023cb..c0434c448 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -64,13 +64,13 @@ components: ["origin", "registry", "director"] --- name: IssuerKey description: |+ - [Deprecated] A filepath to the file containing a PEM-encoded ecdsa private key which later will be parsed + [Deprecated] Use IssuerKeysDirectory instead. The key file set by this parameter will automatically move to IssuerKeysDirectory. + + A filepath to the file containing a PEM-encoded ecdsa private key which later will be parsed into a JWK and serves as the private key to sign various JWTs issued by this server. A public JWK will be derived from this private key and used as the key for token verification. type: filename -deprecated: true -replacedby: "IssuerKeysDirectory" root_default: /etc/pelican/issuer.jwk default: $ConfigBase/issuer.jwk components: ["origin", "cache", "registry", "director"] diff --git a/launcher_utils/register_namespace_test.go b/launcher_utils/register_namespace_test.go index 39ae51701..19aead30d 100644 --- a/launcher_utils/register_namespace_test.go +++ b/launcher_utils/register_namespace_test.go @@ -27,6 +27,7 @@ import ( "io" "net/http" "net/http/httptest" + "os" "path/filepath" "testing" @@ -50,7 +51,12 @@ func TestRegistration(t *testing.T) { config.ResetIssuerJWKPtr() config.ResetIssuerPrivateKeys() }) - tempConfigDir := t.TempDir() + // Use a temp os directory to better control the deletion of the directory. + // Fixes issue on Windows where we are trying to delete a file in use so this + // better waits for the file/process to be shut down before deletion + tempConfigDir, err := os.MkdirTemp("", "test") + assert.NoError(t, err) + defer os.RemoveAll(tempConfigDir) ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) defer func() { require.NoError(t, egrp.Wait()) }() @@ -63,7 +69,7 @@ func TestRegistration(t *testing.T) { config.InitConfig() viper.Set("Registry.DbLocation", filepath.Join(tempConfigDir, "test.sql")) - err := config.InitServer(ctx, server_structs.OriginType) + err = config.InitServer(ctx, server_structs.OriginType) require.NoError(t, err) err = registry.InitializeDB() @@ -174,7 +180,12 @@ func TestMultiKeysRegistration(t *testing.T) { config.ResetIssuerJWKPtr() config.ResetIssuerPrivateKeys() }) - tempConfigDir := t.TempDir() + // Use a temp os directory to better control the deletion of the directory. + // Fixes issue on Windows where we are trying to delete a file in use so this + // better waits for the file/process to be shut down before deletion + tempConfigDir, err := os.MkdirTemp("", "test") + assert.NoError(t, err) + defer os.RemoveAll(tempConfigDir) ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) defer func() { require.NoError(t, egrp.Wait()) }() @@ -187,7 +198,7 @@ func TestMultiKeysRegistration(t *testing.T) { config.InitConfig() viper.Set("Registry.DbLocation", filepath.Join(tempConfigDir, "test.sql")) - err := config.InitServer(ctx, server_structs.OriginType) + err = config.InitServer(ctx, server_structs.OriginType) require.NoError(t, err) err = registry.InitializeDB() diff --git a/launcher_utils/update_namespace_pubkey.go b/launcher_utils/update_namespace_pubkey.go index 5d6570c81..1f8ff67b9 100644 --- a/launcher_utils/update_namespace_pubkey.go +++ b/launcher_utils/update_namespace_pubkey.go @@ -1,3 +1,21 @@ +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + package launcher_utils import ( @@ -85,14 +103,20 @@ func LaunchIssuerKeysDirRefresh(ctx context.Context, egrp *errgroup.Group) { case <-ticker.C: // Refresh the disk to pick up any new private key config.UpdatePreviousIssuerPrivateJWK() - key, err := config.LoadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()) + newKey, err := config.LoadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()) if err != nil { return err } - log.Debugln("Private keys directory refreshed successfully. The active (latest) private key is", key.KeyID()) - log.Debugln("Previous private key is", config.GetPreviousIssuerPrivateJWK().KeyID()) + prevKeyID := config.GetPreviousIssuerPrivateJWK().KeyID() + newKeyID := newKey.KeyID() - // Update public key in registry db with the new active private key + log.Debugf("Private keys directory refreshed successfully. Previous key: %s, New key: %s", prevKeyID, newKeyID) + if newKeyID == prevKeyID { + // Skip this iteration since the active key has not changed + log.Debugf("Active private key has not changed. Skipping update of namespaces' public keys in the registry database.") + continue + } + // Update public key in registry db when there new active private key extUrlStr := param.Server_ExternalWebUrl.GetString() extUrl, _ := url.Parse(extUrlStr) namespace := server_structs.GetOriginNs(extUrl.Host) diff --git a/origin/origin_ui.go b/origin/origin_ui.go index 15d789dd4..98eeaf863 100644 --- a/origin/origin_ui.go +++ b/origin/origin_ui.go @@ -24,7 +24,6 @@ import ( "time" "github.com/gin-gonic/gin" - "github.com/lestrrat-go/jwx/v2/jwk" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -162,13 +161,15 @@ func handleExports(ctx *gin.Context) { ctx.JSON(http.StatusOK, res) } +var GeneratePEM = config.GeneratePEM + // Create a new private key in the given directory, but without loading it immediately // This new key will be detected and loaded later by the ongoing goroutine `LaunchIssuerKeysDirRefresh`, // which periodically refreshes the keys in the issuer keys directory -func createNewIssuerKey(ctx *gin.Context, generatePEM func(string) (jwk.Key, error)) { +func createNewIssuerKey(ctx *gin.Context) { issuerKeysDir := param.IssuerKeysDirectory.GetString() - _, err := generatePEM(issuerKeysDir) + _, err := GeneratePEM(issuerKeysDir) if err != nil { log.Errorf("Error creating a new private key in a new .pem file: %v", err) ctx.JSON(http.StatusInternalServerError, server_structs.SimpleApiResp{ @@ -180,7 +181,7 @@ func createNewIssuerKey(ctx *gin.Context, generatePEM func(string) (jwk.Key, err ctx.JSON(http.StatusOK, server_structs.SimpleApiResp{ Status: server_structs.RespOK, - Msg: "Created a new issuer key and set it as the active private key", + Msg: "Created a new issuer key", }) } @@ -188,14 +189,7 @@ func RegisterOriginWebAPI(engine *gin.Engine) error { originWebAPI := engine.Group("/api/v1.0/origin_ui") { originWebAPI.GET("/exports", web_ui.AuthHandler, web_ui.AdminAuthHandler, handleExports) - - // For unit test: GeneratePEM will be replaced by a mockup func - defaultGeneratePEM := func(directory string) (jwk.Key, error) { - return config.GeneratePEM(directory) - } - originWebAPI.GET("/newIssuerKey", web_ui.AuthHandler, web_ui.AdminAuthHandler, func(ctx *gin.Context) { - createNewIssuerKey(ctx, defaultGeneratePEM) - }) + originWebAPI.GET("/newIssuerKey", web_ui.AuthHandler, web_ui.AdminAuthHandler, createNewIssuerKey) } // Globus backend specific. Config other origin routes above this line diff --git a/origin/origin_ui_test.go b/origin/origin_ui_test.go index b8287b71c..409a4903e 100644 --- a/origin/origin_ui_test.go +++ b/origin/origin_ui_test.go @@ -1,9 +1,28 @@ +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + package origin import ( "encoding/json" "net/http" "net/http/httptest" + "os" "path/filepath" "testing" @@ -22,15 +41,13 @@ func TestSuccessfulCreateNewIssuerKey(t *testing.T) { server_utils.ResetTestState() tDir := t.TempDir() - viper.Set("IssuerKeysDirectory", filepath.Join(tDir, "test-issuer-keys")) + iksDir := filepath.Join(tDir, "test-issuer-keys") + viper.Set("IssuerKeysDirectory", iksDir) viper.Set("ConfigDir", t.TempDir()) config.InitConfig() router := gin.Default() - router.GET("/api/v1.0/origin_ui/newIssuerKey", func(ctx *gin.Context) { - createNewIssuerKey(ctx, config.GeneratePEM) - }) - + router.GET("/api/v1.0/origin_ui/newIssuerKey", createNewIssuerKey) w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/api/v1.0/origin_ui/newIssuerKey", nil) router.ServeHTTP(w, req) @@ -41,8 +58,13 @@ func TestSuccessfulCreateNewIssuerKey(t *testing.T) { err := json.Unmarshal(w.Body.Bytes(), &response) require.NoError(t, err) assert.Equal(t, server_structs.RespOK, response.Status) - assert.Equal(t, "Created a new issuer key and set it as the active private key", response.Msg) + assert.Equal(t, "Created a new issuer key", response.Msg) + files, err := os.ReadDir(iksDir) + require.NoError(t, err) + require.Len(t, files, 1, "A new .pem file should be created in the directory") + newKeyFile := filepath.Join(iksDir, files[0].Name()) + assert.FileExists(t, newKeyFile, "The new .pem file does not exist") } func TestFailedCreateNewIssuerKey(t *testing.T) { @@ -54,12 +76,14 @@ func TestFailedCreateNewIssuerKey(t *testing.T) { config.InitConfig() router := gin.Default() - mockErrGeneratePEM := func(directory string) (jwk.Key, error) { + router.GET("/api/v1.0/origin_ui/newIssuerKey", createNewIssuerKey) + + // Mock GeneratePEM + originalGeneratePEM := GeneratePEM + GeneratePEM = func(dir string) (jwk.Key, error) { return nil, assert.AnError } - router.GET("/api/v1.0/origin_ui/newIssuerKey", func(ctx *gin.Context) { - createNewIssuerKey(ctx, mockErrGeneratePEM) - }) + defer func() { GeneratePEM = originalGeneratePEM }() w := httptest.NewRecorder() req, _ := http.NewRequest("GET", "/api/v1.0/origin_ui/newIssuerKey", nil) diff --git a/registry/client_commands.go b/registry/client_commands.go index 79a1175b1..ec9c07059 100644 --- a/registry/client_commands.go +++ b/registry/client_commands.go @@ -300,14 +300,12 @@ func NamespacesPubKeyUpdate(privateKey jwk.Key, prefixes []string, siteName stri return errors.Wrap(err, "failed to add public key to new JWKS") } - if log.IsLevelEnabled(log.DebugLevel) { - // Let's check that we can convert to JSON and get the right thing... - jsonbuf, err := json.Marshal(keySet) - if err != nil { - return errors.Wrap(err, "failed to marshal the public key into JWKS JSON") - } - log.Debugln("Constructed JWKS from loading public key:", string(jsonbuf)) + // Let's check that we can convert to JSON and get the right thing... + jsonbuf, err := json.Marshal(keySet) + if err != nil { + return errors.Wrap(err, "failed to marshal the public key into JWKS JSON") } + log.Debugln("Constructed JWKS from loading public key:", string(jsonbuf)) clientNonce, err := generateNonce() if err != nil { @@ -337,7 +335,7 @@ func NamespacesPubKeyUpdate(privateKey jwk.Key, prefixes []string, siteName stri // No error if err = json.Unmarshal(resp, &respData); err != nil { - return errors.Wrapf(err, "Failure when parsing JSON response from the server with a success request. Raw server response is %s", resp) + return errors.Wrapf(err, "Failure when parsing JSON response from the server with a successful request. Raw server response is %s", resp) } // Create client payload by concatenating client_nonce and server_nonce @@ -423,7 +421,7 @@ func NamespacesPubKeyUpdate(privateKey jwk.Key, prefixes []string, siteName stri if err != nil { return errors.Wrapf(err, "Server responded with an error and failed to parse JSON response from the server. Raw response is %s", resp) } - return errors.Wrapf(unmarshalErr, "Failure when parsing JSON response from the server with a success request. Raw server response is %s", resp) + return errors.Wrapf(unmarshalErr, "Failure when parsing JSON response from the server with a successful request. Raw server response is %s", resp) } return nil diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index 75ab3acde..6f5711a82 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -133,10 +133,10 @@ func TestServeNamespaceRegistry(t *testing.T) { // Imitate LaunchIssuerKeysDirRefresh function config.UpdatePreviousIssuerPrivateJWK() - config.GeneratePEM(param.IssuerKeysDirectory.GetString()) + _, err = config.GeneratePEM(param.IssuerKeysDirectory.GetString()) + require.NoError(t, err) privKey2, err = config.LoadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()) require.NoError(t, err) - err = NamespacesPubKeyUpdate(privKey2, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") require.NoError(t, err) }) diff --git a/registry/registry_pubkey_update.go b/registry/registry_pubkey_update.go index 83d0f05ac..2a8e1e2ef 100644 --- a/registry/registry_pubkey_update.go +++ b/registry/registry_pubkey_update.go @@ -1,3 +1,21 @@ +/*************************************************************** + * + * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ***************************************************************/ + package registry import ( @@ -133,6 +151,16 @@ func updateNsKeySignChallengeCommit(ctx *gin.Context, data *RegisteredPrefixUpda return false, nil, errors.Wrap(err, "Invalid existing namespace public key format") } + // Get client's previous signature from payload + prevKeyVerified := false + if data.ClientPrevSignature == "" { + prevKeyVerified = true + } + clientPrevSignature, err := hex.DecodeString(data.ClientPrevSignature) + if err != nil { + return false, nil, errors.Wrap(err, "Failed to decode the client's previous signature") + } + // Check if any key in `clientKeySet` matches a key in `existingKeySet` existingKeysIter := existingKeySet.Keys(ctx) clientKeysIter := clientKeySet.Keys(ctx) @@ -163,15 +191,8 @@ func updateNsKeySignChallengeCommit(ctx *gin.Context, data *RegisteredPrefixUpda if err := existingKey.Raw(&prevRawkey); err != nil { return false, nil, errors.Wrap(err, "failed to generate raw pubkey from client's previous pubkey") } - // Get client's previous signature from payload - var prevKeyVerified bool - if data.ClientPrevSignature == "" { - prevKeyVerified = true - } else { - clientPrevSignature, err := hex.DecodeString(data.ClientPrevSignature) - if err != nil { - return false, nil, errors.Wrap(err, "Failed to decode the client's previous signature") - } + + if data.ClientPrevSignature != "" { prevKeyVerified = verifySignature(clientPayload, clientPrevSignature, (prevRawkey).(*ecdsa.PublicKey)) } From e27231e022a53188e5809fac0ec697d70e31810b Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Wed, 18 Dec 2024 18:52:58 +0000 Subject: [PATCH 31/64] fix linter and go build tests --- docs/parameters.yaml | 2 +- param/parameters.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/parameters.yaml b/docs/parameters.yaml index c0434c448..396094527 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -65,7 +65,7 @@ components: ["origin", "registry", "director"] name: IssuerKey description: |+ [Deprecated] Use IssuerKeysDirectory instead. The key file set by this parameter will automatically move to IssuerKeysDirectory. - + A filepath to the file containing a PEM-encoded ecdsa private key which later will be parsed into a JWK and serves as the private key to sign various JWTs issued by this server. diff --git a/param/parameters.go b/param/parameters.go index 98e5a490d..9b7039265 100644 --- a/param/parameters.go +++ b/param/parameters.go @@ -56,7 +56,6 @@ func GetDeprecated() map[string][]string { "Director.EnableStat": {"Director.CheckOriginPresence"}, "DisableHttpProxy": {"Client.DisableHttpProxy"}, "DisableProxyFallback": {"Client.DisableProxyFallback"}, - "IssuerKey": {"IssuerKeysDirectory"}, "MinimumDownloadSpeed": {"Client.MinimumDownloadSpeed"}, "Origin.EnableDirListing": {"Origin.EnableListings"}, "Origin.EnableFallbackRead": {"Origin.EnableDirectReads"}, From 6ccb591830dd2eb724f44d7df3b7190ed467d2c8 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Thu, 19 Dec 2024 03:39:46 +0000 Subject: [PATCH 32/64] on registry, only rotate out the one previous key, then add the new key, while keep other public keys of this namespace in the db intact --- docs/parameters.yaml | 1 + registry/client_commands.go | 9 +- registry/client_commands_test.go | 128 ++++++++++++++++++++++++++++- registry/registry_pubkey_update.go | 53 ++++++++++-- 4 files changed, 179 insertions(+), 12 deletions(-) diff --git a/docs/parameters.yaml b/docs/parameters.yaml index 396094527..ac4b26786 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -71,6 +71,7 @@ description: |+ A public JWK will be derived from this private key and used as the key for token verification. type: filename +deprecated: true root_default: /etc/pelican/issuer.jwk default: $ConfigBase/issuer.jwk components: ["origin", "cache", "registry", "director"] diff --git a/registry/client_commands.go b/registry/client_commands.go index ec9c07059..00f359f45 100644 --- a/registry/client_commands.go +++ b/registry/client_commands.go @@ -365,7 +365,13 @@ func NamespacesPubKeyUpdate(privateKey jwk.Key, prefixes []string, siteName stri } } - // Send origins all public keys in another key set + // Send origins previous public key and all public keys in another key set + prevKey := config.GetPreviousIssuerPrivateJWK() + prevKeySet := jwk.NewSet() + if err = prevKeySet.AddKey(prevKey); err != nil { + return errors.Wrap(err, "failed to add previous public key to new JWKS") + } + privateKeys := config.GetIssuerPrivateKeys() if len(privateKeys) == 0 { return errors.Wrap(err, "The server doesn't have any private key in memory") @@ -384,6 +390,7 @@ func NamespacesPubKeyUpdate(privateKey jwk.Key, prefixes []string, siteName stri // Create data for the second POST request unidentifiedPayload := map[string]interface{}{ "pubkey": keySet, + "prev_pubkey": prevKeySet, "all_pubkeys": allKeysSet, "prefixes": prefixes, "site_name": siteName, diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index 6f5711a82..4712509b0 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -26,6 +26,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "sort" "testing" "github.com/gin-gonic/gin" @@ -42,7 +43,6 @@ import ( ) func registryMockup(ctx context.Context, t *testing.T, testName string) *httptest.Server { - issuerTempDir := filepath.Join(t.TempDir(), testName) ikey := filepath.Join(issuerTempDir, "issuer-keys") @@ -69,11 +69,37 @@ func registryMockup(ctx context.Context, t *testing.T, testName string) *httptes return svr } +func getSortedKids(jsonStr string) ([]string, error) { + set, err := jwk.Parse([]byte(jsonStr)) + if err != nil { + return nil, err + } + ctx := context.Background() + var kids []string + keysIter := set.Keys(ctx) + for keysIter.Next(ctx) { + key := keysIter.Pair().Value.(jwk.Key) + + kid, ok := key.Get("kid") + if !ok { + continue + } + kids = append(kids, kid.(string)) + } + sort.Strings(kids) + + return kids, nil +} + func TestServeNamespaceRegistry(t *testing.T) { ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) - defer func() { require.NoError(t, egrp.Wait()) }() - defer cancel() - + t.Cleanup(func() { + func() { require.NoError(t, egrp.Wait()) }() + cancel() + config.ResetIssuerJWKPtr() + config.ResetIssuerPrivateKeys() + server_utils.ResetTestState() + }) server_utils.ResetTestState() svr := registryMockup(ctx, t, "serveregistry") @@ -184,6 +210,100 @@ func TestServeNamespaceRegistry(t *testing.T) { server_utils.ResetTestState() } +func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { + ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) + server_utils.ResetTestState() + t.Cleanup(func() { + func() { require.NoError(t, egrp.Wait()) }() + cancel() + config.ResetIssuerJWKPtr() + config.ResetIssuerPrivateKeys() + server_utils.ResetTestState() + }) + + tDir := t.TempDir() + + svr := registryMockup(ctx, t, "serveregistry") + defer func() { + err := ShutdownRegistryDB() + assert.NoError(t, err) + svr.CloseClientConnections() + svr.Close() + }() + + config.ResetIssuerJWKPtr() + config.ResetIssuerPrivateKeys() + privKeys := config.GetIssuerPrivateKeys() + require.Len(t, privKeys, 0) + + // Construct a client that has [p1,p2,p3] and p3 is the active private key + privKey1, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) + require.NotEmpty(t, privKey1) + require.NoError(t, err) + privKey2, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) + require.NoError(t, err) + + prefix := "/mascot/bucky" + err = NamespaceRegister(privKey2, svr.URL+"/api/v1.0/registry", "", prefix, "mock_site_name") + require.NoError(t, err) + + config.UpdatePreviousIssuerPrivateJWK() + privKey3, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) + require.NoError(t, err) + + // Construct a public keys JWKS [p2,p4] to save in registry DB, imitating admin manually adding p4 + registryDbJwks := jwk.NewSet() + pubKey2, err := jwk.PublicKeyOf(privKey2) + require.NoError(t, err) + err = registryDbJwks.AddKey(pubKey2) + require.NoError(t, err) + privKey4, err := config.GeneratePEM(filepath.Join(tDir, "elsewhere")) + require.NoError(t, err) + pubKey4, err := jwk.PublicKeyOf(privKey4) + require.NoError(t, err) + err = registryDbJwks.AddKey(pubKey4) + require.NoError(t, err) + jwksBytes, err := json.Marshal(registryDbJwks) + require.NoError(t, err) + jwksStr := string(jwksBytes) + + // Test functionality of a namespace registered with multi public keys [p2,p4] + err = updateNamespacePubKey(prefix, jwksStr) // set the registered public keys to [p2,p4] + require.NoError(t, err) + ns, err := getNamespaceByPrefix(prefix) + require.NoError(t, err) + require.Equal(t, jwksStr, ns.Pubkey) + + prevKey := config.GetPreviousIssuerPrivateJWK() + require.Equal(t, privKey2.KeyID(), prevKey.KeyID()) + privKeys = config.GetIssuerPrivateKeys() + require.Len(t, privKeys, 3) + + // Client allKeys:[p1,p2,p3] prevKey:p2 activeKey:p3 ---UPDATE--> Registry [p2,p4] + // => should update Registry to [p3,p4] (rotate out prevKey:p2, rotate in activeKey:p3) + err = NamespacesPubKeyUpdate(privKey3, []string{prefix}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") + require.NoError(t, err) + ns, err = getNamespaceByPrefix(prefix) + require.NoError(t, err) + + expectedJwks := jwk.NewSet() + pubKey3, err := jwk.PublicKeyOf(privKey3) + require.NoError(t, err) + err = expectedJwks.AddKey(pubKey3) + require.NoError(t, err) + err = expectedJwks.AddKey(pubKey4) + require.NoError(t, err) + expectedJwksBytes, err := json.Marshal(expectedJwks) + require.NoError(t, err) + expectedJwksStr := string(expectedJwksBytes) + + expectedKids, err := getSortedKids(expectedJwksStr) + require.NoError(t, err) + actualKids, err := getSortedKids(ns.Pubkey) + require.NoError(t, err) + require.Equal(t, expectedKids, actualKids) +} + func TestRegistryKeyChainingOSDF(t *testing.T) { ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) defer func() { require.NoError(t, egrp.Wait()) }() diff --git a/registry/registry_pubkey_update.go b/registry/registry_pubkey_update.go index 2a8e1e2ef..27f5b390f 100644 --- a/registry/registry_pubkey_update.go +++ b/registry/registry_pubkey_update.go @@ -44,10 +44,38 @@ type RegisteredPrefixUpdate struct { ServerSignature string `json:"server_signature"` Pubkey json.RawMessage `json:"pubkey"` + PrevPubkey json.RawMessage `json:"prev_pubkey"` AllPubkeys json.RawMessage `json:"all_pubkeys"` Prefixes []string `json:"prefixes"` } +func getKeyByKid(keySet jwk.Set, kidToFind string) (jwk.Key, error) { + for i := 0; i < keySet.Len(); i++ { + key, _ := keySet.Key(i) + kid, ok := key.Get("kid") + if ok && kid == kidToFind { + return key, nil + } + } + return nil, fmt.Errorf("key with kid %s not found in the set", kidToFind) +} + +func constructUpdatedKeySet(keySet jwk.Set, kidToRemove string, keyToAdd jwk.Key) error { + keyToRemove, err := getKeyByKid(keySet, kidToRemove) + if err != nil { + log.Warnf("Key %s not found in the key set, skipping removal: %v", kidToRemove, err) + } else { + // Remove the key only if it exists + if err := keySet.RemoveKey(keyToRemove); err != nil { + return errors.Wrap(err, "failed to remove key from set") + } + } + if err := keySet.AddKey(keyToAdd); err != nil { + return errors.Wrap(err, "failed to add key to the key set") + } + return nil +} + // Generate server nonce for key-sign challenge when updating the public key of registered namespace(s) func updateNsKeySignChallengeInit(data *RegisteredPrefixUpdate) (map[string]interface{}, error) { serverNonce, err := generateNonce() @@ -144,8 +172,8 @@ func updateNsKeySignChallengeCommit(ctx *gin.Context, data *RegisteredPrefixUpda return false, nil, errors.Wrapf(err, "Invalid in-memory public keys format of the client") } // Parse `existingNs.Pubkey` as a JWKS - existingKeySet := jwk.NewSet() - err = json.Unmarshal([]byte(existingNs.Pubkey), &existingKeySet) + registryDbKeySet := jwk.NewSet() + err = json.Unmarshal([]byte(existingNs.Pubkey), ®istryDbKeySet) if err != nil { log.Errorf("Failed to parse existing namespace public key as JWKS: %v", err) return false, nil, errors.Wrap(err, "Invalid existing namespace public key format") @@ -161,8 +189,8 @@ func updateNsKeySignChallengeCommit(ctx *gin.Context, data *RegisteredPrefixUpda return false, nil, errors.Wrap(err, "Failed to decode the client's previous signature") } - // Check if any key in `clientKeySet` matches a key in `existingKeySet` - existingKeysIter := existingKeySet.Keys(ctx) + // Check if any key in `clientKeySet` matches a key in `registryDbKeySet` + existingKeysIter := registryDbKeySet.Keys(ctx) clientKeysIter := clientKeySet.Keys(ctx) matchFound := false @@ -223,11 +251,22 @@ func updateNsKeySignChallengeCommit(ctx *gin.Context, data *RegisteredPrefixUpda if err != nil { return false, nil, errors.Wrapf(err, "Failed to convert public key from json to string format for the prefix %s", prefix) } - pubkeyDbString := string(pubkeyData) + clientPubkeyString := string(pubkeyData) + + // Construct the updated JWKS to be stored in registry db + prevKey, _ := validateJwks(string(data.PrevPubkey)) + err = constructUpdatedKeySet(registryDbKeySet, prevKey.KeyID(), key) + if err != nil { + return false, nil, errors.Wrap(err, "Failed to construct the updated JWKS to be stored in registry db") + } + registryDbKeySetJson, err := json.Marshal(registryDbKeySet) + if err != nil { + return false, nil, errors.Wrap(err, "failed to marshal registryDbKeySet to JSON") + } // Perform the update action when origin provides a new key - if pubkeyDbString != existingNs.Pubkey { - err = updateNamespacePubKey(prefix, pubkeyDbString) + if clientPubkeyString != existingNs.Pubkey { + err = updateNamespacePubKey(prefix, string(registryDbKeySetJson)) if err != nil { log.Errorf("Failed to update the public key of namespace %s: %v", prefix, err) return false, nil, errors.Wrap(err, "Server encountered an error updating the public key of an existing namespace") From c26d95b56dee70d9b8c81c05ef01690ef48d7491 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Thu, 19 Dec 2024 03:50:12 +0000 Subject: [PATCH 33/64] remove IssuerKey deprecated: true to avoid several failed tests --- docs/parameters.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/parameters.yaml b/docs/parameters.yaml index ac4b26786..396094527 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -71,7 +71,6 @@ description: |+ A public JWK will be derived from this private key and used as the key for token verification. type: filename -deprecated: true root_default: /etc/pelican/issuer.jwk default: $ConfigBase/issuer.jwk components: ["origin", "cache", "registry", "director"] From 9496706762fee78594d6b2ff308d78867563e4b0 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Thu, 19 Dec 2024 05:12:22 +0000 Subject: [PATCH 34/64] attempt to fix failed TestRegistryKeyChainingOSDF on GH Action test (1.21.x, macos-latest) --- registry/client_commands_test.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index 1021f63bc..990383eab 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -68,12 +68,11 @@ func registryMockup(ctx context.Context, t *testing.T, testName string) *httptes return svr } -func getSortedKids(jsonStr string) ([]string, error) { +func getSortedKids(ctx context.Context, jsonStr string) ([]string, error) { set, err := jwk.Parse([]byte(jsonStr)) if err != nil { return nil, err } - ctx := context.Background() var kids []string keysIter := set.Keys(ctx) for keysIter.Next(ctx) { @@ -296,17 +295,22 @@ func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { require.NoError(t, err) expectedJwksStr := string(expectedJwksBytes) - expectedKids, err := getSortedKids(expectedJwksStr) + expectedKids, err := getSortedKids(ctx, expectedJwksStr) require.NoError(t, err) - actualKids, err := getSortedKids(ns.Pubkey) + actualKids, err := getSortedKids(ctx, ns.Pubkey) require.NoError(t, err) require.Equal(t, expectedKids, actualKids) } func TestRegistryKeyChainingOSDF(t *testing.T) { ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) - defer func() { require.NoError(t, egrp.Wait()) }() - defer cancel() + t.Cleanup(func() { + func() { require.NoError(t, egrp.Wait()) }() + cancel() + config.ResetIssuerJWKPtr() + config.ResetIssuerPrivateKeys() + server_utils.ResetTestState() + }) server_utils.ResetTestState() _, err := config.SetPreferredPrefix(config.OsdfPrefix) From 1ee5574ae6798435948d161f1beb8a3527ebc447 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Thu, 19 Dec 2024 05:32:37 +0000 Subject: [PATCH 35/64] attempt to fix repeated-cache-access-not-found on GH Action test-ubuntu --- director/director_test.go | 5 +++-- director/origin_api_test.go | 5 +++-- director/stat_test.go | 5 +++-- token/token_create_test.go | 5 +++-- web_ui/prometheus_test.go | 10 +++++----- 5 files changed, 17 insertions(+), 13 deletions(-) diff --git a/director/director_test.go b/director/director_test.go index 31597d9ac..fee5dfdb4 100644 --- a/director/director_test.go +++ b/director/director_test.go @@ -45,6 +45,7 @@ import ( "github.com/stretchr/testify/require" "github.com/pelicanplatform/pelican/config" + "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/server_structs" "github.com/pelicanplatform/pelican/server_utils" "github.com/pelicanplatform/pelican/test_utils" @@ -1272,8 +1273,8 @@ func TestDiscoverOriginCache(t *testing.T) { viper.Set("Server.ExternalWebUrl", mockDirectorUrl) tDir := t.TempDir() - kfile := filepath.Join(tDir, "testKey") - viper.Set("IssuerKey", kfile) + kDir := filepath.Join(tDir, "testKeyDir") + viper.Set(param.IssuerKeysDirectory.GetName(), kDir) viper.Set("ConfigDir", t.TempDir()) config.InitConfig() diff --git a/director/origin_api_test.go b/director/origin_api_test.go index 1c3c350d7..5f676d305 100644 --- a/director/origin_api_test.go +++ b/director/origin_api_test.go @@ -35,6 +35,7 @@ import ( "golang.org/x/sync/errgroup" "github.com/pelicanplatform/pelican/config" + "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/server_structs" "github.com/pelicanplatform/pelican/server_utils" "github.com/pelicanplatform/pelican/test_utils" @@ -50,10 +51,10 @@ func TestVerifyAdvertiseToken(t *testing.T) { server_utils.ResetTestState() tDir := t.TempDir() - kfile := filepath.Join(tDir, "t-key") + kDir := filepath.Join(tDir, "t-issuer-keys") //Setup a private key and a token - viper.Set("IssuerKey", kfile) + viper.Set(param.IssuerKeysDirectory.GetName(), kDir) viper.Set("Federation.DirectorURL", "https://director-url.org") diff --git a/director/stat_test.go b/director/stat_test.go index 49ea263ef..d5270c70b 100644 --- a/director/stat_test.go +++ b/director/stat_test.go @@ -36,6 +36,7 @@ import ( "github.com/stretchr/testify/require" "github.com/pelicanplatform/pelican/config" + "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/server_structs" "github.com/pelicanplatform/pelican/server_utils" "github.com/pelicanplatform/pelican/utils" @@ -827,8 +828,8 @@ func TestSendHeadReq(t *testing.T) { mockOriginAd.URL = *realServerUrl tDir := t.TempDir() - kfile := filepath.Join(tDir, "testKey") - viper.Set("IssuerKey", kfile) + kDir := filepath.Join(tDir, "testKeyDir") + viper.Set(param.IssuerKeysDirectory.GetName(), kDir) viper.Set("ConfigDir", t.TempDir()) config.InitConfig() diff --git a/token/token_create_test.go b/token/token_create_test.go index eb55c61e3..9d2249125 100644 --- a/token/token_create_test.go +++ b/token/token_create_test.go @@ -34,6 +34,7 @@ import ( "github.com/stretchr/testify/require" "github.com/pelicanplatform/pelican/config" + "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/server_structs" "github.com/pelicanplatform/pelican/test_utils" "github.com/pelicanplatform/pelican/token_scopes" @@ -208,8 +209,8 @@ func TestCreateToken(t *testing.T) { config.ResetConfig() viper.Set("IssuerUrl", "https://my-issuer.com") tDir := t.TempDir() - kfile := filepath.Join(tDir, "testKey") - viper.Set("IssuerKey", kfile) + kDir := filepath.Join(tDir, "testKeyDir") + viper.Set(param.IssuerKeysDirectory.GetName(), kDir) viper.Set("ConfigDir", t.TempDir()) config.InitConfig() err := config.InitServer(ctx, server_structs.DirectorType) diff --git a/web_ui/prometheus_test.go b/web_ui/prometheus_test.go index d92f226c0..9995417d2 100644 --- a/web_ui/prometheus_test.go +++ b/web_ui/prometheus_test.go @@ -63,9 +63,9 @@ func TestPrometheusUnprotected(t *testing.T) { // Create temp dir for the origin key file tDir := t.TempDir() - kfile := filepath.Join(tDir, "testKey") + kDir := filepath.Join(tDir, "testKeyDir") //Setup a private key - viper.Set("IssuerKey", kfile) + viper.Set(param.IssuerKeysDirectory.GetName(), kDir) viper.Set("ConfigDir", t.TempDir()) config.InitConfig() err := config.InitServer(ctx, server_structs.OriginType) @@ -109,9 +109,9 @@ func TestPrometheusProtectionCookieAuth(t *testing.T) { // Create temp dir for the origin key file tDir := t.TempDir() - kfile := filepath.Join(tDir, "testKey") - //Setup a private key - viper.Set("IssuerKey", kfile) + kDir := filepath.Join(tDir, "testKeyDir") + // Setup a private key directory + viper.Set(param.IssuerKeysDirectory.GetName(), kDir) viper.Set("ConfigDir", t.TempDir()) config.InitConfig() err := config.InitServer(ctx, server_structs.OriginType) From 8605720599a84c7307d185f305a9ba75b8c454ca Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Thu, 19 Dec 2024 15:36:37 +0000 Subject: [PATCH 36/64] fix deprecated-replacedby binding check --- docs/parameters.yaml | 2 ++ param/parameters.go | 1 + 2 files changed, 3 insertions(+) diff --git a/docs/parameters.yaml b/docs/parameters.yaml index c7c645c75..9e38581ab 100644 --- a/docs/parameters.yaml +++ b/docs/parameters.yaml @@ -71,6 +71,8 @@ description: |+ A public JWK will be derived from this private key and used as the key for token verification. type: filename +deprecated: true +replacedby: none root_default: /etc/pelican/issuer.jwk default: $ConfigBase/issuer.jwk components: ["origin", "cache", "registry", "director"] diff --git a/param/parameters.go b/param/parameters.go index 218a72727..531b5e25a 100644 --- a/param/parameters.go +++ b/param/parameters.go @@ -56,6 +56,7 @@ func GetDeprecated() map[string][]string { "Director.EnableStat": {"Director.CheckOriginPresence"}, "DisableHttpProxy": {"Client.DisableHttpProxy"}, "DisableProxyFallback": {"Client.DisableProxyFallback"}, + "IssuerKey": {"none"}, "MinimumDownloadSpeed": {"Client.MinimumDownloadSpeed"}, "Origin.EnableDirListing": {"Origin.EnableListings"}, "Origin.EnableFallbackRead": {"Origin.EnableDirectReads"}, From fc24a6596a35c2385f0d5e3fad7176ac70364f52 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Thu, 19 Dec 2024 17:22:11 +0000 Subject: [PATCH 37/64] minor semantic improvement --- registry/registry_pubkey_update.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/registry/registry_pubkey_update.go b/registry/registry_pubkey_update.go index 27f5b390f..3edaeccc2 100644 --- a/registry/registry_pubkey_update.go +++ b/registry/registry_pubkey_update.go @@ -190,14 +190,14 @@ func updateNsKeySignChallengeCommit(ctx *gin.Context, data *RegisteredPrefixUpda } // Check if any key in `clientKeySet` matches a key in `registryDbKeySet` - existingKeysIter := registryDbKeySet.Keys(ctx) + registryDbKeysIter := registryDbKeySet.Keys(ctx) clientKeysIter := clientKeySet.Keys(ctx) matchFound := false - for existingKeysIter.Next(ctx) { - existingKey := existingKeysIter.Pair().Value.(jwk.Key) + for registryDbKeysIter.Next(ctx) { + registryDbKey := registryDbKeysIter.Pair().Value.(jwk.Key) - existingKid, ok := existingKey.Get("kid") + registryDbKid, ok := registryDbKey.Get("kid") if !ok { log.Warnf("Skipping registry db existing key without 'kid'") continue @@ -212,11 +212,11 @@ func updateNsKeySignChallengeCommit(ctx *gin.Context, data *RegisteredPrefixUpda continue } - if existingKid == clientKid { + if registryDbKid == clientKid { // Verify the Proof of Possession of client's previous active private key // Get client's previous public key recorded in db var prevRawkey interface{} - if err := existingKey.Raw(&prevRawkey); err != nil { + if err := registryDbKey.Raw(&prevRawkey); err != nil { return false, nil, errors.Wrap(err, "failed to generate raw pubkey from client's previous pubkey") } @@ -228,7 +228,7 @@ func updateNsKeySignChallengeCommit(ctx *gin.Context, data *RegisteredPrefixUpda matchFound = true break } else { - log.Debugf("Client cannot prove that it possesses the key it claims, key id: %s", existingKid) + log.Debugf("This key is not client's previous active key that it wants to rotate out, or client lacks proof of possession of claimed previous active key (kid: %s)", registryDbKid) } } } @@ -283,6 +283,9 @@ func updateNsKeySignChallengeCommit(ctx *gin.Context, data *RegisteredPrefixUpda log.Infof("The public key of prefix %s hasn't changed -- nothing to update!", prefix) return false, returnMsg, nil } + } else { + log.Errorf("Prefix %s is not registered", prefix) + return false, nil, errors.Errorf("Prefix %s is not registered", prefix) } } } From 2657575ef13fdc416351497e9ef27bd9954a2a57 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Thu, 19 Dec 2024 21:43:13 +0000 Subject: [PATCH 38/64] attempt to fix API timeout "/api/v1.0/registry/updateNamespacesPubKey" --- registry/client_commands_test.go | 2 +- registry/registry_db.go | 2 +- registry/registry_pubkey_update.go | 70 ++++++++++++++---------------- 3 files changed, 35 insertions(+), 39 deletions(-) diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index 990383eab..6fe86aa5f 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -266,7 +266,7 @@ func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { jwksStr := string(jwksBytes) // Test functionality of a namespace registered with multi public keys [p2,p4] - err = updateNamespacePubKey(prefix, jwksStr) // set the registered public keys to [p2,p4] + err = setNamespacePubKey(prefix, jwksStr) // set the registered public keys to [p2,p4] require.NoError(t, err) ns, err := getNamespaceByPrefix(prefix) require.NoError(t, err) diff --git a/registry/registry_db.go b/registry/registry_db.go index 61a8c8ff7..f05565bcc 100644 --- a/registry/registry_db.go +++ b/registry/registry_db.go @@ -438,7 +438,7 @@ func updateNamespaceStatusById(id int, status server_structs.RegistrationStatus, return db.Model(ns).Where("id = ?", id).Update("admin_metadata", string(adminMetadataByte)).Error } -func updateNamespacePubKey(prefix string, pubkeyDbString string) error { +func setNamespacePubKey(prefix string, pubkeyDbString string) error { if prefix == "" { return errors.New("invalid prefix. Prefix must not be empty") } diff --git a/registry/registry_pubkey_update.go b/registry/registry_pubkey_update.go index 3edaeccc2..520c01474 100644 --- a/registry/registry_pubkey_update.go +++ b/registry/registry_pubkey_update.go @@ -105,38 +105,38 @@ func updateNsKeySignChallengeInit(data *RegisteredPrefixUpdate) (map[string]inte } // Update the public key of registered prefix(es) if the http request passed client and server verification for nonce. -// It returns whether registration is created, the response data, and an error if any -func updateNsKeySignChallengeCommit(ctx *gin.Context, data *RegisteredPrefixUpdate) (bool, map[string]interface{}, error) { +// It returns the response data, and an error if any +func updateNsKeySignChallengeCommit(ctx *gin.Context, data *RegisteredPrefixUpdate) (map[string]interface{}, error) { // Validate the client's jwks as a set here key, err := validateJwks(string(data.Pubkey)) if err != nil { - return false, nil, badRequestError{Message: err.Error()} + return nil, badRequestError{Message: err.Error()} } var rawkey interface{} // This is the raw key, like *rsa.PrivateKey or *ecdsa.PrivateKey if err := key.Raw(&rawkey); err != nil { - return false, nil, errors.Wrap(err, "failed to generate raw pubkey from jwks") + return nil, errors.Wrap(err, "failed to generate raw pubkey from jwks") } // Verify the Proof of Possession of the client and server's active private keys clientPayload := []byte(data.ClientNonce + data.ServerNonce) clientSignature, err := hex.DecodeString(data.ClientSignature) if err != nil { - return false, nil, errors.Wrap(err, "Failed to decode the client's signature") + return nil, errors.Wrap(err, "Failed to decode the client's signature") } clientVerified := verifySignature(clientPayload, clientSignature, (rawkey).(*ecdsa.PublicKey)) serverPayload, err := hex.DecodeString(data.ServerPayload) if err != nil { - return false, nil, errors.Wrap(err, "Failed to decode the server's payload") + return nil, errors.Wrap(err, "Failed to decode the server's payload") } serverSignature, err := hex.DecodeString(data.ServerSignature) if err != nil { - return false, nil, errors.Wrap(err, "Failed to decode the server's signature") + return nil, errors.Wrap(err, "Failed to decode the server's signature") } serverPrivateKey, err := loadServerKeys() if err != nil { - return false, nil, errors.Wrap(err, "Failed to decode the server's private key") + return nil, errors.Wrap(err, "Failed to decode the server's private key") } serverPubkey := serverPrivateKey.PublicKey serverVerified := verifySignature(serverPayload, serverSignature, &serverPubkey) @@ -149,14 +149,14 @@ func updateNsKeySignChallengeCommit(ctx *gin.Context, data *RegisteredPrefixUpda exists, err := namespaceExistsByPrefix(prefix) if err != nil { log.Errorf("Failed to check if namespace already exists: %v", err) - return false, nil, errors.Wrap(err, "Server encountered an error checking if namespace already exists") + return nil, errors.Wrap(err, "Server encountered an error checking if namespace already exists") } if exists { // Update the namespace's public key with the latest one when authorized origin provides a new key existingNs, err := getNamespaceByPrefix(prefix) if err != nil { log.Errorf("Failed to get existing namespace to update: %v", err) - return false, nil, errors.Wrap(err, "Server encountered an error getting existing namespace to update") + return nil, errors.Wrap(err, "Server encountered an error getting existing namespace to update") } // Check the origin is authorized to update (possessing the public key used for prefix initial registration) @@ -169,14 +169,14 @@ func updateNsKeySignChallengeCommit(ctx *gin.Context, data *RegisteredPrefixUpda } if err != nil { log.Errorf("Failed to parse in-memory public keys of the client: %v", err) - return false, nil, errors.Wrapf(err, "Invalid in-memory public keys format of the client") + return nil, errors.Wrapf(err, "Invalid in-memory public keys format of the client") } // Parse `existingNs.Pubkey` as a JWKS registryDbKeySet := jwk.NewSet() err = json.Unmarshal([]byte(existingNs.Pubkey), ®istryDbKeySet) if err != nil { log.Errorf("Failed to parse existing namespace public key as JWKS: %v", err) - return false, nil, errors.Wrap(err, "Invalid existing namespace public key format") + return nil, errors.Wrap(err, "Invalid existing namespace public key format") } // Get client's previous signature from payload @@ -186,7 +186,7 @@ func updateNsKeySignChallengeCommit(ctx *gin.Context, data *RegisteredPrefixUpda } clientPrevSignature, err := hex.DecodeString(data.ClientPrevSignature) if err != nil { - return false, nil, errors.Wrap(err, "Failed to decode the client's previous signature") + return nil, errors.Wrap(err, "Failed to decode the client's previous signature") } // Check if any key in `clientKeySet` matches a key in `registryDbKeySet` @@ -217,7 +217,7 @@ func updateNsKeySignChallengeCommit(ctx *gin.Context, data *RegisteredPrefixUpda // Get client's previous public key recorded in db var prevRawkey interface{} if err := registryDbKey.Raw(&prevRawkey); err != nil { - return false, nil, errors.Wrap(err, "failed to generate raw pubkey from client's previous pubkey") + return nil, errors.Wrap(err, "failed to generate raw pubkey from client's previous pubkey") } if data.ClientPrevSignature != "" { @@ -239,7 +239,7 @@ func updateNsKeySignChallengeCommit(ctx *gin.Context, data *RegisteredPrefixUpda } if !matchFound { - return false, nil, permissionDeniedError{ + return nil, permissionDeniedError{ Message: fmt.Sprintf("The client that tries to prefix '%s' cannot be authorized: either it doesn't contain any public key matching the existing namespace's public key in db, or it fails to pass the proof of possession verification", prefix), } } @@ -249,7 +249,7 @@ func updateNsKeySignChallengeCommit(ctx *gin.Context, data *RegisteredPrefixUpda // Process origin's new public key pubkeyData, err := json.Marshal(data.Pubkey) if err != nil { - return false, nil, errors.Wrapf(err, "Failed to convert public key from json to string format for the prefix %s", prefix) + return nil, errors.Wrapf(err, "Failed to convert public key from json to string format for the prefix %s", prefix) } clientPubkeyString := string(pubkeyData) @@ -257,64 +257,64 @@ func updateNsKeySignChallengeCommit(ctx *gin.Context, data *RegisteredPrefixUpda prevKey, _ := validateJwks(string(data.PrevPubkey)) err = constructUpdatedKeySet(registryDbKeySet, prevKey.KeyID(), key) if err != nil { - return false, nil, errors.Wrap(err, "Failed to construct the updated JWKS to be stored in registry db") + return nil, errors.Wrap(err, "failed to construct the updated JWKS to be stored in registry db") } registryDbKeySetJson, err := json.Marshal(registryDbKeySet) if err != nil { - return false, nil, errors.Wrap(err, "failed to marshal registryDbKeySet to JSON") + return nil, errors.Wrap(err, "failed to marshal registryDbKeySet to JSON") } // Perform the update action when origin provides a new key if clientPubkeyString != existingNs.Pubkey { - err = updateNamespacePubKey(prefix, string(registryDbKeySetJson)) + err = setNamespacePubKey(prefix, string(registryDbKeySetJson)) if err != nil { log.Errorf("Failed to update the public key of namespace %s: %v", prefix, err) - return false, nil, errors.Wrap(err, "Server encountered an error updating the public key of an existing namespace") + return nil, errors.Wrap(err, "Server encountered an error updating the public key of an existing namespace") } returnMsg := map[string]interface{}{ "message": fmt.Sprintf("Updated the public key of namespace %s:", prefix), } log.Infof("Updated the public key of namespace %s:", prefix) - return false, returnMsg, nil + return returnMsg, nil } else { returnMsg := map[string]interface{}{ "message": fmt.Sprintf("The public key of prefix %s hasn't changed -- nothing to update!", prefix), } log.Infof("The public key of prefix %s hasn't changed -- nothing to update!", prefix) - return false, returnMsg, nil + return returnMsg, nil } } else { log.Errorf("Prefix %s is not registered", prefix) - return false, nil, errors.Errorf("Prefix %s is not registered", prefix) + return nil, errors.Errorf("Prefix %s is not registered", prefix) } } } - return false, nil, errors.Errorf("Unable to verify the client's public key, or an encountered an error with its own: "+ + return nil, errors.Errorf("Unable to verify the client's public key, or an encountered an error with its own: "+ "server verified:%t, client verified:%t", serverVerified, clientVerified) } // Handle the registered namespace public key update with nonce generation and verifcation, regardless of // using OIDC Authorization or not -func updateNsKeySignChallenge(ctx *gin.Context, data *RegisteredPrefixUpdate) (bool, map[string]interface{}, error) { +func updateNsKeySignChallenge(ctx *gin.Context, data *RegisteredPrefixUpdate) (map[string]interface{}, error) { if data.ClientNonce != "" && data.ClientPayload != "" && data.ClientSignature != "" && data.ServerNonce != "" && data.ServerPayload != "" && data.ServerSignature != "" { - created, res, err := updateNsKeySignChallengeCommit(ctx, data) + res, err := updateNsKeySignChallengeCommit(ctx, data) if err != nil { - return false, nil, err + return nil, err } else { - return created, res, nil + return res, nil } } else if data.ClientNonce != "" { res, err := updateNsKeySignChallengeInit(data) if err != nil { - return false, nil, err + return nil, err } else { - return false, res, nil + return res, nil } } else { - return false, nil, badRequestError{Message: "Key sign challenge is missing parameters"} + return nil, badRequestError{Message: "Key sign challenge is missing parameters"} } } @@ -329,7 +329,7 @@ func updateNamespacesPubKey(ctx *gin.Context) { return } - created, res, err := updateNsKeySignChallenge(ctx, &reqData) + res, err := updateNsKeySignChallenge(ctx, &reqData) if err != nil { if errors.As(err, &permissionDeniedError{}) { ctx.JSON(http.StatusForbidden, @@ -350,10 +350,6 @@ func updateNamespacesPubKey(ctx *gin.Context) { log.Warningf("Failed to complete key sign challenge without identity requirement: %v", err) } } else { - if created { - ctx.JSON(http.StatusCreated, res) - } else { - ctx.JSON(http.StatusOK, res) - } + ctx.JSON(http.StatusOK, res) } } From 9d839866f19a5da0da3fdf5b63d65a4b104657c3 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Thu, 19 Dec 2024 21:59:07 +0000 Subject: [PATCH 39/64] another attempt to fix timeout --- registry/registry_pubkey_update.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/registry/registry_pubkey_update.go b/registry/registry_pubkey_update.go index 520c01474..89d7beaba 100644 --- a/registry/registry_pubkey_update.go +++ b/registry/registry_pubkey_update.go @@ -337,19 +337,23 @@ func updateNamespacesPubKey(ctx *gin.Context) { Status: server_structs.RespFailed, Msg: fmt.Sprintf("You don't have permission to update the registered public key of the prefix: %v", err), }) + return } else if errors.As(err, &badRequestError{}) { ctx.JSON(http.StatusBadRequest, server_structs.SimpleApiResp{ Status: server_structs.RespFailed, Msg: fmt.Sprintf("Bad request for key-sign challenge: %v", err), }) + return } else { ctx.JSON(http.StatusInternalServerError, server_structs.SimpleApiResp{ Status: server_structs.RespFailed, Msg: fmt.Sprintf("Server encountered an error during key-sign challenge: %v", err), }) log.Warningf("Failed to complete key sign challenge without identity requirement: %v", err) + return } } else { ctx.JSON(http.StatusOK, res) + return } } From f1123dd4cacdd6d9ea5fcc143fd2be68a849ff36 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 20 Dec 2024 01:57:13 +0000 Subject: [PATCH 40/64] 3rd attempt to fix timeout --- registry/client_commands_test.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index 6fe86aa5f..223071946 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -304,22 +304,12 @@ func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { func TestRegistryKeyChainingOSDF(t *testing.T) { ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) - t.Cleanup(func() { - func() { require.NoError(t, egrp.Wait()) }() - cancel() - config.ResetIssuerJWKPtr() - config.ResetIssuerPrivateKeys() - server_utils.ResetTestState() - }) + defer func() { require.NoError(t, egrp.Wait()) }() + defer cancel() server_utils.ResetTestState() _, err := config.SetPreferredPrefix(config.OsdfPrefix) assert.NoError(t, err) - viper.Set("Federation.DirectorUrl", "https://osdf-director.osg-htc.org") - viper.Set("Federation.RegistryUrl", "https://osdf-registry.osg-htc.org") - viper.Set("Federation.JwkUrl", "https://osg-htc.org/osdf/public_signing_key.jwks") - viper.Set("Federation.BrokerUrl", "https://osdf-director.osg-htc.org") - // On by default, but just to make things explicit viper.Set("Registry.RequireKeyChaining", true) From 08481cd82bab3af59e5e25a73aaecb58d9eacb7d Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 20 Dec 2024 02:47:50 +0000 Subject: [PATCH 41/64] 4th attempt to fix timeout --- registry/client_commands_test.go | 91 +------------------------------- 1 file changed, 1 insertion(+), 90 deletions(-) diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index 223071946..4c9f43a84 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -208,9 +208,8 @@ func TestServeNamespaceRegistry(t *testing.T) { server_utils.ResetTestState() } -func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { +func TestRegistryKeyChainingOSDF(t *testing.T) { ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) - server_utils.ResetTestState() t.Cleanup(func() { func() { require.NoError(t, egrp.Wait()) }() cancel() @@ -219,94 +218,6 @@ func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { server_utils.ResetTestState() }) - tDir := t.TempDir() - - svr := registryMockup(ctx, t, "serveregistry") - defer func() { - err := ShutdownRegistryDB() - assert.NoError(t, err) - svr.CloseClientConnections() - svr.Close() - }() - - config.ResetIssuerJWKPtr() - config.ResetIssuerPrivateKeys() - privKeys := config.GetIssuerPrivateKeys() - require.Len(t, privKeys, 0) - - // Construct a client that has [p1,p2,p3] and p3 is the active private key - privKey1, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) - require.NotEmpty(t, privKey1) - require.NoError(t, err) - privKey2, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) - require.NoError(t, err) - - prefix := "/mascot/bucky" - err = NamespaceRegister(privKey2, svr.URL+"/api/v1.0/registry", "", prefix, "mock_site_name") - require.NoError(t, err) - - config.UpdatePreviousIssuerPrivateJWK() - privKey3, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) - require.NoError(t, err) - - // Construct a public keys JWKS [p2,p4] to save in registry DB, imitating admin manually adding p4 - registryDbJwks := jwk.NewSet() - pubKey2, err := jwk.PublicKeyOf(privKey2) - require.NoError(t, err) - err = registryDbJwks.AddKey(pubKey2) - require.NoError(t, err) - privKey4, err := config.GeneratePEM(filepath.Join(tDir, "elsewhere")) - require.NoError(t, err) - pubKey4, err := jwk.PublicKeyOf(privKey4) - require.NoError(t, err) - err = registryDbJwks.AddKey(pubKey4) - require.NoError(t, err) - jwksBytes, err := json.Marshal(registryDbJwks) - require.NoError(t, err) - jwksStr := string(jwksBytes) - - // Test functionality of a namespace registered with multi public keys [p2,p4] - err = setNamespacePubKey(prefix, jwksStr) // set the registered public keys to [p2,p4] - require.NoError(t, err) - ns, err := getNamespaceByPrefix(prefix) - require.NoError(t, err) - require.Equal(t, jwksStr, ns.Pubkey) - - prevKey := config.GetPreviousIssuerPrivateJWK() - require.Equal(t, privKey2.KeyID(), prevKey.KeyID()) - privKeys = config.GetIssuerPrivateKeys() - require.Len(t, privKeys, 3) - - // Client allKeys:[p1,p2,p3] prevKey:p2 activeKey:p3 ---UPDATE--> Registry [p2,p4] - // => should update Registry to [p3,p4] (rotate out prevKey:p2, rotate in activeKey:p3) - err = NamespacesPubKeyUpdate(privKey3, []string{prefix}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") - require.NoError(t, err) - ns, err = getNamespaceByPrefix(prefix) - require.NoError(t, err) - - expectedJwks := jwk.NewSet() - pubKey3, err := jwk.PublicKeyOf(privKey3) - require.NoError(t, err) - err = expectedJwks.AddKey(pubKey3) - require.NoError(t, err) - err = expectedJwks.AddKey(pubKey4) - require.NoError(t, err) - expectedJwksBytes, err := json.Marshal(expectedJwks) - require.NoError(t, err) - expectedJwksStr := string(expectedJwksBytes) - - expectedKids, err := getSortedKids(ctx, expectedJwksStr) - require.NoError(t, err) - actualKids, err := getSortedKids(ctx, ns.Pubkey) - require.NoError(t, err) - require.Equal(t, expectedKids, actualKids) -} - -func TestRegistryKeyChainingOSDF(t *testing.T) { - ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) - defer func() { require.NoError(t, egrp.Wait()) }() - defer cancel() - server_utils.ResetTestState() _, err := config.SetPreferredPrefix(config.OsdfPrefix) assert.NoError(t, err) From f3c7c99cb44e16c9e5aee792b017f7fb257b3b02 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 20 Dec 2024 03:13:13 +0000 Subject: [PATCH 42/64] 5th attempt to fix timeout --- registry/client_commands_test.go | 98 +++++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 1 deletion(-) diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index 4c9f43a84..e5d03d302 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -208,8 +208,9 @@ func TestServeNamespaceRegistry(t *testing.T) { server_utils.ResetTestState() } -func TestRegistryKeyChainingOSDF(t *testing.T) { +func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) + server_utils.ResetTestState() t.Cleanup(func() { func() { require.NoError(t, egrp.Wait()) }() cancel() @@ -218,7 +219,102 @@ func TestRegistryKeyChainingOSDF(t *testing.T) { server_utils.ResetTestState() }) + tDir := t.TempDir() + + svr := registryMockup(ctx, t, "serveregistry") + defer func() { + err := ShutdownRegistryDB() + assert.NoError(t, err) + svr.CloseClientConnections() + svr.Close() + }() + + config.ResetIssuerJWKPtr() + config.ResetIssuerPrivateKeys() + privKeys := config.GetIssuerPrivateKeys() + require.Len(t, privKeys, 0) + + // Construct a client that has [p1,p2,p3] and p3 is the active private key + privKey1, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) + require.NotEmpty(t, privKey1) + require.NoError(t, err) + privKey2, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) + require.NoError(t, err) + + prefix := "/mascot/bucky" + err = NamespaceRegister(privKey2, svr.URL+"/api/v1.0/registry", "", prefix, "mock_site_name") + require.NoError(t, err) + + config.UpdatePreviousIssuerPrivateJWK() + privKey3, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) + require.NoError(t, err) + + // Construct a public keys JWKS [p2,p4] to save in registry DB, imitating admin manually adding p4 + registryDbJwks := jwk.NewSet() + pubKey2, err := jwk.PublicKeyOf(privKey2) + require.NoError(t, err) + err = registryDbJwks.AddKey(pubKey2) + require.NoError(t, err) + privKey4, err := config.GeneratePEM(filepath.Join(tDir, "elsewhere")) + require.NoError(t, err) + pubKey4, err := jwk.PublicKeyOf(privKey4) + require.NoError(t, err) + err = registryDbJwks.AddKey(pubKey4) + require.NoError(t, err) + jwksBytes, err := json.Marshal(registryDbJwks) + require.NoError(t, err) + jwksStr := string(jwksBytes) + + // Test functionality of a namespace registered with multi public keys [p2,p4] + err = setNamespacePubKey(prefix, jwksStr) // set the registered public keys to [p2,p4] + require.NoError(t, err) + ns, err := getNamespaceByPrefix(prefix) + require.NoError(t, err) + require.Equal(t, jwksStr, ns.Pubkey) + + prevKey := config.GetPreviousIssuerPrivateJWK() + require.Equal(t, privKey2.KeyID(), prevKey.KeyID()) + privKeys = config.GetIssuerPrivateKeys() + require.Len(t, privKeys, 3) + + // Client allKeys:[p1,p2,p3] prevKey:p2 activeKey:p3 ---UPDATE--> Registry [p2,p4] + // => should update Registry to [p3,p4] (rotate out prevKey:p2, rotate in activeKey:p3) + err = NamespacesPubKeyUpdate(privKey3, []string{prefix}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") + require.NoError(t, err) + ns, err = getNamespaceByPrefix(prefix) + require.NoError(t, err) + + expectedJwks := jwk.NewSet() + pubKey3, err := jwk.PublicKeyOf(privKey3) + require.NoError(t, err) + err = expectedJwks.AddKey(pubKey3) + require.NoError(t, err) + err = expectedJwks.AddKey(pubKey4) + require.NoError(t, err) + expectedJwksBytes, err := json.Marshal(expectedJwks) + require.NoError(t, err) + expectedJwksStr := string(expectedJwksBytes) + + expectedKids, err := getSortedKids(ctx, expectedJwksStr) + require.NoError(t, err) + actualKids, err := getSortedKids(ctx, ns.Pubkey) + require.NoError(t, err) + require.Equal(t, expectedKids, actualKids) +} + +func TestRegistryKeyChainingOSDF(t *testing.T) { + ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) + t.Cleanup(func() { + func() { require.NoError(t, egrp.Wait()) }() + cancel() + config.ResetIssuerJWKPtr() + config.ResetIssuerPrivateKeys() + server_utils.ResetTestState() + }) + config.ResetIssuerJWKPtr() + config.ResetIssuerPrivateKeys() server_utils.ResetTestState() + _, err := config.SetPreferredPrefix(config.OsdfPrefix) assert.NoError(t, err) // On by default, but just to make things explicit From 7cc659167b1f5542d77c3c966ffcbaa8affbc9f8 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 20 Dec 2024 03:58:39 +0000 Subject: [PATCH 43/64] 6th attempt to fix timeout --- .github/workflows/test-template.yml | 2 +- registry/client_commands_test.go | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-template.yml b/.github/workflows/test-template.yml index 12e09f25f..9e3900e0d 100644 --- a/.github/workflows/test-template.yml +++ b/.github/workflows/test-template.yml @@ -42,7 +42,7 @@ jobs: - name: Test run: | make web-build - go test -tags=${{ inputs.tags }} -timeout 15m -coverpkg=./... -coverprofile=${{ inputs.coverprofile }} -covermode=count ./... + go test -tags=${{ inputs.tags }} -timeout 20m -coverpkg=./... -coverprofile=${{ inputs.coverprofile }} -covermode=count ./... - name: Get total code coverage if: github.event_name == 'pull_request' id: cc diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index e5d03d302..fb2c153cc 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -303,6 +303,8 @@ func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { } func TestRegistryKeyChainingOSDF(t *testing.T) { + config.ResetIssuerJWKPtr() + config.ResetIssuerPrivateKeys() ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) t.Cleanup(func() { func() { require.NoError(t, egrp.Wait()) }() @@ -311,12 +313,15 @@ func TestRegistryKeyChainingOSDF(t *testing.T) { config.ResetIssuerPrivateKeys() server_utils.ResetTestState() }) - config.ResetIssuerJWKPtr() - config.ResetIssuerPrivateKeys() - server_utils.ResetTestState() + server_utils.ResetTestState() _, err := config.SetPreferredPrefix(config.OsdfPrefix) assert.NoError(t, err) + viper.Set("Federation.DirectorUrl", "https://osdf-director.osg-htc.org") + viper.Set("Federation.RegistryUrl", "https://osdf-registry.osg-htc.org") + viper.Set("Federation.JwkUrl", "https://osg-htc.org/osdf/public_signing_key.jwks") + viper.Set("Federation.BrokerUrl", "https://osdf-director.osg-htc.org") + // On by default, but just to make things explicit viper.Set("Registry.RequireKeyChaining", true) From 0955e187a9409c42bdf045473ad74d4d909faef4 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 20 Dec 2024 04:30:30 +0000 Subject: [PATCH 44/64] 7th attempt to fix timeout --- .github/workflows/test-template.yml | 2 +- registry/client_commands_test.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-template.yml b/.github/workflows/test-template.yml index 9e3900e0d..12e09f25f 100644 --- a/.github/workflows/test-template.yml +++ b/.github/workflows/test-template.yml @@ -42,7 +42,7 @@ jobs: - name: Test run: | make web-build - go test -tags=${{ inputs.tags }} -timeout 20m -coverpkg=./... -coverprofile=${{ inputs.coverprofile }} -covermode=count ./... + go test -tags=${{ inputs.tags }} -timeout 15m -coverpkg=./... -coverprofile=${{ inputs.coverprofile }} -covermode=count ./... - name: Get total code coverage if: github.event_name == 'pull_request' id: cc diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index fb2c153cc..239998951 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -45,8 +45,10 @@ import ( func registryMockup(ctx context.Context, t *testing.T, testName string) *httptest.Server { issuerTempDir := filepath.Join(t.TempDir(), testName) - ikey := filepath.Join(issuerTempDir, "issuer-keys") - viper.Set("IssuerKeysDirectory", ikey) + ikey := filepath.Join(issuerTempDir, "issuer.jwk") + viper.Set("IssuerKey", ikey) + ikeyDir := filepath.Join(issuerTempDir, "issuer-keys") + viper.Set("IssuerKeysDirectory", ikeyDir) viper.Set("Registry.DbLocation", filepath.Join(issuerTempDir, "test.sql")) viper.Set("Server.WebPort", 8444) config.InitConfigDir(viper.GetViper()) From 616277862c2b45d7df1ecb553113b07fab751f68 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 20 Dec 2024 05:38:13 +0000 Subject: [PATCH 45/64] 8th attempt to fix timeout --- registry/client_commands_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index 239998951..27e90e354 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -45,8 +45,6 @@ import ( func registryMockup(ctx context.Context, t *testing.T, testName string) *httptest.Server { issuerTempDir := filepath.Join(t.TempDir(), testName) - ikey := filepath.Join(issuerTempDir, "issuer.jwk") - viper.Set("IssuerKey", ikey) ikeyDir := filepath.Join(issuerTempDir, "issuer-keys") viper.Set("IssuerKeysDirectory", ikeyDir) viper.Set("Registry.DbLocation", filepath.Join(issuerTempDir, "test.sql")) @@ -368,6 +366,7 @@ func TestRegistryKeyChainingOSDF(t *testing.T) { assert.Contains(t, err.Error(), "A superspace or subspace of this namespace /topo/foo already exists in the OSDF topology: /topo/foo. To register a Pelican equivalence, you need to present your identity.") // Now we create a new key and try to use it to register a super/sub space. These shouldn't succeed + viper.Set("IssuerKey", t.TempDir()+"/keychaining.jwk") viper.Set("IssuerKeysDirectory", t.TempDir()+"/keychaining") viper.Set("ConfigDir", t.TempDir()) config.InitConfig() @@ -440,6 +439,7 @@ func TestRegistryKeyChaining(t *testing.T) { require.NoError(t, err) // Now we create a new key and try to use it to register a super/sub space. These shouldn't succeed + viper.Set("IssuerKey", t.TempDir()+"/keychaining.jwk") viper.Set("IssuerKeysDirectory", t.TempDir()+"/keychaining") viper.Set("ConfigDir", t.TempDir()) config.InitConfig() From e9a0fb23358003191d5fa5ebf7ad6ce6790965ec Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 20 Dec 2024 06:15:56 +0000 Subject: [PATCH 46/64] 9th attempt to fix timeout --- registry/client_commands_test.go | 107 ------------------------------- 1 file changed, 107 deletions(-) diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index 27e90e354..447fa6721 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -302,112 +302,6 @@ func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { require.Equal(t, expectedKids, actualKids) } -func TestRegistryKeyChainingOSDF(t *testing.T) { - config.ResetIssuerJWKPtr() - config.ResetIssuerPrivateKeys() - ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) - t.Cleanup(func() { - func() { require.NoError(t, egrp.Wait()) }() - cancel() - config.ResetIssuerJWKPtr() - config.ResetIssuerPrivateKeys() - server_utils.ResetTestState() - }) - - server_utils.ResetTestState() - _, err := config.SetPreferredPrefix(config.OsdfPrefix) - assert.NoError(t, err) - viper.Set("Federation.DirectorUrl", "https://osdf-director.osg-htc.org") - viper.Set("Federation.RegistryUrl", "https://osdf-registry.osg-htc.org") - viper.Set("Federation.JwkUrl", "https://osg-htc.org/osdf/public_signing_key.jwks") - viper.Set("Federation.BrokerUrl", "https://osdf-director.osg-htc.org") - - // On by default, but just to make things explicit - viper.Set("Registry.RequireKeyChaining", true) - - registrySvr := registryMockup(ctx, t, "OSDFkeychaining") - topoSvr := topologyMockup(t, []string{"/topo/foo"}) - viper.Set("Federation.TopologyNamespaceURL", topoSvr.URL) - err = migrateTopologyTestTable() - require.NoError(t, err) - err = PopulateTopology(ctx) - require.NoError(t, err) - - defer func() { - err := ShutdownRegistryDB() - assert.NoError(t, err) - registrySvr.CloseClientConnections() - registrySvr.Close() - topoSvr.CloseClientConnections() - topoSvr.Close() - }() - - _, err = config.GetIssuerPublicJWKS() - require.NoError(t, err) - privKey, err := config.GetIssuerPrivateJWK() - require.NoError(t, err) - - // Start by registering /foo/bar with the default key - err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/foo/bar", "") - require.NoError(t, err) - - // Perform one test with a subspace and the same key -- should succeed - err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/foo/bar/test", "") - require.NoError(t, err) - - // If the namespace is a subspace from the topology and is registered without the identity - // we deny it - err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/topo/foo/bar", "") - require.Error(t, err) - assert.Contains(t, err.Error(), "A superspace or subspace of this namespace /topo/foo/bar already exists in the OSDF topology: /topo/foo. To register a Pelican equivalence, you need to present your identity.") - - err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/topo/foo", "") - require.Error(t, err) - assert.Contains(t, err.Error(), "A superspace or subspace of this namespace /topo/foo already exists in the OSDF topology: /topo/foo. To register a Pelican equivalence, you need to present your identity.") - - // Now we create a new key and try to use it to register a super/sub space. These shouldn't succeed - viper.Set("IssuerKey", t.TempDir()+"/keychaining.jwk") - viper.Set("IssuerKeysDirectory", t.TempDir()+"/keychaining") - viper.Set("ConfigDir", t.TempDir()) - config.InitConfig() - err = config.InitServer(ctx, server_structs.RegistryType) - require.NoError(t, err) - - _, err = config.GetIssuerPublicJWKS() - require.NoError(t, err) - privKey, err = config.GetIssuerPrivateJWK() - require.NoError(t, err) - - err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/foo/bar/baz", "") - require.ErrorContains(t, err, "Cannot register a namespace that is suffixed or prefixed") - - err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/foo", "") - require.ErrorContains(t, err, "Cannot register a namespace that is suffixed or prefixed") - - // Make sure we can register things similar but distinct in prefix and suffix - err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/fo", "") - require.NoError(t, err) - err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/foo/barz", "") - require.NoError(t, err) - - // Now turn off token chaining and retry -- no errors should occur - viper.Set("Registry.RequireKeyChaining", false) - err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/foo/bar/baz", "") - require.NoError(t, err) - - err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/foo", "") - require.NoError(t, err) - - // However, topology check should be independent of key chaining check - err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/topo", "") - require.Error(t, err) - assert.Contains(t, err.Error(), "A superspace or subspace of this namespace /topo already exists in the OSDF topology: /topo/foo. To register a Pelican equivalence, you need to present your identity.") - - _, err = config.SetPreferredPrefix(config.PelicanPrefix) - assert.NoError(t, err) - server_utils.ResetTestState() -} - func TestRegistryKeyChaining(t *testing.T) { ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) defer func() { require.NoError(t, egrp.Wait()) }() @@ -439,7 +333,6 @@ func TestRegistryKeyChaining(t *testing.T) { require.NoError(t, err) // Now we create a new key and try to use it to register a super/sub space. These shouldn't succeed - viper.Set("IssuerKey", t.TempDir()+"/keychaining.jwk") viper.Set("IssuerKeysDirectory", t.TempDir()+"/keychaining") viper.Set("ConfigDir", t.TempDir()) config.InitConfig() From 752ab32b74678464261b0bf87063b19a1030e730 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 20 Dec 2024 07:13:17 +0000 Subject: [PATCH 47/64] 10th attempt to fix timeout --- .github/workflows/test-template.yml | 2 +- registry/client_commands_test.go | 111 +++++++++++++++++++++++++++- 2 files changed, 109 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-template.yml b/.github/workflows/test-template.yml index 12e09f25f..bae3fcdf2 100644 --- a/.github/workflows/test-template.yml +++ b/.github/workflows/test-template.yml @@ -42,7 +42,7 @@ jobs: - name: Test run: | make web-build - go test -tags=${{ inputs.tags }} -timeout 15m -coverpkg=./... -coverprofile=${{ inputs.coverprofile }} -covermode=count ./... + go test -tags=${{ inputs.tags }} -timeout 5m -coverpkg=./... -coverprofile=${{ inputs.coverprofile }} -covermode=count ./... - name: Get total code coverage if: github.event_name == 'pull_request' id: cc diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index 447fa6721..73a35b901 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -302,6 +302,111 @@ func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { require.Equal(t, expectedKids, actualKids) } +func TestRegistryKeyChainingOSDF(t *testing.T) { + config.ResetIssuerJWKPtr() + config.ResetIssuerPrivateKeys() + ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) + t.Cleanup(func() { + func() { require.NoError(t, egrp.Wait()) }() + cancel() + config.ResetIssuerJWKPtr() + config.ResetIssuerPrivateKeys() + server_utils.ResetTestState() + }) + + server_utils.ResetTestState() + _, err := config.SetPreferredPrefix(config.OsdfPrefix) + assert.NoError(t, err) + viper.Set("Federation.DirectorUrl", "https://osdf-director.osg-htc.org") + viper.Set("Federation.RegistryUrl", "https://osdf-registry.osg-htc.org") + viper.Set("Federation.JwkUrl", "https://osg-htc.org/osdf/public_signing_key.jwks") + viper.Set("Federation.BrokerUrl", "https://osdf-director.osg-htc.org") + + // On by default, but just to make things explicit + viper.Set("Registry.RequireKeyChaining", true) + + registrySvr := registryMockup(ctx, t, "OSDFkeychaining") + topoSvr := topologyMockup(t, []string{"/topo/foo"}) + viper.Set("Federation.TopologyNamespaceURL", topoSvr.URL) + err = migrateTopologyTestTable() + require.NoError(t, err) + err = PopulateTopology(ctx) + require.NoError(t, err) + + defer func() { + err := ShutdownRegistryDB() + assert.NoError(t, err) + registrySvr.CloseClientConnections() + registrySvr.Close() + topoSvr.CloseClientConnections() + topoSvr.Close() + }() + + privKey, err := config.GetIssuerPrivateJWK() + require.NoError(t, err) + _, err = config.GetIssuerPublicJWKS() + require.NoError(t, err) + + // Start by registering /foo/bar with the default key + err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/foo/bar", "") + require.NoError(t, err) + + // Perform one test with a subspace and the same key -- should succeed + err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/foo/bar/test", "") + require.NoError(t, err) + + // If the namespace is a subspace from the topology and is registered without the identity + // we deny it + err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/topo/foo/bar", "") + require.Error(t, err) + assert.Contains(t, err.Error(), "A superspace or subspace of this namespace /topo/foo/bar already exists in the OSDF topology: /topo/foo. To register a Pelican equivalence, you need to present your identity.") + + err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/topo/foo", "") + require.Error(t, err) + assert.Contains(t, err.Error(), "A superspace or subspace of this namespace /topo/foo already exists in the OSDF topology: /topo/foo. To register a Pelican equivalence, you need to present your identity.") + + // Now we create a new key and try to use it to register a super/sub space. These shouldn't succeed + viper.Set("IssuerKeysDirectory", t.TempDir()+"/keychaining2") + viper.Set("ConfigDir", t.TempDir()) + config.InitConfig() + err = config.InitServer(ctx, server_structs.RegistryType) + require.NoError(t, err) + + privKey, err = config.GetIssuerPrivateJWK() + require.NoError(t, err) + _, err = config.GetIssuerPublicJWKS() + require.NoError(t, err) + + err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/foo/bar/baz", "") + require.ErrorContains(t, err, "Cannot register a namespace that is suffixed or prefixed") + + err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/foo", "") + require.ErrorContains(t, err, "Cannot register a namespace that is suffixed or prefixed") + + // Make sure we can register things similar but distinct in prefix and suffix + err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/fo", "") + require.NoError(t, err) + err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/foo/barz", "") + require.NoError(t, err) + + // Now turn off token chaining and retry -- no errors should occur + viper.Set("Registry.RequireKeyChaining", false) + err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/foo/bar/baz", "") + require.NoError(t, err) + + err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/foo", "") + require.NoError(t, err) + + // However, topology check should be independent of key chaining check + err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/topo", "") + require.Error(t, err) + assert.Contains(t, err.Error(), "A superspace or subspace of this namespace /topo already exists in the OSDF topology: /topo/foo. To register a Pelican equivalence, you need to present your identity.") + + _, err = config.SetPreferredPrefix(config.PelicanPrefix) + assert.NoError(t, err) + server_utils.ResetTestState() +} + func TestRegistryKeyChaining(t *testing.T) { ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) defer func() { require.NoError(t, egrp.Wait()) }() @@ -333,16 +438,16 @@ func TestRegistryKeyChaining(t *testing.T) { require.NoError(t, err) // Now we create a new key and try to use it to register a super/sub space. These shouldn't succeed - viper.Set("IssuerKeysDirectory", t.TempDir()+"/keychaining") + viper.Set("IssuerKeysDirectory", t.TempDir()+"/keychaining2") viper.Set("ConfigDir", t.TempDir()) config.InitConfig() err = config.InitServer(ctx, server_structs.RegistryType) require.NoError(t, err) - _, err = config.GetIssuerPublicJWKS() - require.NoError(t, err) privKey, err = config.GetIssuerPrivateJWK() require.NoError(t, err) + _, err = config.GetIssuerPublicJWKS() + require.NoError(t, err) err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/foo/bar/baz", "") require.ErrorContains(t, err, "Cannot register a namespace that is suffixed or prefixed") From 0b1b7c1c8bea87c0a26effd0f06b341fca49b727 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 20 Dec 2024 07:33:34 +0000 Subject: [PATCH 48/64] 10.5th attempt to fix timeout --- registry/client_commands_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index 73a35b901..4250cce50 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -424,10 +424,10 @@ func TestRegistryKeyChaining(t *testing.T) { registrySvr.Close() }() - _, err := config.GetIssuerPublicJWKS() - require.NoError(t, err) privKey, err := config.GetIssuerPrivateJWK() require.NoError(t, err) + _, err = config.GetIssuerPublicJWKS() + require.NoError(t, err) // Start by registering /foo/bar with the default key err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/foo/bar", "") From d6dc0e13e7eb219ea427fb23ea8a1b5d5fed08d2 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 20 Dec 2024 08:11:05 +0000 Subject: [PATCH 49/64] 11th attempt to fix timeout --- .github/workflows/test-template.yml | 2 +- registry/client_commands_test.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-template.yml b/.github/workflows/test-template.yml index bae3fcdf2..12e09f25f 100644 --- a/.github/workflows/test-template.yml +++ b/.github/workflows/test-template.yml @@ -42,7 +42,7 @@ jobs: - name: Test run: | make web-build - go test -tags=${{ inputs.tags }} -timeout 5m -coverpkg=./... -coverprofile=${{ inputs.coverprofile }} -covermode=count ./... + go test -tags=${{ inputs.tags }} -timeout 15m -coverpkg=./... -coverprofile=${{ inputs.coverprofile }} -covermode=count ./... - name: Get total code coverage if: github.event_name == 'pull_request' id: cc diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index 4250cce50..f78ee68ea 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -45,6 +45,8 @@ import ( func registryMockup(ctx context.Context, t *testing.T, testName string) *httptest.Server { issuerTempDir := filepath.Join(t.TempDir(), testName) + ikey := filepath.Join(issuerTempDir, "issuer.jwk") + viper.Set("IssuerKey", ikey) ikeyDir := filepath.Join(issuerTempDir, "issuer-keys") viper.Set("IssuerKeysDirectory", ikeyDir) viper.Set("Registry.DbLocation", filepath.Join(issuerTempDir, "test.sql")) From 5eb651f421b87e7e15fb260498d1d5f0aebb8f26 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 20 Dec 2024 13:53:03 +0000 Subject: [PATCH 50/64] 12th attempt to fix timeout --- registry/client_commands_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index f78ee68ea..4250cce50 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -45,8 +45,6 @@ import ( func registryMockup(ctx context.Context, t *testing.T, testName string) *httptest.Server { issuerTempDir := filepath.Join(t.TempDir(), testName) - ikey := filepath.Join(issuerTempDir, "issuer.jwk") - viper.Set("IssuerKey", ikey) ikeyDir := filepath.Join(issuerTempDir, "issuer-keys") viper.Set("IssuerKeysDirectory", ikeyDir) viper.Set("Registry.DbLocation", filepath.Join(issuerTempDir, "test.sql")) From 85bc504e67986bceca716dba477721593c393144 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 20 Dec 2024 19:30:27 +0000 Subject: [PATCH 51/64] only run ./registry/registry_db_test.go and client_commands_test.go --- .github/workflows/test-template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-template.yml b/.github/workflows/test-template.yml index 12e09f25f..e11168660 100644 --- a/.github/workflows/test-template.yml +++ b/.github/workflows/test-template.yml @@ -42,7 +42,7 @@ jobs: - name: Test run: | make web-build - go test -tags=${{ inputs.tags }} -timeout 15m -coverpkg=./... -coverprofile=${{ inputs.coverprofile }} -covermode=count ./... + go test -tags=${{ inputs.tags }} -timeout 15m -coverpkg=./... -coverprofile=${{ inputs.coverprofile }} -covermode=count ./registry/registry_db_test.go ./registry/client_commands_test.go - name: Get total code coverage if: github.event_name == 'pull_request' id: cc @@ -67,7 +67,7 @@ jobs: git reset --hard ${{ github.event.pull_request.base.sha }} make web-build go generate ./... - go test -tags=${{ inputs.tags }} -timeout 15m -coverpkg=./... -coverprofile=base_coverage.out -covermode=count ./... + go test -tags=${{ inputs.tags }} -timeout 15m -coverpkg=./... -coverprofile=base_coverage.out -covermode=count ./registry/registry_db_test.go ./registry/client_commands_test.go go tool cover -func=base_coverage.out > unit-base.txt git reset --hard $HEAD - name: Get base code coverage value From 1fec0cef8bfd2a43f4362ebf98f6aa55635b2c6d Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 20 Dec 2024 19:57:37 +0000 Subject: [PATCH 52/64] comment out TestServeNamespaceRegistry --- .github/workflows/test-template.yml | 4 +- registry/client_commands_test.go | 227 ++++++++++++++-------------- 2 files changed, 115 insertions(+), 116 deletions(-) diff --git a/.github/workflows/test-template.yml b/.github/workflows/test-template.yml index e11168660..55b3723da 100644 --- a/.github/workflows/test-template.yml +++ b/.github/workflows/test-template.yml @@ -42,7 +42,7 @@ jobs: - name: Test run: | make web-build - go test -tags=${{ inputs.tags }} -timeout 15m -coverpkg=./... -coverprofile=${{ inputs.coverprofile }} -covermode=count ./registry/registry_db_test.go ./registry/client_commands_test.go + go test -tags=${{ inputs.tags }} -timeout 5m -coverpkg=./... -coverprofile=${{ inputs.coverprofile }} -covermode=count ./registry/registry_db_test.go ./registry/client_commands_test.go - name: Get total code coverage if: github.event_name == 'pull_request' id: cc @@ -67,7 +67,7 @@ jobs: git reset --hard ${{ github.event.pull_request.base.sha }} make web-build go generate ./... - go test -tags=${{ inputs.tags }} -timeout 15m -coverpkg=./... -coverprofile=base_coverage.out -covermode=count ./registry/registry_db_test.go ./registry/client_commands_test.go + go test -tags=${{ inputs.tags }} -timeout 5m -coverpkg=./... -coverprofile=base_coverage.out -covermode=count ./registry/registry_db_test.go ./registry/client_commands_test.go go tool cover -func=base_coverage.out > unit-base.txt git reset --hard $HEAD - name: Get base code coverage value diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index 4250cce50..e614313c7 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -26,7 +26,6 @@ import ( "net/http/httptest" "os" "path/filepath" - "sort" "testing" "github.com/gin-gonic/gin" @@ -68,26 +67,26 @@ func registryMockup(ctx context.Context, t *testing.T, testName string) *httptes return svr } -func getSortedKids(ctx context.Context, jsonStr string) ([]string, error) { - set, err := jwk.Parse([]byte(jsonStr)) - if err != nil { - return nil, err - } - var kids []string - keysIter := set.Keys(ctx) - for keysIter.Next(ctx) { - key := keysIter.Pair().Value.(jwk.Key) - - kid, ok := key.Get("kid") - if !ok { - continue - } - kids = append(kids, kid.(string)) - } - sort.Strings(kids) - - return kids, nil -} +// func getSortedKids(ctx context.Context, jsonStr string) ([]string, error) { +// set, err := jwk.Parse([]byte(jsonStr)) +// if err != nil { +// return nil, err +// } +// var kids []string +// keysIter := set.Keys(ctx) +// for keysIter.Next(ctx) { +// key := keysIter.Pair().Value.(jwk.Key) + +// kid, ok := key.Get("kid") +// if !ok { +// continue +// } +// kids = append(kids, kid.(string)) +// } +// sort.Strings(kids) + +// return kids, nil +// } func TestServeNamespaceRegistry(t *testing.T) { ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) @@ -208,99 +207,99 @@ func TestServeNamespaceRegistry(t *testing.T) { server_utils.ResetTestState() } -func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { - ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) - server_utils.ResetTestState() - t.Cleanup(func() { - func() { require.NoError(t, egrp.Wait()) }() - cancel() - config.ResetIssuerJWKPtr() - config.ResetIssuerPrivateKeys() - server_utils.ResetTestState() - }) - - tDir := t.TempDir() - - svr := registryMockup(ctx, t, "serveregistry") - defer func() { - err := ShutdownRegistryDB() - assert.NoError(t, err) - svr.CloseClientConnections() - svr.Close() - }() - - config.ResetIssuerJWKPtr() - config.ResetIssuerPrivateKeys() - privKeys := config.GetIssuerPrivateKeys() - require.Len(t, privKeys, 0) - - // Construct a client that has [p1,p2,p3] and p3 is the active private key - privKey1, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) - require.NotEmpty(t, privKey1) - require.NoError(t, err) - privKey2, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) - require.NoError(t, err) - - prefix := "/mascot/bucky" - err = NamespaceRegister(privKey2, svr.URL+"/api/v1.0/registry", "", prefix, "mock_site_name") - require.NoError(t, err) - - config.UpdatePreviousIssuerPrivateJWK() - privKey3, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) - require.NoError(t, err) - - // Construct a public keys JWKS [p2,p4] to save in registry DB, imitating admin manually adding p4 - registryDbJwks := jwk.NewSet() - pubKey2, err := jwk.PublicKeyOf(privKey2) - require.NoError(t, err) - err = registryDbJwks.AddKey(pubKey2) - require.NoError(t, err) - privKey4, err := config.GeneratePEM(filepath.Join(tDir, "elsewhere")) - require.NoError(t, err) - pubKey4, err := jwk.PublicKeyOf(privKey4) - require.NoError(t, err) - err = registryDbJwks.AddKey(pubKey4) - require.NoError(t, err) - jwksBytes, err := json.Marshal(registryDbJwks) - require.NoError(t, err) - jwksStr := string(jwksBytes) - - // Test functionality of a namespace registered with multi public keys [p2,p4] - err = setNamespacePubKey(prefix, jwksStr) // set the registered public keys to [p2,p4] - require.NoError(t, err) - ns, err := getNamespaceByPrefix(prefix) - require.NoError(t, err) - require.Equal(t, jwksStr, ns.Pubkey) - - prevKey := config.GetPreviousIssuerPrivateJWK() - require.Equal(t, privKey2.KeyID(), prevKey.KeyID()) - privKeys = config.GetIssuerPrivateKeys() - require.Len(t, privKeys, 3) - - // Client allKeys:[p1,p2,p3] prevKey:p2 activeKey:p3 ---UPDATE--> Registry [p2,p4] - // => should update Registry to [p3,p4] (rotate out prevKey:p2, rotate in activeKey:p3) - err = NamespacesPubKeyUpdate(privKey3, []string{prefix}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") - require.NoError(t, err) - ns, err = getNamespaceByPrefix(prefix) - require.NoError(t, err) - - expectedJwks := jwk.NewSet() - pubKey3, err := jwk.PublicKeyOf(privKey3) - require.NoError(t, err) - err = expectedJwks.AddKey(pubKey3) - require.NoError(t, err) - err = expectedJwks.AddKey(pubKey4) - require.NoError(t, err) - expectedJwksBytes, err := json.Marshal(expectedJwks) - require.NoError(t, err) - expectedJwksStr := string(expectedJwksBytes) - - expectedKids, err := getSortedKids(ctx, expectedJwksStr) - require.NoError(t, err) - actualKids, err := getSortedKids(ctx, ns.Pubkey) - require.NoError(t, err) - require.Equal(t, expectedKids, actualKids) -} +// func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { +// ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) +// server_utils.ResetTestState() +// t.Cleanup(func() { +// func() { require.NoError(t, egrp.Wait()) }() +// cancel() +// config.ResetIssuerJWKPtr() +// config.ResetIssuerPrivateKeys() +// server_utils.ResetTestState() +// }) + +// tDir := t.TempDir() + +// svr := registryMockup(ctx, t, "serveregistry") +// defer func() { +// err := ShutdownRegistryDB() +// assert.NoError(t, err) +// svr.CloseClientConnections() +// svr.Close() +// }() + +// config.ResetIssuerJWKPtr() +// config.ResetIssuerPrivateKeys() +// privKeys := config.GetIssuerPrivateKeys() +// require.Len(t, privKeys, 0) + +// // Construct a client that has [p1,p2,p3] and p3 is the active private key +// privKey1, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) +// require.NotEmpty(t, privKey1) +// require.NoError(t, err) +// privKey2, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) +// require.NoError(t, err) + +// prefix := "/mascot/bucky" +// err = NamespaceRegister(privKey2, svr.URL+"/api/v1.0/registry", "", prefix, "mock_site_name") +// require.NoError(t, err) + +// config.UpdatePreviousIssuerPrivateJWK() +// privKey3, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) +// require.NoError(t, err) + +// // Construct a public keys JWKS [p2,p4] to save in registry DB, imitating admin manually adding p4 +// registryDbJwks := jwk.NewSet() +// pubKey2, err := jwk.PublicKeyOf(privKey2) +// require.NoError(t, err) +// err = registryDbJwks.AddKey(pubKey2) +// require.NoError(t, err) +// privKey4, err := config.GeneratePEM(filepath.Join(tDir, "elsewhere")) +// require.NoError(t, err) +// pubKey4, err := jwk.PublicKeyOf(privKey4) +// require.NoError(t, err) +// err = registryDbJwks.AddKey(pubKey4) +// require.NoError(t, err) +// jwksBytes, err := json.Marshal(registryDbJwks) +// require.NoError(t, err) +// jwksStr := string(jwksBytes) + +// // Test functionality of a namespace registered with multi public keys [p2,p4] +// err = setNamespacePubKey(prefix, jwksStr) // set the registered public keys to [p2,p4] +// require.NoError(t, err) +// ns, err := getNamespaceByPrefix(prefix) +// require.NoError(t, err) +// require.Equal(t, jwksStr, ns.Pubkey) + +// prevKey := config.GetPreviousIssuerPrivateJWK() +// require.Equal(t, privKey2.KeyID(), prevKey.KeyID()) +// privKeys = config.GetIssuerPrivateKeys() +// require.Len(t, privKeys, 3) + +// // Client allKeys:[p1,p2,p3] prevKey:p2 activeKey:p3 ---UPDATE--> Registry [p2,p4] +// // => should update Registry to [p3,p4] (rotate out prevKey:p2, rotate in activeKey:p3) +// err = NamespacesPubKeyUpdate(privKey3, []string{prefix}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") +// require.NoError(t, err) +// ns, err = getNamespaceByPrefix(prefix) +// require.NoError(t, err) + +// expectedJwks := jwk.NewSet() +// pubKey3, err := jwk.PublicKeyOf(privKey3) +// require.NoError(t, err) +// err = expectedJwks.AddKey(pubKey3) +// require.NoError(t, err) +// err = expectedJwks.AddKey(pubKey4) +// require.NoError(t, err) +// expectedJwksBytes, err := json.Marshal(expectedJwks) +// require.NoError(t, err) +// expectedJwksStr := string(expectedJwksBytes) + +// expectedKids, err := getSortedKids(ctx, expectedJwksStr) +// require.NoError(t, err) +// actualKids, err := getSortedKids(ctx, ns.Pubkey) +// require.NoError(t, err) +// require.Equal(t, expectedKids, actualKids) +// } func TestRegistryKeyChainingOSDF(t *testing.T) { config.ResetIssuerJWKPtr() From 9d03bd455a441daa4fff5a0d8ce41ab643be2c5c Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 20 Dec 2024 20:14:37 +0000 Subject: [PATCH 53/64] comment out TestServeNamespaceRegistry --- registry/client_commands_test.go | 408 +++++++++++++++---------------- 1 file changed, 203 insertions(+), 205 deletions(-) diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index e614313c7..0a9dbdf48 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -21,11 +21,9 @@ package registry import ( "context" "encoding/json" - "io" - "net/http" "net/http/httptest" - "os" "path/filepath" + "sort" "testing" "github.com/gin-gonic/gin" @@ -67,29 +65,149 @@ func registryMockup(ctx context.Context, t *testing.T, testName string) *httptes return svr } -// func getSortedKids(ctx context.Context, jsonStr string) ([]string, error) { -// set, err := jwk.Parse([]byte(jsonStr)) -// if err != nil { -// return nil, err -// } -// var kids []string -// keysIter := set.Keys(ctx) -// for keysIter.Next(ctx) { -// key := keysIter.Pair().Value.(jwk.Key) - -// kid, ok := key.Get("kid") -// if !ok { -// continue -// } -// kids = append(kids, kid.(string)) -// } -// sort.Strings(kids) - -// return kids, nil +func getSortedKids(ctx context.Context, jsonStr string) ([]string, error) { + set, err := jwk.Parse([]byte(jsonStr)) + if err != nil { + return nil, err + } + var kids []string + keysIter := set.Keys(ctx) + for keysIter.Next(ctx) { + key := keysIter.Pair().Value.(jwk.Key) + + kid, ok := key.Get("kid") + if !ok { + continue + } + kids = append(kids, kid.(string)) + } + sort.Strings(kids) + + return kids, nil +} + +// func TestServeNamespaceRegistry(t *testing.T) { +// ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) +// t.Cleanup(func() { +// func() { require.NoError(t, egrp.Wait()) }() +// cancel() +// config.ResetIssuerJWKPtr() +// config.ResetIssuerPrivateKeys() +// server_utils.ResetTestState() +// }) +// server_utils.ResetTestState() + +// svr := registryMockup(ctx, t, "serveregistry") +// defer func() { +// err := ShutdownRegistryDB() +// assert.NoError(t, err) +// svr.CloseClientConnections() +// svr.Close() +// }() + +// _, err := config.GetIssuerPublicJWKS() +// require.NoError(t, err) +// privKey, err := config.GetIssuerPrivateJWK() +// require.NoError(t, err) + +// //Test functionality of registering a namespace (without identity) +// err = NamespaceRegister(privKey, svr.URL+"/api/v1.0/registry", "", "/foo/bar", "mock_site_name") +// require.NoError(t, err) +// var privKey2 jwk.Key + +// //Test we can list the namespace without an error +// t.Run("Test namespace list", func(t *testing.T) { +// //Set up a buffer to capture stdout +// var stdoutCapture string +// oldStdout := os.Stdout +// r, w, _ := os.Pipe() +// os.Stdout = w + +// //List the namespaces +// err = NamespaceList(svr.URL + "/api/v1.0/registry") +// require.NoError(t, err) +// w.Close() +// os.Stdout = oldStdout + +// capturedOutput := make([]byte, 1024) +// n, _ := r.Read(capturedOutput) +// stdoutCapture = string(capturedOutput[:n]) +// assert.Contains(t, stdoutCapture, `"prefix":"/foo/bar"`) +// }) + +// t.Run("Test register namespace with sitename", func(t *testing.T) { +// res, err := http.Get(svr.URL + "/api/v1.0/registry/foo/bar") +// require.NoError(t, err) +// require.Equal(t, http.StatusOK, res.StatusCode) +// data, err := io.ReadAll(res.Body) +// require.NoError(t, err) +// ns := server_structs.Namespace{} +// err = json.Unmarshal(data, &ns) +// require.NoError(t, err) +// assert.Equal(t, ns.AdminMetadata.SiteName, "mock_site_name") +// }) + +// t.Run("test-registered-namespace-pubkey-update-with-new-active-key", func(t *testing.T) { +// activeKey, err := config.GetIssuerPrivateJWK() +// require.NoError(t, err) +// require.Equal(t, privKey.KeyID(), activeKey.KeyID()) + +// // Imitate LaunchIssuerKeysDirRefresh function +// config.UpdatePreviousIssuerPrivateJWK() +// _, err = config.GeneratePEM(param.IssuerKeysDirectory.GetString()) +// require.NoError(t, err) +// privKey2, err = config.LoadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()) +// require.NoError(t, err) +// err = NamespacesPubKeyUpdate(privKey2, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") +// require.NoError(t, err) +// }) + +// t.Run("test-registered-namespace-pubkey-update-with-nonsense-key", func(t *testing.T) { +// tempDir := filepath.Join(t.TempDir(), "in_the_middle_of_nowhere") +// privKey3, err := config.GeneratePEM(tempDir) +// require.NoError(t, err) +// err = NamespacesPubKeyUpdate(privKey3, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") +// require.ErrorContains(t, err, "it doesn't contain any public key matching the existing namespace's public key in db") +// }) + +// t.Run("test-registered-namespace-pubkey-update-with-imposter-key", func(t *testing.T) { +// privKey4, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) +// require.NoError(t, err) +// config.UpdatePreviousIssuerPrivateJWK() +// // Both active key and previous key are set to privKey4 +// err = NamespacesPubKeyUpdate(privKey4, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") +// require.ErrorContains(t, err, "it fails to pass the proof of possession verification") + +// // Revert the active key changes happened in this subtest +// config.SetActiveKey(privKey) +// config.UpdatePreviousIssuerPrivateJWK() +// config.SetActiveKey(privKey2) +// }) + +// t.Run("Test namespace delete", func(t *testing.T) { +// //Test functionality of namespace delete +// err = NamespaceDelete(svr.URL+"/api/v1.0/registry/foo/bar", "/foo/bar") +// require.NoError(t, err) +// var stdoutCapture string +// oldStdout := os.Stdout +// r, w, _ := os.Pipe() +// os.Stdout = w +// err = NamespaceGet(svr.URL + "/api/v1.0/registry") +// require.NoError(t, err) +// w.Close() +// os.Stdout = oldStdout + +// capturedOutput := make([]byte, 1024) +// n, _ := r.Read(capturedOutput) +// stdoutCapture = string(capturedOutput[:n]) +// assert.Equal(t, "[]\n", stdoutCapture) +// }) +// server_utils.ResetTestState() // } -func TestServeNamespaceRegistry(t *testing.T) { +func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) + server_utils.ResetTestState() t.Cleanup(func() { func() { require.NoError(t, egrp.Wait()) }() cancel() @@ -97,7 +215,8 @@ func TestServeNamespaceRegistry(t *testing.T) { config.ResetIssuerPrivateKeys() server_utils.ResetTestState() }) - server_utils.ResetTestState() + + tDir := t.TempDir() svr := registryMockup(ctx, t, "serveregistry") defer func() { @@ -107,200 +226,79 @@ func TestServeNamespaceRegistry(t *testing.T) { svr.Close() }() - _, err := config.GetIssuerPublicJWKS() + config.ResetIssuerJWKPtr() + config.ResetIssuerPrivateKeys() + privKeys := config.GetIssuerPrivateKeys() + require.Len(t, privKeys, 0) + + // Construct a client that has [p1,p2,p3] and p3 is the active private key + privKey1, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) + require.NotEmpty(t, privKey1) require.NoError(t, err) - privKey, err := config.GetIssuerPrivateJWK() + privKey2, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) require.NoError(t, err) - //Test functionality of registering a namespace (without identity) - err = NamespaceRegister(privKey, svr.URL+"/api/v1.0/registry", "", "/foo/bar", "mock_site_name") + prefix := "/mascot/bucky" + err = NamespaceRegister(privKey2, svr.URL+"/api/v1.0/registry", "", prefix, "mock_site_name") require.NoError(t, err) - var privKey2 jwk.Key - - //Test we can list the namespace without an error - t.Run("Test namespace list", func(t *testing.T) { - //Set up a buffer to capture stdout - var stdoutCapture string - oldStdout := os.Stdout - r, w, _ := os.Pipe() - os.Stdout = w - //List the namespaces - err = NamespaceList(svr.URL + "/api/v1.0/registry") - require.NoError(t, err) - w.Close() - os.Stdout = oldStdout + config.UpdatePreviousIssuerPrivateJWK() + privKey3, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) + require.NoError(t, err) - capturedOutput := make([]byte, 1024) - n, _ := r.Read(capturedOutput) - stdoutCapture = string(capturedOutput[:n]) - assert.Contains(t, stdoutCapture, `"prefix":"/foo/bar"`) - }) + // Construct a public keys JWKS [p2,p4] to save in registry DB, imitating admin manually adding p4 + registryDbJwks := jwk.NewSet() + pubKey2, err := jwk.PublicKeyOf(privKey2) + require.NoError(t, err) + err = registryDbJwks.AddKey(pubKey2) + require.NoError(t, err) + privKey4, err := config.GeneratePEM(filepath.Join(tDir, "elsewhere")) + require.NoError(t, err) + pubKey4, err := jwk.PublicKeyOf(privKey4) + require.NoError(t, err) + err = registryDbJwks.AddKey(pubKey4) + require.NoError(t, err) + jwksBytes, err := json.Marshal(registryDbJwks) + require.NoError(t, err) + jwksStr := string(jwksBytes) - t.Run("Test register namespace with sitename", func(t *testing.T) { - res, err := http.Get(svr.URL + "/api/v1.0/registry/foo/bar") - require.NoError(t, err) - require.Equal(t, http.StatusOK, res.StatusCode) - data, err := io.ReadAll(res.Body) - require.NoError(t, err) - ns := server_structs.Namespace{} - err = json.Unmarshal(data, &ns) - require.NoError(t, err) - assert.Equal(t, ns.AdminMetadata.SiteName, "mock_site_name") - }) + // Test functionality of a namespace registered with multi public keys [p2,p4] + err = setNamespacePubKey(prefix, jwksStr) // set the registered public keys to [p2,p4] + require.NoError(t, err) + ns, err := getNamespaceByPrefix(prefix) + require.NoError(t, err) + require.Equal(t, jwksStr, ns.Pubkey) - t.Run("test-registered-namespace-pubkey-update-with-new-active-key", func(t *testing.T) { - activeKey, err := config.GetIssuerPrivateJWK() - require.NoError(t, err) - require.Equal(t, privKey.KeyID(), activeKey.KeyID()) - - // Imitate LaunchIssuerKeysDirRefresh function - config.UpdatePreviousIssuerPrivateJWK() - _, err = config.GeneratePEM(param.IssuerKeysDirectory.GetString()) - require.NoError(t, err) - privKey2, err = config.LoadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()) - require.NoError(t, err) - err = NamespacesPubKeyUpdate(privKey2, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") - require.NoError(t, err) - }) + prevKey := config.GetPreviousIssuerPrivateJWK() + require.Equal(t, privKey2.KeyID(), prevKey.KeyID()) + privKeys = config.GetIssuerPrivateKeys() + require.Len(t, privKeys, 3) - t.Run("test-registered-namespace-pubkey-update-with-nonsense-key", func(t *testing.T) { - tempDir := filepath.Join(t.TempDir(), "in_the_middle_of_nowhere") - privKey3, err := config.GeneratePEM(tempDir) - require.NoError(t, err) - err = NamespacesPubKeyUpdate(privKey3, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") - require.ErrorContains(t, err, "it doesn't contain any public key matching the existing namespace's public key in db") - }) + // Client allKeys:[p1,p2,p3] prevKey:p2 activeKey:p3 ---UPDATE--> Registry [p2,p4] + // => should update Registry to [p3,p4] (rotate out prevKey:p2, rotate in activeKey:p3) + err = NamespacesPubKeyUpdate(privKey3, []string{prefix}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") + require.NoError(t, err) + ns, err = getNamespaceByPrefix(prefix) + require.NoError(t, err) - t.Run("test-registered-namespace-pubkey-update-with-imposter-key", func(t *testing.T) { - privKey4, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) - require.NoError(t, err) - config.UpdatePreviousIssuerPrivateJWK() - // Both active key and previous key are set to privKey4 - err = NamespacesPubKeyUpdate(privKey4, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") - require.ErrorContains(t, err, "it fails to pass the proof of possession verification") - - // Revert the active key changes happened in this subtest - config.SetActiveKey(privKey) - config.UpdatePreviousIssuerPrivateJWK() - config.SetActiveKey(privKey2) - }) + expectedJwks := jwk.NewSet() + pubKey3, err := jwk.PublicKeyOf(privKey3) + require.NoError(t, err) + err = expectedJwks.AddKey(pubKey3) + require.NoError(t, err) + err = expectedJwks.AddKey(pubKey4) + require.NoError(t, err) + expectedJwksBytes, err := json.Marshal(expectedJwks) + require.NoError(t, err) + expectedJwksStr := string(expectedJwksBytes) - t.Run("Test namespace delete", func(t *testing.T) { - //Test functionality of namespace delete - err = NamespaceDelete(svr.URL+"/api/v1.0/registry/foo/bar", "/foo/bar") - require.NoError(t, err) - var stdoutCapture string - oldStdout := os.Stdout - r, w, _ := os.Pipe() - os.Stdout = w - err = NamespaceGet(svr.URL + "/api/v1.0/registry") - require.NoError(t, err) - w.Close() - os.Stdout = oldStdout - - capturedOutput := make([]byte, 1024) - n, _ := r.Read(capturedOutput) - stdoutCapture = string(capturedOutput[:n]) - assert.Equal(t, "[]\n", stdoutCapture) - }) - server_utils.ResetTestState() + expectedKids, err := getSortedKids(ctx, expectedJwksStr) + require.NoError(t, err) + actualKids, err := getSortedKids(ctx, ns.Pubkey) + require.NoError(t, err) + require.Equal(t, expectedKids, actualKids) } -// func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { -// ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) -// server_utils.ResetTestState() -// t.Cleanup(func() { -// func() { require.NoError(t, egrp.Wait()) }() -// cancel() -// config.ResetIssuerJWKPtr() -// config.ResetIssuerPrivateKeys() -// server_utils.ResetTestState() -// }) - -// tDir := t.TempDir() - -// svr := registryMockup(ctx, t, "serveregistry") -// defer func() { -// err := ShutdownRegistryDB() -// assert.NoError(t, err) -// svr.CloseClientConnections() -// svr.Close() -// }() - -// config.ResetIssuerJWKPtr() -// config.ResetIssuerPrivateKeys() -// privKeys := config.GetIssuerPrivateKeys() -// require.Len(t, privKeys, 0) - -// // Construct a client that has [p1,p2,p3] and p3 is the active private key -// privKey1, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) -// require.NotEmpty(t, privKey1) -// require.NoError(t, err) -// privKey2, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) -// require.NoError(t, err) - -// prefix := "/mascot/bucky" -// err = NamespaceRegister(privKey2, svr.URL+"/api/v1.0/registry", "", prefix, "mock_site_name") -// require.NoError(t, err) - -// config.UpdatePreviousIssuerPrivateJWK() -// privKey3, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) -// require.NoError(t, err) - -// // Construct a public keys JWKS [p2,p4] to save in registry DB, imitating admin manually adding p4 -// registryDbJwks := jwk.NewSet() -// pubKey2, err := jwk.PublicKeyOf(privKey2) -// require.NoError(t, err) -// err = registryDbJwks.AddKey(pubKey2) -// require.NoError(t, err) -// privKey4, err := config.GeneratePEM(filepath.Join(tDir, "elsewhere")) -// require.NoError(t, err) -// pubKey4, err := jwk.PublicKeyOf(privKey4) -// require.NoError(t, err) -// err = registryDbJwks.AddKey(pubKey4) -// require.NoError(t, err) -// jwksBytes, err := json.Marshal(registryDbJwks) -// require.NoError(t, err) -// jwksStr := string(jwksBytes) - -// // Test functionality of a namespace registered with multi public keys [p2,p4] -// err = setNamespacePubKey(prefix, jwksStr) // set the registered public keys to [p2,p4] -// require.NoError(t, err) -// ns, err := getNamespaceByPrefix(prefix) -// require.NoError(t, err) -// require.Equal(t, jwksStr, ns.Pubkey) - -// prevKey := config.GetPreviousIssuerPrivateJWK() -// require.Equal(t, privKey2.KeyID(), prevKey.KeyID()) -// privKeys = config.GetIssuerPrivateKeys() -// require.Len(t, privKeys, 3) - -// // Client allKeys:[p1,p2,p3] prevKey:p2 activeKey:p3 ---UPDATE--> Registry [p2,p4] -// // => should update Registry to [p3,p4] (rotate out prevKey:p2, rotate in activeKey:p3) -// err = NamespacesPubKeyUpdate(privKey3, []string{prefix}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") -// require.NoError(t, err) -// ns, err = getNamespaceByPrefix(prefix) -// require.NoError(t, err) - -// expectedJwks := jwk.NewSet() -// pubKey3, err := jwk.PublicKeyOf(privKey3) -// require.NoError(t, err) -// err = expectedJwks.AddKey(pubKey3) -// require.NoError(t, err) -// err = expectedJwks.AddKey(pubKey4) -// require.NoError(t, err) -// expectedJwksBytes, err := json.Marshal(expectedJwks) -// require.NoError(t, err) -// expectedJwksStr := string(expectedJwksBytes) - -// expectedKids, err := getSortedKids(ctx, expectedJwksStr) -// require.NoError(t, err) -// actualKids, err := getSortedKids(ctx, ns.Pubkey) -// require.NoError(t, err) -// require.Equal(t, expectedKids, actualKids) -// } - func TestRegistryKeyChainingOSDF(t *testing.T) { config.ResetIssuerJWKPtr() config.ResetIssuerPrivateKeys() From 08d30447626630bdf3b8870040ca035632574030 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 20 Dec 2024 20:23:13 +0000 Subject: [PATCH 54/64] comment out both TestServeNamespaceRegistry and TestMultiPubKeysRegisteredOnNamespace --- registry/client_commands_test.go | 206 +++++++++++++++---------------- 1 file changed, 101 insertions(+), 105 deletions(-) diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index 0a9dbdf48..c206be50c 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -20,20 +20,16 @@ package registry import ( "context" - "encoding/json" "net/http/httptest" "path/filepath" - "sort" "testing" "github.com/gin-gonic/gin" - "github.com/lestrrat-go/jwx/v2/jwk" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/pelicanplatform/pelican/config" - "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/server_structs" "github.com/pelicanplatform/pelican/server_utils" "github.com/pelicanplatform/pelican/test_utils" @@ -65,26 +61,26 @@ func registryMockup(ctx context.Context, t *testing.T, testName string) *httptes return svr } -func getSortedKids(ctx context.Context, jsonStr string) ([]string, error) { - set, err := jwk.Parse([]byte(jsonStr)) - if err != nil { - return nil, err - } - var kids []string - keysIter := set.Keys(ctx) - for keysIter.Next(ctx) { - key := keysIter.Pair().Value.(jwk.Key) - - kid, ok := key.Get("kid") - if !ok { - continue - } - kids = append(kids, kid.(string)) - } - sort.Strings(kids) - - return kids, nil -} +// func getSortedKids(ctx context.Context, jsonStr string) ([]string, error) { +// set, err := jwk.Parse([]byte(jsonStr)) +// if err != nil { +// return nil, err +// } +// var kids []string +// keysIter := set.Keys(ctx) +// for keysIter.Next(ctx) { +// key := keysIter.Pair().Value.(jwk.Key) + +// kid, ok := key.Get("kid") +// if !ok { +// continue +// } +// kids = append(kids, kid.(string)) +// } +// sort.Strings(kids) + +// return kids, nil +// } // func TestServeNamespaceRegistry(t *testing.T) { // ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) @@ -205,99 +201,99 @@ func getSortedKids(ctx context.Context, jsonStr string) ([]string, error) { // server_utils.ResetTestState() // } -func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { - ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) - server_utils.ResetTestState() - t.Cleanup(func() { - func() { require.NoError(t, egrp.Wait()) }() - cancel() - config.ResetIssuerJWKPtr() - config.ResetIssuerPrivateKeys() - server_utils.ResetTestState() - }) +// func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { +// ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) +// server_utils.ResetTestState() +// t.Cleanup(func() { +// func() { require.NoError(t, egrp.Wait()) }() +// cancel() +// config.ResetIssuerJWKPtr() +// config.ResetIssuerPrivateKeys() +// server_utils.ResetTestState() +// }) - tDir := t.TempDir() +// tDir := t.TempDir() - svr := registryMockup(ctx, t, "serveregistry") - defer func() { - err := ShutdownRegistryDB() - assert.NoError(t, err) - svr.CloseClientConnections() - svr.Close() - }() +// svr := registryMockup(ctx, t, "serveregistry") +// defer func() { +// err := ShutdownRegistryDB() +// assert.NoError(t, err) +// svr.CloseClientConnections() +// svr.Close() +// }() - config.ResetIssuerJWKPtr() - config.ResetIssuerPrivateKeys() - privKeys := config.GetIssuerPrivateKeys() - require.Len(t, privKeys, 0) +// config.ResetIssuerJWKPtr() +// config.ResetIssuerPrivateKeys() +// privKeys := config.GetIssuerPrivateKeys() +// require.Len(t, privKeys, 0) - // Construct a client that has [p1,p2,p3] and p3 is the active private key - privKey1, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) - require.NotEmpty(t, privKey1) - require.NoError(t, err) - privKey2, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) - require.NoError(t, err) +// // Construct a client that has [p1,p2,p3] and p3 is the active private key +// privKey1, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) +// require.NotEmpty(t, privKey1) +// require.NoError(t, err) +// privKey2, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) +// require.NoError(t, err) - prefix := "/mascot/bucky" - err = NamespaceRegister(privKey2, svr.URL+"/api/v1.0/registry", "", prefix, "mock_site_name") - require.NoError(t, err) +// prefix := "/mascot/bucky" +// err = NamespaceRegister(privKey2, svr.URL+"/api/v1.0/registry", "", prefix, "mock_site_name") +// require.NoError(t, err) - config.UpdatePreviousIssuerPrivateJWK() - privKey3, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) - require.NoError(t, err) +// config.UpdatePreviousIssuerPrivateJWK() +// privKey3, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) +// require.NoError(t, err) - // Construct a public keys JWKS [p2,p4] to save in registry DB, imitating admin manually adding p4 - registryDbJwks := jwk.NewSet() - pubKey2, err := jwk.PublicKeyOf(privKey2) - require.NoError(t, err) - err = registryDbJwks.AddKey(pubKey2) - require.NoError(t, err) - privKey4, err := config.GeneratePEM(filepath.Join(tDir, "elsewhere")) - require.NoError(t, err) - pubKey4, err := jwk.PublicKeyOf(privKey4) - require.NoError(t, err) - err = registryDbJwks.AddKey(pubKey4) - require.NoError(t, err) - jwksBytes, err := json.Marshal(registryDbJwks) - require.NoError(t, err) - jwksStr := string(jwksBytes) +// // Construct a public keys JWKS [p2,p4] to save in registry DB, imitating admin manually adding p4 +// registryDbJwks := jwk.NewSet() +// pubKey2, err := jwk.PublicKeyOf(privKey2) +// require.NoError(t, err) +// err = registryDbJwks.AddKey(pubKey2) +// require.NoError(t, err) +// privKey4, err := config.GeneratePEM(filepath.Join(tDir, "elsewhere")) +// require.NoError(t, err) +// pubKey4, err := jwk.PublicKeyOf(privKey4) +// require.NoError(t, err) +// err = registryDbJwks.AddKey(pubKey4) +// require.NoError(t, err) +// jwksBytes, err := json.Marshal(registryDbJwks) +// require.NoError(t, err) +// jwksStr := string(jwksBytes) - // Test functionality of a namespace registered with multi public keys [p2,p4] - err = setNamespacePubKey(prefix, jwksStr) // set the registered public keys to [p2,p4] - require.NoError(t, err) - ns, err := getNamespaceByPrefix(prefix) - require.NoError(t, err) - require.Equal(t, jwksStr, ns.Pubkey) +// // Test functionality of a namespace registered with multi public keys [p2,p4] +// err = setNamespacePubKey(prefix, jwksStr) // set the registered public keys to [p2,p4] +// require.NoError(t, err) +// ns, err := getNamespaceByPrefix(prefix) +// require.NoError(t, err) +// require.Equal(t, jwksStr, ns.Pubkey) - prevKey := config.GetPreviousIssuerPrivateJWK() - require.Equal(t, privKey2.KeyID(), prevKey.KeyID()) - privKeys = config.GetIssuerPrivateKeys() - require.Len(t, privKeys, 3) +// prevKey := config.GetPreviousIssuerPrivateJWK() +// require.Equal(t, privKey2.KeyID(), prevKey.KeyID()) +// privKeys = config.GetIssuerPrivateKeys() +// require.Len(t, privKeys, 3) - // Client allKeys:[p1,p2,p3] prevKey:p2 activeKey:p3 ---UPDATE--> Registry [p2,p4] - // => should update Registry to [p3,p4] (rotate out prevKey:p2, rotate in activeKey:p3) - err = NamespacesPubKeyUpdate(privKey3, []string{prefix}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") - require.NoError(t, err) - ns, err = getNamespaceByPrefix(prefix) - require.NoError(t, err) +// // Client allKeys:[p1,p2,p3] prevKey:p2 activeKey:p3 ---UPDATE--> Registry [p2,p4] +// // => should update Registry to [p3,p4] (rotate out prevKey:p2, rotate in activeKey:p3) +// err = NamespacesPubKeyUpdate(privKey3, []string{prefix}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") +// require.NoError(t, err) +// ns, err = getNamespaceByPrefix(prefix) +// require.NoError(t, err) - expectedJwks := jwk.NewSet() - pubKey3, err := jwk.PublicKeyOf(privKey3) - require.NoError(t, err) - err = expectedJwks.AddKey(pubKey3) - require.NoError(t, err) - err = expectedJwks.AddKey(pubKey4) - require.NoError(t, err) - expectedJwksBytes, err := json.Marshal(expectedJwks) - require.NoError(t, err) - expectedJwksStr := string(expectedJwksBytes) +// expectedJwks := jwk.NewSet() +// pubKey3, err := jwk.PublicKeyOf(privKey3) +// require.NoError(t, err) +// err = expectedJwks.AddKey(pubKey3) +// require.NoError(t, err) +// err = expectedJwks.AddKey(pubKey4) +// require.NoError(t, err) +// expectedJwksBytes, err := json.Marshal(expectedJwks) +// require.NoError(t, err) +// expectedJwksStr := string(expectedJwksBytes) - expectedKids, err := getSortedKids(ctx, expectedJwksStr) - require.NoError(t, err) - actualKids, err := getSortedKids(ctx, ns.Pubkey) - require.NoError(t, err) - require.Equal(t, expectedKids, actualKids) -} +// expectedKids, err := getSortedKids(ctx, expectedJwksStr) +// require.NoError(t, err) +// actualKids, err := getSortedKids(ctx, ns.Pubkey) +// require.NoError(t, err) +// require.Equal(t, expectedKids, actualKids) +// } func TestRegistryKeyChainingOSDF(t *testing.T) { config.ResetIssuerJWKPtr() From 6809e1d97fec4dacf69b779f6456a3d192e73ded Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 20 Dec 2024 22:04:05 +0000 Subject: [PATCH 55/64] test improvement --- config/config.go | 4 ++++ config/init_server_creds.go | 8 ++++++++ registry/client_commands_test.go | 25 ++++++++++--------------- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/config/config.go b/config/config.go index cf7c94f6a..abea37432 100644 --- a/config/config.go +++ b/config/config.go @@ -1641,6 +1641,10 @@ func ResetConfig() { globalFedErr = nil ResetIssuerJWKPtr() + ResetIssuerPrivateKeys() + ResetPreviousIssuerPrivateJWK() + ResetCurrentIssuerKeysDir() + ResetClientInitialized() // other than what's above, resetting Origin exports will be done by ResetTestState() in server_utils pkg diff --git a/config/init_server_creds.go b/config/init_server_creds.go index 7fb09463a..701751002 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -84,6 +84,10 @@ func UpdatePreviousIssuerPrivateJWK() { } } +func ResetPreviousIssuerPrivateJWK() { + previousIssuerPrivateJWK.Store(nil) +} + // Set a private key as the active private key in use func SetActiveKey(key jwk.Key) { issuerPrivateJWK.Store(&key) @@ -124,6 +128,10 @@ func setCurrentIssuerKeysDir(dir string) { currentIssuerKeysDir.Store(dir) } +func ResetCurrentIssuerKeysDir() { + currentIssuerKeysDir.Store(nil) +} + // Helper function to generate random string func createRandomString(length int) string { const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index c206be50c..8708d0e9d 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -296,24 +296,17 @@ func registryMockup(ctx context.Context, t *testing.T, testName string) *httptes // } func TestRegistryKeyChainingOSDF(t *testing.T) { - config.ResetIssuerJWKPtr() - config.ResetIssuerPrivateKeys() + server_utils.ResetTestState() + ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) t.Cleanup(func() { - func() { require.NoError(t, egrp.Wait()) }() + func() { assert.NoError(t, egrp.Wait()) }() cancel() - config.ResetIssuerJWKPtr() - config.ResetIssuerPrivateKeys() server_utils.ResetTestState() }) - server_utils.ResetTestState() _, err := config.SetPreferredPrefix(config.OsdfPrefix) assert.NoError(t, err) - viper.Set("Federation.DirectorUrl", "https://osdf-director.osg-htc.org") - viper.Set("Federation.RegistryUrl", "https://osdf-registry.osg-htc.org") - viper.Set("Federation.JwkUrl", "https://osg-htc.org/osdf/public_signing_key.jwks") - viper.Set("Federation.BrokerUrl", "https://osdf-director.osg-htc.org") // On by default, but just to make things explicit viper.Set("Registry.RequireKeyChaining", true) @@ -401,11 +394,15 @@ func TestRegistryKeyChainingOSDF(t *testing.T) { } func TestRegistryKeyChaining(t *testing.T) { + server_utils.ResetTestState() + ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) - defer func() { require.NoError(t, egrp.Wait()) }() - defer cancel() + t.Cleanup(func() { + func() { assert.NoError(t, egrp.Wait()) }() + cancel() + server_utils.ResetTestState() + }) - server_utils.ResetTestState() // On by default, but just to make things explicit viper.Set("Registry.RequireKeyChaining", true) @@ -455,6 +452,4 @@ func TestRegistryKeyChaining(t *testing.T) { err = NamespaceRegister(privKey, registrySvr.URL+"/api/v1.0/registry", "", "/foo", "") require.NoError(t, err) - - server_utils.ResetTestState() } From 2c1ebd3a8feef7b8eafc3f54ae2da0091b3ebd9e Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 20 Dec 2024 22:21:04 +0000 Subject: [PATCH 56/64] fix ResetCurrentIssuerKeysDir --- config/init_server_creds.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/init_server_creds.go b/config/init_server_creds.go index 701751002..88ffd6371 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -129,7 +129,7 @@ func setCurrentIssuerKeysDir(dir string) { } func ResetCurrentIssuerKeysDir() { - currentIssuerKeysDir.Store(nil) + currentIssuerKeysDir.Store("") } // Helper function to generate random string From ba42296842d05f94e9ca2bf965539c1c56e2d2d3 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 20 Dec 2024 22:48:56 +0000 Subject: [PATCH 57/64] revert target tests in test-template.yml --- .github/workflows/test-template.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-template.yml b/.github/workflows/test-template.yml index 55b3723da..f697a3fe6 100644 --- a/.github/workflows/test-template.yml +++ b/.github/workflows/test-template.yml @@ -42,7 +42,7 @@ jobs: - name: Test run: | make web-build - go test -tags=${{ inputs.tags }} -timeout 5m -coverpkg=./... -coverprofile=${{ inputs.coverprofile }} -covermode=count ./registry/registry_db_test.go ./registry/client_commands_test.go + go test -tags=${{ inputs.tags }} -timeout 5m -coverpkg=./... -coverprofile=${{ inputs.coverprofile }} -covermode=count ./... - name: Get total code coverage if: github.event_name == 'pull_request' id: cc @@ -67,7 +67,7 @@ jobs: git reset --hard ${{ github.event.pull_request.base.sha }} make web-build go generate ./... - go test -tags=${{ inputs.tags }} -timeout 5m -coverpkg=./... -coverprofile=base_coverage.out -covermode=count ./registry/registry_db_test.go ./registry/client_commands_test.go + go test -tags=${{ inputs.tags }} -timeout 5m -coverpkg=./... -coverprofile=base_coverage.out -covermode=count ./... go tool cover -func=base_coverage.out > unit-base.txt git reset --hard $HEAD - name: Get base code coverage value From 7280e9da55ec2e4e8a500860631e6e371fb558cd Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Sat, 21 Dec 2024 11:17:15 -0600 Subject: [PATCH 58/64] Avoid deadlock when waiting on exit The current code had the logic reversed -- first it waited for completion, then it cancelled the running goroutines. --- registry/client_commands_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index 8708d0e9d..a54261489 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -300,8 +300,8 @@ func TestRegistryKeyChainingOSDF(t *testing.T) { ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) t.Cleanup(func() { - func() { assert.NoError(t, egrp.Wait()) }() cancel() + assert.NoError(t, egrp.Wait()) server_utils.ResetTestState() }) @@ -398,8 +398,8 @@ func TestRegistryKeyChaining(t *testing.T) { ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) t.Cleanup(func() { - func() { assert.NoError(t, egrp.Wait()) }() cancel() + assert.NoError(t, egrp.Wait()) server_utils.ResetTestState() }) From b6593cea5d5e183c8bf440674348fe6eed48d546 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Mon, 23 Dec 2024 15:53:11 +0000 Subject: [PATCH 59/64] bring back TestServeNamespaceRegistry and TestMultiPubKeysRegisteredOnNamespace --- registry/client_commands_test.go | 473 ++++++++++++++++--------------- 1 file changed, 240 insertions(+), 233 deletions(-) diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index a54261489..a109f36a3 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -20,16 +20,23 @@ package registry import ( "context" + "encoding/json" + "io" + "net/http" "net/http/httptest" + "os" "path/filepath" + "sort" "testing" "github.com/gin-gonic/gin" + "github.com/lestrrat-go/jwx/v2/jwk" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/pelicanplatform/pelican/config" + "github.com/pelicanplatform/pelican/param" "github.com/pelicanplatform/pelican/server_structs" "github.com/pelicanplatform/pelican/server_utils" "github.com/pelicanplatform/pelican/test_utils" @@ -61,239 +68,239 @@ func registryMockup(ctx context.Context, t *testing.T, testName string) *httptes return svr } -// func getSortedKids(ctx context.Context, jsonStr string) ([]string, error) { -// set, err := jwk.Parse([]byte(jsonStr)) -// if err != nil { -// return nil, err -// } -// var kids []string -// keysIter := set.Keys(ctx) -// for keysIter.Next(ctx) { -// key := keysIter.Pair().Value.(jwk.Key) - -// kid, ok := key.Get("kid") -// if !ok { -// continue -// } -// kids = append(kids, kid.(string)) -// } -// sort.Strings(kids) - -// return kids, nil -// } - -// func TestServeNamespaceRegistry(t *testing.T) { -// ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) -// t.Cleanup(func() { -// func() { require.NoError(t, egrp.Wait()) }() -// cancel() -// config.ResetIssuerJWKPtr() -// config.ResetIssuerPrivateKeys() -// server_utils.ResetTestState() -// }) -// server_utils.ResetTestState() - -// svr := registryMockup(ctx, t, "serveregistry") -// defer func() { -// err := ShutdownRegistryDB() -// assert.NoError(t, err) -// svr.CloseClientConnections() -// svr.Close() -// }() - -// _, err := config.GetIssuerPublicJWKS() -// require.NoError(t, err) -// privKey, err := config.GetIssuerPrivateJWK() -// require.NoError(t, err) - -// //Test functionality of registering a namespace (without identity) -// err = NamespaceRegister(privKey, svr.URL+"/api/v1.0/registry", "", "/foo/bar", "mock_site_name") -// require.NoError(t, err) -// var privKey2 jwk.Key - -// //Test we can list the namespace without an error -// t.Run("Test namespace list", func(t *testing.T) { -// //Set up a buffer to capture stdout -// var stdoutCapture string -// oldStdout := os.Stdout -// r, w, _ := os.Pipe() -// os.Stdout = w - -// //List the namespaces -// err = NamespaceList(svr.URL + "/api/v1.0/registry") -// require.NoError(t, err) -// w.Close() -// os.Stdout = oldStdout - -// capturedOutput := make([]byte, 1024) -// n, _ := r.Read(capturedOutput) -// stdoutCapture = string(capturedOutput[:n]) -// assert.Contains(t, stdoutCapture, `"prefix":"/foo/bar"`) -// }) - -// t.Run("Test register namespace with sitename", func(t *testing.T) { -// res, err := http.Get(svr.URL + "/api/v1.0/registry/foo/bar") -// require.NoError(t, err) -// require.Equal(t, http.StatusOK, res.StatusCode) -// data, err := io.ReadAll(res.Body) -// require.NoError(t, err) -// ns := server_structs.Namespace{} -// err = json.Unmarshal(data, &ns) -// require.NoError(t, err) -// assert.Equal(t, ns.AdminMetadata.SiteName, "mock_site_name") -// }) - -// t.Run("test-registered-namespace-pubkey-update-with-new-active-key", func(t *testing.T) { -// activeKey, err := config.GetIssuerPrivateJWK() -// require.NoError(t, err) -// require.Equal(t, privKey.KeyID(), activeKey.KeyID()) - -// // Imitate LaunchIssuerKeysDirRefresh function -// config.UpdatePreviousIssuerPrivateJWK() -// _, err = config.GeneratePEM(param.IssuerKeysDirectory.GetString()) -// require.NoError(t, err) -// privKey2, err = config.LoadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()) -// require.NoError(t, err) -// err = NamespacesPubKeyUpdate(privKey2, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") -// require.NoError(t, err) -// }) - -// t.Run("test-registered-namespace-pubkey-update-with-nonsense-key", func(t *testing.T) { -// tempDir := filepath.Join(t.TempDir(), "in_the_middle_of_nowhere") -// privKey3, err := config.GeneratePEM(tempDir) -// require.NoError(t, err) -// err = NamespacesPubKeyUpdate(privKey3, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") -// require.ErrorContains(t, err, "it doesn't contain any public key matching the existing namespace's public key in db") -// }) - -// t.Run("test-registered-namespace-pubkey-update-with-imposter-key", func(t *testing.T) { -// privKey4, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) -// require.NoError(t, err) -// config.UpdatePreviousIssuerPrivateJWK() -// // Both active key and previous key are set to privKey4 -// err = NamespacesPubKeyUpdate(privKey4, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") -// require.ErrorContains(t, err, "it fails to pass the proof of possession verification") - -// // Revert the active key changes happened in this subtest -// config.SetActiveKey(privKey) -// config.UpdatePreviousIssuerPrivateJWK() -// config.SetActiveKey(privKey2) -// }) - -// t.Run("Test namespace delete", func(t *testing.T) { -// //Test functionality of namespace delete -// err = NamespaceDelete(svr.URL+"/api/v1.0/registry/foo/bar", "/foo/bar") -// require.NoError(t, err) -// var stdoutCapture string -// oldStdout := os.Stdout -// r, w, _ := os.Pipe() -// os.Stdout = w -// err = NamespaceGet(svr.URL + "/api/v1.0/registry") -// require.NoError(t, err) -// w.Close() -// os.Stdout = oldStdout - -// capturedOutput := make([]byte, 1024) -// n, _ := r.Read(capturedOutput) -// stdoutCapture = string(capturedOutput[:n]) -// assert.Equal(t, "[]\n", stdoutCapture) -// }) -// server_utils.ResetTestState() -// } - -// func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { -// ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) -// server_utils.ResetTestState() -// t.Cleanup(func() { -// func() { require.NoError(t, egrp.Wait()) }() -// cancel() -// config.ResetIssuerJWKPtr() -// config.ResetIssuerPrivateKeys() -// server_utils.ResetTestState() -// }) - -// tDir := t.TempDir() - -// svr := registryMockup(ctx, t, "serveregistry") -// defer func() { -// err := ShutdownRegistryDB() -// assert.NoError(t, err) -// svr.CloseClientConnections() -// svr.Close() -// }() - -// config.ResetIssuerJWKPtr() -// config.ResetIssuerPrivateKeys() -// privKeys := config.GetIssuerPrivateKeys() -// require.Len(t, privKeys, 0) - -// // Construct a client that has [p1,p2,p3] and p3 is the active private key -// privKey1, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) -// require.NotEmpty(t, privKey1) -// require.NoError(t, err) -// privKey2, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) -// require.NoError(t, err) - -// prefix := "/mascot/bucky" -// err = NamespaceRegister(privKey2, svr.URL+"/api/v1.0/registry", "", prefix, "mock_site_name") -// require.NoError(t, err) - -// config.UpdatePreviousIssuerPrivateJWK() -// privKey3, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) -// require.NoError(t, err) - -// // Construct a public keys JWKS [p2,p4] to save in registry DB, imitating admin manually adding p4 -// registryDbJwks := jwk.NewSet() -// pubKey2, err := jwk.PublicKeyOf(privKey2) -// require.NoError(t, err) -// err = registryDbJwks.AddKey(pubKey2) -// require.NoError(t, err) -// privKey4, err := config.GeneratePEM(filepath.Join(tDir, "elsewhere")) -// require.NoError(t, err) -// pubKey4, err := jwk.PublicKeyOf(privKey4) -// require.NoError(t, err) -// err = registryDbJwks.AddKey(pubKey4) -// require.NoError(t, err) -// jwksBytes, err := json.Marshal(registryDbJwks) -// require.NoError(t, err) -// jwksStr := string(jwksBytes) - -// // Test functionality of a namespace registered with multi public keys [p2,p4] -// err = setNamespacePubKey(prefix, jwksStr) // set the registered public keys to [p2,p4] -// require.NoError(t, err) -// ns, err := getNamespaceByPrefix(prefix) -// require.NoError(t, err) -// require.Equal(t, jwksStr, ns.Pubkey) - -// prevKey := config.GetPreviousIssuerPrivateJWK() -// require.Equal(t, privKey2.KeyID(), prevKey.KeyID()) -// privKeys = config.GetIssuerPrivateKeys() -// require.Len(t, privKeys, 3) - -// // Client allKeys:[p1,p2,p3] prevKey:p2 activeKey:p3 ---UPDATE--> Registry [p2,p4] -// // => should update Registry to [p3,p4] (rotate out prevKey:p2, rotate in activeKey:p3) -// err = NamespacesPubKeyUpdate(privKey3, []string{prefix}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") -// require.NoError(t, err) -// ns, err = getNamespaceByPrefix(prefix) -// require.NoError(t, err) - -// expectedJwks := jwk.NewSet() -// pubKey3, err := jwk.PublicKeyOf(privKey3) -// require.NoError(t, err) -// err = expectedJwks.AddKey(pubKey3) -// require.NoError(t, err) -// err = expectedJwks.AddKey(pubKey4) -// require.NoError(t, err) -// expectedJwksBytes, err := json.Marshal(expectedJwks) -// require.NoError(t, err) -// expectedJwksStr := string(expectedJwksBytes) - -// expectedKids, err := getSortedKids(ctx, expectedJwksStr) -// require.NoError(t, err) -// actualKids, err := getSortedKids(ctx, ns.Pubkey) -// require.NoError(t, err) -// require.Equal(t, expectedKids, actualKids) -// } +func getSortedKids(ctx context.Context, jsonStr string) ([]string, error) { + set, err := jwk.Parse([]byte(jsonStr)) + if err != nil { + return nil, err + } + var kids []string + keysIter := set.Keys(ctx) + for keysIter.Next(ctx) { + key := keysIter.Pair().Value.(jwk.Key) + + kid, ok := key.Get("kid") + if !ok { + continue + } + kids = append(kids, kid.(string)) + } + sort.Strings(kids) + + return kids, nil +} + +func TestServeNamespaceRegistry(t *testing.T) { + ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) + t.Cleanup(func() { + func() { require.NoError(t, egrp.Wait()) }() + cancel() + config.ResetIssuerJWKPtr() + config.ResetIssuerPrivateKeys() + server_utils.ResetTestState() + }) + server_utils.ResetTestState() + + svr := registryMockup(ctx, t, "serveregistry") + defer func() { + err := ShutdownRegistryDB() + assert.NoError(t, err) + svr.CloseClientConnections() + svr.Close() + }() + + _, err := config.GetIssuerPublicJWKS() + require.NoError(t, err) + privKey, err := config.GetIssuerPrivateJWK() + require.NoError(t, err) + + //Test functionality of registering a namespace (without identity) + err = NamespaceRegister(privKey, svr.URL+"/api/v1.0/registry", "", "/foo/bar", "mock_site_name") + require.NoError(t, err) + var privKey2 jwk.Key + + //Test we can list the namespace without an error + t.Run("Test namespace list", func(t *testing.T) { + //Set up a buffer to capture stdout + var stdoutCapture string + oldStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + + //List the namespaces + err = NamespaceList(svr.URL + "/api/v1.0/registry") + require.NoError(t, err) + w.Close() + os.Stdout = oldStdout + + capturedOutput := make([]byte, 1024) + n, _ := r.Read(capturedOutput) + stdoutCapture = string(capturedOutput[:n]) + assert.Contains(t, stdoutCapture, `"prefix":"/foo/bar"`) + }) + + t.Run("Test register namespace with sitename", func(t *testing.T) { + res, err := http.Get(svr.URL + "/api/v1.0/registry/foo/bar") + require.NoError(t, err) + require.Equal(t, http.StatusOK, res.StatusCode) + data, err := io.ReadAll(res.Body) + require.NoError(t, err) + ns := server_structs.Namespace{} + err = json.Unmarshal(data, &ns) + require.NoError(t, err) + assert.Equal(t, ns.AdminMetadata.SiteName, "mock_site_name") + }) + + t.Run("test-registered-namespace-pubkey-update-with-new-active-key", func(t *testing.T) { + activeKey, err := config.GetIssuerPrivateJWK() + require.NoError(t, err) + require.Equal(t, privKey.KeyID(), activeKey.KeyID()) + + // Imitate LaunchIssuerKeysDirRefresh function + config.UpdatePreviousIssuerPrivateJWK() + _, err = config.GeneratePEM(param.IssuerKeysDirectory.GetString()) + require.NoError(t, err) + privKey2, err = config.LoadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()) + require.NoError(t, err) + err = NamespacesPubKeyUpdate(privKey2, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") + require.NoError(t, err) + }) + + t.Run("test-registered-namespace-pubkey-update-with-nonsense-key", func(t *testing.T) { + tempDir := filepath.Join(t.TempDir(), "in_the_middle_of_nowhere") + privKey3, err := config.GeneratePEM(tempDir) + require.NoError(t, err) + err = NamespacesPubKeyUpdate(privKey3, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") + require.ErrorContains(t, err, "it doesn't contain any public key matching the existing namespace's public key in db") + }) + + t.Run("test-registered-namespace-pubkey-update-with-imposter-key", func(t *testing.T) { + privKey4, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) + require.NoError(t, err) + config.UpdatePreviousIssuerPrivateJWK() + // Both active key and previous key are set to privKey4 + err = NamespacesPubKeyUpdate(privKey4, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") + require.ErrorContains(t, err, "it fails to pass the proof of possession verification") + + // Revert the active key changes happened in this subtest + config.SetActiveKey(privKey) + config.UpdatePreviousIssuerPrivateJWK() + config.SetActiveKey(privKey2) + }) + + t.Run("Test namespace delete", func(t *testing.T) { + //Test functionality of namespace delete + err = NamespaceDelete(svr.URL+"/api/v1.0/registry/foo/bar", "/foo/bar") + require.NoError(t, err) + var stdoutCapture string + oldStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + err = NamespaceGet(svr.URL + "/api/v1.0/registry") + require.NoError(t, err) + w.Close() + os.Stdout = oldStdout + + capturedOutput := make([]byte, 1024) + n, _ := r.Read(capturedOutput) + stdoutCapture = string(capturedOutput[:n]) + assert.Equal(t, "[]\n", stdoutCapture) + }) + server_utils.ResetTestState() +} + +func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { + ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) + server_utils.ResetTestState() + t.Cleanup(func() { + func() { require.NoError(t, egrp.Wait()) }() + cancel() + config.ResetIssuerJWKPtr() + config.ResetIssuerPrivateKeys() + server_utils.ResetTestState() + }) + + tDir := t.TempDir() + + svr := registryMockup(ctx, t, "serveregistry") + defer func() { + err := ShutdownRegistryDB() + assert.NoError(t, err) + svr.CloseClientConnections() + svr.Close() + }() + + config.ResetIssuerJWKPtr() + config.ResetIssuerPrivateKeys() + privKeys := config.GetIssuerPrivateKeys() + require.Len(t, privKeys, 0) + + // Construct a client that has [p1,p2,p3] and p3 is the active private key + privKey1, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) + require.NotEmpty(t, privKey1) + require.NoError(t, err) + privKey2, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) + require.NoError(t, err) + + prefix := "/mascot/bucky" + err = NamespaceRegister(privKey2, svr.URL+"/api/v1.0/registry", "", prefix, "mock_site_name") + require.NoError(t, err) + + config.UpdatePreviousIssuerPrivateJWK() + privKey3, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) + require.NoError(t, err) + + // Construct a public keys JWKS [p2,p4] to save in registry DB, imitating admin manually adding p4 + registryDbJwks := jwk.NewSet() + pubKey2, err := jwk.PublicKeyOf(privKey2) + require.NoError(t, err) + err = registryDbJwks.AddKey(pubKey2) + require.NoError(t, err) + privKey4, err := config.GeneratePEM(filepath.Join(tDir, "elsewhere")) + require.NoError(t, err) + pubKey4, err := jwk.PublicKeyOf(privKey4) + require.NoError(t, err) + err = registryDbJwks.AddKey(pubKey4) + require.NoError(t, err) + jwksBytes, err := json.Marshal(registryDbJwks) + require.NoError(t, err) + jwksStr := string(jwksBytes) + + // Test functionality of a namespace registered with multi public keys [p2,p4] + err = setNamespacePubKey(prefix, jwksStr) // set the registered public keys to [p2,p4] + require.NoError(t, err) + ns, err := getNamespaceByPrefix(prefix) + require.NoError(t, err) + require.Equal(t, jwksStr, ns.Pubkey) + + prevKey := config.GetPreviousIssuerPrivateJWK() + require.Equal(t, privKey2.KeyID(), prevKey.KeyID()) + privKeys = config.GetIssuerPrivateKeys() + require.Len(t, privKeys, 3) + + // Client allKeys:[p1,p2,p3] prevKey:p2 activeKey:p3 ---UPDATE--> Registry [p2,p4] + // => should update Registry to [p3,p4] (rotate out prevKey:p2, rotate in activeKey:p3) + err = NamespacesPubKeyUpdate(privKey3, []string{prefix}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") + require.NoError(t, err) + ns, err = getNamespaceByPrefix(prefix) + require.NoError(t, err) + + expectedJwks := jwk.NewSet() + pubKey3, err := jwk.PublicKeyOf(privKey3) + require.NoError(t, err) + err = expectedJwks.AddKey(pubKey3) + require.NoError(t, err) + err = expectedJwks.AddKey(pubKey4) + require.NoError(t, err) + expectedJwksBytes, err := json.Marshal(expectedJwks) + require.NoError(t, err) + expectedJwksStr := string(expectedJwksBytes) + + expectedKids, err := getSortedKids(ctx, expectedJwksStr) + require.NoError(t, err) + actualKids, err := getSortedKids(ctx, ns.Pubkey) + require.NoError(t, err) + require.Equal(t, expectedKids, actualKids) +} func TestRegistryKeyChainingOSDF(t *testing.T) { server_utils.ResetTestState() From 896aa4b19e25fe3f6c81222166626b6c5f39ce18 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Mon, 23 Dec 2024 18:24:13 +0000 Subject: [PATCH 60/64] comment out TestMultiPubKeysRegisteredOnNamespace --- registry/client_commands_test.go | 227 +++++++++++++++---------------- 1 file changed, 113 insertions(+), 114 deletions(-) diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index a109f36a3..b99855812 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -26,7 +26,6 @@ import ( "net/http/httptest" "os" "path/filepath" - "sort" "testing" "github.com/gin-gonic/gin" @@ -68,26 +67,26 @@ func registryMockup(ctx context.Context, t *testing.T, testName string) *httptes return svr } -func getSortedKids(ctx context.Context, jsonStr string) ([]string, error) { - set, err := jwk.Parse([]byte(jsonStr)) - if err != nil { - return nil, err - } - var kids []string - keysIter := set.Keys(ctx) - for keysIter.Next(ctx) { - key := keysIter.Pair().Value.(jwk.Key) - - kid, ok := key.Get("kid") - if !ok { - continue - } - kids = append(kids, kid.(string)) - } - sort.Strings(kids) - - return kids, nil -} +// func getSortedKids(ctx context.Context, jsonStr string) ([]string, error) { +// set, err := jwk.Parse([]byte(jsonStr)) +// if err != nil { +// return nil, err +// } +// var kids []string +// keysIter := set.Keys(ctx) +// for keysIter.Next(ctx) { +// key := keysIter.Pair().Value.(jwk.Key) + +// kid, ok := key.Get("kid") +// if !ok { +// continue +// } +// kids = append(kids, kid.(string)) +// } +// sort.Strings(kids) + +// return kids, nil +// } func TestServeNamespaceRegistry(t *testing.T) { ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) @@ -208,99 +207,99 @@ func TestServeNamespaceRegistry(t *testing.T) { server_utils.ResetTestState() } -func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { - ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) - server_utils.ResetTestState() - t.Cleanup(func() { - func() { require.NoError(t, egrp.Wait()) }() - cancel() - config.ResetIssuerJWKPtr() - config.ResetIssuerPrivateKeys() - server_utils.ResetTestState() - }) - - tDir := t.TempDir() - - svr := registryMockup(ctx, t, "serveregistry") - defer func() { - err := ShutdownRegistryDB() - assert.NoError(t, err) - svr.CloseClientConnections() - svr.Close() - }() - - config.ResetIssuerJWKPtr() - config.ResetIssuerPrivateKeys() - privKeys := config.GetIssuerPrivateKeys() - require.Len(t, privKeys, 0) - - // Construct a client that has [p1,p2,p3] and p3 is the active private key - privKey1, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) - require.NotEmpty(t, privKey1) - require.NoError(t, err) - privKey2, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) - require.NoError(t, err) - - prefix := "/mascot/bucky" - err = NamespaceRegister(privKey2, svr.URL+"/api/v1.0/registry", "", prefix, "mock_site_name") - require.NoError(t, err) - - config.UpdatePreviousIssuerPrivateJWK() - privKey3, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) - require.NoError(t, err) - - // Construct a public keys JWKS [p2,p4] to save in registry DB, imitating admin manually adding p4 - registryDbJwks := jwk.NewSet() - pubKey2, err := jwk.PublicKeyOf(privKey2) - require.NoError(t, err) - err = registryDbJwks.AddKey(pubKey2) - require.NoError(t, err) - privKey4, err := config.GeneratePEM(filepath.Join(tDir, "elsewhere")) - require.NoError(t, err) - pubKey4, err := jwk.PublicKeyOf(privKey4) - require.NoError(t, err) - err = registryDbJwks.AddKey(pubKey4) - require.NoError(t, err) - jwksBytes, err := json.Marshal(registryDbJwks) - require.NoError(t, err) - jwksStr := string(jwksBytes) - - // Test functionality of a namespace registered with multi public keys [p2,p4] - err = setNamespacePubKey(prefix, jwksStr) // set the registered public keys to [p2,p4] - require.NoError(t, err) - ns, err := getNamespaceByPrefix(prefix) - require.NoError(t, err) - require.Equal(t, jwksStr, ns.Pubkey) - - prevKey := config.GetPreviousIssuerPrivateJWK() - require.Equal(t, privKey2.KeyID(), prevKey.KeyID()) - privKeys = config.GetIssuerPrivateKeys() - require.Len(t, privKeys, 3) - - // Client allKeys:[p1,p2,p3] prevKey:p2 activeKey:p3 ---UPDATE--> Registry [p2,p4] - // => should update Registry to [p3,p4] (rotate out prevKey:p2, rotate in activeKey:p3) - err = NamespacesPubKeyUpdate(privKey3, []string{prefix}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") - require.NoError(t, err) - ns, err = getNamespaceByPrefix(prefix) - require.NoError(t, err) - - expectedJwks := jwk.NewSet() - pubKey3, err := jwk.PublicKeyOf(privKey3) - require.NoError(t, err) - err = expectedJwks.AddKey(pubKey3) - require.NoError(t, err) - err = expectedJwks.AddKey(pubKey4) - require.NoError(t, err) - expectedJwksBytes, err := json.Marshal(expectedJwks) - require.NoError(t, err) - expectedJwksStr := string(expectedJwksBytes) - - expectedKids, err := getSortedKids(ctx, expectedJwksStr) - require.NoError(t, err) - actualKids, err := getSortedKids(ctx, ns.Pubkey) - require.NoError(t, err) - require.Equal(t, expectedKids, actualKids) -} +// func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { +// ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) +// server_utils.ResetTestState() +// t.Cleanup(func() { +// func() { require.NoError(t, egrp.Wait()) }() +// cancel() +// config.ResetIssuerJWKPtr() +// config.ResetIssuerPrivateKeys() +// server_utils.ResetTestState() +// }) + +// tDir := t.TempDir() + +// svr := registryMockup(ctx, t, "serveregistry") +// defer func() { +// err := ShutdownRegistryDB() +// assert.NoError(t, err) +// svr.CloseClientConnections() +// svr.Close() +// }() + +// config.ResetIssuerJWKPtr() +// config.ResetIssuerPrivateKeys() +// privKeys := config.GetIssuerPrivateKeys() +// require.Len(t, privKeys, 0) + +// // Construct a client that has [p1,p2,p3] and p3 is the active private key +// privKey1, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) +// require.NotEmpty(t, privKey1) +// require.NoError(t, err) +// privKey2, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) +// require.NoError(t, err) + +// prefix := "/mascot/bucky" +// err = NamespaceRegister(privKey2, svr.URL+"/api/v1.0/registry", "", prefix, "mock_site_name") +// require.NoError(t, err) + +// config.UpdatePreviousIssuerPrivateJWK() +// privKey3, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) +// require.NoError(t, err) + +// // Construct a public keys JWKS [p2,p4] to save in registry DB, imitating admin manually adding p4 +// registryDbJwks := jwk.NewSet() +// pubKey2, err := jwk.PublicKeyOf(privKey2) +// require.NoError(t, err) +// err = registryDbJwks.AddKey(pubKey2) +// require.NoError(t, err) +// privKey4, err := config.GeneratePEM(filepath.Join(tDir, "elsewhere")) +// require.NoError(t, err) +// pubKey4, err := jwk.PublicKeyOf(privKey4) +// require.NoError(t, err) +// err = registryDbJwks.AddKey(pubKey4) +// require.NoError(t, err) +// jwksBytes, err := json.Marshal(registryDbJwks) +// require.NoError(t, err) +// jwksStr := string(jwksBytes) + +// // Test functionality of a namespace registered with multi public keys [p2,p4] +// err = setNamespacePubKey(prefix, jwksStr) // set the registered public keys to [p2,p4] +// require.NoError(t, err) +// ns, err := getNamespaceByPrefix(prefix) +// require.NoError(t, err) +// require.Equal(t, jwksStr, ns.Pubkey) + +// prevKey := config.GetPreviousIssuerPrivateJWK() +// require.Equal(t, privKey2.KeyID(), prevKey.KeyID()) +// privKeys = config.GetIssuerPrivateKeys() +// require.Len(t, privKeys, 3) + +// // Client allKeys:[p1,p2,p3] prevKey:p2 activeKey:p3 ---UPDATE--> Registry [p2,p4] +// // => should update Registry to [p3,p4] (rotate out prevKey:p2, rotate in activeKey:p3) +// err = NamespacesPubKeyUpdate(privKey3, []string{prefix}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") +// require.NoError(t, err) +// ns, err = getNamespaceByPrefix(prefix) +// require.NoError(t, err) + +// expectedJwks := jwk.NewSet() +// pubKey3, err := jwk.PublicKeyOf(privKey3) +// require.NoError(t, err) +// err = expectedJwks.AddKey(pubKey3) +// require.NoError(t, err) +// err = expectedJwks.AddKey(pubKey4) +// require.NoError(t, err) +// expectedJwksBytes, err := json.Marshal(expectedJwks) +// require.NoError(t, err) +// expectedJwksStr := string(expectedJwksBytes) + +// expectedKids, err := getSortedKids(ctx, expectedJwksStr) +// require.NoError(t, err) +// actualKids, err := getSortedKids(ctx, ns.Pubkey) +// require.NoError(t, err) +// require.Equal(t, expectedKids, actualKids) +// } func TestRegistryKeyChainingOSDF(t *testing.T) { server_utils.ResetTestState() From 67b62f090058a64dc51f770edc2264471d321826 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Mon, 23 Dec 2024 18:59:35 +0000 Subject: [PATCH 61/64] comment out TestServeNamespaceRegistry --- registry/client_commands_test.go | 408 +++++++++++++++---------------- 1 file changed, 203 insertions(+), 205 deletions(-) diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index b99855812..e1984c430 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -21,11 +21,9 @@ package registry import ( "context" "encoding/json" - "io" - "net/http" "net/http/httptest" - "os" "path/filepath" + "sort" "testing" "github.com/gin-gonic/gin" @@ -67,29 +65,149 @@ func registryMockup(ctx context.Context, t *testing.T, testName string) *httptes return svr } -// func getSortedKids(ctx context.Context, jsonStr string) ([]string, error) { -// set, err := jwk.Parse([]byte(jsonStr)) -// if err != nil { -// return nil, err -// } -// var kids []string -// keysIter := set.Keys(ctx) -// for keysIter.Next(ctx) { -// key := keysIter.Pair().Value.(jwk.Key) - -// kid, ok := key.Get("kid") -// if !ok { -// continue -// } -// kids = append(kids, kid.(string)) -// } -// sort.Strings(kids) - -// return kids, nil +func getSortedKids(ctx context.Context, jsonStr string) ([]string, error) { + set, err := jwk.Parse([]byte(jsonStr)) + if err != nil { + return nil, err + } + var kids []string + keysIter := set.Keys(ctx) + for keysIter.Next(ctx) { + key := keysIter.Pair().Value.(jwk.Key) + + kid, ok := key.Get("kid") + if !ok { + continue + } + kids = append(kids, kid.(string)) + } + sort.Strings(kids) + + return kids, nil +} + +// func TestServeNamespaceRegistry(t *testing.T) { +// ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) +// t.Cleanup(func() { +// func() { require.NoError(t, egrp.Wait()) }() +// cancel() +// config.ResetIssuerJWKPtr() +// config.ResetIssuerPrivateKeys() +// server_utils.ResetTestState() +// }) +// server_utils.ResetTestState() + +// svr := registryMockup(ctx, t, "serveregistry") +// defer func() { +// err := ShutdownRegistryDB() +// assert.NoError(t, err) +// svr.CloseClientConnections() +// svr.Close() +// }() + +// _, err := config.GetIssuerPublicJWKS() +// require.NoError(t, err) +// privKey, err := config.GetIssuerPrivateJWK() +// require.NoError(t, err) + +// //Test functionality of registering a namespace (without identity) +// err = NamespaceRegister(privKey, svr.URL+"/api/v1.0/registry", "", "/foo/bar", "mock_site_name") +// require.NoError(t, err) +// var privKey2 jwk.Key + +// //Test we can list the namespace without an error +// t.Run("Test namespace list", func(t *testing.T) { +// //Set up a buffer to capture stdout +// var stdoutCapture string +// oldStdout := os.Stdout +// r, w, _ := os.Pipe() +// os.Stdout = w + +// //List the namespaces +// err = NamespaceList(svr.URL + "/api/v1.0/registry") +// require.NoError(t, err) +// w.Close() +// os.Stdout = oldStdout + +// capturedOutput := make([]byte, 1024) +// n, _ := r.Read(capturedOutput) +// stdoutCapture = string(capturedOutput[:n]) +// assert.Contains(t, stdoutCapture, `"prefix":"/foo/bar"`) +// }) + +// t.Run("Test register namespace with sitename", func(t *testing.T) { +// res, err := http.Get(svr.URL + "/api/v1.0/registry/foo/bar") +// require.NoError(t, err) +// require.Equal(t, http.StatusOK, res.StatusCode) +// data, err := io.ReadAll(res.Body) +// require.NoError(t, err) +// ns := server_structs.Namespace{} +// err = json.Unmarshal(data, &ns) +// require.NoError(t, err) +// assert.Equal(t, ns.AdminMetadata.SiteName, "mock_site_name") +// }) + +// t.Run("test-registered-namespace-pubkey-update-with-new-active-key", func(t *testing.T) { +// activeKey, err := config.GetIssuerPrivateJWK() +// require.NoError(t, err) +// require.Equal(t, privKey.KeyID(), activeKey.KeyID()) + +// // Imitate LaunchIssuerKeysDirRefresh function +// config.UpdatePreviousIssuerPrivateJWK() +// _, err = config.GeneratePEM(param.IssuerKeysDirectory.GetString()) +// require.NoError(t, err) +// privKey2, err = config.LoadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()) +// require.NoError(t, err) +// err = NamespacesPubKeyUpdate(privKey2, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") +// require.NoError(t, err) +// }) + +// t.Run("test-registered-namespace-pubkey-update-with-nonsense-key", func(t *testing.T) { +// tempDir := filepath.Join(t.TempDir(), "in_the_middle_of_nowhere") +// privKey3, err := config.GeneratePEM(tempDir) +// require.NoError(t, err) +// err = NamespacesPubKeyUpdate(privKey3, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") +// require.ErrorContains(t, err, "it doesn't contain any public key matching the existing namespace's public key in db") +// }) + +// t.Run("test-registered-namespace-pubkey-update-with-imposter-key", func(t *testing.T) { +// privKey4, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) +// require.NoError(t, err) +// config.UpdatePreviousIssuerPrivateJWK() +// // Both active key and previous key are set to privKey4 +// err = NamespacesPubKeyUpdate(privKey4, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") +// require.ErrorContains(t, err, "it fails to pass the proof of possession verification") + +// // Revert the active key changes happened in this subtest +// config.SetActiveKey(privKey) +// config.UpdatePreviousIssuerPrivateJWK() +// config.SetActiveKey(privKey2) +// }) + +// t.Run("Test namespace delete", func(t *testing.T) { +// //Test functionality of namespace delete +// err = NamespaceDelete(svr.URL+"/api/v1.0/registry/foo/bar", "/foo/bar") +// require.NoError(t, err) +// var stdoutCapture string +// oldStdout := os.Stdout +// r, w, _ := os.Pipe() +// os.Stdout = w +// err = NamespaceGet(svr.URL + "/api/v1.0/registry") +// require.NoError(t, err) +// w.Close() +// os.Stdout = oldStdout + +// capturedOutput := make([]byte, 1024) +// n, _ := r.Read(capturedOutput) +// stdoutCapture = string(capturedOutput[:n]) +// assert.Equal(t, "[]\n", stdoutCapture) +// }) +// server_utils.ResetTestState() // } -func TestServeNamespaceRegistry(t *testing.T) { +func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) + server_utils.ResetTestState() t.Cleanup(func() { func() { require.NoError(t, egrp.Wait()) }() cancel() @@ -97,7 +215,8 @@ func TestServeNamespaceRegistry(t *testing.T) { config.ResetIssuerPrivateKeys() server_utils.ResetTestState() }) - server_utils.ResetTestState() + + tDir := t.TempDir() svr := registryMockup(ctx, t, "serveregistry") defer func() { @@ -107,200 +226,79 @@ func TestServeNamespaceRegistry(t *testing.T) { svr.Close() }() - _, err := config.GetIssuerPublicJWKS() + config.ResetIssuerJWKPtr() + config.ResetIssuerPrivateKeys() + privKeys := config.GetIssuerPrivateKeys() + require.Len(t, privKeys, 0) + + // Construct a client that has [p1,p2,p3] and p3 is the active private key + privKey1, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) + require.NotEmpty(t, privKey1) require.NoError(t, err) - privKey, err := config.GetIssuerPrivateJWK() + privKey2, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) require.NoError(t, err) - //Test functionality of registering a namespace (without identity) - err = NamespaceRegister(privKey, svr.URL+"/api/v1.0/registry", "", "/foo/bar", "mock_site_name") + prefix := "/mascot/bucky" + err = NamespaceRegister(privKey2, svr.URL+"/api/v1.0/registry", "", prefix, "mock_site_name") require.NoError(t, err) - var privKey2 jwk.Key - //Test we can list the namespace without an error - t.Run("Test namespace list", func(t *testing.T) { - //Set up a buffer to capture stdout - var stdoutCapture string - oldStdout := os.Stdout - r, w, _ := os.Pipe() - os.Stdout = w - - //List the namespaces - err = NamespaceList(svr.URL + "/api/v1.0/registry") - require.NoError(t, err) - w.Close() - os.Stdout = oldStdout + config.UpdatePreviousIssuerPrivateJWK() + privKey3, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) + require.NoError(t, err) - capturedOutput := make([]byte, 1024) - n, _ := r.Read(capturedOutput) - stdoutCapture = string(capturedOutput[:n]) - assert.Contains(t, stdoutCapture, `"prefix":"/foo/bar"`) - }) + // Construct a public keys JWKS [p2,p4] to save in registry DB, imitating admin manually adding p4 + registryDbJwks := jwk.NewSet() + pubKey2, err := jwk.PublicKeyOf(privKey2) + require.NoError(t, err) + err = registryDbJwks.AddKey(pubKey2) + require.NoError(t, err) + privKey4, err := config.GeneratePEM(filepath.Join(tDir, "elsewhere")) + require.NoError(t, err) + pubKey4, err := jwk.PublicKeyOf(privKey4) + require.NoError(t, err) + err = registryDbJwks.AddKey(pubKey4) + require.NoError(t, err) + jwksBytes, err := json.Marshal(registryDbJwks) + require.NoError(t, err) + jwksStr := string(jwksBytes) - t.Run("Test register namespace with sitename", func(t *testing.T) { - res, err := http.Get(svr.URL + "/api/v1.0/registry/foo/bar") - require.NoError(t, err) - require.Equal(t, http.StatusOK, res.StatusCode) - data, err := io.ReadAll(res.Body) - require.NoError(t, err) - ns := server_structs.Namespace{} - err = json.Unmarshal(data, &ns) - require.NoError(t, err) - assert.Equal(t, ns.AdminMetadata.SiteName, "mock_site_name") - }) + // Test functionality of a namespace registered with multi public keys [p2,p4] + err = setNamespacePubKey(prefix, jwksStr) // set the registered public keys to [p2,p4] + require.NoError(t, err) + ns, err := getNamespaceByPrefix(prefix) + require.NoError(t, err) + require.Equal(t, jwksStr, ns.Pubkey) - t.Run("test-registered-namespace-pubkey-update-with-new-active-key", func(t *testing.T) { - activeKey, err := config.GetIssuerPrivateJWK() - require.NoError(t, err) - require.Equal(t, privKey.KeyID(), activeKey.KeyID()) - - // Imitate LaunchIssuerKeysDirRefresh function - config.UpdatePreviousIssuerPrivateJWK() - _, err = config.GeneratePEM(param.IssuerKeysDirectory.GetString()) - require.NoError(t, err) - privKey2, err = config.LoadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()) - require.NoError(t, err) - err = NamespacesPubKeyUpdate(privKey2, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") - require.NoError(t, err) - }) + prevKey := config.GetPreviousIssuerPrivateJWK() + require.Equal(t, privKey2.KeyID(), prevKey.KeyID()) + privKeys = config.GetIssuerPrivateKeys() + require.Len(t, privKeys, 3) - t.Run("test-registered-namespace-pubkey-update-with-nonsense-key", func(t *testing.T) { - tempDir := filepath.Join(t.TempDir(), "in_the_middle_of_nowhere") - privKey3, err := config.GeneratePEM(tempDir) - require.NoError(t, err) - err = NamespacesPubKeyUpdate(privKey3, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") - require.ErrorContains(t, err, "it doesn't contain any public key matching the existing namespace's public key in db") - }) + // Client allKeys:[p1,p2,p3] prevKey:p2 activeKey:p3 ---UPDATE--> Registry [p2,p4] + // => should update Registry to [p3,p4] (rotate out prevKey:p2, rotate in activeKey:p3) + err = NamespacesPubKeyUpdate(privKey3, []string{prefix}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") + require.NoError(t, err) + ns, err = getNamespaceByPrefix(prefix) + require.NoError(t, err) - t.Run("test-registered-namespace-pubkey-update-with-imposter-key", func(t *testing.T) { - privKey4, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) - require.NoError(t, err) - config.UpdatePreviousIssuerPrivateJWK() - // Both active key and previous key are set to privKey4 - err = NamespacesPubKeyUpdate(privKey4, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") - require.ErrorContains(t, err, "it fails to pass the proof of possession verification") - - // Revert the active key changes happened in this subtest - config.SetActiveKey(privKey) - config.UpdatePreviousIssuerPrivateJWK() - config.SetActiveKey(privKey2) - }) + expectedJwks := jwk.NewSet() + pubKey3, err := jwk.PublicKeyOf(privKey3) + require.NoError(t, err) + err = expectedJwks.AddKey(pubKey3) + require.NoError(t, err) + err = expectedJwks.AddKey(pubKey4) + require.NoError(t, err) + expectedJwksBytes, err := json.Marshal(expectedJwks) + require.NoError(t, err) + expectedJwksStr := string(expectedJwksBytes) - t.Run("Test namespace delete", func(t *testing.T) { - //Test functionality of namespace delete - err = NamespaceDelete(svr.URL+"/api/v1.0/registry/foo/bar", "/foo/bar") - require.NoError(t, err) - var stdoutCapture string - oldStdout := os.Stdout - r, w, _ := os.Pipe() - os.Stdout = w - err = NamespaceGet(svr.URL + "/api/v1.0/registry") - require.NoError(t, err) - w.Close() - os.Stdout = oldStdout - - capturedOutput := make([]byte, 1024) - n, _ := r.Read(capturedOutput) - stdoutCapture = string(capturedOutput[:n]) - assert.Equal(t, "[]\n", stdoutCapture) - }) - server_utils.ResetTestState() + expectedKids, err := getSortedKids(ctx, expectedJwksStr) + require.NoError(t, err) + actualKids, err := getSortedKids(ctx, ns.Pubkey) + require.NoError(t, err) + require.Equal(t, expectedKids, actualKids) } -// func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { -// ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) -// server_utils.ResetTestState() -// t.Cleanup(func() { -// func() { require.NoError(t, egrp.Wait()) }() -// cancel() -// config.ResetIssuerJWKPtr() -// config.ResetIssuerPrivateKeys() -// server_utils.ResetTestState() -// }) - -// tDir := t.TempDir() - -// svr := registryMockup(ctx, t, "serveregistry") -// defer func() { -// err := ShutdownRegistryDB() -// assert.NoError(t, err) -// svr.CloseClientConnections() -// svr.Close() -// }() - -// config.ResetIssuerJWKPtr() -// config.ResetIssuerPrivateKeys() -// privKeys := config.GetIssuerPrivateKeys() -// require.Len(t, privKeys, 0) - -// // Construct a client that has [p1,p2,p3] and p3 is the active private key -// privKey1, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) -// require.NotEmpty(t, privKey1) -// require.NoError(t, err) -// privKey2, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) -// require.NoError(t, err) - -// prefix := "/mascot/bucky" -// err = NamespaceRegister(privKey2, svr.URL+"/api/v1.0/registry", "", prefix, "mock_site_name") -// require.NoError(t, err) - -// config.UpdatePreviousIssuerPrivateJWK() -// privKey3, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) -// require.NoError(t, err) - -// // Construct a public keys JWKS [p2,p4] to save in registry DB, imitating admin manually adding p4 -// registryDbJwks := jwk.NewSet() -// pubKey2, err := jwk.PublicKeyOf(privKey2) -// require.NoError(t, err) -// err = registryDbJwks.AddKey(pubKey2) -// require.NoError(t, err) -// privKey4, err := config.GeneratePEM(filepath.Join(tDir, "elsewhere")) -// require.NoError(t, err) -// pubKey4, err := jwk.PublicKeyOf(privKey4) -// require.NoError(t, err) -// err = registryDbJwks.AddKey(pubKey4) -// require.NoError(t, err) -// jwksBytes, err := json.Marshal(registryDbJwks) -// require.NoError(t, err) -// jwksStr := string(jwksBytes) - -// // Test functionality of a namespace registered with multi public keys [p2,p4] -// err = setNamespacePubKey(prefix, jwksStr) // set the registered public keys to [p2,p4] -// require.NoError(t, err) -// ns, err := getNamespaceByPrefix(prefix) -// require.NoError(t, err) -// require.Equal(t, jwksStr, ns.Pubkey) - -// prevKey := config.GetPreviousIssuerPrivateJWK() -// require.Equal(t, privKey2.KeyID(), prevKey.KeyID()) -// privKeys = config.GetIssuerPrivateKeys() -// require.Len(t, privKeys, 3) - -// // Client allKeys:[p1,p2,p3] prevKey:p2 activeKey:p3 ---UPDATE--> Registry [p2,p4] -// // => should update Registry to [p3,p4] (rotate out prevKey:p2, rotate in activeKey:p3) -// err = NamespacesPubKeyUpdate(privKey3, []string{prefix}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") -// require.NoError(t, err) -// ns, err = getNamespaceByPrefix(prefix) -// require.NoError(t, err) - -// expectedJwks := jwk.NewSet() -// pubKey3, err := jwk.PublicKeyOf(privKey3) -// require.NoError(t, err) -// err = expectedJwks.AddKey(pubKey3) -// require.NoError(t, err) -// err = expectedJwks.AddKey(pubKey4) -// require.NoError(t, err) -// expectedJwksBytes, err := json.Marshal(expectedJwks) -// require.NoError(t, err) -// expectedJwksStr := string(expectedJwksBytes) - -// expectedKids, err := getSortedKids(ctx, expectedJwksStr) -// require.NoError(t, err) -// actualKids, err := getSortedKids(ctx, ns.Pubkey) -// require.NoError(t, err) -// require.Equal(t, expectedKids, actualKids) -// } - func TestRegistryKeyChainingOSDF(t *testing.T) { server_utils.ResetTestState() From 7933e3c449901cc07ba2644336b3332fe3100673 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Fri, 27 Dec 2024 14:57:00 -0600 Subject: [PATCH 62/64] generate a unique filename using a POSIX mkstemp-like logic; fix race condition in updating previous active private key; a few comments --- cmd/generate_keygen.go | 2 +- config/init_server_creds.go | 101 ++++++++++++++++++++++++------------ 2 files changed, 70 insertions(+), 33 deletions(-) diff --git a/cmd/generate_keygen.go b/cmd/generate_keygen.go index 7db4b428b..f2c4832b5 100644 --- a/cmd/generate_keygen.go +++ b/cmd/generate_keygen.go @@ -70,7 +70,7 @@ func keygenMain(cmd *cobra.Command, args []string) error { viper.Set(param.IssuerKeysDirectory.GetName(), privateKeyPath) - // GetIssuerPublicJWKS will generate the private key at IssuerKey if it does not exist + // GetIssuerPublicJWKS will generate the private key in IssuerKeysDirectory if it does not exist // and parse the private key and generate the corresponding public key for us pubkey, err := config.GetIssuerPublicJWKS() if err != nil { diff --git a/config/init_server_creds.go b/config/init_server_creds.go index 88ffd6371..a083ac6b7 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -28,8 +28,8 @@ import ( "crypto/x509/pkix" "encoding/pem" "fmt" + "io" "math/big" - mathrand "math/rand" "os" "os/exec" "path/filepath" @@ -61,6 +61,9 @@ var ( // Used to ensure initialization func init() is only called once initOnce sync.Once + + // Prevent race condition in issuer's private key update + updateKeyMutex sync.Mutex ) // Reset the atomic pointer to issuer private jwk @@ -68,6 +71,7 @@ func ResetIssuerJWKPtr() { issuerPrivateJWK.Store(nil) } +// Get issuer's previous active private key func GetPreviousIssuerPrivateJWK() jwk.Key { previousKey := previousIssuerPrivateJWK.Load() if previousKey == nil { @@ -76,7 +80,13 @@ func GetPreviousIssuerPrivateJWK() jwk.Key { return *previousKey } +// Transfer the current issuer's private key to the previous one. This is called before updating +// the current key to maintain a reference to the previous key for previous public key validation. func UpdatePreviousIssuerPrivateJWK() { + // ensure previousIssuerPrivateJWK isn't being updated concurrently + updateKeyMutex.Lock() + defer updateKeyMutex.Unlock() + // Save the current key to previousIssuerPrivateJWK currentKey := issuerPrivateJWK.Load() if currentKey != nil { @@ -84,6 +94,7 @@ func UpdatePreviousIssuerPrivateJWK() { } } +// Reset the atomic pointer to issuer's private key func ResetPreviousIssuerPrivateJWK() { previousIssuerPrivateJWK.Store(nil) } @@ -117,6 +128,8 @@ func GetIssuerPrivateKeys() map[string]jwk.Key { return *keysPtr } +// Get the current directory that stores issuer's all private keys +// We record this value to handle runtime changes to the issuer keys directory func getCurrentIssuerKeysDir() string { if val := currentIssuerKeysDir.Load(); val != nil { return val.(string) @@ -124,24 +137,16 @@ func getCurrentIssuerKeysDir() string { return "" } +// Set the current directory that stores issuer's all private keys func setCurrentIssuerKeysDir(dir string) { currentIssuerKeysDir.Store(dir) } +// Reset the current directory that stores issuer's all private keys func ResetCurrentIssuerKeysDir() { currentIssuerKeysDir.Store("") } -// Helper function to generate random string -func createRandomString(length int) string { - const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" - b := make([]byte, length) - for i := range b { - b[i] = charset[mathrand.Intn(len(charset))] - } - return string(b) -} - // Return a pointer to an ECDSA private key or RSA private key read from keyLocation. // // This can be used to load ECDSA or RSA private key for various purposes, @@ -590,7 +595,7 @@ func GenerateCert() error { return nil } -// Helper function to move the existing single key file to new directory +// Helper function to copy the existing single key file to new directory func migratePrivateKey(newDir string) error { legacyPrivateKeyFile := param.IssuerKey.GetString() if _, err := os.Stat(legacyPrivateKeyFile); os.IsNotExist(err) { @@ -602,29 +607,42 @@ func migratePrivateKey(newDir string) error { return errors.Wrapf(err, "failed to create destination directory %s", newDir) } - // Get the key id of the existing key - key, err := loadSinglePEM(legacyPrivateKeyFile) + // Verify there is a valid key in the legacyPrivateKeyFile + _, err := loadSinglePEM(legacyPrivateKeyFile) if err != nil { - log.Warnf("Failed to load key %s: %v", key.KeyID(), err) + log.Warnf("Failed to load the key at %s: %v", legacyPrivateKeyFile, err) return err } - // Rename the existing private key file and set destination path - fileName := fmt.Sprintf("migrated_%d_%s.pem", - time.Now().UnixNano(), - createRandomString(4)) - destPath := filepath.Join(newDir, fileName) + // Generate a unique filename using mkstemp logic for migration destination + fileNamePattern := fmt.Sprintf("migrated_%d_*.pem", + time.Now().UnixNano()) - // Check if a file with the same name already exists in the destination - if _, err := os.Stat(destPath); err == nil { - return errors.Errorf("destination file already exists: %s", destPath) + destFile, err := os.CreateTemp(newDir, fileNamePattern) + if err != nil { + return errors.Wrapf(err, "failed to create migration destination file in %s", newDir) } + defer destFile.Close() - // Move the file - if err := os.Rename(legacyPrivateKeyFile, destPath); err != nil { - return errors.Wrapf(err, "failed to move %s to %s", legacyPrivateKeyFile, destPath) + // Copy the file contents + srcFile, err := os.Open(legacyPrivateKeyFile) + if err != nil { + return errors.Wrapf(err, "failed to open source file %s", legacyPrivateKeyFile) } + defer srcFile.Close() + if _, err := io.Copy(destFile, srcFile); err != nil { + return errors.Wrapf(err, "failed to copy file contents") + } + + // Preserve file permissions + sourceInfo, err := srcFile.Stat() + if err != nil { + return errors.Wrap(err, "failed to get source file info") + } + if err := os.Chmod(destFile.Name(), sourceInfo.Mode()); err != nil { + return errors.Wrapf(err, "failed to set permissions on %s", destFile.Name()) + } return nil } @@ -727,18 +745,37 @@ func loadPEMFiles(dir string) (jwk.Key, error) { // Create a new .pem file (combining GeneratePrivateKey and LoadPrivateKey functions) func GeneratePEM(dir string) (jwk.Key, error) { - filename := fmt.Sprintf("pelican_generated_%d_%s.pem", - time.Now().UnixNano(), - createRandomString(4)) + // Generate a unique filename using a POSIX mkstemp-like logic + // Create a temp file, store its filename, then immediately delete this temp file + filenamePattern := fmt.Sprintf("pelican_generated_%d_*.pem", + time.Now().UnixNano()) + gid, err := GetDaemonGID() + if err != nil { + return nil, errors.Wrap(err, "failed to get deamon gid") + } + if err := MkdirAll(dir, 0750, -1, gid); err != nil { + return nil, errors.Wrapf(err, "failed to set the permission of %s", dir) + } + tempFile, err := os.CreateTemp(dir, filenamePattern) + if err != nil { + return nil, errors.Wrap(err, "failed to remove temp file") + } + keyPath := tempFile.Name() + if err := tempFile.Close(); err != nil { + return nil, errors.Wrap(err, "failed to close temp file") + } + if err := os.Remove(keyPath); err != nil { + return nil, errors.Wrap(err, "failed to remove temp file") + } - keyPath := filepath.Join(dir, filename) if err := GeneratePrivateKey(keyPath, elliptic.P256(), false); err != nil { - return nil, errors.Wrap(err, "failed to generate new private key") + return nil, errors.Wrapf(err, "failed to generate new private key at %s", keyPath) } key, err := loadSinglePEM(keyPath) if err != nil { - log.Warnf("Failed to load key %s: %v", keyPath, err) + log.Errorf("Failed to load key %s: %v", keyPath, err) + return nil, errors.Wrapf(err, "failed to load key from %s", keyPath) } log.Debugf("Generated private key %s", key.KeyID()) From e089e623aa80873d88072fb9e673fc019723253c Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Tue, 31 Dec 2024 04:56:33 +0000 Subject: [PATCH 63/64] fix and refine private key i/o relevant funcs, remove risky api, create illusion for IssuerKey if it exists, minor test impr --- config/config.go | 1 - config/encrypted_test.go | 1 + config/init_server_creds.go | 200 ++++++++---------------- config/init_server_creds_test.go | 9 -- origin/origin_ui.go | 25 --- origin/origin_ui_test.go | 99 ------------ registry/client_commands.go | 3 +- registry/client_commands_test.go | 252 ++++++++++++++++--------------- 8 files changed, 198 insertions(+), 392 deletions(-) delete mode 100644 origin/origin_ui_test.go diff --git a/config/config.go b/config/config.go index abea37432..89220b0b5 100644 --- a/config/config.go +++ b/config/config.go @@ -1643,7 +1643,6 @@ func ResetConfig() { ResetIssuerJWKPtr() ResetIssuerPrivateKeys() ResetPreviousIssuerPrivateJWK() - ResetCurrentIssuerKeysDir() ResetClientInitialized() diff --git a/config/encrypted_test.go b/config/encrypted_test.go index fa6f9295c..6fc3cefff 100644 --- a/config/encrypted_test.go +++ b/config/encrypted_test.go @@ -97,6 +97,7 @@ func TestDecryptString(t *testing.T) { require.NoError(t, err) assert.NotEmpty(t, getEncrypt) + ResetConfig() newKeyDir := filepath.Join(tmp, "new-issuer-keys") viper.Set(param.IssuerKeysDirectory.GetName(), newKeyDir) diff --git a/config/init_server_creds.go b/config/init_server_creds.go index a083ac6b7..8ccfb832c 100644 --- a/config/init_server_creds.go +++ b/config/init_server_creds.go @@ -28,7 +28,7 @@ import ( "crypto/x509/pkix" "encoding/pem" "fmt" - "io" + "io/fs" "math/big" "os" "os/exec" @@ -56,9 +56,6 @@ var ( // Representing private keys (from all .pem files) in the directory cache issuerPrivateKeys atomic.Pointer[map[string]jwk.Key] - // Record the value of "IssuerKeysDirectory" param in case it changes during runtime - currentIssuerKeysDir atomic.Value - // Used to ensure initialization func init() is only called once initOnce sync.Once @@ -128,23 +125,16 @@ func GetIssuerPrivateKeys() map[string]jwk.Key { return *keysPtr } -// Get the current directory that stores issuer's all private keys -// We record this value to handle runtime changes to the issuer keys directory -func getCurrentIssuerKeysDir() string { - if val := currentIssuerKeysDir.Load(); val != nil { - return val.(string) +// Helper function to create a directory and set proper permissions to save private keys +func createDirForKeys(dir string) error { + gid, err := GetDaemonGID() + if err != nil { + return errors.Wrap(err, "failed to get deamon gid") } - return "" -} - -// Set the current directory that stores issuer's all private keys -func setCurrentIssuerKeysDir(dir string) { - currentIssuerKeysDir.Store(dir) -} - -// Reset the current directory that stores issuer's all private keys -func ResetCurrentIssuerKeysDir() { - currentIssuerKeysDir.Store("") + if err := MkdirAll(dir, 0750, -1, gid); err != nil { + return errors.Wrapf(err, "failed to set the permission of %s", dir) + } + return nil } // Return a pointer to an ECDSA private key or RSA private key read from keyLocation. @@ -595,67 +585,11 @@ func GenerateCert() error { return nil } -// Helper function to copy the existing single key file to new directory -func migratePrivateKey(newDir string) error { - legacyPrivateKeyFile := param.IssuerKey.GetString() - if _, err := os.Stat(legacyPrivateKeyFile); os.IsNotExist(err) { - return nil // No existing key exists - } - - // Create the destination directory if it doesn't exist - if err := os.MkdirAll(newDir, 0750); err != nil { - return errors.Wrapf(err, "failed to create destination directory %s", newDir) - } - - // Verify there is a valid key in the legacyPrivateKeyFile - _, err := loadSinglePEM(legacyPrivateKeyFile) - if err != nil { - log.Warnf("Failed to load the key at %s: %v", legacyPrivateKeyFile, err) - return err - } - - // Generate a unique filename using mkstemp logic for migration destination - fileNamePattern := fmt.Sprintf("migrated_%d_*.pem", - time.Now().UnixNano()) - - destFile, err := os.CreateTemp(newDir, fileNamePattern) - if err != nil { - return errors.Wrapf(err, "failed to create migration destination file in %s", newDir) - } - defer destFile.Close() - - // Copy the file contents - srcFile, err := os.Open(legacyPrivateKeyFile) - if err != nil { - return errors.Wrapf(err, "failed to open source file %s", legacyPrivateKeyFile) - } - defer srcFile.Close() - - if _, err := io.Copy(destFile, srcFile); err != nil { - return errors.Wrapf(err, "failed to copy file contents") - } - - // Preserve file permissions - sourceInfo, err := srcFile.Stat() - if err != nil { - return errors.Wrap(err, "failed to get source file info") - } - if err := os.Chmod(destFile.Name(), sourceInfo.Mode()); err != nil { - return errors.Wrapf(err, "failed to set permissions on %s", destFile.Name()) - } - return nil -} - -// Helper function to initialize the in-memory map to save all private keys and migrate existing private key file -func initKeysMap(privateKeysDir string) error { +// Helper function to initialize the in-memory map to save all private keys +func initKeysMap() error { initialMap := make(map[string]jwk.Key) issuerPrivateKeys.Store(&initialMap) - setCurrentIssuerKeysDir(privateKeysDir) - migrateErr := migratePrivateKey(privateKeysDir) - if migrateErr != nil { - return errors.Wrap(migrateErr, "failed to migrate existing private key file") - } return nil } @@ -684,50 +618,68 @@ func loadSinglePEM(path string) (jwk.Key, error) { return key, nil } -// Helper function to load/refresh all .pem files from specified directory and find the most recent modified private key +// Helper function to load/refresh all key files from both legacy IssuerKey file and specified directory +// find the most recent private key based on lexicographical order of their filenames func loadPEMFiles(dir string) (jwk.Key, error) { var mostRecentKey jwk.Key - var mostRecentModTime time.Time + var mostRecentFileName string + latestKeys := getIssuerPrivateKeysCopy() - files, err := os.ReadDir(dir) - if err != nil { - // It's fine if error is "directory not found". We'll create this dir using GeneratePEM func - if os.IsNotExist(err) { - newKey, err := GeneratePEMandSetActiveKey(dir) + // Load legacy private key if it exists - parsing the file at IssuerKey act as if it is included in IssuerKeysDirectory + issuerKeyPath := param.IssuerKey.GetString() + if issuerKeyPath != "" { + if _, err := os.Stat(issuerKeyPath); err == nil { + issuerKey, err := loadSinglePEM(issuerKeyPath) if err != nil { - return nil, errors.Wrapf(err, "failed to create a new .pem file to save private key") + log.Warnf("Failed to load key %s: %v", issuerKeyPath, err) + } else { + latestKeys[issuerKey.KeyID()] = issuerKey + if mostRecentFileName == "" || filepath.Base(issuerKeyPath) > mostRecentFileName { + mostRecentFileName = filepath.Base(issuerKeyPath) + mostRecentKey = issuerKey + } } - return newKey, nil } - return nil, errors.Wrap(err, "failed to read directory that stores private keys") } - latestKeys := getIssuerPrivateKeysCopy() - for _, file := range files { - if filepath.Ext(file.Name()) != ".pem" { - continue + // Ensure input directory dir exists, if not, create it with proper permissions + if _, err := os.Stat(dir); os.IsNotExist(err) { + if err := createDirForKeys(dir); err != nil { + return nil, errors.Wrapf(err, "failed to create directory and set permissions: %s", dir) } - // parse the private key in this file and add to the in-memory keys map - keyPath := filepath.Join(dir, file.Name()) - key, err := loadSinglePEM(keyPath) + } + // Traverse the directory for .pem files in lexical order + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { if err != nil { - log.Warnf("Failed to load key %s: %v", keyPath, err) - continue + return err } - latestKeys[key.KeyID()] = key - // find the most recent modified file - fileInfo, err := file.Info() - if err != nil { - continue - } - if fileInfo.ModTime().After(mostRecentModTime) { - mostRecentModTime = fileInfo.ModTime() - mostRecentKey = key + if !d.IsDir() && filepath.Ext(d.Name()) == ".pem" { + // Parse the private key in this file and add to the in-memory keys map + key, err := loadSinglePEM(path) + if err != nil { + log.Warnf("Failed to load key %s: %v", path, err) + return nil // Skip this file and continue + } + + latestKeys[key.KeyID()] = key + + // Update the most recent key based on lexicographical order of filenames + if mostRecentFileName == "" || d.Name() > mostRecentFileName { + mostRecentFileName = d.Name() + mostRecentKey = key + } } + return nil + }) + + if err != nil { + return nil, errors.Wrapf(err, "failed to traverse directory %s that stores private keys", dir) } - // The directory exists but contains no .pem files - if len(latestKeys) == 0 { + + // Create a new private key and set as active when neither legacy private key at IssuerKey + // nor any .pem file at IssuerKeysDirectory exists + if len(latestKeys) == 0 || mostRecentKey == nil { newKey, err := GeneratePEMandSetActiveKey(dir) if err != nil { return nil, errors.Wrapf(err, "failed to create a new .pem file to save private key") @@ -749,12 +701,8 @@ func GeneratePEM(dir string) (jwk.Key, error) { // Create a temp file, store its filename, then immediately delete this temp file filenamePattern := fmt.Sprintf("pelican_generated_%d_*.pem", time.Now().UnixNano()) - gid, err := GetDaemonGID() - if err != nil { - return nil, errors.Wrap(err, "failed to get deamon gid") - } - if err := MkdirAll(dir, 0750, -1, gid); err != nil { - return nil, errors.Wrapf(err, "failed to set the permission of %s", dir) + if err := createDirForKeys(dir); err != nil { + return nil, errors.Wrapf(err, "failed to create directory and set permissions: %s", dir) } tempFile, err := os.CreateTemp(dir, filenamePattern) if err != nil { @@ -783,7 +731,6 @@ func GeneratePEM(dir string) (jwk.Key, error) { } // Generate a new .pem file and then set the private key it contains as the active one -// This function is also used in origin API to let admin generate new private key func GeneratePEMandSetActiveKey(dir string) (jwk.Key, error) { newKey, err := GeneratePEM(dir) if err != nil { @@ -804,7 +751,7 @@ func LoadIssuerPrivateKey(issuerKeysDir string) (jwk.Key, error) { // Ensure initKeysMap is only called once across the program’s runtime var initErr error initOnce.Do(func() { - initErr = initKeysMap(issuerKeysDir) + initErr = initKeysMap() }) if initErr != nil { return nil, errors.Wrap(initErr, "failed to initialize and/or migrate existing private key file") @@ -855,24 +802,9 @@ func loadIssuerPublicJWKS(existingJWKS string, issuerKeysDir string) (jwk.Set, e func GetIssuerPrivateJWK() (jwk.Key, error) { key := issuerPrivateJWK.Load() issuerKeysDir := param.IssuerKeysDirectory.GetString() - currentIssuerKeysDir := getCurrentIssuerKeysDir() - // Handles runtime changes to the issuer keys directory (configured via "IssuerKeysDirectory" parameter). - // When the directory path changes: - // 1. Resets the active private key - // 2. Clears the in-memory cache of all private keys - // Note: This does not affect the actual key files on disk. - // Primarily used in the "invalid-token-sig-key", "diff-secrets-yield-diff-result" test scenario. - var isDirChanged bool - if currentIssuerKeysDir != issuerKeysDir { - log.Debugf("The private keys dir generated by IssuerKeysDirectory param has changed from '%s' to '%s'", currentIssuerKeysDir, issuerKeysDir) - ResetIssuerJWKPtr() - ResetIssuerPrivateKeys() - setCurrentIssuerKeysDir(issuerKeysDir) - isDirChanged = true - } - - // Re-scan the private keys dir when no active private key in memory or dir changes - if key == nil || isDirChanged { + + // Re-scan the private keys dir when no active private key in memory + if key == nil { newKey, err := LoadIssuerPrivateKey(issuerKeysDir) if err != nil { return nil, errors.Wrap(err, "Failed to load issuer private key") diff --git a/config/init_server_creds_test.go b/config/init_server_creds_test.go index 45dadf227..6086cdbbd 100644 --- a/config/init_server_creds_test.go +++ b/config/init_server_creds_test.go @@ -28,14 +28,10 @@ import ( "os" "path/filepath" "testing" - "time" "github.com/pkg/errors" - "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/pelicanplatform/pelican/param" ) // encrypt should be ecdsa|rsa @@ -178,10 +174,6 @@ func TestMultiPrivateKey(t *testing.T) { require.NoError(t, err) require.NotNil(t, key) - // Wait for 1 second to avoid duplicated private key filenames - // because they are named after unix epoch timestamp - time.Sleep(1 * time.Second) - // Create another private key secondKey, err := GeneratePEMandSetActiveKey(issuerKeysDir) require.NoError(t, err) @@ -189,7 +181,6 @@ func TestMultiPrivateKey(t *testing.T) { assert.NotEqual(t, key.KeyID(), secondKey.KeyID()) // Check if the active private key points to the latest key - viper.Set(param.IssuerKeysDirectory.GetName(), issuerKeysDir) latestKey, err := GetIssuerPrivateJWK() require.NoError(t, err) assert.Equal(t, secondKey.KeyID(), latestKey.KeyID()) diff --git a/origin/origin_ui.go b/origin/origin_ui.go index 98eeaf863..07a16dc25 100644 --- a/origin/origin_ui.go +++ b/origin/origin_ui.go @@ -161,35 +161,10 @@ func handleExports(ctx *gin.Context) { ctx.JSON(http.StatusOK, res) } -var GeneratePEM = config.GeneratePEM - -// Create a new private key in the given directory, but without loading it immediately -// This new key will be detected and loaded later by the ongoing goroutine `LaunchIssuerKeysDirRefresh`, -// which periodically refreshes the keys in the issuer keys directory -func createNewIssuerKey(ctx *gin.Context) { - issuerKeysDir := param.IssuerKeysDirectory.GetString() - - _, err := GeneratePEM(issuerKeysDir) - if err != nil { - log.Errorf("Error creating a new private key in a new .pem file: %v", err) - ctx.JSON(http.StatusInternalServerError, server_structs.SimpleApiResp{ - Status: server_structs.RespFailed, - Msg: "Error creating a new private key in a new .pem file"}) - return - } - - ctx.JSON(http.StatusOK, - server_structs.SimpleApiResp{ - Status: server_structs.RespOK, - Msg: "Created a new issuer key", - }) -} - func RegisterOriginWebAPI(engine *gin.Engine) error { originWebAPI := engine.Group("/api/v1.0/origin_ui") { originWebAPI.GET("/exports", web_ui.AuthHandler, web_ui.AdminAuthHandler, handleExports) - originWebAPI.GET("/newIssuerKey", web_ui.AuthHandler, web_ui.AdminAuthHandler, createNewIssuerKey) } // Globus backend specific. Config other origin routes above this line diff --git a/origin/origin_ui_test.go b/origin/origin_ui_test.go deleted file mode 100644 index 409a4903e..000000000 --- a/origin/origin_ui_test.go +++ /dev/null @@ -1,99 +0,0 @@ -/*************************************************************** - * - * Copyright (C) 2024, Pelican Project, Morgridge Institute for Research - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You may - * obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ***************************************************************/ - -package origin - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "testing" - - "github.com/gin-gonic/gin" - "github.com/lestrrat-go/jwx/v2/jwk" - "github.com/spf13/viper" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/pelicanplatform/pelican/config" - "github.com/pelicanplatform/pelican/server_structs" - "github.com/pelicanplatform/pelican/server_utils" -) - -func TestSuccessfulCreateNewIssuerKey(t *testing.T) { - server_utils.ResetTestState() - - tDir := t.TempDir() - iksDir := filepath.Join(tDir, "test-issuer-keys") - viper.Set("IssuerKeysDirectory", iksDir) - viper.Set("ConfigDir", t.TempDir()) - config.InitConfig() - - router := gin.Default() - router.GET("/api/v1.0/origin_ui/newIssuerKey", createNewIssuerKey) - w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", "/api/v1.0/origin_ui/newIssuerKey", nil) - router.ServeHTTP(w, req) - - assert.Equal(t, http.StatusOK, w.Code) - - var response server_structs.SimpleApiResp - err := json.Unmarshal(w.Body.Bytes(), &response) - require.NoError(t, err) - assert.Equal(t, server_structs.RespOK, response.Status) - assert.Equal(t, "Created a new issuer key", response.Msg) - - files, err := os.ReadDir(iksDir) - require.NoError(t, err) - require.Len(t, files, 1, "A new .pem file should be created in the directory") - newKeyFile := filepath.Join(iksDir, files[0].Name()) - assert.FileExists(t, newKeyFile, "The new .pem file does not exist") -} - -func TestFailedCreateNewIssuerKey(t *testing.T) { - server_utils.ResetTestState() - - tDir := t.TempDir() - viper.Set("IssuerKeysDirectory", filepath.Join(tDir, "test-issuer-keys")) - viper.Set("ConfigDir", t.TempDir()) - config.InitConfig() - - router := gin.Default() - router.GET("/api/v1.0/origin_ui/newIssuerKey", createNewIssuerKey) - - // Mock GeneratePEM - originalGeneratePEM := GeneratePEM - GeneratePEM = func(dir string) (jwk.Key, error) { - return nil, assert.AnError - } - defer func() { GeneratePEM = originalGeneratePEM }() - - w := httptest.NewRecorder() - req, _ := http.NewRequest("GET", "/api/v1.0/origin_ui/newIssuerKey", nil) - router.ServeHTTP(w, req) - - assert.Equal(t, http.StatusInternalServerError, w.Code) - - var response server_structs.SimpleApiResp - err := json.Unmarshal(w.Body.Bytes(), &response) - require.NoError(t, err) - assert.Equal(t, server_structs.RespFailed, response.Status) - assert.Equal(t, "Error creating a new private key in a new .pem file", response.Msg) -} diff --git a/registry/client_commands.go b/registry/client_commands.go index 00f359f45..87535143c 100644 --- a/registry/client_commands.go +++ b/registry/client_commands.go @@ -288,8 +288,7 @@ func NamespacesPubKeyUpdate(privateKey jwk.Key, prefixes []string, siteName stri if err != nil { return errors.Wrapf(err, "failed to generate public key for namespace registration") } - err = jwk.AssignKeyID(publicKey) - if err != nil { + if err = jwk.AssignKeyID(publicKey); err != nil { return errors.Wrap(err, "failed to assign key ID to public key") } if err = publicKey.Set("alg", "ES256"); err != nil { diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index e1984c430..190613e20 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -21,7 +21,10 @@ package registry import ( "context" "encoding/json" + "io" + "net/http" "net/http/httptest" + "os" "path/filepath" "sort" "testing" @@ -40,13 +43,15 @@ import ( ) func registryMockup(ctx context.Context, t *testing.T, testName string) *httptest.Server { - issuerTempDir := filepath.Join(t.TempDir(), testName) + tDir := t.TempDir() + issuerTempDir := filepath.Join(tDir, testName) ikeyDir := filepath.Join(issuerTempDir, "issuer-keys") viper.Set("IssuerKeysDirectory", ikeyDir) viper.Set("Registry.DbLocation", filepath.Join(issuerTempDir, "test.sql")) viper.Set("Server.WebPort", 8444) - config.InitConfigDir(viper.GetViper()) + viper.Set("ConfigDir", tDir) + config.InitConfig() err := config.InitServer(ctx, server_structs.RegistryType) require.NoError(t, err) @@ -86,124 +91,124 @@ func getSortedKids(ctx context.Context, jsonStr string) ([]string, error) { return kids, nil } -// func TestServeNamespaceRegistry(t *testing.T) { -// ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) -// t.Cleanup(func() { -// func() { require.NoError(t, egrp.Wait()) }() -// cancel() -// config.ResetIssuerJWKPtr() -// config.ResetIssuerPrivateKeys() -// server_utils.ResetTestState() -// }) -// server_utils.ResetTestState() - -// svr := registryMockup(ctx, t, "serveregistry") -// defer func() { -// err := ShutdownRegistryDB() -// assert.NoError(t, err) -// svr.CloseClientConnections() -// svr.Close() -// }() - -// _, err := config.GetIssuerPublicJWKS() -// require.NoError(t, err) -// privKey, err := config.GetIssuerPrivateJWK() -// require.NoError(t, err) - -// //Test functionality of registering a namespace (without identity) -// err = NamespaceRegister(privKey, svr.URL+"/api/v1.0/registry", "", "/foo/bar", "mock_site_name") -// require.NoError(t, err) -// var privKey2 jwk.Key - -// //Test we can list the namespace without an error -// t.Run("Test namespace list", func(t *testing.T) { -// //Set up a buffer to capture stdout -// var stdoutCapture string -// oldStdout := os.Stdout -// r, w, _ := os.Pipe() -// os.Stdout = w - -// //List the namespaces -// err = NamespaceList(svr.URL + "/api/v1.0/registry") -// require.NoError(t, err) -// w.Close() -// os.Stdout = oldStdout - -// capturedOutput := make([]byte, 1024) -// n, _ := r.Read(capturedOutput) -// stdoutCapture = string(capturedOutput[:n]) -// assert.Contains(t, stdoutCapture, `"prefix":"/foo/bar"`) -// }) - -// t.Run("Test register namespace with sitename", func(t *testing.T) { -// res, err := http.Get(svr.URL + "/api/v1.0/registry/foo/bar") -// require.NoError(t, err) -// require.Equal(t, http.StatusOK, res.StatusCode) -// data, err := io.ReadAll(res.Body) -// require.NoError(t, err) -// ns := server_structs.Namespace{} -// err = json.Unmarshal(data, &ns) -// require.NoError(t, err) -// assert.Equal(t, ns.AdminMetadata.SiteName, "mock_site_name") -// }) - -// t.Run("test-registered-namespace-pubkey-update-with-new-active-key", func(t *testing.T) { -// activeKey, err := config.GetIssuerPrivateJWK() -// require.NoError(t, err) -// require.Equal(t, privKey.KeyID(), activeKey.KeyID()) - -// // Imitate LaunchIssuerKeysDirRefresh function -// config.UpdatePreviousIssuerPrivateJWK() -// _, err = config.GeneratePEM(param.IssuerKeysDirectory.GetString()) -// require.NoError(t, err) -// privKey2, err = config.LoadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()) -// require.NoError(t, err) -// err = NamespacesPubKeyUpdate(privKey2, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") -// require.NoError(t, err) -// }) - -// t.Run("test-registered-namespace-pubkey-update-with-nonsense-key", func(t *testing.T) { -// tempDir := filepath.Join(t.TempDir(), "in_the_middle_of_nowhere") -// privKey3, err := config.GeneratePEM(tempDir) -// require.NoError(t, err) -// err = NamespacesPubKeyUpdate(privKey3, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") -// require.ErrorContains(t, err, "it doesn't contain any public key matching the existing namespace's public key in db") -// }) - -// t.Run("test-registered-namespace-pubkey-update-with-imposter-key", func(t *testing.T) { -// privKey4, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) -// require.NoError(t, err) -// config.UpdatePreviousIssuerPrivateJWK() -// // Both active key and previous key are set to privKey4 -// err = NamespacesPubKeyUpdate(privKey4, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") -// require.ErrorContains(t, err, "it fails to pass the proof of possession verification") - -// // Revert the active key changes happened in this subtest -// config.SetActiveKey(privKey) -// config.UpdatePreviousIssuerPrivateJWK() -// config.SetActiveKey(privKey2) -// }) - -// t.Run("Test namespace delete", func(t *testing.T) { -// //Test functionality of namespace delete -// err = NamespaceDelete(svr.URL+"/api/v1.0/registry/foo/bar", "/foo/bar") -// require.NoError(t, err) -// var stdoutCapture string -// oldStdout := os.Stdout -// r, w, _ := os.Pipe() -// os.Stdout = w -// err = NamespaceGet(svr.URL + "/api/v1.0/registry") -// require.NoError(t, err) -// w.Close() -// os.Stdout = oldStdout - -// capturedOutput := make([]byte, 1024) -// n, _ := r.Read(capturedOutput) -// stdoutCapture = string(capturedOutput[:n]) -// assert.Equal(t, "[]\n", stdoutCapture) -// }) -// server_utils.ResetTestState() -// } +func TestServeNamespaceRegistry(t *testing.T) { + ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) + t.Cleanup(func() { + func() { require.NoError(t, egrp.Wait()) }() + cancel() + config.ResetIssuerJWKPtr() + config.ResetIssuerPrivateKeys() + server_utils.ResetTestState() + }) + server_utils.ResetTestState() + + svr := registryMockup(ctx, t, "serveregistry") + defer func() { + err := ShutdownRegistryDB() + assert.NoError(t, err) + svr.CloseClientConnections() + svr.Close() + }() + + _, err := config.GetIssuerPublicJWKS() + require.NoError(t, err) + privKey, err := config.GetIssuerPrivateJWK() + require.NoError(t, err) + + //Test functionality of registering a namespace (without identity) + err = NamespaceRegister(privKey, svr.URL+"/api/v1.0/registry", "", "/foo/bar", "mock_site_name") + require.NoError(t, err) + var privKey2 jwk.Key + + //Test we can list the namespace without an error + t.Run("Test namespace list", func(t *testing.T) { + //Set up a buffer to capture stdout + var stdoutCapture string + oldStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + + //List the namespaces + err = NamespaceList(svr.URL + "/api/v1.0/registry") + require.NoError(t, err) + w.Close() + os.Stdout = oldStdout + + capturedOutput := make([]byte, 1024) + n, _ := r.Read(capturedOutput) + stdoutCapture = string(capturedOutput[:n]) + assert.Contains(t, stdoutCapture, `"prefix":"/foo/bar"`) + }) + + t.Run("Test register namespace with sitename", func(t *testing.T) { + res, err := http.Get(svr.URL + "/api/v1.0/registry/foo/bar") + require.NoError(t, err) + require.Equal(t, http.StatusOK, res.StatusCode) + data, err := io.ReadAll(res.Body) + require.NoError(t, err) + ns := server_structs.Namespace{} + err = json.Unmarshal(data, &ns) + require.NoError(t, err) + assert.Equal(t, ns.AdminMetadata.SiteName, "mock_site_name") + }) + + t.Run("test-registered-namespace-pubkey-update-with-new-active-key", func(t *testing.T) { + activeKey, err := config.GetIssuerPrivateJWK() + require.NoError(t, err) + require.Equal(t, privKey.KeyID(), activeKey.KeyID()) + + // Imitate LaunchIssuerKeysDirRefresh function + config.UpdatePreviousIssuerPrivateJWK() + _, err = config.GeneratePEM(param.IssuerKeysDirectory.GetString()) + require.NoError(t, err) + privKey2, err = config.LoadIssuerPrivateKey(param.IssuerKeysDirectory.GetString()) + require.NoError(t, err) + err = NamespacesPubKeyUpdate(privKey2, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") + require.NoError(t, err) + }) + + t.Run("test-registered-namespace-pubkey-update-with-nonsense-key", func(t *testing.T) { + tempDir := filepath.Join(t.TempDir(), "in_the_middle_of_nowhere") + privKey3, err := config.GeneratePEM(tempDir) + require.NoError(t, err) + err = NamespacesPubKeyUpdate(privKey3, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") + require.ErrorContains(t, err, "it doesn't contain any public key matching the existing namespace's public key in db") + }) + + t.Run("test-registered-namespace-pubkey-update-with-imposter-key", func(t *testing.T) { + privKey4, err := config.GeneratePEMandSetActiveKey(param.IssuerKeysDirectory.GetString()) + require.NoError(t, err) + config.UpdatePreviousIssuerPrivateJWK() + // Both active key and previous key are set to privKey4 + err = NamespacesPubKeyUpdate(privKey4, []string{"/foo/bar"}, "mock_site_name", svr.URL+"/api/v1.0/registry/updateNamespacesPubKey") + require.ErrorContains(t, err, "it fails to pass the proof of possession verification") + + // Revert the active key changes happened in this subtest + config.SetActiveKey(privKey) + config.UpdatePreviousIssuerPrivateJWK() + config.SetActiveKey(privKey2) + }) + + t.Run("Test namespace delete", func(t *testing.T) { + //Test functionality of namespace delete + err = NamespaceDelete(svr.URL+"/api/v1.0/registry/foo/bar", "/foo/bar") + require.NoError(t, err) + var stdoutCapture string + oldStdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + err = NamespaceGet(svr.URL + "/api/v1.0/registry") + require.NoError(t, err) + w.Close() + os.Stdout = oldStdout + + capturedOutput := make([]byte, 1024) + n, _ := r.Read(capturedOutput) + stdoutCapture = string(capturedOutput[:n]) + assert.Equal(t, "[]\n", stdoutCapture) + }) + server_utils.ResetTestState() +} func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) @@ -356,8 +361,11 @@ func TestRegistryKeyChainingOSDF(t *testing.T) { assert.Contains(t, err.Error(), "A superspace or subspace of this namespace /topo/foo already exists in the OSDF topology: /topo/foo. To register a Pelican equivalence, you need to present your identity.") // Now we create a new key and try to use it to register a super/sub space. These shouldn't succeed - viper.Set("IssuerKeysDirectory", t.TempDir()+"/keychaining2") - viper.Set("ConfigDir", t.TempDir()) + config.ResetIssuerJWKPtr() + config.ResetIssuerPrivateKeys() + tDir2 := t.TempDir() + viper.Set("IssuerKeysDirectory", tDir2+"/keychaining2") + viper.Set("ConfigDir", tDir2) config.InitConfig() err = config.InitServer(ctx, server_structs.RegistryType) require.NoError(t, err) From a878208201225a69b349bd1dff7369889bf6c1e4 Mon Sep 17 00:00:00 2001 From: Howard Zhong Date: Tue, 31 Dec 2024 19:00:52 +0000 Subject: [PATCH 64/64] attempt to solve timeout --- registry/client_commands_test.go | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/registry/client_commands_test.go b/registry/client_commands_test.go index 190613e20..4ffb06ce5 100644 --- a/registry/client_commands_test.go +++ b/registry/client_commands_test.go @@ -94,10 +94,12 @@ func getSortedKids(ctx context.Context, jsonStr string) ([]string, error) { func TestServeNamespaceRegistry(t *testing.T) { ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) t.Cleanup(func() { - func() { require.NoError(t, egrp.Wait()) }() + if r := recover(); r != nil { + t.Errorf("Test panicked: %v", r) + } + cancel() - config.ResetIssuerJWKPtr() - config.ResetIssuerPrivateKeys() + assert.NoError(t, egrp.Wait()) server_utils.ResetTestState() }) server_utils.ResetTestState() @@ -214,16 +216,18 @@ func TestMultiPubKeysRegisteredOnNamespace(t *testing.T) { ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) server_utils.ResetTestState() t.Cleanup(func() { - func() { require.NoError(t, egrp.Wait()) }() + if r := recover(); r != nil { + t.Errorf("Test panicked: %v", r) + } + cancel() - config.ResetIssuerJWKPtr() - config.ResetIssuerPrivateKeys() + assert.NoError(t, egrp.Wait()) server_utils.ResetTestState() }) tDir := t.TempDir() - svr := registryMockup(ctx, t, "serveregistry") + svr := registryMockup(ctx, t, "MultiPubKeysRegisteredOnNamespace") defer func() { err := ShutdownRegistryDB() assert.NoError(t, err) @@ -309,6 +313,10 @@ func TestRegistryKeyChainingOSDF(t *testing.T) { ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) t.Cleanup(func() { + if r := recover(); r != nil { + t.Errorf("Test panicked: %v", r) + } + cancel() assert.NoError(t, egrp.Wait()) server_utils.ResetTestState() @@ -410,6 +418,10 @@ func TestRegistryKeyChaining(t *testing.T) { ctx, cancel, egrp := test_utils.TestContext(context.Background(), t) t.Cleanup(func() { + if r := recover(); r != nil { + t.Errorf("Test panicked: %v", r) + } + cancel() assert.NoError(t, egrp.Wait()) server_utils.ResetTestState()