diff --git a/.gitignore b/.gitignore index 3054e9eb3c2..b117122c4bb 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,5 @@ msi __pycache__ *.py[cod] *.egg-info + +.idea diff --git a/cmd/crowdsec-cli/config_show.go b/cmd/crowdsec-cli/config_show.go index 634ca77410e..c277173c387 100644 --- a/cmd/crowdsec-cli/config_show.go +++ b/cmd/crowdsec-cli/config_show.go @@ -100,6 +100,7 @@ API Client: {{- if .API.Server }} Local API Server{{if and .API.Server.Enable (not (ValueBool .API.Server.Enable))}} (disabled){{end}}: - Listen URL : {{.API.Server.ListenURI}} + - Listen Socket : {{.API.Server.ListenSocket}} - Profile File : {{.API.Server.ProfilesPath}} {{- if .API.Server.TLS }} diff --git a/cmd/crowdsec-cli/lapi.go b/cmd/crowdsec-cli/lapi.go index 0bb4a31b72a..bc01158e31d 100644 --- a/cmd/crowdsec-cli/lapi.go +++ b/cmd/crowdsec-cli/lapi.go @@ -44,7 +44,9 @@ func (cli *cliLapi) status() error { password := strfmt.Password(cfg.API.Client.Credentials.Password) login := cfg.API.Client.Credentials.Login - apiurl, err := url.Parse(cfg.API.Client.Credentials.URL) + origURL := cfg.API.Client.Credentials.URL + + apiURL, err := url.Parse(origURL) if err != nil { return fmt.Errorf("parsing api url: %w", err) } @@ -59,7 +61,7 @@ func (cli *cliLapi) status() error { return fmt.Errorf("failed to get scenarios: %w", err) } - Client, err = apiclient.NewDefaultClient(apiurl, + Client, err = apiclient.NewDefaultClient(apiURL, LAPIURLPrefix, fmt.Sprintf("crowdsec/%s", version.String()), nil) @@ -74,7 +76,8 @@ func (cli *cliLapi) status() error { } log.Infof("Loaded credentials from %s", cfg.API.Client.CredentialsFilePath) - log.Infof("Trying to authenticate with username %s on %s", login, apiurl) + // use the original string because apiURL would print 'http://unix/' + log.Infof("Trying to authenticate with username %s on %s", login, origURL) _, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t) if err != nil { @@ -101,23 +104,7 @@ func (cli *cliLapi) register(apiURL string, outputFile string, machine string) e password := strfmt.Password(generatePassword(passwordLength)) - if apiURL == "" { - if cfg.API.Client == nil || cfg.API.Client.Credentials == nil || cfg.API.Client.Credentials.URL == "" { - return fmt.Errorf("no Local API URL. Please provide it in your configuration or with the -u parameter") - } - - apiURL = cfg.API.Client.Credentials.URL - } - /*URL needs to end with /, but user doesn't care*/ - if !strings.HasSuffix(apiURL, "/") { - apiURL += "/" - } - /*URL needs to start with http://, but user doesn't care*/ - if !strings.HasPrefix(apiURL, "http://") && !strings.HasPrefix(apiURL, "https://") { - apiURL = "http://" + apiURL - } - - apiurl, err := url.Parse(apiURL) + apiurl, err := prepareAPIURL(cfg.API.Client, apiURL) if err != nil { return fmt.Errorf("parsing api url: %w", err) } @@ -173,6 +160,27 @@ func (cli *cliLapi) register(apiURL string, outputFile string, machine string) e return nil } +func prepareAPIURL(clientCfg *csconfig.LocalApiClientCfg, apiURL string) (*url.URL, error) { + if apiURL == "" { + if clientCfg == nil || clientCfg.Credentials == nil || clientCfg.Credentials.URL == "" { + return nil, fmt.Errorf("no Local API URL. Please provide it in your configuration or with the -u parameter") + } + apiURL = clientCfg.Credentials.URL + } + + // URL needs to end with /, but user doesn't care + if !strings.HasSuffix(apiURL, "/") { + apiURL += "/" + } + + // URL needs to start with http://, but user doesn't care + if !strings.HasPrefix(apiURL, "http://") && !strings.HasPrefix(apiURL, "https://") && !strings.HasPrefix(apiURL, "/") { + apiURL = "http://" + apiURL + } + + return url.Parse(apiURL) +} + func (cli *cliLapi) newStatusCmd() *cobra.Command { cmdLapiStatus := &cobra.Command{ Use: "status", diff --git a/cmd/crowdsec-cli/lapi_test.go b/cmd/crowdsec-cli/lapi_test.go new file mode 100644 index 00000000000..3d4523a2d21 --- /dev/null +++ b/cmd/crowdsec-cli/lapi_test.go @@ -0,0 +1,47 @@ +package main + +import ( + "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" +) + +func TestPrepareAPIURL_NoProtocol(t *testing.T) { + url, err := prepareAPIURL(nil, "localhost:81") + require.NoError(t, err) + assert.Equal(t, "http://localhost:81/", url.String()) +} + +func TestPrepareAPIURL_Http(t *testing.T) { + url, err := prepareAPIURL(nil, "http://localhost:81") + require.NoError(t, err) + assert.Equal(t, "http://localhost:81/", url.String()) +} + +func TestPrepareAPIURL_Https(t *testing.T) { + url, err := prepareAPIURL(nil, "https://localhost:81") + require.NoError(t, err) + assert.Equal(t, "https://localhost:81/", url.String()) +} + +func TestPrepareAPIURL_UnixSocket(t *testing.T) { + url, err := prepareAPIURL(nil, "/path/socket") + require.NoError(t, err) + assert.Equal(t, "/path/socket/", url.String()) +} + +func TestPrepareAPIURL_Empty(t *testing.T) { + _, err := prepareAPIURL(nil, "") + require.Error(t, err) +} + +func TestPrepareAPIURL_Empty_ConfigOverride(t *testing.T) { + url, err := prepareAPIURL(&csconfig.LocalApiClientCfg{ + Credentials: &csconfig.ApiCredentialsCfg{ + URL: "localhost:80", + }, + }, "") + require.NoError(t, err) + assert.Equal(t, "http://localhost:80/", url.String()) +} diff --git a/cmd/crowdsec-cli/machines.go b/cmd/crowdsec-cli/machines.go index 7c9b9708c92..a75e6e7a566 100644 --- a/cmd/crowdsec-cli/machines.go +++ b/cmd/crowdsec-cli/machines.go @@ -317,8 +317,8 @@ func (cli *cliMachines) add(args []string, machinePassword string, dumpFile stri if apiURL == "" { if clientCfg != nil && clientCfg.Credentials != nil && clientCfg.Credentials.URL != "" { apiURL = clientCfg.Credentials.URL - } else if serverCfg != nil && serverCfg.ListenURI != "" { - apiURL = "http://" + serverCfg.ListenURI + } else if serverCfg.ClientURL() != "" { + apiURL = serverCfg.ClientURL() } else { return fmt.Errorf("unable to dump an api URL. Please provide it in your configuration or with the -u parameter") } diff --git a/docker/test/tests/test_tls.py b/docker/test/tests/test_tls.py index 591afe0d303..fe899b000af 100644 --- a/docker/test/tests/test_tls.py +++ b/docker/test/tests/test_tls.py @@ -22,8 +22,7 @@ def test_missing_key_file(crowdsec, flavor): } with crowdsec(flavor=flavor, environment=env, wait_status=Status.EXITED) as cs: - # XXX: this message appears twice, is that normal? - cs.wait_for_log("*while starting API server: missing TLS key file*") + cs.wait_for_log("*local API server stopped with error: missing TLS key file*") def test_missing_cert_file(crowdsec, flavor): @@ -35,7 +34,7 @@ def test_missing_cert_file(crowdsec, flavor): } with crowdsec(flavor=flavor, environment=env, wait_status=Status.EXITED) as cs: - cs.wait_for_log("*while starting API server: missing TLS cert file*") + cs.wait_for_log("*local API server stopped with error: missing TLS cert file*") def test_tls_missing_ca(crowdsec, flavor, certs_dir): diff --git a/pkg/apiclient/auth_jwt.go b/pkg/apiclient/auth_jwt.go index 2ead10cf6da..5314d502826 100644 --- a/pkg/apiclient/auth_jwt.go +++ b/pkg/apiclient/auth_jwt.go @@ -70,9 +70,14 @@ func (t *JWTTransport) refreshJwtToken() error { req.Header.Add("Content-Type", "application/json") + transport := t.Transport + if transport == nil { + transport = http.DefaultTransport + } + client := &http.Client{ Transport: &retryRoundTripper{ - next: http.DefaultTransport, + next: transport, maxAttempts: 5, withBackOff: true, retryStatusCodes: []int{http.StatusTooManyRequests, http.StatusServiceUnavailable, http.StatusGatewayTimeout, http.StatusInternalServerError}, diff --git a/pkg/apiclient/client.go b/pkg/apiclient/client.go index b487f68a698..7add3157ffd 100644 --- a/pkg/apiclient/client.go +++ b/pkg/apiclient/client.go @@ -5,8 +5,10 @@ import ( "crypto/tls" "crypto/x509" "fmt" + "net" "net/http" "net/url" + "strings" "github.com/golang-jwt/jwt/v4" @@ -67,11 +69,15 @@ func NewClient(config *Config) (*ApiClient, error) { MachineID: &config.MachineID, Password: &config.Password, Scenarios: config.Scenarios, - URL: config.URL, UserAgent: config.UserAgent, VersionPrefix: config.VersionPrefix, UpdateScenario: config.UpdateScenario, } + transport, baseUrl := createTransport(config.URL) + if transport != nil { + t.Transport = transport + } + t.URL = baseUrl tlsconfig := tls.Config{InsecureSkipVerify: InsecureSkipVerify} tlsconfig.RootCAs = CaCertPool @@ -84,7 +90,7 @@ func NewClient(config *Config) (*ApiClient, error) { ht.TLSClientConfig = &tlsconfig } - c := &ApiClient{client: t.Client(), BaseURL: config.URL, UserAgent: config.UserAgent, URLPrefix: config.VersionPrefix, PapiURL: config.PapiURL} + c := &ApiClient{client: t.Client(), BaseURL: baseUrl, UserAgent: config.UserAgent, URLPrefix: config.VersionPrefix, PapiURL: config.PapiURL} c.common.client = c c.Decisions = (*DecisionsService)(&c.common) c.Alerts = (*AlertsService)(&c.common) @@ -98,23 +104,26 @@ func NewClient(config *Config) (*ApiClient, error) { } func NewDefaultClient(URL *url.URL, prefix string, userAgent string, client *http.Client) (*ApiClient, error) { + transport, baseUrl := createTransport(URL) if client == nil { client = &http.Client{} - if ht, ok := http.DefaultTransport.(*http.Transport); ok { - tlsconfig := tls.Config{InsecureSkipVerify: InsecureSkipVerify} - tlsconfig.RootCAs = CaCertPool - - if Cert != nil { - tlsconfig.Certificates = []tls.Certificate{*Cert} + if transport != nil { + client.Transport = transport + } else { + if ht, ok := http.DefaultTransport.(*http.Transport); ok { + tlsconfig := tls.Config{InsecureSkipVerify: InsecureSkipVerify} + tlsconfig.RootCAs = CaCertPool + if Cert != nil { + tlsconfig.Certificates = []tls.Certificate{*Cert} + } + ht.TLSClientConfig = &tlsconfig + client.Transport = ht } - - ht.TLSClientConfig = &tlsconfig - client.Transport = ht } } - c := &ApiClient{client: client, BaseURL: URL, UserAgent: userAgent, URLPrefix: prefix} + c := &ApiClient{client: client, BaseURL: baseUrl, UserAgent: userAgent, URLPrefix: prefix} c.common.client = c c.Decisions = (*DecisionsService)(&c.common) c.Alerts = (*AlertsService)(&c.common) @@ -128,18 +137,24 @@ func NewDefaultClient(URL *url.URL, prefix string, userAgent string, client *htt } func RegisterClient(config *Config, client *http.Client) (*ApiClient, error) { + transport, baseUrl := createTransport(config.URL) if client == nil { client = &http.Client{} + if transport != nil { + client.Transport = transport + } else { + tlsconfig := tls.Config{InsecureSkipVerify: InsecureSkipVerify} + if Cert != nil { + tlsconfig.RootCAs = CaCertPool + tlsconfig.Certificates = []tls.Certificate{*Cert} + } + http.DefaultTransport.(*http.Transport).TLSClientConfig = &tlsconfig + } + } else if client.Transport == nil && transport != nil { + client.Transport = transport } - tlsconfig := tls.Config{InsecureSkipVerify: InsecureSkipVerify} - if Cert != nil { - tlsconfig.RootCAs = CaCertPool - tlsconfig.Certificates = []tls.Certificate{*Cert} - } - - http.DefaultTransport.(*http.Transport).TLSClientConfig = &tlsconfig - c := &ApiClient{client: client, BaseURL: config.URL, UserAgent: config.UserAgent, URLPrefix: config.VersionPrefix} + c := &ApiClient{client: client, BaseURL: baseUrl, UserAgent: config.UserAgent, URLPrefix: config.VersionPrefix} c.common.client = c c.Decisions = (*DecisionsService)(&c.common) c.Alerts = (*AlertsService)(&c.common) @@ -158,6 +173,26 @@ func RegisterClient(config *Config, client *http.Client) (*ApiClient, error) { return c, nil } +func createTransport(url *url.URL) (*http.Transport, *url.URL) { + urlString := url.String() + + // TCP transport + if !strings.HasPrefix(urlString, "/") { + return nil, url + } + + // Unix transport + url.Path = "/" + url.Host = "unix" + url.Scheme = "http" + + return &http.Transport{ + DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial("unix", strings.TrimSuffix(urlString, "/")) + }, + }, url +} + type Response struct { Response *http.Response //add our pagination stuff diff --git a/pkg/apiclient/client_test.go b/pkg/apiclient/client_test.go index dc6eae16926..95dcc80a404 100644 --- a/pkg/apiclient/client_test.go +++ b/pkg/apiclient/client_test.go @@ -3,10 +3,13 @@ package apiclient import ( "context" "fmt" + "net" "net/http" "net/http/httptest" "net/url" + "path" "runtime" + "strings" "testing" log "github.com/sirupsen/logrus" @@ -34,12 +37,49 @@ func setupWithPrefix(urlPrefix string) (*http.ServeMux, string, func()) { apiHandler := http.NewServeMux() apiHandler.Handle(baseURLPath+"/", http.StripPrefix(baseURLPath, mux)) - // server is a test HTTP server used to provide mock API responses. server := httptest.NewServer(apiHandler) return mux, server.URL, server.Close } +// toUNCPath converts a Windows file path to a UNC path. +func toUNCPath(path string) (string, error) { + colonIdx := strings.Index(path, ":") + if colonIdx == -1 { + return "", fmt.Errorf("invalid path format, missing drive letter: %s", path) + } + + // convert backslashes to forward slashes + remaining := strings.ReplaceAll(path[colonIdx+1:], "\\", "/") + + uncPath := fmt.Sprintf("//localhost/%s$%s", path[colonIdx-1:colonIdx], remaining) + + return uncPath, nil +} + +func setupUnixSocketWithPrefix(socket string, urlPrefix string) (mux *http.ServeMux, serverURL string, teardown func()) { + var err error + if runtime.GOOS == "windows" { + socket, err = toUNCPath(socket) + if err != nil { + log.Fatalf("converting to UNC path: %s", err) + } + } + + mux = http.NewServeMux() + baseURLPath := "/" + urlPrefix + + apiHandler := http.NewServeMux() + apiHandler.Handle(baseURLPath+"/", http.StripPrefix(baseURLPath, mux)) + + server := httptest.NewUnstartedServer(apiHandler) + l, _ := net.Listen("unix", socket) + _ = server.Listener.Close() + server.Listener = l + server.Start() + return mux, socket, server.Close +} + func testMethod(t *testing.T, r *http.Request, want string) { t.Helper() assert.Equal(t, want, r.Method) @@ -77,6 +117,46 @@ func TestNewClientOk(t *testing.T) { assert.Equal(t, http.StatusOK, resp.Response.StatusCode) } +func TestNewClientOk_UnixSocket(t *testing.T) { + tmpDir := t.TempDir() + socket := path.Join(tmpDir, "socket") + + mux, urlx, teardown := setupUnixSocketWithPrefix(socket, "v1") + defer teardown() + apiURL, err := url.Parse(urlx) + if err != nil { + t.Fatalf("parsing api url: %s", apiURL) + } + client, err := NewClient(&Config{ + MachineID: "test_login", + Password: "test_password", + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), + URL: apiURL, + VersionPrefix: "v1", + }) + if err != nil { + t.Fatalf("new api client: %s", err) + } + /*mock login*/ + mux.HandleFunc("/watchers/login", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"code": 200, "expire": "2030-01-02T15:04:05Z", "token": "oklol"}`)) + }) + + mux.HandleFunc("/alerts", func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "GET") + w.WriteHeader(http.StatusOK) + }) + + _, resp, err := client.Alerts.List(context.Background(), AlertsListOpts{}) + if err != nil { + t.Fatalf("test Unable to list alerts : %+v", err) + } + if resp.Response.StatusCode != http.StatusOK { + t.Fatalf("Alerts.List returned status: %d, want %d", resp.Response.StatusCode, http.StatusCreated) + } +} + func TestNewClientKo(t *testing.T) { mux, urlx, teardown := setup() defer teardown() @@ -131,6 +211,28 @@ func TestNewDefaultClient(t *testing.T) { log.Printf("err-> %s", err) } +func TestNewDefaultClient_UnixSocket(t *testing.T) { + tmpDir := t.TempDir() + socket := path.Join(tmpDir, "socket") + mux, urlx, teardown := setupUnixSocketWithPrefix(socket, "v1") + defer teardown() + apiURL, err := url.Parse(urlx) + if err != nil { + t.Fatalf("parsing api url: %s", apiURL) + } + client, err := NewDefaultClient(apiURL, "/v1", "", nil) + if err != nil { + t.Fatalf("new api client: %s", err) + } + mux.HandleFunc("/alerts", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + w.Write([]byte(`{"code": 401, "message" : "brr"}`)) + }) + _, _, err = client.Alerts.List(context.Background(), AlertsListOpts{}) + assert.Contains(t, err.Error(), `performing request: API error: brr`) + log.Printf("err-> %s", err) +} + func TestNewClientRegisterKO(t *testing.T) { apiURL, err := url.Parse("http://127.0.0.1:4242/") require.NoError(t, err) @@ -143,10 +245,10 @@ func TestNewClientRegisterKO(t *testing.T) { VersionPrefix: "v1", }, &http.Client{}) - if runtime.GOOS != "windows" { - cstest.RequireErrorContains(t, err, "dial tcp 127.0.0.1:4242: connect: connection refused") - } else { + if runtime.GOOS == "windows" { cstest.RequireErrorContains(t, err, " No connection could be made because the target machine actively refused it.") + } else { + cstest.RequireErrorContains(t, err, "dial tcp 127.0.0.1:4242: connect: connection refused") } } @@ -178,6 +280,37 @@ func TestNewClientRegisterOK(t *testing.T) { log.Printf("->%T", client) } +func TestNewClientRegisterOK_UnixSocket(t *testing.T) { + log.SetLevel(log.TraceLevel) + tmpDir := t.TempDir() + socket := path.Join(tmpDir, "socket") + mux, urlx, teardown := setupUnixSocketWithPrefix(socket, "v1") + defer teardown() + + /*mock login*/ + mux.HandleFunc("/watchers", func(w http.ResponseWriter, r *http.Request) { + testMethod(t, r, "POST") + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"code": 200, "expire": "2030-01-02T15:04:05Z", "token": "oklol"}`)) + }) + + apiURL, err := url.Parse(urlx) + if err != nil { + t.Fatalf("parsing api url: %s", apiURL) + } + client, err := RegisterClient(&Config{ + MachineID: "test_login", + Password: "test_password", + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), + URL: apiURL, + VersionPrefix: "v1", + }, &http.Client{}) + if err != nil { + t.Fatalf("while registering client : %s", err) + } + log.Printf("->%T", client) +} + func TestNewClientBadAnswer(t *testing.T) { log.SetLevel(log.TraceLevel) diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 19a0085d2dc..0207a2fa1dc 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -32,6 +32,7 @@ const keyLength = 32 type APIServer struct { URL string + UnixSocket string TLS *csconfig.TLSCfg dbClient *database.Client logFile string @@ -176,6 +177,13 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) { router.ForwardedByClientIP = false + // set the remore address of the request to 127.0.0.1 if it comes from a unix socket + router.Use(func(c *gin.Context) { + if c.Request.RemoteAddr == "@" { + c.Request.RemoteAddr = "127.0.0.1:65535" + } + }) + if config.TrustedProxies != nil && config.UseForwardedForHeaders { if err = router.SetTrustedProxies(*config.TrustedProxies); err != nil { return nil, fmt.Errorf("while setting trusted_proxies: %w", err) @@ -267,6 +275,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) { return &APIServer{ URL: config.ListenURI, + UnixSocket: config.ListenSocket, TLS: config.TLS, logFile: logFile, dbClient: dbClient, @@ -353,29 +362,29 @@ func (s *APIServer) Run(apiReady chan bool) error { }) } - s.httpServerTomb.Go(func() error { s.listenAndServeURL(apiReady); return nil }) + s.httpServerTomb.Go(func() error { return s.listenAndServeLAPI(apiReady) }) + + if err := s.httpServerTomb.Wait(); err != nil { + return fmt.Errorf("local API server stopped with error: %w", err) + } return nil } -// listenAndServeURL starts the http server and blocks until it's closed +// listenAndServeLAPI starts the http server and blocks until it's closed // it also updates the URL field with the actual address the server is listening on // it's meant to be run in a separate goroutine -func (s *APIServer) listenAndServeURL(apiReady chan bool) { - serverError := make(chan error, 1) - - go func() { - listener, err := net.Listen("tcp", s.URL) - if err != nil { - serverError <- fmt.Errorf("listening on %s: %w", s.URL, err) - return - } - - s.URL = listener.Addr().String() - log.Infof("CrowdSec Local API listening on %s", s.URL) - apiReady <- true +func (s *APIServer) listenAndServeLAPI(apiReady chan bool) error { + var ( + tcpListener net.Listener + unixListener net.Listener + err error + serverError = make(chan error, 2) + listenerClosed = make(chan struct{}) + ) - if s.TLS != nil && (s.TLS.CertFilePath != "" || s.TLS.KeyFilePath != "") { + startServer := func(listener net.Listener, canTLS bool) { + if canTLS && s.TLS != nil && (s.TLS.CertFilePath != "" || s.TLS.KeyFilePath != "") { if s.TLS.KeyFilePath == "" { serverError <- errors.New("missing TLS key file") return @@ -392,24 +401,62 @@ func (s *APIServer) listenAndServeURL(apiReady chan bool) { } if err != nil && err != http.ErrServerClosed { - serverError <- fmt.Errorf("while serving local API: %w", err) + serverError <- err + } + } + + // Starting TCP listener + go func() { + if s.URL == "" { return } + + tcpListener, err = net.Listen("tcp", s.URL) + if err != nil { + serverError <- fmt.Errorf("listening on %s: %v", s.URL, err) + return + } + log.Infof("CrowdSec Local API listening on %s", s.URL) + startServer(tcpListener, true) + }() + + // Starting Unix socket listener + go func() { + if s.UnixSocket == "" { + return + } + + _ = os.RemoveAll(s.UnixSocket) + unixListener, err = net.Listen("unix", s.UnixSocket) + if err != nil { + serverError <- fmt.Errorf("while creating unix listener: %v", err) + return + } + log.Infof("CrowdSec Local API listening on Unix socket %s", s.UnixSocket) + startServer(unixListener, false) }() + apiReady <- true + select { case err := <-serverError: - log.Fatalf("while starting API server: %s", err) + return err case <-s.httpServerTomb.Dying(): log.Infof("Shutting down API server") - // do we need a graceful shutdown here? ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() if err := s.httpServer.Shutdown(ctx); err != nil { - log.Errorf("while shutting down http server: %s", err) + log.Errorf("while shutting down http server: %v", err) + } + close(listenerClosed) + case <-listenerClosed: + if s.UnixSocket != "" { + _ = os.RemoveAll(s.UnixSocket) } } + + return nil } func (s *APIServer) Close() { diff --git a/pkg/apiserver/controllers/v1/alerts.go b/pkg/apiserver/controllers/v1/alerts.go index ad183e4ba80..a767ac9ae00 100644 --- a/pkg/apiserver/controllers/v1/alerts.go +++ b/pkg/apiserver/controllers/v1/alerts.go @@ -323,7 +323,7 @@ func (c *Controller) DeleteAlertByID(gctx *gin.Context) { var err error incomingIP := gctx.ClientIP() - if incomingIP != "127.0.0.1" && incomingIP != "::1" && !networksContainIP(c.TrustedIPs, incomingIP) { + if incomingIP != "127.0.0.1" && incomingIP != "::1" && !networksContainIP(c.TrustedIPs, incomingIP) && !isUnixSocket(gctx) { gctx.JSON(http.StatusForbidden, gin.H{"message": fmt.Sprintf("access forbidden from this IP (%s)", incomingIP)}) return } @@ -349,7 +349,7 @@ func (c *Controller) DeleteAlertByID(gctx *gin.Context) { // DeleteAlerts deletes alerts from the database based on the specified filter func (c *Controller) DeleteAlerts(gctx *gin.Context) { incomingIP := gctx.ClientIP() - if incomingIP != "127.0.0.1" && incomingIP != "::1" && !networksContainIP(c.TrustedIPs, incomingIP) { + if incomingIP != "127.0.0.1" && incomingIP != "::1" && !networksContainIP(c.TrustedIPs, incomingIP) && !isUnixSocket(gctx) { gctx.JSON(http.StatusForbidden, gin.H{"message": fmt.Sprintf("access forbidden from this IP (%s)", incomingIP)}) return } diff --git a/pkg/apiserver/controllers/v1/utils.go b/pkg/apiserver/controllers/v1/utils.go index 6f14dd9204e..3f0269ba0a2 100644 --- a/pkg/apiserver/controllers/v1/utils.go +++ b/pkg/apiserver/controllers/v1/utils.go @@ -2,7 +2,9 @@ package v1 import ( "errors" + "net" "net/http" + "strings" jwt "github.com/appleboy/gin-jwt/v2" "github.com/gin-gonic/gin" @@ -25,6 +27,14 @@ func getBouncerFromContext(ctx *gin.Context) (*ent.Bouncer, error) { return bouncerInfo, nil } +func isUnixSocket(c *gin.Context) bool { + if localAddr, ok := c.Request.Context().Value(http.LocalAddrContextKey).(net.Addr); ok { + return strings.HasPrefix(localAddr.Network(), "unix") + } + + return false +} + func getMachineIDFromContext(ctx *gin.Context) (string, error) { claims := jwt.ExtractClaims(ctx) if claims == nil { @@ -47,8 +57,14 @@ func getMachineIDFromContext(ctx *gin.Context) (string, error) { func (c *Controller) AbortRemoteIf(option bool) gin.HandlerFunc { return func(gctx *gin.Context) { + if !option { + return + } + if isUnixSocket(gctx) { + return + } incomingIP := gctx.ClientIP() - if option && incomingIP != "127.0.0.1" && incomingIP != "::1" { + if incomingIP != "127.0.0.1" && incomingIP != "::1" { gctx.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"}) gctx.Abort() } diff --git a/pkg/apiserver/middlewares/v1/api_key.go b/pkg/apiserver/middlewares/v1/api_key.go index 4e273371bfe..bef79eb3fd0 100644 --- a/pkg/apiserver/middlewares/v1/api_key.go +++ b/pkg/apiserver/middlewares/v1/api_key.go @@ -139,8 +139,10 @@ func (a *APIKey) MiddlewareFunc() gin.HandlerFunc { return func(c *gin.Context) { var bouncer *ent.Bouncer + clientIP := c.ClientIP() + logger := log.WithFields(log.Fields{ - "ip": c.ClientIP(), + "ip": clientIP, }) if c.Request.TLS != nil && len(c.Request.TLS.PeerCertificates) > 0 { @@ -160,7 +162,7 @@ func (a *APIKey) MiddlewareFunc() gin.HandlerFunc { }) if bouncer.IPAddress == "" { - if err := a.DbClient.UpdateBouncerIP(c.ClientIP(), bouncer.ID); err != nil { + if err := a.DbClient.UpdateBouncerIP(clientIP, bouncer.ID); err != nil { logger.Errorf("Failed to update ip address for '%s': %s\n", bouncer.Name, err) c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"}) c.Abort() @@ -170,10 +172,10 @@ func (a *APIKey) MiddlewareFunc() gin.HandlerFunc { } //Don't update IP on HEAD request, as it's used by the appsec to check the validity of the API key provided - if bouncer.IPAddress != c.ClientIP() && bouncer.IPAddress != "" && c.Request.Method != http.MethodHead { - log.Warningf("new IP address detected for bouncer '%s': %s (old: %s)", bouncer.Name, c.ClientIP(), bouncer.IPAddress) + if bouncer.IPAddress != clientIP && bouncer.IPAddress != "" && c.Request.Method != http.MethodHead { + log.Warningf("new IP address detected for bouncer '%s': %s (old: %s)", bouncer.Name, clientIP, bouncer.IPAddress) - if err := a.DbClient.UpdateBouncerIP(c.ClientIP(), bouncer.ID); err != nil { + if err := a.DbClient.UpdateBouncerIP(clientIP, bouncer.ID); err != nil { logger.Errorf("Failed to update ip address for '%s': %s\n", bouncer.Name, err) c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"}) c.Abort() @@ -199,6 +201,5 @@ func (a *APIKey) MiddlewareFunc() gin.HandlerFunc { } c.Set(BouncerContextKey, bouncer) - c.Next() } } diff --git a/pkg/apiserver/middlewares/v1/jwt.go b/pkg/apiserver/middlewares/v1/jwt.go index 6fe053713bc..8f4c6659483 100644 --- a/pkg/apiserver/middlewares/v1/jwt.go +++ b/pkg/apiserver/middlewares/v1/jwt.go @@ -213,18 +213,20 @@ func (j *JWT) Authenticator(c *gin.Context) (interface{}, error) { } } + clientIP := c.ClientIP() + if auth.clientMachine.IpAddress == "" { - err = j.DbClient.UpdateMachineIP(c.ClientIP(), auth.clientMachine.ID) + err = j.DbClient.UpdateMachineIP(clientIP, auth.clientMachine.ID) if err != nil { log.Errorf("Failed to update ip address for '%s': %s\n", auth.machineID, err) return nil, jwt.ErrFailedAuthentication } } - if auth.clientMachine.IpAddress != c.ClientIP() && auth.clientMachine.IpAddress != "" { - log.Warningf("new IP address detected for machine '%s': %s (old: %s)", auth.clientMachine.MachineId, c.ClientIP(), auth.clientMachine.IpAddress) + if auth.clientMachine.IpAddress != clientIP && auth.clientMachine.IpAddress != "" { + log.Warningf("new IP address detected for machine '%s': %s (old: %s)", auth.clientMachine.MachineId, clientIP, auth.clientMachine.IpAddress) - err = j.DbClient.UpdateMachineIP(c.ClientIP(), auth.clientMachine.ID) + err = j.DbClient.UpdateMachineIP(clientIP, auth.clientMachine.ID) if err != nil { log.Errorf("Failed to update ip address for '%s': %s\n", auth.clientMachine.MachineId, err) return nil, jwt.ErrFailedAuthentication @@ -233,13 +235,13 @@ func (j *JWT) Authenticator(c *gin.Context) (interface{}, error) { useragent := strings.Split(c.Request.UserAgent(), "/") if len(useragent) != 2 { - log.Warningf("bad user agent '%s' from '%s'", c.Request.UserAgent(), c.ClientIP()) + log.Warningf("bad user agent '%s' from '%s'", c.Request.UserAgent(), clientIP) return nil, jwt.ErrFailedAuthentication } if err := j.DbClient.UpdateMachineVersion(useragent[1], auth.clientMachine.ID); err != nil { log.Errorf("unable to update machine '%s' version '%s': %s", auth.clientMachine.MachineId, useragent[1], err) - log.Errorf("bad user agent from : %s", c.ClientIP()) + log.Errorf("bad user agent from : %s", clientIP) return nil, jwt.ErrFailedAuthentication } diff --git a/pkg/csconfig/api.go b/pkg/csconfig/api.go index 7fd1f588897..cf829a5aa6c 100644 --- a/pkg/csconfig/api.go +++ b/pkg/csconfig/api.go @@ -141,12 +141,25 @@ func (l *LocalApiClientCfg) Load() error { } if l.Credentials != nil && l.Credentials.URL != "" { - if !strings.HasSuffix(l.Credentials.URL, "/") { + // don't append a trailing slash if the URL is a unix socket + if strings.HasPrefix(l.Credentials.URL, "http") && !strings.HasSuffix(l.Credentials.URL, "/") { l.Credentials.URL += "/" } } - if l.Credentials.Login != "" && (l.Credentials.CertPath != "" || l.Credentials.KeyPath != "") { + // is the configuration asking for client authentication via TLS? + credTLSClientAuth := l.Credentials.CertPath != "" || l.Credentials.KeyPath != "" + + // is the configuration asking for TLS encryption and server authentication? + credTLS := credTLSClientAuth || l.Credentials.CACertPath != "" + + credSocket := strings.HasPrefix(l.Credentials.URL, "/") + + if credTLS && credSocket { + return errors.New("cannot use TLS with a unix socket") + } + + if credTLSClientAuth && l.Credentials.Login != "" { return errors.New("user/password authentication and TLS authentication are mutually exclusive") } @@ -187,10 +200,9 @@ func (l *LocalApiClientCfg) Load() error { return nil } -func (lapiCfg *LocalApiServerCfg) GetTrustedIPs() ([]net.IPNet, error) { +func (c *LocalApiServerCfg) GetTrustedIPs() ([]net.IPNet, error) { trustedIPs := make([]net.IPNet, 0) - - for _, ip := range lapiCfg.TrustedIPs { + for _, ip := range c.TrustedIPs { cidr := toValidCIDR(ip) _, ipNet, err := net.ParseCIDR(cidr) @@ -225,6 +237,7 @@ type CapiWhitelist struct { type LocalApiServerCfg struct { Enable *bool `yaml:"enable"` ListenURI string `yaml:"listen_uri,omitempty"` // 127.0.0.1:8080 + ListenSocket string `yaml:"listen_socket,omitempty"` TLS *TLSCfg `yaml:"tls"` DbConfig *DatabaseCfg `yaml:"-"` LogDir string `yaml:"-"` @@ -248,6 +261,19 @@ type LocalApiServerCfg struct { CapiWhitelists *CapiWhitelist `yaml:"-"` } +func (c *LocalApiServerCfg) ClientURL() string { + if c == nil { + return "" + } + if c.ListenSocket != "" { + return c.ListenSocket + } + if c.ListenURI != "" { + return fmt.Sprintf("http://%s", c.ListenURI) + } + return "" +} + func (c *Config) LoadAPIServer(inCli bool) error { if c.DisableAPI { log.Warning("crowdsec local API is disabled from flag") @@ -273,8 +299,8 @@ func (c *Config) LoadAPIServer(inCli bool) error { return nil } - if c.API.Server.ListenURI == "" { - return errors.New("no listen_uri specified") + if c.API.Server.ListenURI == "" && c.API.Server.ListenSocket == "" { + return errors.New("no listen_uri or listen_socket specified") } // inherit log level from common, then api->server @@ -393,21 +419,21 @@ func parseCapiWhitelists(fd io.Reader) (*CapiWhitelist, error) { return ret, nil } -func (s *LocalApiServerCfg) LoadCapiWhitelists() error { - if s.CapiWhitelistsPath == "" { +func (c *LocalApiServerCfg) LoadCapiWhitelists() error { + if c.CapiWhitelistsPath == "" { return nil } - fd, err := os.Open(s.CapiWhitelistsPath) + fd, err := os.Open(c.CapiWhitelistsPath) if err != nil { return fmt.Errorf("while opening capi whitelist file: %w", err) } defer fd.Close() - s.CapiWhitelists, err = parseCapiWhitelists(fd) + c.CapiWhitelists, err = parseCapiWhitelists(fd) if err != nil { - return fmt.Errorf("while parsing capi whitelist file '%s': %w", s.CapiWhitelistsPath, err) + return fmt.Errorf("while parsing capi whitelist file '%s': %w", c.CapiWhitelistsPath, err) } return nil diff --git a/test/bats/01_crowdsec_lapi.bats b/test/bats/01_crowdsec_lapi.bats index 233340e500f..1b7940615ed 100644 --- a/test/bats/01_crowdsec_lapi.bats +++ b/test/bats/01_crowdsec_lapi.bats @@ -32,20 +32,20 @@ teardown() { } @test "lapi (no .api.server.listen_uri)" { - rune -0 config_set 'del(.api.server.listen_uri)' + rune -0 config_set 'del(.api.server.listen_socket) | del(.api.server.listen_uri)' rune -1 "${CROWDSEC}" -no-cs - assert_stderr --partial "no listen_uri specified" + assert_stderr --partial "no listen_uri or listen_socket specified" } @test "lapi (bad .api.server.listen_uri)" { - rune -0 config_set '.api.server.listen_uri="127.0.0.1:-80"' + rune -0 config_set 'del(.api.server.listen_socket) | .api.server.listen_uri="127.0.0.1:-80"' rune -1 "${CROWDSEC}" -no-cs - assert_stderr --partial "while starting API server: listening on 127.0.0.1:-80: listen tcp: address -80: invalid port" + assert_stderr --partial "local API server stopped with error: listening on 127.0.0.1:-80: listen tcp: address -80: invalid port" } @test "lapi (listen on random port)" { config_set '.common.log_media="stdout"' - rune -0 config_set '.api.server.listen_uri="127.0.0.1:0"' + rune -0 config_set 'del(.api.server.listen_socket) | .api.server.listen_uri="127.0.0.1:0"' rune -0 wait-for --err "CrowdSec Local API listening on 127.0.0.1:" "${CROWDSEC}" -no-cs } diff --git a/test/bats/01_cscli.bats b/test/bats/01_cscli.bats index 60a65b98d58..5f7ff30a149 100644 --- a/test/bats/01_cscli.bats +++ b/test/bats/01_cscli.bats @@ -100,10 +100,14 @@ teardown() { # check that LAPI configuration is loaded (human and json, not shows in raw) + sock=$(config_get '.api.server.listen_socket') + rune -0 cscli config show -o human assert_line --regexp ".*- URL +: http://127.0.0.1:8080/" assert_line --regexp ".*- Login +: githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})?" assert_line --regexp ".*- Credentials File +: .*/local_api_credentials.yaml" + assert_line --regexp ".*- Listen URL +: 127.0.0.1:8080" + assert_line --regexp ".*- Listen Socket +: $sock" rune -0 cscli config show -o json rune -0 jq -c '.API.Client.Credentials | [.url,.login[0:32]]' <(output) @@ -212,7 +216,6 @@ teardown() { assert_stderr --partial "Loaded credentials from" assert_stderr --partial "Trying to authenticate with username" - assert_stderr --partial " on http://127.0.0.1:8080/" assert_stderr --partial "You can successfully interact with Local API (LAPI)" } diff --git a/test/bats/09_socket.bats b/test/bats/09_socket.bats new file mode 100644 index 00000000000..f770abaad2e --- /dev/null +++ b/test/bats/09_socket.bats @@ -0,0 +1,158 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + sockdir=$(TMPDIR="$BATS_FILE_TMPDIR" mktemp -u) + export sockdir + mkdir -p "$sockdir" + socket="$sockdir/crowdsec_api.sock" + export socket + LOCAL_API_CREDENTIALS=$(config_get '.api.client.credentials_path') + export LOCAL_API_CREDENTIALS +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + config_set ".api.server.listen_socket=strenv(socket)" +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli - connects from existing machine with socket" { + config_set "$LOCAL_API_CREDENTIALS" ".url=strenv(socket)" + + ./instance-crowdsec start + + rune -0 cscli lapi status + assert_stderr --regexp "Trying to authenticate with username .* on $socket" + assert_stderr --partial "You can successfully interact with Local API (LAPI)" +} + +@test "crowdsec - listen on both socket and TCP" { + ./instance-crowdsec start + + rune -0 cscli lapi status + assert_stderr --regexp "Trying to authenticate with username .* on http://127.0.0.1:8080/" + assert_stderr --partial "You can successfully interact with Local API (LAPI)" + + config_set "$LOCAL_API_CREDENTIALS" ".url=strenv(socket)" + + rune -0 cscli lapi status + assert_stderr --regexp "Trying to authenticate with username .* on $socket" + assert_stderr --partial "You can successfully interact with Local API (LAPI)" +} + +@test "cscli - authenticate new machine with socket" { + # verify that if a listen_uri and a socket are set, the socket is used + # by default when creating a local machine. + + rune -0 cscli machines delete "$(cscli machines list -o json | jq -r '.[].machineId')" + + # this one should be using the socket + rune -0 cscli machines add --auto --force + + using=$(config_get "$LOCAL_API_CREDENTIALS" ".url") + + assert [ "$using" = "$socket" ] + + # disable the agent because it counts as a first authentication + config_disable_agent + ./instance-crowdsec start + + # the machine does not have an IP yet + + rune -0 cscli machines list -o json + rune -0 jq -r '.[].ipAddress' <(output) + assert_output null + + # upon first authentication, it's assigned to localhost + + rune -0 cscli lapi status + + rune -0 cscli machines list -o json + rune -0 jq -r '.[].ipAddress' <(output) + assert_output 127.0.0.1 +} + +bouncer_http() { + URI="$1" + curl -fs -H "X-Api-Key: $API_KEY" "http://localhost:8080$URI" +} + +bouncer_socket() { + URI="$1" + curl -fs -H "X-Api-Key: $API_KEY" --unix-socket "$socket" "http://localhost$URI" +} + +@test "lapi - connects from existing bouncer with socket" { + ./instance-crowdsec start + API_KEY=$(cscli bouncers add testbouncer -o raw) + export API_KEY + + # the bouncer does not have an IP yet + + rune -0 cscli bouncers list -o json + rune -0 jq -r '.[].ip_address' <(output) + assert_output "" + + # upon first authentication, it's assigned to localhost + + rune -0 bouncer_socket '/v1/decisions' + assert_output 'null' + refute_stderr + + rune -0 cscli bouncers list -o json + rune -0 jq -r '.[].ip_address' <(output) + assert_output "127.0.0.1" + + # we can still use TCP of course + + rune -0 bouncer_http '/v1/decisions' + assert_output 'null' + refute_stderr +} + +@test "lapi - listen on socket only" { + config_set "del(.api.server.listen_uri)" + + mkdir -p "$sockdir" + + # agent is not able to connect right now + config_disable_agent + ./instance-crowdsec start + + API_KEY=$(cscli bouncers add testbouncer -o raw) + export API_KEY + + # now we can't + + rune -1 cscli lapi status + assert_stderr --partial "connection refused" + + rune -7 bouncer_http '/v1/decisions' + refute_output + refute_stderr + + # here we can + + config_set "$LOCAL_API_CREDENTIALS" ".url=strenv(socket)" + + rune -0 cscli lapi status + + rune -0 bouncer_socket '/v1/decisions' + assert_output 'null' + refute_stderr +} diff --git a/test/bats/30_machines_tls.bats b/test/bats/30_machines_tls.bats index 311293ca70c..6909c89cb1f 100644 --- a/test/bats/30_machines_tls.bats +++ b/test/bats/30_machines_tls.bats @@ -120,7 +120,50 @@ teardown() { rune -0 jq -c '[. | length, .[0].machineId[0:32], .[0].isValidated, .[0].ipAddress, .[0].auth_type]' <(output) assert_output '[1,"localhost@127.0.0.1",true,"127.0.0.1","tls"]' - cscli machines delete localhost@127.0.0.1 + rune -0 cscli machines delete localhost@127.0.0.1 +} + +@test "a machine can still connect with a unix socket, no TLS" { + sock=$(config_get '.api.server.listen_socket') + export sock + + # an agent is a machine too + config_disable_agent + ./instance-crowdsec start + + rune -0 cscli machines add with-socket --auto --force + rune -0 cscli lapi status + + rune -0 cscli machines list -o json + rune -0 jq -c '[. | length, .[0].machineId[0:32], .[0].isValidated, .[0].ipAddress, .[0].auth_type]' <(output) + assert_output '[1,"with-socket",true,"127.0.0.1","password"]' + + # TLS cannot be used with a unix socket + + config_set "${CONFIG_DIR}/local_api_credentials.yaml" ' + .ca_cert_path=strenv(tmpdir) + "/bundle.pem" + ' + + rune -1 cscli lapi status + assert_stderr --partial "loading api client: cannot use TLS with a unix socket" + + config_set "${CONFIG_DIR}/local_api_credentials.yaml" ' + del(.ca_cert_path) | + .key_path=strenv(tmpdir) + "/agent-key.pem" + ' + + rune -1 cscli lapi status + assert_stderr --partial "loading api client: cannot use TLS with a unix socket" + + config_set "${CONFIG_DIR}/local_api_credentials.yaml" ' + del(.key_path) | + .cert_path=strenv(tmpdir) + "/agent.pem" + ' + + rune -1 cscli lapi status + assert_stderr --partial "loading api client: cannot use TLS with a unix socket" + + rune -0 cscli machines delete with-socket } @test "invalid cert for agent" { diff --git a/test/lib/config/config-global b/test/lib/config/config-global index 68346c18875..0caf0591f7d 100755 --- a/test/lib/config/config-global +++ b/test/lib/config/config-global @@ -58,6 +58,7 @@ config_prepare() { # remove trailing slash from CONFIG_DIR # since it's assumed to be missing during the tests yq e -i ' + .api.server.listen_socket="/run/crowdsec.sock" | .config_paths.config_dir |= sub("/$", "") ' "${CONFIG_DIR}/config.yaml" } diff --git a/test/lib/config/config-local b/test/lib/config/config-local index e3b7bc685d4..e5cfaf997be 100755 --- a/test/lib/config/config-local +++ b/test/lib/config/config-local @@ -57,7 +57,6 @@ config_generate() { cp ../config/profiles.yaml \ ../config/simulation.yaml \ - ../config/local_api_credentials.yaml \ ../config/online_api_credentials.yaml \ "${CONFIG_DIR}/" @@ -95,6 +94,7 @@ config_generate() { .db_config.db_path=strenv(DATA_DIR)+"/crowdsec.db" | .db_config.use_wal=true | .api.client.credentials_path=strenv(CONFIG_DIR)+"/local_api_credentials.yaml" | + .api.server.listen_socket=strenv(DATA_DIR)+"/crowdsec.sock" | .api.server.profiles_path=strenv(CONFIG_DIR)+"/profiles.yaml" | .api.server.console_path=strenv(CONFIG_DIR)+"/console.yaml" | del(.api.server.online_client) @@ -119,7 +119,8 @@ make_init_data() { ./bin/preload-hub-items - "$CSCLI" --warning machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto --force + # force TCP, the default would be unix socket + "$CSCLI" --warning machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --url http://127.0.0.1:8080 --auto --force mkdir -p "$LOCAL_INIT_DIR"