-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add search endpoint - connect to postgres
- Loading branch information
1 parent
84612ff
commit 668face
Showing
10 changed files
with
312 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| package datasources | ||
|
|
||
| import ( | ||
| "context" | ||
| ) | ||
|
|
||
| // Datasource knows how make different kinds of queries/actions on the underlying actual datastore. | ||
| // This abstraction allows the rest of the system to stay datastore agnostic. | ||
| type Datasource interface { | ||
| Suggest(ctx context.Context, suggestForThis string) ([]string, error) | ||
|
|
||
| // Close closes (connections to) the datasource gracefully | ||
| Close() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| package postgres | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
|
|
||
| "github.com/jackc/pgx/v5" | ||
| pgxgeom "github.com/twpayne/pgx-geom" | ||
|
|
||
| "strings" | ||
| "time" | ||
| ) | ||
|
|
||
| type Postgres struct { | ||
| db *pgx.Conn | ||
| ctx context.Context | ||
|
|
||
| queryTimeout time.Duration | ||
| searchIndex string | ||
| } | ||
|
|
||
| func NewPostgres(dbConn string, queryTimeout time.Duration, searchIndex string) (*Postgres, error) { | ||
| ctx := context.Background() | ||
| db, err := pgx.Connect(ctx, dbConn) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("unable to connect to database: %w", err) | ||
| } | ||
| // add support for Go <-> PostGIS conversions | ||
| if err := pgxgeom.Register(ctx, db); err != nil { | ||
| return nil, err | ||
| } | ||
| return &Postgres{db, ctx, queryTimeout, searchIndex}, nil | ||
| } | ||
|
|
||
| func (p *Postgres) Close() { | ||
| _ = p.db.Close(p.ctx) | ||
| } | ||
|
|
||
| func (p *Postgres) Suggest(ctx context.Context, suggestForThis string) ([]string, error) { | ||
| queryCtx, cancel := context.WithTimeout(ctx, p.queryTimeout) | ||
| defer cancel() | ||
|
|
||
| // Prepare dynamic full-text search query | ||
| // Split terms by spaces and append :* to each term | ||
| terms := strings.Fields(suggestForThis) | ||
| for i, term := range terms { | ||
| terms[i] = term + ":*" | ||
| } | ||
| searchTerm := strings.Join(terms, " & ") | ||
|
|
||
| sqlQuery := fmt.Sprintf( | ||
| `SELECT | ||
| r.display_name AS display_name, | ||
| max(r.rank) AS rank, | ||
| max(r.highlighted_text) AS highlighted_text | ||
| FROM ( | ||
| SELECT display_name, | ||
| ts_rank_cd(ts, to_tsquery('%[1]s'), 1) AS rank, | ||
| ts_headline('dutch', suggest, to_tsquery('%[2]s')) AS highlighted_text | ||
| FROM | ||
| %[3]s | ||
| WHERE ts @@ to_tsquery('%[4]s') LIMIT 500 | ||
| ) r | ||
| GROUP BY display_name | ||
| ORDER BY rank DESC, display_name ASC LIMIT 50`, | ||
| searchTerm, searchTerm, p.searchIndex, searchTerm) | ||
|
|
||
| // Execute query | ||
| rows, err := p.db.Query(queryCtx, sqlQuery) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("query '%s' failed: %w", sqlQuery, err) | ||
| } | ||
| defer rows.Close() | ||
|
|
||
| if queryCtx.Err() != nil { | ||
| return nil, queryCtx.Err() | ||
| } | ||
|
|
||
| var suggestions []string | ||
| for rows.Next() { | ||
| var displayName, highlightedText string | ||
| var rank float64 | ||
|
|
||
| // Scan all selected columns | ||
| if err := rows.Scan(&displayName, &rank, &highlightedText); err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| suggestions = append(suggestions, highlightedText) // or displayName, whichever you want to return | ||
| } | ||
|
|
||
| return suggestions, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| package search | ||
|
|
||
| import ( | ||
| stdjson "encoding/json" | ||
| "io" | ||
| "log" | ||
| "net/http" | ||
| "os" | ||
| "strconv" | ||
|
|
||
| "github.com/PDOK/gomagpie/internal/engine" | ||
| perfjson "github.com/goccy/go-json" | ||
| ) | ||
|
|
||
| var ( | ||
| disableJSONPerfOptimization, _ = strconv.ParseBool(os.Getenv("DISABLE_JSON_PERF_OPTIMIZATION")) | ||
| ) | ||
|
|
||
| // serveJSON serves JSON *WITHOUT* OpenAPI validation by writing directly to the response output stream | ||
| func serveJSON(input any, contentType string, w http.ResponseWriter) { | ||
| w.Header().Set(engine.HeaderContentType, contentType) | ||
|
|
||
| if err := getEncoder(w).Encode(input); err != nil { | ||
| handleJSONEncodingFailure(err, w) | ||
| return | ||
| } | ||
| } | ||
|
|
||
| type jsonEncoder interface { | ||
| Encode(input any) error | ||
| } | ||
|
|
||
| // Create JSONEncoder. Note escaping of '<', '>' and '&' is disabled (HTMLEscape is false). | ||
| // Especially the '&' is important since we use this character in the next/prev links. | ||
| func getEncoder(w io.Writer) jsonEncoder { | ||
| if disableJSONPerfOptimization { | ||
| // use Go stdlib JSON encoder | ||
| encoder := stdjson.NewEncoder(w) | ||
| encoder.SetEscapeHTML(false) | ||
| return encoder | ||
| } | ||
| // use ~7% overall faster 3rd party JSON encoder (in case of issues switch back to stdlib using env variable) | ||
| encoder := perfjson.NewEncoder(w) | ||
| encoder.SetEscapeHTML(false) | ||
| return encoder | ||
| } | ||
|
|
||
| func handleJSONEncodingFailure(err error, w http.ResponseWriter) { | ||
| log.Printf("JSON encoding failed: %v", err) | ||
| engine.RenderProblem(engine.ProblemServerError, w, "Failed to write JSON response") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.