diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3031eb8..1d1e7e9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,11 +24,16 @@ jobs: ELASTICSEARCH_PROTOCOL: http ELASTICSEARCH_HOST: 127.0.0.1 ELASTICSEARCH_PORT: 9200 + continue-on-error: ${{ matrix.experimental }} strategy: - fail-fast: false matrix: cfengine: [ "lucee@5", "adobe@2018", "adobe@2021" ] - ELASTICSEARCH_VERSION: [ "7.17.8", "8.5.3" ] + ELASTICSEARCH_VERSION: [ "7.17.8", "8.7.0" ] + experimental: [ false ] + include: + - cfengine: "adobe@2023" + ELASTICSEARCH_VERSION: "8.7.0" + experimental: true steps: - name: Checkout Repository uses: actions/checkout@v3.2.0 diff --git a/ModuleConfig.cfc b/ModuleConfig.cfc index 9eb26e7..1d606ac 100644 --- a/ModuleConfig.cfc +++ b/ModuleConfig.cfc @@ -1,105 +1,102 @@ /** -********************************************************************************* -* Your Copyright -******************************************************************************** -*/ -component{ + ********************************************************************************* + * Your Copyright + ******************************************************************************** + */ +component { - // Module Properties - this.title = "cbElasticSearch"; - this.author = "Jon Clausen "; - this.webURL = ""; - this.description = "Coldbox Module with Fluent API for ElasticSearch"; - this.version = "@build.version@+@build.number@"; - // If true, looks for views in the parent first, if not found, then in the module. Else vice-versa - this.viewParentLookup = true; - // If true, looks for layouts in the parent first, if not found, then in module. Else vice-versa - this.layoutParentLookup = true; - // Module Entry Point - this.entryPoint = "cbElasticsearch"; - // Model Namespace - this.modelNamespace = "cbElasticsearch"; - // CF Mapping - this.cfmapping = "cbElasticsearch"; - // Auto-map models - this.autoMapModels = true; - // Module Dependencies That Must Be Loaded First, use internal names or aliases - this.dependencies = [ "hyper" ]; - // Auto-parse parent settings - this.parseParentSettings = true; + // Module Properties + this.title = "cbElasticSearch"; + this.author = "Jon Clausen "; + this.webURL = ""; + this.description = "Coldbox Module with Fluent API for ElasticSearch"; + this.version = "@build.version@+@build.number@"; + // If true, looks for views in the parent first, if not found, then in the module. Else vice-versa + this.viewParentLookup = true; + // If true, looks for layouts in the parent first, if not found, then in module. Else vice-versa + this.layoutParentLookup = true; + // Module Entry Point + this.entryPoint = "cbElasticsearch"; + // Model Namespace + this.modelNamespace = "cbElasticsearch"; + // CF Mapping + this.cfmapping = "cbElasticsearch"; + // Auto-map models + this.autoMapModels = true; + // Module Dependencies That Must Be Loaded First, use internal names or aliases + this.dependencies = [ "hyper" ]; + // Auto-parse parent settings + this.parseParentSettings = true; - variables.configStruct = {}; + variables.configStruct = {}; - function configure(){ + function configure(){ + // Default settings + settings = { + // The default hosts - an array of host connections + // - REST-based clients (e.g. JEST): round robin connections will be used + // - Socket-based clients (e.g. Transport): cluster-aware routing used + versionTarget : getSystemSetting( "ELASTICSEARCH_VERSION", "" ), + hosts : [ + // The default connection is made to http://127.0.0.1:9200 + { + serverProtocol : getSystemSetting( "ELASTICSEARCH_PROTOCOL", "http" ), + serverName : getSystemSetting( "ELASTICSEARCH_HOST", "127.0.0.1" ), + serverPort : getSystemSetting( "ELASTICSEARCH_PORT", 9200 ) + } + ], + // The default credentials for access, if any - may also be overridden when searching index collections + defaultCredentials : { + "username" : getSystemSetting( "ELASTICSEARCH_USERNAME", "" ), + "password" : getSystemSetting( "ELASTICSEARCH_PASSWORD", "" ) + }, + // The default index + defaultIndex : getSystemSetting( "ELASTICSEARCH_INDEX", "cbElasticsearch" ), + // The default number of shards to use when creating an index + defaultIndexShards : getSystemSetting( "ELASTICSEARCH_SHARDS", 5 ), + // The default number of index replicas to create + defaultIndexReplicas : getSystemSetting( "ELASTICSEARCH_REPLICAS", 0 ), + // Whether to use separate threads for client transactions + multiThreaded : true, + // The maximum amount of time to wait until releasing a connection (in seconds) + maxConnectionIdleTime : 30, + // The maximum number of connections allowed per route ( e.g. search URI endpoint ) + maxConnectionsPerRoute : 10, + // The maxium number of connections, in total for all Elasticsearch requests + maxConnections : getSystemSetting( "ELASTICSEARCH_MAX_CONNECTIONS", 100 ), + // Read timeout - the read timeout in milliseconds + readTimeout : getSystemSetting( "ELASTICSEARCH_READ_TIMEOUT", 3000 ), + // Connection timeout - timeout attempts to connect to elasticsearch after this timeout + connectionTimeout : getSystemSetting( "ELASTICSEARCH_CONNECT_TIMEOUT", 3000 ) + }; - // Default settings - settings = { - // The default hosts - an array of host connections - // - REST-based clients (e.g. JEST): round robin connections will be used - // - Socket-based clients (e.g. Transport): cluster-aware routing used - versionTarget = getSystemSetting( "ELASTICSEARCH_VERSION", '' ), - hosts = [ - //The default connection is made to http://127.0.0.1:9200 - { - serverProtocol: getSystemSetting( "ELASTICSEARCH_PROTOCOL", "http" ), - serverName: getSystemSetting( "ELASTICSEARCH_HOST", "127.0.0.1" ), - serverPort: getSystemSetting( "ELASTICSEARCH_PORT", 9200 ) - } - ], - // The default credentials for access, if any - may also be overridden when searching index collections - defaultCredentials = { - "username" : getSystemSetting( "ELASTICSEARCH_USERNAME", "" ), - "password" : getSystemSetting( "ELASTICSEARCH_PASSWORD", "" ) - }, - // The default index - defaultIndex = getSystemSetting( "ELASTICSEARCH_INDEX", "cbElasticsearch" ), - // The default number of shards to use when creating an index - defaultIndexShards = getSystemSetting( "ELASTICSEARCH_SHARDS", 5 ), - // The default number of index replicas to create - defaultIndexReplicas = getSystemSetting( "ELASTICSEARCH_REPLICAS", 0 ), - // Whether to use separate threads for client transactions - multiThreaded = true, - // The maximum amount of time to wait until releasing a connection (in seconds) - maxConnectionIdleTime = 30, - // The maximum number of connections allowed per route ( e.g. search URI endpoint ) - maxConnectionsPerRoute = 10, - // The maxium number of connections, in total for all Elasticsearch requests - maxConnections = getSystemSetting( "ELASTICSEARCH_MAX_CONNECTIONS", 100 ), - // Read timeout - the read timeout in milliseconds - readTimeout = getSystemSetting( "ELASTICSEARCH_READ_TIMEOUT", 3000 ), - // Connection timeout - timeout attempts to connect to elasticsearch after this timeout - connectionTimeout = getSystemSetting( "ELASTICSEARCH_CONNECT_TIMEOUT", 3000 ) - }; - - // Custom Declared Points - interceptorSettings = { - customInterceptionPoints = [ + // Custom Declared Points + interceptorSettings = { + customInterceptionPoints : [ "cbElasticsearchPreSave", "cbElasticsearchPostSave" ] - }; - - // Custom Declared Interceptors - interceptors = []; - - } + }; - /** - * Fired when the module is registered and activated. - */ - function onLoad(){ - /** - * Main Configuration Object Singleton - **/ - binder.map( "Config@cbElasticsearch" ) - .to( '#this.cfmapping#.models.Config' ) - .threadSafe() - .asSingleton(); + // Custom Declared Interceptors + interceptors = []; + } - binder.map( "Client@cbElasticsearch" ) - .to( '#this.cfmapping#.models.io.HyperClient' ); + /** + * Fired when the module is registered and activated. + */ + function onLoad(){ + /** + * Main Configuration Object Singleton + **/ + binder + .map( "Config@cbElasticsearch" ) + .to( "#this.cfmapping#.models.Config" ) + .threadSafe() + .asSingleton(); - } + binder.map( "Client@cbElasticsearch" ).to( "#this.cfmapping#.models.io.HyperClient" ); + } } diff --git a/box.json b/box.json index 45d7bd3..46f4de8 100644 --- a/box.json +++ b/box.json @@ -2,7 +2,7 @@ "name":"Elasticsearch for the Coldbox Framework", "author":"Ortus Solutions document.getFields()["interestCost"] ); // 5.50 +``` + +### Runtime Fields + +Elasticsearch also allows the creation of runtime fields, which are fields defined in the index mapping but populated at search time via a script. + +{% hint style="info" %} +See [Managing-Indices](../Indices/Managing-Indices.md#creating-runtime-fields) for more information on creating runtime fields. +{% endhint %} + +Runtime fields can be fetched via the `setFields()` or `addField()` methods, and will appear in the `Document` object's `fields` struct. This example retrieves the `"fuel_usage_in_mpg"` runtime field as well as the indexed `"make"` and `"model"` fields: + +```js +var hits = searchBuilder.new( "itinerary" ) + .setFields( [ "fuel_mpg", "make", "model" ] ) + .execute() + .getHits(); +// OR +var hits = searchBuilder.new( "itinerary" ) + .addField( "fuel_mpg" ) + .addField( "make" ) + .addField( "model" ) + .execute() + .getHits(); +``` + +Once you have a search response, you can use the `.getFields()` method to retrieve the specified fields from the search document: + +```js +for( hit in hits ){ + var result = hit.getFields(); + writeOutput( "This #result.make# #result.model# gets #fuel_mpg#/gallon" ); +} +``` + +To access document `fields` as well as the `_source` properties, use`hit.getDocument( includeFields = true)`: + +```js +var result = searchBuilder.execute(); +for( hit in result.getHits() ){ + var document = document.getDocument( includeFields = true ); + writeOutput( "This #document.make# #document.model# gets #fuel_mpg#/gallon" ); +} +``` + ### Advanced Query DSL The SearchBuilder also allows full use of the [Elasticsearch query language](https://www.elastic.co/guide/en/elasticsearch/reference/current/_introducing_the_query_language.html), allowing full configuration of your search queries. There are several methods to provide the raw query language to the Search Builder. One is during instantiation. @@ -154,6 +236,24 @@ var search = getInstance( "SearchBuilder@cbElasticsearch" ) .execute(); ``` +After instantion, you can use the `.param()` and `.bodyParam()` methods to set [query parameters](https://www.elastic.co/guide/en/elasticsearch/reference/8.7/search-search.html#search-search-api-query-params) and [body parameters](https://www.elastic.co/guide/en/elasticsearch/reference/8.7/search-search.html#search-search-api-request-body), respectively. + +```js +var response = getInstance( "SearchBuilder@cbElasticsearch" ) + .new( "bookshop" ) + .sort( "publishDate DESC" ) + // match everything + .setQuery( { "match_all": {} } ) + // Query parameter: return the document version with each hit + .param( "version", true ) + // Body parameter: return a relevance score for each document, despite our custom sort + .bodyParam( "track_scores", true ); + // Body parameter: filter by minimum relevance score + .bodyParam( "min_score", 3 ) + // run the search + .execute(); +``` + {% hint style="info" %} For more information on Elasticsearch query DSL, the [Search in Depth Documentation](https://www.elastic.co/guide/en/elasticsearch/guide/current/search-in-depth.html) is an excellent starting point. {% endhint %} @@ -238,6 +338,36 @@ SearchBuilder.highlight( { }) ``` +## Terms Enum + +On occasion, you may wish to show a set of terms matching a partial string. This is similar to aggregations, only filtered by the provided string and intended for autocompletion. + +To retrieve this data, you can use the client's `getTermsEnum()` method: + +```js +var terms = getInstance( "HyperClient@cbElasticsearch" ) + .getTermsEnum( + indexName = "hotels", + field = "city", + match = "alb", + size = 50, + caseInsensitive = true + ); +``` + +For advanced lookups, you can use the second argument to pass a struct of custom options: + +```js +var terms = getInstance( "HyperClient@cbElasticsearch" ) + .getTermsEnum( ["cities","towns"], { + "field" : "name", + "string" : "west", + "size" : 50, + "timeout" : "10s" + } ); +``` + + ## `SearchBuilder` Function Reference * `new([string index], [string type], [struct properties])` - Populates a new SearchBuilder object. diff --git a/models/Document.cfc b/models/Document.cfc index ae88240..effa401 100644 --- a/models/Document.cfc +++ b/models/Document.cfc @@ -34,7 +34,7 @@ component accessors="true" { /** * The structural representation of the document object **/ - property name="memento"; + property name="_source"; /** * The pipeline used to process this document @@ -46,6 +46,10 @@ component accessors="true" { */ property name="params"; + /** + * Specifically selected or generated fields. Script fields, runtime fields, and selected fields will end up here. + */ + property name="fields" type="struct"; function onDIComplete(){ reset(); @@ -60,8 +64,9 @@ component accessors="true" { 0 ); variables.highlights = {}; - variables.memento = {}; + variables._source = {}; variables.params = {}; + variables.fields = {}; var nullDefaults = [ "id", "score" ]; @@ -95,30 +100,28 @@ component accessors="true" { * @refresh */ function create( any refresh = false ){ - var createOptions = { - "_index" : variables.index - }; - if( !isNull( variables.id ) && len( variables.id ) ){ + var createOptions = { "_index" : variables.index }; + if ( !isNull( variables.id ) && len( variables.id ) ) { createOptions[ "_id" ] = variables.id; } variables.params[ "refresh" ] = "wait_for"; - if( !isNull( variables.pipeline ) ){ + if ( !isNull( variables.pipeline ) ) { variables.params[ "pipeline" ] = variables.pipeline; } var response = getClient().processBulkOperation( [ { - "operation" : { "create" : createOptions }, - "source" : getDocument() - } + "operation" : { "create" : createOptions }, + "source" : getDocument() + } ], variables.params - ); + ); - if( response.errors ){ + if ( response.errors ) { var result = response.items[ 1 ][ "create" ]; throw( type = "cbElasticsearch.invalidRequest", @@ -128,14 +131,13 @@ component accessors="true" { ); } - if( arguments.refresh ){ - var idx = response.items[ 1 ][ "create" ][ "_index" ]; + if ( arguments.refresh ) { + var idx = response.items[ 1 ][ "create" ][ "_index" ]; var docId = response.items[ 1 ][ "create" ][ "_id" ]; return getClient().get( id = docId, index = idx ); } else { return this; } - } /** @@ -210,11 +212,11 @@ component accessors="true" { } // we need to duplicate so that we can remove any passed `_id` key - variables.memento = duplicate( arguments.properties ); + variables._source = duplicate( arguments.properties ); - if ( structKeyExists( variables.memento, "_id" ) ) { - variables.id = variables.memento[ "_id" ]; - structDelete( variables.memento, "_id" ); + if ( structKeyExists( variables._source, "_id" ) ) { + variables.id = variables._source[ "_id" ]; + structDelete( variables._source, "_id" ); } return this; @@ -225,19 +227,19 @@ component accessors="true" { * @properties struct the structural representation of the document **/ public Document function populate( required struct properties ){ - if ( isNull( variables.memento ) ) { - variables.memento = {}; + if ( isNull( variables._source ) ) { + variables._source = {}; } structAppend( - variables.memento, + variables._source, duplicate( arguments.properties ), true ); - if ( structKeyExists( variables.memento, "_id" ) ) { - setId( variables.memento[ "_id" ] ); - structDelete( variables.memento, "_id" ); + if ( structKeyExists( variables._source, "_id" ) ) { + setId( variables._source[ "_id" ] ); + structDelete( variables._source, "_id" ); } return this; @@ -250,7 +252,7 @@ component accessors="true" { * @value string the key value **/ public Document function setValue( required string name, required any value ){ - variables.memento[ arguments.name ] = arguments.value; + variables._source[ arguments.name ] = arguments.value; return this; } @@ -261,12 +263,12 @@ component accessors="true" { **/ public any function getValue( required string key, any default ){ // null return if the key does not exist - if ( !structKeyExists( variables.memento, arguments.key ) && isNull( arguments.default ) ) { + if ( !structKeyExists( variables._source, arguments.key ) && isNull( arguments.default ) ) { return; - } else if ( !structKeyExists( variables.memento, arguments.key ) && !isNull( arguments.default ) ) { + } else if ( !structKeyExists( variables._source, arguments.key ) && !isNull( arguments.default ) ) { return arguments.default; } else { - return variables.memento[ arguments.key ]; + return variables._source[ arguments.key ]; } } @@ -274,13 +276,17 @@ component accessors="true" { /** * Convenience method for a flattened struct of the memento * @includeKey boolean Whether to include the document key in the returned packet + * @includeFields boolean Include values from `"fields"` array, such as runtime or script fields **/ - public struct function getDocument( boolean includeKey = false ){ - var documentObject = duplicate( variables.memento ); + public struct function getDocument( boolean includeKey = false, boolean includeFields = false ){ + var documentObject = duplicate( variables._source ); if ( arguments.includeKey && !isNull( variables.id ) ) { documentObject[ "_id" ] = variables.id; } + if ( arguments.includeFields ){ + structAppend(documentObject, getFields(), true ); + } return documentObject; } @@ -309,4 +315,11 @@ component accessors="true" { ); } + /** + * Get the document _source properties as a struct. + */ + public struct function getMemento(){ + return variables._source; + } + } diff --git a/models/ILMPolicyBuilder.cfc b/models/ILMPolicyBuilder.cfc index 7051fa2..9ef5864 100644 --- a/models/ILMPolicyBuilder.cfc +++ b/models/ILMPolicyBuilder.cfc @@ -1,321 +1,312 @@ -component accessors="true"{ +component accessors="true" { - property name="policyName"; + property name="policyName"; - property name="phases"; + property name="phases"; - property name="meta"; + property name="meta"; - - /** + + /** * Client provider **/ - Client function getClient() provider="Client@cbElasticsearch"{} - - /** - * Creates a new policy builder instance - * - * @policyName string - * @phases a struct of phases ( optional ) - * @meta optional struct of meta - */ - ILMPolicyBuilder function new( - required string policyName, - struct phases, - struct meta - ){ - - structAppend( variables, arguments, true ); - param variables.phases = {}; - param variables.meta = {}; - - return this; - } - - /** - * Sets the configuration for the ILM Hot Phase - * https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-index-lifecycle.html - * - * @config a raw struct containing the phase configuration - * @priority numeric a priority to set for this index during the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-set-priority.html - * @rollover any either a raw rollover struct or a numeric (GB)/ string representing the size at which the index should rollover documents to the next phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-rollover.html - * @shards numeric the number of shards to shrink to in the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-shrink.html - * @searchableSnapshot string the name of a snapshot respository to create during this phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-searchable-snapshot.html - * @downsample any whether to downsample the repository. Either a numeric or string may be passed ( e.g. 1(days) or `1d` ) which denotes the fixed interval of the @timestamp to downsample to https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-downsample.html - * @forceMerge numeric The number of segments to force merge to during this phase. This action makes the index read-only https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-forcemerge.html - * @readOnly boolean Whether to make the index read-only during the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-readonly.html - * @unfollow boolean Whether to convert from a follower index ot a regular index at this phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-unfollow.html - */ - ILMPolicyBuilder function hotPhase( - struct config, - numeric priority, - any rollover, - numeric shards, - string searchableSnapshot, - any downsample, - numeric forceMerge, - boolean readOnly, - boolean unfollow - ){ - arguments.phaseName = "hot"; - return setPhase( argumentCollection = arguments ); - } - - /** - * Sets the configuration for the ILM Warm Phase - * https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-index-lifecycle.html - * - * @config a raw struct containing the phase configuration - * @age any Either a numeric of the number of days or a string interval to use as the threshold at which data is transitioned to this tier - * @priority numeric a priority to set for this index during the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-set-priority.html - * @shards numeric the number of shards to shrink to in the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-shrink.html - * @downsample any whether to downsample the repository. Either a numeric or string may be passed ( e.g. 1(days) or `1d` ) which denotes the fixed interval of the @timestamp to downsample to https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-downsample.html - * @allocate any if a numeric is provided it is applied as the number of replicas. Otherwise a struct config may be provided https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-allocate.html - * @migrate boolean moves the data to the phase-configured tier. Defaults to true so only use this argument if disabling migration https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-migrate.html - * @forceMerge numeric The number of segments to force merge to during this phase. This action makes the index read-only https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-forcemerge.html - * @readOnly boolean Whether to make the index read-only during the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-readonly.html - * @unfollow boolean Whether to convert from a follower index ot a regular index at this phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-unfollow.html - */ - ILMPolicyBuilder function warmPhase( - struct config, - any age, - numeric priority, - numeric shards, - any downsample, - any allocate, - boolean migrate, - numeric forceMerge, - boolean readOnly, - boolean unfollow - ){ - - verifyAgePolicy( argumentCollection = arguments ); - arguments.phaseName = "warm"; - return setPhase( argumentCollection = arguments ); - } - - /** - * Sets the configuration for the ILM Cold Phase - * https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-index-lifecycle.html - * - * @config a raw struct containing the phase configuration - * @age any Either a numeric of the number of days or a string interval to use as the threshold at which data is transitioned to this tier - * @priority numeric a priority to set for this index during the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-set-priority.html - * @searchableSnapshot string the name of a snapshot respository to create during this phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-searchable-snapshot.html - * @downsample any whether to downsample the repository. Either a numeric or string may be passed ( e.g. 1(days) or `1d` ) which denotes the fixed interval of the @timestamp to downsample to https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-downsample.html - * @allocate any if a numeric is provided it is applied as the number of replicas. Otherwise a struct config may be provided https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-allocate.html - * @migrate boolean moves the data to the phase-configured tier. Defaults to true so only use this argument if disabling migration https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-migrate.html - * @readOnly boolean Whether to make the index read-only during the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-readonly.html - * @unfollow boolean Whether to convert from a follower index ot a regular index at this phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-unfollow.html - */ - ILMPolicyBuilder function coldPhase( - struct config, - any age, - numeric priority, - string searchableSnapshot, - any downsample, - any allocate, - boolean migrate, - boolean readOnly, - boolean unfollow - ){ - verifyAgePolicy( argumentCollection = arguments ); - - arguments.phaseName = "cold"; - return setPhase( argumentCollection = arguments ); - } - /** - * Sets the configuration for the ILM Freeze Phase - * https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-index-lifecycle.html - * - * @config a raw struct containing the phase configuration - * @age any Either a numeric of the number of days or a string interval to use as the threshold at which data is transitioned to this tier - * @searchableSnapshot string the name of a snapshot respository to create during this phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-searchable-snapshot.html - * @unfollow boolean Whether to convert from a follower index ot a regular index at this phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-unfollow.html - */ - ILMPolicyBuilder function frozenPhase( - struct config, - any age, - string searchableSnapshot, - boolean unfollow - ){ - verifyAgePolicy( argumentCollection = arguments ); - - arguments.phaseName = "freeze"; - return setPhase( argumentCollection = arguments ); - - } - - /** - * Sets the configuration for the deletion phase - * - * @config a raw struct containing the phase configuration - * @age any Either a numeric of the number of days or a string interval to use as the threshold at which data is transitioned to this tier - * @waitForSnapshot string the name of the SLM policy to execute that the delete action should wait for https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-wait-for-snapshot.html - * @deleteSnapshot boolean Whether to delete the snapshot created in the previous phase - * - */ - ILMPolicyBuilder function withDeletion( - struct config, - any age, - string waitForSnapshot, - boolean deleteSnapshot - ){ - verifyAgePolicy( argumentCollection = arguments ); - - var maxAge = arguments.age ?: config.max_age; - if( isNumeric( maxAge ) ) maxAge = javacast( "string", maxAge & "d" ); - - var phase = { - "min_age": maxAge, - "actions": { - "delete": {} - } - }; - - if( !isNull( arguments.waitForSnapshot ) ){ - phase.actions[ "wait_for_snapshot" ] = { - "policy" : arguments.waitForSnapshot - }; - } - - if( !isNull( arguments.deleteSnapshot ) ){ - phase.actions.delete[ "delete_searchable_snapshot" ] = javacast( "boolean", arguments.deleteSnapshot ); - } - - variables.phases[ "delete" ] = phase; - - return this; - - } - - /** - * Sets the configuration for an ILM phase - * https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-index-lifecycle.html - * - * @config a raw struct containing the phase configuration - * @priority numeric a priority to set for this index during the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-set-priority.html - * @rollover any either a raw rollover struct or a numeric (GB)/ string representing the size at which the index should rollover documents to the next phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-rollover.html - * @shards numeric the number of shards to shrink to in the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-shrink.html - * @searchableSnapshot string the name of a snapshot respository to create during this phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-searchable-snapshot.html - * @downsample any whether to downsample the repository. Either a numeric or string may be passed ( e.g. 1(days) or `1d` ) which denotes the fixed interval of the @timestamp to downsample to https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-downsample.html - * @forceMerge numeric The number of segments to force merge to during this phase. This action makes the index read-only https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-forcemerge.html - * @readOnly boolean Whether to make the index read-only during the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-readonly.html - * @unfollow boolean Whether to convert from a follower index ot a regular index at this phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-unfollow.html - */ - ILMPolicyBuilder function setPhase( - required string phaseName, - struct config, - numeric priority, - any age, - any rollover, - numeric shards, - string searchableSnapshot, - any downsample, - numeric forceMerge, - boolean readOnly = false, - boolean unfollow = false - ){ - - var phase = arguments.config ?: { "actions" : {} }; - - if( !isNull( arguments.priority ) ){ - phase.actions[ "set_priority" ] = { "priority" : arguments.priority }; - } - - if( !isNull( arguments.age ) ){ - phase[ "min_age" ] = arguments.age; - if( isNumeric( phase.min_age ) ) phase.min_age = javacast( "string", phase.min_age & "d" ); - } - - if( !isNull( arguments.rollover ) ){ - var rolloverSize = arguments.rollover; - if( isNumeric( rolloverSize ) ) rolloverSize = javacast( "string", rolloverSize & "gb" ); - phase.actions[ "rollover" ] = { "max_primary_shard_size" : rolloverSize }; - } - - if( !isNull( arguments.shards ) ){ - phase.actions[ "shrink" ] = { - "number_of_shards" : arguments.shards - }; - } - - if( !isNull( arguments.searchableSnapshot ) ){ - phase.actions[ "searchable_snapshot" ] = { - "shapshot_repository" : arguments.searchableSnapshot - }; - } - - if( !isNull( arguments.downsample ) ){ - if( getClient().isMajorVersion( 7 ) ){ - getClient().getLog().warn( "Elasticsearch versions below version 8 do not support lifecycle phase downsampling. The argument with a value of #arguments.downsample# in phase #arguments.phaseName# for policy #variables.policyName# is being ignored." ); - } else { - var interval = arguments.downsample; - if( isNumeric( interval ) ) interval = javacast( "string", interval & "h" ); - phase.actions[ "downsample" ] = { "fixed_interval" : interval }; - } - } - - if( !isNull( arguments.forceMerge ) ){ - phase.actions[ "forcemerge" ] = { "max_num_segments": arguments.forceMerge }; - } - - if( arguments.readOnly ){ - phase.actions[ "readonly" ] = {}; - } - - if( arguments.unfollow ){ - phase.actions[ "unfollow" ] = {}; - } - - variables.phases[ arguments.phaseName ] = phase; - - return this; - - } - - /** - * Returns the configured policy DSL - */ - struct function getDSL(){ - var policy = { - "phases" : variables.phases - }; - - if( !variables.meta.isEmpty() ){ - policy[ "_meta" ] = variables.meta - } - - return policy; - } - - /** - * Creates or Updates the Policy - */ - ILMPolicyBuilder function save(){ - getClient().applyILMPolicy( variables.policyName, getDSL() ); - return this; - } - - /** - * Returns the existing ILM policy - */ - struct function get(){ - return getClient().getILMPolicy( variables.policyName ); - } - - /** - * Verifies the age is set for a policy - * - * @config struct - * @age string - */ - private void function verifyAgePolicy( struct config, string age ){ - if( ( isNull( arguments.age ) && isNull( arguments.config ) ) || ( !isNull( arguments.config ) && !arguments.config.keyExists( "max_age" ) ) ){ - throw( - type = "cbElasticsearch.ILMPolicy.InvalidPolicyException", - message = "This ILM Phase requires an age parameter at which to transition documents" - ); - } - } -} \ No newline at end of file + Client function getClient() provider="Client@cbElasticsearch"{ + } + + /** + * Creates a new policy builder instance + * + * @policyName string + * @phases a struct of phases ( optional ) + * @meta optional struct of meta + */ + ILMPolicyBuilder function new( + required string policyName, + struct phases, + struct meta + ){ + structAppend( variables, arguments, true ); + param variables.phases = {}; + param variables.meta = {}; + + return this; + } + + /** + * Sets the configuration for the ILM Hot Phase + * https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-index-lifecycle.html + * + * @config a raw struct containing the phase configuration + * @priority numeric a priority to set for this index during the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-set-priority.html + * @rollover any either a raw rollover struct or a numeric (GB)/ string representing the size at which the index should rollover documents to the next phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-rollover.html + * @shards numeric the number of shards to shrink to in the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-shrink.html + * @searchableSnapshot string the name of a snapshot respository to create during this phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-searchable-snapshot.html + * @downsample any whether to downsample the repository. Either a numeric or string may be passed ( e.g. 1(days) or `1d` ) which denotes the fixed interval of the @timestamp to downsample to https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-downsample.html + * @forceMerge numeric The number of segments to force merge to during this phase. This action makes the index read-only https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-forcemerge.html + * @readOnly boolean Whether to make the index read-only during the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-readonly.html + * @unfollow boolean Whether to convert from a follower index ot a regular index at this phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-unfollow.html + */ + ILMPolicyBuilder function hotPhase( + struct config, + numeric priority, + any rollover, + numeric shards, + string searchableSnapshot, + any downsample, + numeric forceMerge, + boolean readOnly, + boolean unfollow + ){ + arguments.phaseName = "hot"; + return setPhase( argumentCollection = arguments ); + } + + /** + * Sets the configuration for the ILM Warm Phase + * https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-index-lifecycle.html + * + * @config a raw struct containing the phase configuration + * @age any Either a numeric of the number of days or a string interval to use as the threshold at which data is transitioned to this tier + * @priority numeric a priority to set for this index during the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-set-priority.html + * @shards numeric the number of shards to shrink to in the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-shrink.html + * @downsample any whether to downsample the repository. Either a numeric or string may be passed ( e.g. 1(days) or `1d` ) which denotes the fixed interval of the @timestamp to downsample to https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-downsample.html + * @allocate any if a numeric is provided it is applied as the number of replicas. Otherwise a struct config may be provided https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-allocate.html + * @migrate boolean moves the data to the phase-configured tier. Defaults to true so only use this argument if disabling migration https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-migrate.html + * @forceMerge numeric The number of segments to force merge to during this phase. This action makes the index read-only https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-forcemerge.html + * @readOnly boolean Whether to make the index read-only during the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-readonly.html + * @unfollow boolean Whether to convert from a follower index ot a regular index at this phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-unfollow.html + */ + ILMPolicyBuilder function warmPhase( + struct config, + any age, + numeric priority, + numeric shards, + any downsample, + any allocate, + boolean migrate, + numeric forceMerge, + boolean readOnly, + boolean unfollow + ){ + verifyAgePolicy( argumentCollection = arguments ); + arguments.phaseName = "warm"; + return setPhase( argumentCollection = arguments ); + } + + /** + * Sets the configuration for the ILM Cold Phase + * https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-index-lifecycle.html + * + * @config a raw struct containing the phase configuration + * @age any Either a numeric of the number of days or a string interval to use as the threshold at which data is transitioned to this tier + * @priority numeric a priority to set for this index during the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-set-priority.html + * @searchableSnapshot string the name of a snapshot respository to create during this phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-searchable-snapshot.html + * @downsample any whether to downsample the repository. Either a numeric or string may be passed ( e.g. 1(days) or `1d` ) which denotes the fixed interval of the @timestamp to downsample to https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-downsample.html + * @allocate any if a numeric is provided it is applied as the number of replicas. Otherwise a struct config may be provided https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-allocate.html + * @migrate boolean moves the data to the phase-configured tier. Defaults to true so only use this argument if disabling migration https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-migrate.html + * @readOnly boolean Whether to make the index read-only during the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-readonly.html + * @unfollow boolean Whether to convert from a follower index ot a regular index at this phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-unfollow.html + */ + ILMPolicyBuilder function coldPhase( + struct config, + any age, + numeric priority, + string searchableSnapshot, + any downsample, + any allocate, + boolean migrate, + boolean readOnly, + boolean unfollow + ){ + verifyAgePolicy( argumentCollection = arguments ); + + arguments.phaseName = "cold"; + return setPhase( argumentCollection = arguments ); + } + /** + * Sets the configuration for the ILM Freeze Phase + * https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-index-lifecycle.html + * + * @config a raw struct containing the phase configuration + * @age any Either a numeric of the number of days or a string interval to use as the threshold at which data is transitioned to this tier + * @searchableSnapshot string the name of a snapshot respository to create during this phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-searchable-snapshot.html + * @unfollow boolean Whether to convert from a follower index ot a regular index at this phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-unfollow.html + */ + ILMPolicyBuilder function frozenPhase( + struct config, + any age, + string searchableSnapshot, + boolean unfollow + ){ + verifyAgePolicy( argumentCollection = arguments ); + + arguments.phaseName = "freeze"; + return setPhase( argumentCollection = arguments ); + } + + /** + * Sets the configuration for the deletion phase + * + * @config a raw struct containing the phase configuration + * @age any Either a numeric of the number of days or a string interval to use as the threshold at which data is transitioned to this tier + * @waitForSnapshot string the name of the SLM policy to execute that the delete action should wait for https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-wait-for-snapshot.html + * @deleteSnapshot boolean Whether to delete the snapshot created in the previous phase + * + */ + ILMPolicyBuilder function withDeletion( + struct config, + any age, + string waitForSnapshot, + boolean deleteSnapshot + ){ + verifyAgePolicy( argumentCollection = arguments ); + + var maxAge = arguments.age ?: config.max_age; + if ( isNumeric( maxAge ) ) maxAge = javacast( "string", maxAge & "d" ); + + var phase = { "min_age" : maxAge, "actions" : { "delete" : {} } }; + + if ( !isNull( arguments.waitForSnapshot ) ) { + phase.actions[ "wait_for_snapshot" ] = { "policy" : arguments.waitForSnapshot }; + } + + if ( !isNull( arguments.deleteSnapshot ) ) { + phase.actions.delete[ "delete_searchable_snapshot" ] = javacast( "boolean", arguments.deleteSnapshot ); + } + + variables.phases[ "delete" ] = phase; + + return this; + } + + /** + * Sets the configuration for an ILM phase + * https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-index-lifecycle.html + * + * @config a raw struct containing the phase configuration + * @priority numeric a priority to set for this index during the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-set-priority.html + * @rollover any either a raw rollover struct or a numeric (GB)/ string representing the size at which the index should rollover documents to the next phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-rollover.html + * @shards numeric the number of shards to shrink to in the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-shrink.html + * @searchableSnapshot string the name of a snapshot respository to create during this phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-searchable-snapshot.html + * @downsample any whether to downsample the repository. Either a numeric or string may be passed ( e.g. 1(days) or `1d` ) which denotes the fixed interval of the @timestamp to downsample to https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-downsample.html + * @forceMerge numeric The number of segments to force merge to during this phase. This action makes the index read-only https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-forcemerge.html + * @readOnly boolean Whether to make the index read-only during the phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-readonly.html + * @unfollow boolean Whether to convert from a follower index ot a regular index at this phase https://www.elastic.co/guide/en/elasticsearch/reference/current/ilm-unfollow.html + */ + ILMPolicyBuilder function setPhase( + required string phaseName, + struct config, + numeric priority, + any age, + any rollover, + numeric shards, + string searchableSnapshot, + any downsample, + numeric forceMerge, + boolean readOnly = false, + boolean unfollow = false + ){ + var phase = arguments.config ?: { "actions" : {} }; + + if ( !isNull( arguments.priority ) ) { + phase.actions[ "set_priority" ] = { "priority" : arguments.priority }; + } + + if ( !isNull( arguments.age ) ) { + phase[ "min_age" ] = arguments.age; + if ( isNumeric( phase.min_age ) ) phase.min_age = javacast( "string", phase.min_age & "d" ); + } + + if ( !isNull( arguments.rollover ) ) { + var rolloverSize = arguments.rollover; + if ( isNumeric( rolloverSize ) ) rolloverSize = javacast( "string", rolloverSize & "gb" ); + phase.actions[ "rollover" ] = { "max_primary_shard_size" : rolloverSize }; + } + + if ( !isNull( arguments.shards ) ) { + phase.actions[ "shrink" ] = { "number_of_shards" : arguments.shards }; + } + + if ( !isNull( arguments.searchableSnapshot ) ) { + phase.actions[ "searchable_snapshot" ] = { "shapshot_repository" : arguments.searchableSnapshot }; + } + + if ( !isNull( arguments.downsample ) ) { + if ( getClient().isMajorVersion( 7 ) ) { + getClient() + .getLog() + .warn( + "Elasticsearch versions below version 8 do not support lifecycle phase downsampling. The argument with a value of #arguments.downsample# in phase #arguments.phaseName# for policy #variables.policyName# is being ignored." + ); + } else { + var interval = arguments.downsample; + if ( isNumeric( interval ) ) interval = javacast( "string", interval & "h" ); + phase.actions[ "downsample" ] = { "fixed_interval" : interval }; + } + } + + if ( !isNull( arguments.forceMerge ) ) { + phase.actions[ "forcemerge" ] = { "max_num_segments" : arguments.forceMerge }; + } + + if ( arguments.readOnly ) { + phase.actions[ "readonly" ] = {}; + } + + if ( arguments.unfollow ) { + phase.actions[ "unfollow" ] = {}; + } + + variables.phases[ arguments.phaseName ] = phase; + + return this; + } + + /** + * Returns the configured policy DSL + */ + struct function getDSL(){ + var policy = { "phases" : variables.phases }; + + if ( !variables.meta.isEmpty() ) { + policy[ "_meta" ] = variables.meta + } + + return policy; + } + + /** + * Creates or Updates the Policy + */ + ILMPolicyBuilder function save(){ + getClient().applyILMPolicy( variables.policyName, getDSL() ); + return this; + } + + /** + * Returns the existing ILM policy + */ + struct function get(){ + return getClient().getILMPolicy( variables.policyName ); + } + + /** + * Verifies the age is set for a policy + * + * @config struct + * @age string + */ + private void function verifyAgePolicy( struct config, string age ){ + if ( + ( isNull( arguments.age ) && isNull( arguments.config ) ) || ( + !isNull( arguments.config ) && !arguments.config.keyExists( "max_age" ) + ) + ) { + throw( + type = "cbElasticsearch.ILMPolicy.InvalidPolicyException", + message = "This ILM Phase requires an age parameter at which to transition documents" + ); + } + } + +} diff --git a/models/IndexBuilder.cfc b/models/IndexBuilder.cfc index 2bb9058..5bbf3ae 100644 --- a/models/IndexBuilder.cfc +++ b/models/IndexBuilder.cfc @@ -32,13 +32,11 @@ component accessors="true" { function reset( indexName ){ variables.settings = {}; - if( isNull( arguments.indexName ) || !getClient().indexExists( arguments.indexName ) ){ - variables.settings.append( - { - "number_of_shards" : javacast( "int", getConfig().get( "defaultIndexShards" ) ), - "number_of_replicas" : javacast( "int", getConfig().get( "defaultIndexReplicas" ) ) - } - ); + if ( isNull( arguments.indexName ) || !getClient().indexExists( arguments.indexName ) ) { + variables.settings.append( { + "number_of_shards" : javacast( "int", getConfig().get( "defaultIndexShards" ) ), + "number_of_replicas" : javacast( "int", getConfig().get( "defaultIndexReplicas" ) ) + } ); } variables.mappings = {}; @@ -103,14 +101,17 @@ component accessors="true" { * @properties {Struct} Index mapping. Defines the fields and types used in the index. * @settings {Struct} Key/value struct of index settings such as `number_of_shards`. */ - boolean function patch( required string name, any properties, struct settings ){ + boolean function patch( + required string name, + any properties, + struct settings + ){ reset( arguments.name ); return this.populate( argumentCollection = arguments ).save(); } IndexBuilder function populate( string name, any properties, struct settings ){ - reset( arguments.name ?: javacast( "null", 0 ) ); if ( !isNull( arguments.name ) ) { diff --git a/models/SearchBuilder.cfc b/models/SearchBuilder.cfc index ddcb230..833883e 100644 --- a/models/SearchBuilder.cfc +++ b/models/SearchBuilder.cfc @@ -50,6 +50,20 @@ component accessors="true" { **/ property name="script"; + /** + * Property containing elasticsearch "script_fields" definition for runtime scripted fields + * + * https://www.elastic.co/guide/en/elasticsearch/reference/current/search-fields.html#script-fields + */ + property name="scriptFields" type="struct"; + + /** + * Property containing "fields" array of fields to return for each hit + * + * https://www.elastic.co/guide/en/elasticsearch/reference/current/search-fields.html + */ + property name="fields" type="array"; + /** * When performing matching searches, the type of match to specify **/ @@ -60,6 +74,11 @@ component accessors="true" { */ property name="params"; + /** + * Body parameters which will be passed to transform the execution output + */ + property name="body" type="struct"; + /** * Whether to preflight the query prior to execution( recommended ) - ensures consistent formatting to prevent errors **/ @@ -75,8 +94,8 @@ component accessors="true" { property name="suggest"; // Optional search defaults - property name="maxRows"; - property name="startRow"; + property name="size"; + property name="from"; function onDIComplete(){ @@ -102,9 +121,10 @@ component accessors="true" { variables.highlight = {}; variables.suggest = {}; variables.params = []; + variables.body = {}; - variables.maxRows = 25; - variables.startRow = 0; + variables.size = 25; + variables.from = 0; variables.preflight = true; @@ -150,6 +170,28 @@ component accessors="true" { return getClient().deleteByQuery( this ); } + /** + * Backwards compatible setter for max result size + * + * @deprecated + * + * @value Max number of records to retrieve. + */ + SearchBuilder function setMaxRows( required numeric value ){ + variables.size = arguments.value; + return this; + } + /** + * Backwards compatible setter for result start offset + * + * @deprecated + * + * @value Starting document offset. + */ + SearchBuilder function setStartRow( required numeric value ){ + variables.from = arguments.value; + return this; + } /** * Populates a new SearchBuilder object @@ -171,13 +213,15 @@ component accessors="true" { if ( !isNull( arguments.properties ) ) { for ( var propName in arguments.properties ) { switch ( propName ) { + case "from": case "offset": case "startRow": { - variables.startRow = arguments.properties[ propName ]; + variables.from = arguments.properties[ propName ]; break; } + case "size": case "maxRows": { - variables.maxRows = arguments.properties[ propName ]; + variables.size = arguments.properties[ propName ]; break; } case "query": { @@ -247,7 +291,7 @@ component accessors="true" { required any name, required any value, numeric boost, - string operator = "must", + string operator = "must", boolean caseInsensitive = false ){ param variables.query.bool = {}; @@ -262,9 +306,9 @@ component accessors="true" { return { "wildcard" : { "#key#" : { - "value" : reFind( "^(?![a-zA-Z0-9 ,.&$']*[^a-zA-Z0-9 ,.&$']).*$", value ) - ? "*" & value & "*" - : value, + "value" : reFind( "^(?![a-zA-Z0-9 ,.&$']*[^a-zA-Z0-9 ,.&$']).*$", value ) + ? "*" & value & "*" + : value, "case_insensitive" : javacast( "boolean", caseInsensitive ) } } @@ -276,9 +320,9 @@ component accessors="true" { var wildcard = { "wildcard" : { "#name#" : { - "value" : reFind( "^(?![a-zA-Z0-9 ,.&$']*[^a-zA-Z0-9 ,.&$']).*$", value ) - ? "*" & value & "*" - : value, + "value" : reFind( "^(?![a-zA-Z0-9 ,.&$']*[^a-zA-Z0-9 ,.&$']).*$", value ) + ? "*" & value & "*" + : value, "case_insensitive" : javacast( "boolean", arguments.caseInsensitive ) } } @@ -393,11 +437,27 @@ component accessors="true" { * `range` filter for date ranges * @name string the key to match * @start string the preformatted date string to start the range - * @end string the preformatted date string to end the range + * @end string the preformatted date string to end the range + * @operator string opeartor for the filter operation: `must` or `should` **/ - SearchBuilder function filterRange( required string name, string start, string end ){ + SearchBuilder function filterRange( + required string name, + string start, + string end, + operator = "must" + ){ if ( isNull( arguments.start ) && isNull( arguments.end ) ) { - throw( type = "", message = "" ); + throw( + type = "cbElasticsearch.SearchBuilder.InvalidParamTypeException", + message = "A start or end is required to use filterRange" + ); + } + + if ( arguments.operator != "must" && arguments.operator != "should" ) { + throw( + type = "cbElasticsearch.SearchBuilder.InvalidParamTypeException", + message = "The operator should be either `must` or `should`." + ); } var properties = {}; @@ -407,9 +467,16 @@ component accessors="true" { if ( !isNull( arguments.end ) ) { properties[ "lte" ] = arguments.end; } - param variables.query.bool = {}; - param variables.query.bool.filter = {}; - param variables.query.bool.filter.range = { "#arguments.name#" : properties }; + + param variables.query.bool = {}; + param variables.query.bool.filter = {}; + param variables.query.bool.filter.bool = {}; + + if ( !variables.query.bool.filter.bool.keyExists( arguments.operator ) ) { + variables.query.bool.filter.bool[ arguments.operator ] = []; + } + + variables.query.bool.filter.bool[ arguments.operator ].append( { "range" : { "#arguments.name#" : properties } } ); return this; } @@ -560,7 +627,10 @@ component accessors="true" { numeric boost ){ if ( isNull( arguments.start ) && isNull( arguments.end ) ) { - throw( type = "", message = "" ); + throw( + type = "cbElasticsearch.SearchBuilder.InvalidParamTypeException", + message = "A start or end is required to use dateMatch" + ); } var properties = {}; @@ -804,6 +874,39 @@ component accessors="true" { return this; } + /** + * Generic setter for any/all request properties. + * + * For example, `set( "size", 100 )` or `set( "min_score" : 1 )`. + * + * Example https://www.elastic.co/guide/en/elasticsearch/reference/8.7/search-search.html#search-search-api-request-body + * + * @name the name of the parameter to set. + * @value the value of the parameter + */ + SearchBuilder function set( required string name, required any value ){ + if( variables.keyExists( arguments.name ) ){ + variables[ arguments.name ] = arguments.value; + } else { + variables.body[ arguments.name ] = arguments.value; + } + + return this; + } + + /** + * Adds a body parameter to the request (such as filtering by min_score, forcing a relevance score return, etc.) + * + * Example https://www.elastic.co/guide/en/elasticsearch/reference/8.7/search-search.html#search-search-api-request-body + * + * @name the name of the body parameter to set + * @value the value of the parameter + */ + SearchBuilder function bodyParam( required string name, required any value ){ + set( arguments.name, arguments.value ); + return this; + } + /** * Adds highlighting to search * @@ -1073,8 +1176,8 @@ component accessors="true" { if ( !isNull( variables.query ) && !structIsEmpty( variables.query ) ) { dsl[ "query" ] = variables.query; - dsl[ "from" ] = variables.startRow; - dsl[ "size" ] = variables.maxRows; + dsl[ "from" ] = variables.from; + dsl[ "size" ] = variables.size; } if ( !isNull( variables.highlight ) && !structIsEmpty( variables.highlight ) ) { @@ -1101,6 +1204,16 @@ component accessors="true" { dsl[ "script" ] = variables.script; } + structAppend( dsl, variables.body, true ); + + if ( !isNull( variables.scriptFields ) ) { + dsl[ "script_fields" ] = variables.scriptFields; + } + + if ( !isNull( variables.fields ) ) { + dsl[ "fields" ] = variables.fields; + } + if ( !isNull( variables.sorting ) ) { // we used a linked hashmap for sorting to maintain order dsl[ "sort" ] = createObject( "java", "java.util.LinkedHashMap" ).init(); @@ -1165,4 +1278,46 @@ component accessors="true" { return this; } + public SearchBuilder function setFields( array fields = [] ){ + variables.fields = arguments.fields; + return this; + } + + /** + * Append a dynamic script field to the search. + * + * @name Name of the script field + * @script Script to use. `{ "script" : { "lang": "painless", "source" : } }` + * @source Which _source values to include in the response. `true` for all, `false` for none, or a wildcard-capable string: `source = "author.*"` + */ + public SearchBuilder function addScriptField( required string name, struct script, any source = true ){ + if ( isNull( variables.scriptFields ) ){ + variables.scriptFields = {}; + } + variables.scriptFields[ arguments.name ] = arguments.script; + setSource( arguments.source ); + return this; + } + + /** + * Append a field name or object in the list of fields to return. + * + * Especially useful for runtime fields. + * + * Example: + * ``` + * addField( { "field": "@timestamp", "format": "epoch_millis" } ) + * ``` + * + * @see https://www.elastic.co/guide/en/elasticsearch/reference/current/runtime-retrieving-fields.html#runtime-search-dayofweek + * + * @value string|struct Field name to retrieve OR struct config + */ + public SearchBuilder function addField( required any value ){ + if ( isNull( variables.fields ) ){ + variables.fields = []; + } + variables.fields.append( arguments.value ); + return this; + } } diff --git a/models/SearchResult.cfc b/models/SearchResult.cfc index 34234cb..e73bd4a 100644 --- a/models/SearchResult.cfc +++ b/models/SearchResult.cfc @@ -143,7 +143,8 @@ component accessors="true" { variables.hits = []; for ( var hit in arguments.hits ) { - var doc = newDocument().populate( hit[ "_source" ] ); + var documentSource = hit.keyExists( "_source" ) ? hit[ "_source" ] : {}; + var doc = newDocument().populate( documentSource ); doc.setIndex( hit[ "_index" ] ); @@ -161,6 +162,10 @@ component accessors="true" { doc.setHighlights( hit[ "highlight" ] ); } + if ( structKeyExists( hit, "fields" ) ) { + doc.setFields( hit[ "fields" ] ); + } + arrayAppend( variables.hits, doc ); } diff --git a/models/cache/Provider.cfc b/models/cache/Provider.cfc index df62e2a..10b1818 100644 --- a/models/cache/Provider.cfc +++ b/models/cache/Provider.cfc @@ -610,7 +610,7 @@ component arrayAppend( documents, document ); } - var transactionResult = getClient().saveAll( documents, true, { "refresh" : "wait_for" } ); + var transactionResult = getClient().saveAll( documents, true, { "refresh" : "wait_for" } ); var te = getTickCount(); @@ -667,7 +667,7 @@ component document.setId( objectKey ); try { - var future = document.save( refresh=true ); + var future = document.save( refresh = true ); } catch ( any e ) { if ( isTimeoutException( e ) && getConfiguration().ignoreElasticsearchTimeouts ) { // log it diff --git a/models/io/HyperClient.cfc b/models/io/HyperClient.cfc index b5a9b14..c6f774c 100644 --- a/models/io/HyperClient.cfc +++ b/models/io/HyperClient.cfc @@ -59,7 +59,7 @@ component accessors="true" threadSafe singleton { /** * Pipeline provider **/ - cbElasticsearch.models.Document function newPipeline() provider="Pipeline@cbElasticsearch"{ + cbElasticsearch.models.Document function newPipeline() provider="Pipeline@cbElasticsearch"{ } /** @@ -146,7 +146,7 @@ component accessors="true" threadSafe singleton { } } - if( isMajorVersion( 6 ) ){ + if ( isMajorVersion( 6 ) ) { throw( type = "cbElasticsearch.UnsupportedVersionException", message = "Support for Elasticsearch Version 6 was removed in cbElasticsearch v3. Please use version 2 of this module for support for Elasticsearch versions < 7" @@ -240,9 +240,9 @@ component accessors="true" threadSafe singleton { * @interfaced **/ boolean function indexMappingExists( required string indexName ){ - try{ + try { var mappings = getMappings( arguments.indexName ); - } catch( any e ){ + } catch ( any e ) { return false; } @@ -255,7 +255,9 @@ component accessors="true" threadSafe singleton { * @indexName string the name of the index ( optional ) if null returns all settings for the server */ struct function getSettings( string indexName ){ - var response = variables.nodePool.newRequest( !isNull( arguments.indexName ) ? arguments.indexName & "/_settings" : "_settings" , "GET" ).send(); + var response = variables.nodePool + .newRequest( !isNull( arguments.indexName ) ? arguments.indexName & "/_settings" : "_settings", "GET" ) + .send(); if ( response.getStatusCode() != 200 ) { onResponseFailure( response ); @@ -311,6 +313,7 @@ component accessors="true" threadSafe singleton { && structKeyExists( indexDSL, "mappings" ) && !structIsEmpty( indexDSL.mappings ) && !structKeyExists( indexDSL.mappings, "properties" ) + && !structKeyExists( indexDSL.mappings, "runtime" ) ) { if ( indexDSL.mappings.keyArray().len() > 1 ) { throw( @@ -354,11 +357,11 @@ component accessors="true" threadSafe singleton { indexResult[ "mappings" ] = applyMappings( indexName, indexDSL.mappings ); } } - if( structKeyExists( indexDSL, "settings" ) && !structIsEmpty( indexDSL.settings ) ){ + if ( structKeyExists( indexDSL, "settings" ) && !structIsEmpty( indexDSL.settings ) ) { var requestBuilder = variables.nodePool - .newRequest( indexName & "/_settings", "PUT" ) - .setBody( getUtil().toJSON( indexDSL.settings ) ) - .asJSON(); + .newRequest( indexName & "/_settings", "PUT" ) + .setBody( getUtil().toJSON( indexDSL.settings ) ) + .asJSON(); var response = requestBuilder.send(); @@ -482,39 +485,45 @@ component accessors="true" threadSafe singleton { /** * Trigger an index refresh on the given index/indices. - * + * * @indexName string|array Index name or alias. Can accept an array of index/alias names. * @params struct Struct of query parameters to influence the request. For example: `{ "ignore_unavailable" : true }` */ struct function refreshIndex( required any indexName, struct params = {} ){ - if ( isArray( arguments.indexName ) ){ arguments.indexName = arrayToList( arguments.indexName ); } + if ( isArray( arguments.indexName ) ) { + arguments.indexName = arrayToList( arguments.indexName ); + } var refreshRequest = variables.nodePool.newRequest( "/#arguments.indexName#/_refresh", "post" ); return refreshRequest - .withQueryParams( arguments.params ) - .send() - .json(); + .withQueryParams( arguments.params ) + .send() + .json(); } /** * Get statistics for the given index/indices. - * + * * @indexName string|array Index name or alias. Can accept an array of index/alias names. * @metrics array Array of index metrics to retrieve. I.e. `[ "completion","refresh", "request_cache" ]`. * @params struct Struct of query parameters to influence the request. For example: `{ "expand_wildcards" : "none", "level" : "shards" } */ - struct function getIndexStats( any indexName, array metrics = [], struct params = {} ){ - if ( isArray( arguments.indexName ) ){ arguments.indexName = arrayToList( arguments.indexName ); } - - var endpoint = [ - "_stats" - ]; - if ( !isNull( arguments.indexName ) && arguments.indexName != "" ){ + struct function getIndexStats( + any indexName, + array metrics = [], + struct params = {} + ){ + if ( isArray( arguments.indexName ) ) { + arguments.indexName = arrayToList( arguments.indexName ); + } + + var endpoint = [ "_stats" ]; + if ( !isNull( arguments.indexName ) && arguments.indexName != "" ) { endpoint.prepend( arguments.indexName ); } endpoint.append( arrayToList( metrics ) ); var statsRequest = variables.nodePool.newRequest( arrayToList( endpoint, "/" ), "get" ); - + return statsRequest .withQueryParams( arguments.params ) .send() @@ -570,8 +579,8 @@ component accessors="true" threadSafe singleton { .keyArray() .each( function( indexName ){ if ( - structKeyExists( aliasesResult[ indexName ], "aliases" ) - && + structKeyExists( aliasesResult[ indexName ], "aliases" ) + && !structIsEmpty( aliasesResult[ indexName ].aliases ) ) { // we need to scope this for the ACF compiler @@ -579,11 +588,11 @@ component accessors="true" threadSafe singleton { indexObj.aliases .keyArray() .each( function( alias ){ - if( !aliasesMap.aliases.keyExists( alias ) ){ + if ( !aliasesMap.aliases.keyExists( alias ) ) { aliasesMap.aliases[ alias ] = []; } aliasesMap.aliases[ alias ].append( { - "index" : indexName, + "index" : indexName, "attributes" : indexObj.aliases[ alias ] } ); } ); @@ -674,7 +683,7 @@ component accessors="true" threadSafe singleton { struct function applyMappings( required string indexName, required struct mappings ){ var mappingResults = {}; - if( arguments.mappings.keyExists( "properties" ) ){ + if ( arguments.mappings.keyExists( "properties" ) ) { arguments.mappings = arguments.mappings.properties; } @@ -860,10 +869,7 @@ component accessors="true" threadSafe singleton { * @return Document The saved cbElasticsearch Document object * @interfaced **/ - cbElasticsearch.models.Document function save( - required cbElasticsearch.models.Document document, - any refresh - ){ + cbElasticsearch.models.Document function save( required cbElasticsearch.models.Document document, any refresh ){ if ( isNull( arguments.document.getId() ) ) { var saveRequest = variables.nodePool.newRequest( "#arguments.document.getIndex()#/_doc", "POST" ); } else { @@ -900,7 +906,11 @@ component accessors="true" threadSafe singleton { arguments.document.setId( saveResult[ "_id" ] ); - if ( arguments.keyExists( "refresh" ) && arguments.refresh == true && !isNull( arguments.document.getPipeline() ) ) { + if ( + arguments.keyExists( "refresh" ) && arguments.refresh == true && !isNull( + arguments.document.getPipeline() + ) + ) { arguments.document = this.get( saveResult[ "_id" ], arguments.document.getIndex() ); } @@ -1345,58 +1355,46 @@ component accessors="true" threadSafe singleton { /** * Determines whether a snapshot repository exists * - * @name + * @name */ function snapshotRepositoryExists( required string name ){ return variables.nodePool - .newRequest( "_snapshot/#arguments.name#" ) - .send() - .getStatusCode() == "200"; + .newRequest( "_snapshot/#arguments.name#" ) + .send() + .getStatusCode() == "200"; } /** * Creates or Updates a Snapshot Repository */ - function applySnapshotRepository( - required name, - required definition - ){ - if( isSimpleValue( arguments.definition ) ){ + function applySnapshotRepository( required name, required definition ){ + if ( isSimpleValue( arguments.definition ) ) { arguments.definition = { - "type" : "fs", - "settings" : { - "location" : arguments.definition - } + "type" : "fs", + "settings" : { "location" : arguments.definition } }; } - var response = variables.nodePool - .newRequest( "_snapshot/#arguments.name#", "PUT" ) - .setBody( - getUtil().toJSON( arguments.definition ) - ) - .asJSON() - .send(); - - return response.getStatusCode() == 200 - ? response.json() - : onResponseFailure( response ); + var response = variables.nodePool + .newRequest( "_snapshot/#arguments.name#", "PUT" ) + .setBody( getUtil().toJSON( arguments.definition ) ) + .asJSON() + .send(); + return response.getStatusCode() == 200 + ? response.json() + : onResponseFailure( response ); } /** * Deletes a Snapshot Repository */ - function deleteSnapshotRepository( - required name - ){ - var response = variables.nodePool - .newRequest( "_snapshot/#arguments.name#", "DELETE" ) - .send(); + function deleteSnapshotRepository( required name ){ + var response = variables.nodePool.newRequest( "_snapshot/#arguments.name#", "DELETE" ).send(); return response.getStatusCode() == 200 - ? response.json() - : onResponseFailure( response ); + ? response.json() + : onResponseFailure( response ); } /** @@ -1406,13 +1404,13 @@ component accessors="true" threadSafe singleton { /** * Determines whether an index template exists * - * @name + * @name */ boolean function indexTemplateExists( required string name ){ return variables.nodePool - .newRequest( "_index_template/#arguments.name#" ) - .send() - .getStatusCode() == "200"; + .newRequest( "_index_template/#arguments.name#" ) + .send() + .getStatusCode() == "200"; } /** @@ -1421,20 +1419,16 @@ component accessors="true" threadSafe singleton { * @name string * @definition struct */ - any function applyIndexTemplate( required string name, required struct definition ){ - - var response = variables.nodePool - .newRequest( "_index_template/#arguments.name#", "PUT" ) - .setBody( - getUtil().toJSON( arguments.definition ) - ) - .asJSON() - .send(); - - return response.getStatusCode() == 200 - ? response.json() - : onResponseFailure( response ); + any function applyIndexTemplate( required string name, required struct definition ){ + var response = variables.nodePool + .newRequest( "_index_template/#arguments.name#", "PUT" ) + .setBody( getUtil().toJSON( arguments.definition ) ) + .asJSON() + .send(); + return response.getStatusCode() == 200 + ? response.json() + : onResponseFailure( response ); } /** @@ -1442,13 +1436,11 @@ component accessors="true" threadSafe singleton { * @name string */ any function deleteIndexTemplate( required string name ){ - var response = variables.nodePool - .newRequest( "_index_template/#arguments.name#", "DELETE" ) - .send(); + var response = variables.nodePool.newRequest( "_index_template/#arguments.name#", "DELETE" ).send(); return response.getStatusCode() == 200 - ? response.json() - : onResponseFailure( response ); + ? response.json() + : onResponseFailure( response ); } /** @@ -1458,13 +1450,13 @@ component accessors="true" threadSafe singleton { /** * Determines whether an component template exists * - * @name + * @name */ boolean function componentTemplateExists( required string name ){ return variables.nodePool - .newRequest( "_component_template/#arguments.name#" ) - .send() - .getStatusCode() == "200"; + .newRequest( "_component_template/#arguments.name#" ) + .send() + .getStatusCode() == "200"; } /** @@ -1473,22 +1465,22 @@ component accessors="true" threadSafe singleton { * @name string * @definition struct */ - any function applyComponentTemplate( required string name, required struct definition ){ - var response = variables.nodePool - .newRequest( "_component_template/#arguments.name#", "PUT" ) - .setBody( - getUtil().toJSON( - !definition.keyExists( "template" ) - ? { "template" : arguments.definition } - : arguments.definition - ) - ) - .asJSON() - .send(); + any function applyComponentTemplate( required string name, required struct definition ){ + var response = variables.nodePool + .newRequest( "_component_template/#arguments.name#", "PUT" ) + .setBody( + getUtil().toJSON( + !definition.keyExists( "template" ) + ? { "template" : arguments.definition } + : arguments.definition + ) + ) + .asJSON() + .send(); return response.getStatusCode() == 200 - ? response.json() - : onResponseFailure( response ); + ? response.json() + : onResponseFailure( response ); } /** @@ -1496,13 +1488,11 @@ component accessors="true" threadSafe singleton { * @name string */ any function deleteComponentTemplate( required string name ){ - var response = variables.nodePool - .newRequest( "_component_template/#arguments.name#", "DELETE" ) - .send(); + var response = variables.nodePool.newRequest( "_component_template/#arguments.name#", "DELETE" ).send(); return response.getStatusCode() == 200 - ? response.json() - : onResponseFailure( response ); + ? response.json() + : onResponseFailure( response ); } /** @@ -1512,28 +1502,26 @@ component accessors="true" threadSafe singleton { /** * Checks whether a named ILM policy exists * - * @name + * @name */ boolean function ILMPolicyExists( required string name ){ return variables.nodePool - .newRequest( "_ilm/policy/#arguments.name#" ) - .send() - .getStatusCode() == 200; + .newRequest( "_ilm/policy/#arguments.name#" ) + .send() + .getStatusCode() == 200; } /** * Get an ILM policy by name * * @name string - */ + */ any function getILMPolicy( required string name ){ - var response = variables.nodePool - .newRequest( "_ilm/policy/#arguments.name#" ) - .send(); + var response = variables.nodePool.newRequest( "_ilm/policy/#arguments.name#" ).send(); return response.getStatusCode() == 200 - ? response.json() - : onResponseFailure( response ); + ? response.json() + : onResponseFailure( response ); } /** @@ -1542,73 +1530,59 @@ component accessors="true" threadSafe singleton { * @name string * @policy object Either a struct defining the policy or a policy object */ - any function applyILMPolicy( - required string name, - required any policy - ){ + any function applyILMPolicy( required string name, required any policy ){ var response = variables.nodePool - .newRequest( "_ilm/policy/#arguments.name#", "PUT" ) - .setBody( getUtil().toJSON( { "policy" : arguments.policy } ) ) - .asJSON() - .send(); - - return response.getStatusCode() == 200 - ? response.json() - : onResponseFailure( response ); + .newRequest( "_ilm/policy/#arguments.name#", "PUT" ) + .setBody( getUtil().toJSON( { "policy" : arguments.policy } ) ) + .asJSON() + .send(); + return response.getStatusCode() == 200 + ? response.json() + : onResponseFailure( response ); } /** * Deletes an ILM policy * - * @name + * @name */ - any function deleteILMPolicy( - required string name - ){ - var response = variables.nodePool - .newRequest( "_ilm/policy/#arguments.name#", "DELETE" ) - .send(); - + any function deleteILMPolicy( required string name ){ + var response = variables.nodePool.newRequest( "_ilm/policy/#arguments.name#", "DELETE" ).send(); + return response.getStatusCode() == 200 - ? response.json() - : onResponseFailure( response ); + ? response.json() + : onResponseFailure( response ); } - + /** - * Data Streams Support - */ + * Data Streams Support + */ /** * Checks to see whether a data stream exists * - * @name + * @name */ - boolean function dataStreamExists( - required string name - ){ + boolean function dataStreamExists( required string name ){ return variables.nodePool - .newRequest( "_data_stream/#arguments.name#" ) - .send() - .getStatusCode() == "200"; + .newRequest( "_data_stream/#arguments.name#" ) + .send() + .getStatusCode() == "200"; } /** * Ensures the existence of a data stream * - * @name + * @name */ - any function ensureDataStream( - required string name - ){ - var response = variables.nodePool - .newRequest( "_data_stream/#arguments.name#", "PUT" ) - .send(); + any function ensureDataStream( required string name ){ + var response = variables.nodePool.newRequest( "_data_stream/#arguments.name#", "PUT" ).send(); return response.getStatusCode() == 200 - ? response.json() - : onResponseFailure( response ); + ? response.json() + : onResponseFailure( response ); } /** @@ -1616,33 +1590,27 @@ component accessors="true" threadSafe singleton { * * @indexName */ - any function migrateToDataStream( - required string indexName - ){ + any function migrateToDataStream( required string indexName ){ var response = variables.nodePool - .newRequest( "_data_stream/_migrate/#arguments.indexName#", "POST" ) - .send(); + .newRequest( "_data_stream/_migrate/#arguments.indexName#", "POST" ) + .send(); return response.getStatusCode() == 200 - ? response.json() - : onResponseFailure( response ); + ? response.json() + : onResponseFailure( response ); } /** * Gets a datastream definition * - * @name + * @name */ - any function getDataStream( - required string name - ){ - var response = variables.nodePool - .newRequest( "_data_stream/#arguments.name#" ) - .send(); + any function getDataStream( required string name ){ + var response = variables.nodePool.newRequest( "_data_stream/#arguments.name#" ).send(); return response.getStatusCode() == 200 - ? response.json() - : onResponseFailure( response ); + ? response.json() + : onResponseFailure( response ); } /** @@ -1651,22 +1619,18 @@ component accessors="true" threadSafe singleton { * @name string the name of the stream * @template string the name of the index template to use for this data stream */ - any function deleteDataStream( - required string name - ){ - var response = variables.nodePool - .newRequest( "_data_stream/#arguments.name#", "DELETE" ) - .send(); - + any function deleteDataStream( required string name ){ + var response = variables.nodePool.newRequest( "_data_stream/#arguments.name#", "DELETE" ).send(); + return response.getStatusCode() == 200 - ? response.json() - : onResponseFailure( response ); + ? response.json() + : onResponseFailure( response ); } /** * Handles response errors from Elasticsearch * - * @response + * @response */ function onResponseFailure( required Hyper.models.HyperResponse response ){ return getUtil().handleResponseError( response = arguments.response ); @@ -1712,4 +1676,29 @@ component accessors="true" threadSafe singleton { return listGetAt( variables.versionTarget, 1, "." ) == versionNumber; } + /** + * Retrieve an enum of field terms from the index matching the provided string. + * + * @indexName string|array Index name or array of index names to query on + * @field string|struct If string, field name to query. Otherwise, a struct of query options where only "field" is required. + * + * @see https://www.elastic.co/guide/en/elasticsearch/reference/8.7/search-terms-enum.html + */ + function getTermsEnum( required indexName, required any field, any match, numeric size = 10, boolean caseInsensitive = true ){ + if ( isArray( arguments.indexName ) ){ arguments.indexName = arrayToList( arguments.indexName ); } + var termsRequest = variables.nodePool.newRequest( "/#arguments.indexName#/_terms_enum", "post" ); + + var opts = { + "size" : arguments.size, + "case_insensitive": arguments.caseInsensitive, + "field" : !isNull( arguments.match ) ? arguments.match : javaCast( "null", 0 ) + }; + if ( !isSimpleValue( arguments.field ) ){ opts = arguments.field; } + + return termsRequest + .setBody( opts ) + .send() + .json(); + } + } diff --git a/models/logging/LogstashAppender.cfc b/models/logging/LogstashAppender.cfc index 44fcce6..c70a404 100644 --- a/models/logging/LogstashAppender.cfc +++ b/models/logging/LogstashAppender.cfc @@ -7,8 +7,8 @@ component hint ="This a logstash appender for Elasticsearch" { - property name="util" inject="Util@cbelasticsearch"; - property name="cachebox" inject="cachebox"; + property name="util" inject="Util@cbelasticsearch"; + property name="cachebox" inject="cachebox"; property name="asyncManager" inject="box:AsyncManager"; /** @@ -42,31 +42,31 @@ component instance.DEFAULTS = { // Data stream components - "dataStreamPattern" : "logs-coldbox-*", - "dataStream" : "logs-coldbox-logstash-appender", - "ILMPolicyName" : "cbelasticsearch-logs", + "dataStreamPattern" : "logs-coldbox-*", + "dataStream" : "logs-coldbox-logstash-appender", + "ILMPolicyName" : "cbelasticsearch-logs", "componentTemplateName" : "cbelasticsearch-logs-mappings", - "indexTemplateName" : "cbelasticsearch-logs", - "pipelineName" : "cbelasticsearch-logs", + "indexTemplateName" : "cbelasticsearch-logs", + "pipelineName" : "cbelasticsearch-logs", // Retention of logs in number of days - "retentionDays" : 365, + "retentionDays" : 365, // optional lifecycle full policy - "lifecyclePolicy" : javacast( "null", 0 ), + "lifecyclePolicy" : javacast( "null", 0 ), // the application name to use for this instance - "applicationName" : applicationName, + "applicationName" : applicationName, // The release version - "releaseVersion" : "", + "releaseVersion" : "", // The number of shards for the backing indices - "indexShards" : 1, + "indexShards" : 1, // The number of replicas for the backing indices - "indexReplicas" : 0, + "indexReplicas" : 0, // The maximum shard size at which a rollover of the oldest data will occur - "rolloverSize" : "10gb", + "rolloverSize" : "10gb", // v2 migration fields - "index" : javacast( "null", 0 ), - "migrateIndices" : false, + "index" : javacast( "null", 0 ), + "migrateIndices" : false, // Whether to throw an error if an attempt to save a log entry fails - "throwOnError" : true + "throwOnError" : true }; for ( var configKey in structKeyArray( instance.Defaults ) ) { @@ -80,11 +80,12 @@ component } // Attempt to retreive the package version from the `box.json` - if( !len( getProperty( "releaseVersion" ) ) && fileExists( expandPath( "/box.json" ) ) ){ - try{ - var packageInfo = deSerializeJSON( fileRead( expandPath( "/box.json" ) ) ); + if ( !len( getProperty( "releaseVersion" ) ) && fileExists( expandPath( "/box.json" ) ) ) { + try { + var packageInfo = deserializeJSON( fileRead( expandPath( "/box.json" ) ) ); setProperty( "releaseVersion", packageInfo.version ?: "" ); - } catch( any e ){} + } catch ( any e ) { + } } application.wirebox.autowire( this ); @@ -122,15 +123,12 @@ component * Write an entry into the appender. */ public void function logMessage( required any logEvent ) output=false{ - var logObj = marshallLogObject( argumentCollection = arguments ); - try{ - newDocument() - .new( index = getProperty( "dataStream" ), properties = logObj ) - .create(); - } catch( any e ){ - if( getProperty( "throwOnError" ) ){ + try { + newDocument().new( index = getProperty( "dataStream" ), properties = logObj ).create(); + } catch ( any e ) { + if ( getProperty( "throwOnError" ) ) { rethrow; } else { var eLogEvent = new coldbox.system.logging.LogEvent( @@ -139,72 +137,63 @@ component extraInfo = { "logData" : logObj, "exception" : e }, category = e.type ); - var appendersMap = application.wirebox.getLogbox().getAppendersMap(); + var appendersMap = application.wirebox.getLogbox().getAppendersMap(); // Log errors out to other appenders besides this one - var safeAppenders = appendersMap.keyArray().filter( function( key ){ return key != getName(); } ); + var safeAppenders = appendersMap + .keyArray() + .filter( function( key ){ + return key != getName(); + } ); saveAppenders.each( function( appenderName ){ appendersMap[ appenderName ].logMessage( eLogEvent ); } ); } } - } - public struct function marshallLogObject( required any logEvent ) output=false { + public struct function marshallLogObject( required any logEvent ) output=false{ var loge = arguments.logEvent; var extraInfo = loge.getExtraInfo(); var level = lCase( severityToString( loge.getSeverity() ) ); var message = loge.getMessage(); var loggerCat = loge.getCategory(); - var tzInfo = getTimezoneInfo(); + var tzInfo = getTimezoneInfo(); var logObj = { "@timestamp" : dateTimeFormat( loge.getTimestamp(), "yyyy-mm-dd'T'hh:nn:ssZZ" ), - "log" : { - "level" : level, - "logger" : getName(), + "log" : { + "level" : level, + "logger" : getName(), "category" : loggerCat }, "message" : message, - "event" : { - "created" : dateTimeFormat( loge.getTimestamp(), "yyyy-mm-dd'T'hh:nn:ssZZ" ), + "event" : { + "created" : dateTimeFormat( loge.getTimestamp(), "yyyy-mm-dd'T'hh:nn:ssZZ" ), "severity" : loge.getSeverity(), - "category" : loggerCat, - "dataset" : "cfml", - "timezone" : tzInfo.timezone ?: createObject( "java", "java.util.TimeZone" ).getDefault().getId() + "category" : loggerCat, + "dataset" : "cfml", + "timezone" : tzInfo.timezone ?: createObject( "java", "java.util.TimeZone" ).getDefault().getId() }, - "file" : { - "path" : CGI.CF_TEMPLATE_PATH - }, - "url" : { + "file" : { "path" : CGI.CF_TEMPLATE_PATH }, + "url" : { "domain" : CGI.SERVER_NAME, - "path" : CGI.PATH_INFO, - "port" : CGI.SERVER_PORT, - "query" : CGI.QUERY_STRING, - "scheme" : lcase( listFirst( CGI.SERVER_PROTOCOL, "/" ) ) + "path" : CGI.PATH_INFO, + "port" : CGI.SERVER_PORT, + "query" : CGI.QUERY_STRING, + "scheme" : lCase( listFirst( CGI.SERVER_PROTOCOL, "/" ) ) }, - "http" : { - "request" : { - "referer" : CGI.HTTP_REFERER - } - }, - "labels" : { - "application" : getProperty( "applicationName" ) - }, - "package": { - "name" : getProperty( "applicationName" ), + "http" : { "request" : { "referer" : CGI.HTTP_REFERER } }, + "labels" : { "application" : getProperty( "applicationName" ) }, + "package" : { + "name" : getProperty( "applicationName" ), "version" : javacast( "string", getProperty( "releaseVersion" ) ), - "type" : "cfml", - "path" : expandPath( "/" ) + "type" : "cfml", + "path" : expandPath( "/" ) }, "host" : { "name" : CGI.HTTP_HOST, "hostname" : CGI.SERVER_NAME }, - "client" : { - "ip" : CGI.REMOTE_ADDR - }, - "user" : {}, - "user_agent" : { - "original" : CGI.HTTP_USER_AGENT - } + "client" : { "ip" : CGI.REMOTE_ADDR }, + "user" : {}, + "user_agent" : { "original" : CGI.HTTP_USER_AGENT } }; if ( propertyExists( "userInfoUDF" ) ) { @@ -212,29 +201,38 @@ component if ( isClosure( udf ) ) { try { logObj.user[ "info" ] = udf(); - if( !isSimpleValue( logObj.user.info ) ){ - if( isStruct( logObj.user.info ) ){ - var userKeys = [ "email", "domain", "full_name", "hash", "id", "name", "roles", "username" ]; - userKeys.each( - function( key ){ - if( key == "username" ) key = "name"; - if( logObj.user.info.keyExists( key ) ){ - logObj.user[ key ] = logObj.user.info[ key ]; - } + if ( !isSimpleValue( logObj.user.info ) ) { + if ( isStruct( logObj.user.info ) ) { + var userKeys = [ + "email", + "domain", + "full_name", + "hash", + "id", + "name", + "roles", + "username" + ]; + userKeys.each( function( key ){ + if ( key == "username" ) key = "name"; + if ( logObj.user.info.keyExists( key ) ) { + logObj.user[ key ] = logObj.user.info[ key ]; } - ); + } ); } - logObj.user.info = variables.util.toJSON( logObj.user.info ); + logObj.user.info = variables.util.toJSON( logObj.user.info ); } } catch ( any e ) { - logObj[ "user" ] = { "error" : "An error occurred when attempting to run the userInfoUDF provided. The message received was #e.message#" }; + logObj[ "user" ] = { + "error" : "An error occurred when attempting to run the userInfoUDF provided. The message received was #e.message#" + }; } } } if ( structKeyExists( application, "cbController" ) ) { var event = application.cbController.getRequestService().getContext(); - var rc = event.getCollection(); + var rc = event.getCollection(); structAppend( local.logObj.event, { @@ -243,15 +241,24 @@ component event.getCurrentRoutedModule() != "" ? " from the " & event.getCurrentRoutedModule() & "module router." : "" ) : javacast( "null", 0 ), "extension" : rc.keyExists( "format" ) ? rc.format : javacast( "null", 0 ), - "url" : ( event.getCurrentRoutedURL() != "" ) ? event.getCurrentRoutedURL() : javacast( "null", 0 ), - "layout" : ( event.getCurrentLayout() != "" ) ? event.getCurrentLayout() : javacast( "null", 0 ), - "module" : event.getCurrentModule(), - "view" : event.getCurrentView() + "url" : ( event.getCurrentRoutedURL() != "" ) ? event.getCurrentRoutedURL() : javacast( + "null", + 0 + ), + "layout" : ( event.getCurrentLayout() != "" ) ? event.getCurrentLayout() : javacast( + "null", + 0 + ), + "module" : event.getCurrentModule(), + "view" : event.getCurrentView() }, true ); - logObj.url[ "full" ] = ( event.getCurrentRoutedURL() != "" ) ? event.getCurrentRoutedURL() : javacast( "null", 0 ); + logObj.url[ "full" ] = ( event.getCurrentRoutedURL() != "" ) ? event.getCurrentRoutedURL() : javacast( + "null", + 0 + ); logObj.package[ "reference" ] = event.getHTMLBaseURL(); @@ -270,7 +277,8 @@ component "Detail" ) ) { - structAppend( local.logObj, + structAppend( + local.logObj, parseException( exception = extraInfo, level = level, @@ -317,96 +325,92 @@ component * Verify or create the logging index */ private void function ensureDataStream() output=false{ - - var dataStreamName = getProperty( "dataStream" ); - var dataStreamPattern = getProperty( "dataStreamPattern" ); + var dataStreamName = getProperty( "dataStream" ); + var dataStreamPattern = getProperty( "dataStreamPattern" ); var componentTemplateName = getProperty( "componentTemplateName" ); - var indexTemplateName = getProperty( "indexTemplateName" ); + var indexTemplateName = getProperty( "indexTemplateName" ); - var policyMeta = { - "description" : "Lifecyle Policy for cbElasticsearch logs" - }; - var policyBuilder = policyBuilder().new( policyName=getProperty( "ILMPolicyName" ), meta=policyMeta ); + var policyMeta = { "description" : "Lifecyle Policy for cbElasticsearch logs" }; + var policyBuilder = policyBuilder().new( policyName = getProperty( "ILMPolicyName" ), meta = policyMeta ); // Put our ILM Policy - if( propertyExists( "lifecyclePolicy" ) ){ + if ( propertyExists( "lifecyclePolicy" ) ) { policyBuilder.setPhases( getProperty( "lifecyclePolicy" ) ); } else { - policyBuilder.withDeletion( - age = getProperty( "retentionDays" ) - ); + policyBuilder.withDeletion( age = getProperty( "retentionDays" ) ); } policyBuilder.save(); // Create our pipeline to handle data from older versions of the appender - getClient().newPipeline() - .setId( getProperty( "pipelineName" ) ) - .setDescription( "Ingest pipeline for cbElasticsearch logstash appender" ) - .addProcessor( - { - "script": { - "lang": "painless", - "source": reReplace( fileRead( expandPath( "/cbelasticsearch/models/logging/scripts/v2MigrationProcessor.painless" ) ), - "\n|\r|\t", - "", - "ALL" - ) - } - } - ).save() + getClient() + .newPipeline() + .setId( getProperty( "pipelineName" ) ) + .setDescription( "Ingest pipeline for cbElasticsearch logstash appender" ) + .addProcessor( { + "script" : { + "lang" : "painless", + "source" : reReplace( + fileRead( + expandPath( "/cbelasticsearch/models/logging/scripts/v2MigrationProcessor.painless" ) + ), + "\n|\r|\t", + "", + "ALL" + ) + } + } ) + .save() // Upsert our component template - getClient().applyComponentTemplate( - componentTemplateName, - getComponentTemplate() - ); + getClient().applyComponentTemplate( componentTemplateName, getComponentTemplate() ); // Upsert the current version of our template getClient().applyIndexTemplate( indexTemplateName, { "index_patterns" : [ dataStreamPattern ], - "composed_of" : [ + "composed_of" : [ "logs-mappings", "data-streams-mappings", "logs-settings", componentTemplateName ], "data_stream" : {}, - "priority" : 150, - "_meta" : { - "description" : "Index Template for cbElasticsearch Logs" - } + "priority" : 150, + "_meta" : { "description" : "Index Template for cbElasticsearch Logs" } } ); - if( !getClient().dataStreamExists( dataStreamName ) ){ + if ( !getClient().dataStreamExists( dataStreamName ) ) { getClient().ensureDataStream( dataStreamName ); } // Check for any previous indices created matching the pattern and migrate them to the datastream - if( propertyExists( "index" ) && getProperty( "migrateIndices" ) ){ + if ( propertyExists( "index" ) && getProperty( "migrateIndices" ) ) { var existingIndexPrefix = getProperty( "index" ); - var existingIndices = getClient().getIndices().keyArray().filter( - function( index ){ + var existingIndices = getClient() + .getIndices() + .keyArray() + .filter( function( index ){ return len( index ) >= len( existingIndexPrefix ) && left( index, len( existingIndexPrefix ) ) == existingIndexPrefix; + } ); + variables.asyncManager.allApply( existingIndices, function( index ){ + try { + getClient().reindex( + index, + { "index" : dataStreamName, "op_type" : "create" }, + true + ); + getClient().deleteIndex( index ); + } catch ( any e ) { + // Print to StdError to bypass LogBox, since we are in an appender + createObject( "java", "java.lang.System" ).err.println( + "[ERROR] Index Migration Between the Previous Index of #index# to the data stream #dataStreamName# could not be completed. The error received was: #e.message#" + ); } - ); - variables.asyncManager.allApply( - existingIndices, - function( index ){ - try{ - getClient().reindex( index, { "index" : dataStreamName, "op_type" : "create" }, true ); - getClient().deleteIndex( index ); - } catch( any e ){ - // Print to StdError to bypass LogBox, since we are in an appender - createObject( "java", "java.lang.System" ).err.println( "[ERROR] Index Migration Between the Previous Index of #index# to the data stream #dataStreamName# could not be completed. The error received was: #e.message#" ); - } - } - ); + } ); } - } /** @@ -432,14 +436,13 @@ component var logstashException = { "error" : { - "level" : arguments.level, - "type" : arguments.exception.type.toString(), - "message" : message & " " & arguments.exception.detail, + "level" : arguments.level, + "type" : arguments.exception.type.toString(), + "message" : message & " " & arguments.exception.detail, "stack_trace" : isSimpleValue( arguments.exception.StackTrace ) ? listToArray( - arguments.exception.StackTrace, - "#chr( 13 )##chr( 10 )#" - ) : arguments.exception.StackTrace - + arguments.exception.StackTrace, + "#chr( 13 )##chr( 10 )#" + ) : arguments.exception.StackTrace } }; @@ -604,10 +607,10 @@ component public function getComponentTemplate(){ return { "settings" : { - "number_of_shards" : getProperty( "indexShards" ), - "number_of_replicas" : getProperty( "indexReplicas" ), - "index.lifecycle.name": getProperty( "ILMPolicyName" ), - "index.default_pipeline": getProperty( "pipelineName" ) + "number_of_shards" : getProperty( "indexShards" ), + "number_of_replicas" : getProperty( "indexReplicas" ), + "index.lifecycle.name" : getProperty( "ILMPolicyName" ), + "index.default_pipeline" : getProperty( "pipelineName" ) }, "mappings" : { "dynamic_templates" : [ @@ -635,7 +638,7 @@ component } ], "properties" : { - "geoip" : { + "geoip" : { "dynamic" : true, "properties" : { "ip" : { "type" : "ip" }, @@ -645,22 +648,20 @@ component } }, "log" : { - "type" : "object", - "properties" : { - "category" : { "type" : "keyword" } - } + "type" : "object", + "properties" : { "category" : { "type" : "keyword" } } }, "event" : { - "type" : "object", + "type" : "object", "properties" : { - "created" : { "type" : "date", "format" : "date_time_no_millis" }, - "layout" : { "type" : "keyword" }, - "module" : { "type" : "keyword" }, - "view" : { "type" : "keyword" } + "created" : { "type" : "date", "format" : "date_time_no_millis" }, + "layout" : { "type" : "keyword" }, + "module" : { "type" : "keyword" }, + "view" : { "type" : "keyword" } } }, // Customized properties - "stachebox" : { + "stachebox" : { "type" : "object", "properties" : { "signature" : { "type" : "keyword" }, diff --git a/models/migrations/Manager.cfc b/models/migrations/Manager.cfc index 0dc3ee6..b033a43 100644 --- a/models/migrations/Manager.cfc +++ b/models/migrations/Manager.cfc @@ -2,8 +2,14 @@ component { property name="wirebox" inject="wirebox"; property name="migrationsIndex" default=".cfmigrations"; - property name="indexShards" type="numeric" default=1; - property name="indexReplicas" type="numeric" default=0; + property + name ="indexShards" + type ="numeric" + default=1; + property + name ="indexReplicas" + type ="numeric" + default=0; public Manager function init(){ for ( var key in arguments ) { @@ -26,9 +32,9 @@ component { wirebox .getInstance( "IndexBuilder@cbelasticsearch" ) .new( - name = variables.migrationsIndex, + name = variables.migrationsIndex, settings = { - "number_of_shards" : variables.indexShards, + "number_of_shards" : variables.indexShards, "number_of_replicas" : variables.indexReplicas }, properties = { @@ -65,7 +71,7 @@ component { searchBuilder .setQuery( { "match_all" : {} } ) .setSourceIncludes( [ "name", "migrationRan" ] ) - .setMaxRows( searchBuilder.count() ) + .setSize( searchBuilder.count() ) .sort( "migrationRan desc" ); return searchBuilder .execute() diff --git a/models/util/Util.cfc b/models/util/Util.cfc index 5beb0af..3d2b034 100644 --- a/models/util/Util.cfc +++ b/models/util/Util.cfc @@ -130,26 +130,19 @@ component accessors="true" singleton { } void function preflightLogEntry( required struct logObj ){ - - if( !arguments.logObj.keyExists( "@timestamp" ) ){ + if ( !arguments.logObj.keyExists( "@timestamp" ) ) { arguments.logObj[ "@timestamp" ] = dateTimeFormat( now(), "yyyy-mm-dd'T'hh:nn:ssZZ" ) } // ensure consistent casing for search - if( logObj.keyExists( "labels" ) ){ - logObj[ "labels" ][ "environment" ] = lcase( logObj.labels.environment ?: variables.appEnvironment ); + if ( logObj.keyExists( "labels" ) ) { + logObj[ "labels" ][ "environment" ] = lCase( logObj.labels.environment ?: variables.appEnvironment ); } else { - logObj[ "labels" ] = { - "environment" : variables.appEnvironment - } + logObj[ "labels" ] = { "environment" : variables.appEnvironment } } - if( LogObj.keyExists( "error" ) ){ - var errorStringify = [ - "frames", - "extrainfo", - "stack_trace" - ]; + if ( LogObj.keyExists( "error" ) ) { + var errorStringify = [ "frames", "extrainfo", "stack_trace" ]; errorStringify.each( function( key ){ if ( logObj.error.keyExists( key ) && !isSimpleValue( logObj.error[ key ] ) ) { @@ -159,7 +152,6 @@ component accessors="true" singleton { } generateLogEntrySignature( logObj ); - } /** @@ -184,13 +176,16 @@ component accessors="true" singleton { ]; var sigContent = ""; signable.each( function( key ){ - logObj.findKey( listLast( key, "." ), "all" ) - .filter( function( found ){ return found.path == key } ) - .each( function( found ){ - if( !isNull( found.value ) && len( found.value ) ){ - sigContent &= found.value; - } - } ); + logObj + .findKey( listLast( key, "." ), "all" ) + .filter( function( found ){ + return found.path == key + } ) + .each( function( found ){ + if ( !isNull( found.value ) && len( found.value ) ) { + sigContent &= found.value; + } + } ); } ); if ( len( sigContent ) ) { arguments.logObj.stachebox[ "signature" ] = hash( sigContent ); diff --git a/server-adobe@2023.json b/server-adobe@2023.json new file mode 100644 index 0000000..4284271 --- /dev/null +++ b/server-adobe@2023.json @@ -0,0 +1,24 @@ +{ + "name":"cbelasticsearch-adobe@2023", + "app":{ + "serverHomeDirectory":".engine/adobe2023", + "cfengine":"adobe@2023.0.0-beta.1" + }, + "web":{ + "http":{ + "port":"60299" + }, + "rewrites":{ + "enable":"true" + }, + "webroot":"test-harness", + "aliases":{ + "/moduleroot/cbelasticsearch":"../", + "/root":"./test-harness" + } + }, + "openBrowser":"false", + "scripts":{ + "onServerInstall":"cfpm install zip,debugger" + } +} diff --git a/test-harness/box.json b/test-harness/box.json index 9aba809..6df7c18 100644 --- a/test-harness/box.json +++ b/test-harness/box.json @@ -5,7 +5,7 @@ "private":true, "description":"", "dependencies":{ - "coldbox":"^6" + "coldbox":"be" }, "devDependencies":{ "testbox":"*", diff --git a/test-harness/tests/specs/unit/HyperClientTest.cfc b/test-harness/tests/specs/unit/HyperClientTest.cfc index 11bf41c..bd1b135 100644 --- a/test-harness/tests/specs/unit/HyperClientTest.cfc +++ b/test-harness/tests/specs/unit/HyperClientTest.cfc @@ -21,10 +21,6 @@ component extends="coldbox.system.testing.BaseTestCase" { function run(){ describe( "Performs cbElasticsearch HyperClient tests", function(){ - afterEach( function(){ - // we give ourselves a few seconds before each next test for updates to persist - sleep( 500 ); - } ); it( "Tests the ability to create an index", function(){ var builderProperties = { @@ -34,7 +30,8 @@ component extends="coldbox.system.testing.BaseTestCase" { "type" : "text", "fields" : { "kw" : { "type" : "keyword" } } }, - "createdTime" : { "type" : "date", "format" : "date_time_no_millis" } + "createdTime" : { "type" : "date", "format" : "date_time_no_millis" }, + "price" : { "type" : "float" } } } }; @@ -52,10 +49,6 @@ component extends="coldbox.system.testing.BaseTestCase" { } ); describe( "Index mappings and settings and utility methods", function(){ - afterEach( function(){ - // we give ourselves a few seconds before each next test for updates to persist - sleep( 500 ); - } ); it( "Tests the ability to update mappings in an index", function(){ @@ -172,10 +165,6 @@ component extends="coldbox.system.testing.BaseTestCase" { } ); describe( "Document tests", function(){ - afterEach( function(){ - // we give ourselves a few seconds before each next test for updates to persist - sleep( 500 ); - } ); it( "Tests the ability to insert a document in to an index", function(){ @@ -544,11 +533,6 @@ component extends="coldbox.system.testing.BaseTestCase" { describe( "Search tests", function(){ - afterEach( function(){ - // we give ourselves a few seconds before each next test for updates to persist - sleep( 500 ); - } ); - it( "Tests the ability to process a search on an index", function(){ expect( variables ).toHaveKey( "testDocumentId" ); @@ -643,14 +627,83 @@ component extends="coldbox.system.testing.BaseTestCase" { expect( firstResult.getScore() ).toBeGT( secondResult.getScore() ); } ); - } ); - describe( "More fun with documents", function(){ + it( "Tests custom script fields", function(){ + getWirebox().getInstance( "Document@cbElasticsearch" ).new( + variables.testIndexName, + "testdocs", { + "_id" : createUUID(), + "title" : "My Test Document", + "createdTime" : dateTimeFormat( now(), "yyyy-mm-dd'T'hh:nn:ssZZ" ), + "price" : 9.99 + } ) + .save( refresh = true ); + var searchBuilder = getWirebox().getInstance( "SearchBuilder@cbElasticsearch" ).new( + variables.testIndexName, + "testdocs", + { "match_all" : {} } + ); + + searchBuilder.addScriptField( "interestCost", { + "script": { + "lang": "painless", + "source": "return doc['price'].size() != 0 ? doc['price'].value * (params.interestRate/100) : null;", + "params": { "interestRate": 5.5 } + } + } ); + + var hits = variables.model.executeSearch( searchBuilder ).getHits(); + expect( hits.len() ).toBeGT( 0 ); + for( hit in hits ){ + expect( hit.getFields() ).toHaveKey( "interestCost" ); + expect( hit.getDocument( includeFields = true ) ).toHaveKey( "interestCost" ); + expect( hit.getDocument() ).notToHaveKey( "interestCost" ); + } + } ); - afterEach( function(){ - // we give ourselves a few seconds before each next test for updates to persist - sleep( 500 ); + it( "Tests runtime fields", function(){ + getWirebox().getInstance( "IndexBuilder@cbElasticsearch" ) + .patch( name = variables.testIndexName, properties = { + "mappings" : { + "runtime" : { + "price_in_cents" : { + "type" : "long", + "script" : { + "source" : "if( doc['price'].size() != 0) { emit(Math.round(doc['price'].value * 100 )); }" + } + } + } + } + } ); + getWirebox().getInstance( "Document@cbElasticsearch" ).new( + variables.testIndexName, + "testdocs", + { + "_id" : createUUID(), + "title" : "My Test Document", + "createdTime" : dateTimeFormat( now(), "yyyy-mm-dd'T'hh:nn:ssZZ" ), + "price" : 9.99 + } ) + .save( refresh = true ); + var searchBuilder = getWirebox().getInstance( "SearchBuilder@cbElasticsearch" ).new( + variables.testIndexName, + "testdocs" + ) + .filterTerm( "price_in_cents", "999" ) + .addField( "price_in_cents" ); + + var hits = variables.model.executeSearch( searchBuilder ).getHits(); + expect( hits.len() ).toBeGT( 0 ); + for( hit in hits ){ + expect( hit.getFields() ).toHaveKey( "price_in_cents" ); + expect( hit.getDocument( includeFields = true ) ).toHaveKey( "price_in_cents" ); + expect( hit.getDocument() ).notToHaveKey( "price_in_cents" ); + expect( hit.getMemento() ).notToHaveKey( "price_in_cents" ); + } } ); + } ); + + describe( "More fun with documents", function(){ it( "Tests the ability to patch a document with a single field value", function(){ expect( variables ).toHaveKey( "testDocumentId" ); @@ -929,10 +982,6 @@ component extends="coldbox.system.testing.BaseTestCase" { } ); describe( "Post index creation tests", function(){ - afterEach( function(){ - // we give ourselves a few seconds before each next test for updates to persist - sleep( 500 ); - } ); it( "Tests refreshIndex method ", function(){ expect( variables ).toHaveKey( "testIndexName" ); @@ -1619,6 +1668,32 @@ component extends="coldbox.system.testing.BaseTestCase" { } ); } ); + + describe( "General requests", function() { + it( "can query terms enum with options struct", function() { + var result = getInstance( "HyperClient@cbElasticsearch" ) + .getTermsEnum( [ variables.testIndexName ], { + "field" : "title", + "size" : 50 + } ); + expect( result ).toBeStruct() + .toHaveKey( "terms" ) + .toHaveKey( "_shards" ); + }); + it( "can query terms enum with simple arguments", function() { + var result = getInstance( "HyperClient@cbElasticsearch" ) + .getTermsEnum( + indexName = variables.testIndexName, + field = "title", + match = "doc", + size = 50, + caseInsensitive = false + ); + expect( result ).toBeStruct() + .toHaveKey( "terms" ) + .toHaveKey( "_shards" ); + }); + }); } ); } diff --git a/test-harness/tests/specs/unit/SearchBuilderTest.cfc b/test-harness/tests/specs/unit/SearchBuilderTest.cfc index c8eb99c..7ae62d9 100644 --- a/test-harness/tests/specs/unit/SearchBuilderTest.cfc +++ b/test-harness/tests/specs/unit/SearchBuilderTest.cfc @@ -21,7 +21,8 @@ component extends="coldbox.system.testing.BaseTestCase" { "_all" : { "enabled" : false }, "properties" : { "title" : { "type" : "text" }, - "createdTime" : { "type" : "date", "format" : "date_time_no_millis" } + "createdTime" : { "type" : "date", "format" : "date_time_no_millis" }, + "price" : { "type" : "float" } } } } @@ -275,13 +276,17 @@ component extends="coldbox.system.testing.BaseTestCase" { var searchBuilder = variables.model.new( variables.testIndexName, "testdocs" ); var dateStart = dateTimeFormat( now(), "yyyy-mm-dd'T'hh:nn:ssXXX" ); var dateEnd = dateTimeFormat( now(), "yyyy-mm-dd'T'hh:nn:ssXXX" ); - searchBuilder.filterRange( "createdTime", dateStart, dateEnd, 2 ); + searchBuilder.filterRange( "createdTime", dateStart, dateEnd ); expect( searchBuilder.getQuery() ).toBeStruct().toHaveKey( "bool" ); expect( searchBuilder.getQuery().bool ).toHaveKey( "filter" ); - expect( searchBuilder.getQuery().bool.filter ).toBeStruct().toHaveKey( "range" ); - expect( searchBuilder.getQuery().bool.filter.range ).toBeStruct().toHaveKey( "createdTime" ); - expect( searchBuilder.getQuery().bool.filter.range.createdTime ) + expect( searchBuilder.getQuery().bool.filter ).toHaveKey( "bool" ); + expect( searchBuilder.getQuery().bool.filter.bool ).toHaveKey( "must" ); + expect( searchBuilder.getQuery().bool.filter.bool.must ).toBeArray(); + expect( searchBuilder.getQuery().bool.filter.bool.must ).toHaveLength( 1 ); + expect( searchBuilder.getQuery().bool.filter.bool.must[ 1 ] ).toBeStruct().toHaveKey( "range" ); + expect( searchBuilder.getQuery().bool.filter.bool.must[ 1 ].range ).toBeStruct().toHaveKey( "createdTime" ); + expect( searchBuilder.getQuery().bool.filter.bool.must[ 1 ].range.createdTime ) .toBeStruct() .toHaveKey( "gte" ) .toHaveKey( "lte" ); @@ -754,6 +759,93 @@ component extends="coldbox.system.testing.BaseTestCase" { expect( searchBuilder.getDSL()[ "_source" ][ "excludes" ] ).toBe( [ "*.description" ] ); } ); + it( "Tests bodyParam() method for root dsl", function(){ + var searchBuilder = variables.model.new( variables.testIndexName, "testdocs" ); + + searchBuilder.bodyParam( "explain", true ); + searchBuilder.bodyParam( "post_filter", { + "term": { "color": "red" } + } ); + + expect( searchBuilder.getDSL() ).toBeStruct(); + expect( searchBuilder.getDSL() ).toHaveKey( "explain" ); + expect( searchBuilder.getDSL().explain ).toBeTrue( "supports booleans in root DSL" ); + expect( searchBuilder.getDSL() ).toHaveKey( "post_filter" ); + expect( searchBuilder.getDSL().post_filter ).toBeStruct( "supports structs in root DSL" ); + } ); + + it( "Tests bodyParam() and set() methods for root dsl", function(){ + var searchBuilder = variables.model.new( variables.testIndexName, "testdocs" ); + + searchBuilder.set( "min_score", 3 ); + searchBuilder.set( "track_scores", true ); + searchBuilder.set( "min_score", 3 ); + searchBuilder.set( "docvalue_fields", [ + "user.id", + "http.response.*", + { + "field": "date", + "format": "epoch_millis" + } + ] ); + + expect( searchBuilder.getDSL() ).toBeStruct(); + expect( searchBuilder.getDSL() ).toHaveKey( "track_scores" ); + expect( searchBuilder.getDSL().track_scores ).toBeTrue( "supports booleans in root DSL" ); + expect( searchBuilder.getDSL() ).toHaveKey( "min_score" ); + expect( searchBuilder.getDSL().min_score ).toBe( 3, "supports numeric value in root DSL" ); + expect( searchBuilder.getDSL() ).toHaveKey( "docvalue_fields" ); + expect( searchBuilder.getDSL().docvalue_fields ).toBeArray( "supports array value in root DSL"); + + }); + + it( "Tests the setScriptFields() method", function(){ + var searchBuilder = variables.model.new( variables.testIndexName, "testdocs" ); + + searchBuilder.setScriptFields( { + "with5PercentDiscount": { + "script": { + "lang": "painless", + "source": "doc['price'].value * 2" + } + } + } ); + + expect( searchBuilder.getDSL() ).toBeStruct(); + expect( searchBuilder.getDSL() ).toHaveKey( "script_fields" ); + expect( searchBuilder.getDSL()[ "script_fields" ] ).toHaveKey( "with5PercentDiscount" ); + } ); + + it( "Tests the addScriptField() method", function(){ + var searchBuilder = variables.model.new( variables.testIndexName, "testdocs" ); + + searchBuilder.addScriptField( "with5PercentDiscount", { + "script": { + "lang": "painless", + "source": "doc['price'].value * 2" + } + } ); + + expect( searchBuilder.getDSL() ).toBeStruct().toHaveKey( "script_fields" ); + expect( searchBuilder.getDSL()[ "script_fields" ] ).toHaveKey( "with5PercentDiscount" ); + } ); + + it( "Tests the addField() method for retrieving runtime or other fields", function(){ + var search = variables.model.new( variables.testIndexName, "testdocs" ); + + search.addField( "day_of_week" ) + .addField( "name_full" ) + .addField( { + "field": "@timestamp", + "format": "epoch_millis" + } ); + + expect( search.getDSL() ).toBeStruct().toHaveKey( "fields" ); + expect( search.getDSL()[ "fields" ] ).toBeArray() + .toInclude( "day_of_week" ) + .toInclude( "name_full" ); + } ); + it( "Tests the both the setSourceIncludes() and setSourceExcludes() methods", function(){ var searchBuilder = variables.model.new( variables.testIndexName, "testdocs" ); @@ -768,6 +860,67 @@ component extends="coldbox.system.testing.BaseTestCase" { expect( searchBuilder.getDSL()[ "_source" ][ "excludes" ] ).toBe( [ "*.description" ] ); } ); + it( "Tests the pagination maxrows/startrow via .new() properties", function(){ + var searchBuilder = variables.model.new( variables.testIndexName, "testdocs", { + maxRows : 15, + startRow : 16 + } ); + searchBuilder.setQuery( { "match_all": {} } ); + debug( searchBuilder.getDSL() ); + expect( searchBuilder.getDSL() ).toBeStruct(); + expect( searchBuilder.getDSL() ).toHaveKey( "from" ); + expect( searchBuilder.getDSL() ).toHaveKey( "size" ); + expect( searchBuilder.getDSL().from ).toBe( 16 ); + expect( searchBuilder.getDSL().size ).toBe( 15 ); + + expect( searchBuilder.execute() ).toBeInstanceOf( "cbElasticsearch.models.SearchResult" ); + } ); + + it( "Tests the pagination size/from via .new() properties", function(){ + var searchBuilder = variables.model.new( variables.testIndexName, "testdocs", { + size : 15, + from : 16 + } ); + searchBuilder.setQuery( { "match_all": {} } ); + debug( searchBuilder.getDSL() ); + expect( searchBuilder.getDSL() ).toBeStruct(); + expect( searchBuilder.getDSL() ).toHaveKey( "from" ); + expect( searchBuilder.getDSL() ).toHaveKey( "size" ); + expect( searchBuilder.getDSL().from ).toBe( 16 ); + expect( searchBuilder.getDSL().size ).toBe( 15 ); + + expect( searchBuilder.execute() ).toBeInstanceOf( "cbElasticsearch.models.SearchResult" ); + } ); + + it( "Tests pagination via setStartRow()/setMaxRows()", function(){ + var searchBuilder = variables.model.new( variables.testIndexName, "testdocs" ); + searchBuilder.setQuery( { "match_all": {} } ); + searchBuilder.setStartRow( 51 ); + searchBuilder.setMaxRows( 50 ); + + expect( searchBuilder.getDSL() ).toBeStruct(); + expect( searchBuilder.getDSL() ).toHaveKey( "from" ); + expect( searchBuilder.getDSL() ).toHaveKey( "size" ); + expect( searchBuilder.getDSL().from ).toBe( 51 ); + expect( searchBuilder.getDSL().size ).toBe( 50 ); + + expect( searchBuilder.execute() ).toBeInstanceOf( "cbElasticsearch.models.SearchResult" ); + }); + it( "Tests pagination via setFrom()/setSize()", function(){ + var searchBuilder = variables.model.new( variables.testIndexName, "testdocs" ); + searchBuilder.setQuery( { "match_all": {} } ); + searchBuilder.setFrom( 51 ); + searchBuilder.setSize( 50 ); + + expect( searchBuilder.getDSL() ).toBeStruct(); + expect( searchBuilder.getDSL() ).toHaveKey( "from" ); + expect( searchBuilder.getDSL() ).toHaveKey( "size" ); + expect( searchBuilder.getDSL().from ).toBe( 51 ); + expect( searchBuilder.getDSL().size ).toBe( 50 ); + + expect( searchBuilder.execute() ).toBeInstanceOf( "cbElasticsearch.models.SearchResult" ); + }); + describe( "suggestions", function(){ describe( "suggestTerm", function(){ it( "can add a term suggestion for a field", function(){