diff --git a/bin/rolling_index b/bin/rolling_index index 53d6b9c..07ce8ac 100755 --- a/bin/rolling_index +++ b/bin/rolling_index @@ -3,7 +3,7 @@ require_relative '../config/environment' require 'daemons' -QUERY = { q: '*:*', sort: 'timestamp asc', fl: 'id', rows: Settings.rolling_indexer.batch_size } +QUERY = { q: '*:*', sort: 'timestamp asc', fl: 'id,timestamp', rows: Settings.rolling_indexer.batch_size } Daemons.run_proc( File.basename(__FILE__), @@ -40,12 +40,16 @@ Daemons.run_proc( end end.compact - solr_conn.add(solr_docs, add_attributes: { commitWithin: 1000 }) + solr_conn.add(solr_docs, add_attributes: { commitWithin: Settings.rolling_indexer.commit_within.to_i }) end_time = Time.now batch_run_seconds = (end_time - start_time).round(3) + first_doc = response['response']['docs'].first + first_doc_str = "#{first_doc['id']} (#{first_doc['timestamp']})" + last_doc = response['response']['docs'].last + last_doc_str = "#{last_doc['id']} (#{last_doc['timestamp']})" # The Daemons gem will redirect this to its log - puts "#{end_time}\tIndexed #{Settings.rolling_indexer.batch_size} documents in #{batch_run_seconds}" + puts "#{end_time}\tIndexed #{Settings.rolling_indexer.batch_size} documents in #{batch_run_seconds} (#{first_doc_str} - #{last_doc_str})" sleep(Settings.rolling_indexer.pause_time_between_batches) end diff --git a/config/settings.yml b/config/settings.yml index 6cfb3fc..b407812 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -2,8 +2,12 @@ date_format_str: '%Y-%m-%d %H:%M:%S.%L' rolling_indexer: batch_size: 500 - pause_time_between_docs: .2 - pause_time_between_batches: 0 + # in seconds + pause_time_between_docs: 0.2 + # commitWithin does a soft commit by default, so a little more than commitWithin is desired (seconds) + pause_time_between_batches: 11 + # milliseconds + commitWithin: 1000 ssl: cert_file: ~