Set correct exchange time to OptionUniverse instances

Common/Data/UniverseSelection/OptionUniverse.cs

@@ -242,7 +242,7 @@ public override BaseData Reader(SubscriptionDataConfig config, StreamReader stre
                 CacheSymbol(key, symbol);
             }
 
-            return new OptionUniverse(date, symbol, remainingLine);
+            return new OptionUniverse(date.ConvertTo(config.DataTimeZone, config.ExchangeTimeZone), symbol, remainingLine);
         }
 
         /// <summary>

Common/Python/PandasConverter.DataFrameGenerator.cs

@@ -33,7 +33,7 @@ public partial class PandasConverter
         private class DataFrameGenerator
         {
             private static readonly string[] MultiBaseDataCollectionDataFrameNames = new[] { "collection_symbol", "time" };
-            private static readonly string[] MultiCanonicalOptionDataFrameNames = new[] { "canonical", "time" };
+            private static readonly string[] MultiCanonicalSymbolsDataFrameNames = new[] { "canonical", "time" };
             private static readonly string[] SingleBaseDataCollectionDataFrameNames = new[] { "time" };
 
             private readonly Type _dataType;
@@ -82,7 +82,7 @@ protected void AddData(IEnumerable<Slice> slices)
                 {
                     foreach (var data in slice.AllData)
                     {
-                        if (_flatten && IsBaseDataCollection(data.GetType()))
+                        if (_flatten && IsCollection(data.GetType()))
                         {
                             AddCollection(data.Symbol, data.EndTime, (data as IEnumerable).Cast<ISymbolProvider>());
                             continue;
@@ -152,9 +152,9 @@ protected void AddData<T>(IEnumerable<T> data)
                 where T : ISymbolProvider
             {
                 var type = typeof(T);
-                var isBaseDataCollection = IsBaseDataCollection(type);
+                var isCollection = IsCollection(type);
 
-                if (_flatten && isBaseDataCollection)
+                if (_flatten && isCollection)
                 {
                     foreach (var collection in data)
                     {
@@ -220,8 +220,8 @@ public PyObject GenerateDataFrame(int? levels = null, bool sort = true, bool fil
                         var keys = collectionsDataFrames
                             .Select(x => new object[] { x.Item1, x.Item2 })
                             .Concat(pandasDataDataFrames.Select(x => new object[] { x, DateTime.MinValue }));
-                        var names = _collections.Any(x => x.Symbol.SecurityType.IsOption() && x.Symbol.IsCanonical())
-                            ? MultiCanonicalOptionDataFrameNames
+                        var names = _collections.Any(x => x.Symbol.IsCanonical())
+                            ? MultiCanonicalSymbolsDataFrameNames
                             : MultiBaseDataCollectionDataFrameNames;
 
                         return ConcatDataFrames(dataFrames, keys, names, sort, dropna: true);
@@ -306,13 +306,13 @@ private void AddCollection(Symbol symbol, DateTime time, IEnumerable<ISymbolProv
             }
 
             /// <summary>
-            /// Determines whether the type is considered a base data collection for flattening.
+            /// Determines whether the type is considered a collection for flattening.
             /// Any object that is a <see cref="BaseData"/> and implements <see cref="IEnumerable{ISymbolProvider}"/>
             /// is considered a base data collection.
             /// This allows detecting collections of cases like <see cref="OptionUniverse"/> (which is a direct subclass of
             /// <see cref="BaseDataCollection"/>) and <see cref="OptionChain"/>, which is a collection of <see cref="OptionContract"/>
             /// </summary>
-            private static bool IsBaseDataCollection(Type type)
+            private static bool IsCollection(Type type)
             {
                 return type.IsAssignableTo(typeof(BaseData)) &&
                     type.GetInterfaces().Any(x => x.IsGenericType &&

Engine/DataFeeds/BacktestingChainProvider.cs

@@ -109,7 +109,6 @@ private IEnumerable<Symbol> GetOptionSymbols(Symbol canonicalSymbol, DateTime da
             var marketHoursDataBase = MarketHoursDatabase.FromDataFolder();
             var marketHoursEntry = marketHoursDataBase.GetEntry(canonicalSymbol.ID.Market, canonicalSymbol, canonicalSymbol.SecurityType);
 
-            date = date.Date;
             var previousTradingDate = Time.GetStartTimeForTradeBars(marketHoursEntry.ExchangeHours, date, Time.OneDay, 1,
                 extendedMarketHours: false, marketHoursEntry.DataTimeZone);
             var request = new HistoryRequest(
@@ -132,7 +131,7 @@ private IEnumerable<Symbol> GetOptionSymbols(Symbol canonicalSymbol, DateTime da
                 return Enumerable.Empty<Symbol>();
             }
 
-            return history.TakeLast(1).GetUniverseData().SelectMany(x => x.Values.Single().Where(x => x.Symbol.SecurityType.IsOption())).Select(x => x.Symbol);
+            return history.GetUniverseData().SelectMany(x => x.Values.Single().Where(x => x.Symbol.SecurityType.IsOption())).Select(x => x.Symbol);
         }
 
         /// <summary>

Engine/DataFeeds/SubscriptionDataReader.cs

@@ -321,10 +321,14 @@ public bool MoveNext()
                     // Advance the time keeper either until the current instance time (to synchronize) or until the source changes.
                     // Note: use time instead of end time to avoid skipping instances that all have the same timestamps in the same file (e.g. universe data)
                     var currentSource = _source;
-                    var nextExchangeDate = _config.Resolution == Resolution.Daily && _timeKeeper.IsExchangeBehindData()
+                    var nextExchangeDate = _config.Resolution == Resolution.Daily
+                        && _timeKeeper.IsExchangeBehindData()
+                        && !_config.Type.IsAssignableTo(typeof(BaseDataCollection))
                         // If daily and exchange is behind data, data for date X will have a start time within date X-1,
                         // so we use the actual date from end time. e.g. a daily bar for Jan15 can have a start time of Jan14 8PM
                         // (exchange tz 4 hours behind data tz) and end time would be Jan15 8PM.
+                        // This doesn't apply to universe files (BaseDataCollection check) because they are not read in the same way
+                        // price daily files are read: they are read in a collection with end time of X+1. We don't want to skip them or advance time yet.
                         ? instance.EndTime
                         : instance.Time;
                     while (_timeKeeper.ExchangeTime < nextExchangeDate && currentSource == _source)
@@ -334,10 +338,6 @@ public bool MoveNext()
 
                     // Source change, check if we should emit the current instance
                     if (currentSource != _source
-                        // Don't skip for base data collections (like universe files): those daily files are not read in the
-                        // same way price daily files are read. e.g universe files for date X are loaded in a collection with end time of X+1,
-                        // we don't what to skip them
-                        && !_config.Type.IsAssignableTo(typeof(BaseDataCollection))
                         && (
                             // After a mapping for every resolution except daily:
                             // For other resolutions, the instance that triggered the exchange date change should be skipped,