Seed runtime-added currency conversion rates immediately

Algorithm.CSharp/RuntimeCurrencyConversionSeedingRegressionAlgorithm.cs

@@ -0,0 +1,174 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.Collections.Generic;
+using System.Linq;
+using QuantConnect.Brokerages;
+using QuantConnect.Data;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Regression algorithm asserting that a currency added at runtime (here BTCEUR, from a scheduled event) has its
+    /// conversion rate seeded right away, so using it immediately no longer throws because the rate is still 0.
+    /// </summary>
+    public class RuntimeCurrencyConversionSeedingRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        private Symbol _ltcusd;
+        private bool _addedAtRuntime;
+        private bool _assertedSeeded;
+
+        /// <summary>
+        /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
+        /// </summary>
+        public override void Initialize()
+        {
+            SetStartDate(2018, 4, 5);
+            SetEndDate(2018, 4, 5);
+            SetBrokerageModel(BrokerageName.GDAX, AccountType.Cash);
+            SetCash(100000);
+
+            // Account currency asset that funds the loop
+            _ltcusd = AddCrypto("LTCUSD", Resolution.Minute).Symbol;
+
+            // Add a non-account-currency asset at runtime, mirroring users that add assets from a scheduled event
+            Schedule.On(DateRules.EveryDay(), TimeRules.At(10, 0), () =>
+            {
+                if (_addedAtRuntime)
+                {
+                    return;
+                }
+                _addedAtRuntime = true;
+                AddCrypto("BTCEUR", Resolution.Minute);
+            });
+        }
+
+        /// <summary>
+        /// Runs right after the runtime-added security is wired up, the earliest point it can be used
+        /// </summary>
+        public override void OnSecuritiesChanged(SecurityChanges changes)
+        {
+            if (!changes.AddedSecurities.Any(security => security.Symbol.Value == "BTCEUR"))
+            {
+                return;
+            }
+            _assertedSeeded = true;
+
+            // With the fix these are already seeded here. Without it they would still be 0 and the conversion below would throw.
+            var eur = Portfolio.CashBook["EUR"];
+            var btc = Portfolio.CashBook["BTC"];
+            if (eur.ConversionRate == 0 || btc.ConversionRate == 0)
+            {
+                throw new RegressionTestException(
+                    $"Runtime-added currency conversion rates were not seeded (EUR={eur.ConversionRate}, BTC={btc.ConversionRate})");
+            }
+
+            if (Portfolio.CashBook.ConvertToAccountCurrency(100m, "EUR") <= 0)
+            {
+                throw new RegressionTestException("Expected a positive EUR -> account currency conversion");
+            }
+        }
+
+        /// <summary>
+        /// OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
+        /// </summary>
+        /// <param name="slice">Slice object keyed by symbol containing the stock data</param>
+        public override void OnData(Slice slice)
+        {
+            if (!_addedAtRuntime || Portfolio.Invested)
+            {
+                return;
+            }
+
+            if (Securities[_ltcusd].Price != 0)
+            {
+                SetHoldings(_ltcusd, 0.5);
+            }
+        }
+
+        /// <summary>
+        /// Makes sure the seeding path was actually exercised so the test can't silently pass
+        /// </summary>
+        public override void OnEndOfAlgorithm()
+        {
+            if (!_assertedSeeded)
+            {
+                throw new RegressionTestException("BTCEUR was never added at runtime, the seeding path was not exercised");
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public List<Language> Languages { get; } = new() { Language.CSharp };
+
+        /// <summary>
+        /// Data Points count of all timeslices of algorithm
+        /// </summary>
+        public long DataPoints => 6005;
+
+        /// <summary>
+        /// Data Points count of the algorithm history
+        /// </summary>
+        public int AlgorithmHistoryDataPoints => 591;
+
+        /// <summary>
+        /// Final status of the algorithm
+        /// </summary>
+        public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Orders", "1"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "0%"},
+            {"Drawdown", "0%"},
+            {"Expectancy", "0"},
+            {"Start Equity", "100000.00"},
+            {"End Equity", "99064.52"},
+            {"Net Profit", "0%"},
+            {"Sharpe Ratio", "0"},
+            {"Sortino Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "0%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "0"},
+            {"Tracking Error", "0"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "$149.18"},
+            {"Estimated Strategy Capacity", "$160000.00"},
+            {"Lowest Capacity Asset", "LTCUSD 2XR"},
+            {"Portfolio Turnover", "50.20%"},
+            {"Drawdown Recovery", "0"},
+            {"OrderListHash", "69d27a394cffbd938ec23fbb451f37ae"}
+        };
+    }
+}

Algorithm.CSharp/ScheduledUniverseSelectionModelRegressionAlgorithm.cs

@@ -199,7 +199,7 @@ private void ExpectRemovals(SecurityChanges changes, params string[] tickers)
         /// <summary>
         /// Data Points count of the algorithm history
         /// </summary>
-        public int AlgorithmHistoryDataPoints => 0;
+        public int AlgorithmHistoryDataPoints => 50;
 
         /// <summary>
         /// Final status of the algorithm

Engine/DataFeeds/CurrencySubscriptionDataConfigManager.cs

@@ -131,7 +131,9 @@ public IEnumerable<SubscriptionDataConfig> GetPendingSubscriptionDataConfigs()
         /// <summary>
         /// Checks the current <see cref="SubscriptionDataConfig"/> and adds new necessary currency pair feeds to provide real time conversion data
         /// </summary>
-        public void EnsureCurrencySubscriptionDataConfigs(SecurityChanges securityChanges, IBrokerageModel brokerageModel)
+        /// <returns>True if new currency conversion feeds were introduced, false otherwise. Lets callers skip
+        /// follow up work like seeding the new conversion rates when nothing was added</returns>
+        public bool EnsureCurrencySubscriptionDataConfigs(SecurityChanges securityChanges, IBrokerageModel brokerageModel)
         {
             _ensureCurrencyDataFeeds = false;
             // remove any 'to be added' if the security has already been added
@@ -150,6 +152,8 @@ public void EnsureCurrencySubscriptionDataConfigs(SecurityChanges securityChange
                 _toBeAddedCurrencySubscriptionDataConfigs.Add(config);
             }
             _pendingSubscriptionDataConfigs = _toBeAddedCurrencySubscriptionDataConfigs.Any();
+
+            return newConfigs.Count > 0;
         }
     }
 }

Engine/DataFeeds/UniverseSelection.cs

@@ -128,7 +128,7 @@ public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTi
 
                     // if the input is already fundamental data we just need to filter it and pass it through
                     var hasFundamentalData = universeData.Data.Count > 0 && universeData.Data[0] is Fundamental;
-                    if(hasFundamentalData)
+                    if (hasFundamentalData)
                     {
                         // Remove selected symbols that does not have fine fundamental data
                         var anyDoesNotHaveFundamentalData = false;
@@ -137,7 +137,8 @@ public SecurityChanges ApplyUniverseSelection(Universe universe, DateTime dateTi
                         // which do not use coarse data as underlying, in which case it could happen that we try to load fine fundamental data that is missing, but no problem,
                         // 'FineFundamentalSubscriptionEnumeratorFactory' won't emit it
                         var set = selectSymbolsResult.ToHashSet();
-                        fineCollection.Data.AddRange(universeData.Data.OfType<Fundamental>().Where(fundamental => {
+                        fineCollection.Data.AddRange(universeData.Data.OfType<Fundamental>().Where(fundamental =>
+                        {
                             // we remove to we distict by symbol
                             if (set.Remove(fundamental.Symbol))
                             {
@@ -360,7 +361,7 @@ public bool AddPendingInternalDataFeeds(DateTime utcStart)
                         resolution = supportedResolutions.OrderByDescending(x => x).First();
                     }
 
-                    var subscriptionList = new List<Tuple<Type, TickType>>() {subscriptionType};
+                    var subscriptionList = new List<Tuple<Type, TickType>>() { subscriptionType };
                     var dataConfig = _algorithm.SubscriptionManager.SubscriptionDataConfigService.Add(
                         securityBenchmark.Security.Symbol,
                         resolution,
@@ -418,9 +419,50 @@ public bool AddPendingInternalDataFeeds(DateTime utcStart)
         /// <summary>
         /// Checks the current subscriptions and adds necessary currency pair feeds to provide real time conversion data
         /// </summary>
-        public void EnsureCurrencyDataFeeds(SecurityChanges securityChanges)
+        /// <param name="securityChanges">The security changes to consume</param>
+        /// <param name="seedNewCurrencies">Whether to seed the conversion rate of newly added currencies with their last
+        /// known price. The setup handler passes false because it performs its own (optionally white-listed) seeding</param>
+        public void EnsureCurrencyDataFeeds(SecurityChanges securityChanges, bool seedNewCurrencies = true)
         {
-            _currencySubscriptionDataConfigManager.EnsureCurrencySubscriptionDataConfigs(securityChanges, _algorithm.BrokerageModel);
+            var newCurrencyFeedsAdded = _currencySubscriptionDataConfigManager.EnsureCurrencySubscriptionDataConfigs(securityChanges, _algorithm.BrokerageModel);
+
+            // Only scan the cashbook and seed when a new conversion feed was actually introduced
+            if (!seedNewCurrencies || !newCurrencyFeedsAdded)
+            {
+                return;
+            }
+
+            // Seed the new conversion rates with their last known price so they are non-zero right away, instead of
+            // waiting for the first conversion pair bar to arrive. Otherwise a conversion needed in that gap would
+            // throw. This is the same thing BaseSetupHandler does during setup, but for cashes added at runtime.
+            var cashToUpdate = _algorithm.Portfolio.CashBook.Values
+                .Where(cash => cash.CurrencyConversion != null && cash.ConversionRate == 0)
+                .ToList();
+
+            if (cashToUpdate.Count == 0)
+            {
+                return;
+            }
+
+            var securitiesToUpdate = cashToUpdate
+                .SelectMany(cash => cash.CurrencyConversion.ConversionRateSecurities)
+                .Distinct()
+                .ToList();
+
+            try
+            {
+                AlgorithmUtils.SeedSecurities(securitiesToUpdate, _algorithm);
+
+                foreach (var cash in cashToUpdate)
+                {
+                    cash.Update();
+                }
+            }
+            catch (Exception err)
+            {
+                // Seeding must never break the algorithm, the rate will be set on the first conversion pair bar
+                Log.Error($"UniverseSelection.EnsureCurrencyDataFeeds(): failed to seed runtime currency conversion rate(s): {err.Message}");
+            }
         }
 
         /// <summary>

Engine/Setup/BaseSetupHandler.cs

@@ -84,8 +84,10 @@ public static void SetupCurrencyConversions(
             IReadOnlyCollection<string> currenciesToUpdateWhiteList = null)
         {
             // this is needed to have non-zero currency conversion rates during warmup
-            // will also set the Cash.ConversionRateSecurity
-            universeSelection.EnsureCurrencyDataFeeds(SecurityChanges.None);
+            // will also set the Cash.ConversionRateSecurity.
+            // We don't let it seed the conversion rates here because we do that right below,
+            // where we can also limit the seeding to a specific white list of currencies
+            universeSelection.EnsureCurrencyDataFeeds(SecurityChanges.None, seedNewCurrencies: false);
 
             // now set conversion rates
             Func<Cash, bool> cashToUpdateFilter = currenciesToUpdateWhiteList == null

Tests/Engine/Setup/BaseSetupHandlerTests.cs

@@ -16,6 +16,7 @@
 
 using NUnit.Framework;
 using QuantConnect.Data;
+using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Util;
 using QuantConnect.Data.Auxiliary;
 using QuantConnect.Lean.Engine.Setup;
@@ -110,5 +111,48 @@ public void CurrencyConversionRateResolvedForWhiteListedCurrenciesOnly()
             Assert.AreEqual(0, algorithm.Portfolio.CashBook["EUR"].ConversionRate);
             Assert.AreEqual(0, algorithm.Portfolio.CashBook["USDT"].ConversionRate);
         }
+
+        [Test]
+        public void RuntimeCurrencyConversionRateIsSeeded()
+        {
+            // When a currency requiring a conversion feed is introduced at runtime (a universe adding a security
+            // whose quote currency isn't in the cashbook yet), the runtime path used to wire up the conversion
+            // subscription without seeding its price, leaving the rate at 0 until the first pair bar. After the fix
+            // it seeds the new conversion security right away, just like BaseSetupHandler does during setup.
+
+            var historyProvider = new SubscriptionDataReaderHistoryProvider();
+
+            var algorithm = new BrokerageSetupHandlerTests.TestAlgorithm { UniverseSettings = { Resolution = Resolution.Minute } };
+
+            historyProvider.Initialize(new HistoryProviderInitializeParameters(
+                null,
+                null,
+                TestGlobals.DataProvider,
+                TestGlobals.DataCacheProvider,
+                TestGlobals.MapFileProvider,
+                TestGlobals.FactorFileProvider,
+                null,
+                false,
+                new DataPermissionManager(),
+                algorithm.ObjectStore,
+                algorithm.Settings));
+            algorithm.SetHistoryProvider(historyProvider);
+
+            algorithm.SetStartDate(2015, 1, 24);
+            algorithm.SetCash("USD", 0);
+
+            // Run setup so the engine is in the post-setup (runtime) state
+            BaseSetupHandler.SetupCurrencyConversions(algorithm, algorithm.DataManager.UniverseSelection);
+
+            // Introduce a new currency at runtime and drive the runtime path that wires up its conversion feed
+            algorithm.SetCash("BTC", 10);
+            algorithm.DataManager.UniverseSelection.EnsureCurrencyDataFeeds(SecurityChanges.None);
+
+            // The new currency should already have a non-zero rate, without waiting for a live bar
+            Assert.IsNotNull(algorithm.Portfolio.CashBook["BTC"].CurrencyConversion);
+            Assert.AreNotEqual(0, algorithm.Portfolio.CashBook["BTC"].ConversionRate);
+            Assert.IsTrue(algorithm.Portfolio.CashBook["BTC"].ValueInAccountCurrency > 0);
+            Assert.DoesNotThrow(() => algorithm.Portfolio.CashBook.ConvertToAccountCurrency(10, "BTC"));
+        }
     }
 }