Lean  $LEAN_TAG$
LeanData.cs
1 /*
2  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
3  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
4  *
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14 */
15 
16 using System;
17 using System.Collections.Generic;
18 using System.Globalization;
19 using System.IO;
20 using System.Linq;
21 using System.Runtime.CompilerServices;
22 using NodaTime;
23 using QuantConnect.Data;
24 using QuantConnect.Data.Auxiliary;
25 using QuantConnect.Data.Consolidators;
26 using QuantConnect.Data.Market;
27 using QuantConnect.Data.UniverseSelection;
28 using QuantConnect.Logging;
29 using QuantConnect.Securities;
30 using QuantConnect.Securities.Future;
31 using QuantConnect.Securities.Option;
32 using static QuantConnect.StringExtensions;
33 
34 namespace QuantConnect.Util
35 {
36  /// <summary>
37  /// Provides methods for generating lean data file content
38  /// </summary>
39  public static class LeanData
40  {
41  /// <summary>
42  /// The different <see cref="SecurityType"/> used for data paths
43  /// </summary>
44  /// <remarks>This includes 'alternative'</remarks>
45  public static HashSet<string> SecurityTypeAsDataPath => Enum.GetNames(typeof(SecurityType))
46  .Select(x => x.ToLowerInvariant()).Union(new[] { "alternative" }).ToHashSet();
47 
48  /// <summary>
49  /// Converts the specified base data instance into a lean data file csv line.
50  /// This method takes into account the fake that base data instances typically
51  /// are time stamped in the exchange time zone, but need to be written to disk
52  /// in the data time zone.
53  /// </summary>
54  public static string GenerateLine(IBaseData data, Resolution resolution, DateTimeZone exchangeTimeZone, DateTimeZone dataTimeZone)
55  {
56  var clone = data.Clone();
57  clone.Time = data.Time.ConvertTo(exchangeTimeZone, dataTimeZone);
58  return GenerateLine(clone, clone.Symbol.ID.SecurityType, resolution);
59  }
60 
61  /// <summary>
62  /// Helper method that will parse a given data line in search of an associated date time
63  /// </summary>
64  public static DateTime ParseTime(string line, DateTime date, Resolution resolution)
65  {
66  switch (resolution)
67  {
68  case Resolution.Tick:
69  case Resolution.Second:
70  case Resolution.Minute:
71  var index = line.IndexOf(',', StringComparison.InvariantCulture);
72  return date.AddTicks(Convert.ToInt64(10000 * decimal.Parse(line.AsSpan(0, index))));
73  case Resolution.Hour:
74  case Resolution.Daily:
75  return DateTime.ParseExact(line.AsSpan(0, DateFormat.TwelveCharacter.Length), DateFormat.TwelveCharacter, CultureInfo.InvariantCulture);
76  default:
77  throw new ArgumentOutOfRangeException(nameof(resolution), resolution, null);
78  }
79  }
80 
81  /// <summary>
82  /// Converts the specified base data instance into a lean data file csv line
83  /// </summary>
84  public static string GenerateLine(IBaseData data, SecurityType securityType, Resolution resolution)
85  {
86  var milliseconds = data.Time.TimeOfDay.TotalMilliseconds.ToString(CultureInfo.InvariantCulture);
87  var longTime = data.Time.ToStringInvariant(DateFormat.TwelveCharacter);
88 
89  switch (securityType)
90  {
91  case SecurityType.Equity:
92  switch (resolution)
93  {
94  case Resolution.Tick:
95  var tick = (Tick) data;
96  if (tick.TickType == TickType.Trade)
97  {
98  return ToCsv(milliseconds, Scale(tick.LastPrice), tick.Quantity, tick.ExchangeCode, tick.SaleCondition, tick.Suspicious ? "1" : "0");
99  }
100  if (tick.TickType == TickType.Quote)
101  {
102  return ToCsv(milliseconds, Scale(tick.BidPrice), tick.BidSize, Scale(tick.AskPrice), tick.AskSize, tick.ExchangeCode, tick.SaleCondition, tick.Suspicious ? "1" : "0");
103  }
104  break;
105  case Resolution.Minute:
106  case Resolution.Second:
107  var tradeBar = data as TradeBar;
108  if (tradeBar != null)
109  {
110  return ToCsv(milliseconds, Scale(tradeBar.Open), Scale(tradeBar.High), Scale(tradeBar.Low), Scale(tradeBar.Close), tradeBar.Volume);
111  }
112  var quoteBar = data as QuoteBar;
113  if (quoteBar != null)
114  {
115  return ToCsv(milliseconds,
116  ToScaledCsv(quoteBar.Bid), quoteBar.LastBidSize,
117  ToScaledCsv(quoteBar.Ask), quoteBar.LastAskSize);
118  }
119  break;
120 
121  case Resolution.Hour:
122  case Resolution.Daily:
123  var bigTradeBar = data as TradeBar;
124  if (bigTradeBar != null)
125  {
126  return ToCsv(longTime, Scale(bigTradeBar.Open), Scale(bigTradeBar.High), Scale(bigTradeBar.Low), Scale(bigTradeBar.Close), bigTradeBar.Volume);
127  }
128  var bigQuoteBar = data as QuoteBar;
129  if (bigQuoteBar != null)
130  {
131  return ToCsv(longTime,
132  ToScaledCsv(bigQuoteBar.Bid), bigQuoteBar.LastBidSize,
133  ToScaledCsv(bigQuoteBar.Ask), bigQuoteBar.LastAskSize);
134  }
135  break;
136  }
137  break;
138 
139  case SecurityType.Crypto:
140  case SecurityType.CryptoFuture:
141  switch (resolution)
142  {
143  case Resolution.Tick:
144  var tick = data as Tick;
145  if (tick == null)
146  {
147  throw new ArgumentException($"{securityType} tick could not be created", nameof(data));
148  }
149  if (tick.TickType == TickType.Trade)
150  {
151  return ToCsv(milliseconds, tick.LastPrice, tick.Quantity, tick.Suspicious ? "1" : "0");
152  }
153  if (tick.TickType == TickType.Quote)
154  {
155  return ToCsv(milliseconds, tick.BidPrice, tick.BidSize, tick.AskPrice, tick.AskSize, tick.Suspicious ? "1" : "0");
156  }
157  throw new ArgumentException($"{securityType} tick could not be created");
158  case Resolution.Second:
159  case Resolution.Minute:
160  var quoteBar = data as QuoteBar;
161  if (quoteBar != null)
162  {
163  return ToCsv(milliseconds,
164  ToNonScaledCsv(quoteBar.Bid), quoteBar.LastBidSize,
165  ToNonScaledCsv(quoteBar.Ask), quoteBar.LastAskSize);
166  }
167  var tradeBar = data as TradeBar;
168  if (tradeBar != null)
169  {
170  return ToCsv(milliseconds, tradeBar.Open, tradeBar.High, tradeBar.Low, tradeBar.Close, tradeBar.Volume);
171  }
172  throw new ArgumentException($"{securityType} minute/second bar could not be created", nameof(data));
173 
174  case Resolution.Hour:
175  case Resolution.Daily:
176  var bigQuoteBar = data as QuoteBar;
177  if (bigQuoteBar != null)
178  {
179  return ToCsv(longTime,
180  ToNonScaledCsv(bigQuoteBar.Bid), bigQuoteBar.LastBidSize,
181  ToNonScaledCsv(bigQuoteBar.Ask), bigQuoteBar.LastAskSize);
182  }
183  var bigTradeBar = data as TradeBar;
184  if (bigTradeBar != null)
185  {
186  return ToCsv(longTime,
187  bigTradeBar.Open,
188  bigTradeBar.High,
189  bigTradeBar.Low,
190  bigTradeBar.Close,
191  bigTradeBar.Volume);
192  }
193  throw new ArgumentException($"{securityType} hour/daily bar could not be created", nameof(data));
194  }
195  break;
196  case SecurityType.Forex:
197  case SecurityType.Cfd:
198  switch (resolution)
199  {
200  case Resolution.Tick:
201  var tick = data as Tick;
202  if (tick == null)
203  {
204  throw new ArgumentException("Expected data of type 'Tick'", nameof(data));
205  }
206  return ToCsv(milliseconds, tick.BidPrice, tick.AskPrice);
207 
208  case Resolution.Second:
209  case Resolution.Minute:
210  var bar = data as QuoteBar;
211  if (bar == null)
212  {
213  throw new ArgumentException("Expected data of type 'QuoteBar'", nameof(data));
214  }
215  return ToCsv(milliseconds,
216  ToNonScaledCsv(bar.Bid), bar.LastBidSize,
217  ToNonScaledCsv(bar.Ask), bar.LastAskSize);
218 
219  case Resolution.Hour:
220  case Resolution.Daily:
221  var bigBar = data as QuoteBar;
222  if (bigBar == null)
223  {
224  throw new ArgumentException("Expected data of type 'QuoteBar'", nameof(data));
225  }
226  return ToCsv(longTime,
227  ToNonScaledCsv(bigBar.Bid), bigBar.LastBidSize,
228  ToNonScaledCsv(bigBar.Ask), bigBar.LastAskSize);
229  }
230  break;
231 
232  case SecurityType.Index:
233  switch (resolution)
234  {
235  case Resolution.Tick:
236  var tick = (Tick) data;
237  return ToCsv(milliseconds, tick.LastPrice, tick.Quantity, string.Empty, string.Empty, "0");
238  case Resolution.Second:
239  case Resolution.Minute:
240  var bar = data as TradeBar;
241  if (bar == null)
242  {
243  throw new ArgumentException("Expected data of type 'TradeBar'", nameof(data));
244  }
245  return ToCsv(milliseconds, bar.Open, bar.High, bar.Low, bar.Close, bar.Volume);
246  case Resolution.Hour:
247  case Resolution.Daily:
248  var bigTradeBar = data as TradeBar;
249  return ToCsv(longTime, bigTradeBar.Open, bigTradeBar.High, bigTradeBar.Low, bigTradeBar.Close, bigTradeBar.Volume);
250  }
251  break;
252 
253  case SecurityType.Option:
254  case SecurityType.IndexOption:
255  switch (resolution)
256  {
257  case Resolution.Tick:
258  var tick = (Tick)data;
259  if (tick.TickType == TickType.Trade)
260  {
261  return ToCsv(milliseconds,
262  Scale(tick.LastPrice), tick.Quantity, tick.ExchangeCode, tick.SaleCondition, tick.Suspicious ? "1" : "0");
263  }
264  if (tick.TickType == TickType.Quote)
265  {
266  return ToCsv(milliseconds,
267  Scale(tick.BidPrice), tick.BidSize, Scale(tick.AskPrice), tick.AskSize, tick.ExchangeCode, tick.Suspicious ? "1" : "0");
268  }
269  if (tick.TickType == TickType.OpenInterest)
270  {
271  return ToCsv(milliseconds, tick.Value);
272  }
273  break;
274 
275  case Resolution.Second:
276  case Resolution.Minute:
277  // option and future data can be quote or trade bars
278  var quoteBar = data as QuoteBar;
279  if (quoteBar != null)
280  {
281  return ToCsv(milliseconds,
282  ToScaledCsv(quoteBar.Bid), quoteBar.LastBidSize,
283  ToScaledCsv(quoteBar.Ask), quoteBar.LastAskSize);
284  }
285  var tradeBar = data as TradeBar;
286  if (tradeBar != null)
287  {
288  return ToCsv(milliseconds,
289  Scale(tradeBar.Open), Scale(tradeBar.High), Scale(tradeBar.Low), Scale(tradeBar.Close), tradeBar.Volume);
290  }
291  var openInterest = data as OpenInterest;
292  if (openInterest != null)
293  {
294  return ToCsv(milliseconds, openInterest.Value);
295  }
296  break;
297 
298  case Resolution.Hour:
299  case Resolution.Daily:
300  // option and future data can be quote or trade bars
301  var bigQuoteBar = data as QuoteBar;
302  if (bigQuoteBar != null)
303  {
304  return ToCsv(longTime,
305  ToScaledCsv(bigQuoteBar.Bid), bigQuoteBar.LastBidSize,
306  ToScaledCsv(bigQuoteBar.Ask), bigQuoteBar.LastAskSize);
307  }
308  var bigTradeBar = data as TradeBar;
309  if (bigTradeBar != null)
310  {
311  return ToCsv(longTime, ToScaledCsv(bigTradeBar), bigTradeBar.Volume);
312  }
313  var bigOpenInterest = data as OpenInterest;
314  if (bigOpenInterest != null)
315  {
316  return ToCsv(longTime, bigOpenInterest.Value);
317  }
318  break;
319 
320  default:
321  throw new ArgumentOutOfRangeException(nameof(resolution), resolution, null);
322  }
323  break;
324 
325  case SecurityType.FutureOption:
326  switch (resolution)
327  {
328  case Resolution.Tick:
329  var tick = (Tick)data;
330  if (tick.TickType == TickType.Trade)
331  {
332  return ToCsv(milliseconds,
333  tick.LastPrice, tick.Quantity, tick.ExchangeCode, tick.SaleCondition, tick.Suspicious ? "1" : "0");
334  }
335  if (tick.TickType == TickType.Quote)
336  {
337  return ToCsv(milliseconds,
338  tick.BidPrice, tick.BidSize, tick.AskPrice, tick.AskSize, tick.ExchangeCode, tick.Suspicious ? "1" : "0");
339  }
340  if (tick.TickType == TickType.OpenInterest)
341  {
342  return ToCsv(milliseconds, tick.Value);
343  }
344  break;
345 
346  case Resolution.Second:
347  case Resolution.Minute:
348  // option and future data can be quote or trade bars
349  var quoteBar = data as QuoteBar;
350  if (quoteBar != null)
351  {
352  return ToCsv(milliseconds,
353  ToNonScaledCsv(quoteBar.Bid), quoteBar.LastBidSize,
354  ToNonScaledCsv(quoteBar.Ask), quoteBar.LastAskSize);
355  }
356  var tradeBar = data as TradeBar;
357  if (tradeBar != null)
358  {
359  return ToCsv(milliseconds,
360  tradeBar.Open, tradeBar.High, tradeBar.Low, tradeBar.Close, tradeBar.Volume);
361  }
362  var openInterest = data as OpenInterest;
363  if (openInterest != null)
364  {
365  return ToCsv(milliseconds, openInterest.Value);
366  }
367  break;
368 
369  case Resolution.Hour:
370  case Resolution.Daily:
371  // option and future data can be quote or trade bars
372  var bigQuoteBar = data as QuoteBar;
373  if (bigQuoteBar != null)
374  {
375  return ToCsv(longTime,
376  ToNonScaledCsv(bigQuoteBar.Bid), bigQuoteBar.LastBidSize,
377  ToNonScaledCsv(bigQuoteBar.Ask), bigQuoteBar.LastAskSize);
378  }
379  var bigTradeBar = data as TradeBar;
380  if (bigTradeBar != null)
381  {
382  return ToCsv(longTime, ToNonScaledCsv(bigTradeBar), bigTradeBar.Volume);
383  }
384  var bigOpenInterest = data as OpenInterest;
385  if (bigOpenInterest != null)
386  {
387  return ToCsv(longTime, bigOpenInterest.Value);
388  }
389  break;
390 
391  default:
392  throw new ArgumentOutOfRangeException(nameof(resolution), resolution, null);
393  }
394  break;
395 
396  case SecurityType.Future:
397  switch (resolution)
398  {
399  case Resolution.Tick:
400  var tick = (Tick)data;
401  if (tick.TickType == TickType.Trade)
402  {
403  return ToCsv(milliseconds,
404  tick.LastPrice, tick.Quantity, tick.ExchangeCode, tick.SaleCondition, tick.Suspicious ? "1": "0");
405  }
406  if (tick.TickType == TickType.Quote)
407  {
408  return ToCsv(milliseconds,
409  tick.BidPrice, tick.BidSize, tick.AskPrice, tick.AskSize, tick.ExchangeCode, tick.Suspicious ? "1" : "0");
410  }
411  if (tick.TickType == TickType.OpenInterest)
412  {
413  return ToCsv(milliseconds, tick.Value);
414  }
415  break;
416 
417  case Resolution.Second:
418  case Resolution.Minute:
419  // option and future data can be quote or trade bars
420  var quoteBar = data as QuoteBar;
421  if (quoteBar != null)
422  {
423  return ToCsv(milliseconds,
424  ToNonScaledCsv(quoteBar.Bid), quoteBar.LastBidSize,
425  ToNonScaledCsv(quoteBar.Ask), quoteBar.LastAskSize);
426  }
427  var tradeBar = data as TradeBar;
428  if (tradeBar != null)
429  {
430  return ToCsv(milliseconds,
431  tradeBar.Open, tradeBar.High, tradeBar.Low, tradeBar.Close, tradeBar.Volume);
432  }
433  var openInterest = data as OpenInterest;
434  if (openInterest != null)
435  {
436  return ToCsv(milliseconds, openInterest.Value);
437  }
438  break;
439 
440  case Resolution.Hour:
441  case Resolution.Daily:
442  // option and future data can be quote or trade bars
443  var bigQuoteBar = data as QuoteBar;
444  if (bigQuoteBar != null)
445  {
446  return ToCsv(longTime,
447  ToNonScaledCsv(bigQuoteBar.Bid), bigQuoteBar.LastBidSize,
448  ToNonScaledCsv(bigQuoteBar.Ask), bigQuoteBar.LastAskSize);
449  }
450  var bigTradeBar = data as TradeBar;
451  if (bigTradeBar != null)
452  {
453  return ToCsv(longTime, ToNonScaledCsv(bigTradeBar), bigTradeBar.Volume);
454  }
455  var bigOpenInterest = data as OpenInterest;
456  if (bigOpenInterest != null)
457  {
458  return ToCsv(longTime, bigOpenInterest.Value);
459  }
460  break;
461 
462  default:
463  throw new ArgumentOutOfRangeException(nameof(resolution), resolution, null);
464  }
465  break;
466 
467  default:
468  throw new ArgumentOutOfRangeException(nameof(securityType), securityType, null);
469  }
470 
471  throw new NotImplementedException(Invariant(
472  $"LeanData.GenerateLine has not yet been implemented for security type: {securityType} at resolution: {resolution}"
473  ));
474  }
475 
476  /// <summary>
477  /// Gets the data type required for the specified combination of resolution and tick type
478  /// </summary>
479  /// <param name="resolution">The resolution, if Tick, the Type returned is always Tick</param>
480  /// <param name="tickType">The <see cref="TickType"/> that primarily dictates the type returned</param>
481  /// <returns>The Type used to create a subscription</returns>
482  public static Type GetDataType(Resolution resolution, TickType tickType)
483  {
484  if (resolution == Resolution.Tick) return typeof(Tick);
485  if (tickType == TickType.OpenInterest) return typeof(OpenInterest);
486  if (tickType == TickType.Quote) return typeof(QuoteBar);
487  return typeof(TradeBar);
488  }
489 
490 
491  /// <summary>
492  /// Determines if the Type is a 'common' type used throughout lean
493  /// This method is helpful in creating <see cref="SubscriptionDataConfig"/>
494  /// </summary>
495  /// <param name="baseDataType">The Type to check</param>
496  /// <returns>A bool indicating whether the type is of type <see cref="TradeBar"/>
497  /// <see cref="QuoteBar"/> or <see cref="OpenInterest"/></returns>
498  public static bool IsCommonLeanDataType(Type baseDataType)
499  {
500  if (baseDataType == typeof(Tick) ||
501  baseDataType == typeof(TradeBar) ||
502  baseDataType == typeof(QuoteBar) ||
503  baseDataType == typeof(OpenInterest))
504  {
505  return true;
506  }
507 
508  return false;
509  }
510 
511  /// <summary>
512  /// Helper method to determine if a configuration set is valid
513  /// </summary>
514  public static bool IsValidConfiguration(SecurityType securityType, Resolution resolution, TickType tickType)
515  {
516  if (securityType == SecurityType.Equity && (resolution == Resolution.Daily || resolution == Resolution.Hour))
517  {
518  return tickType != TickType.Quote;
519  }
520  return true;
521  }
522 
523  /// <summary>
524  /// Generates the full zip file path rooted in the <paramref name="dataDirectory"/>
525  /// </summary>
526  public static string GenerateZipFilePath(string dataDirectory, Symbol symbol, DateTime date, Resolution resolution, TickType tickType)
527  {
528  // we could call 'GenerateRelativeZipFilePath' but we don't to avoid an extra string & path combine we are doing to drop right away
529  return Path.Combine(dataDirectory, GenerateRelativeZipFileDirectory(symbol, resolution), GenerateZipFileName(symbol, date, resolution, tickType));
530  }
531 
532  /// <summary>
533  /// Generates the full zip file path rooted in the <paramref name="dataDirectory"/>
534  /// </summary>
535  public static string GenerateZipFilePath(string dataDirectory, string symbol, SecurityType securityType, string market, DateTime date, Resolution resolution)
536  {
537  return Path.Combine(dataDirectory, GenerateRelativeZipFilePath(symbol, securityType, market, date, resolution));
538  }
539 
540  /// <summary>
541  /// Generates the relative zip directory for the specified symbol/resolution
542  /// </summary>
543  public static string GenerateRelativeZipFileDirectory(Symbol symbol, Resolution resolution)
544  {
545  var isHourOrDaily = resolution == Resolution.Hour || resolution == Resolution.Daily;
546  var securityType = symbol.SecurityType.SecurityTypeToLower();
547 
548  var market = symbol.ID.Market.ToLowerInvariant();
549  var res = resolution.ResolutionToLower();
550  var directory = Path.Combine(securityType, market, res);
551  switch (symbol.ID.SecurityType)
552  {
553  case SecurityType.Base:
554  case SecurityType.Equity:
555  case SecurityType.Index:
556  case SecurityType.Forex:
557  case SecurityType.Cfd:
558  case SecurityType.Crypto:
559  return !isHourOrDaily ? Path.Combine(directory, symbol.Value.ToLowerInvariant()) : directory;
560 
561  case SecurityType.IndexOption:
562  // For index options, we use the canonical option ticker since it can differ from the underlying's ticker.
563  return !isHourOrDaily ? Path.Combine(directory, symbol.ID.Symbol.ToLowerInvariant()) : directory;
564 
565  case SecurityType.Option:
566  // options uses the underlying symbol for pathing.
567  return !isHourOrDaily ? Path.Combine(directory, symbol.Underlying.Value.ToLowerInvariant()) : directory;
568 
569  case SecurityType.FutureOption:
570  // For futures options, we use the canonical option ticker plus the underlying's expiry
571  // since it can differ from the underlying's ticker. We differ from normal futures
572  // because the option chain can be extraordinarily large compared to equity option chains.
573  var futureOptionPath = Path.Combine(symbol.ID.Symbol, symbol.Underlying.ID.Date.ToStringInvariant(DateFormat.EightCharacter))
574  .ToLowerInvariant();
575 
576  return Path.Combine(directory, futureOptionPath);
577 
578  case SecurityType.Future:
579  case SecurityType.CryptoFuture:
580  return !isHourOrDaily ? Path.Combine(directory, symbol.ID.Symbol.ToLowerInvariant()) : directory;
581 
582  case SecurityType.Commodity:
583  default:
584  throw new ArgumentOutOfRangeException();
585  }
586  }
587 
588  /// <summary>
589  /// Generates relative factor file paths for equities
590  /// </summary>
591  public static string GenerateRelativeFactorFilePath(Symbol symbol)
592  {
593  return Path.Combine(Globals.DataFolder,
594  "equity",
595  symbol.ID.Market,
596  "factor_files",
597  symbol.Value.ToLowerInvariant() + ".csv");
598  }
599 
600  /// <summary>
601  /// Generates the relative zip file path rooted in the /Data directory
602  /// </summary>
603  public static string GenerateRelativeZipFilePath(Symbol symbol, DateTime date, Resolution resolution, TickType tickType)
604  {
605  return Path.Combine(GenerateRelativeZipFileDirectory(symbol, resolution), GenerateZipFileName(symbol, date, resolution, tickType));
606  }
607 
608  /// <summary>
609  /// Generates the relative zip file path rooted in the /Data directory
610  /// </summary>
611  public static string GenerateRelativeZipFilePath(string symbol, SecurityType securityType, string market, DateTime date, Resolution resolution)
612  {
613  var directory = Path.Combine(securityType.SecurityTypeToLower(), market.ToLowerInvariant(), resolution.ResolutionToLower());
614  if (resolution != Resolution.Daily && resolution != Resolution.Hour)
615  {
616  directory = Path.Combine(directory, symbol.ToLowerInvariant());
617  }
618 
619  return Path.Combine(directory, GenerateZipFileName(symbol, securityType, date, resolution));
620  }
621 
622  /// <summary>
623  /// Generate's the zip entry name to hold the specified data.
624  /// </summary>
625  public static string GenerateZipEntryName(Symbol symbol, DateTime date, Resolution resolution, TickType tickType)
626  {
627  var formattedDate = date.ToStringInvariant(DateFormat.EightCharacter);
628  var isHourOrDaily = resolution == Resolution.Hour || resolution == Resolution.Daily;
629 
630  switch (symbol.ID.SecurityType)
631  {
632  case SecurityType.Base:
633  case SecurityType.Equity:
634  case SecurityType.Index:
635  case SecurityType.Forex:
636  case SecurityType.Cfd:
637  case SecurityType.Crypto:
638  if (resolution == Resolution.Tick && symbol.SecurityType == SecurityType.Equity)
639  {
640  return Invariant($"{formattedDate}_{symbol.Value.ToLowerInvariant()}_{tickType}_{resolution}.csv");
641  }
642 
643  if (isHourOrDaily)
644  {
645  return $"{symbol.Value.ToLowerInvariant()}.csv";
646  }
647 
648  return Invariant($"{formattedDate}_{symbol.Value.ToLowerInvariant()}_{resolution.ResolutionToLower()}_{tickType.TickTypeToLower()}.csv");
649 
650  case SecurityType.Option:
651  var optionPath = symbol.Underlying.Value.ToLowerInvariant();
652 
653  if (isHourOrDaily)
654  {
655  return string.Join("_",
656  optionPath,
657  tickType.TickTypeToLower(),
658  symbol.ID.OptionStyle.OptionStyleToLower(),
659  symbol.ID.OptionRight.OptionRightToLower(),
660  Scale(symbol.ID.StrikePrice),
661  symbol.ID.Date.ToStringInvariant(DateFormat.EightCharacter)
662  ) + ".csv";
663  }
664 
665  return string.Join("_",
666  formattedDate,
667  optionPath,
668  resolution.ResolutionToLower(),
669  tickType.TickTypeToLower(),
670  symbol.ID.OptionStyle.OptionStyleToLower(),
671  symbol.ID.OptionRight.OptionRightToLower(),
672  Scale(symbol.ID.StrikePrice),
673  symbol.ID.Date.ToStringInvariant(DateFormat.EightCharacter)
674  ) + ".csv";
675 
676  case SecurityType.IndexOption:
677  case SecurityType.FutureOption:
678  // We want the future/index option ticker as the lookup name inside the ZIP file
679  var optionTickerBasedPath = symbol.ID.Symbol.ToLowerInvariant();
680 
681  if (isHourOrDaily)
682  {
683  return string.Join("_",
684  optionTickerBasedPath,
685  tickType.TickTypeToLower(),
686  symbol.ID.OptionStyle.OptionStyleToLower(),
687  symbol.ID.OptionRight.OptionRightToLower(),
688  Scale(symbol.ID.StrikePrice),
689  symbol.ID.Date.ToStringInvariant(DateFormat.EightCharacter)
690  ) + ".csv";
691  }
692 
693  return string.Join("_",
694  formattedDate,
695  optionTickerBasedPath,
696  resolution.ResolutionToLower(),
697  tickType.TickTypeToLower(),
698  symbol.ID.OptionStyle.OptionStyleToLower(),
699  symbol.ID.OptionRight.OptionRightToLower(),
700  Scale(symbol.ID.StrikePrice),
701  symbol.ID.Date.ToStringInvariant(DateFormat.EightCharacter)
702  ) + ".csv";
703 
704  case SecurityType.Future:
705  case SecurityType.CryptoFuture:
706  if (symbol.HasUnderlying)
707  {
708  symbol = symbol.Underlying;
709  }
710 
711  string expirationTag;
712  var expiryDate = symbol.ID.Date;
713  if(expiryDate != SecurityIdentifier.DefaultDate)
714  {
715  var monthsToAdd = FuturesExpiryUtilityFunctions.GetDeltaBetweenContractMonthAndContractExpiry(symbol.ID.Symbol, expiryDate.Date);
716  var contractYearMonth = expiryDate.AddMonths(monthsToAdd).ToStringInvariant(DateFormat.YearMonth);
717 
718  expirationTag = $"{contractYearMonth}_{expiryDate.ToStringInvariant(DateFormat.EightCharacter)}";
719  }
720  else
721  {
722  expirationTag = "perp";
723  }
724 
725  if (isHourOrDaily)
726  {
727  return string.Join("_",
728  symbol.ID.Symbol.ToLowerInvariant(),
729  tickType.TickTypeToLower(),
730  expirationTag
731  ) + ".csv";
732  }
733 
734  return string.Join("_",
735  formattedDate,
736  symbol.ID.Symbol.ToLowerInvariant(),
737  resolution.ResolutionToLower(),
738  tickType.TickTypeToLower(),
739  expirationTag
740  ) + ".csv";
741 
742  case SecurityType.Commodity:
743  default:
744  throw new ArgumentOutOfRangeException();
745  }
746  }
747 
748  /// <summary>
749  /// Generates the zip file name for the specified date of data.
750  /// </summary>
751  public static string GenerateZipFileName(Symbol symbol, DateTime date, Resolution resolution, TickType tickType)
752  {
753  var tickTypeString = tickType.TickTypeToLower();
754  var formattedDate = date.ToStringInvariant(DateFormat.EightCharacter);
755  var isHourOrDaily = resolution == Resolution.Hour || resolution == Resolution.Daily;
756 
757  switch (symbol.ID.SecurityType)
758  {
759  case SecurityType.Base:
760  case SecurityType.Index:
761  case SecurityType.Equity:
762  case SecurityType.Forex:
763  case SecurityType.Cfd:
764  if (isHourOrDaily)
765  {
766  return $"{symbol.Value.ToLowerInvariant()}.zip";
767  }
768 
769  return $"{formattedDate}_{tickTypeString}.zip";
770  case SecurityType.Crypto:
771  if (isHourOrDaily)
772  {
773  return $"{symbol.Value.ToLowerInvariant()}_{tickTypeString}.zip";
774  }
775 
776  return $"{formattedDate}_{tickTypeString}.zip";
777  case SecurityType.Option:
778  if (isHourOrDaily)
779  {
780  // see TryParsePath: he knows tick type position is 3
781  var optionPath = symbol.Underlying.Value.ToLowerInvariant();
782  return $"{optionPath}_{date.Year}_{tickTypeString}_{symbol.ID.OptionStyle.OptionStyleToLower()}.zip";
783  }
784 
785  return $"{formattedDate}_{tickTypeString}_{symbol.ID.OptionStyle.OptionStyleToLower()}.zip";
786 
787  case SecurityType.IndexOption:
788  case SecurityType.FutureOption:
789  if (isHourOrDaily)
790  {
791  // see TryParsePath: he knows tick type position is 3
792  var optionTickerBasedPath = symbol.ID.Symbol.ToLowerInvariant();
793  return $"{optionTickerBasedPath}_{date.Year}_{tickTypeString}_{symbol.ID.OptionStyle.OptionStyleToLower()}.zip";
794  }
795 
796  return $"{formattedDate}_{tickTypeString}_{symbol.ID.OptionStyle.OptionStyleToLower()}.zip";
797 
798  case SecurityType.Future:
799  case SecurityType.CryptoFuture:
800  if (isHourOrDaily)
801  {
802  return $"{symbol.ID.Symbol.ToLowerInvariant()}_{tickTypeString}.zip";
803  }
804 
805  return $"{formattedDate}_{tickTypeString}.zip";
806 
807  case SecurityType.Commodity:
808  default:
809  throw new ArgumentOutOfRangeException();
810  }
811  }
812 
813  /// <summary>
814  /// Creates the zip file name for a QC zip data file
815  /// </summary>
816  public static string GenerateZipFileName(string symbol, SecurityType securityType, DateTime date, Resolution resolution, TickType? tickType = null)
817  {
818  if (resolution == Resolution.Hour || resolution == Resolution.Daily)
819  {
820  return $"{symbol.ToLowerInvariant()}.zip";
821  }
822 
823  var zipFileName = date.ToStringInvariant(DateFormat.EightCharacter);
824 
825  if (tickType == null)
826  {
827  if (securityType == SecurityType.Forex || securityType == SecurityType.Cfd) {
828  tickType = TickType.Quote;
829  }
830  else
831  {
832  tickType = TickType.Trade;
833  }
834  }
835 
836  var suffix = Invariant($"_{tickType.Value.TickTypeToLower()}.zip");
837  return zipFileName + suffix;
838  }
839 
840  /// <summary>
841  /// Gets the tick type most commonly associated with the specified security type
842  /// </summary>
843  /// <param name="securityType">The security type</param>
844  /// <returns>The most common tick type for the specified security type</returns>
845  public static TickType GetCommonTickType(SecurityType securityType)
846  {
847  if (securityType == SecurityType.Forex || securityType == SecurityType.Cfd || securityType == SecurityType.Crypto)
848  {
849  return TickType.Quote;
850  }
851  return TickType.Trade;
852  }
853 
854  /// <summary>
855  /// Creates a symbol from the specified zip entry name
856  /// </summary>
857  /// <param name="symbol">The root symbol of the output symbol</param>
858  /// <param name="resolution">The resolution of the data source producing the zip entry name</param>
859  /// <param name="zipEntryName">The zip entry name to be parsed</param>
860  /// <returns>A new symbol representing the zip entry name</returns>
861  public static Symbol ReadSymbolFromZipEntry(Symbol symbol, Resolution resolution, string zipEntryName)
862  {
863  var isHourlyOrDaily = resolution == Resolution.Hour || resolution == Resolution.Daily;
864  var parts = zipEntryName.Replace(".csv", string.Empty).Split('_');
865  switch (symbol.ID.SecurityType)
866  {
867  case SecurityType.Option:
868  case SecurityType.FutureOption:
869  case SecurityType.IndexOption:
870  if (isHourlyOrDaily)
871  {
872  var style = parts[2].ParseOptionStyle();
873  var right = parts[3].ParseOptionRight();
874  var strike = Parse.Decimal(parts[4]) / 10000m;
875  var expiry = Parse.DateTimeExact(parts[5], DateFormat.EightCharacter);
876  return Symbol.CreateOption(symbol.Underlying, symbol.ID.Symbol, symbol.ID.Market, style, right, strike, expiry);
877  }
878  else
879  {
880  var style = parts[4].ParseOptionStyle();
881  var right = parts[5].ParseOptionRight();
882  var strike = Parse.Decimal(parts[6]) / 10000m;
883  var expiry = DateTime.ParseExact(parts[7], DateFormat.EightCharacter, CultureInfo.InvariantCulture);
884  return Symbol.CreateOption(symbol.Underlying, symbol.ID.Symbol, symbol.ID.Market, style, right, strike, expiry);
885  }
886 
887  case SecurityType.Future:
888  if (isHourlyOrDaily)
889  {
890  var expiryYearMonth = Parse.DateTimeExact(parts[2], DateFormat.YearMonth);
891  var futureExpiryFunc = FuturesExpiryFunctions.FuturesExpiryFunction(symbol);
892  var futureExpiry = futureExpiryFunc(expiryYearMonth);
893  return Symbol.CreateFuture(parts[0], symbol.ID.Market, futureExpiry);
894  }
895  else
896  {
897  var expiryYearMonth = Parse.DateTimeExact(parts[4], DateFormat.YearMonth);
898  var futureExpiryFunc = FuturesExpiryFunctions.FuturesExpiryFunction(symbol);
899  var futureExpiry = futureExpiryFunc(expiryYearMonth);
900  return Symbol.CreateFuture(parts[1], symbol.ID.Market, futureExpiry);
901  }
902 
903  default:
904  throw new NotImplementedException(Invariant(
905  $"ReadSymbolFromZipEntry is not implemented for {symbol.ID.SecurityType} {symbol.ID.Market} {resolution}"
906  ));
907  }
908  }
909 
910  /// <summary>
911  /// Scale and convert the resulting number to deci-cents int.
912  /// </summary>
913  private static long Scale(decimal value)
914  {
915  return (long)(value*10000);
916  }
917 
918  /// <summary>
919  /// Create a csv line from the specified arguments
920  /// </summary>
921  private static string ToCsv(params object[] args)
922  {
923  // use culture neutral formatting for decimals
924  for (var i = 0; i < args.Length; i++)
925  {
926  var value = args[i];
927  if (value is decimal)
928  {
929  args[i] = ((decimal) value).Normalize();
930  }
931  }
932 
933  var argsFormatted = args.Select(x => Convert.ToString(x, CultureInfo.InvariantCulture));
934  return string.Join(",", argsFormatted);
935  }
936 
937  /// <summary>
938  /// Creates a scaled csv line for the bar, if null fills in empty strings
939  /// </summary>
940  private static string ToScaledCsv(IBar bar)
941  {
942  if (bar == null)
943  {
944  return ToCsv(string.Empty, string.Empty, string.Empty, string.Empty);
945  }
946 
947  return ToCsv(Scale(bar.Open), Scale(bar.High), Scale(bar.Low), Scale(bar.Close));
948  }
949 
950 
951  /// <summary>
952  /// Creates a non scaled csv line for the bar, if null fills in empty strings
953  /// </summary>
954  private static string ToNonScaledCsv(IBar bar)
955  {
956  if (bar == null)
957  {
958  return ToCsv(string.Empty, string.Empty, string.Empty, string.Empty);
959  }
960 
961  return ToCsv(bar.Open, bar.High, bar.Low, bar.Close);
962  }
963 
964  /// <summary>
965  /// Get the <see cref="TickType"/> for common Lean data types.
966  /// If not a Lean common data type, return a TickType of Trade.
967  /// </summary>
968  /// <param name="type">A Type used to determine the TickType</param>
969  /// <param name="securityType">The SecurityType used to determine the TickType</param>
970  /// <returns>A TickType corresponding to the type</returns>
971  public static TickType GetCommonTickTypeForCommonDataTypes(Type type, SecurityType securityType)
972  {
973  if (type == typeof(TradeBar))
974  {
975  return TickType.Trade;
976  }
977  if (type == typeof(QuoteBar))
978  {
979  return TickType.Quote;
980  }
981  if (type == typeof(OpenInterest))
982  {
983  return TickType.OpenInterest;
984  }
985  if (type == typeof(ZipEntryName))
986  {
987  return TickType.Quote;
988  }
989  if (type == typeof(Tick))
990  {
991  return GetCommonTickType(securityType);
992  }
993 
994  return TickType.Trade;
995  }
996 
997  /// <summary>
998  /// Matches a data path security type with the <see cref="SecurityType"/>
999  /// </summary>
1000  /// <remarks>This includes 'alternative'</remarks>
1001  /// <param name="securityType">The data path security type</param>
1002  /// <returns>The matching security type for the given data path</returns>
1003  public static SecurityType ParseDataSecurityType(string securityType)
1004  {
1005  if (securityType.Equals("alternative", StringComparison.InvariantCultureIgnoreCase))
1006  {
1007  return SecurityType.Base;
1008  }
1009  return (SecurityType) Enum.Parse(typeof(SecurityType), securityType, true);
1010  }
1011 
1012  /// <summary>
1013  /// Parses file name into a <see cref="Security"/> and DateTime
1014  /// </summary>
1015  /// <param name="fileName">File name to be parsed</param>
1016  /// <param name="securityType">The securityType as parsed from the fileName</param>
1017  /// <param name="market">The market as parsed from the fileName</param>
1018  public static bool TryParseSecurityType(string fileName, out SecurityType securityType, out string market)
1019  {
1020  securityType = SecurityType.Base;
1021  market = string.Empty;
1022 
1023  try
1024  {
1025  var info = SplitDataPath(fileName);
1026 
1027  // find the securityType and parse it
1028  var typeString = info.Find(x => SecurityTypeAsDataPath.Contains(x.ToLowerInvariant()));
1029  securityType = ParseDataSecurityType(typeString);
1030 
1031  var existingMarkets = Market.SupportedMarkets();
1032  var foundMarket = info.Find(x => existingMarkets.Contains(x.ToLowerInvariant()));
1033  if (foundMarket != null)
1034  {
1035  market = foundMarket;
1036  }
1037  }
1038  catch (Exception e)
1039  {
1040  Log.Error($"LeanData.TryParsePath(): Error encountered while parsing the path {fileName}. Error: {e.GetBaseException()}");
1041  return false;
1042  }
1043 
1044  return true;
1045  }
1046 
1047  /// <summary>
1048  /// Parses file name into a <see cref="Security"/> and DateTime
1049  /// </summary>
1050  /// <param name="filePath">File path to be parsed</param>
1051  /// <param name="symbol">The symbol as parsed from the fileName</param>
1052  /// <param name="date">Date of data in the file path. Only returned if the resolution is lower than Hourly</param>
1053  /// <param name="resolution">The resolution of the symbol as parsed from the filePath</param>
1054  /// <param name="tickType">The tick type</param>
1055  /// <param name="dataType">The data type</param>
1056  public static bool TryParsePath(string filePath, out Symbol symbol, out DateTime date,
1057  out Resolution resolution, out TickType tickType, out Type dataType)
1058  {
1059  symbol = default;
1060  tickType = default;
1061  dataType = default;
1062  date = default;
1063  resolution = default;
1064 
1065  try
1066  {
1067  if (!TryParsePath(filePath, out symbol, out date, out resolution))
1068  {
1069  return false;
1070  }
1071 
1072  tickType = GetCommonTickType(symbol.SecurityType);
1073  var fileName = Path.GetFileNameWithoutExtension(filePath);
1074  if (fileName.Contains("_"))
1075  {
1076  // example: 20140606_openinterest_american.zip
1077  var tickTypePosition = 1;
1078  if (resolution >= Resolution.Hour && symbol.SecurityType.IsOption())
1079  {
1080  // daily and hourly have the year too, example: aapl_2014_openinterest_american.zip
1081  // see GenerateZipFileName he's creating these paths
1082  tickTypePosition = 2;
1083  }
1084  tickType = (TickType)Enum.Parse(typeof(TickType), fileName.Split('_')[tickTypePosition], true);
1085  }
1086 
1087  dataType = GetDataType(resolution, tickType);
1088  return true;
1089  }
1090  catch (Exception ex)
1091  {
1092  Log.Debug($"LeanData.TryParsePath(): Error encountered while parsing the path {filePath}. Error: {ex.GetBaseException()}");
1093  }
1094  return false;
1095  }
1096 
1097  /// <summary>
1098  /// Parses file name into a <see cref="Security"/> and DateTime
1099  /// </summary>
1100  /// <param name="fileName">File name to be parsed</param>
1101  /// <param name="symbol">The symbol as parsed from the fileName</param>
1102  /// <param name="date">Date of data in the file path. Only returned if the resolution is lower than Hourly</param>
1103  /// <param name="resolution">The resolution of the symbol as parsed from the filePath</param>
1104  public static bool TryParsePath(string fileName, out Symbol symbol, out DateTime date, out Resolution resolution)
1105  {
1106  symbol = null;
1107  resolution = Resolution.Daily;
1108  date = default(DateTime);
1109 
1110  try
1111  {
1112  var info = SplitDataPath(fileName);
1113 
1114  // find where the useful part of the path starts - i.e. the securityType
1115  var startIndex = info.FindIndex(x => SecurityTypeAsDataPath.Contains(x.ToLowerInvariant()));
1116 
1117  if(startIndex == -1)
1118  {
1119  if (Log.DebuggingEnabled)
1120  {
1121  Log.Debug($"LeanData.TryParsePath(): Failed to parse '{fileName}' unexpected SecurityType");
1122  }
1123  // SPDB & MHDB folders
1124  return false;
1125  }
1126  var securityType = ParseDataSecurityType(info[startIndex]);
1127 
1128  var market = Market.USA;
1129  string ticker;
1130  var isUniverses = false;
1131  if (!Enum.TryParse(info[startIndex + 2], true, out resolution))
1132  {
1133  resolution = Resolution.Daily;
1134  isUniverses = info[startIndex + 2].Equals("universes", StringComparison.InvariantCultureIgnoreCase);
1135  if (securityType != SecurityType.Base)
1136  {
1137  if (!isUniverses)
1138  {
1139  if (Log.DebuggingEnabled)
1140  {
1141  Log.Debug($"LeanData.TryParsePath(): Failed to parse '{fileName}' unexpected Resolution");
1142  }
1143  // only acept a failure to parse resolution if we are facing a universes path
1144  return false;
1145  }
1146  securityType = SecurityType.Base;
1147  }
1148  }
1149 
1150  if (securityType == SecurityType.Base)
1151  {
1152  // the last part of the path is the file name
1153  var fileNameNoPath = info[info.Count - 1].Split('_').First();
1154 
1155  if (!DateTime.TryParseExact(fileNameNoPath,
1156  DateFormat.EightCharacter,
1157  DateTimeFormatInfo.InvariantInfo,
1158  DateTimeStyles.None,
1159  out date))
1160  {
1161  // if parsing the date failed we assume filename is ticker
1162  ticker = fileNameNoPath;
1163  }
1164  else
1165  {
1166  // ticker must be the previous part of the path
1167  ticker = info[info.Count - 2];
1168  }
1169  }
1170  else
1171  {
1172  // Gather components used to create the security
1173  market = info[startIndex + 1];
1174  var components = info[startIndex + 3].Split('_');
1175 
1176  // Remove the ticktype from the ticker (Only exists in Crypto and Future data but causes no issues)
1177  ticker = components[0];
1178 
1179  if (resolution < Resolution.Hour)
1180  {
1181  // Future options are special and have the following format Market/Resolution/Ticker/FutureExpiry/Date
1182  var dateIndex = securityType == SecurityType.FutureOption ? startIndex + 5 : startIndex + 4;
1183  date = Parse.DateTimeExact(info[dateIndex].Substring(0, 8), DateFormat.EightCharacter);
1184  }
1185  // If resolution is Daily or Hour for options and index options, we can only get the year from the path
1186  else if (securityType == SecurityType.Option || securityType == SecurityType.IndexOption)
1187  {
1188  var year = int.Parse(components[1], CultureInfo.InvariantCulture);
1189  date = new DateTime(year, 01, 01);
1190  }
1191  }
1192 
1193  // Future Options cannot use Symbol.Create
1194  if (securityType == SecurityType.FutureOption)
1195  {
1196  // Future options have underlying FutureExpiry date as the parent dir for the zips, we need this for our underlying
1197  var underlyingFutureExpiryDate = Parse.DateTimeExact(info[startIndex + 4].Substring(0, 8), DateFormat.EightCharacter);
1198 
1199  var underlyingTicker = OptionSymbol.MapToUnderlying(ticker, securityType);
1200  // Create our underlying future and then the Canonical option for this future
1201  var underlyingFuture = Symbol.CreateFuture(underlyingTicker, market, underlyingFutureExpiryDate);
1202  symbol = Symbol.CreateCanonicalOption(underlyingFuture);
1203  }
1204  else if(securityType == SecurityType.IndexOption)
1205  {
1206  var underlyingTicker = OptionSymbol.MapToUnderlying(ticker, securityType);
1207  // Create our underlying index and then the Canonical option
1208  var underlyingIndex = Symbol.Create(underlyingTicker, SecurityType.Index, market);
1209  symbol = Symbol.CreateCanonicalOption(underlyingIndex, ticker, market, null);
1210  }
1211  else
1212  {
1213  Type dataType = null;
1214  if (isUniverses && info[startIndex + 3].Equals("etf", StringComparison.InvariantCultureIgnoreCase))
1215  {
1216  dataType = typeof(ETFConstituentUniverse);
1217  }
1218  symbol = CreateSymbol(ticker, securityType, market, dataType, date);
1219  }
1220 
1221  }
1222  catch (Exception ex)
1223  {
1224  Log.Debug($"LeanData.TryParsePath(): Error encountered while parsing the path {fileName}. Error: {ex.GetBaseException()}");
1225  return false;
1226  }
1227 
1228  return true;
1229  }
1230 
1231  /// <summary>
1232  /// Creates a new Symbol based on parsed data path information.
1233  /// </summary>
1234  /// <param name="ticker">The parsed ticker symbol.</param>
1235  /// <param name="securityType">The parsed type of security.</param>
1236  /// <param name="market">The parsed market or exchange.</param>
1237  /// <param name="dataType">Optional type used for generating the base data SID (applicable only for SecurityType.Base).</param>
1238  /// <param name="mappingResolveDate">The date used in path parsing to create the correct symbol.</param>
1239  /// <returns>A unique security identifier.</returns>
1240  /// <example>
1241  /// <code>
1242  /// path: equity/usa/minute/spwr/20071223_trade.zip
1243  /// ticker: spwr
1244  /// securityType: equity
1245  /// market: usa
1246  /// mappingResolveDate: 2007/12/23
1247  /// </code>
1248  /// </example>
1249  private static Symbol CreateSymbol(string ticker, SecurityType securityType, string market, Type dataType, DateTime mappingResolveDate = default)
1250  {
1251  if (mappingResolveDate != default && (securityType == SecurityType.Equity || securityType == SecurityType.Option))
1252  {
1253  var symbol = new Symbol(SecurityIdentifier.GenerateEquity(ticker, market, mappingResolveDate: mappingResolveDate), ticker);
1254  return securityType == SecurityType.Option ? Symbol.CreateCanonicalOption(symbol) : symbol;
1255  }
1256  else
1257  {
1258  return Symbol.Create(ticker, securityType, market, baseDataType: dataType);
1259  }
1260  }
1261 
1262  private static List<string> SplitDataPath(string fileName)
1263  {
1264  var pathSeparators = new[] { '/', '\\' };
1265 
1266  // Removes file extension
1267  fileName = fileName.Replace(fileName.GetExtension(), string.Empty);
1268 
1269  // remove any relative file path
1270  while (fileName.First() == '.' || pathSeparators.Any(x => x == fileName.First()))
1271  {
1272  fileName = fileName.Remove(0, 1);
1273  }
1274 
1275  // split path into components
1276  return fileName.Split(pathSeparators, StringSplitOptions.RemoveEmptyEntries).ToList();
1277  }
1278 
1279  /// <summary>
1280  /// Aggregates a list of second/minute bars at the requested resolution
1281  /// </summary>
1282  /// <param name="bars">List of <see cref="TradeBar"/>s</param>
1283  /// <param name="symbol">Symbol of all tradeBars</param>
1284  /// <param name="resolution">Desired resolution for new <see cref="TradeBar"/>s</param>
1285  /// <returns>List of aggregated <see cref="TradeBar"/>s</returns>
1286  public static IEnumerable<TradeBar> AggregateTradeBars(IEnumerable<TradeBar> bars, Symbol symbol, TimeSpan resolution)
1287  {
1288  return Aggregate(new TradeBarConsolidator(resolution), bars, symbol);
1289  }
1290 
1291  /// <summary>
1292  /// Aggregates a list of second/minute bars at the requested resolution
1293  /// </summary>
1294  /// <param name="bars">List of <see cref="QuoteBar"/>s</param>
1295  /// <param name="symbol">Symbol of all QuoteBars</param>
1296  /// <param name="resolution">Desired resolution for new <see cref="QuoteBar"/>s</param>
1297  /// <returns>List of aggregated <see cref="QuoteBar"/>s</returns>
1298  public static IEnumerable<QuoteBar> AggregateQuoteBars(IEnumerable<QuoteBar> bars, Symbol symbol, TimeSpan resolution)
1299  {
1300  return Aggregate(new QuoteBarConsolidator(resolution), bars, symbol);
1301  }
1302 
1303  /// <summary>
1304  /// Aggregates a list of ticks at the requested resolution
1305  /// </summary>
1306  /// <param name="ticks">List of quote ticks</param>
1307  /// <param name="symbol">Symbol of all ticks</param>
1308  /// <param name="resolution">Desired resolution for new <see cref="QuoteBar"/>s</param>
1309  /// <returns>List of aggregated <see cref="QuoteBar"/>s</returns>
1310  public static IEnumerable<QuoteBar> AggregateTicks(IEnumerable<Tick> ticks, Symbol symbol, TimeSpan resolution)
1311  {
1312  return Aggregate(new TickQuoteBarConsolidator(resolution), ticks, symbol);
1313  }
1314 
1315  /// <summary>
1316  /// Aggregates a list of ticks at the requested resolution
1317  /// </summary>
1318  /// <param name="ticks">List of trade ticks</param>
1319  /// <param name="symbol">Symbol of all ticks</param>
1320  /// <param name="resolution">Desired resolution for new <see cref="TradeBar"/>s</param>
1321  /// <returns>List of aggregated <see cref="TradeBar"/>s</returns>
1322  public static IEnumerable<TradeBar> AggregateTicksToTradeBars(IEnumerable<Tick> ticks, Symbol symbol, TimeSpan resolution)
1323  {
1324  return Aggregate(new TickConsolidator(resolution), ticks, symbol);
1325  }
1326 
1327  /// <summary>
1328  /// Helper method to return the start time and period of a bar the given point time should be part of
1329  /// </summary>
1330  /// <param name="exchangeTimeZoneDate">The point in time we want to get the bar information about</param>
1331  /// <param name="exchange">The associated security exchange</param>
1332  /// <param name="extendedMarketHours">True if extended market hours should be taken into consideration</param>
1333  /// <returns>The calendar information that holds a start time and a period</returns>
1334  public static CalendarInfo GetDailyCalendar(DateTime exchangeTimeZoneDate, SecurityExchange exchange, bool extendedMarketHours)
1335  {
1336  return GetDailyCalendar(exchangeTimeZoneDate, exchange.Hours, extendedMarketHours);
1337  }
1338 
1339  /// <summary>
1340  /// Helper method to return the start time and period of a bar the given point time should be part of
1341  /// </summary>
1342  /// <param name="exchangeTimeZoneDate">The point in time we want to get the bar information about</param>
1343  /// <param name="exchangeHours">The associated exchange hours</param>
1344  /// <param name="extendedMarketHours">True if extended market hours should be taken into consideration</param>
1345  /// <returns>The calendar information that holds a start time and a period</returns>
1346  public static CalendarInfo GetDailyCalendar(DateTime exchangeTimeZoneDate, SecurityExchangeHours exchangeHours, bool extendedMarketHours)
1347  {
1348  var startTime = exchangeHours.GetPreviousMarketOpen(exchangeTimeZoneDate, extendedMarketHours);
1349  var endTime = exchangeHours.GetNextMarketClose(startTime, extendedMarketHours);
1350 
1351  // Let's not consider regular market gaps like when market closes at 16:15 and opens again at 16:30
1352  while (true)
1353  {
1354  var potentialEnd = exchangeHours.GetNextMarketClose(endTime, extendedMarketHours);
1355  if (potentialEnd.Date != endTime.Date)
1356  {
1357  break;
1358  }
1359  endTime = potentialEnd;
1360  }
1361 
1362  var period = endTime - startTime;
1363  return new CalendarInfo(startTime, period);
1364  }
1365 
1366  /// <summary>
1367  /// Helper method that defines the types of options that should use scale factor
1368  /// </summary>
1369  [MethodImpl(MethodImplOptions.AggressiveInlining)]
1370  public static bool OptionUseScaleFactor(Symbol symbol)
1371  {
1372  return symbol.SecurityType == SecurityType.Option || symbol.SecurityType == SecurityType.IndexOption;
1373  }
1374 
1375  /// <summary>
1376  /// Helper to separate filename and entry from a given key for DataProviders
1377  /// </summary>
1378  /// <param name="key">The key to parse</param>
1379  /// <param name="fileName">File name extracted</param>
1380  /// <param name="entryName">Entry name extracted</param>
1381  public static void ParseKey(string key, out string fileName, out string entryName)
1382  {
1383  // Default scenario, no entryName included in key
1384  entryName = null; // default to all entries
1385  fileName = key;
1386 
1387  if (key == null)
1388  {
1389  return;
1390  }
1391 
1392  // Try extracting an entry name; Anything after a # sign
1393  var hashIndex = key.LastIndexOf("#", StringComparison.Ordinal);
1394  if (hashIndex != -1)
1395  {
1396  entryName = key.Substring(hashIndex + 1);
1397  fileName = key.Substring(0, hashIndex);
1398  }
1399  }
1400 
1401  /// <summary>
1402  /// Helper method to aggregate ticks or bars into lower frequency resolutions
1403  /// </summary>
1404  /// <typeparam name="T">Output type</typeparam>
1405  /// <typeparam name="K">Input type</typeparam>
1406  /// <param name="consolidator">The consolidator to use</param>
1407  /// <param name="dataPoints">The data point source</param>
1408  /// <param name="symbol">The symbol to output</param>
1409  private static IEnumerable<T> Aggregate<T, K>(PeriodCountConsolidatorBase<K, T> consolidator, IEnumerable<K> dataPoints, Symbol symbol)
1410  where T : BaseData
1411  where K : BaseData
1412  {
1413  IBaseData lastAggregated = null;
1414  var getConsolidatedBar = () =>
1415  {
1416  if (lastAggregated != consolidator.Consolidated && consolidator.Consolidated != null)
1417  {
1418  // if there's a new aggregated bar we set the symbol & return it
1419  lastAggregated = consolidator.Consolidated;
1420  lastAggregated.Symbol = symbol;
1421  return lastAggregated;
1422  }
1423  return null;
1424  };
1425 
1426  foreach (var dataPoint in dataPoints)
1427  {
1428  consolidator.Update(dataPoint);
1429  var consolidated = getConsolidatedBar();
1430  if (consolidated != null)
1431  {
1432  yield return (T)consolidated;
1433  }
1434  }
1435 
1436  // flush any partial bar
1437  consolidator.Scan(Time.EndOfTime);
1438  var lastConsolidated = getConsolidatedBar();
1439  if (lastConsolidated != null)
1440  {
1441  yield return (T)lastConsolidated;
1442  }
1443 
1444  // cleanup
1445  consolidator.DisposeSafely();
1446  }
1447  }
1448 }