17 using System.Collections.Generic;
21 using System.Net.Http;
22 using Newtonsoft.Json;
23 using Newtonsoft.Json.Linq;
25 using RestSharp.Extensions;
35 using System.Threading;
36 using System.Net.Http.Headers;
37 using System.Collections.Concurrent;
39 using Newtonsoft.Json.Serialization;
48 private readonly BlockingCollection<Lazy<HttpClient>> _clientPool;
49 private string _dataFolder;
56 ContractResolver =
new DefaultContractResolver
58 NamingStrategy =
new CamelCaseNamingStrategy
60 ProcessDictionaryKeys =
false,
61 OverrideSpecifiedNames =
true
76 _clientPool =
new BlockingCollection<Lazy<HttpClient>>(
new ConcurrentQueue<Lazy<HttpClient>>(), 5);
77 for (
int i = 0; i < _clientPool.BoundedCapacity; i++)
79 _clientPool.Add(
new Lazy<HttpClient>());
86 public virtual void Initialize(
int userId,
string token,
string dataFolder)
89 _dataFolder = dataFolder?.Replace(
"\\",
"/", StringComparison.InvariantCulture);
92 JsonConvert.DefaultSettings = () =>
new JsonSerializerSettings
114 var request =
new RestRequest(
"projects/create", Method.POST)
116 RequestFormat = DataFormat.Json
121 if (
string.IsNullOrEmpty(organizationId))
123 jsonParams = JsonConvert.SerializeObject(
new
131 jsonParams = JsonConvert.SerializeObject(
new
139 request.AddParameter(
"application/json", jsonParams, ParameterType.RequestBody);
153 var request =
new RestRequest(
"projects/read", Method.POST)
155 RequestFormat = DataFormat.Json
158 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
161 }), ParameterType.RequestBody);
174 var request =
new RestRequest(
"projects/read", Method.POST)
176 RequestFormat = DataFormat.Json
194 var request =
new RestRequest(
"files/create", Method.POST)
196 RequestFormat = DataFormat.Json
199 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
204 }), ParameterType.RequestBody);
221 var request =
new RestRequest(
"files/update", Method.POST)
223 RequestFormat = DataFormat.Json
226 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
230 newName = newFileName
231 }), ParameterType.RequestBody);
248 var request =
new RestRequest(
"files/update", Method.POST)
250 RequestFormat = DataFormat.Json
253 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
257 content = newFileContents
258 }), ParameterType.RequestBody);
273 var request =
new RestRequest(
"files/read", Method.POST)
275 RequestFormat = DataFormat.Json
278 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
281 }), ParameterType.RequestBody);
294 var request =
new RestRequest(
"projects/nodes/read", Method.POST)
296 RequestFormat = DataFormat.Json
299 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
302 }), ParameterType.RequestBody);
317 var request =
new RestRequest(
"projects/nodes/update", Method.POST)
319 RequestFormat = DataFormat.Json
322 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
326 }), ParameterType.RequestBody);
341 var request =
new RestRequest(
"files/read", Method.POST)
343 RequestFormat = DataFormat.Json
346 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
350 }), ParameterType.RequestBody);
361 var request =
new RestRequest(
"lean/versions/read", Method.POST)
363 RequestFormat = DataFormat.Json
379 var request =
new RestRequest(
"files/delete", Method.POST)
381 RequestFormat = DataFormat.Json
384 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
388 }), ParameterType.RequestBody);
402 var request =
new RestRequest(
"projects/delete", Method.POST)
404 RequestFormat = DataFormat.Json
407 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
410 }), ParameterType.RequestBody);
424 var request =
new RestRequest(
"compile/create", Method.POST)
426 RequestFormat = DataFormat.Json
429 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
432 }), ParameterType.RequestBody);
447 var request =
new RestRequest(
"compile/read", Method.POST)
449 RequestFormat = DataFormat.Json
452 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
456 }), ParameterType.RequestBody);
470 throw new NotImplementedException($
"{nameof(Api)} does not support sending notifications");
483 var request =
new RestRequest(
"backtests/create", Method.POST)
485 RequestFormat = DataFormat.Json
488 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
493 }), ParameterType.RequestBody);
498 result.Backtest.Success = result.Success;
499 result.Backtest.Errors = result.Errors;
502 return result.Backtest;
515 var request =
new RestRequest(
"backtests/read", Method.POST)
517 RequestFormat = DataFormat.Json
520 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
524 }), ParameterType.RequestBody);
531 result.Backtest =
new Backtest { BacktestId = backtestId };
534 else if (getCharts && result.Backtest.Completed)
537 var updatedCharts =
new Dictionary<string, Chart>();
540 foreach (var chart
in result.Backtest.Charts)
542 if (!chart.Value.Series.IsNullOrEmpty())
547 var chartRequest =
new RestRequest(
"backtests/read", Method.POST)
549 RequestFormat = DataFormat.Json
552 chartRequest.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
557 }), ParameterType.RequestBody);
562 updatedCharts.Add(chart.Key, chartResponse.Backtest.Charts[chart.Key]);
567 foreach(var updatedChart
in updatedCharts)
569 result.Backtest.Charts[updatedChart.Key] = updatedChart.Value;
574 result.Backtest.Success = result.Success;
575 result.Backtest.Errors = result.Errors;
578 return result.Backtest;
591 public List<ApiOrderResponse>
ReadBacktestOrders(
int projectId,
string backtestId,
int start = 0,
int end = 100)
593 var request =
new RestRequest(
"backtests/orders/read", Method.POST)
595 RequestFormat = DataFormat.Json
598 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
604 }), ParameterType.RequestBody);
606 return MakeRequestOrThrow<OrdersResponseWrapper>(request, nameof(
ReadBacktestOrders)).Orders;
621 var request =
new RestRequest(
"backtests/chart/read", Method.POST)
623 RequestFormat = DataFormat.Json
626 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
634 }), ParameterType.RequestBody);
639 var finish = DateTime.UtcNow.AddMinutes(1);
640 while (DateTime.UtcNow < finish && result.Chart ==
null)
660 var request =
new RestRequest(
"backtests/update", Method.POST)
662 RequestFormat = DataFormat.Json
665 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
671 }), ParameterType.RequestBody);
686 var request =
new RestRequest(
"backtests/list", Method.POST)
688 RequestFormat = DataFormat.Json
691 var obj =
new Dictionary<string, object>()
693 {
"projectId", projectId },
694 {
"includeStatistics", includeStatistics }
697 request.AddParameter(
"application/json", JsonConvert.SerializeObject(obj), ParameterType.RequestBody);
712 var request =
new RestRequest(
"backtests/delete", Method.POST)
714 RequestFormat = DataFormat.Json
717 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
721 }), ParameterType.RequestBody);
736 var request =
new RestRequest(
"backtests/tags/update", Method.POST)
738 RequestFormat = DataFormat.Json
741 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
746 }), ParameterType.RequestBody);
776 Dictionary<string, object> brokerageSettings,
777 string versionId =
"-1",
778 Dictionary<string, object> dataProviders =
null)
780 var request =
new RestRequest(
"live/create", Method.POST)
782 RequestFormat = DataFormat.Json
785 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
794 ), ParameterType.RequestBody);
824 return CreateLiveAlgorithm(projectId, compileId, nodeId, ConvertToDictionary(brokerageSettings), versionId, dataProviders !=
null ? ConvertToDictionary(dataProviders) :
null);
831 private static Dictionary<string, object> ConvertToDictionary(PyObject brokerageSettings)
835 var stringBrokerageSettings = brokerageSettings.ToString();
836 return JsonConvert.DeserializeObject<Dictionary<string, object>>(stringBrokerageSettings);
849 DateTime? startTime =
null,
850 DateTime? endTime =
null)
853 if (status.HasValue &&
859 throw new ArgumentException(
860 "The Api only supports Algorithm Statuses of Running, Stopped, RuntimeError and Liquidated");
863 var request =
new RestRequest(
"live/list", Method.POST)
865 RequestFormat = DataFormat.Json
871 JObject obj =
new JObject
873 {
"start", epochStartTime },
874 {
"end", epochEndTime }
879 obj.Add(
"status", status.ToString());
882 request.AddParameter(
"application/json", JsonConvert.SerializeObject(obj), ParameterType.RequestBody);
897 var request =
new RestRequest(
"live/read", Method.POST)
899 RequestFormat = DataFormat.Json
902 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
906 }), ParameterType.RequestBody);
919 var request =
new RestRequest(
"live/portfolio/read", Method.POST)
921 RequestFormat = DataFormat.Json
924 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
927 }), ParameterType.RequestBody);
942 public List<ApiOrderResponse>
ReadLiveOrders(
int projectId,
int start = 0,
int end = 100)
944 var request =
new RestRequest(
"live/orders/read", Method.POST)
946 RequestFormat = DataFormat.Json
949 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
954 }), ParameterType.RequestBody);
956 return MakeRequestOrThrow<OrdersResponseWrapper>(request, nameof(
ReadLiveOrders)).Orders;
967 var request =
new RestRequest(
"live/update/liquidate", Method.POST)
969 RequestFormat = DataFormat.Json
972 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
975 }), ParameterType.RequestBody);
989 var request =
new RestRequest(
"live/update/stop", Method.POST)
991 RequestFormat = DataFormat.Json
994 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
997 }), ParameterType.RequestBody);
1013 var logLinesNumber = endLine - startLine;
1014 if (logLinesNumber > 250)
1016 throw new ArgumentException($
"The maximum number of log lines allowed is 250. But the number of log lines was {logLinesNumber}.");
1019 var request =
new RestRequest(
"live/logs/read", Method.POST)
1021 RequestFormat = DataFormat.Json
1024 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1031 }), ParameterType.RequestBody);
1048 var request =
new RestRequest(
"live/chart/read", Method.POST)
1050 RequestFormat = DataFormat.Json
1053 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1060 }), ParameterType.RequestBody);
1065 var finish = DateTime.UtcNow.AddMinutes(1);
1066 while(DateTime.UtcNow < finish && result.Chart ==
null)
1084 var request =
new RestRequest(
"live/insights/read", Method.POST)
1086 RequestFormat = DataFormat.Json,
1089 var diff = end - start;
1092 throw new ArgumentException($
"The difference between the start and end index of the insights must be smaller than 100, but it was {diff}.");
1099 JObject obj =
new JObject
1101 {
"projectId", projectId },
1106 request.AddParameter(
"application/json", JsonConvert.SerializeObject(obj), ParameterType.RequestBody);
1120 if (filePath ==
null)
1122 throw new ArgumentException(
"Api.ReadDataLink(): Filepath must not be null");
1128 var request =
new RestRequest(
"data/read", Method.POST)
1130 RequestFormat = DataFormat.Json
1133 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1138 }), ParameterType.RequestBody);
1150 if (filePath ==
null)
1152 throw new ArgumentException(
"Api.ReadDataDirectory(): Filepath must not be null");
1160 if (filePath.Count(x => x ==
'/') < 3)
1162 throw new ArgumentException($
"Api.ReadDataDirectory(): Data directory requested must be at least" +
1163 $
" three directories deep. FilePath: {filePath}");
1166 var request =
new RestRequest(
"data/list", Method.POST)
1168 RequestFormat = DataFormat.Json
1171 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1174 }), ParameterType.RequestBody);
1185 var request =
new RestRequest(
"data/prices", Method.POST)
1187 RequestFormat = DataFormat.Json
1190 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1193 }), ParameterType.RequestBody);
1207 var request =
new RestRequest(
"backtests/read/report", Method.POST)
1209 RequestFormat = DataFormat.Json
1212 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1216 }), ParameterType.RequestBody);
1219 var finish = DateTime.UtcNow.AddMinutes(1);
1220 while (DateTime.UtcNow < finish && !report.
Success)
1222 Thread.Sleep(10000);
1241 if (!dataLink.Success)
1243 Log.
Trace($
"Api.DownloadData(): Failed to get link for {filePath}. " +
1244 $
"Errors: {string.Join(',', dataLink.Errors)}");
1249 var directory = Path.GetDirectoryName(filePath);
1250 if (!Directory.Exists(directory))
1252 Directory.CreateDirectory(directory);
1255 var client = BorrowClient();
1259 var uri =
new Uri(dataLink.Link);
1260 using var dataStream = client.Value.GetStreamAsync(uri);
1263 dataStream.Result.CopyTo(fileStream);
1267 Log.
Error($
"Api.DownloadData(): Failed to download zip for path ({filePath})");
1272 ReturnClient(client);
1288 ChartSubscription =
"*"
1319 public virtual void SendStatistics(
string algorithmId, decimal unrealized, decimal fees, decimal netProfit, decimal holdings, decimal equity, decimal netReturn, decimal volume,
int trades,
double sharpe)
1331 public virtual void SendUserEmail(
string algorithmId,
string subject,
string body)
1344 public virtual string Download(
string address, IEnumerable<KeyValuePair<string, string>> headers,
string userName,
string password)
1346 return Encoding.UTF8.GetString(
DownloadBytes(address, headers, userName, password));
1358 public virtual byte[]
DownloadBytes(
string address, IEnumerable<KeyValuePair<string, string>> headers,
string userName,
string password)
1360 var client = BorrowClient();
1363 client.Value.DefaultRequestHeaders.Clear();
1366 client.Value.DefaultRequestHeaders.TryAddWithoutValidation(
"user-agent",
"QCAlgorithm.Download(): User Agent Header");
1368 if (headers !=
null)
1370 foreach (var header
in headers)
1372 client.Value.DefaultRequestHeaders.Add(header.Key, header.Value);
1376 if (!userName.IsNullOrEmpty() || !password.IsNullOrEmpty())
1378 var credentials = Convert.ToBase64String(Encoding.ASCII.GetBytes($
"{userName}:{password}"));
1379 client.Value.DefaultRequestHeaders.Authorization =
new AuthenticationHeaderValue(
"Basic", credentials);
1382 return client.Value.GetByteArrayAsync(
new Uri(address)).Result;
1384 catch (Exception exception)
1386 var message = $
"Api.DownloadBytes(): Failed to download data from {address}";
1387 if (!userName.IsNullOrEmpty() || !password.IsNullOrEmpty())
1389 message += $
" with username: {userName} and password {password}";
1392 throw new WebException($
"{message}. Please verify the source for missing http:// or https://", exception);
1396 client.Value.DefaultRequestHeaders.Clear();
1397 ReturnClient(client);
1408 _clientPool.CompleteAdding();
1409 foreach (var client
in _clientPool.GetConsumingEnumerable())
1411 if (client.IsValueCreated)
1413 client.Value.DisposeSafely();
1416 _clientPool.DisposeSafely();
1427 var data = $
"{token}:{timestamp.ToStringInvariant()}";
1428 return data.ToSHA256();
1437 var request =
new RestRequest(
"account/read", Method.POST)
1439 RequestFormat = DataFormat.Json
1442 if (organizationId !=
null)
1444 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new { organizationId }), ParameterType.RequestBody);
1458 var request =
new RestRequest(
"organizations/read", Method.POST)
1460 RequestFormat = DataFormat.Json
1463 if (organizationId !=
null)
1465 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new { organizationId }), ParameterType.RequestBody);
1469 return response.Organization;
1490 decimal? targetValue,
1493 HashSet<OptimizationParameter> parameters,
1494 IReadOnlyList<Constraint> constraints)
1496 var request =
new RestRequest(
"optimizations/estimate", Method.POST)
1498 RequestFormat = DataFormat.Json
1501 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1515 return response.Estimate;
1539 decimal? targetValue,
1542 HashSet<OptimizationParameter> parameters,
1543 IReadOnlyList<Constraint> constraints,
1544 decimal estimatedCost,
1548 var request =
new RestRequest(
"optimizations/create", Method.POST)
1550 RequestFormat = DataFormat.Json
1553 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1570 return result.Optimizations.FirstOrDefault();
1580 var request =
new RestRequest(
"optimizations/list", Method.POST)
1582 RequestFormat = DataFormat.Json
1585 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1588 }), ParameterType.RequestBody);
1591 return result.Optimizations;
1601 var request =
new RestRequest(
"optimizations/read", Method.POST)
1603 RequestFormat = DataFormat.Json
1606 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1609 }), ParameterType.RequestBody);
1612 return response.Optimization;
1622 var request =
new RestRequest(
"optimizations/abort", Method.POST)
1624 RequestFormat = DataFormat.Json
1627 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1630 }), ParameterType.RequestBody);
1644 var request =
new RestRequest(
"optimizations/update", Method.POST)
1646 RequestFormat = DataFormat.Json
1649 var obj =
new JObject
1651 {
"optimizationId", optimizationId }
1654 if (name.HasValue())
1656 obj.Add(
"name", name);
1659 request.AddParameter(
"application/json", JsonConvert.SerializeObject(obj), ParameterType.RequestBody);
1672 var request =
new RestRequest(
"optimizations/delete", Method.POST)
1674 RequestFormat = DataFormat.Json
1677 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1680 }), ParameterType.RequestBody);
1693 public bool GetObjectStore(
string organizationId, List<string> keys,
string destinationFolder =
null)
1695 var request =
new RestRequest(
"object/get", Method.POST)
1697 RequestFormat = DataFormat.Json
1700 request.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1704 }), ParameterType.RequestBody);
1708 if (result ==
null || !result.Success)
1710 Log.
Error($
"Api.GetObjectStore(): Failed to get the jobId to request the download URL for the object store files."
1711 + (result !=
null ? $
" Errors: {string.Join(",
", result.Errors)}" :
""));
1715 var jobId = result.JobId;
1716 var getUrlRequest =
new RestRequest(
"object/get", Method.POST)
1718 RequestFormat = DataFormat.Json
1720 getUrlRequest.AddParameter(
"application/json", JsonConvert.SerializeObject(
new
1724 }), ParameterType.RequestBody);
1726 var frontier = DateTime.UtcNow + TimeSpan.FromMinutes(5);
1727 while (
string.IsNullOrEmpty(result?.Url) && (DateTime.UtcNow < frontier))
1733 if (result ==
null ||
string.IsNullOrEmpty(result.Url))
1735 Log.
Error($
"Api.GetObjectStore(): Failed to get the download URL from the jobId {jobId}."
1736 + (result !=
null ? $
" Errors: {string.Join(",
", result.Errors)}" :
""));
1740 var directory = destinationFolder ?? Directory.GetCurrentDirectory();
1741 var client = BorrowClient();
1745 if (client.Value.Timeout != TimeSpan.FromMinutes(20))
1747 client.Value.Timeout = TimeSpan.FromMinutes(20);
1751 var uri =
new Uri(result.Url);
1752 using var byteArray = client.Value.GetByteArrayAsync(uri);
1758 Log.
Error($
"Api.GetObjectStore(): Failed to download zip for path ({directory}). Error: {e.Message}");
1763 ReturnClient(client);
1778 var request =
new RestRequest(
"object/properties", Method.POST)
1780 RequestFormat = DataFormat.Json
1783 request.AddParameter(
"organizationId", organizationId);
1784 request.AddParameter(
"key", key);
1788 if (result ==
null || !result.Success)
1790 Log.
Error($
"Api.ObjectStore(): Failed to get the properties for the object store key {key}." + (result !=
null ? $
" Errors: {string.Join(",
", result.Errors)}" :
""));
1804 var request =
new RestRequest(
"object/set", Method.POST)
1806 RequestFormat = DataFormat.Json
1809 request.AddParameter(
"organizationId", organizationId);
1810 request.AddParameter(
"key", key);
1811 request.AddFileBytes(
"objectData", objectData,
"objectData");
1812 request.AlwaysMultipartFormData =
true;
1826 var request =
new RestRequest(
"object/delete", Method.POST)
1828 RequestFormat = DataFormat.Json
1831 var obj =
new Dictionary<string, object>
1833 {
"organizationId", organizationId },
1837 request.AddParameter(
"application/json", JsonConvert.SerializeObject(obj), ParameterType.RequestBody);
1851 var request =
new RestRequest(
"object/list", Method.POST)
1853 RequestFormat = DataFormat.Json
1856 var obj =
new Dictionary<string, object>
1858 {
"organizationId", organizationId },
1862 request.AddParameter(
"application/json", JsonConvert.SerializeObject(obj), ParameterType.RequestBody);
1876 if (filePath ==
null)
1878 Log.
Error(
"Api.FormatPathForDataRequest(): Cannot format null string");
1884 dataFolder = dataFolder.Replace(
"\\",
"/", StringComparison.InvariantCulture);
1885 filePath = filePath.Replace(
"\\",
"/", StringComparison.InvariantCulture);
1888 if (filePath.StartsWith(dataFolder, StringComparison.InvariantCulture))
1890 filePath = filePath.Substring(dataFolder.Length);
1894 filePath = filePath.TrimStart(
'/');
1901 private T MakeRequestOrThrow<T>(RestRequest request,
string callerName)
1906 var errors =
string.Empty;
1907 if (result !=
null && result.Errors !=
null && result.Errors.Count > 0)
1909 errors = $
". Errors: ['{string.Join(",
", result.Errors)}']";
1911 throw new WebException($
"{callerName} api request failed{errors}");
1920 private Lazy<HttpClient> BorrowClient()
1922 using var cancellationTokenSource =
new CancellationTokenSource(TimeSpan.FromMinutes(10));
1923 return _clientPool.Take(cancellationTokenSource.Token);
1929 private void ReturnClient(Lazy<HttpClient> client)
1931 _clientPool.Add(client);