PageRenderTime 26ms CodeModel.GetById 22ms RepoModel.GetById 0ms app.codeStats 0ms

/Plugins/BuildServerIntegration/AppVeyorIntegration/AppVeyorAdapter.cs

http://github.com/spdr870/gitextensions
C# | 542 lines | 458 code | 74 blank | 10 comment | 55 complexity | 843ab92a8e1061adcff27f40f56e5926 MD5 | raw file
Possible License(s): GPL-3.0
  1. using System;
  2. using System.Collections.Generic;
  3. using System.ComponentModel.Composition;
  4. using System.IO;
  5. using System.Linq;
  6. using System.Net.Http;
  7. using System.Net.Http.Headers;
  8. using System.Reactive.Concurrency;
  9. using System.Reactive.Linq;
  10. using System.Threading;
  11. using System.Threading.Tasks;
  12. using GitCommands.Utils;
  13. using GitUI;
  14. using GitUIPluginInterfaces;
  15. using GitUIPluginInterfaces.BuildServerIntegration;
  16. using JetBrains.Annotations;
  17. using Newtonsoft.Json.Linq;
  18. namespace AppVeyorIntegration
  19. {
  20. [MetadataAttribute]
  21. [AttributeUsage(AttributeTargets.Class)]
  22. public class AppVeyorIntegrationMetadata : BuildServerAdapterMetadataAttribute
  23. {
  24. public AppVeyorIntegrationMetadata(string buildServerType)
  25. : base(buildServerType)
  26. {
  27. }
  28. public override string CanBeLoaded
  29. {
  30. get
  31. {
  32. if (EnvUtils.IsNet4FullOrHigher())
  33. {
  34. return null;
  35. }
  36. return ".Net 4 full framework required";
  37. }
  38. }
  39. }
  40. [Export(typeof(IBuildServerAdapter))]
  41. [AppVeyorIntegrationMetadata(PluginName)]
  42. [PartCreationPolicy(CreationPolicy.NonShared)]
  43. internal class AppVeyorAdapter : IBuildServerAdapter
  44. {
  45. public const string PluginName = "AppVeyor";
  46. private const uint ProjectsToRetrieveCount = 25;
  47. private const string WebSiteUrl = "https://ci.appveyor.com";
  48. private const string ApiBaseUrl = WebSiteUrl + "/api/projects/";
  49. private IBuildServerWatcher _buildServerWatcher;
  50. private HttpClient _httpClientAppVeyor;
  51. private List<AppVeyorBuildInfo> _allBuilds = new List<AppVeyorBuildInfo>();
  52. private HashSet<ObjectId> _fetchBuilds;
  53. private string _accountToken;
  54. private static readonly Dictionary<string, Project> Projects = new Dictionary<string, Project>();
  55. private Func<ObjectId, bool> _isCommitInRevisionGrid;
  56. private bool _shouldLoadTestResults;
  57. public void Initialize(
  58. IBuildServerWatcher buildServerWatcher,
  59. ISettingsSource config,
  60. Func<ObjectId, bool> isCommitInRevisionGrid = null)
  61. {
  62. if (_buildServerWatcher != null)
  63. {
  64. throw new InvalidOperationException("Already initialized");
  65. }
  66. _buildServerWatcher = buildServerWatcher;
  67. _isCommitInRevisionGrid = isCommitInRevisionGrid;
  68. var accountName = config.GetString("AppVeyorAccountName", null);
  69. _accountToken = config.GetString("AppVeyorAccountToken", null);
  70. var projectNamesSetting = config.GetString("AppVeyorProjectName", null);
  71. if (accountName.IsNullOrWhiteSpace() && projectNamesSetting.IsNullOrWhiteSpace())
  72. {
  73. return;
  74. }
  75. _shouldLoadTestResults = config.GetBool("AppVeyorLoadTestsResults", false);
  76. _fetchBuilds = new HashSet<ObjectId>();
  77. _httpClientAppVeyor = GetHttpClient(WebSiteUrl, _accountToken);
  78. var useAllProjects = string.IsNullOrWhiteSpace(projectNamesSetting);
  79. string[] projectNames = null;
  80. if (!useAllProjects)
  81. {
  82. projectNames = _buildServerWatcher.ReplaceVariables(projectNamesSetting)
  83. .Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
  84. }
  85. if (Projects.Count == 0 ||
  86. (!useAllProjects && Projects.Keys.Intersect(projectNames).Count() != projectNames.Length))
  87. {
  88. Projects.Clear();
  89. if (_accountToken.IsNullOrWhiteSpace())
  90. {
  91. FillProjectsFromSettings(accountName, projectNames);
  92. }
  93. else
  94. {
  95. if (accountName.IsNullOrWhiteSpace())
  96. {
  97. return;
  98. }
  99. ThreadHelper.JoinableTaskFactory.Run(
  100. async () =>
  101. {
  102. var result = await GetResponseAsync(_httpClientAppVeyor, ApiBaseUrl, CancellationToken.None).ConfigureAwait(false);
  103. if (result.IsNullOrWhiteSpace())
  104. {
  105. return;
  106. }
  107. var projects = JArray.Parse(result);
  108. foreach (var project in projects)
  109. {
  110. var projectId = project["slug"].ToString();
  111. projectId = accountName.Combine("/", projectId);
  112. var projectName = project["name"].ToString();
  113. var projectObj = new Project
  114. {
  115. Name = projectName,
  116. Id = projectId,
  117. QueryUrl = BuildQueryUrl(projectId)
  118. };
  119. if (useAllProjects || projectNames.Contains(projectObj.Name))
  120. {
  121. Projects.Add(projectObj.Name, projectObj);
  122. }
  123. }
  124. });
  125. }
  126. }
  127. var builds = Projects.Where(p => useAllProjects || projectNames.Contains(p.Value.Name)).Select(p => p.Value);
  128. _allBuilds =
  129. FilterBuilds(builds.SelectMany(project => QueryBuildsResults(project)));
  130. }
  131. private static void FillProjectsFromSettings(string accountName, [InstantHandle] IEnumerable<string> projectNames)
  132. {
  133. foreach (var projectName in projectNames)
  134. {
  135. var projectId = accountName.Combine("/", projectName);
  136. Projects.Add(projectName, new Project
  137. {
  138. Name = projectName,
  139. Id = projectId,
  140. QueryUrl = BuildQueryUrl(projectId)
  141. });
  142. }
  143. }
  144. private static HttpClient GetHttpClient(string baseUrl, string accountToken)
  145. {
  146. var httpClient = new HttpClient(new HttpClientHandler { UseDefaultCredentials = true })
  147. {
  148. Timeout = TimeSpan.FromMinutes(2),
  149. BaseAddress = new Uri(baseUrl, UriKind.Absolute),
  150. };
  151. if (accountToken.IsNotNullOrWhitespace())
  152. {
  153. httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", accountToken);
  154. }
  155. return httpClient;
  156. }
  157. private static string BuildQueryUrl(string projectId)
  158. {
  159. return ApiBaseUrl + projectId + "/history?recordsNumber=" + ProjectsToRetrieveCount;
  160. }
  161. private class Project
  162. {
  163. public string Name;
  164. public string Id;
  165. public string QueryUrl;
  166. }
  167. private IEnumerable<AppVeyorBuildInfo> QueryBuildsResults(Project project)
  168. {
  169. try
  170. {
  171. return ThreadHelper.JoinableTaskFactory.Run(
  172. async () =>
  173. {
  174. var result = await GetResponseAsync(_httpClientAppVeyor, project.QueryUrl, CancellationToken.None).ConfigureAwait(false);
  175. if (string.IsNullOrWhiteSpace(result))
  176. {
  177. return Enumerable.Empty<AppVeyorBuildInfo>();
  178. }
  179. var builds = JObject.Parse(result)["builds"].Children();
  180. var baseApiUrl = ApiBaseUrl + project.Id;
  181. var baseWebUrl = WebSiteUrl + "/project/" + project.Id + "/build/";
  182. var buildDetails = new List<AppVeyorBuildInfo>();
  183. foreach (var b in builds)
  184. {
  185. try
  186. {
  187. if (!ObjectId.TryParse((b["pullRequestHeadCommitId"] ?? b["commitId"]).ToObject<string>(), out var objectId) || !_isCommitInRevisionGrid(objectId))
  188. {
  189. continue;
  190. }
  191. var pullRequestId = b["pullRequestId"];
  192. var version = b["version"].ToObject<string>();
  193. var status = ParseBuildStatus(b["status"].ToObject<string>());
  194. long? duration = null;
  195. if (status == BuildInfo.BuildStatus.Success || status == BuildInfo.BuildStatus.Failure)
  196. {
  197. duration = GetBuildDuration(b);
  198. }
  199. buildDetails.Add(new AppVeyorBuildInfo
  200. {
  201. Id = version,
  202. BuildId = b["buildId"].ToObject<string>(),
  203. Branch = b["branch"].ToObject<string>(),
  204. CommitId = objectId,
  205. CommitHashList = new[] { objectId },
  206. Status = status,
  207. StartDate = b["started"]?.ToObject<DateTime>() ?? DateTime.MinValue,
  208. BaseWebUrl = baseWebUrl,
  209. Url = WebSiteUrl + "/project/" + project.Id + "/build/" + version,
  210. BaseApiUrl = baseApiUrl,
  211. AppVeyorBuildReportUrl = baseApiUrl + "/build/" + version,
  212. PullRequestText = pullRequestId != null ? " PR#" + pullRequestId.Value<string>() : string.Empty,
  213. Duration = duration,
  214. TestsResultText = string.Empty
  215. });
  216. }
  217. catch (Exception)
  218. {
  219. // Failure on reading data on a build detail should not prevent to display the others build results
  220. }
  221. }
  222. return buildDetails;
  223. });
  224. }
  225. catch
  226. {
  227. return Enumerable.Empty<AppVeyorBuildInfo>();
  228. }
  229. }
  230. /// <summary>
  231. /// Gets a unique key which identifies this build server.
  232. /// </summary>
  233. public string UniqueKey => _httpClientAppVeyor.BaseAddress.Host;
  234. public IObservable<BuildInfo> GetFinishedBuildsSince(IScheduler scheduler, DateTime? sinceDate = null)
  235. {
  236. // AppVeyor api is different than TeamCity one and all build results are fetch in one call without
  237. // filter parameters possible (so this call is useless!)
  238. return Observable.Empty<BuildInfo>();
  239. }
  240. public IObservable<BuildInfo> GetRunningBuilds(IScheduler scheduler)
  241. {
  242. return GetBuilds(scheduler);
  243. }
  244. private IObservable<BuildInfo> GetBuilds(IScheduler scheduler)
  245. {
  246. return Observable.Create<BuildInfo>((observer, cancellationToken) =>
  247. Task.Run(
  248. () => scheduler.Schedule(() => ObserveBuilds(observer, cancellationToken))));
  249. }
  250. private void ObserveBuilds(IObserver<BuildInfo> observer, CancellationToken cancellationToken)
  251. {
  252. try
  253. {
  254. if (_allBuilds == null)
  255. {
  256. return;
  257. }
  258. // Display all builds found
  259. foreach (var build in _allBuilds)
  260. {
  261. UpdateDisplay(observer, build);
  262. }
  263. // Update finished build with tests results
  264. if (_shouldLoadTestResults)
  265. {
  266. foreach (var build in _allBuilds.Where(b => b.Status == BuildInfo.BuildStatus.Success
  267. || b.Status == BuildInfo.BuildStatus.Failure))
  268. {
  269. UpdateDescription(build, cancellationToken);
  270. UpdateDisplay(observer, build);
  271. }
  272. }
  273. // Manage in progress builds...
  274. var inProgressBuilds = _allBuilds.Where(b => b.Status == BuildInfo.BuildStatus.InProgress).ToList();
  275. _allBuilds = null;
  276. do
  277. {
  278. Thread.Sleep(5000);
  279. foreach (var build in inProgressBuilds)
  280. {
  281. UpdateDescription(build, cancellationToken);
  282. UpdateDisplay(observer, build);
  283. }
  284. inProgressBuilds = inProgressBuilds.Where(b => b.Status == BuildInfo.BuildStatus.InProgress).ToList();
  285. }
  286. while (inProgressBuilds.Any());
  287. observer.OnCompleted();
  288. }
  289. catch (OperationCanceledException)
  290. {
  291. // Do nothing, the observer is already stopped
  292. }
  293. catch (Exception ex)
  294. {
  295. observer.OnError(ex);
  296. }
  297. }
  298. private static void UpdateDisplay(IObserver<BuildInfo> observer, AppVeyorBuildInfo build)
  299. {
  300. build.UpdateDescription();
  301. observer.OnNext(build);
  302. }
  303. private List<AppVeyorBuildInfo> FilterBuilds(IEnumerable<AppVeyorBuildInfo> allBuilds)
  304. {
  305. var filteredBuilds = new List<AppVeyorBuildInfo>();
  306. foreach (var build in allBuilds.OrderByDescending(b => b.StartDate))
  307. {
  308. if (!_fetchBuilds.Contains(build.CommitId))
  309. {
  310. filteredBuilds.Add(build);
  311. _fetchBuilds.Add(build.CommitId);
  312. }
  313. }
  314. return filteredBuilds;
  315. }
  316. private void UpdateDescription(AppVeyorBuildInfo buildDetails, CancellationToken cancellationToken)
  317. {
  318. var buildDetailsParsed = ThreadHelper.JoinableTaskFactory.Run(() => FetchBuildDetailsManagingVersionUpdateAsync(buildDetails, cancellationToken));
  319. if (buildDetailsParsed == null)
  320. {
  321. return;
  322. }
  323. var buildData = buildDetailsParsed["build"];
  324. var buildDescription = buildData["jobs"].Last();
  325. var status = buildDescription["status"].ToObject<string>();
  326. buildDetails.Status = ParseBuildStatus(status);
  327. buildDetails.ChangeProgressCounter();
  328. if (!buildDetails.IsRunning)
  329. {
  330. buildDetails.Duration = GetBuildDuration(buildData);
  331. }
  332. int testCount = buildDescription["testsCount"].ToObject<int>();
  333. if (testCount != 0)
  334. {
  335. int failedTestCount = buildDescription["failedTestsCount"].ToObject<int>();
  336. int skippedTestCount = testCount - buildDescription["passedTestsCount"].ToObject<int>();
  337. var testResults = " : " + testCount + " tests";
  338. if (failedTestCount != 0 || skippedTestCount != 0)
  339. {
  340. testResults += string.Format(" ( {0} failed, {1} skipped )", failedTestCount, skippedTestCount);
  341. }
  342. buildDetails.TestsResultText = " " + testResults;
  343. }
  344. }
  345. private static long GetBuildDuration(JToken buildData)
  346. {
  347. var startTime = buildData["started"].ToObject<DateTime>();
  348. var updateTime = buildData["updated"].ToObject<DateTime>();
  349. return (long)(updateTime - startTime).TotalMilliseconds;
  350. }
  351. private async Task<JObject> FetchBuildDetailsManagingVersionUpdateAsync(AppVeyorBuildInfo buildDetails, CancellationToken cancellationToken)
  352. {
  353. try
  354. {
  355. return JObject.Parse(await GetResponseAsync(_httpClientAppVeyor, buildDetails.AppVeyorBuildReportUrl, cancellationToken).ConfigureAwait(false));
  356. }
  357. catch
  358. {
  359. var buildHistoryUrl = buildDetails.BaseApiUrl + "/history?recordsNumber=1&startBuildId=" + (int.Parse(buildDetails.BuildId) + 1);
  360. var builds = JObject.Parse(await GetResponseAsync(_httpClientAppVeyor, buildHistoryUrl, cancellationToken).ConfigureAwait(false));
  361. var version = builds["builds"][0]["version"].ToObject<string>();
  362. buildDetails.Id = version;
  363. buildDetails.AppVeyorBuildReportUrl = buildDetails.BaseApiUrl + "/build/" + version;
  364. buildDetails.Url = buildDetails.BaseWebUrl + version;
  365. return JObject.Parse(await GetResponseAsync(_httpClientAppVeyor, buildDetails.AppVeyorBuildReportUrl, cancellationToken).ConfigureAwait(false));
  366. }
  367. }
  368. private static BuildInfo.BuildStatus ParseBuildStatus(string statusValue)
  369. {
  370. switch (statusValue)
  371. {
  372. case "success":
  373. return BuildInfo.BuildStatus.Success;
  374. case "failed":
  375. return BuildInfo.BuildStatus.Failure;
  376. case "cancelled":
  377. return BuildInfo.BuildStatus.Stopped;
  378. case "queued":
  379. case "running":
  380. return BuildInfo.BuildStatus.InProgress;
  381. default:
  382. return BuildInfo.BuildStatus.Unknown;
  383. }
  384. }
  385. private Task<Stream> GetStreamAsync(HttpClient httpClient, string restServicePath, CancellationToken cancellationToken)
  386. {
  387. cancellationToken.ThrowIfCancellationRequested();
  388. return httpClient.GetAsync(restServicePath, HttpCompletionOption.ResponseHeadersRead, cancellationToken)
  389. .ContinueWith(
  390. task => GetStreamFromHttpResponseAsync(httpClient, task, restServicePath, cancellationToken),
  391. cancellationToken,
  392. restServicePath.Contains("github") ? TaskContinuationOptions.None : TaskContinuationOptions.AttachedToParent,
  393. TaskScheduler.Current)
  394. .Unwrap();
  395. }
  396. private Task<Stream> GetStreamFromHttpResponseAsync(HttpClient httpClient, Task<HttpResponseMessage> task, string restServicePath, CancellationToken cancellationToken)
  397. {
  398. var retry = task.IsCanceled && !cancellationToken.IsCancellationRequested;
  399. if (retry)
  400. {
  401. return GetStreamAsync(httpClient, restServicePath, cancellationToken);
  402. }
  403. if (task.Status == TaskStatus.RanToCompletion && task.CompletedResult().IsSuccessStatusCode)
  404. {
  405. return task.CompletedResult().Content.ReadAsStreamAsync();
  406. }
  407. return null;
  408. }
  409. private Task<string> GetResponseAsync(HttpClient httpClient, string relativePath, CancellationToken cancellationToken)
  410. {
  411. var getStreamTask = GetStreamAsync(httpClient, relativePath, cancellationToken);
  412. var taskContinuationOptions = relativePath.Contains("github") ? TaskContinuationOptions.None : TaskContinuationOptions.AttachedToParent;
  413. return getStreamTask.ContinueWith(
  414. task =>
  415. {
  416. if (task.Status != TaskStatus.RanToCompletion)
  417. {
  418. return string.Empty;
  419. }
  420. using (var responseStream = task.Result)
  421. {
  422. return new StreamReader(responseStream).ReadToEnd();
  423. }
  424. },
  425. cancellationToken,
  426. taskContinuationOptions,
  427. TaskScheduler.Current);
  428. }
  429. public void Dispose()
  430. {
  431. GC.SuppressFinalize(this);
  432. _httpClientAppVeyor?.Dispose();
  433. }
  434. }
  435. internal sealed class AppVeyorBuildInfo : BuildInfo
  436. {
  437. private static readonly IBuildDurationFormatter _buildDurationFormatter = new BuildDurationFormatter();
  438. private int _buildProgressCount;
  439. public string BuildId { get; set; }
  440. public ObjectId CommitId { get; set; }
  441. public string AppVeyorBuildReportUrl { get; set; }
  442. public string Branch { get; set; }
  443. public string BaseApiUrl { get; set; }
  444. public string BaseWebUrl { get; set; }
  445. public string PullRequestText { get; set; }
  446. public string TestsResultText { get; set; }
  447. public bool IsRunning => Status == BuildStatus.InProgress;
  448. public void ChangeProgressCounter()
  449. {
  450. _buildProgressCount = (_buildProgressCount % 3) + 1;
  451. }
  452. public void UpdateDescription()
  453. {
  454. Description = Id + " " + DisplayStatus + " " + _buildDurationFormatter.Format(Duration) + TestsResultText + PullRequestText;
  455. }
  456. private string DisplayStatus
  457. {
  458. get
  459. {
  460. if (Status != BuildStatus.InProgress)
  461. {
  462. return Status.ToString("G");
  463. }
  464. return "In progress" + new string('.', _buildProgressCount) + new string(' ', 3 - _buildProgressCount);
  465. }
  466. }
  467. }
  468. }