PageRenderTime 55ms CodeModel.GetById 27ms RepoModel.GetById 0ms app.codeStats 0ms

/Plugins/BuildServerIntegration/JenkinsIntegration/JenkinsAdapter.cs

http://github.com/spdr870/gitextensions
C# | 554 lines | 450 code | 71 blank | 33 comment | 71 complexity | f77062f9412b8f4de8714773f2007eab MD5 | raw file
Possible License(s): GPL-3.0
  1. using System;
  2. using System.Collections.Generic;
  3. using System.ComponentModel.Composition;
  4. using System.Diagnostics;
  5. using System.IO;
  6. using System.Linq;
  7. using System.Net;
  8. using System.Net.Http;
  9. using System.Net.Http.Headers;
  10. using System.Reactive.Concurrency;
  11. using System.Reactive.Linq;
  12. using System.Text;
  13. using System.Text.RegularExpressions;
  14. using System.Threading;
  15. using System.Threading.Tasks;
  16. using GitCommands.Utils;
  17. using GitUI;
  18. using GitUIPluginInterfaces;
  19. using GitUIPluginInterfaces.BuildServerIntegration;
  20. using JetBrains.Annotations;
  21. using Microsoft.VisualStudio.Threading;
  22. using Newtonsoft.Json.Linq;
  23. namespace JenkinsIntegration
  24. {
  25. [MetadataAttribute]
  26. [AttributeUsage(AttributeTargets.Class)]
  27. public class JenkinsIntegrationMetadata : BuildServerAdapterMetadataAttribute
  28. {
  29. public JenkinsIntegrationMetadata(string buildServerType)
  30. : base(buildServerType)
  31. {
  32. }
  33. public override string CanBeLoaded
  34. {
  35. get
  36. {
  37. if (EnvUtils.IsNet4FullOrHigher())
  38. {
  39. return null;
  40. }
  41. return ".Net 4 full framework required";
  42. }
  43. }
  44. }
  45. [Export(typeof(IBuildServerAdapter))]
  46. [JenkinsIntegrationMetadata(PluginName)]
  47. [PartCreationPolicy(CreationPolicy.NonShared)]
  48. internal class JenkinsAdapter : IBuildServerAdapter
  49. {
  50. public const string PluginName = "Jenkins";
  51. private static readonly IBuildDurationFormatter _buildDurationFormatter = new BuildDurationFormatter();
  52. private IBuildServerWatcher _buildServerWatcher;
  53. private HttpClient _httpClient;
  54. private readonly Dictionary<string, JenkinsCacheInfo> _lastBuildCache = new Dictionary<string, JenkinsCacheInfo>();
  55. private readonly List<string> _projectsUrls = new List<string>();
  56. private Regex _ignoreBuilds;
  57. public void Initialize(IBuildServerWatcher buildServerWatcher, ISettingsSource config, Func<ObjectId, bool> isCommitInRevisionGrid = null)
  58. {
  59. if (_buildServerWatcher != null)
  60. {
  61. throw new InvalidOperationException("Already initialized");
  62. }
  63. _buildServerWatcher = buildServerWatcher;
  64. var projectName = config.GetString("ProjectName", null);
  65. var hostName = config.GetString("BuildServerUrl", null);
  66. if (!string.IsNullOrEmpty(hostName) && !string.IsNullOrEmpty(projectName))
  67. {
  68. var baseAddress = hostName.Contains("://")
  69. ? new Uri(hostName, UriKind.Absolute)
  70. : new Uri($"{Uri.UriSchemeHttp}://{hostName}:8080", UriKind.Absolute);
  71. _httpClient = new HttpClient(new HttpClientHandler { UseDefaultCredentials = true })
  72. {
  73. Timeout = TimeSpan.FromMinutes(2),
  74. BaseAddress = baseAddress
  75. };
  76. var buildServerCredentials = buildServerWatcher.GetBuildServerCredentials(this, true);
  77. UpdateHttpClientOptions(buildServerCredentials);
  78. string[] projectUrls = _buildServerWatcher.ReplaceVariables(projectName)
  79. .Split(new[] { '|' }, StringSplitOptions.RemoveEmptyEntries);
  80. foreach (var projectUrl in projectUrls.Select(s => baseAddress + "job/" + s.Trim() + "/"))
  81. {
  82. AddGetBuildUrl(projectUrl);
  83. }
  84. }
  85. var ignoreBuilds = config.GetString("IgnoreBuildBranch", string.Empty);
  86. _ignoreBuilds = ignoreBuilds.IsNotNullOrWhitespace() ? new Regex(ignoreBuilds) : null;
  87. }
  88. /// <summary>
  89. /// Gets a unique key which identifies this build server.
  90. /// </summary>
  91. public string UniqueKey => _httpClient.BaseAddress.Host;
  92. private void AddGetBuildUrl(string projectUrl)
  93. {
  94. if (!_projectsUrls.Contains(projectUrl))
  95. {
  96. _projectsUrls.Add(projectUrl);
  97. _lastBuildCache[projectUrl] = new JenkinsCacheInfo();
  98. }
  99. }
  100. public class ResponseInfo
  101. {
  102. public string Url { get; set; }
  103. public long Timestamp { get; set; }
  104. public IEnumerable<JToken> JobDescription { get; set; }
  105. }
  106. public class JenkinsCacheInfo
  107. {
  108. public long Timestamp = -1;
  109. }
  110. private async Task<ResponseInfo> GetBuildInfoTaskAsync(string projectUrl, bool fullInfo, CancellationToken cancellationToken)
  111. {
  112. string t = null;
  113. long timestamp = 0;
  114. IEnumerable<JToken> s = Enumerable.Empty<JToken>();
  115. try
  116. {
  117. t = await GetResponseAsync(FormatToGetJson(projectUrl, fullInfo), cancellationToken).ConfigureAwait(false);
  118. }
  119. catch
  120. {
  121. // Could be cancelled or failed. Explicitly assign 't' to reveal the intended behavior of following code
  122. // for this case.
  123. t = null;
  124. }
  125. if (t.IsNotNullOrWhitespace() && !cancellationToken.IsCancellationRequested)
  126. {
  127. JObject jobDescription = JObject.Parse(t);
  128. if (jobDescription["builds"] != null)
  129. {
  130. // Freestyle jobs
  131. s = jobDescription["builds"];
  132. }
  133. else if (jobDescription["jobs"] != null)
  134. {
  135. // Multi-branch pipeline
  136. s = jobDescription["jobs"]
  137. .SelectMany(j => j["builds"]);
  138. foreach (var j in jobDescription["jobs"])
  139. {
  140. var ts = j["lastBuild"]["timestamp"];
  141. if (ts != null)
  142. {
  143. timestamp = Math.Max(timestamp, ts.ToObject<long>());
  144. }
  145. }
  146. }
  147. // else: The server had no response (overloaded?) or a multi-branch pipeline is not configured
  148. if (timestamp == 0 && jobDescription["lastBuild"]?["timestamp"] != null)
  149. {
  150. timestamp = jobDescription["lastBuild"]["timestamp"].ToObject<long>();
  151. }
  152. }
  153. return new ResponseInfo
  154. {
  155. Url = projectUrl,
  156. Timestamp = timestamp,
  157. JobDescription = s
  158. };
  159. }
  160. public IObservable<BuildInfo> GetFinishedBuildsSince(IScheduler scheduler, DateTime? sinceDate = null)
  161. {
  162. // GetBuilds() will return the same builds as for GetRunningBuilds().
  163. // Multiple calls will fetch same info multiple times and make debugging very confusing
  164. // Similar as for AppVeyor
  165. return Observable.Empty<BuildInfo>();
  166. }
  167. public IObservable<BuildInfo> GetRunningBuilds(IScheduler scheduler)
  168. {
  169. return GetBuilds(scheduler, null, true);
  170. }
  171. private IObservable<BuildInfo> GetBuilds(IScheduler scheduler, DateTime? sinceDate = null, bool? running = null)
  172. {
  173. return Observable.Create<BuildInfo>((observer, cancellationToken) =>
  174. ThreadHelper.JoinableTaskFactory.RunAsync(async () =>
  175. {
  176. await TaskScheduler.Default;
  177. return scheduler.Schedule(() => ObserveBuilds(sinceDate, running, observer, cancellationToken));
  178. }).Task);
  179. }
  180. private void ObserveBuilds(DateTime? sinceDate, bool? running, IObserver<BuildInfo> observer, CancellationToken cancellationToken)
  181. {
  182. // Note that 'running' is ignored (attempt to fetch data when updated)
  183. // Similar for 'sinceDate', not supported in Jenkins API
  184. try
  185. {
  186. var allBuildInfos = new List<JoinableTask<ResponseInfo>>();
  187. var latestBuildInfos = new List<JoinableTask<ResponseInfo>>();
  188. foreach (var projectUrl in _projectsUrls)
  189. {
  190. if (_lastBuildCache[projectUrl].Timestamp <= 0)
  191. {
  192. // This job must be updated, no need to to check the latest builds
  193. allBuildInfos.Add(ThreadHelper.JoinableTaskFactory.RunAsync(() => GetBuildInfoTaskAsync(projectUrl, true, cancellationToken)));
  194. }
  195. else
  196. {
  197. latestBuildInfos.Add(ThreadHelper.JoinableTaskFactory.RunAsync(() => GetBuildInfoTaskAsync(projectUrl, false, cancellationToken)));
  198. }
  199. }
  200. // Check the latest build on the server to the existing build cache
  201. // The simple request will limit the load on the Jenkins server
  202. // To fetch just new builds is possible too, but it will make the solution more complicated
  203. // Similar, the build results could be cached so they are available when switching repos
  204. foreach (var info in latestBuildInfos)
  205. {
  206. if (!info.Task.IsFaulted)
  207. {
  208. if (info.Join().Timestamp > _lastBuildCache[info.Join().Url].Timestamp)
  209. {
  210. // The cache has at least one newer job, query the status
  211. allBuildInfos.Add(ThreadHelper.JoinableTaskFactory.RunAsync(() => GetBuildInfoTaskAsync(info.Task.CompletedResult().Url, true, cancellationToken)));
  212. }
  213. }
  214. }
  215. if (allBuildInfos.All(t => t.Task.IsCanceled))
  216. {
  217. observer.OnCompleted();
  218. return;
  219. }
  220. foreach (var build in allBuildInfos)
  221. {
  222. if (build.Task.IsFaulted)
  223. {
  224. Debug.Assert(build.Task.Exception != null, "build.Task.Exception != null");
  225. observer.OnError(build.Task.Exception);
  226. continue;
  227. }
  228. if (build.Task.IsCanceled || build.Join().Timestamp <= 0)
  229. {
  230. // No valid information received for the build
  231. continue;
  232. }
  233. _lastBuildCache[build.Join().Url].Timestamp = build.Join().Timestamp;
  234. // Present information in reverse, so the latest job is displayed (i.e. new inprogress on one commit)
  235. // (for multi-branch pipeline, ignore the corner case with multiple branches with inprogress builds on one commit)
  236. foreach (var buildDetails in build.Join().JobDescription.Reverse())
  237. {
  238. if (cancellationToken.IsCancellationRequested)
  239. {
  240. return;
  241. }
  242. try
  243. {
  244. var buildInfo = CreateBuildInfo((JObject)buildDetails);
  245. if (buildInfo != null)
  246. {
  247. observer.OnNext(buildInfo);
  248. if (buildInfo.Status == BuildInfo.BuildStatus.InProgress)
  249. {
  250. // Need to make a full request next time
  251. _lastBuildCache[build.Join().Url].Timestamp = 0;
  252. }
  253. }
  254. }
  255. catch
  256. {
  257. // Ignore unexpected responses
  258. }
  259. }
  260. }
  261. // Complete the job, it will be run again with Observe.Retry() (every 10th sec)
  262. observer.OnCompleted();
  263. }
  264. catch (OperationCanceledException)
  265. {
  266. // Do nothing, the observer is already stopped
  267. }
  268. catch (Exception ex)
  269. {
  270. // Cancelling a sub-task is similar to cancelling this task
  271. if (!(ex.InnerException is OperationCanceledException))
  272. {
  273. observer.OnError(ex);
  274. }
  275. }
  276. }
  277. private const string _jenkinsTreeBuildInfo = "number,result,timestamp,url,actions[lastBuiltRevision[SHA1,branch[name]],totalCount,failCount,skipCount],building,duration";
  278. [CanBeNull]
  279. private BuildInfo CreateBuildInfo(JObject buildDescription)
  280. {
  281. var idValue = buildDescription["number"].ToObject<string>();
  282. var statusValue = buildDescription["result"].ToObject<string>();
  283. var startDateTicks = buildDescription["timestamp"].ToObject<long>();
  284. var webUrl = buildDescription["url"].ToObject<string>();
  285. var action = buildDescription["actions"];
  286. var commitHashList = new List<ObjectId>();
  287. string testResults = string.Empty;
  288. foreach (var element in action)
  289. {
  290. if (element["lastBuiltRevision"] != null)
  291. {
  292. commitHashList.Add(ObjectId.Parse(element["lastBuiltRevision"]["SHA1"].ToObject<string>()));
  293. var branches = element["lastBuiltRevision"]["branch"];
  294. if (_ignoreBuilds != null && branches != null)
  295. {
  296. // Ignore build events for specified branches
  297. foreach (var branch in branches)
  298. {
  299. var name = branch["name"];
  300. if (name != null)
  301. {
  302. var name2 = name.ToObject<string>();
  303. if (name2.IsNotNullOrWhitespace() && _ignoreBuilds.IsMatch(name2))
  304. {
  305. return null;
  306. }
  307. }
  308. }
  309. }
  310. }
  311. if (element["totalCount"] != null)
  312. {
  313. int testCount = element["totalCount"].ToObject<int>();
  314. if (testCount != 0)
  315. {
  316. int failedTestCount = element["failCount"].ToObject<int>();
  317. int skippedTestCount = element["skipCount"].ToObject<int>();
  318. testResults = $"{testCount} tests ({failedTestCount} failed, {skippedTestCount} skipped)";
  319. }
  320. }
  321. }
  322. var isRunning = buildDescription["building"].ToObject<bool>();
  323. long? buildDuration;
  324. if (isRunning)
  325. {
  326. buildDuration = null;
  327. }
  328. else
  329. {
  330. buildDuration = buildDescription["duration"].ToObject<long>();
  331. }
  332. var status = isRunning ? BuildInfo.BuildStatus.InProgress : ParseBuildStatus(statusValue);
  333. var statusText = status.ToString("G");
  334. var buildInfo = new BuildInfo
  335. {
  336. Id = idValue,
  337. StartDate = TimestampToDateTime(startDateTicks),
  338. Duration = buildDuration,
  339. Status = status,
  340. CommitHashList = commitHashList.ToArray(),
  341. Url = webUrl
  342. };
  343. var durationText = _buildDurationFormatter.Format(buildInfo.Duration);
  344. buildInfo.Description = $"#{idValue} {durationText} {testResults} {statusText}";
  345. return buildInfo;
  346. }
  347. public static DateTime TimestampToDateTime(long timestamp)
  348. {
  349. return new DateTime(1970, 1, 1, 0, 0, 0, DateTime.Now.Kind).AddMilliseconds(timestamp);
  350. }
  351. private static AuthenticationHeaderValue CreateBasicHeader(string username, string password)
  352. {
  353. byte[] byteArray = Encoding.UTF8.GetBytes(string.Format("{0}:{1}", username, password));
  354. return new AuthenticationHeaderValue("Basic", Convert.ToBase64String(byteArray));
  355. }
  356. private static BuildInfo.BuildStatus ParseBuildStatus(string statusValue)
  357. {
  358. switch (statusValue)
  359. {
  360. case "SUCCESS":
  361. return BuildInfo.BuildStatus.Success;
  362. case "FAILURE":
  363. return BuildInfo.BuildStatus.Failure;
  364. case "UNSTABLE":
  365. return BuildInfo.BuildStatus.Unstable;
  366. case "ABORTED":
  367. return BuildInfo.BuildStatus.Stopped;
  368. default:
  369. return BuildInfo.BuildStatus.Unknown;
  370. }
  371. }
  372. private async Task<Stream> GetStreamAsync(string restServicePath, CancellationToken cancellationToken)
  373. {
  374. cancellationToken.ThrowIfCancellationRequested();
  375. var response = await _httpClient.GetAsync(restServicePath, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
  376. return await GetStreamFromHttpResponseAsync(response);
  377. async Task<Stream> GetStreamFromHttpResponseAsync(HttpResponseMessage resp)
  378. {
  379. bool unauthorized = resp.StatusCode == HttpStatusCode.Unauthorized;
  380. if (resp.IsSuccessStatusCode)
  381. {
  382. var httpContent = resp.Content;
  383. if (httpContent.Headers.ContentType.MediaType == "text/html")
  384. {
  385. // Jenkins responds with an HTML login page when guest access is denied.
  386. unauthorized = true;
  387. }
  388. else
  389. {
  390. return await httpContent.ReadAsStreamAsync();
  391. }
  392. }
  393. else if (resp.StatusCode == HttpStatusCode.NotFound)
  394. {
  395. // The url does not exist, no jobs to retrieve
  396. return null;
  397. }
  398. else if (resp.StatusCode == HttpStatusCode.Forbidden)
  399. {
  400. unauthorized = true;
  401. }
  402. if (unauthorized)
  403. {
  404. var buildServerCredentials = _buildServerWatcher.GetBuildServerCredentials(this, false);
  405. if (buildServerCredentials != null)
  406. {
  407. UpdateHttpClientOptions(buildServerCredentials);
  408. return await GetStreamAsync(restServicePath, cancellationToken);
  409. }
  410. throw new OperationCanceledException(resp.ReasonPhrase);
  411. }
  412. throw new HttpRequestException(resp.ReasonPhrase);
  413. }
  414. }
  415. private void UpdateHttpClientOptions(IBuildServerCredentials buildServerCredentials)
  416. {
  417. var useGuestAccess = buildServerCredentials == null || buildServerCredentials.UseGuestAccess;
  418. _httpClient.DefaultRequestHeaders.Authorization = useGuestAccess
  419. ? null : CreateBasicHeader(buildServerCredentials.Username, buildServerCredentials.Password);
  420. }
  421. private async Task<string> GetResponseAsync(string relativePath, CancellationToken cancellationToken)
  422. {
  423. using (var responseStream = await GetStreamAsync(relativePath, cancellationToken).ConfigureAwait(false))
  424. {
  425. using (var reader = new StreamReader(responseStream))
  426. {
  427. return await reader.ReadToEndAsync();
  428. }
  429. }
  430. }
  431. private static string FormatToGetJson(string restServicePath, bool buildsInfo = false)
  432. {
  433. string buildTree = "lastBuild[timestamp]";
  434. int depth = 1;
  435. int postIndex = restServicePath.IndexOf('?');
  436. if (postIndex >= 0)
  437. {
  438. int endLen = restServicePath.Length - postIndex;
  439. if (restServicePath.EndsWith("/"))
  440. {
  441. endLen--;
  442. }
  443. string post = restServicePath.Substring(postIndex, endLen);
  444. if (post == "?m")
  445. {
  446. // Multi pipeline project
  447. buildTree = "jobs[" + buildTree;
  448. if (buildsInfo)
  449. {
  450. depth = 2;
  451. buildTree += ",builds[" + _jenkinsTreeBuildInfo + "]";
  452. }
  453. buildTree += "]";
  454. }
  455. else
  456. {
  457. // user defined format (will likely require changes in the code)
  458. buildTree = post;
  459. }
  460. restServicePath = restServicePath.Substring(0, postIndex);
  461. }
  462. else
  463. {
  464. // Freestyle project
  465. if (buildsInfo)
  466. {
  467. buildTree += ",builds[" + _jenkinsTreeBuildInfo + "]";
  468. }
  469. }
  470. if (!restServicePath.EndsWith("/"))
  471. {
  472. restServicePath += "/";
  473. }
  474. restServicePath += "api/json?depth=" + depth + "&tree=" + buildTree;
  475. return restServicePath;
  476. }
  477. public void Dispose()
  478. {
  479. GC.SuppressFinalize(this);
  480. _httpClient?.Dispose();
  481. }
  482. }
  483. }