PageRenderTime 531ms CodeModel.GetById 55ms RepoModel.GetById 2ms app.codeStats 0ms

/Compiler.d

https://bitbucket.org/h3r3tic/xfbuild
D | 487 lines | 357 code | 78 blank | 52 comment | 62 complexity | 2e4c1661e64f33c7192630999a909678 MD5 | raw file
  1. module xfbuild.Compiler;
  2. private {
  3. import xfbuild.GlobalParams;
  4. import xfbuild.Module;
  5. import xfbuild.Process;
  6. import xfbuild.Misc;
  7. import xfbuild.BuildException;
  8. version (MultiThreaded) {
  9. import xfbuild.MT;
  10. }
  11. //import xf.utils.Profiler;
  12. import tango.core.Exception;
  13. import tango.io.device.File;
  14. import tango.io.stream.Buffered;
  15. import tango.sys.Process;
  16. import tango.io.stream.Lines;
  17. import tango.text.Regex;
  18. import tango.util.log.Trace;
  19. import Path = tango.io.Path;
  20. import Ascii = tango.text.Ascii;
  21. // TODO: better logging
  22. import tango.io.Stdout;
  23. }
  24. private {
  25. /+Regex importSemanticStartRegex;
  26. Regex importSemanticEndRegex;+/
  27. //Regex moduleSemantic1Regex;
  28. //Regex verboseRegex;
  29. }
  30. bool isVerboseMsg(char[] msg) {
  31. return
  32. msg.startsWith(`parse`)
  33. || msg.startsWith(`semantic`)
  34. || msg.startsWith(`function`)
  35. || msg.startsWith(`import`)
  36. || msg.startsWith(`library`)
  37. || msg.startsWith(`code`);
  38. }
  39. static this() {
  40. /+importSemanticStartRegex = Regex(`^Import::semantic\('([a-zA-Z0-9._]+)'\)$`);
  41. importSemanticEndRegex = Regex(`^-Import::semantic\('([a-zA-Z0-9._]+)', '(.+)'\)$`);+/
  42. //moduleSemantic1Regex = Regex(`^semantic\s+([a-zA-Z0-9._]+)$`);
  43. //verboseRegex = Regex(`^parse|semantic|function|import|library|code.*`);
  44. }
  45. class CompilerError : BuildException {
  46. this (char[] msg) {
  47. super (msg);
  48. }
  49. this(char[]m,char[]fl,long ln,Exception next=null){
  50. super(m,fl,ln,next);
  51. }
  52. }
  53. // TODO: Cache the escaped paths?
  54. private char[] unescapePath(char[] path) {
  55. char[] res = (new char[path.length])[0..0];
  56. for (int i = 0; i < path.length; ++i) {
  57. switch (path[i]) {
  58. case '\\': ++i;
  59. // fall through
  60. default:
  61. // Stdout.formatln("concatenating {}", path[i]).flush;
  62. res ~= path[i];
  63. // Stdout.formatln("done").flush;
  64. }
  65. }
  66. return res;
  67. }
  68. void compileAndTrackDeps(
  69. Module[] compileArray,
  70. ref Module[char[]] modules,
  71. ref Module[] compileMore,
  72. size_t affinity
  73. ) {
  74. Module getModule(char[] name, char[] path, bool* newlyEncountered = null) {
  75. Module worker() {
  76. if (auto mp = name in modules) {
  77. return *mp;
  78. } else {
  79. path = Path.standard(path);
  80. // If there's a corresponding .d file, compile that instead of trying to process a .di
  81. if (path.length > 3 && path[$-3..$] == ".di") {
  82. if (Path.exists(path[0..$-1]) && Path.isFile(path[0..$-1])) {
  83. path = path[0..$-1];
  84. }
  85. }
  86. auto mod = new Module;
  87. mod.name = name.dup;
  88. mod.path = path.dup;
  89. mod.timeModified = Path.modified(mod.path).ticks;
  90. assert (modules !is null);
  91. modules[mod.name] = mod;
  92. compileMore ~= mod;
  93. return mod;
  94. }
  95. }
  96. version (MultiThreaded) {
  97. synchronized (.threadPool) return worker();
  98. } else {
  99. return worker();
  100. }
  101. }
  102. char[][] opts;
  103. if (globalParams.manageHeaders)
  104. opts ~= "-H";
  105. char[] depsFileName;
  106. if (globalParams.useDeps) {
  107. depsFileName = compileArray[0].name ~ ".moduleDeps";
  108. opts ~= ["-deps=" ~ depsFileName];
  109. }
  110. if (globalParams.moduleByModule){
  111. foreach(mod;compileArray){
  112. try{
  113. compile(opts,[mod], (char[] line) {
  114. if (!isVerboseMsg(line) && TextUtil.trim(line).length) {
  115. Stderr(line).newline;
  116. }
  117. },
  118. globalParams.compilerName != "increBuild", // ==moveObjects?
  119. affinity
  120. );
  121. } catch(ProcessExecutionException e){
  122. throw new CompilerError("Error compiling "~mod.name,__FILE__,__LINE__,e);
  123. }
  124. }
  125. } else {
  126. try{
  127. compile(opts, compileArray, (char[] line) {
  128. if (!isVerboseMsg(line) && TextUtil.trim(line).length) {
  129. Stderr(line).newline;
  130. }
  131. },
  132. globalParams.compilerName != "increBuild", // ==moveObjects?
  133. affinity
  134. );
  135. } catch (ProcessExecutionException e) {
  136. char[] mods;
  137. foreach(i,m;compileArray){
  138. if (i!=0) mods~=",";
  139. mods~=m.name;
  140. }
  141. throw new CompilerError("Error compiling "~mods,__FILE__,
  142. __LINE__,e);
  143. }
  144. }
  145. // This must be done after the compilation so if the compiler errors out,
  146. // then we will keep the old deps instead of clearing them
  147. foreach (mod; compileArray) {
  148. mod.deps = null;
  149. }
  150. if (globalParams.useDeps) {
  151. scope depsRawFile = new File(depsFileName, File.ReadExisting);
  152. scope depsFile = new BufferedInput(depsRawFile);
  153. scope (exit) {
  154. depsRawFile.close();
  155. Path.remove(depsFileName);
  156. }
  157. //profile!("deps parsing")({
  158. foreach (line; new Lines!(char)(depsFile)) {
  159. auto arr = line.decomposeString(cast(char[])null, ` (`, null, `) : `, null, ` : `, null, ` (`, null, `)`, null);
  160. if (arr !is null) {
  161. char[] modName = arr[0].dup;
  162. char[] modPath = unescapePath(arr[1]);
  163. //char[] prot = arr[2];
  164. if (!isIgnored(modName)) {
  165. assert (modPath.length > 0);
  166. Module m = getModule(modName, modPath);
  167. char[] depName = arr[3].dup;
  168. char[] depPath = unescapePath(arr[4]);
  169. if (depName != "object" && !isIgnored(depName)) {
  170. assert (depPath.length > 0);
  171. Module depMod = getModule(depName, depPath);
  172. //Stdout.formatln("Module {} depends on {}", m.name, depMod.name);
  173. m.addDep(depMod);
  174. }
  175. }
  176. }
  177. }
  178. //});
  179. }
  180. foreach (mod; compileArray) {
  181. mod.timeDep = mod.timeModified;
  182. mod.wasCompiled = true;
  183. mod.needRecompile = false;
  184. // remove unwanted headers
  185. if (!mod.isHeader) {
  186. auto path = mod.path;
  187. foreach (unwanted; globalParams.noHeaders) {
  188. if (unwanted == mod.name || mod.name is null) {
  189. if (".d" == path[$-2..$]) {
  190. path = path ~ "i";
  191. if (Path.exists(path) && Path.isFile(path)) {
  192. Path.remove(path);
  193. }
  194. }
  195. }
  196. }
  197. }
  198. }
  199. }
  200. void compile(
  201. char[][] extraArgs,
  202. Module[] compileArray,
  203. void delegate(char[]) stdout,
  204. bool moveObjects,
  205. size_t affinity,
  206. ) {
  207. void execute(char[][] args, size_t affinity) {
  208. executeCompilerViaResponseFile(args[0], args[1..$], affinity);
  209. /+scope process = new Process(true, args);
  210. .execute(process);
  211. foreach(line; new Lines!(char)(process.stdout)) {
  212. stdout(TextUtil.trim(line));
  213. }
  214. Stderr.copy(process.stderr).flush;
  215. checkProcessFail(process);
  216. //Stdout.formatln("process finished");+/
  217. }
  218. if (compileArray.length)
  219. {
  220. if(!globalParams.useOP && !globalParams.useOQ)
  221. {
  222. void doGroup(Module[] group)
  223. {
  224. char[][] args;
  225. args ~= globalParams.compilerName;
  226. args ~= globalParams.compilerOptions;
  227. args ~= "-c";
  228. args ~= extraArgs;
  229. foreach(m; group)
  230. args ~= m.path;
  231. execute(args, affinity);
  232. if (moveObjects) {
  233. foreach(m; group)
  234. Path.rename(m.lastName ~ globalParams.objExt, m.objFile);
  235. }
  236. }
  237. int[char[]] lastNames;
  238. Module[][] passes;
  239. foreach(m; compileArray)
  240. {
  241. char[] lastName = Ascii.toLower(m.lastName.dup);
  242. int group;
  243. if(lastName in lastNames)
  244. group = ++lastNames[lastName];
  245. else
  246. group = lastNames[lastName] = 0;
  247. if(passes.length <= group) passes.length = group + 1;
  248. passes[group] ~= m;
  249. }
  250. foreach(pass; passes)
  251. {
  252. if(!pass.length)
  253. continue;
  254. doGroup(pass);
  255. }
  256. }
  257. else
  258. {
  259. char[][] args;
  260. args ~= globalParams.compilerName;
  261. args ~= globalParams.compilerOptions;
  262. if (globalParams.compilerName != "increBuild") {
  263. args ~= "-c";
  264. if (!globalParams.useOQ) {
  265. args ~= "-op";
  266. } else {
  267. args ~= "-oq";
  268. args ~= "-od" ~ globalParams.objPath;
  269. }
  270. }
  271. args ~= extraArgs;
  272. foreach(m; compileArray)
  273. args ~= m.path;
  274. auto compiled = compileArray.dup;
  275. execute(args, affinity);
  276. if (moveObjects) {
  277. if (!globalParams.useOQ) {
  278. try {
  279. foreach(m; compiled) {
  280. try {
  281. Path.rename(m.objFileInFolder, m.objFile);
  282. } catch (IOException) {
  283. // If the source file being compiled (and hence the
  284. // object file as well) and the object directory are
  285. // on different volumes, just renaming the file is an
  286. // invalid operation on *nix (cross-device link).
  287. // Hence, try copy/remove before erroring out.
  288. Path.copy(m.objFileInFolder, m.objFile);
  289. Path.remove(m.objFileInFolder);
  290. }
  291. }
  292. } catch (IOException e) {
  293. throw new CompilerError(e.msg);
  294. }
  295. }
  296. }
  297. }
  298. }
  299. }
  300. import tango.util.container.HashSet;
  301. void compile(ref Module[char[]] modules/+, ref Module[] moduleStack+/)
  302. {
  303. /+if (globalParams.verbose) {
  304. Stdout.formatln("compile called with: {}", modules.keys);
  305. }+/
  306. Module[] compileArray;
  307. //profile!("finding modules to be compiled")({
  308. bool[Module][Module] revDeps;
  309. foreach (mname, m; modules) {
  310. foreach (d; m.deps) {
  311. revDeps[d][m] = true;
  312. }
  313. }
  314. auto toCompile = new HashSet!(Module);
  315. {
  316. Module[] checkDeps;
  317. /+foreach (mod; moduleStack) {
  318. toCompile.add(mod);
  319. checkDeps ~= mod;
  320. }+/
  321. foreach (mname, mod; modules) {
  322. if (mod.needRecompile) {
  323. toCompile.add(mod);
  324. checkDeps ~= mod;
  325. }
  326. }
  327. while (checkDeps.length > 0) {
  328. auto mod = checkDeps[$-1];
  329. checkDeps = checkDeps[0..$-1];
  330. if (!(mod in revDeps)) {
  331. //Stdout.formatln("Module {} is not used by anything", mod.name);
  332. } else {
  333. foreach (rd, _dummy; revDeps[mod]) {
  334. if (!toCompile.contains(rd)) {
  335. toCompile.add(rd);
  336. checkDeps ~= rd;
  337. }
  338. }
  339. }
  340. }
  341. }
  342. compileArray = toCompile.toArray;
  343. if (globalParams.verbose) {
  344. Stdout.formatln("Modules to be compiled: {}", compileArray);
  345. }
  346. //});
  347. Module[] compileMore;
  348. bool firstPass = true;
  349. while (compileArray) {
  350. if (globalParams.reverseModuleOrder) {
  351. compileArray.reverse;
  352. }
  353. compileMore = null;
  354. Module[] compileNow = compileArray;
  355. Module[] compileLater = null;
  356. if (compileNow.length > globalParams.maxModulesToCompile) {
  357. compileNow = compileArray[0..globalParams.maxModulesToCompile];
  358. compileLater = compileArray[globalParams.maxModulesToCompile .. $];
  359. }
  360. //profile!("compileAndTrackDeps")({
  361. version (MultiThreaded) {
  362. int threads = globalParams.threadsToUse;
  363. // HACK: because affinity is stored in size_t
  364. // which is also what WinAPI expects;
  365. // TODO: do this properly one day :P
  366. if (threads > size_t.sizeof * 8) {
  367. threads = size_t.sizeof * 8;
  368. }
  369. Module[][] threadNow = new Module[][threads];
  370. Module[][] threadLater = new Module[][threads];
  371. foreach (th; mtFor(.threadPool, 0, threads)) {
  372. auto mods = compileNow[compileNow.length * th / threads .. compileNow.length * (th+1) / threads];
  373. if (globalParams.verbose) {
  374. Trace.formatln("Thread {}: compiling {} modules", th, mods.length);
  375. }
  376. if (mods.length > 0) {
  377. compileAndTrackDeps(
  378. mods,
  379. modules,
  380. threadLater[th],
  381. getNthAffinityMaskBit(th)
  382. );
  383. }
  384. }
  385. foreach (later; threadLater) {
  386. compileLater ~= later;
  387. }
  388. } else {
  389. compileAndTrackDeps(compileNow, modules, compileLater, size_t.max);
  390. }
  391. //});
  392. //Stdout.formatln("compileMore: {}", compileMore);
  393. auto next = compileLater ~ compileMore;
  394. /*
  395. In the second pass, the modules from the first one will be compiled anyway
  396. we'll pass them again to the compiler so it has a chance of better symbol placement
  397. */
  398. if (firstPass && next.length > 0) {
  399. compileArray ~= next;
  400. } else {
  401. compileArray = next;
  402. }
  403. firstPass = false;
  404. }
  405. }