/src/xf/nucleus/kdef/KDefProcessor.d
D | 882 lines | 686 code | 162 blank | 34 comment | 116 complexity | 3400da440c64b57863d50e6708197f95 MD5 | raw file
- module xf.nucleus.kdef.KDefProcessor;
-
- private {
- import tango.core.Tuple;
-
- import
- xf.nucleus.kdef.model.IKDefFileParser,
- xf.nucleus.kdef.model.KDefInvalidation,
- xf.nucleus.kdef.KDefParserBase,
- xf.nucleus.kdef.Common,
- xf.nucleus.kdef.ParamUtils;
-
- import
- xf.nucleus.kernel.KernelDef;
-
- import
- xf.nucleus.Defs,
- xf.nucleus.TypeSystem,
- xf.nucleus.TypeConversion,
- xf.nucleus.Function,
- xf.nucleus.Param,
- xf.nucleus.Value,
- xf.nucleus.KernelImpl,
- xf.nucleus.SurfaceDef,
- xf.nucleus.MaterialDef,
- Dep = xf.nucleus.DepTracker;
-
- import
- xf.mem.ScratchAllocator,
- xf.mem.StackBuffer,
- xf.mem.Gather;
-
- import xf.utils.Graph : findTopologicalOrder, CycleHandlingMode;
- import xf.utils.Memory : alloc, free, append;
- import xf.nucleus.Log : error = nucleusError, log = nucleusLog;
-
- import tango.io.Stdout; // for dumpInfo, could be moved outside, to another mod
- import tango.io.vfs.model.Vfs;
-
- alias char[] string;
- }
-
-
-
- class KDefProcessor {
- alias DgScratchAllocator Allocator;
-
-
- this (IKDefFileParser fileParser) {
- this.fileParser = fileParser;
- }
-
-
- static Allocator modAlloc(KDefModule mod) {
- return mod.mem;
- }
-
-
- void processFile(string path) {
- if (auto mod = path in _modules) {
- if ((*mod).processing) {
- throw new Exception("Cyclic import in kernel module '" ~ path ~ "'");
- }
- } else {
- log.trace("KDefProcessor.parseFile({})", path);
- auto mod = fileParser.parseFile(path);
- _modules[path] = mod;
- mod.processing = true;
- mod.filePath = path;
- process(mod, modAlloc(mod));
- mod.processing = false;
- }
- }
-
-
- KDefModule getModuleForPath(string path) {
- if (auto mod = path in _modules) {
- return *mod;
- } else {
- return null;
- }
- }
-
-
- void doSemantics() {
- doKernelSemantics();
- }
-
-
- void dispose() {
- foreach (ref mod; _modules) {
- delete mod;
- }
- // TODO(?): do more thorough cleaning
- _modules = null;
- }
-
-
- void dumpInfo() {
- Stdout.formatln("got {} modules:", _modules.keys.length);
- foreach (name, mod; _modules) {
- Stdout.formatln("mod {}:", name);
- dumpInfo(mod);
- }
- }
-
-
- struct KernelInfo {
- KernelImpl* impl;
- KDefModule mod;
- }
-
- KernelInfo[string] kernels;
-
-
- int converters(int delegate(ref SemanticConverter) dg) {
- foreach (name, mod; _modules) {
- foreach (ref conv; mod.converters) {
- if (auto r = dg(conv)) {
- return r;
- }
- }
- }
-
- return 0;
- }
-
-
- KernelImpl getKernel(string name) {
- if (auto impl = name in this.kernels) {
- return *impl.impl;
- } else {
- error("Unknown kernel: '{}'.", name);
- assert (false);
- }
- }
-
-
- // BUG: this is slow
- KernelImpl getKernel(KernelImplId id) {
- assert (id.isValid);
-
- foreach (k, ref v; this.kernels) {
- assert (v.impl.id.isValid);
- if (v.impl.id == id) {
- return *v.impl;
- }
- }
-
- error("Unknown kernel id: '{}'.", id.value);
- assert (false);
- }
-
-
- bool getKernel(string name, KernelImpl* res) {
- if (auto impl = name in this.kernels) {
- *res = *impl.impl;
- return true;
- } else {
- return false;
- }
- }
-
-
- int kernelImpls(int delegate(ref KernelImpl) dg) {
- foreach (n, k; this.kernels) {
- if (int r = dg(*k.impl)) {
- return r;
- }
- }
-
- return 0;
- }
-
-
- int surfaces(int delegate(ref string, ref SurfaceDef) dg) {
- foreach (name, mod; _modules) {
- foreach (name, ref surf; mod.surfaces) {
- string meh = name;
- if (auto r = dg(meh, surf)) {
- return r;
- }
- }
- }
-
- return 0;
- }
-
-
- int materials(int delegate(ref string, ref MaterialDef) dg) {
- foreach (name, mod; _modules) {
- foreach (name, ref surf; mod.materials) {
- string meh = name;
- if (auto r = dg(meh, surf)) {
- return r;
- }
- }
- }
-
- return 0;
- }
-
-
- int modules(int delegate(ref string, ref KDefModule) dg) {
- foreach (name, mod; _modules) {
- if (int r = dg(name, mod)) {
- return r;
- }
- }
-
- return 0;
- }
-
-
- KDefInvalidationInfo invalidateDifferences(KDefProcessor other) {
- KDefInvalidationInfo res;
-
- foreach (modName, mod; other._modules) {
- if (!(modName in this._modules)) {
- if (mod.converters.length > 0) {
- res.anyConverters = true;
- }
- }
- }
-
- foreach (modName, mod; _modules) {
- if (auto otherMod = modName in other._modules) {
- foreach (name, ref o; mod.kernels) {
- if (auto o2 = name in otherMod.kernels) {
- if (o != *o2) {
- o.invalidate();
- }
- } else {
- o.invalidate();
- }
- }
-
- foreach (name, ref o; mod.surfaces) {
- if (auto o2 = name in otherMod.surfaces) {
- if (o != *o2) {
- o.invalidate();
- }
- } else {
- o.invalidate();
- }
- }
-
- foreach (name, ref o; mod.materials) {
- if (auto o2 = name in otherMod.materials) {
- if (o != *o2) {
- o.invalidate();
- }
- } else {
- o.invalidate();
- }
- }
-
- if (!res.anyConverters) {
- if (mod.converters.length != otherMod.converters.length) {
- res.anyConverters = true;
- } else {
- foreach (i, ref c; mod.converters) {
- if (c != otherMod.converters[i]) {
- res.anyConverters = true;
- break;
- }
- }
- }
- }
- } else {
- // If the module is missing from the other processor,
- // invalidate all of the current module's items
-
- if (mod.converters.length > 0) {
- res.anyConverters = true;
- }
-
- foreach (name, ref o; mod.kernels) {
- o.invalidate();
- }
-
- foreach (name, ref o; mod.surfaces) {
- o.invalidate();
- }
-
- foreach (name, ref o; mod.materials) {
- o.invalidate();
- }
- }
- }
-
- Dep.invalidateGraph((void delegate(Dep.DepTracker*) depSink) {
- void process(Dep.DepTracker* tr) {
- if (!tr.valid) {
- depSink(tr);
- }
- }
-
- foreach (modName, mod; _modules) {
- foreach (name, ref o; mod.kernels) {
- process(o.dependentOnThis);
- }
-
- foreach (name, ref o; mod.surfaces) {
- process(o.dependentOnThis);
- }
-
- foreach (name, ref o; mod.materials) {
- process(o.dependentOnThis);
- }
- }
- });
-
- return res;
- }
-
-
- private {
- enum Processed {
- Not,
- InProgress,
- Done
- }
-
-
- void doKernelSemantics() {
- Processed[KernelImpl] processed;
-
- foreach (name, mod; _modules) {
- foreach (kname, ref kimpl; mod.kernels) {
- if (kname in this.kernels) {
- error(
- "Multiple definitions of kernel '{}' in:\n {}\n and\n {}.",
- kname,
- mod.filePath,
- this.kernels[kname].mod.filePath
- );
- } else {
- this.kernels[kname] = KernelInfo(
- &kimpl,
- mod
- );
- }
-
- if (KernelImpl.Type.Kernel == kimpl.type) {
- if (auto func = cast(Function)kimpl.kernel.func) {
- static assert (is(typeof(mod) == KDefModule));
- func.code._module = cast(void*)mod;
- }
-
- kimpl.kernel._module = cast(void*)mod;
- } else {
- GraphDef(kimpl.graph)._module = cast(void*)mod;
- }
- }
- }
-
-
- foreach (name, mod; _modules) {
- foreach (ref surf; mod.surfaces) {
- getKernel(surf.reflKernelName)
- .dependentOnThis.add(surf.dependentOnThis);
- }
-
- foreach (ref mat; mod.materials) {
- getKernel(mat.materialKernelName)
- .dependentOnThis.add(mat.dependentOnThis);
- }
- }
-
-
- foreach (n, ref k; kernels) {
- _curSemModule = k.mod;
- doKernelSemantics(*k.impl, modAlloc(k.mod), processed);
- }
- }
-
- void inheritKernel(KernelDef sub, KernelImpl supr) {
- iterKernelInputs(supr, (Param* p) {
- if (!sub.func.params.get(p.name)) {
- sub.func.params.add(*p);
- }
- });
-
- iterKernelOutputs(supr, (Param* p) {
- if (!sub.func.params.get(p.name)) {
- sub.func.params.add(*p);
- }
- });
- }
-
- void getPlainSemanticForNode(Param* par, Semantic* plainSem, KernelImpl supr) {
- findOutputSemantic(
- par,
-
- // getFormalParamSemantic
- (string name) {
- Semantic* res;
-
- iterKernelInputs(supr, (Param* p) {
- if (p.isInput && p.name == name) {
- res = p.semantic();
- }
- });
-
- if (res) {
- return *res;
- }
-
- error(
- "simplifyParamSemantics: output param '{}' refers to a"
- " nonexistent formal parameter '{}'.",
- par.name,
- name
- );
- assert (false);
- },
-
- // getActualParamSemantic
- (string name) {
- error(
- "Cannot derive node params from {}: param {} has a semantic expression.",
- supr.name,
- par.name
- );
- return Semantic.init;
- },
-
- plainSem
- );
- }
-
- void inheritKernelInputs(GraphDefNode node, KernelImpl supr) {
- iterKernelInputs(supr, (Param* p) {
- if (!node.params.get(p.name)) {
- assert (p.hasPlainSemantic);
- node.params.add(*p).dir = ParamDirection.Out;
- }
- });
- }
-
- void inheritKernelOutputs(GraphDefNode node, KernelImpl supr) {
- iterKernelOutputs(supr, (Param* p) {
- if (!node.params.get(p.name)) {
- if (!p.hasPlainSemantic) {
- auto pnew = node.params.add(ParamDirection.In, p.name);
- pnew.hasPlainSemantic = true;
- getPlainSemanticForNode(p, pnew.semantic, supr);
- pnew.copyValueFrom(p);
- pnew.annotation = p.annotation;
- } else {
- node.params.add(*p).dir = ParamDirection.In;
- }
- }
- });
- }
-
-
- void iterKernelParams(
- KernelImpl impl,
- bool wantInputs,
- void delegate(Param*) sink
- ) {
- switch (impl.type) {
- case KernelImpl.Type.Kernel: {
- foreach (ref p; impl.kernel.func.params) {
- if (wantInputs == p.isInput) {
- sink(&p);
- }
- }
- } break;
-
- case KernelImpl.Type.Graph: {
- foreach (nodeName, node; GraphDef(impl.graph).nodes) {
- if ((wantInputs ? "input" : "output") == node.type) {
- foreach (ref p; node.params) {
- sink(&p);
- }
- }
- }
- } break;
-
- default: assert (false);
- }
- }
-
- void iterKernelInputs(KernelImpl impl, void delegate(Param*) sink) {
- iterKernelParams(impl, true, sink);
- }
-
- void iterKernelOutputs(KernelImpl impl, void delegate(Param*) sink) {
- iterKernelParams(impl, false, sink);
- }
-
-
- void doKernelSemantics(ref KernelDef k, Allocator allocator, ref Processed[KernelImpl] processed) {
- if (k.superKernel.length > 0) {
- auto superKernel = getKernel(k.superKernel);
- doKernelSemantics(superKernel, allocator, processed);
- inheritKernel(k, superKernel);
-
- superKernel.dependentOnThis.add(
- k.dependentOnThis
- );
- }
- }
-
-
- void doKernelSemantics(GraphDef graph, Allocator allocator, ref Processed[KernelImpl] processed) {
- KernelImpl superKernel;
-
- if (graph.superKernel.length > 0) {
- superKernel = getKernel(graph.superKernel);
- doKernelSemantics(superKernel, allocator, processed);
-
- superKernel.dependentOnThis.add(
- graph.dependentOnThis
- );
- }
-
- foreach (nodeName, node; graph.nodes) {
- switch (node.type) {
- case "input": {
- if (!superKernel.isNull) {
- inheritKernelInputs(node, superKernel);
- }
- } break;
-
- case "output": {
- if (!superKernel.isNull) {
- inheritKernelOutputs(node, superKernel);
- }
- } break;
-
- case "kernel": {
- auto kernelVar = node.getVar("kernel");
- if (kernelVar is null) {
- // TODO: err
- error("No kernel defined for a kernel node.");
- }
- if (auto ident = cast(IdentifierValue)kernelVar) {
- node.kernelImpl = getKernel(ident.value);
- }
- else if (auto literal = cast(KernelDefValue)kernelVar) {
- if (!cast(Function)literal.kernelDef.func) {
- // TODO: err
- error(
- "Graph nodes must use concrete kernel literals."
- );
- }
-
- doKernelSemantics(literal.kernelDef, allocator, processed);
- node.kernelImpl = KernelImpl(literal.kernelDef);
- node.kernelImpl.kernel.func.name = "literal";
- node.kernelImpl.kernel._module = cast(void*)_curSemModule;
- }
- else {
- error(
- "The 'kernel' var in a graph node must be"
- " either an identifier or a concrete kernel literal,"
- " not a '{}'", kernelVar.classinfo.name
- );
- }
-
- node.kernelImpl.dependentOnThis.add(
- graph.dependentOnThis
- );
- } break;
-
- default: break;
- }
- }
- }
-
- void doKernelSemantics(ref KernelImpl k, Allocator allocator, ref Processed[KernelImpl] processed) {
- if (auto p = k in processed) {
- switch (*p) {
- case Processed.InProgress: {
- char[] list;
- foreach (ki, p2; processed) {
- if (Processed.InProgress == p2) {
- list ~= " '"~ki.name~"'\n";
- }
- }
-
- // TODO: keep track of the exact eval order
- error(
- "Recursive prosessing of a kernel '{}'. From:\n{}",
- k.name,
- list
- );
- } break;
-
- case Processed.Not: break;
- case Processed.Done: return;
-
- default: assert (false);
- }
- }
-
- processed[k] = Processed.InProgress;
- scope (success) processed[k] = Processed.Done;
-
- switch (k.type) {
- case KernelImpl.Type.Kernel: {
- doKernelSemantics(k.kernel, allocator, processed);
- } break;
-
- case KernelImpl.Type.Graph: {
- doKernelSemantics(GraphDef(k.graph), allocator, processed);
- } break;
-
- default: assert (false);
- }
- }
-
- // --------------------------------------------------------------------------------------------------------------------------------
- //
-
- struct SimpleArray(T) {
- private {
- T[] _arr;
- size_t _len;
- }
-
-
- size_t length() {
- return _len;
- }
-
- T[] data() {
- return _arr[0.._len];
- }
-
- void add(T t) {
- _arr.append(t, &_len);
- }
-
- void dispose() {
- _arr.free();
- }
- }
-
-
- void dispose2dArray(T)(ref SimpleArray!(T)[] arr) {
- foreach (ref item; arr) {
- item.dispose();
- }
- arr.free();
- }
-
-
- //
- // --------------------------------------------------------------------------------------------------------------------------------
-
- void dumpInfo(KDefModule mod) {
- Stdout.formatln("* path: {}", mod.filePath);
- Stdout.formatln("* kernel impls:");
-
- foreach (n, impl; mod.kernels) {
- dumpInfo(impl);
- }
- }
-
-
- void dumpInfo(KernelImpl kimpl) {
- switch (kimpl.type) {
- case KernelImpl.Type.Graph:
- return dumpInfo(GraphDef(kimpl.graph));
- case KernelImpl.Type.Kernel:
- return dumpInfo(kimpl.kernel);
- default: assert (false);
- }
- }
-
-
- void dumpInfo(AbstractFunction func) {
- Stdout.formatln("\tfunc: {} {{", func.name);
- foreach (param; func.params) {
- Stdout.format("\t\t{}", param);
- /+if (param.defaultValue) {
- Stdout.formatln(" = {}", param.defaultValue);
- }+/
- Stdout.newline;
- }
- if (auto qf = cast(Function)func) {
- //Stdout.formatln("{} code:", qf.code.language);
- Stdout("----").newline;
- char[] code;
- qf.code.writeOut((char[] s) {
- code ~= s;
- });
- Stdout(code).newline;
- Stdout("----").newline;
- }
- Stdout("}").newline;
- }
-
-
- void dumpInfo(GraphDef graph) {
- Stdout("graph {").newline;
- Stdout.formatln("file: {}", (cast(KDefModule)graph._module).filePath);
- Stdout.formatln("bytes: {}..{}", graph._firstByte, graph._firstByte + graph._lengthBytes);
- foreach (nodeName, _node; graph.nodes) {
- Stdout.formatln("\tnode: {}", nodeName);
- }
- /+foreach (graphName, subGraph; graph.graphs) {
- Stdout.formatln("\tsub-graph: ");
- dumpInfo(subGraph);
- }+/
- Stdout("}").newline;
- }
-
-
- void dumpInfo(KernelDef kernel) {
- dumpInfo(kernel.func);
- }
-
-
- void process(Scope sc, Allocator allocator) {
- gatherArrays!(string, Value)(sc.mem,
- (void delegate(lazy string, lazy Value) gen) {
- foreach (stmt_; sc.statements) {
- if (auto stmt = cast(AssignStatement)stmt_) {
- gen(stmt.name, stmt.value);
- }
- }
- },
- (string[] names, Value[] values) {
- sc.doAssign(names, values);
- });
-
- gatherArrays!(string, string)(sc.mem,
- (void delegate(lazy string, lazy string) gen) {
- foreach (stmt_; sc.statements) {
- if (auto stmt = cast(ConnectStatement)stmt_) {
- if (auto graph = cast(GraphDef)sc) {
- gen(stmt.from, stmt.to);
- } else {
- throw new Exception("connections only allowed at graph scope");
- }
- }
- }
- },
- (string[] from, string[] to) {
- (cast(GraphDef)sc).doConnect(from, to);
- });
-
- gatherArrays!(string)(sc.mem,
- (void delegate(lazy string) gen) {
- foreach (stmt_; sc.statements) {
- if (auto stmt = cast(NoAutoFlowStatement)stmt_) {
- if (auto graph = cast(GraphDef)sc) {
- gen(stmt.to);
- } else {
- error("Can't use noauto outside of graph definitions.");
- }
- }
- }
- },
- (string[] to) {
- (cast(GraphDef)sc).setNoAuto(to);
- });
-
-
-
- foreach (stmt_; sc.statements) {
- if (auto stmt = cast(ConnectStatement)stmt_) {
- // nothing
- } else if (auto stmt = cast(NoAutoFlowStatement)stmt_) {
- // nothing
- } else if (auto stmt = cast(AssignStatement)stmt_) {
- if (auto scopeValue = cast(IScopeValue)stmt.value) {
- process(scopeValue.toScope, allocator);
- }
-
- if (auto kernelValue = cast(KernelDefValue)stmt.value) {
- kernelValue.kernelDef.func.name = stmt.name;
- if (auto mod = cast(KDefModule)sc) {
- mod.kernels[stmt.name] = KernelImpl(kernelValue.kernelDef);
- }
- }
-
- if (auto graphValue = cast(GraphDefValue)stmt.value) {
- GraphDef(graphValue.graphDef)._name = stmt.name;
- if (auto mod = cast(KDefModule)sc) {
- mod.kernels[stmt.name] = KernelImpl(graphValue.graphDef);
- }
- }
-
- if (auto surfValue = cast(SurfaceDefValue)stmt.value) {
- surfValue.surface.name = stmt.name;
- if (auto mod = cast(KDefModule)sc) {
- mod.surfaces[stmt.name] = surfValue.surface;
- }
- }
-
- if (auto matValue = cast(MaterialDefValue)stmt.value) {
- matValue.material.name = stmt.name;
- if (auto mod = cast(KDefModule)sc) {
- mod.materials[stmt.name] = matValue.material;
- }
- }
-
- if (auto nodeValue = cast(GraphDefNodeValue)stmt.value) {
- auto node = nodeValue.node;
-
- switch (node.type) {
- case "input":
- case "data":
- case "output": {
- node.params = ParamList(allocator._allocator);
- }
- default: break;
- }
-
- buildConcreteParams(node, &node.params);
- }
-
- if (auto traitValue = cast(TraitDefValue)stmt.value) {
- traitValue.value.name = stmt.name;
- if (auto mod = cast(KDefModule)sc) {
- mod.traitDefs ~= traitValue.value;
- }
- }
- } else if (auto stmt = cast(ImportStatement)stmt_) {
- auto path = stmt.path;
- processFile(path);
- auto names = stmt.what;
- auto mod = _modules[path];
-
- if (names !is null) {
- assert (false, "TODO: selective imports");
- } else {
- sc.doImport((void delegate(Statement) dg) {
- foreach (st; mod.statements) {
- dg(st);
- }
- });
- }
- } else if (auto stmt = cast(ConverterDeclStatement)stmt_) {
- if (auto mod = cast(KDefModule)sc) {
- assert (2 == stmt.func.params.length); // there's also hidden context
-
- auto from = stmt.func.params[0];
- auto to = stmt.func.params[1];
-
- /+if ("Cg" == stmt.func.code.language) {
- if (!from.semantic.hasTrait("domain")) {
- from.semantic.addTrait("domain", "gpu");
- }
-
- if (!to.semantic.hasTrait("domain")) {
- to.semantic.addTrait("domain", "gpu");
- }
- }+/
-
- string name = stmt.func.name;
- if (name is null) {
- // Will get renamed for overloads by codegen
- name = "converter";
- }
-
- mod.converters ~= SemanticConverter(stmt.func, stmt.cost);
- } else {
- throw new Exception("only can has converters at module scope ktnx");
- }
- } else {
- throw new Exception("Unhandled statement: " ~ stmt_.toString);
- }
- }
- }
-
-
- IKDefFileParser fileParser;
-
- // indexed by path
- KDefModule[string] _modules;
-
- KDefModule _curSemModule;
- }
- }