1 /** 2 * D Documentation Generator 3 * Copyright: © 2014 Economic Modeling Specialists, Intl., © 2015 Ferdinand Majerech 4 * Authors: Brian Schott, Ferdinand Majerech 5 * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt Boost License 1.0) 6 */ 7 module visitor; 8 9 import std.algorithm; 10 import std.array: appender, empty, array, popBack, back, popFront, front; 11 import dparse.ast; 12 import dparse.lexer; 13 import std.file; 14 import std.path; 15 import std.stdio; 16 import std.string: format, join; 17 import std.typecons; 18 19 import config; 20 import ddoc.comments; 21 import item; 22 import symboldatabase; 23 import unittest_preprocessor; 24 import writer; 25 26 /** 27 * Generates documentation for a (single) module. 28 */ 29 class DocVisitor(Writer) : ASTVisitor 30 { 31 /** 32 * Params: 33 * 34 * config = Configuration data, including macros and the output directory. 35 * database = Stores information about modules and symbols for e.g. cross-referencing. 36 * unitTestMapping = The mapping of declaration addresses to their documentation unittests 37 * fileBytes = The source code of the module as a byte array. 38 * writer = Handles writing into generated files. 39 */ 40 this(ref const Config config, SymbolDatabase database, 41 TestRange[][size_t] unitTestMapping, const(ubyte[]) fileBytes, Writer writer) 42 { 43 this.config = &config; 44 this.database = database; 45 this.unitTestMapping = unitTestMapping; 46 this.fileBytes = fileBytes; 47 this.writer = writer; 48 49 this.writer.processCode = &crossReference; 50 } 51 52 override void visit(const Unittest){} 53 54 override void visit(const Module mod) 55 { 56 import std.conv : to; 57 import std.range: join; 58 assert(mod.moduleDeclaration !is null, "DataGatherVisitor should have caught this"); 59 pushAttributes(); 60 stack = cast(string[]) mod.moduleDeclaration.moduleName.identifiers.map!(a => a.text).array; 61 writer.prepareModule(stack); 62 63 moduleName = stack.join(".").to!string; 64 65 scope(exit) { writer.finishModule(); } 66 67 // The module is the first and only top-level "symbol". 68 bool dummyFirst; 69 string link; 70 auto fileWriter = writer.pushSymbol(stack, database, dummyFirst, link); 71 scope(exit) { writer.popSymbol(); } 72 73 writer.writeHeader(fileWriter, moduleName, stack.length - 1); 74 writer.writeBreadcrumbs(fileWriter, stack, database); 75 writer.writeTOC(fileWriter, moduleName); 76 writer.writeSymbolStart(fileWriter, link); 77 78 prevComments.length = 1; 79 80 const comment = mod.moduleDeclaration.comment; 81 memberStack.length = 1; 82 83 mod.accept(this); 84 85 writer.writeSymbolDescription(fileWriter, 86 { 87 memberStack.back.writePublicImports(fileWriter, writer); 88 89 if (comment.length) 90 { 91 writer.readAndWriteComment(fileWriter, comment, prevComments, 92 null, getUnittestDocTuple(mod.moduleDeclaration)); 93 } 94 }); 95 96 memberStack.back.write(fileWriter, writer); 97 writer.writeSymbolEnd(fileWriter); 98 } 99 100 override void visit(const EnumDeclaration ed) 101 { 102 enum formattingCode = q{ 103 fileWriter.put("enum " ~ ad.name.text); 104 if (ad.type !is null) 105 { 106 fileWriter.put(" : "); 107 formatter.format(ad.type); 108 } 109 }; 110 visitAggregateDeclaration!(formattingCode, "enums")(ed); 111 } 112 113 override void visit(const EnumMember member) 114 { 115 memberStack.back.values ~= Item("#", member.name.text, member.comment, null, member); 116 } 117 118 override void visit(const ClassDeclaration cd) 119 { 120 enum formattingCode = q{ 121 fileWriter.put("class " ~ ad.name.text); 122 if (ad.templateParameters !is null) 123 formatter.format(ad.templateParameters); 124 if (ad.baseClassList !is null) 125 formatter.format(ad.baseClassList); 126 if (ad.constraint !is null) 127 formatter.format(ad.constraint); 128 }; 129 visitAggregateDeclaration!(formattingCode, "classes")(cd); 130 } 131 132 override void visit(const TemplateDeclaration td) 133 { 134 enum formattingCode = q{ 135 fileWriter.put("template " ~ ad.name.text); 136 if (ad.templateParameters !is null) 137 formatter.format(ad.templateParameters); 138 if (ad.constraint) 139 formatter.format(ad.constraint); 140 }; 141 visitAggregateDeclaration!(formattingCode, "templates")(td); 142 } 143 144 override void visit(const StructDeclaration sd) 145 { 146 enum formattingCode = q{ 147 fileWriter.put("struct " ~ ad.name.text); 148 if (ad.templateParameters) 149 formatter.format(ad.templateParameters); 150 if (ad.constraint) 151 formatter.format(ad.constraint); 152 }; 153 visitAggregateDeclaration!(formattingCode, "structs")(sd); 154 } 155 156 override void visit(const InterfaceDeclaration id) 157 { 158 enum formattingCode = q{ 159 fileWriter.put("interface " ~ ad.name.text); 160 if (ad.templateParameters !is null) 161 formatter.format(ad.templateParameters); 162 if (ad.baseClassList !is null) 163 formatter.format(ad.baseClassList); 164 if (ad.constraint !is null) 165 formatter.format(ad.constraint); 166 }; 167 visitAggregateDeclaration!(formattingCode, "interfaces")(id); 168 } 169 170 override void visit(const AliasDeclaration ad) 171 { 172 if (ad.comment is null) 173 return; 174 bool first; 175 if (ad.declaratorIdentifierList !is null) 176 foreach (name; ad.declaratorIdentifierList.identifiers) 177 { 178 string itemURL; 179 auto fileWriter = pushSymbol(name.text, first, itemURL); 180 scope(exit) popSymbol(fileWriter); 181 182 string type, summary; 183 writer.writeSymbolDescription(fileWriter, 184 { 185 type = writeAliasType(fileWriter, name.text, ad.type); 186 summary = writer.readAndWriteComment(fileWriter, ad.comment, prevComments); 187 }); 188 189 memberStack[$ - 2].aliases ~= Item(itemURL, name.text, summary, type); 190 } 191 else foreach (initializer; ad.initializers) 192 { 193 string itemURL; 194 auto fileWriter = pushSymbol(initializer.name.text, first, itemURL); 195 scope(exit) popSymbol(fileWriter); 196 197 string type, summary; 198 writer.writeSymbolDescription(fileWriter, 199 { 200 type = writeAliasType(fileWriter, initializer.name.text, initializer.type); 201 summary = writer.readAndWriteComment(fileWriter, ad.comment, prevComments); 202 }); 203 204 memberStack[$ - 2].aliases ~= Item(itemURL, initializer.name.text, summary, type); 205 } 206 } 207 208 override void visit(const VariableDeclaration vd) 209 { 210 // Write the variable attributes, type, name. 211 void writeVariableHeader(R)(ref R dst, string typeStr, string nameStr, string defStr = "", string templStr = "") 212 { 213 writer.writeCodeBlock(dst, 214 { 215 assert(attributeStack.length > 0, 216 "Attributes stack must not be empty when writing variable attributes"); 217 auto formatter = writer.newFormatter(dst); 218 scope(exit) { destroy(formatter.sink); } 219 // Attributes like public, etc. 220 writeAttributes(dst, formatter, attributeStack.back); 221 dst.put(typeStr); 222 dst.put(` `); 223 dst.put(nameStr); 224 if (templStr.length) 225 dst.put(templStr); 226 if (defStr.length) 227 dst.put(defStr); 228 }); 229 } 230 bool first; 231 foreach (const Declarator dec; vd.declarators) 232 { 233 if (vd.comment is null && dec.comment is null) 234 continue; 235 string itemURL; 236 auto fileWriter = pushSymbol(dec.name.text, first, itemURL); 237 scope(exit) popSymbol(fileWriter); 238 239 string typeStr = writer.formatNode(vd.type); 240 string summary; 241 writer.writeSymbolDescription(fileWriter, 242 { 243 import dparse.formatter : fmt = format; 244 import std.array : Appender; 245 string defStr = dec.initializer 246 ? { Appender!string app; app.put(" = "); fmt(&app, dec.initializer); return app.data; }() 247 : ""; 248 string templStr = dec.templateParameters 249 ? { Appender!string app; fmt(&app, dec.templateParameters); return app.data;}() 250 : ""; 251 writeVariableHeader(fileWriter, typeStr, dec.name.text, defStr, templStr); 252 summary = writer.readAndWriteComment(fileWriter, 253 dec.comment is null ? vd.comment : dec.comment, 254 prevComments); 255 }); 256 257 memberStack[$ - 2].variables ~= Item(itemURL, dec.name.text, summary, typeStr, dec); 258 } 259 if (vd.comment !is null && vd.autoDeclaration !is null) 260 { 261 foreach (part; vd.autoDeclaration.parts) with (part) 262 { 263 string itemURL; 264 auto fileWriter = pushSymbol(identifier.text, first, itemURL); 265 scope(exit) popSymbol(fileWriter); 266 267 // TODO this was hastily updated to get harbored-mod to compile 268 // after a libdparse update. Revisit and validate/fix any errors. 269 string[] storageClasses; 270 vd.storageClasses.each!(stor => storageClasses ~= str(stor.token.type)); 271 272 string typeStr = storageClasses.canFind("enum") ? null : "auto"; 273 string summary; 274 writer.writeSymbolDescription(fileWriter, 275 { 276 writeVariableHeader(fileWriter, typeStr, identifier.text); 277 summary = writer.readAndWriteComment(fileWriter, vd.comment, prevComments); 278 }); 279 auto i = Item(itemURL, identifier.text, summary, typeStr); 280 if (storageClasses.canFind("enum")) 281 memberStack[$ - 2].enums ~= i; 282 else 283 memberStack[$ - 2].variables ~= i; 284 285 // string storageClass; 286 // foreach (attr; vd.attributes) 287 // { 288 // if (attr.storageClass !is null) 289 // storageClass = str(attr.storageClass.token.type); 290 // } 291 // auto i = Item(name, ident.text, 292 // summary, storageClass == "enum" ? null : "auto"); 293 // if (storageClass == "enum") 294 // memberStack[$ - 2].enums ~= i; 295 // else 296 // memberStack[$ - 2].variables ~= i; 297 } 298 } 299 } 300 301 override void visit(const StructBody sb) 302 { 303 pushAttributes(); 304 sb.accept(this); 305 popAttributes(); 306 } 307 308 override void visit(const BlockStatement bs) 309 { 310 pushAttributes(); 311 bs.accept(this); 312 popAttributes(); 313 } 314 315 override void visit(const Declaration dec) 316 { 317 attributeStack.back ~= dec.attributes; 318 dec.accept(this); 319 if (dec.attributeDeclaration is null) 320 attributeStack.back = attributeStack.back[0 .. $ - dec.attributes.length]; 321 } 322 323 override void visit(const AttributeDeclaration dec) 324 { 325 attributeStack.back ~= dec.attribute; 326 } 327 328 override void visit(const Constructor cons) 329 { 330 if (cons.comment is null) 331 return; 332 writeFnDocumentation("this", cons, attributeStack.back); 333 } 334 335 override void visit(const FunctionDeclaration fd) 336 { 337 if (fd.comment is null) 338 return; 339 writeFnDocumentation(fd.name.text, fd, attributeStack.back); 340 } 341 342 override void visit(const ImportDeclaration imp) 343 { 344 // public attribute must be specified explicitly for public imports. 345 foreach (attr; attributeStack.back) 346 if (attr.attribute.type == tok!"public") 347 { 348 foreach (i; imp.singleImports) 349 { 350 const nameParts = i.identifierChain.identifiers.map!(t => t.text).array; 351 const name = nameParts.join("."); 352 const knownModule = database.moduleNames.canFind(name); 353 const link = knownModule ? writer.moduleLink(nameParts) : null; 354 memberStack.back.publicImports ~= 355 Item(link, name, null, null, imp); 356 } 357 return; 358 } 359 //TODO handle imp.importBindings as well? Need to figure out how it works. 360 } 361 362 // Optimization: don't allow visit() for these AST nodes to result in visit() 363 // calls for their subnodes. This avoids most of the dynamic cast overhead. 364 override void visit(const AssignExpression assignExpression) {} 365 override void visit(const CmpExpression cmpExpression) {} 366 override void visit(const TernaryExpression ternaryExpression) {} 367 override void visit(const IdentityExpression identityExpression) {} 368 override void visit(const InExpression inExpression) {} 369 370 alias visit = ASTVisitor.visit; 371 372 private: 373 /// Get the current protection attribute. 374 IdType currentProtection() 375 out(result) 376 { 377 assert([tok!"private", tok!"package", tok!"protected", tok!"public"].canFind(result), 378 "Unknown protection attribute"); 379 } 380 body 381 { 382 foreach (a; attributeStack.back.filter!(a => a.attribute.type.isProtection)) 383 { 384 return a.attribute.type; 385 } 386 return tok!"public"; 387 } 388 389 /** Writes attributes to the range dst using formatter to format code. 390 * 391 * Params: 392 * 393 * dst = Range to write to. 394 * formatter = Formatter to format the attributes with. 395 * attrs = Attributes to write. 396 */ 397 void writeAttributes(R, F)(ref R dst, F formatter, const(Attribute)[] attrs) 398 { 399 import dparse.lexer : isProtection, tok; 400 switch (currentProtection()) 401 { 402 case tok!"private": dst.put("private "); break; 403 case tok!"package": dst.put("package "); break; 404 case tok!"protected": dst.put("protected "); break; 405 default: dst.put("public "); break; 406 } 407 foreach (a; attrs.filter!(a => !a.attribute.type.isProtection)) 408 { 409 formatter.format(a); 410 dst.put(" "); 411 } 412 } 413 414 415 void visitAggregateDeclaration(string formattingCode, string name, A)(const A ad) 416 { 417 bool first; 418 if (ad.comment is null) 419 return; 420 421 string itemURL; 422 auto fileWriter = pushSymbol(ad.name.text, first, itemURL); 423 scope(exit) popSymbol(fileWriter); 424 425 string summary; 426 writer.writeSymbolDescription(fileWriter, 427 { 428 writer.writeCodeBlock(fileWriter, 429 { 430 auto formatter = writer.newFormatter(fileWriter); 431 scope(exit) destroy(formatter.sink); 432 assert(attributeStack.length > 0, 433 "Attributes stack must not be empty when writing aggregate attributes"); 434 writeAttributes(fileWriter, formatter, attributeStack.back); 435 mixin(formattingCode); 436 }); 437 438 summary = writer.readAndWriteComment(fileWriter, ad.comment, prevComments, 439 null, getUnittestDocTuple(ad)); 440 }); 441 442 mixin(`memberStack[$ - 2].` ~ name ~ ` ~= Item(itemURL, ad.name.text, summary);`); 443 444 prevComments.length = prevComments.length + 1; 445 ad.accept(this); 446 prevComments.popBack(); 447 448 memberStack.back.write(fileWriter, writer); 449 } 450 451 /** 452 * Params: 453 * t = The declaration. 454 * Returns: An array of tuples where the first item is the contents of the 455 * unittest block and the second item is the doc comment for the 456 * unittest block. This array may be empty. 457 */ 458 Tuple!(string, string)[] getUnittestDocTuple(T)(const T t) 459 { 460 immutable size_t index = cast(size_t) (cast(void*) t); 461 // writeln("Searching for unittest associated with ", index); 462 auto tupArray = index in unitTestMapping; 463 if (tupArray is null) 464 return []; 465 // writeln("Found a doc unit test for ", cast(size_t) &t); 466 Tuple!(string, string)[] rVal; 467 foreach (tup; *tupArray) 468 rVal ~= tuple(cast(string) fileBytes[tup[0] + 2 .. tup[1]], tup[2]); 469 return rVal; 470 } 471 472 /** 473 * 474 */ 475 void writeFnDocumentation(Fn)(string name, Fn fn, const(Attribute)[] attrs) 476 { 477 bool first; 478 string itemURL; 479 auto fileWriter = pushSymbol(name, first, itemURL); 480 scope(exit) popSymbol(fileWriter); 481 482 string summary; 483 writer.writeSymbolDescription(fileWriter, 484 { 485 auto formatter = writer.newFormatter(fileWriter); 486 scope(exit) destroy(formatter.sink); 487 488 // Write the function signature. 489 writer.writeCodeBlock(fileWriter, 490 { 491 assert(attributeStack.length > 0, 492 "Attributes stack must not be empty when writing " ~ 493 "function attributes"); 494 // Attributes like public, etc. 495 writeAttributes(fileWriter, formatter, attrs); 496 // Return type and function name, with special case fo constructor 497 static if (__traits(hasMember, typeof(fn), "returnType")) 498 { 499 if (fn.returnType) 500 { 501 formatter.format(fn.returnType); 502 fileWriter.put(" "); 503 } 504 formatter.format(fn.name); 505 } 506 else 507 { 508 fileWriter.put("this"); 509 } 510 // Template params 511 if (fn.templateParameters !is null) 512 formatter.format(fn.templateParameters); 513 // Function params 514 if (fn.parameters !is null) 515 formatter.format(fn.parameters); 516 // Attributes like const, nothrow, etc. 517 foreach (a; fn.memberFunctionAttributes) 518 { 519 fileWriter.put(" "); 520 formatter.format(a); 521 } 522 // Template constraint 523 if (fn.constraint) 524 { 525 fileWriter.put(" "); 526 formatter.format(fn.constraint); 527 } 528 }); 529 530 summary = writer.readAndWriteComment(fileWriter, fn.comment, 531 prevComments, fn.functionBody, getUnittestDocTuple(fn)); 532 }); 533 string fdName; 534 static if (__traits(hasMember, typeof(fn), "name")) 535 fdName = fn.name.text; 536 else 537 fdName = "this"; 538 auto fnItem = Item(itemURL, fdName, summary, null, fn); 539 memberStack[$ - 2].functions ~= fnItem; 540 prevComments.length = prevComments.length + 1; 541 fn.accept(this); 542 543 // The function may have nested functions/classes/etc, so at the very 544 // least we need to close their files, and once public/private works even 545 // document them. 546 memberStack.back.write(fileWriter, writer); 547 prevComments.popBack(); 548 } 549 550 /** 551 * Writes an alias' type to the given range and returns it. 552 * Params: 553 * dst = The range to write to 554 * name = the name of the alias 555 * t = the aliased type 556 * Returns: A string reperesentation of the given type. 557 */ 558 string writeAliasType(R)(ref R dst, string name, const Type t) 559 { 560 if (t is null) 561 return null; 562 string formatted = writer.formatNode(t); 563 writer.writeCodeBlock(dst, 564 { 565 dst.put("alias %s = ".format(name)); 566 dst.put(formatted); 567 }); 568 return formatted; 569 } 570 571 572 /** Generate links from symbols in input to files documenting those symbols. 573 * 574 * Note: The current implementation is far from perfect. It doesn't try to parse 575 * input; it just searches for alphanumeric words and patterns like 576 * "alnumword.otheralnumword" and asks SymbolDatabase to find a reference to them. 577 * 578 * TODO: Improve this by trying to parse input as D code first, only falling back 579 * to current implementation if the parsing fails. Parsing would only be used to 580 * correctly detect names, but must not reformat any code from input. 581 * 582 * Params: 583 * 584 * input = String to find symbols in. 585 * 586 * Returns: 587 * 588 * string with symbols replaced by links (links' format depends on Writer). 589 */ 590 string crossReference(string input) @trusted nothrow 591 { 592 import std.ascii : isAlphaNum; 593 bool isNameCharacter(dchar c) 594 { 595 char c8 = cast(char)c; 596 return c8 == c && (c8.isAlphaNum || "_.".canFind(c8)); 597 } 598 599 auto app = appender!string(); 600 dchar prevC = '\0'; 601 dchar c; 602 603 // Scan a symbol name. When done, both c and input.front will be set to 604 // the first character after the name. 605 string scanName() 606 { 607 auto scanApp = appender!string(); 608 while(!input.empty) 609 { 610 c = input.front; 611 if (!isNameCharacter(c) && isNameCharacter(prevC)) 612 break; 613 614 scanApp.put(c); 615 prevC = c; 616 input.popFront(); 617 } 618 return scanApp.data; 619 } 620 621 // There should be no UTF decoding errors as we validate text when loading 622 // with std.file.readText(). 623 try while(!input.empty) 624 { 625 c = input.front; 626 if (isNameCharacter(c) && !isNameCharacter(prevC)) 627 { 628 string name = scanName(); 629 630 auto link = database.crossReference(writer, stack, name); 631 size_t partIdx = 0; 632 633 if (link.length) 634 writer.writeLink(app, link, { app.put(name); }); 635 // Attempt to cross-reference individual parts of the name 636 // (e.g. "variable.method" will not match anything if 637 // "variable" is a local variable "method" by itself may 638 // still match something) 639 else foreach (part; name.splitter(".")) 640 { 641 if (partIdx++ > 0) 642 app.put("."); 643 644 link = database.crossReference(writer, stack, part); 645 if (link.length) 646 writer.writeLink(app, link, { app.put(part); }); 647 else 648 app.put(part); 649 } 650 } 651 652 if (input.empty) 653 break; 654 655 // Even if scanName was called above, c is the first character 656 // *after* scanName. 657 app.put(c); 658 prevC = c; 659 // Must check again because scanName might have exhausted the input. 660 input.popFront(); 661 } 662 catch(Exception e) 663 { 664 import std.exception: assumeWontThrow; 665 writeln("Unexpected exception when cross-referencing: ", e.msg) 666 .assumeWontThrow; 667 } 668 669 return app.data; 670 } 671 672 /** 673 * Params: 674 * 675 * name = The symbol's name 676 * first = Set to true if this is the first time that pushSymbol has been 677 * called for this name. 678 * itemURL = URL to use in the Item for this symbol will be written here. 679 * 680 * Returns: A range to write the symbol's documentation to. 681 */ 682 auto pushSymbol(string name, ref bool first, ref string itemURL) 683 { 684 import std.array : array, join; 685 import std.string : format; 686 stack ~= name; 687 memberStack.length = memberStack.length + 1; 688 689 // Sets first 690 auto result = writer.pushSymbol(stack, database, first, itemURL); 691 692 if (first) 693 { 694 writer.writeHeader(result, name, writer.moduleNameLength); 695 writer.writeBreadcrumbs(result, stack, database); 696 writer.writeTOC(result, moduleName); 697 } 698 else 699 { 700 writer.writeSeparator(result); 701 } 702 writer.writeSymbolStart(result, itemURL); 703 return result; 704 } 705 706 void popSymbol(R)(ref R dst) 707 { 708 writer.writeSymbolEnd(dst); 709 stack.popBack(); 710 memberStack.popBack(); 711 writer.popSymbol(); 712 } 713 714 void pushAttributes() { attributeStack.length = attributeStack.length + 1; } 715 716 void popAttributes() { attributeStack.popBack(); } 717 718 719 /// The module name in "package.package.module" format. 720 string moduleName; 721 722 const(Attribute)[][] attributeStack; 723 Comment[] prevComments; 724 /** Namespace stack of the current symbol, 725 * 726 * E.g. ["package", "subpackage", "module", "Class", "member"] 727 */ 728 string[] stack; 729 /** Every item of this stack corresponds to a parent module/class/etc of the 730 * current symbol, but not package. 731 * 732 * Each Members struct is used to accumulate all members of that module/class/etc 733 * so the list of all members can be generated. 734 */ 735 Members[] memberStack; 736 TestRange[][size_t] unitTestMapping; 737 const(ubyte[]) fileBytes; 738 const(Config)* config; 739 /// Information about modules and symbols for e.g. cross-referencing. 740 SymbolDatabase database; 741 Writer writer; 742 }