1 /** 2 * D Documentation Generator 3 * Copyright: © 2014 Economic Modeling Specialists, Intl., © 2015 Ferdinand Majerech 4 * Authors: Brian Schott, Ferdinand Majerech 5 * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt Boost License 1.0) 6 */ 7 module visitor; 8 9 import std.algorithm; 10 import std.array: appender, empty, array, popBack, back, popFront, front; 11 import dparse.ast; 12 import dparse.lexer; 13 import std.file; 14 import std.path; 15 import std.stdio; 16 import std.string: format; 17 import std.typecons; 18 19 import config; 20 import ddoc.comments; 21 import item; 22 import symboldatabase; 23 import unittest_preprocessor; 24 import writer; 25 26 /** 27 * Generates documentation for a (single) module. 28 */ 29 class DocVisitor(Writer) : ASTVisitor 30 { 31 /** 32 * Params: 33 * 34 * config = Configuration data, including macros and the output directory. 35 * database = Stores information about modules and symbols for e.g. cross-referencing. 36 * unitTestMapping = The mapping of declaration addresses to their documentation unittests 37 * fileBytes = The source code of the module as a byte array. 38 * writer = Handles writing into generated files. 39 */ 40 this(ref const Config config, SymbolDatabase database, 41 TestRange[][size_t] unitTestMapping, const(ubyte[]) fileBytes, Writer writer) 42 { 43 this.config = &config; 44 this.database = database; 45 this.unitTestMapping = unitTestMapping; 46 this.fileBytes = fileBytes; 47 this.writer = writer; 48 49 this.writer.processCode = &crossReference; 50 } 51 52 override void visit(const Unittest){} 53 54 override void visit(const Module mod) 55 { 56 import std.conv : to; 57 import std.range: join; 58 assert(mod.moduleDeclaration !is null, "DataGatherVisitor should have caught this"); 59 pushAttributes(); 60 stack = cast(string[]) mod.moduleDeclaration.moduleName.identifiers.map!(a => a.text).array; 61 writer.prepareModule(stack); 62 63 moduleName = stack.join(".").to!string; 64 65 scope(exit) { writer.finishModule(); } 66 67 // The module is the first and only top-level "symbol". 68 bool dummyFirst; 69 string link; 70 auto fileWriter = writer.pushSymbol(stack, database, dummyFirst, link); 71 scope(exit) { writer.popSymbol(); } 72 73 writer.writeHeader(fileWriter, moduleName, stack.length - 1); 74 writer.writeBreadcrumbs(fileWriter, stack, database); 75 writer.writeTOC(fileWriter, moduleName); 76 writer.writeSymbolStart(fileWriter, link); 77 78 prevComments.length = 1; 79 80 const comment = mod.moduleDeclaration.comment; 81 memberStack.length = 1; 82 83 mod.accept(this); 84 85 writer.writeSymbolDescription(fileWriter, 86 { 87 memberStack.back.writePublicImports(fileWriter, writer); 88 89 if (comment !is null) 90 { 91 writer.readAndWriteComment(fileWriter, comment, prevComments, 92 null, getUnittestDocTuple(mod.moduleDeclaration)); 93 } 94 }); 95 96 memberStack.back.write(fileWriter, writer); 97 writer.writeSymbolEnd(fileWriter); 98 } 99 100 override void visit(const EnumDeclaration ed) 101 { 102 enum formattingCode = q{ 103 fileWriter.put("enum " ~ ad.name.text); 104 if (ad.type !is null) 105 { 106 fileWriter.put(" : "); 107 formatter.format(ad.type); 108 } 109 }; 110 visitAggregateDeclaration!(formattingCode, "enums")(ed); 111 } 112 113 override void visit(const EnumMember member) 114 { 115 // Document all enum members even if they have no doc comments. 116 if (member.comment is null) 117 { 118 memberStack.back.values ~= Item("#", member.name.text, ""); 119 return; 120 } 121 auto dummy = appender!string(); 122 // No interest in detailed docs for an enum member. 123 string summary = writer.readAndWriteComment(dummy, member.comment, 124 prevComments, null, getUnittestDocTuple(member)); 125 memberStack.back.values ~= Item("#", member.name.text, summary); 126 } 127 128 override void visit(const ClassDeclaration cd) 129 { 130 enum formattingCode = q{ 131 fileWriter.put("class " ~ ad.name.text); 132 if (ad.templateParameters !is null) 133 formatter.format(ad.templateParameters); 134 if (ad.baseClassList !is null) 135 formatter.format(ad.baseClassList); 136 if (ad.constraint !is null) 137 formatter.format(ad.constraint); 138 }; 139 visitAggregateDeclaration!(formattingCode, "classes")(cd); 140 } 141 142 override void visit(const TemplateDeclaration td) 143 { 144 enum formattingCode = q{ 145 fileWriter.put("template " ~ ad.name.text); 146 if (ad.templateParameters !is null) 147 formatter.format(ad.templateParameters); 148 if (ad.constraint) 149 formatter.format(ad.constraint); 150 }; 151 visitAggregateDeclaration!(formattingCode, "templates")(td); 152 } 153 154 override void visit(const StructDeclaration sd) 155 { 156 enum formattingCode = q{ 157 fileWriter.put("struct " ~ ad.name.text); 158 if (ad.templateParameters) 159 formatter.format(ad.templateParameters); 160 if (ad.constraint) 161 formatter.format(ad.constraint); 162 }; 163 visitAggregateDeclaration!(formattingCode, "structs")(sd); 164 } 165 166 override void visit(const InterfaceDeclaration id) 167 { 168 enum formattingCode = q{ 169 fileWriter.put("interface " ~ ad.name.text); 170 if (ad.templateParameters !is null) 171 formatter.format(ad.templateParameters); 172 if (ad.baseClassList !is null) 173 formatter.format(ad.baseClassList); 174 if (ad.constraint !is null) 175 formatter.format(ad.constraint); 176 }; 177 visitAggregateDeclaration!(formattingCode, "interfaces")(id); 178 } 179 180 override void visit(const AliasDeclaration ad) 181 { 182 if (ad.comment is null) 183 return; 184 bool first; 185 if (ad.declaratorIdentifierList !is null) 186 foreach (name; ad.declaratorIdentifierList.identifiers) 187 { 188 string itemURL; 189 auto fileWriter = pushSymbol(name.text, first, itemURL); 190 scope(exit) popSymbol(fileWriter); 191 192 string type, summary; 193 writer.writeSymbolDescription(fileWriter, 194 { 195 type = writeAliasType(fileWriter, name.text, ad.type); 196 summary = writer.readAndWriteComment(fileWriter, ad.comment, prevComments); 197 }); 198 199 memberStack[$ - 2].aliases ~= Item(itemURL, name.text, summary, type); 200 } 201 else foreach (initializer; ad.initializers) 202 { 203 string itemURL; 204 auto fileWriter = pushSymbol(initializer.name.text, first, itemURL); 205 scope(exit) popSymbol(fileWriter); 206 207 string type, summary; 208 writer.writeSymbolDescription(fileWriter, 209 { 210 type = writeAliasType(fileWriter, initializer.name.text, initializer.type); 211 summary = writer.readAndWriteComment(fileWriter, ad.comment, prevComments); 212 }); 213 214 memberStack[$ - 2].aliases ~= Item(itemURL, initializer.name.text, summary, type); 215 } 216 } 217 218 override void visit(const VariableDeclaration vd) 219 { 220 // Write the variable attributes, type, name. 221 void writeVariableHeader(R)(ref R dst, string typeStr, string nameStr, string defStr = "") 222 { 223 writer.writeCodeBlock(dst, 224 { 225 assert(attributeStack.length > 0, 226 "Attributes stack must not be empty when writing variable attributes"); 227 auto formatter = writer.newFormatter(dst); 228 scope(exit) { destroy(formatter.sink); } 229 // Attributes like public, etc. 230 writeAttributes(dst, formatter, attributeStack.back); 231 dst.put(typeStr); 232 dst.put(` `); 233 dst.put(nameStr); 234 dst.put(defStr); 235 }); 236 } 237 bool first; 238 foreach (const Declarator dec; vd.declarators) 239 { 240 if (vd.comment is null && dec.comment is null) 241 continue; 242 string itemURL; 243 auto fileWriter = pushSymbol(dec.name.text, first, itemURL); 244 scope(exit) popSymbol(fileWriter); 245 246 string typeStr = writer.formatNode(vd.type); 247 string summary; 248 writer.writeSymbolDescription(fileWriter, 249 { 250 string defStr; 251 if (dec.initializer) 252 { 253 import dparse.formatter : fmt = format; 254 import std.array : Appender; 255 Appender!string app; 256 app.put(" = "); 257 fmt(&app, dec.initializer); 258 defStr = app.data; 259 } 260 writeVariableHeader(fileWriter, typeStr, dec.name.text, defStr); 261 summary = writer.readAndWriteComment(fileWriter, 262 dec.comment is null ? vd.comment : dec.comment, 263 prevComments); 264 }); 265 266 memberStack[$ - 2].variables ~= Item(itemURL, dec.name.text, summary, typeStr, dec); 267 } 268 if (vd.comment !is null && vd.autoDeclaration !is null) 269 { 270 foreach (part; vd.autoDeclaration.parts) with (part) 271 { 272 string itemURL; 273 auto fileWriter = pushSymbol(identifier.text, first, itemURL); 274 scope(exit) popSymbol(fileWriter); 275 276 // TODO this was hastily updated to get hgen to compile 277 // after a libdparse update. Revisit and validate/fix any errors. 278 string[] storageClasses; 279 foreach(stor; vd.storageClasses) 280 { 281 storageClasses ~= str(stor.token.type); 282 } 283 284 string typeStr = storageClasses.canFind("enum") ? null : "auto"; 285 string summary; 286 writer.writeSymbolDescription(fileWriter, 287 { 288 writeVariableHeader(fileWriter, typeStr, identifier.text); 289 summary = writer.readAndWriteComment(fileWriter, vd.comment, prevComments); 290 }); 291 auto i = Item(itemURL, identifier.text, summary, typeStr); 292 if (storageClasses.canFind("enum")) 293 memberStack[$ - 2].enums ~= i; 294 else 295 memberStack[$ - 2].variables ~= i; 296 297 // string storageClass; 298 // foreach (attr; vd.attributes) 299 // { 300 // if (attr.storageClass !is null) 301 // storageClass = str(attr.storageClass.token.type); 302 // } 303 // auto i = Item(name, ident.text, 304 // summary, storageClass == "enum" ? null : "auto"); 305 // if (storageClass == "enum") 306 // memberStack[$ - 2].enums ~= i; 307 // else 308 // memberStack[$ - 2].variables ~= i; 309 } 310 } 311 } 312 313 override void visit(const StructBody sb) 314 { 315 pushAttributes(); 316 sb.accept(this); 317 popAttributes(); 318 } 319 320 override void visit(const BlockStatement bs) 321 { 322 pushAttributes(); 323 bs.accept(this); 324 popAttributes(); 325 } 326 327 override void visit(const Declaration dec) 328 { 329 attributeStack.back ~= dec.attributes; 330 dec.accept(this); 331 if (dec.attributeDeclaration is null) 332 attributeStack.back = attributeStack.back[0 .. $ - dec.attributes.length]; 333 } 334 335 override void visit(const AttributeDeclaration dec) 336 { 337 attributeStack.back ~= dec.attribute; 338 } 339 340 override void visit(const Constructor cons) 341 { 342 if (cons.comment is null) 343 return; 344 writeFnDocumentation("this", cons, attributeStack.back); 345 } 346 347 override void visit(const FunctionDeclaration fd) 348 { 349 if (fd.comment is null) 350 return; 351 writeFnDocumentation(fd.name.text, fd, attributeStack.back); 352 } 353 354 override void visit(const ImportDeclaration imp) 355 { 356 // public attribute must be specified explicitly for public imports. 357 foreach(attr; attributeStack.back) if(attr.attribute.type == tok!"public") 358 { 359 foreach(i; imp.singleImports) 360 { 361 import std.conv; 362 // Using 'dup' here because of std.algorithm's apparent 363 // inability to work with const arrays. Probably not an 364 // issue (imports are not hugely common), but keep the 365 // possible GC overhead in mind. 366 auto nameParts = i.identifierChain.identifiers 367 .dup.map!(t => t.text).array; 368 const name = nameParts.joiner(".").to!string; 369 370 const knownModule = database.moduleNames.canFind(name); 371 const link = knownModule ? writer.moduleLink(nameParts) 372 : null; 373 memberStack.back.publicImports ~= 374 Item(link, name, null, null, imp); 375 } 376 return; 377 } 378 //TODO handle imp.importBindings as well? Need to figure out how it works. 379 } 380 381 // Optimization: don't allow visit() for these AST nodes to result in visit() 382 // calls for their subnodes. This avoids most of the dynamic cast overhead. 383 override void visit(const AssignExpression assignExpression) {} 384 override void visit(const CmpExpression cmpExpression) {} 385 override void visit(const TernaryExpression ternaryExpression) {} 386 override void visit(const IdentityExpression identityExpression) {} 387 override void visit(const InExpression inExpression) {} 388 389 alias visit = ASTVisitor.visit; 390 391 private: 392 /// Get the current protection attribute. 393 IdType currentProtection() 394 out(result) 395 { 396 assert([tok!"private", tok!"package", tok!"protected", tok!"public"].canFind(result), 397 "Unknown protection attribute"); 398 } 399 body 400 { 401 foreach(a; attributeStack.back.filter!(a => a.attribute.type.isProtection)) 402 { 403 return a.attribute.type; 404 } 405 return tok!"public"; 406 } 407 408 /** Writes attributes to the range dst using formatter to format code. 409 * 410 * Params: 411 * 412 * dst = Range to write to. 413 * formatter = Formatter to format the attributes with. 414 * attrs = Attributes to write. 415 */ 416 final void writeAttributes(R, F)(ref R dst, F formatter, const(Attribute)[] attrs) 417 { 418 import dparse.lexer: IdType, isProtection, tok; 419 IdType protection = currentProtection(); 420 switch (protection) 421 { 422 case tok!"private": dst.put("private "); break; 423 case tok!"package": dst.put("package "); break; 424 case tok!"protected": dst.put("protected "); break; 425 default: dst.put(""); break; 426 } 427 foreach (a; attrs.filter!(a => !a.attribute.type.isProtection)) 428 { 429 formatter.format(a); 430 dst.put(" "); 431 } 432 } 433 434 435 void visitAggregateDeclaration(string formattingCode, string name, A)(const A ad) 436 { 437 bool first; 438 if (ad.comment is null) 439 return; 440 441 string itemURL; 442 auto fileWriter = pushSymbol(ad.name.text, first, itemURL); 443 scope(exit) popSymbol(fileWriter); 444 445 string summary; 446 writer.writeSymbolDescription(fileWriter, 447 { 448 writer.writeCodeBlock(fileWriter, 449 { 450 auto formatter = writer.newFormatter(fileWriter); 451 scope(exit) destroy(formatter.sink); 452 assert(attributeStack.length > 0, 453 "Attributes stack must not be empty when writing aggregate attributes"); 454 writeAttributes(fileWriter, formatter, attributeStack.back); 455 mixin(formattingCode); 456 }); 457 458 summary = writer.readAndWriteComment(fileWriter, ad.comment, prevComments, 459 null, getUnittestDocTuple(ad)); 460 }); 461 462 mixin(`memberStack[$ - 2].` ~ name ~ ` ~= Item(itemURL, ad.name.text, summary);`); 463 464 prevComments.length = prevComments.length + 1; 465 ad.accept(this); 466 prevComments.popBack(); 467 468 memberStack.back.write(fileWriter, writer); 469 } 470 471 /** 472 * Params: 473 * t = The declaration. 474 * Returns: An array of tuples where the first item is the contents of the 475 * unittest block and the second item is the doc comment for the 476 * unittest block. This array may be empty. 477 */ 478 Tuple!(string, string)[] getUnittestDocTuple(T)(const T t) 479 { 480 immutable size_t index = cast(size_t) (cast(void*) t); 481 // writeln("Searching for unittest associated with ", index); 482 auto tupArray = index in unitTestMapping; 483 if (tupArray is null) 484 return []; 485 // writeln("Found a doc unit test for ", cast(size_t) &t); 486 Tuple!(string, string)[] rVal; 487 foreach (tup; *tupArray) 488 rVal ~= tuple(cast(string) fileBytes[tup[0] + 2 .. tup[1]], tup[2]); 489 return rVal; 490 } 491 492 /** 493 * 494 */ 495 void writeFnDocumentation(Fn)(string name, Fn fn, const(Attribute)[] attrs) 496 { 497 bool first; 498 string itemURL; 499 auto fileWriter = pushSymbol(name, first, itemURL); 500 scope(exit) popSymbol(fileWriter); 501 502 string summary; 503 writer.writeSymbolDescription(fileWriter, 504 { 505 auto formatter = writer.newFormatter(fileWriter); 506 scope(exit) destroy(formatter.sink); 507 508 // Write the function signature. 509 writer.writeCodeBlock(fileWriter, 510 { 511 assert(attributeStack.length > 0, 512 "Attributes stack must not be empty when writing " ~ 513 "function attributes"); 514 // Attributes like public, etc. 515 writeAttributes(fileWriter, formatter, attrs); 516 // Return type and function name, with special case fo constructor 517 static if (__traits(hasMember, typeof(fn), "returnType")) 518 { 519 if (fn.returnType) 520 { 521 formatter.format(fn.returnType); 522 fileWriter.put(" "); 523 } 524 formatter.format(fn.name); 525 } 526 else 527 { 528 fileWriter.put("this"); 529 } 530 // Template params 531 if (fn.templateParameters !is null) 532 formatter.format(fn.templateParameters); 533 // Function params 534 if (fn.parameters !is null) 535 formatter.format(fn.parameters); 536 // Attributes like const, nothrow, etc. 537 foreach (a; fn.memberFunctionAttributes) 538 { 539 fileWriter.put(" "); 540 formatter.format(a); 541 } 542 // Template constraint 543 if (fn.constraint) 544 { 545 fileWriter.put(" "); 546 formatter.format(fn.constraint); 547 } 548 }); 549 550 summary = writer.readAndWriteComment(fileWriter, fn.comment, 551 prevComments, fn.functionBody, getUnittestDocTuple(fn)); 552 }); 553 string fdName; 554 static if (__traits(hasMember, typeof(fn), "name")) 555 fdName = fn.name.text; 556 else 557 fdName = "this"; 558 auto fnItem = Item(itemURL, fdName, summary, null, fn); 559 memberStack[$ - 2].functions ~= fnItem; 560 prevComments.length = prevComments.length + 1; 561 fn.accept(this); 562 563 // The function may have nested functions/classes/etc, so at the very 564 // least we need to close their files, and once public/private works even 565 // document them. 566 memberStack.back.write(fileWriter, writer); 567 prevComments.popBack(); 568 } 569 570 /** 571 * Writes an alias' type to the given range and returns it. 572 * Params: 573 * dst = The range to write to 574 * name = the name of the alias 575 * t = the aliased type 576 * Returns: A string reperesentation of the given type. 577 */ 578 string writeAliasType(R)(ref R dst, string name, const Type t) 579 { 580 if (t is null) 581 return null; 582 string formatted = writer.formatNode(t); 583 writer.writeCodeBlock(dst, 584 { 585 dst.put("alias %s = ".format(name)); 586 dst.put(formatted); 587 }); 588 return formatted; 589 } 590 591 592 /** Generate links from symbols in input to files documenting those symbols. 593 * 594 * Note: The current implementation is far from perfect. It doesn't try to parse 595 * input; it just searches for alphanumeric words and patterns like 596 * "alnumword.otheralnumword" and asks SymbolDatabase to find a reference to them. 597 * 598 * TODO: Improve this by trying to parse input as D code first, only falling back 599 * to current implementation if the parsing fails. Parsing would only be used to 600 * correctly detect names, but must not reformat any code from input. 601 * 602 * Params: 603 * 604 * input = String to find symbols in. 605 * 606 * Returns: 607 * 608 * string with symbols replaced by links (links' format depends on Writer). 609 */ 610 string crossReference(string input) @trusted nothrow 611 { 612 import std.ascii; 613 bool isNameCharacter(dchar c) 614 { 615 char c8 = cast(char)c; 616 return c8 == c && (c8.isAlphaNum || "_.".canFind(c8)); 617 } 618 619 auto app = appender!string(); 620 dchar prevC = '\0'; 621 dchar c; 622 623 // Scan a symbol name. When done, both c and input.front will be set to 624 // the first character after the name. 625 string scanName() 626 { 627 auto scanApp = appender!string(); 628 while(!input.empty) 629 { 630 c = input.front; 631 if(!isNameCharacter(c) && isNameCharacter(prevC)) { break; } 632 633 scanApp.put(c); 634 prevC = c; 635 input.popFront(); 636 } 637 return scanApp.data; 638 } 639 640 // There should be no UTF decoding errors as we validate text when loading 641 // with std.file.readText(). 642 try while(!input.empty) 643 { 644 c = input.front; 645 if(isNameCharacter(c) && !isNameCharacter(prevC)) 646 { 647 string name = scanName(); 648 649 auto link = database.crossReference(writer, stack, name); 650 size_t partIdx = 0; 651 652 if(link !is null) writer.writeLink(app, link, { app.put(name); }); 653 // Attempt to cross-reference individual parts of the name 654 // (e.g. "variable.method" will not match anything if 655 // "variable" is a local variable "method" by itself may 656 // still match something) 657 else foreach(part; name.splitter(".")) 658 { 659 if(partIdx++ > 0) { app.put("."); } 660 661 link = database.crossReference(writer, stack, part); 662 if(link !is null) writer.writeLink(app, link, { app.put(part); }); 663 else { app.put(part); } 664 } 665 } 666 667 if(input.empty) { break; } 668 669 // Even if scanName was called above, c is the first character 670 // *after* scanName. 671 app.put(c); 672 prevC = c; 673 // Must check again because scanName might have exhausted the input. 674 input.popFront(); 675 } 676 catch(Exception e) 677 { 678 import std.exception: assumeWontThrow; 679 writeln("Unexpected exception when cross-referencing: ", e.msg) 680 .assumeWontThrow; 681 } 682 683 return app.data; 684 } 685 686 /** 687 * Params: 688 * 689 * name = The symbol's name 690 * first = Set to true if this is the first time that pushSymbol has been 691 * called for this name. 692 * itemURL = URL to use in the Item for this symbol will be written here. 693 * 694 * Returns: A range to write the symbol's documentation to. 695 */ 696 auto pushSymbol(string name, ref bool first, ref string itemURL) 697 { 698 import std.array : array, join; 699 import std.string : format; 700 stack ~= name; 701 memberStack.length = memberStack.length + 1; 702 703 // Sets first 704 auto result = writer.pushSymbol(stack, database, first, itemURL); 705 706 if(first) 707 { 708 writer.writeHeader(result, name, writer.moduleNameLength); 709 writer.writeBreadcrumbs(result, stack, database); 710 writer.writeTOC(result, moduleName); 711 } 712 else 713 { 714 writer.writeSeparator(result); 715 } 716 writer.writeSymbolStart(result, itemURL); 717 return result; 718 } 719 720 void popSymbol(R)(ref R dst) 721 { 722 writer.writeSymbolEnd(dst); 723 stack.popBack(); 724 memberStack.popBack(); 725 writer.popSymbol(); 726 } 727 728 void pushAttributes() { attributeStack.length = attributeStack.length + 1; } 729 730 void popAttributes() { attributeStack.popBack(); } 731 732 733 /// The module name in "package.package.module" format. 734 string moduleName; 735 736 const(Attribute)[][] attributeStack; 737 Comment[] prevComments; 738 /** Namespace stack of the current symbol, 739 * 740 * E.g. ["package", "subpackage", "module", "Class", "member"] 741 */ 742 string[] stack; 743 /** Every item of this stack corresponds to a parent module/class/etc of the 744 * current symbol, but not package. 745 * 746 * Each Members struct is used to accumulate all members of that module/class/etc 747 * so the list of all members can be generated. 748 */ 749 Members[] memberStack; 750 TestRange[][size_t] unitTestMapping; 751 const(ubyte[]) fileBytes; 752 const(Config)* config; 753 /// Information about modules and symbols for e.g. cross-referencing. 754 SymbolDatabase database; 755 Writer writer; 756 }