main source code

1 /**
2  * D Documentation Generator
3  * Copyright: © 2014 Economic Modeling Specialists, Intl.
4  *            © 2021 Eugene Stulin
5  * Authors: Brian Schott and other members of the D community,
6  *          Eugene Stulin
7  * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt Boost License 1.0)
8  */
9 module main;
10 
11 import std.algorithm,
12        std.array,
13        std.conv,
14        std.datetime,
15        std.exception,
16        std.file,
17        std.getopt,
18        std.path,
19        std.stdio,
20        std.string;
21 
22 import dparse.ast,
23        dparse.lexer,
24        dparse.parser;
25 
26 import ddoc.lexer;
27 
28 import allocator;
29 import config;
30 import macros;
31 import symboldatabase;
32 import tocbuilder;
33 import unittest_preprocessor;
34 import visitor;
35 import writer;
36 
37 immutable string defaultConfigPath = "hgen.cfg";
38 immutable string linuxConfigDir;
39 immutable string linuxConfigPath;
40 immutable string language;
41 
42 
43 shared static this() {
44     version (linux) {
45         import std.process : environment;
46         auto home = environment["HOME"];
47         linuxConfigDir = std.path.buildPath(home, ".config", "hgen");
48         linuxConfigPath = std.path.buildPath(linuxConfigDir, "hgen.cfg");
49     }
50     auto LANG = environment.get("LANG", "en_US.UTF-8");
51     auto langAndCharset = LANG.split(".").array;
52     auto tempLang = "";
53     if (langAndCharset.length > 0) {
54         auto langAndCountry = langAndCharset[0].split("_").array;
55         if (langAndCountry.length > 0) {
56             tempLang = langAndCountry[0];
57         }
58     }
59     language = tempLang=="" ? "en" : tempLang;
60 }
61 
62 
63 int main(string[] args) {
64     const startTime = Clock.currStdTime;
65 
66     Config config;
67 
68     config.loadConfigFile(getDefaultConfigPath());
69     config.loadCLI(args);
70     
71     if (config.doHelp) {
72         writeln(helpTranslations.get(language, "en"));
73         return 0;
74     }
75 
76     if (config.showVersion) {
77         writeln(versionString);
78         return 0;
79     }
80 
81     scope(exit) {
82         if (config.performance) {
83             writefln("Time spent: %.3fs",
84                     (Clock.currStdTime - startTime) / 10_000_000.0);
85             writefln("Peak memory usage (KiB): %s", peakMemoryUsageK());
86         }
87     }
88 
89     if (config.doGenerateCSSPath !is null) {
90         writefln("Generating CSS file '%s'", config.doGenerateCSSPath);
91         return writeProtected(config.doGenerateCSSPath, stylecss, "CSS");
92     }
93     if (config.doGenerateConfig) {
94         writefln("Generating config file '%s'", defaultConfigPath);
95         return writeProtected(defaultConfigPath, defaultConfigString, "config");
96     }
97 
98     version(linux) {
99         if (config.doGenerateConfigForLinux) {
100             if (!std.file.exists(linuxConfigPath)) {
101                 mkdirRecurse(linuxConfigDir);
102                 writefln("Generating config file '%s'", linuxConfigPath);
103                 return writeProtected(
104                     linuxConfigPath, defaultConfigString, "config"
105                 );
106             }
107             return 0;
108         }
109     }
110 
111     try {
112         config.macros = readMacros(config.macroFileNames);
113     } catch (Exception e) {
114         stderr.writeln(e.msg);
115         return 1;
116     }
117 
118     switch(config.format) {
119         case "html-simple":
120             generateDocumentation!HTMLWriterSimple(config);
121             break;
122         case "html-aggregated":
123             generateDocumentation!HTMLWriterAggregated(config);
124             break;
125         default:
126             writeln("Unknown format: ", config.format);
127     }
128 
129     return 0;
130 }
131 
132 
133 /*******************************************************************************
134  * Returns path to config file by default: in current directory or (for Linux)
135  * in ~/.config/hgen/ if in the current directory the file doesn't exist.
136  */
137 string getDefaultConfigPath() {
138     string configPath = defaultConfigPath;
139     version (linux) {
140         if (!exists(defaultConfigPath) && exists(linuxConfigPath)) {
141             configPath = linuxConfigPath;
142         }
143     }
144     return configPath;
145 }
146 
147 
148 /// Used to write default CSS/config with overwrite checking.
149 int writeProtected(string path, string content, string type) {
150     if (path.exists) {
151         writefln("'%s' exists. Overwrite? (y/N)", path);
152         import std.ascii: toLower;
153         char overwrite;
154         readf("%s", &overwrite);
155         if(overwrite.toLower != 'y') {
156             writefln("Exited without overwriting '%s'", path);
157             return 1;
158         }
159         writefln("Overwriting '%s'", path);
160     }
161     try {
162         std.file.write(path, content);
163     } catch(Exception e) {
164         writefln("Failed to write default %s to file `%s` : %s",
165             type, path, e.msg);
166         return 1;
167     }
168     return 0;
169 }
170 
171 
172 private string[string] readMacros(const string[] macroFiles) {
173     import ddoc.macros : defaultMacros = DEFAULT_MACROS;
174 
175     string[string] result;
176     defaultMacros.byKeyValue.each!(a => result[a.key] = a.value);
177     result["D"]    = `<code class="d_inline_code">$0</code>`;
178     result["HTTP"] = "<a href=\"http://$1\">$+</a>";
179     result["WEB"]  = "$(HTTP $1,$2)";
180     uniformCodeStyle(result);
181     macroFiles.each!(mf => mf.readMacroFile(result));
182     return result;
183 }
184 
185 
186 private void uniformCodeStyle(ref string[string] macros) {
187     macros[`D_CODE`] = `<pre><code class="hljs_d">$0</code></pre>`;
188     macros[`D`] = `<b>$0</b>`;
189     macros[`D_INLINECODE`] = `<b>$0</b>`;
190     macros[`D_COMMENT`] = `$0`;
191     macros[`D_KEYWORD`] = `$0`;
192     macros[`D_PARAM`] = macros[`D_INLINECODE`];
193 }
194 
195 
196 private void generateDocumentation(Writer)(ref Config config) {
197     string[] files = getFilesToProcess(config);
198     import std.stdio : writeln;
199     stdout.writeln("Writing documentation to ", config.outputDirectory);
200 
201     mkdirRecurse(config.outputDirectory);
202 
203     File search = File(buildPath(config.outputDirectory, "search.js"), "w");
204     search.writeln(`"use strict";`);
205     search.writeln(`var items = [`);
206 
207     auto database =
208         gatherData(config, new Writer(config, search, null, null), files);
209 
210     TocItem[] tocItems = buildTree(
211         database.moduleNames, database.moduleNameToLink
212     );
213 
214     enum noFile = "missing file";
215     string[] tocAdditionals =
216         config.tocAdditionalFileNames.map!(
217             path => path.exists ? readText(path) : noFile
218         ).array ~ config.tocAdditionalStrings;
219 
220     if (!tocAdditionals.empty) {
221         foreach(ref text; tocAdditionals) {
222             auto html = new Writer(config, search, null, null);
223             auto writer = appender!string();
224             html.readAndWriteComment(writer, text);
225             text = writer.data;
226         }
227     }
228 
229     foreach (f; database.moduleFiles) {
230         writeln("Generating documentation for ", f);
231         try {
232             writeDocumentation!Writer(
233                 config, database, f, search, tocItems, tocAdditionals
234             );
235         } catch (DdocParseException e) {
236             stderr.writeln(
237                 "Could not generate documentation for ",
238                 f, ": ", e.msg, ": ", e.snippet
239             );
240         } catch (Exception e) {
241             stderr.writeln(
242                 "Could not generate documentation for ", f, ": ", e.msg
243             );
244         }
245     }
246     search.writeln(`];`);
247     search.writeln(searchjs);
248 
249     // Write index.html and style.css
250     {
251         writeln("Generating main page");
252         File css = File(buildPath(config.outputDirectory, "style.css"), "w");
253         css.write(getCSS(config.cssFileName));
254         File index = File(buildPath(config.outputDirectory, "index.html"), "w");
255 
256         auto fileWriter = index.lockingTextWriter;
257         auto html = new Writer(config, search, tocItems, tocAdditionals);
258         html.writeHeader(fileWriter, "Index", 0);
259         const projectStr = config.projectName ~ " " ~ config.projectVersion;
260         const heading = projectStr == " " ? "Main Page"
261                                           : (projectStr ~ ": Main Page");
262         html.writeBreadcrumbs(fileWriter, heading);
263         html.writeTOC(fileWriter);
264 
265         // Index content added by the user.
266         if (config.indexFileName !is null) {
267             File indexFile = File(config.indexFileName);
268             ubyte[] indexBytes = new ubyte[cast(uint) indexFile.size];
269             indexFile.rawRead(indexBytes);
270             html.readAndWriteComment(fileWriter, cast(string)indexBytes);
271         }
272 
273         // A full list of all modules.
274         if (database.moduleNames.length <= config.maxModuleListLength) {
275             html.writeModuleList(fileWriter, database);
276         }
277 
278         index.writeln(HTML_END);
279     }
280 }
281 
282 
283 /*******************************************************************************
284  * Get the CSS content to write into style.css.
285  *
286  * If customCSS is not null, try to load from that file.
287  */
288 string getCSS(string customCSS) {
289     if (customCSS is null) {
290         return stylecss;
291     }
292     try {
293         return readText(customCSS);
294     } catch(Exception e) {
295         stderr.writefln("Failed to load custom CSS `%s`: %s", customCSS, e.msg);
296         return stylecss;
297     }
298 }
299 
300 
301 /// Creates documentation for the module at the given path
302 void writeDocumentation(Writer)(
303     ref Config config, SymbolDatabase database, 
304     string path, File search, TocItem[] tocItems, string[] tocAdditionals
305 ) {
306     LexerConfig lexConfig;
307     lexConfig.fileName = path;
308     lexConfig.stringBehavior = StringBehavior.source;
309 
310     File f = File(path);
311     ubyte[] fileBytes = uninitializedArray!(ubyte[])(to!size_t(f.size));
312     import core.memory;
313     scope(exit) { GC.free(fileBytes.ptr); }
314     f.rawRead(fileBytes);
315     StringCache cache = StringCache(1024 * 4);
316     auto tokens = getTokensForParser(fileBytes, lexConfig, &cache).array;
317 
318     import std.functional : toDelegate;
319     import dparse.rollback_allocator;
320     RollbackAllocator allocator;
321 
322     Module m = parseModule(tokens, path, &allocator, toDelegate(&doNothing));
323 
324     TestRange[][size_t] unitTestMapping = getUnittestMap(m);
325     
326     auto htmlWriter  = new Writer(config, search, tocItems, tocAdditionals);
327     auto visitor = new DocVisitor!Writer(
328         config, database, unitTestMapping, fileBytes, htmlWriter
329     );
330     visitor.visit(m);
331 }
332 
333 
334 /*******************************************************************************
335  * Get .d/.di files to process.
336  *
337  * Files that don't exist, are bigger than config.maxFileSizeK or
338  * could not be opened will be ignored.
339  *
340  * Params:
341  *
342  * config = Access to config to get source file and directory paths
343  *         get max file size.
344  * 
345  * Returns: Paths of files to process.
346  */
347 private string[] getFilesToProcess(ref const Config config) {
348     auto paths = config.sourcePaths.dup;
349     auto files = appender!(string[])();
350     void addFile(string path) {
351         const size = path.getSize();
352         if (size > config.maxFileSizeK * 1024) {
353             writefln(
354                 "WARNING: '%s' (%skiB) bigger than max file size (%skiB), " ~
355                 "ignoring", path, size / 1024, config.maxFileSizeK
356             );
357             return;
358         }
359         files.put(path);
360     }
361 
362     foreach (arg; paths) {
363         if (!arg.exists) {
364             stderr.writefln("WARNING: '%s' does not exist, ignoring", arg);
365         } else if (arg.isDir) {
366             foreach (fileName; arg.dirEntries("*.{d,di}", SpanMode.depth)) {
367                 addFile(fileName.expandTilde);
368             }
369         } else if (arg.isFile) {
370             addFile(arg.expandTilde);
371         } else {
372             stderr.writefln("WARNING: Could not open '%s', ignoring", arg);
373         }
374     }
375     return files.data;
376 }
377 
378 
379 /// Special empty function.
380 void doNothing(string, size_t, size_t, string, bool) {}
381 
382 
383 /// String representing the default CSS.
384 immutable string stylecss = import("style.css");
385 /// String representing the script used to search.
386 immutable string searchjs = import("search.js");
387 
388 
389 private ulong peakMemoryUsageK() {
390     version(linux) {
391         try {
392             auto line = File("/proc/self/status")
393                 .byLine().filter!(l => l.startsWith("VmHWM"));
394             enforce(!line.empty, "No VmHWM in /proc/self/status");
395             return line.front.split()[1].to!ulong;
396         } catch(Exception e) {
397             writeln("Failed to get peak memory usage: ", e);
398             return 0;
399         }
400     } else {
401         writeln("peakMemoryUsageK not implemented on non-Linux platforms");
402         return 0;
403     }
404 }