1 /* This file is part of the 'hgen' project.
2  *
3  * Copyright (c) 2014-2022
4  *     Economic Modeling Specialists, Intl.,
5  *     the D Community.
6  *
7  * Main contributors:
8  *     Brian Schott
9  *     Ferdinand Majerech
10  *     Akzhigitov Anton
11  *     Nemanja Boric
12  *     Basile Burg
13  *     Eugene Stulin
14  *
15  * Distributed under the Boost Software License, Version 1.0.
16  *
17  * You should have received a copy of the Boost Software License
18  * along with this program. If not, see <http://www.boost.org/LICENSE_1_0.txt>.
19  * This file is offered as-is, without any warranty.
20  */
21 
22 module main;
23 
24 import std.algorithm,
25        std.array,
26        std.conv,
27        std.datetime,
28        std.exception,
29        std.file,
30        std.getopt,
31        std.path,
32        std.stdio,
33        std.string;
34 
35 import dparse.ast,
36        dparse.lexer,
37        dparse.parser;
38 
39 import ddoc.lexer;
40 
41 import allocator;
42 import config;
43 import macros;
44 import symboldatabase;
45 import tocbuilder;
46 import unittest_preprocessor;
47 import visitor;
48 import writer;
49 
50 immutable string defaultConfigPath = "hgen.cfg";
51 immutable string linuxConfigDir;
52 immutable string linuxConfigPath;
53 immutable string language;
54 
55 
56 shared static this() {
57     version (linux) {
58         import std.process : environment;
59         auto home = environment["HOME"];
60         linuxConfigDir = std.path.buildPath(home, ".config", "hgen");
61         linuxConfigPath = std.path.buildPath(linuxConfigDir, "hgen.cfg");
62     }
63     auto LANG = environment.get("LANG", "en_US.UTF-8");
64     auto langAndCharset = LANG.split(".").array;
65     auto tempLang = "";
66     if (langAndCharset.length > 0) {
67         auto langAndCountry = langAndCharset[0].split("_").array;
68         if (langAndCountry.length > 0) {
69             tempLang = langAndCountry[0];
70         }
71     }
72     language = tempLang=="" ? "en" : tempLang;
73 }
74 
75 
76 int main(string[] args) {
77     const startTime = Clock.currStdTime;
78 
79     Config config;
80 
81     config.loadConfigFile(getDefaultConfigPath());
82     config.loadCLI(args);
83     
84     if (config.doHelp) {
85         writeln(helpTranslations.get(language, "en"));
86         return 0;
87     }
88 
89     if (config.showVersion) {
90         writeln(versionString);
91         return 0;
92     }
93 
94     scope(exit) {
95         if (config.performance) {
96             writefln!"Time spent: %.3fs"(
97                 (Clock.currStdTime - startTime) / 10_000_000.0
98             );
99             writefln("Peak memory usage (KiB): %s", peakMemoryUsageK());
100         }
101     }
102 
103     if (config.doGenerateCSSPath !is null) {
104         writefln("Generating CSS file '%s'", config.doGenerateCSSPath);
105         return writeProtected(config.doGenerateCSSPath, stylecss, "CSS");
106     }
107     if (config.doGenerateConfig) {
108         writefln("Generating config file '%s'", defaultConfigPath);
109         return writeProtected(defaultConfigPath, defaultConfigString, "config");
110     }
111 
112     version(linux) {
113         if (config.doGenerateConfigForLinux) {
114             if (!std.file.exists(linuxConfigPath)) {
115                 mkdirRecurse(linuxConfigDir);
116                 writefln("Generating config file '%s'", linuxConfigPath);
117                 return writeProtected(
118                     linuxConfigPath, defaultConfigString, "config"
119                 );
120             }
121             return 0;
122         }
123     }
124 
125     try {
126         config.macros = readMacros(config.macroFileNames);
127     } catch (Exception e) {
128         stderr.writeln(e.msg);
129         return 1;
130     }
131 
132     switch(config.format) {
133         case "html-simple":
134             generateDocumentation!HTMLWriterSimple(config);
135             break;
136         case "html-aggregated":
137             generateDocumentation!HTMLWriterAggregated(config);
138             break;
139         default:
140             writeln("Unknown format: ", config.format);
141     }
142 
143     return 0;
144 }
145 
146 
147 /*******************************************************************************
148  * Returns path to config file by default: in current directory or (for Linux)
149  * in ~/.config/hgen/ if in the current directory the file doesn't exist.
150  */
151 string getDefaultConfigPath() {
152     string configPath = defaultConfigPath;
153     version (linux) {
154         if (!exists(defaultConfigPath) && exists(linuxConfigPath)) {
155             configPath = linuxConfigPath;
156         }
157     }
158     return configPath;
159 }
160 
161 
162 /// Used to write default CSS/config with overwrite checking.
163 int writeProtected(string path, string content, string type) {
164     if (path.exists) {
165         writefln("'%s' exists. Overwrite? (y/N)", path);
166         import std.ascii: toLower;
167         char overwrite;
168         readf("%s", &overwrite);
169         if(overwrite.toLower != 'y') {
170             writefln("Exited without overwriting '%s'", path);
171             return 1;
172         }
173         writefln("Overwriting '%s'", path);
174     }
175     try {
176         std.file.write(path, content);
177     } catch(Exception e) {
178         writefln("Failed to write default %s to file `%s` : %s",
179             type, path, e.msg);
180         return 1;
181     }
182     return 0;
183 }
184 
185 
186 private string[string] readMacros(const string[] macroFiles) {
187     import ddoc.macros : defaultMacros = DEFAULT_MACROS;
188 
189     string[string] result;
190     defaultMacros.byKeyValue.each!(a => result[a.key] = a.value);
191     result["D"]    = `<code class="d_inline_code">$0</code>`;
192     result["HTTP"] = "<a href=\"http://$1\">$+</a>";
193     result["WEB"]  = "$(HTTP $1,$2)";
194     uniformCodeStyle(result);
195     macroFiles.each!(mf => mf.readMacroFile(result));
196     return result;
197 }
198 
199 
200 private void uniformCodeStyle(ref string[string] macros) {
201     macros[`D_CODE`] = `<pre><code class="hljs_d">$0</code></pre>`;
202     macros[`D`] = `<b>$0</b>`;
203     macros[`D_INLINECODE`] = `<b>$0</b>`;
204     macros[`D_COMMENT`] = `$0`;
205     macros[`D_KEYWORD`] = `$0`;
206     macros[`D_PARAM`] = macros[`D_INLINECODE`];
207 }
208 
209 
210 private void generateDocumentation(Writer)(ref Config config) {
211     string[] files = getFilesToProcess(config);
212     import std.stdio : writeln;
213     stdout.writeln("Writing documentation to ", config.outputDirectory);
214 
215     mkdirRecurse(config.outputDirectory);
216 
217     File search = File(buildPath(config.outputDirectory, "search.js"), "w");
218     search.writeln(`"use strict";`);
219     search.writeln(`var items = [`);
220 
221     auto database =
222         gatherData(config, new Writer(config, search, null, null), files);
223 
224     TocItem[] tocItems = buildTree(
225         database.moduleNames, database.moduleNameToLink
226     );
227 
228     enum noFile = "missing file";
229     string[] tocAdditionals =
230         config.tocAdditionalFileNames.map!(
231             path => path.exists ? readText(path) : noFile
232         ).array ~ config.tocAdditionalStrings;
233 
234     if (!tocAdditionals.empty) {
235         foreach(ref text; tocAdditionals) {
236             auto html = new Writer(config, search, null, null);
237             auto writer = appender!string();
238             html.readAndWriteComment(writer, text);
239             text = writer.data;
240         }
241     }
242 
243     foreach (f; database.moduleFiles) {
244         writeln("Generating documentation for ", f);
245         try {
246             writeDocumentation!Writer(
247                 config, database, f, search, tocItems, tocAdditionals
248             );
249         } catch (DdocParseException e) {
250             stderr.writeln(
251                 "Could not generate documentation for ",
252                 f, ": ", e.msg, ": ", e.snippet
253             );
254         } catch (Exception e) {
255             stderr.writeln(
256                 "Could not generate documentation for ", f, ": ", e.msg
257             );
258         }
259     }
260     search.writeln(`];`);
261     search.writeln(searchjs);
262 
263     // Write index.html and style.css
264     {
265         writeln("Generating main page");
266         File css = File(buildPath(config.outputDirectory, "style.css"), "w");
267         css.write(getCSS(config.cssFileName));
268         File index = File(buildPath(config.outputDirectory, "index.html"), "w");
269 
270         auto fileWriter = index.lockingTextWriter;
271         auto html = new Writer(config, search, tocItems, tocAdditionals);
272         html.writeHeader(fileWriter, "Index", 0);
273         const projectStr = config.projectName ~ " " ~ config.projectVersion;
274         const heading = projectStr == " " ? "Main Page"
275                                           : (projectStr ~ ": Main Page");
276         html.writeBreadcrumbs(fileWriter, heading);
277         html.writeTOC(fileWriter);
278 
279         // Index content added by the user.
280         if (config.indexFileName !is null) {
281             File indexFile = File(config.indexFileName);
282             ubyte[] indexBytes = new ubyte[cast(uint) indexFile.size];
283             indexFile.rawRead(indexBytes);
284             html.readAndWriteComment(fileWriter, cast(string)indexBytes);
285         }
286 
287         // A full list of all modules.
288         if (database.moduleNames.length <= config.maxModuleListLength) {
289             html.writeModuleList(fileWriter, database);
290         }
291 
292         index.writeln(HTML_END);
293     }
294 }
295 
296 
297 /*******************************************************************************
298  * Get the CSS content to write into style.css.
299  *
300  * If customCSS is not null, try to load from that file.
301  */
302 string getCSS(string customCSS) {
303     if (customCSS is null) {
304         return stylecss;
305     }
306     try {
307         return readText(customCSS);
308     } catch(Exception e) {
309         stderr.writefln("Failed to load custom CSS `%s`: %s", customCSS, e.msg);
310         return stylecss;
311     }
312 }
313 
314 
315 /// Creates documentation for the module at the given path
316 void writeDocumentation(Writer)(
317     ref Config config, SymbolDatabase database, 
318     string path, File search, TocItem[] tocItems, string[] tocAdditionals
319 ) {
320     LexerConfig lexConfig;
321     lexConfig.fileName = path;
322     lexConfig.stringBehavior = StringBehavior.source;
323 
324     File f = File(path);
325     ubyte[] fileBytes = uninitializedArray!(ubyte[])(to!size_t(f.size));
326     import core.memory;
327     scope(exit) GC.free(fileBytes.ptr);
328     f.rawRead(fileBytes);
329     StringCache cache = StringCache(1024 * 4);
330     auto tokens = getTokensForParser(fileBytes, lexConfig, &cache).array;
331 
332     import std.functional : toDelegate;
333     import dparse.rollback_allocator;
334     RollbackAllocator allocator;
335 
336     Module m = parseModule(tokens, path, &allocator, toDelegate(&doNothing));
337 
338     TestRange[][size_t] unitTestMapping = getUnittestMap(m);
339     
340     auto htmlWriter  = new Writer(config, search, tocItems, tocAdditionals);
341     auto visitor = new DocVisitor!Writer(
342         config, database, unitTestMapping, fileBytes, htmlWriter
343     );
344     visitor.visit(m);
345 }
346 
347 
348 /*******************************************************************************
349  * Get .d/.di files to process.
350  *
351  * Files that don't exist, are bigger than config.maxFileSizeK or
352  * could not be opened will be ignored.
353  *
354  * Params:
355  *
356  * config = Access to config to get source file and directory paths
357  *         get max file size.
358  * 
359  * Returns: Paths of files to process.
360  */
361 private string[] getFilesToProcess(ref const Config config) {
362     auto paths = config.sourcePaths.dup;
363     auto files = appender!(string[])();
364     void addFile(string path) {
365         const size = path.getSize();
366         if (size > config.maxFileSizeK * 1024) {
367             writefln(
368                 "WARNING: '%s' (%skiB) bigger than max file size (%skiB), " ~
369                 "ignoring", path, size / 1024, config.maxFileSizeK
370             );
371             return;
372         }
373         files.put(path);
374     }
375 
376     foreach (arg; paths) {
377         if (!arg.exists) {
378             stderr.writefln("WARNING: '%s' does not exist, ignoring", arg);
379         } else if (arg.isDir) {
380             foreach (fileName; arg.dirEntries("*.{d,di}", SpanMode.depth)) {
381                 addFile(fileName.expandTilde);
382             }
383         } else if (arg.isFile) {
384             addFile(arg.expandTilde);
385         } else {
386             stderr.writefln("WARNING: Could not open '%s', ignoring", arg);
387         }
388     }
389     return files.data;
390 }
391 
392 
393 /// Special empty function.
394 void doNothing(string, size_t, size_t, string, bool) {}
395 
396 
397 /// String representing the default CSS.
398 immutable string stylecss = import("style.css");
399 /// String representing the script used to search.
400 immutable string searchjs = import("search.js");
401 
402 
403 private ulong peakMemoryUsageK() {
404     version(linux) {
405         try {
406             auto line = File("/proc/self/status")
407                 .byLine().filter!(l => l.startsWith("VmHWM"));
408             enforce(!line.empty, "No VmHWM in /proc/self/status");
409             return line.front.split()[1].to!ulong;
410         } catch(Exception e) {
411             writeln("Failed to get peak memory usage: ", e);
412             return 0;
413         }
414     } else {
415         writeln("peakMemoryUsageK not implemented on non-Linux platforms");
416         return 0;
417     }
418 }