Commit 77819826 authored by Albert Gräf's avatar Albert Gräf
Browse files

Help browser: add an index cache to speed up launch times.

Features:

- Rebuilds and caches the search index if there is no cache yet, or if
  the cache is outdated (i.e., any cached directory or its parent was
  removed or modified since the cache was last created).

- Keeps track of the modification times of all cached directories of
  help files, as well as their parent directories, in order to determine
  when the cache is outdated. The parents are tracked to catch changes
  where new sibling directories are added. This may produce some false
  positives and still isn't 100% foolproof, but it is as close as we can
  get if we still want to achieve substantial speedups.

- Automatically rebuilds the index (and cache) from scratch, as soon as
  the help browser is relaunched after changes to the browser
  configuration in the gui prefs. So there's no need any more to
  relaunch Purr Data to have these changes take effect.

Both the index cache and the directory timestamps are maintained as
ordinary text files with a straightforward syntax (basically
colon-delimited csv) in the user's home directory. The files are located
in configuration directories (.purr-data on Linux and Mac,
AppData/Roaming/Purr-Data on Windows). Thus it's easily possible to
modify these files with external tools (e.g., if we want to upgrade the
file format in the future.)
parent ffea3cd5
......@@ -789,6 +789,12 @@ function apply(save_prefs) {
get_bool_elem("browser_init"),
get_autopatch_yoffset()
);
// Also send the browser config to the GUI, so that it will update the
// help browser accordingly.
pdgui.update_browser(
get_bool_elem("browser_doc"),
get_bool_elem("browser_path")
);
// Update the grid on all open windows.
pdgui.update_grid(get_bool_elem("show_grid"));
......
......@@ -74,6 +74,29 @@ exports.set_focused_patchwin = function(cid) {
last_focused = cid;
}
// ico@vt.edu 2020-08-12: check which OS we have since Windows has a different
// windows positioning logic on nw.js 0.14.7 than Linux/OSX. Go figure...
function check_os(name) {
var os = require('os');
//post("os=" + os.platform());
return os.platform() === name ? 1 : 0;
}
exports.check_os = check_os;
// ico@vt.edu 2020-08-14: used to speed-up window size consistency and other
// OS-centric operations by avoiding constant calls to string comparisons.
// Most pertinent calls can be found in index.js' nw_create_window and pdgui.js'
// canvas_check_geometry
var nw_os_is_linux = check_os("linux");
var nw_os_is_osx = check_os("darwin");
var nw_os_is_windows = check_os("win32");
exports.nw_os_is_linux = nw_os_is_linux;
exports.nw_os_is_osx = nw_os_is_osx;
exports.nw_os_is_windows = nw_os_is_windows;
// Keyword index (cf. dialog_search.html)
var fs = require("fs");
......@@ -81,14 +104,30 @@ var path = require("path");
var dive = require("./dive.js"); // small module to recursively search dirs
var elasticlunr = require("./elasticlunr.js"); // lightweight full-text search engine in JavaScript, cf. https://github.com/weixsong/elasticlunr.js/
var index = elasticlunr();
index.addField("title");
index.addField("keywords");
index.addField("description");
//index.addField("body");
index.addField("path");
index.setRef("id");
function init_elasticlunr()
{
index = elasticlunr();
index.addField("title");
index.addField("keywords");
index.addField("description");
//index.addField("body");
index.addField("path");
index.setRef("id");
return index;
}
var index = init_elasticlunr();
var index_cache = new Array();
var index_manif = new Set();
function index_entry_esc(s) {
if (s) {
var t = s.replace(/\\/g, "\\\\").replace(/:/g, "\\:");
return t.replace(/(?:\r\n|\r|\n)/g, "\\n");
} else {
return "";
}
}
function add_doc_to_index(filename, data) {
var title = path.basename(filename, ".pd"),
......@@ -111,6 +150,12 @@ function add_doc_to_index(filename, data) {
if (title.slice(-5) === "-help") {
title = title.slice(0, -5);
}
index_cache[index_cache.length] = [filename, title, keywords, desc]
.map(index_entry_esc).join(":");
var d = path.dirname(filename);
index_manif.add(d);
// Also add the parent directory to catch additions of siblings.
index_manif.add(path.dirname(d));
index.addDoc({
"id": filename,
"title": title,
......@@ -142,21 +187,82 @@ function read_file(err, filename, stat) {
var index_done = false;
var index_started = false;
var index_start_time;
// Filenames for the index cache, relative to the user's homedir.
const cache_basename = nw_os_is_windows
? "~/AppData/Roaming/Purr-Data/search"
: "~/.purr-data/search";
const cache_name = cache_basename + ".index";
const stamps_name = cache_basename + ".stamps";
function finish_index() {
index_done = true;
post("finished building help index");
var have_cache = index_cache.length > 0;
try {
// write the index cache if we have one
if (have_cache) {
var a = new Array();
index_manif.forEach(function(x) {
var st = fs.statSync(x);
a[a.length] = index_entry_esc(x) + ":" + st.mtimeMs;
});
a.sort();
// Make sure that the target dir exists:
try {
fs.mkdirSync(expand_tilde(path.dirname(cache_name)));
} catch (err) {
//console.log(err);
}
fs.writeFileSync(expand_tilde(cache_name),
index_cache.join("\n"), {mode: 0o644});
// also write a manifest with the timestamps of all directories:
fs.writeFileSync(expand_tilde(stamps_name),
a.join("\n"), {mode: 0o644});
}
} catch (err) {
console.log(err);
}
var t = new Date().getTime() / 1000;
post("finished " + (have_cache?"building":"loading") + " help index (" +
(t-index_start_time).toFixed(2) + " secs)");
}
// AG: pilfered from https://stackoverflow.com/questions/21077670
function expand_tilde(filepath) {
if (filepath[0] === '~') {
return path.join(process.env.HOME, filepath.slice(1));
var home = nw_os_is_windows ? process.env.HOMEPATH : process.env.HOME;
return path.join(home, filepath.slice(1));
}
return filepath;
}
// AG: This is supposed to be executed only once, after lib_dir has been set.
function check_timestamps(manif)
{
manif = manif.split('\n');
for (var j = 0, l = manif.length; j < l; j++) {
if (manif[j]) {
var e = manif[j].replace(/\\:/g, "\x1c").split(':')
.map(x => x
.replace(/\x1c/g, ":")
.replace(/\\n/g, "\n")
.replace(/\\\\/g, "\\"));
var dirname = e[0] ? e[0] : null;
var stamp = e[1] ? parseFloat(e[1]) : 0.0;
try {
var st = fs.statSync(dirname);
if (st.mtimeMs > stamp) {
return false;
}
} catch (err) {
return false;
}
}
}
return true;
}
// AG: This is normally executed only once, after lib_dir has been set.
// Note that dive() traverses lib_dir asynchronously, so we report back in
// finish_index() when this is done.
function make_index() {
......@@ -182,8 +288,47 @@ function make_index() {
}
}
index_started = true;
post("building help index in " + doc_path);
dive(doc_path, read_file, browser_path?make_index_cont:finish_index);
index_start_time = new Date().getTime() / 1000;
var idx, manif;
try {
// test for index cache and manifest
idx = fs.readFileSync
(expand_tilde(cache_name), 'utf8');
manif = fs.readFileSync
(expand_tilde(stamps_name), 'utf8');
} catch (err) {
//console.log(err);
}
if (idx && manif && check_timestamps(manif)) {
// index cache is present and up-to-date, load it
post("loading cached help index from " + cache_name);
idx = idx.split('\n');
for (var j = 0, l = idx.length; j < l; j++) {
if (idx[j]) {
var e = idx[j].replace(/\\:/g, "\x1c").split(':')
.map(x => x
.replace(/\x1c/g, ":")
.replace(/\\n/g, "\n")
.replace(/\\\\/g, "\\"));
var filename = e[0] ? e[0] : null;
var title = e[1] ? e[1] : null;
var keywords = e[2] ? e[2] : null;
var descr = e[3] ? e[3] : null;
index.addDoc({
"id": filename,
"title": title,
"keywords": keywords,
"description": descr
});
}
}
finish_index();
} else {
// no index cache, or it is out of date, so (re)build it now, and
// save the new cache along the way
post("building help index in " + doc_path);
dive(doc_path, read_file, browser_path?make_index_cont:finish_index);
}
}
// AG: This is called from dialog_search.html with a callback that expects to
......@@ -206,6 +351,42 @@ function build_index(cb) {
exports.build_index = build_index;
// this doesn't actually rebuild the index, it just clears it, so that it
// will be rebuilt the next time the help browser is opened
function rebuild_index()
{
post("clearing help index (reopen browser to rebuild!)");
index = init_elasticlunr();
index_started = index_done = false;
try {
fs.unlink(expand_tilde(cache_name));
fs.unlink(expand_tilde(stamps_name));
} catch (err) {
//console.log(err);
}
}
// this is called from the gui tab of the prefs dialog
function update_browser(doc_flag, path_flag)
{
var changed = false;
doc_flag = doc_flag?1:0;
path_flag = path_flag?1:0;
if (browser_doc !== doc_flag) {
browser_doc = doc_flag;
changed = true;
}
if (browser_path !== path_flag) {
browser_path = path_flag;
changed = true;
}
if (changed) {
rebuild_index();
}
}
exports.update_browser = update_browser;
// Modules
var cp = require("child_process"); // for starting core Pd from GUI in OSX
......@@ -815,29 +996,6 @@ function check_nw_version(version) {
exports.check_nw_version = check_nw_version;
// ico@vt.edu 2020-08-12: check which OS we have since Windows has a different
// windows positioning logic on nw.js 0.14.7 than Linux/OSX. Go figure...
function check_os(name) {
var os = require('os');
//post("os=" + os.platform());
return os.platform() === name ? 1 : 0;
}
exports.check_os = check_os;
// ico@vt.edu 2020-08-14: used to speed-up window size consistency and other
// OS-centric operations by avoiding constant calls to string comparisons.
// Most pertinent calls can be found in index.js' nw_create_window and pdgui.js'
// canvas_check_geometry
var nw_os_is_linux = check_os("linux");
var nw_os_is_osx = check_os("darwin");
var nw_os_is_windows = check_os("win32");
exports.nw_os_is_linux = nw_os_is_linux;
exports.nw_os_is_osx = nw_os_is_osx;
exports.nw_os_is_windows = nw_os_is_windows;
// ico@vt.edu 2020-08-11: this appears to have to be 25 at all times
// we will leave this here for later if we encounter issues with inconsistencies
// across different nw.js versions...
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment