123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362 | /* Client-side repository search.
*
* One file, three roles, chosen by environment:
* - page script: the UI on /search/<repo>
* - Web Worker: loads the corpus JSON once and runs queries off the main thread
* - node module: exports the pure matching functions for tests
*
* Plain ES2020. No dependencies, no bundler.
*/
(function (global) {
'use strict';
// ---------------------------------------------------------------- core
function escapeRegExp(s) {
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
// Compile a query into a matcher. Returns {re} for regex-style matching
// or {literal, lower} for the fast indexOf path. Throws on bad regex.
function compile(q, opts) {
opts = opts || {};
if (!q) return null;
if (!opts.regex && !opts.word) {
return opts.caseSensitive
? { literal: q }
: { literal: q, lower: q.toLowerCase() };
}
let pattern = opts.regex ? q : escapeRegExp(q);
if (opts.word) pattern = '(?<![\\p{L}\\p{N}_])(?:' + pattern + ')(?![\\p{L}\\p{N}_])';
const flags = 'u' + (opts.caseSensitive ? '' : 'i');
return { re: new RegExp(pattern, flags) };
}
// Find the first match in a line. Returns [start, end] or null.
function matchLine(m, line, lowerLine) {
if (m.re) {
m.re.lastIndex = 0;
const r = m.re.exec(line);
if (!r) return null;
return [r.index, r.index + Math.max(r[0].length, 1)];
}
if (m.lower !== undefined) {
const hay = lowerLine !== undefined ? lowerLine : line.toLowerCase();
const i = hay.indexOf(m.lower);
return i < 0 ? null : [i, i + m.lower.length];
}
const i = line.indexOf(m.literal);
return i < 0 ? null : [i, i + m.literal.length];
}
// Convert a path glob to a RegExp: * matches within a segment, ** across
// segments, ? one char. A glob without a slash matches the basename.
function globToRegExp(glob) {
glob = (glob || '').trim();
if (!glob) return null;
let re = '';
for (let i = 0; i < glob.length; i++) {
const c = glob[i];
if (c === '*') {
if (glob[i + 1] === '*') { re += '.*'; i++; }
else re += '[^/]*';
} else if (c === '?') re += '[^/]';
else re += escapeRegExp(c);
}
const anchored = glob.includes('/') ? '^' + re + '$' : '(^|/)' + re + '$';
return new RegExp(anchored);
}
// Generator yielding {p, matches:[{n, text, s, e}]} per file with hits.
// `files` is the corpus's files array; `lowered` caches lowercase
// copies per file (array parallel to files) when case-insensitive.
function* iterSearch(files, m, pathRe, lowered) {
for (let fi = 0; fi < files.length; fi++) {
const f = files[fi];
if (pathRe && !pathRe.test(f.p)) continue;
let low = null;
if (m.lower !== undefined && lowered) {
low = lowered[fi];
if (!low) {
low = f.l.map((s) => s.toLowerCase());
lowered[fi] = low;
}
}
let matches = null;
for (let i = 0; i < f.l.length; i++) {
const r = matchLine(m, f.l[i], low ? low[i] : undefined);
if (r) {
if (!matches) matches = [];
matches.push({ n: i + 1, text: f.l[i], s: r[0], e: r[1] });
}
}
if (matches) yield { p: f.p, matches: matches };
}
}
// Cap the hits from iterSearch at max matches in all, cutting the last
// file short if need be. Returns (as the generator's final value)
// whether any matches were left out.
function* capped(hits, max) {
let total = 0;
for (const hit of hits) {
if (total + hit.matches.length > max) {
hit.matches = hit.matches.slice(0, max - total);
if (hit.matches.length) yield hit;
return true;
}
total += hit.matches.length;
yield hit;
}
return false;
}
// Synchronous search with a result cap, as the worker runs it.
function searchSync(files, q, opts, maxResults) {
const m = compile(q, opts);
if (!m) return { files: [], total: 0, truncated: false };
const it = capped(iterSearch(files, m, globToRegExp(opts && opts.path), []), maxResults);
const out = [];
let total = 0;
for (;;) {
const step = it.next();
if (step.done) return { files: out, total: total, truncated: step.value };
out.push(step.value);
total += step.value.matches.length;
}
}
const core = { compile, matchLine, globToRegExp, iterSearch, searchSync, escapeRegExp };
// ---------------------------------------------------------------- node
if (typeof module !== 'undefined' && module.exports) {
module.exports = core;
return;
}
// ---------------------------------------------------------------- worker
if (typeof WorkerGlobalScope !== 'undefined' && global instanceof WorkerGlobalScope) {
let files = null;
let lowered = [];
let generation = 0;
async function load(url) {
try {
// same-origin, not omit: on the private site the corpus is
// behind the login the browser has for the page.
const res = await fetch(url, { credentials: 'same-origin' });
if (!res.ok) throw new Error('HTTP ' + res.status);
const data = await res.json();
files = data.files || [];
lowered = new Array(files.length);
let lines = 0;
for (const f of files) lines += f.l.length;
global.postMessage({ type: 'loaded', files: files.length, lines: lines, sha: data.sha });
} catch (err) {
global.postMessage({ type: 'error', message: 'Could not load search data: ' + err.message });
}
}
async function run(msg) {
const id = msg.id;
let m;
let pathRe;
try {
m = compile(msg.q, msg.opts);
pathRe = globToRegExp(msg.opts.path);
} catch (err) {
global.postMessage({ type: 'results', id: id, batch: [], done: true, total: 0, error: 'Invalid pattern: ' + err.message });
return;
}
if (!m || !files) {
global.postMessage({ type: 'results', id: id, batch: [], done: true, total: 0 });
return;
}
const it = capped(iterSearch(files, m, pathRe, lowered), msg.max);
let batch = [];
let total = 0;
let batchLines = 0;
let scanned = 0;
let truncated = false;
const flush = (done) => {
global.postMessage({ type: 'results', id: id, batch: batch, done: done, total: total, truncated: truncated });
batch = [];
batchLines = 0;
};
for (;;) {
const step = it.next();
if (step.done) { truncated = step.value; break; }
const hit = step.value;
total += hit.matches.length;
batch.push(hit);
batchLines += hit.matches.length;
if (batchLines >= 100) flush(false);
if (++scanned % 200 === 0) {
// Yield so a newer query can cancel this one.
await new Promise((r) => setTimeout(r, 0));
if (id !== generation) return;
}
}
if (id === generation) flush(true);
}
global.onmessage = (ev) => {
const msg = ev.data;
if (msg.type === 'load') load(msg.url);
else if (msg.type === 'search') { generation = msg.id; run(msg); }
};
return;
}
// ---------------------------------------------------------------- page
if (typeof document === 'undefined') return;
// The worker is this same script, at its versioned URL.
const workerURL = document.currentScript ? document.currentScript.src : '/static/search.js';
const form = document.getElementById('search');
if (!form) return;
const $ = (id) => document.getElementById(id);
const qEl = $('q'), pathEl = $('path'), regexEl = $('regex'), caseEl = $('case'), wordEl = $('word');
const statusEl = $('status'), resultsEl = $('results');
const codeBase = form.dataset.code;
const maxResults = parseInt(form.dataset.max, 10) || 2000;
const dataURL = form.dataset.data;
form.addEventListener('submit', (ev) => { ev.preventDefault(); kick(); });
let worker;
try {
worker = new Worker(workerURL);
} catch (err) {
statusEl.textContent = 'Search is unavailable: ' + err.message;
return;
}
let loaded = false;
let id = 0;
let pending = null;
let rendered = { files: 0, matches: 0 };
// While a search runs, its progress counts are marked busy, so a screen
// reader announces only the final count instead of every batch.
function setStatus(text, busy) {
statusEl.textContent = text;
statusEl.setAttribute('aria-busy', busy ? 'true' : 'false');
}
function opts() {
return {
regex: regexEl.checked,
caseSensitive: caseEl.checked,
word: wordEl.checked,
path: pathEl.value.trim(),
};
}
function syncURL() {
const u = new URL(location.href);
const set = (k, v) => (v ? u.searchParams.set(k, v) : u.searchParams.delete(k));
set('q', qEl.value);
set('path', pathEl.value.trim());
set('r', regexEl.checked ? '1' : '');
set('c', caseEl.checked ? '1' : '');
set('w', wordEl.checked ? '1' : '');
history.replaceState(null, '', u);
}
function escapeHTML(s) {
return s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"').replace(/'/g, ''');
}
function encodePath(p) {
return p.split('/').map(encodeURIComponent).join('/');
}
function renderBatch(batch) {
const frag = document.createDocumentFragment();
for (const file of batch) {
const det = document.createElement('details');
det.open = true;
det.className = 'hit-file';
const sum = document.createElement('summary');
const link = document.createElement('a');
link.href = codeBase + '/' + encodePath(file.p);
link.textContent = file.p;
sum.appendChild(link);
const cnt = document.createElement('span');
cnt.className = 'count';
cnt.textContent = file.matches.length + (file.matches.length === 1 ? ' match' : ' matches');
sum.appendChild(cnt);
det.appendChild(sum);
const ol = document.createElement('div');
ol.className = 'hits';
let html = '';
const base = escapeHTML(link.href);
for (const m of file.matches) {
const before = escapeHTML(m.text.slice(0, m.s));
const mid = escapeHTML(m.text.slice(m.s, m.e));
const after = escapeHTML(m.text.slice(m.e));
html += '<a class="hit" href="' + base + '#L' + m.n + '"><span class="ln">' + m.n + '</span><code>' +
before + '<mark>' + mid + '</mark>' + after + '</code></a>';
}
ol.innerHTML = html;
det.appendChild(ol);
frag.appendChild(det);
rendered.files++;
rendered.matches += file.matches.length;
}
resultsEl.appendChild(frag);
}
function kick() {
syncURL();
const q = qEl.value;
id++;
resultsEl.textContent = '';
rendered = { files: 0, matches: 0 };
if (!q) { setStatus(loaded ? 'Type to search.' : 'Loading file contents…'); return; }
if (!loaded) { pending = true; setStatus('Loading file contents…'); return; }
setStatus('Searching…', true);
worker.postMessage({ type: 'search', id: id, q: q, opts: opts(), max: maxResults });
}
let timer = null;
const debounced = () => { clearTimeout(timer); timer = setTimeout(kick, 150); };
qEl.addEventListener('input', debounced);
pathEl.addEventListener('input', debounced);
for (const el of [regexEl, caseEl, wordEl]) el.addEventListener('change', kick);
worker.onmessage = (ev) => {
const msg = ev.data;
if (msg.type === 'loaded') {
loaded = true;
setStatus('Ready: ' + msg.files + ' files, ' + msg.lines.toLocaleString() + ' lines.');
if (pending || qEl.value) { pending = null; kick(); }
} else if (msg.type === 'error') {
setStatus(msg.message);
} else if (msg.type === 'results') {
if (msg.id !== id) return;
if (msg.error) { setStatus(msg.error); return; }
renderBatch(msg.batch);
if (msg.done) {
if (msg.total === 0) setStatus('No matches.');
else setStatus(msg.total.toLocaleString() + (msg.total === 1 ? ' match in ' : ' matches in ') +
rendered.files + (rendered.files === 1 ? ' file' : ' files') +
(msg.truncated ? ' (stopped at ' + maxResults.toLocaleString() + ' results)' : '') + '.');
} else {
setStatus('Searching… ' + msg.total.toLocaleString() + ' so far', true);
}
}
};
const params = new URLSearchParams(location.search);
if (params.get('q')) qEl.value = params.get('q');
if (params.get('path')) pathEl.value = params.get('path');
regexEl.checked = params.get('r') === '1';
caseEl.checked = params.get('c') === '1';
wordEl.checked = params.get('w') === '1';
worker.postMessage({ type: 'load', url: dataURL });
if (qEl.value) setStatus('Loading file contents…');
qEl.focus();
})(typeof self !== 'undefined' ? self : this);
|