Compact Manifest

Reducing the serialised size of a manifest of all files in a .lit notebook.

Exploring

js
return (async fn => {
  const {CompactPrefixTree} = await import('https://cdn.skypack.dev/compact-prefix-tree')

  const manifest = await fetch('/manifest.json').then(res => res.json())
  const keys = manifest.nodes.map(n=>n.id)
  const trie = new CompactPrefixTree(keys)

  const before = JSON.stringify(manifest.nodes.reduce((m,n)=>{m[n.id] = null; return m }, {}))
  //console.log(before)
  const after = JSON.stringify(trie.T)
  // console.log(trie.T)
  await lit.fs.writeFile('/testing/compactManifest1.json', after, 'utf8')
  return `${before.length}->${after.length}. A ${100-(after.length/before.length*100)}% reduction in size.`
})()
txtUpdated 55.3w ago
5978->4663. A 21.997323519571765% reduction in size.

Looking up a file, check for existence:

js
return (async (fn) => {
  const { CompactPrefixTree, getWordsFromTrie } = await import(
    "https://cdn.skypack.dev/compact-prefix-tree"
  );

  const json = await fetch("/testing/compactManifest1.json").then((res) =>
    res.json()
  );
  // return json
  const keys = getWordsFromTrie(json);
  const trie = new CompactPrefixTree(Array.from(keys));
  return trie.prefix("testing/log/");
})();

txtUpdated 58.0w ago
{ prefix: 'testing/log/day.lit', isProper: true }
txtUpdated 58.0w ago
{ prefix: '', isProper: false }

A Start

js
return (async (fn) => {
  const { CompactPrefixTree, getWordsFromTrie } = lit.utils.compactPrefixTree;

  const resp = await fetch("/compactManifest.json");
  const data = await resp.json();
  const keyset = getWordsFromTrie(data);
  const list = Array.from(keyset).map(k=>`/${k}`)
  console.log(list[50])
  return list.length
})();

txtUpdated 56.3w ago
/parser/frontmatter.js
159
  • Use the above to sync local file system instead of manifest.
txtUpdated 55.3w ago
Synced 165/168 files in 3.451 seconds. Fetched: 3 Duds: 0 Errors: 3 Skipped: 162
{ duds: [],
  errors: 
   [ '/testing/uploads/B98FC982-EB37-427B-A7D7-D872628323C0.png',
     '/testing/uploads/graph.png',
     '/testing/uploads/EC0FF74E-93A5-496D-A5A7-1F2FC1885F77.png.png' ] }