Reducing the serialised size of a manifest of all files in a .lit notebook.
Exploring
Trying out compact-prefix-tree↗
return (async fn => {
const {CompactPrefixTree} = await import('https://cdn.skypack.dev/compact-prefix-tree')
const manifest = await fetch('/manifest.json').then(res => res.json())
const keys = manifest.nodes.map(n=>n.id)
const trie = new CompactPrefixTree(keys)
const before = JSON.stringify(manifest.nodes.reduce((m,n)=>{m[n.id] = null; return m }, {}))
//console.log(before)
const after = JSON.stringify(trie.T)
// console.log(trie.T)
await lit.fs.writeFile('/testing/compactManifest1.json', after, 'utf8')
return `${before.length}->${after.length}. A ${100-(after.length/before.length*100)}% reduction in size.`
})()
Looking up a file, check for existence:
return (async (fn) => {
const { CompactPrefixTree, getWordsFromTrie } = await import(
"https://cdn.skypack.dev/compact-prefix-tree"
);
const json = await fetch("/testing/compactManifest1.json").then((res) =>
res.json()
);
// return json
const keys = getWordsFromTrie(json);
const trie = new CompactPrefixTree(Array.from(keys));
return trie.prefix("testing/log/");
})();
A Start
return (async (fn) => {
const { CompactPrefixTree, getWordsFromTrie } = lit.utils.compactPrefixTree;
const resp = await fetch("/compactManifest.json");
const data = await resp.json();
const keyset = getWordsFromTrie(data);
const list = Array.from(keyset).map(k=>`/${k}`)
console.log(list[50])
return list.length
})();
- Use the above to sync local file system instead of manifest.
// fetch all remote files and store
// locally if they don't already exist
const getList = async () => {
const { CompactPrefixTree, getWordsFromTrie } = lit.utils.compactPrefixTree;
const resp = await fetch("/compactManifest.json");
const data = await resp.json();
const keyset = getWordsFromTrie(data);
const list = Array.from(keyset).map((k) => `/${k}`);
return list;
};
return (async (fn) => {
const t = Date.now();
const p = lit.utils.path;
const writePLocal = async (...args) => {};
const list = await getList();
const existed = []
const duds = [];
const synced = [];
const errors = [];
const res = await Promise.all(
list.map(async (n) => {
try {
const stats = await lit.fs
.readStat(n)
.then((x) => x)
.catch((err) => {
duds.push(n);
return { local: {}, remote: {} };
});
if (stats.local.stat) {
existed.push(n)
}
else if (stats.remote.stat) {
await lit.fs.writeFile(n, stats.remote.value, {
localOnly: true,
encoding: "utf8",
});
synced.push(n);
} else {
errors.push(n)
}
return n;
} catch (err) {
errors.push(n + " : " + err.message);
}
})
);
console.log(
`Synced ${synced.length + existed.length}/${list.length} files in ${
(Date.now() - t) / 1000
} seconds. Fetched: ${synced.length} Duds: ${duds.length} Errors: ${errors.length} Skipped: ${existed.length}`
);
return { duds, errors };
})();