RotondeDirectoryCrawler
"use strict";
let queue = [];
let knownUsers = {};
let loadedUsers = {};
let userCount = 0;
let fetching = 0;

function getID(id) { return document.getElementById(id); }

function addUser(url, portal) {
	let p = document.createElement('li');
	let a = document.createElement('a');
	a.innerHTML = '@'+portal.name; 
	a.href = url;
	p.appendChild(a);
	getID("userlist").appendChild(p);
	++userCount;
	getID("user-count").innerHTML = userCount;
	--fetching;
	getID("fetch-count").innerHTML = fetching;
}

function cleanURL(url) {
	url = url.trim();
	while(url[url.length-1] == '/') {
		url = url.slice(0, -1);
	}
	return url + '/';
}

async function loadSite(url) {
	if(loadedUsers[url]) return;
	loadedUsers[url] = true;
	try {
		++fetching;
		getID("fetch-count").innerHTML = fetching;
		let archive = new DatArchive(url);
		let data = await archive.readFile('/portal.json');
		let portal = JSON.parse(data);
		addUser(url, portal);
		for(let i=0; i<portal.port.length; ++i) {
			let p = cleanURL(portal.port[i]);
			if(!knownUsers[p]) {
				knownUsers[p] = true;
				queue.push(p);
			}
		}
	} catch(err) {
		--fetching;
		getID("fetch-count").innerHTML = fetching;
	}
}

function tick() {
	if(queue.length > 0) {
		let url = queue.shift();
		loadSite(url);
	}
	requestAnimationFrame(tick);
}

async function main() {
	getID("discover-form").onsubmit = (e)=>{
		e.preventDefault();
		queue = [];
		knownUsers = [];
		loadedUsers = [];
		userCount = 0;
		getID("user-count").innerHTML = '0';
		getID("userlist").innerHTML = '';
		queue.push(cleanURL(getID("root-url").value));
	};
	tick();
}

main()