Parallel crawlers

@inproceedings{cho2002parallel, abstract = {In this paper we study how we can design an effective parallel crawler. As the size of the Web grows, it becomes imperative to parallelize a crawling process, in order to finish downloading pages in a reasonable amount of time. We first propose multiple architectures for a parallel crawler and identify fundamental issues related to parallel crawling. Based on this understanding, we then propose metrics to evaluate a parallel crawler, and compare the proposed architectures using 40 million pages ...}, author = {Cho, J. and Molina, Garcia H. }, booktitle = {Proc. of the 11th International World--Wide Web Conference}, citeulike-article-id = {1586892}, keywords = {crawling, distributed-ir, next-generation}, posted-at = {2008-07-23 06:24:58}, priority = {2}, title = {Parallel crawlers}, url = {http://citeseer.ist.psu.edu/cho02parallel.html}, year = {2002} }

TY - CONF ID - cho2002parallel L3 - citeulike-article-id:1586892 TI - Parallel crawlers T2 - Proc. of the 11th International World--Wide Web Conference N2 - In this paper we study how we can design an effective parallel crawler. As the size of the Web grows, it becomes imperative to parallelize a crawling process, in order to finish downloading pages in a reasonable amount of time. We first propose multiple architectures for a parallel crawler and identify fundamental issues related to parallel crawling. Based on this understanding, we then propose metrics to evaluate a parallel crawler, and compare the proposed architectures using 40 million pages ... KW - crawling KW - distributed-ir KW - next-generation AU - Cho, J AU - Molina, Garcia PY - 2002/// UR - http://citeseer.ist.psu.edu/cho02parallel.html ER -