@article{brewington:keepup,
    author = {Brian Brewington and George Cybenko},
    title = {Keeping up with the changing web},
    journal = {IEEE Computer},
    year = {2000},
    month = {May},
    volume = {33},
    number = {5},
    pages = {52--58},
    publisher = {IEEE Computer Society Press},
    copyright = {IEEE},
    group = {agents, actcomm, cmc, coabs},
    url = {http://csdl.computer.org/comp/mags/co/2000/05/r5052abs.htm},
    urlpdf = {http://csdl.computer.org/dl/mags/co/2000/05/r5052.pdf},
    keyword = {WWW, scalability, information retrieval},
    abstract = {Because information depreciates over time, keeping Web pages
current presents new design challenges. This article quantifies what
"current" means for Web search engines and estimates how often they must
reindex the Web to keep current with its changing pages and structure. \par
Most information--from a newspaper story to a temperature sensor measurement
to a Web page--is dynamic. When monitoring an information source, when do our
previous observations become stale and need refreshing? How can we schedule
these refresh operations to satisfy a required level of currency without
violating resource constraints--such as band-width or computing limitations
on how much data can be observed in a given time? \par The authors
investigate the trade-offs involved in monitoring dynamic information sources
and discuss the Web in detail, estimating how fast documents change and
exploring what constitutes a "current" Web index. For a simple class of
Web-monitoring systems--search engines--they combine their idea of currency
with actual measured data to estimate revisit rates.}
}