# useless stuff User-agent: * Crawl-delay: 15 Disallow: /cgi-bin/ Disallow: /dl/ # # some harvesting fun Disallow: /wavexx/cv. Disallow: /null # # hi googlecrap Disallow: /*.apk$ Disallow: /*.asc$ Disallow: /*.deb$ Disallow: /*.dmg$ Disallow: /*.log$ Disallow: /*.log.*$ Disallow: /*.qpr$ Disallow: /*.rpm$ Disallow: /*.sig$ Disallow: /*.tar.*$ Disallow: /*.xpi$ Disallow: /*.zip$ # # avoid dumb bots getting stuck in long-gone hierarchies Disallow: /~wavexx/software/trend/contrib/ Disallow: /~wavexx/software/trend/rpms/ # # SEO and garbage bots User-agent: AhrefsBot User-agent: AwarioRssBot User-agent: AwarioSmartBot User-agent: BLEXBot User-agent: Barkrowler User-agent: Bytespider User-agent: CensysInspect User-agent: DataForSeoBot User-agent: DomainStatsBot User-agent: DotBot User-agent: Expanse User-agent: GPTBot User-agent: MJ12bot User-agent: Pandalytics User-agent: SEOkicks User-agent: Screaming Frog SEO Spider User-agent: SemrushBot User-agent: WellKnownBot User-agent: ZoominfoBot User-agent: fluid User-agent: serpstatbot Disallow: /