# this is for all bots and spiders User-agent: * # do not follow/index these # prevents a robot looping through unite/archives/css/css/css/css Disallow: /unite/*css*$ # do follow/index these #Allow: