# All robots will spider the domain
User-agent: *
Disallow: 

# Disallow directory /css/
User-agent: *
Disallow: /css/

# Disallow directory /images/
User-agent: *
Disallow: /images/

# Disallow directory /download/
User-agent: *
Disallow: /download/

# Disallow directory /_private/
User-agent: *
Disallow: /_private/

# Disallow directory /cgi-bin/
User-agent: *
Disallow: /cgi-bin/

# Disallow directory /mdb-database/
User-agent: *
Disallow: /mdb-database/

# Disallow directory /public/
User-agent: *
Disallow: /public/