robots.txt에 대한 추가 사례 입니다.

New York Times ( http://times.com/robots.txt )

# robots.txt, www.nytimes.com 9/24/2008
#
User-agent: *
Disallow: /adx/bin/
Disallow: /aponline/
Disallow: /archives/
Disallow: /cnet/
Disallow: /college/
Disallow: /external/
Disallow: /financialtimes/
Disallow: /idg/
Disallow: /indexes/
Disallow: /learning/
Disallow: /library/
Disallow: /nytimes-partners/
Disallow: /pages/college/
Disallow: /paidcontent/
Disallow: /partners/
Disallow: /reuters/
Disallow: /thestreet/
Allow: /pages/
Allow: /2003/
Allow: /2004/
Allow: /2005/
Allow: /top/
Allow: /ref/
Allow: /services/xml/

User-agent: Mediapartners-Google*
Disallow:

Sitemap: http://spiderbites.nytimes.com/sitemaps/www.nytimes.com/sitemap.xml.gz

아마존닷컴 ( http://www.amazon.com/robots.txt )

# Disallow all crawlers access to certain pages.

User-agent: *
Disallow: /exec/obidos/account-access-login
Disallow: /exec/obidos/change-style
Disallow: /exec/obidos/flex-sign-in
Disallow: /exec/obidos/handle-buy-box
Disallow: /exec/obidos/tg/cm/member
Disallow: /gp/cart
Disallow: /gp/flex
Disallow: /gp/product/e-mail-friend
Disallow: /gp/product/product-availability
Disallow: /gp/product/rate-this-item
Disallow: /gp/sign-in
Disallow: /gp/reader
Disallow: /gp/sitbv3/reader
Disallow: /gp/richpub/syltguides/create
Disallow: /gp/gfix
Disallow: /gp/associations/wizard.html
Disallow: /gp/dmusic/order
Disallow: /gp/legacy-handle-buy-box.html
Disallow: /gp/aws/ssop
Disallow: /gp/yourstore
Disallow: /gp/gift-central/organizer/add-wishlist
Disallow: /gp/vote
Disallow: /gp/music/wma-pop-up
Disallow: /gp/customer-images
Disallow: /gp/richpub/listmania/createpipeline
Disallow: /gp/content-form
Disallow: /gp/pdp/invitation/invite
Disallow: /gp/customer-reviews/write-a-review.html
Disallow: /gp/associations/wizard.html
Disallow: /gp/music/clipserve
Disallow: /gp/offer-listing
Disallow: /gp/customer-media/upload
Disallow: /gp/history
Disallow: /gp/item-dispatch
Disallow: /gp/dmusic/order/handle-buy-box.html
Disallow: /gp/recsradio

# Sitemap files
Sitemap: http://www.amazon.com/sitemap-manual-index.xml
Sitemap: http://www.amazon.com/sitemap-manual-tv.xml
Sitemap: http://www.amazon.com/sitemap-us-tbb.xml
Sitemap: http://www.amazon.com/sitemap_artists_index.xml
Sitemap: http://www.amazon.com/sitemap_backfill_dp_index.xml
Sitemap: http://www.amazon.com/sitemap_index_0.xml
Sitemap: http://www.amazon.com/sitemap_index_1.xml
Sitemap: http://www.amazon.com/sitemap_index_2.xml
Sitemap: http://www.amazon.com/sitemap_index_3.xml
Sitemap: http://www.amazon.com/sitemap_index_4.xml
iProspect사 (글로벌 SEO 전문회사) ( http://www.iprospect.com/robots.txt )
User-agent: *


Disallow: /Forrester_Wave/08-Wave-Report-Clean.ppt
Disallow: event/client_connections_2008.htm
Disallow: event/cc_belanger.htm
Disallow: event/cc_schmidt.htm
Disallow: event/cc_colborn.htm
Disallow: event/cc_vanboskirk.htm
Disallow: /forrester
Disallow: /messaging
Disallow: /services/international-search-marketing.htm
Disallow: /services/international-search-engine-marketing.htm
Disallow: /treasures
Disallow: /feeds1
Disallow: /feeds2
Disallow: /feed
Disallow: /media/article_directmag_5_31_06.htm
Disallow: /media/article_directmag_6_15_06.htm
Disallow: /corporate/wbcn_careers.htm
Disallow: /search-engine-marketing-results/case-study05.htm
Disallow: /search-engine-marketing-results/case-studies_conversion.htm
Disallow: /ecampaign
Disallow: /media/request-speaker-marckini.htm
Disallow: /media/request-speaker-tawadros.htm
Disallow: /media/request-speaker-peterson.htm
Disallow: /media/request-speaker-murray.htm
Disallow: /mdg_revision
Disallow: /ecampaigns
Disallow: /premiumPDFs/affiliate_summit.pdf 
Disallow: /premiumPDFs/WhitePaper_2005-Jan_TimeEffect.pdf
Disallow: /premiumPDFs/Nugget_2005-Jul_GoogleSitemaps.pdf
Disallow: /premiumPDFs/Nugget_2005-Jul_MyAllyMyEnemy.pdf
Disallow: /premiumPDFs/Nugget_2005-Aug_GoingGlobal.pdf
Disallow: /premiumPDFs/Nugget_2005-Aug_ReLaunchingYourWebsite.pdf
Disallow: /premiumPDFs/Nugget_2005-Aug_ContextualAds.pdf
Disallow: /premiumPDFs/Nugget_2005-Jul_LinkPopularityGoodLinkvsBadLink.pdf
Disallow: /premiumPDFs/Nugget_04-Nov_Outsource.pdf
Disallow: /premiumPDFs/Nugget_04-Nov_SelectingSEM.pdf
Disallow: /premiumPDFs/Nugget_04-Oct_ClickFraud.pdf
Disallow: /premiumPDFs/Nugget_04-June_SEOvsPPC.pdf
Disallow: /premiumPDFs/Nugget_04-Oct_SEMiterative.pdf
Disallow: /premiumPDFs/math_formulas_march05.pdf
Disallow: /premiumPDFs/iprospect_brochure2005.pdf
Disallow: /premiumPDFs/Nugget_03-Jul_BoostLinkPop.pdf
Disallow: /premiumPDFs/Nugget_03-Jul_Conversion.pdf
Disallow: /premiumPDFs/Nugget_04-Aug_SrchFriendlySites.pdf
Disallow: /premiumPDFs/Nugget_04-Dec_OptimizeFlash.pdf
Disallow: /premiumPDFs/Nugget_04-July_Personas.pdf
Disallow: /premiumPDFs/Nugget_04-June_BuildingLinkPop.pdf
Disallow: /premiumPDFs/Nugget_04-June_PIprograms.pdf
Disallow: /premiumPDFs/Nugget_04-May_SiteRedesign.pdf
Disallow: /premiumPDFs/Nugget_04-Nov_NowWhat.pdf
Disallow: /premiumPDFs/WhitePaper_2004-Dec_SelectSEM.pdf
Disallow: /premiumPDFs/WhitePaper_2004-July_PPCGoogle01.pdf
Disallow: /premiumPDFs/WhitePaper_2005-Jan_Beware.pdf
Disallow: /rawhide_jan2005
Disallow: /index-temp.htm
Disallow: /media/article2002_06_17.htm
Disallow: /media/article2003_12_17.htm
Disallow: /media/article2004_04_08.htm
Disallow: /media/clip_04_0504.htm
Disallow: /media/clip_04_12_03.htm
Disallow: /media/excerpt042304.htm
Disallow: /media/excerpt043004.htm
Disallow: /media/article2003_06_16.htm
Disallow: /media/article2004_05_13.htm
Disallow: /avantmarketer_sep04
Disallow: /adtech_chicago
Disallow: /adtech_nyc
Disallow: /adtech_sanfran
Disallow: /affiliate_summit
Disallow: /as_0605
Disallow: /conference_vt.nh_may2004
Disallow: /contentbiz_may2004
Disallow: /copharm0105
Disallow: /cos_email_0205
Disallow: /cj_0305
Disallow: /ema04
Disallow: /etail0205
Disallow: /frost1004
Disallow: /ses_0205
Disallow: /ses_0205_ben
Disallow: /ses_0205_jt
Disallow: /frost0105
Disallow: /frost_boston04
Disallow: /fn01_convertingtraffic
Disallow: /iaw_sept04
Disallow: /idate2004
Disallow: /internet_planet_june04
Disallow: /jupiter_adforum_july2004
Disallow: /mitx_may2004
Disallow: /morebangforbuck_mdn
Disallow: /morebangforbuck_os
Disallow: /netconnect2004
Disallow: /presentation_ucea
Disallow: /pricing
Disallow: /searchmarketinginfo_01
Disallow: /ses_aug2004
Disallow: /ses_march2004
Disallow: /travelcom-attendees2004_04
Disallow: /wbt04
Disallow: /webinar_sept2004
Disallow: /webinar_aug2004
Disallow: /webinar_july2004
Disallow: /webinar_apr292004
Disallow: /webinar_apr2004
Disallow: /webinar_feb2004_ama
Disallow: /webinar_jan2004
Disallow: /webinar_jan6_05
Disallow: /webinar_0205
Disallow: /webinar_0305
Disallow: /webinar_0405
Disallow: /webinar_0505
Disallow: /webinar_0605
Disallow: /ema_0405
Disallow: /ucea_0205
Disallow: /ir0105
Disallow: /webinar_0106
Disallow: /mayo_0206
Disallow: /as_0106
Disallow: /ses_0206
Disallow: /ses_1205
Disallow: /shop_0106
Disallow: /adtech_0306
Disallow: /adclub_0306
Disallow: /webtrends_aug04
Disallow: /corporate/privacypolicy.htm
Disallow: /corporate/termsofuse.htm
Disallow: /corporate/copyright.htm
Disallow: /corporate/employment_algorithmicanalyst.htm
Disallow: /corporate/employment_conversionservices.htm
Disallow: /corporate/employment_csm.htm
Disallow: /corporate/employment_dataminer.htm
Disallow: /corporate/employment_exectocel.htm
Disallow: /corporate/employment_executiveassistant.htm
Disallow: /corporate/employment_marketingstrategist.htm
Disallow: /corporate/employment_mediarelations.htm
Disallow: /corporate/employment_paidmediaanalyst.htm
Disallow: /corporate/employment_paidmediasales.htm
Disallow: /corporate/employment_receptionist.htm
Disallow: /corporate/employment_regionalsalesmanager.htm
Disallow: /corporate/employment_reportdeveloper.htm
Disallow: /corporate/employment_researchanalyst01.htm
Disallow: /corporate/employment_salescoordinator.htm
Disallow: /corporate/employment_salesengineer.htm
Disallow: /corporate/employment_searchspecialist.htm
Disallow: /corporate/employment_seniorinterfacedesigner.htm
Disallow: /corporate/employment_staffaccountant.htm
Disallow: /corporate/employment_technicalsearchmarketinganalyst.htm
Disallow: /corporate/employment_websitecopyeditor.htm
Disallow: /event/client_connections_2008.htm
Disallow: /media/nwslttr_aug_08.pdf
Disallow: /corporate/muller_website.jpg
Disallow: /corporate/wilson_website.jpg
Disallow: /corporate/wheeler_website.jpg
Disallow: /corporate/murray_website.jpg
Disallow: /corporate/ferrara_website.jpg
Disallow: /corporate/tawadros_website.jpg
Disallow: /corporate/kaminski_website.jpg
Disallow: /event/DebRevision.jpg