Merge pull request #1336 from davidalger/feature/robots-txt

Removes .js and .css files robots.txt disallow rules

Author: vpelipenko

app/code/Magento/Theme/etc/config.xml

@@ -22,11 +22,8 @@
 User-agent: *
 Disallow: /index.php/
 Disallow: /*?
-Disallow: /*.js$
-Disallow: /*.css$
 Disallow: /checkout/
 Disallow: /app/
-Disallow: /js/
 Disallow: /lib/
 Disallow: /*.php$
 Disallow: /pkginfo/