Make robots.txt to block duplicate pages

_config.yml

@@ -11,13 +11,9 @@ sass:
 
 defaults:
   - scope:
-      path: ""
+      path: "*.md"
     values:
       layout: wiki
-  - scope:
-      path: assets/js
-    values:
-      layout: null
 
 collections:
   en-US:

robots.txt

@@ -0,0 +1,15 @@
+---
+---
+
+User-agent: *
+Disallow: /ic-IC/
+{% for collection in site.collections %}
+	{%- unless collection.label == "ic-IC" -%}
+		{%- for item in collection.docs -%}
+
+{% unless item.lang == collection.label %}Disallow: {{ item.url }}
+{% endunless %}
+
+		{%- endfor -%}
+	{%- endunless -%}
+{%- endfor -%}