| ... |
... |
@@ -1,0 +1,113 @@ |
|
1 |
+# Disallow all the website to undesirable bots |
|
2 |
+# Syntax reference: https://developers.google.com/search/docs/advanced/robots/create-robots-txt |
|
3 |
+User-agent: SemrushBot |
|
4 |
+User-agent: SiteAuditBot |
|
5 |
+User-agent: AhrefsBot |
|
6 |
+User-agent: CoherenceBot |
|
7 |
+User-agent: Deskyobot |
|
8 |
+User-agent: magpie-crawler |
|
9 |
+User-agent: MauiBot |
|
10 |
+User-agent: coccocbot-image |
|
11 |
+User-agent: coccocbot-web |
|
12 |
+User-agent: DotBot |
|
13 |
+User-agent: InfoTigerBot |
|
14 |
+User-agent: Mail.RU_Bot |
|
15 |
+User-agent: MJ12bot |
|
16 |
+User-agent: SeznamBot |
|
17 |
+User-agent: SurdotlyBot |
|
18 |
+User-agent: WellKnownBot |
|
19 |
+User-agent: YandexBot |
|
20 |
+User-agent: DataForSeoBot |
|
21 |
+User-agent: SabsimBot |
|
22 |
+User-agent: trendictionbot |
|
23 |
+User-agent: yacybot |
|
24 |
+User-agent: ZoominfoBot |
|
25 |
+Disallow: / |
|
26 |
+ |
|
27 |
+# OW2 Custom |
|
28 |
+User-agent: * |
|
29 |
+Disallow: /view/services/ |
|
30 |
+Disallow: /view/Membership_Joining/On_Line_Registration |
|
31 |
+Disallow: /services/ |
|
32 |
+Disallow: /status/ |
|
33 |
+Disallow: /xmlrpc/ |
|
34 |
+Disallow: /view/XWiki |
|
35 |
+ |
|
36 |
+# XWIKI recommendations |
|
37 |
+# https://www.xwiki.org/xwiki/bin/view/Documentation/AdminGuide/Performances/#HRobots.txt |
|
38 |
+User-agent: * |
|
39 |
+# Prevent bots from executing all actions except "view" since: |
|
40 |
+# 1) we don't want bots to execute stuff in the wiki! |
|
41 |
+# 2) we don't want bots to consume CPU and memory |
|
42 |
+# (for example to perform exports) |
|
43 |
+Disallow: /viewattachrev/ |
|
44 |
+Disallow: /viewrev/ |
|
45 |
+Disallow: /pdf/ |
|
46 |
+Disallow: /tex/ |
|
47 |
+Disallow: /edit/ |
|
48 |
+Disallow: /create/ |
|
49 |
+Disallow: /inline/ |
|
50 |
+Disallow: /preview/ |
|
51 |
+Disallow: /save/ |
|
52 |
+Disallow: /saveandcontinue/ |
|
53 |
+Disallow: /rollback/ |
|
54 |
+Disallow: /deleteversions/ |
|
55 |
+Disallow: /cancel/ |
|
56 |
+Disallow: /delete/ |
|
57 |
+Disallow: /deletespace/ |
|
58 |
+Disallow: /undelete/ |
|
59 |
+Disallow: /reset/ |
|
60 |
+Disallow: /register/ |
|
61 |
+Disallow: /propupdate/ |
|
62 |
+Disallow: /propadd/ |
|
63 |
+Disallow: /propdisable/ |
|
64 |
+Disallow: /propenable/ |
|
65 |
+Disallow: /propdelete/ |
|
66 |
+Disallow: /objectadd/ |
|
67 |
+Disallow: /commentadd/ |
|
68 |
+Disallow: /commentsave/ |
|
69 |
+Disallow: /objectsync/ |
|
70 |
+Disallow: /objectremove/ |
|
71 |
+Disallow: /attach/ |
|
72 |
+Disallow: /upload/ |
|
73 |
+Disallow: /download/ |
|
74 |
+Disallow: /temp/ |
|
75 |
+Disallow: /downloadrev/ |
|
76 |
+Disallow: /dot/ |
|
77 |
+Disallow: /svg/ |
|
78 |
+Disallow: /delattachment/ |
|
79 |
+Disallow: /login/ |
|
80 |
+Disallow: /loginsubmit/ |
|
81 |
+Disallow: /loginerror/ |
|
82 |
+Disallow: /logout/ |
|
83 |
+Disallow: /charting/ |
|
84 |
+Disallow: /lock/ |
|
85 |
+Disallow: /redirect/ |
|
86 |
+Disallow: /admin/ |
|
87 |
+Disallow: /export/ |
|
88 |
+Disallow: /import/ |
|
89 |
+Disallow: /get/ |
|
90 |
+Disallow: /distribution/ |
|
91 |
+Disallow: /imagecaptcha/ |
|
92 |
+Disallow: /unknown/ |
|
93 |
+# Don't index sandbox content since it's sample content |
|
94 |
+Disallow: /view/Sandbox/ |
|
95 |
+# Don't index Admin space since it contains Admin stuff. |
|
96 |
+# Note that the Admin space is protected by permissions |
|
97 |
+# anyway but this acts as a safety net to not have private |
|
98 |
+# info leaked on the internet ;) |
|
99 |
+Disallow: /view/Admin/ |
|
100 |
+# Don't index Stats data (just because it's not useful and |
|
101 |
+# those pages are a bit CPU intensive) |
|
102 |
+Disallow: /view/Stats/ |
|
103 |
+# Don't index Panels data (because we don't want it |
|
104 |
+# indexed on the internet) |
|
105 |
+Disallow: /view/Panels/ |
|
106 |
+# Don't index the search page. |
|
107 |
+Disallow: /Main/Search |
|
108 |
+# Don't index the REST API. |
|
109 |
+Disallow: /xwiki/rest/ |
|
110 |
+# These are just UI elements which can cause infinite loops in |
|
111 |
+# web crawlers. See https://jira.xwiki.org/browse/XWIKI-16915 |
|
112 |
+Disallow: ?*xpage=* |
|
113 |
+ |
