Skip to Content
From version 1278.1
edited by OW2
on 2023/07/31 14:19
Change comment: Uploaded new attachment "robots.txt", version 1.1
To version 1277.3
edited by OW2
on 2023/01/16 08:33
Change comment: There is no comment for this version

Summary

Details

robots.txt
Author
... ... @@ -1,1 +1,0 @@
1 -xwiki:XWiki.wadmin
Size
... ... @@ -1,1 +1,0 @@
1 -2.9 KB
Content
... ... @@ -1,113 +1,0 @@
1 -# Disallow all the website to undesirable bots
2 -# Syntax reference: https://developers.google.com/search/docs/advanced/robots/create-robots-txt
3 -User-agent: SemrushBot
4 -User-agent: SiteAuditBot
5 -User-agent: AhrefsBot
6 -User-agent: CoherenceBot
7 -User-agent: Deskyobot
8 -User-agent: magpie-crawler
9 -User-agent: MauiBot
10 -User-agent: coccocbot-image
11 -User-agent: coccocbot-web
12 -User-agent: DotBot
13 -User-agent: InfoTigerBot
14 -User-agent: Mail.RU_Bot
15 -User-agent: MJ12bot
16 -User-agent: SeznamBot
17 -User-agent: SurdotlyBot
18 -User-agent: WellKnownBot
19 -User-agent: YandexBot
20 -User-agent: DataForSeoBot
21 -User-agent: SabsimBot
22 -User-agent: trendictionbot
23 -User-agent: yacybot
24 -User-agent: ZoominfoBot
25 -Disallow: /
26 -
27 -# OW2 Custom
28 -User-agent: *
29 -Disallow: /view/services/
30 -Disallow: /view/Membership_Joining/On_Line_Registration
31 -Disallow: /services/
32 -Disallow: /status/
33 -Disallow: /xmlrpc/
34 -Disallow: /view/XWiki
35 -
36 -# XWIKI recommendations
37 -# https://www.xwiki.org/xwiki/bin/view/Documentation/AdminGuide/Performances/#HRobots.txt
38 -User-agent: *
39 -# Prevent bots from executing all actions except "view" since:
40 -# 1) we don't want bots to execute stuff in the wiki!
41 -# 2) we don't want bots to consume CPU and memory
42 -# (for example to perform exports)
43 -Disallow: /viewattachrev/
44 -Disallow: /viewrev/
45 -Disallow: /pdf/
46 -Disallow: /tex/
47 -Disallow: /edit/
48 -Disallow: /create/
49 -Disallow: /inline/
50 -Disallow: /preview/
51 -Disallow: /save/
52 -Disallow: /saveandcontinue/
53 -Disallow: /rollback/
54 -Disallow: /deleteversions/
55 -Disallow: /cancel/
56 -Disallow: /delete/
57 -Disallow: /deletespace/
58 -Disallow: /undelete/
59 -Disallow: /reset/
60 -Disallow: /register/
61 -Disallow: /propupdate/
62 -Disallow: /propadd/
63 -Disallow: /propdisable/
64 -Disallow: /propenable/
65 -Disallow: /propdelete/
66 -Disallow: /objectadd/
67 -Disallow: /commentadd/
68 -Disallow: /commentsave/
69 -Disallow: /objectsync/
70 -Disallow: /objectremove/
71 -Disallow: /attach/
72 -Disallow: /upload/
73 -Disallow: /download/
74 -Disallow: /temp/
75 -Disallow: /downloadrev/
76 -Disallow: /dot/
77 -Disallow: /svg/
78 -Disallow: /delattachment/
79 -Disallow: /login/
80 -Disallow: /loginsubmit/
81 -Disallow: /loginerror/
82 -Disallow: /logout/
83 -Disallow: /charting/
84 -Disallow: /lock/
85 -Disallow: /redirect/
86 -Disallow: /admin/
87 -Disallow: /export/
88 -Disallow: /import/
89 -Disallow: /get/
90 -Disallow: /distribution/
91 -Disallow: /imagecaptcha/
92 -Disallow: /unknown/
93 -# Don't index sandbox content since it's sample content
94 -Disallow: /view/Sandbox/
95 -# Don't index Admin space since it contains Admin stuff.
96 -# Note that the Admin space is protected by permissions
97 -# anyway but this acts as a safety net to not have private
98 -# info leaked on the internet ;)
99 -Disallow: /view/Admin/
100 -# Don't index Stats data (just because it's not useful and
101 -# those pages are a bit CPU intensive)
102 -Disallow: /view/Stats/
103 -# Don't index Panels data (because we don't want it
104 -# indexed on the internet)
105 -Disallow: /view/Panels/
106 -# Don't index the search page.
107 -Disallow: /Main/Search
108 -# Don't index the REST API.
109 -Disallow: /xwiki/rest/
110 -# These are just UI elements which can cause infinite loops in
111 -# web crawlers. See https://jira.xwiki.org/browse/XWIKI-16915
112 -Disallow: ?*xpage=*
113 -