|
|
lawn-hark.sh - gopher-lawn - The gopher lawn gopher directory project. |
|
|
 |
git clone git://bitreich.org/gopher-lawn/ git://enlrupgkhuxnvlhsf6lc3fziv5h2hhfrinws65d7roiv6bfj7d652fid.onion/gopher-lawn/ (git://bitreich.org) |
|
|
 |
Log |
|
|
|
Files |
|
|
|
Refs |
|
|
|
Tags |
|
|
|
--- |
|
|
|
lawn-hark.sh (4426B) |
|
|
|
--- |
|
|
|
1 #!/bin/bash |
|
|
|
2 # |
|
|
|
3 # Originally written by pazz0. |
|
|
|
4 # |
|
|
|
5 |
|
|
|
6 errorthreshold=3 |
|
|
|
7 timeout=15 |
|
|
|
8 maxworkers=8 |
|
|
|
9 onionsocksproxy="127.0.0.1:9050" |
|
|
|
10 |
|
|
|
11 function tcpdial() { |
|
|
|
12 if [ -z "${1##*.onion}" ]; |
|
|
|
13 then |
|
|
|
14 nc -w "${timeout}" -X 5 -x "${onionsocksproxy}" "$1" "$2" \ |
|
|
|
15 2>/dev/null |
|
|
|
16 else |
|
|
|
17 nc -w "${timeout}" "$1" "$2" 2>/dev/null |
|
|
|
18 fi |
|
|
|
19 } |
|
|
|
20 |
|
|
|
21 function checkgopher() { |
|
|
|
22 if [ "$#" -gt 3 ]; |
|
|
|
23 then |
|
|
|
24 data="$(printf "%s\t%s\r\n" "$3" "$4" \ |
|
|
|
25 | tcpdial "$1" "$2" \ |
|
|
|
26 | dd bs=128 count=1 2>/dev/null)" |
|
|
|
27 else |
|
|
|
28 data="$(printf "%s\r\n" "$3" \ |
|
|
|
29 | tcpdial "$1" "$2" \ |
|
|
|
30 | dd bs=128 count=1 2>/dev/null)" |
|
|
|
31 fi |
|
|
|
32 |
|
|
|
33 if [ -z "${data}" ]; |
|
|
|
34 then |
|
|
|
35 printf "Can't connect, timeout or no content\n" |
|
|
|
36 return 1 |
|
|
|
37 fi |
|
|
|
38 |
|
|
|
39 if [ -z "${data##3?* *}" ]; |
|
|
|
40 then |
|
|
|
41 printf "Got type '3' on first line\n" |
|
|
|
42 return 1 |
|
|
|
43 fi |
|
|
|
44 |
|
|
|
45 if [ -z "${data##Error: File or directory not found!*}" ] \ |
|
|
|
46 || [ -z "${data##Error: Access denied!*}" ]; |
|
|
|
47 then |
|
|
|
48 printf "Got Gophernicus error\n" |
|
|
|
49 return 1 |
|
|
|
50 fi |
|
|
|
51 |
|
|
|
52 return 0 |
|
|
|
53 } |
|
|
|
54 |
|
|
|
55 function checkhttp() { |
|
|
|
56 # max. one redirect (2) for http->https things |
|
|
|
57 bc="$(curl -L --max-redirs 2 -m "${timeout}" -f "$1" 2>/dev/null \ |
|
|
|
58 | dd bs=16 count=1 2>/dev/null \ |
|
|
|
59 | wc -c \ |
|
|
|
60 | xargs)" |
|
|
|
61 if [ "${bc}" -eq 0 ]; |
|
|
|
62 then |
|
|
|
63 printf "Can't connect, timout, too many redirects or no content\n" |
|
|
|
64 return 1 |
|
|
|
65 fi |
|
|
|
66 |
|
|
|
67 return 0 |
|
|
|
68 } |
|
|
|
69 |
|
|
|
70 function checkcso() { |
|
|
|
71 bc="$(printf "status\r\n" \ |
|
|
|
72 | tcpdial "$1" "$2" \ |
|
|
|
73 | dd bs=16 count=1 2>/dev/null \ |
|
|
|
74 | wc -c \ |
|
|
|
75 | xargs)" |
|
|
|
76 if [ "${bc}" -eq 0 ]; |
|
|
|
77 then |
|
|
|
78 printf "Can't connect, timeout or no content\n" |
|
|
|
79 return 1 |
|
|
|
80 fi |
|
|
|
81 |
|
|
|
82 return 0 |
|
|
|
83 } |
|
|
|
84 |
|
|
|
85 function checkraw() { |
|
|
|
86 bc="$(tcpdial "$1" "$2" \ |
|
|
|
87 | dd bs=16 count=1 2>/dev/null \ |
|
|
|
88 | wc -c \ |
|
|
|
89 | xargs)" |
|
|
|
90 if [ "${bc}" -eq 0 ]; |
|
|
|
91 then |
|
|
|
92 printf "Can't connect, timeout or no content\n" |
|
|
|
93 return 1 |
|
|
|
94 fi |
|
|
|
95 |
|
|
|
96 return 0 |
|
|
|
97 } |
|
|
|
98 |
|
|
|
99 program="$(readlink -f "$0")" |
|
|
|
100 |
|
|
|
101 if [ "${LAWNHARK_WORKER}" = "1" ]; |
|
|
|
102 then |
|
|
|
103 statedir="$1" |
|
|
|
104 checktime="$2" |
|
|
|
105 f="$3" |
|
|
|
106 |
|
|
|
107 type="" |
|
|
|
108 selector="" |
|
|
|
109 host="" |
|
|
|
110 port="" |
|
|
|
111 |
|
|
|
112 while read -r line; |
|
|
|
113 do |
|
|
|
114 value="$(printf '%s\n' "${line}" \ |
|
|
|
115 | cut -f 2- -d ':' \ |
|
|
|
116 | sed -n 's/^[[:space:]]*\(.*\)$/\1/p')" |
|
|
|
117 case "${line}" in |
|
|
|
118 Type:* ) |
|
|
|
119 type="${value}" |
|
|
|
120 ;; |
|
|
|
121 Selector:* ) |
|
|
|
122 selector="${value}" |
|
|
|
123 ;; |
|
|
|
124 Host:* ) |
|
|
|
125 host="${value}" |
|
|
|
126 ;; |
|
|
|
127 Port:* ) |
|
|
|
128 port="${value}" |
|
|
|
129 ;; |
|
|
|
130 esac |
|
|
|
131 done < "$f" |
|
|
|
132 |
|
|
|
133 if [ -z "${type}" ] \ |
|
|
|
134 || [ -z "${host}" ] \ |
|
|
|
135 || [ -z "${port}" ]; |
|
|
|
136 then |
|
|
|
137 flock -x "${program}" printf "ERROR\t%s\tInvalid entry!\n" "${f}" >&2 |
|
|
|
138 exit |
|
|
|
139 fi |
|
|
|
140 |
|
|
|
141 case "${type}" in |
|
|
|
142 cso ) |
|
|
|
143 error="$(checkcso "${host}" "${port}")" |
|
|
|
144 ;; |
|
|
|
145 telnet ) |
|
|
|
146 error="$(checkraw "${host}" "${port}")" |
|
|
|
147 ;; |
|
|
|
148 error ) |
|
|
|
149 error="Type = 'error'" |
|
|
|
150 ;; |
|
|
|
151 link ) |
|
|
|
152 if [ -n "${selector}" ] && [ -z "${selector##URL:*}" ]; |
|
|
|
153 then |
|
|
|
154 url="${selector##URL:}" |
|
|
|
155 case "${url}" in |
|
|
|
156 http://* | https://* ) |
|
|
|
157 error="$(checkhttp "${url}")" |
|
|
|
158 ;; |
|
|
|
159 ssh://* ) |
|
|
|
160 sshhost="${url##ssh://}" |
|
|
|
161 sshhost="${sshhost##*@}" |
|
|
|
162 sshhost="${sshhost%%/*}" |
|
|
|
163 sshport="22" |
|
|
|
164 if [ -z "${sshhost##*:*}" ]; |
|
|
|
165 then |
|
|
|
166 sshport="${sshhost##*:}" |
|
|
|
167 sshhost="${sshhost%%:*}" |
|
|
|
168 fi |
|
|
|
169 error="$(checkraw "${sshhost}" "${sshport}")" |
|
|
|
170 ;; |
|
|
|
171 * ) |
|
|
|
172 flock -x "${program}" printf "TODO\t%s\tCan't handle %s\n" "${f}" "${url}" >&2 |
|
|
|
173 exit |
|
|
|
174 ;; |
|
|
|
175 esac |
|
|
|
176 else |
|
|
|
177 error="$(checkgopher "${host}" "${port}" "${selector}")" |
|
|
|
178 fi |
|
|
|
179 ;; |
|
|
|
180 search ) |
|
|
|
181 error="$(checkgopher "${host}" "${port}" "${selector}" "")" |
|
|
|
182 ;; |
|
|
|
183 text | uuencoded | * ) |
|
|
|
184 error="$(checkgopher "${host}" "${port}" "${selector}")" |
|
|
|
185 ;; |
|
|
|
186 esac |
|
|
|
187 |
|
|
|
188 lastcheck="" |
|
|
|
189 errorcount=0 |
|
|
|
190 statefile="${statedir}/$(basename "$f")" |
|
|
|
191 if [ -f "${statefile}" ]; |
|
|
|
192 then |
|
|
|
193 IFS=" " read -r lastcheck errorcount < "${statefile}" |
|
|
|
194 fi |
|
|
|
195 |
|
|
|
196 if [ -n "${error}" ]; |
|
|
|
197 then |
|
|
|
198 errorcount=$((errorcount + 1)) |
|
|
|
199 else |
|
|
|
200 errorcount=0 |
|
|
|
201 fi |
|
|
|
202 |
|
|
|
203 if [ ${errorcount} -ge ${errorthreshold} ]; |
|
|
|
204 then |
|
|
|
205 flock -x "${program}" printf "ERROR\t%s\t%s\n" "${f}" "${error}" >&2 |
|
|
|
206 fi |
|
|
|
207 |
|
|
|
208 printf "%s\t%s\n" "${checktime}" "${errorcount}" > "${statefile}" |
|
|
|
209 else |
|
|
|
210 checktime="$(date +%s)" |
|
|
|
211 statedir="$1" |
|
|
|
212 |
|
|
|
213 if [ -z "${statedir}" ]; |
|
|
|
214 then |
|
|
|
215 printf "You need to specify a state dir.\n" >&2 |
|
|
|
216 exit 1 |
|
|
|
217 fi |
|
|
|
218 |
|
|
|
219 mkdir -p "${statedir}" |
|
|
|
220 if [ ! -d "${statedir}" ]; |
|
|
|
221 then |
|
|
|
222 printf "%s is not a directory! Aborting.\n" "${statedir}" >&2 |
|
|
|
223 exit 1 |
|
|
|
224 fi |
|
|
|
225 |
|
|
|
226 shift |
|
|
|
227 |
|
|
|
228 for f; |
|
|
|
229 do |
|
|
|
230 printf "%s\0" "${f}" |
|
|
|
231 done | LAWNHARK_WORKER=1 xargs -r -0 -P "${maxworkers}" -L1 "${program}" "${statedir}" "${checktime}" |
|
|
|
232 |
|
|
|
233 # garbage collection |
|
|
|
234 find "${statedir}" -type f | while read -r f; |
|
|
|
235 do |
|
|
|
236 IFS=" " read -r lastcheck errorcount < "${f}" |
|
|
|
237 |
|
|
|
238 if [ ${lastcheck} -ne ${checktime} ]; |
|
|
|
239 then |
|
|
|
240 rm -f "${f}" |
|
|
|
241 fi |
|
|
|
242 done |
|
|
|
243 fi |
|