|
|
zs - zs - Zeitungsschau rss to email converter |
|
|
 |
git clone git://r-36.net/zs (git://r-36.net) |
|
|
 |
Log |
|
|
|
Files |
|
|
|
Refs |
|
|
|
README |
|
|
|
LICENSE |
|
|
|
--- |
|
|
|
zs (6320B) |
|
|
|
--- |
|
|
|
1 #!/usr/bin/env python |
|
|
|
2 # coding=utf-8 |
|
|
|
3 # |
|
|
|
4 # Copy me if you can. |
|
|
|
5 # by 20h |
|
|
|
6 # |
|
|
|
7 |
|
|
|
8 import sys |
|
|
|
9 import os |
|
|
|
10 import zeitungsschau.feed as feed |
|
|
|
11 import zeitungsschau.feeddb as feeddb |
|
|
|
12 import zeitungsschau.opml as opml |
|
|
|
13 import zeitungsschau.feedemail as feedemail |
|
|
|
14 import socket |
|
|
|
15 import http.client |
|
|
|
16 import ssl |
|
|
|
17 import getopt |
|
|
|
18 import pprint |
|
|
|
19 import requests.exceptions |
|
|
|
20 import requests |
|
|
|
21 import smtplib |
|
|
|
22 import lxml |
|
|
|
23 |
|
|
|
24 dodebug = False |
|
|
|
25 |
|
|
|
26 def debug(msg): |
|
|
|
27 global dodebug |
|
|
|
28 if dodebug == True: |
|
|
|
29 print("debug: %s" % (msg)) |
|
|
|
30 |
|
|
|
31 def sendfeed(db, ufeed): |
|
|
|
32 feedemail.send(ufeed, db.cfg["email"], db.cfg["smtphost"], \ |
|
|
|
33 db.cfg["smtpport"], db.cfg["smtpssl"], \ |
|
|
|
34 db.cfg["smtpstarttls"], db.cfg["smtpuser"], \ |
|
|
|
35 db.cfg["smtppassword"], db.cfg["smtpcmd"], \ |
|
|
|
36 db.cfg["smtpuselocal"]) |
|
|
|
37 |
|
|
|
38 def run(db, selfeed=None, dryrun=False, onlychanges=False): |
|
|
|
39 feeduris = db.listfeeds() |
|
|
|
40 |
|
|
|
41 if feeduris != None and selfeed in feeduris: |
|
|
|
42 feeduris = [selfeed] |
|
|
|
43 |
|
|
|
44 for feeduri in feeduris: |
|
|
|
45 if db.ispaused(feeduri): |
|
|
|
46 print("pause %s" % (feeduri)) |
|
|
|
47 continue |
|
|
|
48 |
|
|
|
49 retries = db.getretry(feeduri) |
|
|
|
50 estr = None |
|
|
|
51 if onlychanges != True: |
|
|
|
52 print("fetch %s" % (feeduri)) |
|
|
|
53 curfeed = None |
|
|
|
54 rcode = 0 |
|
|
|
55 |
|
|
|
56 """ |
|
|
|
57 # All errors. |
|
|
|
58 (rcode, curfeed) = feed.fetch(feeduri) |
|
|
|
59 """ |
|
|
|
60 try: |
|
|
|
61 (rcode, curfeed) = feed.fetch(feeduri) |
|
|
|
62 except socket.gaierror: |
|
|
|
63 continue |
|
|
|
64 except socket.timeout: |
|
|
|
65 continue |
|
|
|
66 except TimeoutError: |
|
|
|
67 continue |
|
|
|
68 except ConnectionResetError: |
|
|
|
69 estr = "connreset" |
|
|
|
70 retries += 1 |
|
|
|
71 except requests.exceptions.ConnectionError: |
|
|
|
72 estr = "connreset" |
|
|
|
73 retries += 1 |
|
|
|
74 except requests.exceptions.ReadTimeout: |
|
|
|
75 continue |
|
|
|
76 except requests.exceptions.ChunkedEncodingError: |
|
|
|
77 continue |
|
|
|
78 except lxml.etree.XMLSyntaxError: |
|
|
|
79 estr = "xml error" |
|
|
|
80 retries += 1 |
|
|
|
81 except requests.exceptions.TooManyRedirects: |
|
|
|
82 estr = "redirects" |
|
|
|
83 retries += 1 |
|
|
|
84 except OSError as err: |
|
|
|
85 estr = err.strerror |
|
|
|
86 retries += 1 |
|
|
|
87 |
|
|
|
88 if rcode == 404: |
|
|
|
89 estr = "404" |
|
|
|
90 retries += 1 |
|
|
|
91 |
|
|
|
92 if curfeed == None: |
|
|
|
93 continue |
|
|
|
94 |
|
|
|
95 # retry handling |
|
|
|
96 if estr != None: |
|
|
|
97 if retries > 2: |
|
|
|
98 sys.stderr.write("pause %s %s\n" % \ |
|
|
|
99 (estr, feeduri)) |
|
|
|
100 db.pause(feeduri) |
|
|
|
101 db.setretry(feeduri, retries) |
|
|
|
102 continue |
|
|
|
103 elif retries > 0: |
|
|
|
104 db.setretry(feeduri, 0) |
|
|
|
105 |
|
|
|
106 try: |
|
|
|
107 clen = len(curfeed["articles"]) |
|
|
|
108 except AttributeError: |
|
|
|
109 continue |
|
|
|
110 |
|
|
|
111 if clen == 0: |
|
|
|
112 # This is no target anymore. Thanks NATO for your |
|
|
|
113 # crappy RSS feed! |
|
|
|
114 #print("0 articles -> pause %s" % (feeduri)) |
|
|
|
115 #db.pause(feeduri) |
|
|
|
116 continue |
|
|
|
117 |
|
|
|
118 db.mergefeed(feeduri, curfeed) |
|
|
|
119 ufeed = db.unreadarticles(feeduri) |
|
|
|
120 if len(ufeed["articles"]) > 0 and onlychanges != True: |
|
|
|
121 print("cur %d unread %d" % (clen, \ |
|
|
|
122 len(ufeed["articles"]))) |
|
|
|
123 debug(ufeed) |
|
|
|
124 if dryrun == False: |
|
|
|
125 try: |
|
|
|
126 sendfeed(db, ufeed) |
|
|
|
127 db.setreadarticles(feeduri, ufeed) |
|
|
|
128 except smtplib.SMTPDataError: |
|
|
|
129 return |
|
|
|
130 except smtplib.SMTPSenderRefused: |
|
|
|
131 return |
|
|
|
132 |
|
|
|
133 class ExceptionHook: |
|
|
|
134 instance = None |
|
|
|
135 |
|
|
|
136 def __call__(self, *args, **kwargs): |
|
|
|
137 if self.instance is None: |
|
|
|
138 from IPython.core import ultratb |
|
|
|
139 self.instance = ultratb.FormattedTB(mode='Verbose', |
|
|
|
140 color_scheme='Linux', call_pdb=1) |
|
|
|
141 return self.instance(*args, **kwargs) |
|
|
|
142 |
|
|
|
143 def usage(app): |
|
|
|
144 app = os.path.basename(app) |
|
|
|
145 sys.stderr.write("usage: %s [-dhs] cmd\n" % (app)) |
|
|
|
146 sys.exit(1) |
|
|
|
147 |
|
|
|
148 def main(args): |
|
|
|
149 global dodebug |
|
|
|
150 retval = 0 |
|
|
|
151 |
|
|
|
152 try: |
|
|
|
153 opts, largs = getopt.getopt(args[1:], "hds") |
|
|
|
154 except getopt.GetoptError as err: |
|
|
|
155 print(str(err)) |
|
|
|
156 usage(args[0]) |
|
|
|
157 |
|
|
|
158 silent = False |
|
|
|
159 for o, a in opts: |
|
|
|
160 if o == "-h": |
|
|
|
161 usage(args[0]) |
|
|
|
162 elif o == "-d": |
|
|
|
163 dodebug = True |
|
|
|
164 elif o == "-s": |
|
|
|
165 silent = True |
|
|
|
166 else: |
|
|
|
167 usage(args[0]) |
|
|
|
168 |
|
|
|
169 if len(largs) < 1: |
|
|
|
170 usage(args[0]) |
|
|
|
171 |
|
|
|
172 if dodebug == True: |
|
|
|
173 sys.excepthook = ExceptionHook() |
|
|
|
174 |
|
|
|
175 if largs[0] == "testfeed": |
|
|
|
176 if len(largs) < 2: |
|
|
|
177 print("usage: %s testfeed URI\n" % \ |
|
|
|
178 (os.path.basename(args[0]))) |
|
|
|
179 return 1 |
|
|
|
180 |
|
|
|
181 fe = feed.fetch(largs[1]) |
|
|
|
182 pprint.pprint(fe) |
|
|
|
183 return 0 |
|
|
|
184 |
|
|
|
185 db = feeddb.feeddb() |
|
|
|
186 |
|
|
|
187 if largs[0] == "run": |
|
|
|
188 if len(largs) > 1: |
|
|
|
189 run(db, largs[1], onlychanges=silent) |
|
|
|
190 else: |
|
|
|
191 run(db, onlychanges=silent) |
|
|
|
192 |
|
|
|
193 elif largs[0] == "dryrun": |
|
|
|
194 if len(largs) > 1: |
|
|
|
195 run(db, largs[1], dryrun=True, onlychanges=silent) |
|
|
|
196 else: |
|
|
|
197 run(db, dryrun=True, onlychanges=silent) |
|
|
|
198 |
|
|
|
199 elif largs[0] == "cfg": |
|
|
|
200 if len(largs) < 2: |
|
|
|
201 for k in db.cfg: |
|
|
|
202 print("%s = '%s'" % (k, db.cfg[k])) |
|
|
|
203 elif len(args) < 3: |
|
|
|
204 if largs[1] in db.cfg: |
|
|
|
205 print("%s = '%s'" % (largs[1], \ |
|
|
|
206 db.cfg[largs[1]])) |
|
|
|
207 else: |
|
|
|
208 retval = 1 |
|
|
|
209 else: |
|
|
|
210 db.cfg[largs[1]] = largs[2] |
|
|
|
211 print("%s = '%s'" % (largs[1], db.cfg[largs[1]])) |
|
|
|
212 db.default = False |
|
|
|
213 |
|
|
|
214 elif largs[0] == "cfgdel": |
|
|
|
215 if len(largs) < 2: |
|
|
|
216 usage(args[0]) |
|
|
|
217 if largs[1] in db.cfg: |
|
|
|
218 del db.cfg[largs[1]] |
|
|
|
219 |
|
|
|
220 elif largs[0] == "add": |
|
|
|
221 if len(largs) < 2: |
|
|
|
222 usage(args[0]) |
|
|
|
223 db.addfeed(largs[1]) |
|
|
|
224 |
|
|
|
225 elif largs[0] == "list": |
|
|
|
226 print("\n".join(db.listfeeds())) |
|
|
|
227 |
|
|
|
228 elif largs[0] == "listuuids": |
|
|
|
229 if len(largs) < 2: |
|
|
|
230 usage(args[0]) |
|
|
|
231 rfeed = db.readfeed(largs[1]) |
|
|
|
232 for art in rfeed["articles"]: |
|
|
|
233 print("%s: %s: %s" % (art["uuid"], art["link"],\ |
|
|
|
234 art["title"])) |
|
|
|
235 |
|
|
|
236 elif largs[0] == "unread": |
|
|
|
237 if len(largs) < 3: |
|
|
|
238 usage(args[0]) |
|
|
|
239 db.setarticleunread(largs[1], largs[2]) |
|
|
|
240 |
|
|
|
241 elif largs[0] == "resend": |
|
|
|
242 if len(largs) < 2: |
|
|
|
243 usage(args[0]) |
|
|
|
244 ufeed = db.unreadarticles(largs[1]) |
|
|
|
245 sendfeed(db, ufeed) |
|
|
|
246 db.setreadarticles(largs[1], ufeed) |
|
|
|
247 |
|
|
|
248 elif largs[0] == "del": |
|
|
|
249 if len(largs) < 2: |
|
|
|
250 usage(args[0]) |
|
|
|
251 if db.delfeed(largs[1]) == True: |
|
|
|
252 print("'%s' has been deleted." % (largs[1])) |
|
|
|
253 |
|
|
|
254 elif largs[0] == "reset": |
|
|
|
255 if len(largs) < 2: |
|
|
|
256 usage(args[0]) |
|
|
|
257 db.resetarticles(largs[1]) |
|
|
|
258 |
|
|
|
259 elif largs[0] == "retry": |
|
|
|
260 if len(largs) < 3: |
|
|
|
261 usage(args[0]) |
|
|
|
262 db.setretry(largs[1], int(largs[2])) |
|
|
|
263 |
|
|
|
264 elif largs[0] == "pause": |
|
|
|
265 if len(largs) < 2: |
|
|
|
266 usage(args[0]) |
|
|
|
267 db.pause(largs[1]) |
|
|
|
268 |
|
|
|
269 elif largs[0] == "unpause": |
|
|
|
270 if len(largs) < 2: |
|
|
|
271 usage(args[0]) |
|
|
|
272 db.unpause(largs[1]) |
|
|
|
273 |
|
|
|
274 elif largs[0] == "opmlexport": |
|
|
|
275 if len(largs) > 1: |
|
|
|
276 filen = open(largs[1], "w") |
|
|
|
277 else: |
|
|
|
278 filen = sys.stdout |
|
|
|
279 filen.write(opml.write(db.listfeeds())) |
|
|
|
280 |
|
|
|
281 elif largs[0] == "opmlimport": |
|
|
|
282 if len(largs) > 1: |
|
|
|
283 filen = open(largs[1], "r") |
|
|
|
284 else: |
|
|
|
285 filen = sys.stdin |
|
|
|
286 feedlist = db.listfeeds() |
|
|
|
287 nfeedlist = opml.read(filen.read().encode("utf-8")) |
|
|
|
288 for f in nfeedlist: |
|
|
|
289 if not f in feedlist: |
|
|
|
290 print("import feed: %s" % (f)) |
|
|
|
291 db.addfeed(f) |
|
|
|
292 |
|
|
|
293 del db |
|
|
|
294 return retval |
|
|
|
295 |
|
|
|
296 if __name__ == "__main__": |
|
|
|
297 sys.exit(main(sys.argv)) |
|
|
|
298 |
|