/Demo/scripts/newslist.py

  1#! /usr/bin/env python
  2#######################################################################
  3# Newslist  $Revision: 66429 $
  4#
  5# Syntax:
  6#    newslist [ -a ]
  7#
  8# This is a program to create a directory full of HTML pages
  9# which between them contain links to all the newsgroups available
 10# on your server.
 11#
 12# The -a option causes a complete list of all groups to be read from
 13# the server rather than just the ones which have appeared since last
 14# execution. This recreates the local list from scratch. Use this on
 15# the first invocation of the program, and from time to time thereafter.
 16#   When new groups are first created they may appear on your server as
 17# empty groups. By default, empty groups are ignored by the -a option.
 18# However, these new groups will not be created again, and so will not
 19# appear in the server's list of 'new groups' at a later date. Hence it
 20# won't appear until you do a '-a' after some articles have appeared.
 21#
 22# I should really keep a list of ignored empty groups and re-check them
 23# for articles on every run, but I haven't got around to it yet.
 24#
 25# This assumes an NNTP news feed.
 26#
 27# Feel free to copy, distribute and modify this code for
 28# non-commercial use. If you make any useful modifications, let me
 29# know!
 30#
 31# (c) Quentin Stafford-Fraser 1994
 32# fraser@europarc.xerox.com                     qs101@cl.cam.ac.uk
 33#                                                                     #
 34#######################################################################
 35import sys,nntplib, string, marshal, time, os, posix, string
 36
 37#######################################################################
 38# Check these variables before running!                               #
 39
 40# Top directory.
 41# Filenames which don't start with / are taken as being relative to this.
 42topdir='/anfs/qsbigdisc/web/html/newspage'
 43
 44# The name of your NNTP host
 45# eg.
 46#    newshost = 'nntp-serv.cl.cam.ac.uk'
 47# or use following to get the name from the NNTPSERVER environment
 48# variable:
 49#    newshost = posix.environ['NNTPSERVER']
 50newshost = 'nntp-serv.cl.cam.ac.uk'
 51
 52# The filename for a local cache of the newsgroup list
 53treefile = 'grouptree'
 54
 55# The filename for descriptions of newsgroups
 56# I found a suitable one at ftp.uu.net in /uunet-info/newgroups.gz
 57# You can set this to '' if you don't wish to use one.
 58descfile = 'newsgroups'
 59
 60# The directory in which HTML pages should be created
 61# eg.
 62#   pagedir  = '/usr/local/lib/html/newspage'
 63#   pagedir  = 'pages'
 64pagedir  = topdir
 65
 66# The html prefix which will refer to this directory
 67# eg.
 68#   httppref = '/newspage/',
 69# or leave blank for relative links between pages: (Recommended)
 70#   httppref = ''
 71httppref = ''
 72
 73# The name of the 'root' news page in this directory.
 74# A .html suffix will be added.
 75rootpage = 'root'
 76
 77# Set skipempty to 0 if you wish to see links to empty groups as well.
 78# Only affects the -a option.
 79skipempty = 1
 80
 81# pagelinkicon can contain html to put an icon after links to
 82# further pages. This helps to make important links stand out.
 83# Set to '' if not wanted, or '...' is quite a good one.
 84pagelinkicon='... <img src="http://pelican.cl.cam.ac.uk/icons/page.xbm"> '
 85
 86# ---------------------------------------------------------------------
 87# Less important personal preferences:
 88
 89# Sublistsize controls the maximum number of items the will appear as
 90# an indented sub-list before the whole thing is moved onto a different
 91# page. The smaller this is, the more pages you will have, but the
 92# shorter each will be.
 93sublistsize = 4
 94
 95# That should be all.                                                 #
 96#######################################################################
 97
 98for dir in os.curdir, os.environ['HOME']:
 99    rcfile = os.path.join(dir, '.newslistrc.py')
100    if os.path.exists(rcfile):
101        print rcfile
102        execfile(rcfile)
103        break
104
105from nntplib import NNTP
106from stat import *
107
108rcsrev = '$Revision: 66429 $'
109rcsrev = string.join(filter(lambda s: '$' not in s, string.split(rcsrev)))
110desc = {}
111
112# Make (possibly) relative filenames into absolute ones
113treefile = os.path.join(topdir,treefile)
114descfile = os.path.join(topdir,descfile)
115page = os.path.join(topdir,pagedir)
116
117# First the bits for creating trees ---------------------------
118
119# Addtotree creates/augments a tree from a list of group names
120def addtotree(tree, groups):
121    print 'Updating tree...'
122    for i in groups:
123        parts = string.splitfields(i,'.')
124        makeleaf(tree, parts)
125
126# Makeleaf makes a leaf and the branch leading to it if necessary
127def makeleaf(tree,path):
128    j = path[0]
129    l = len(path)
130
131    if not tree.has_key(j):
132        tree[j] = {}
133    if l == 1:
134        tree[j]['.'] = '.'
135    if l > 1:
136        makeleaf(tree[j],path[1:])
137
138# Then the bits for outputting trees as pages ----------------
139
140# Createpage creates an HTML file named <root>.html containing links
141# to those groups beginning with <root>.
142
143def createpage(root, tree, p):
144    filename = os.path.join(pagedir,root+'.html')
145    if root == rootpage:
146        detail = ''
147    else:
148        detail = ' under ' + root
149    f = open(filename,'w')
150    # f.write('Content-Type: text/html\n')
151    f.write('<TITLE>Newsgroups available' + detail + '</TITLE>\n')
152    f.write('<H1>Newsgroups available' + detail +'</H1>\n')
153    f.write('<A HREF="'+httppref+rootpage+'.html">Back to top level</A><P>\n')
154    printtree(f,tree,0,p)
155    f.write('<I>This page automatically created by \'newslist\' v. '+rcsrev+'.')
156    f.write(time.ctime(time.time()) + '</I><P>')
157    f.close()
158
159# Printtree prints the groups as a bulleted list.  Groups with
160# more than <sublistsize> subgroups will be put on a separate page.
161# Other sets of subgroups are just indented.
162
163def printtree(f, tree, indent, p):
164    global desc
165    l = len(tree)
166
167    if l > sublistsize and indent>0:
168        # Create a new page and a link to it
169        f.write('<LI><B><A HREF="'+httppref+p[1:]+'.html">')
170        f.write(p[1:]+'.*')
171        f.write('</A></B>'+pagelinkicon+'\n')
172        createpage(p[1:], tree, p)
173        return
174
175    kl = tree.keys()
176
177    if l > 1:
178        kl.sort()
179        if indent > 0:
180            # Create a sub-list
181            f.write('<LI>'+p[1:]+'\n<UL>')
182        else:
183            # Create a main list
184            f.write('<UL>')
185        indent = indent + 1
186
187    for i in kl:
188        if i == '.':
189            # Output a newsgroup
190            f.write('<LI><A HREF="news:' + p[1:] + '">'+ p[1:] + '</A> ')
191            if desc.has_key(p[1:]):
192                f.write('     <I>'+desc[p[1:]]+'</I>\n')
193            else:
194                f.write('\n')
195        else:
196            # Output a hierarchy
197            printtree(f,tree[i], indent, p+'.'+i)
198
199    if l > 1:
200        f.write('\n</UL>')
201
202# Reading descriptions file ---------------------------------------
203
204# This returns an array mapping group name to its description
205
206def readdesc(descfile):
207    global desc
208
209    desc = {}
210
211    if descfile == '':
212        return
213
214    try:
215        d = open(descfile, 'r')
216        print 'Reading descriptions...'
217    except (IOError):
218        print 'Failed to open description file ' + descfile
219        return
220    l = d.readline()
221    while l != '':
222        bits = string.split(l)
223        try:
224            grp = bits[0]
225            dsc = string.join(bits[1:])
226            if len(dsc)>1:
227                desc[grp] = dsc
228        except (IndexError):
229            pass
230        l = d.readline()
231
232# Check that ouput directory exists, ------------------------------
233# and offer to create it if not
234
235def checkopdir(pagedir):
236    if not os.path.isdir(pagedir):
237        print 'Directory '+pagedir+' does not exist.'
238        print 'Shall I create it for you? (y/n)'
239        if sys.stdin.readline()[0] == 'y':
240            try:
241                os.mkdir(pagedir,0777)
242            except:
243                print 'Sorry - failed!'
244                sys.exit(1)
245        else:
246            print 'OK. Exiting.'
247            sys.exit(1)
248
249# Read and write current local tree ----------------------------------
250
251def readlocallist(treefile):
252    print 'Reading current local group list...'
253    tree = {}
254    try:
255        treetime = time.localtime(os.stat(treefile)[ST_MTIME])
256    except:
257        print '\n*** Failed to open local group cache '+treefile
258        print 'If this is the first time you have run newslist, then'
259        print 'use the -a option to create it.'
260        sys.exit(1)
261    treedate = '%02d%02d%02d' % (treetime[0] % 100 ,treetime[1], treetime[2])
262    try:
263        dump = open(treefile,'r')
264        tree = marshal.load(dump)
265        dump.close()
266    except (IOError):
267        print 'Cannot open local group list ' + treefile
268    return (tree, treedate)
269
270def writelocallist(treefile, tree):
271    try:
272        dump = open(treefile,'w')
273        groups = marshal.dump(tree,dump)
274        dump.close()
275        print 'Saved list to '+treefile+'\n'
276    except:
277        print 'Sorry - failed to write to local group cache '+treefile
278        print 'Does it (or its directory) have the correct permissions?'
279        sys.exit(1)
280
281# Return list of all groups on server -----------------------------
282
283def getallgroups(server):
284    print 'Getting list of all groups...'
285    treedate='010101'
286    info = server.list()[1]
287    groups = []
288    print 'Processing...'
289    if skipempty:
290        print '\nIgnoring following empty groups:'
291    for i in info:
292        grpname = string.split(i[0])[0]
293        if skipempty and string.atoi(i[1]) < string.atoi(i[2]):
294            print grpname+' ',
295        else:
296            groups.append(grpname)
297    print '\n'
298    if skipempty:
299        print '(End of empty groups)'
300    return groups
301
302# Return list of new groups on server -----------------------------
303
304def getnewgroups(server, treedate):
305    print 'Getting list of new groups since start of '+treedate+'...',
306    info = server.newgroups(treedate,'000001')[1]
307    print 'got %d.' % len(info)
308    print 'Processing...',
309    groups = []
310    for i in info:
311        grpname = string.split(i)[0]
312        groups.append(grpname)
313    print 'Done'
314    return groups
315
316# Now the main program --------------------------------------------
317
318def main():
319    global desc
320
321    tree={}
322
323    # Check that the output directory exists
324    checkopdir(pagedir)
325
326    try:
327        print 'Connecting to '+newshost+'...'
328        if sys.version[0] == '0':
329            s = NNTP.init(newshost)
330        else:
331            s = NNTP(newshost)
332        connected = 1
333    except (nntplib.error_temp, nntplib.error_perm), x:
334        print 'Error connecting to host:', x
335        print 'I\'ll try to use just the local list.'
336        connected = 0
337
338    # If -a is specified, read the full list of groups from server
339    if connected and len(sys.argv) > 1 and sys.argv[1] == '-a':
340
341        groups = getallgroups(s)
342
343    # Otherwise just read the local file and then add
344    # groups created since local file last modified.
345    else:
346
347        (tree, treedate) = readlocallist(treefile)
348        if connected:
349            groups = getnewgroups(s, treedate)
350
351    if connected:
352        addtotree(tree, groups)
353        writelocallist(treefile,tree)
354
355    # Read group descriptions
356    readdesc(descfile)
357
358    print 'Creating pages...'
359    createpage(rootpage, tree, '')
360    print 'Done'
361
362if __name__ == "__main__":
363    main()
364
365# That's all folks
366######################################################################