From zmailer-owner@nic.funet.fi Wed Dec  3 01:26:22 1997
Received: from pobox3.funet.fi ([128.214.46.64]:6716 "EHLO pobox3.funet.fi" ident: "NO-IDENT-SERVICE") by nic.funet.fi with ESMTP id <4628-16327>; Wed, 3 Dec 1997 00:14:23 +0200
Received: from press-gopher.uchicago.edu (press-gopher.uchicago.edu [128.135.234.36])
	by pobox3.funet.fi (8.8.8/8.8.8/FUNET/Rtrs-1.14) with ESMTP id AAA02574;
	Wed, 3 Dec 1997 00:02:30 +0200 (EET)
Received: from rcb@localhost (fake: stdin (HELO localhost) (ident: uid#100@localhost)) by press-gopher.uchicago.edu with SMTP id <41636-8759>; Tue, 2 Dec 1997 15:59:59 -0600
Date:	Tue, 2 Dec 1997 15:59:48 -0600 (CST)
Sender: Roy C Bixler <rcb@press-gopher.uchicago.edu>
From:	Roy C Bixler <rcb@press-gopher.uchicago.edu>
To:	mea@nic.funet.fi
cc:	zmailer@nic.funet.fi
Subject: Re: Release of  2.99.49p9
In-Reply-To: <19971130234800Z15632-16327+3313@nic.funet.fi>
Message-ID: <Pine.GSO.3.96.971202155000.13472A-100000@press-gopher.uchicago.edu>
MIME-Version: 1.0
Content-Type: TEXT/PLAIN; charset=US-ASCII
Return-Path: <zmailer-owner@nic.funet.fi>
X-Orcpt: rfc822;mea
Status: RO

I found one problem with compiling under Linux 2.1.x with glibc 2.0.5c and
gcc 2.7.2.3 and it is that gai_strerror is not found while trying to link
the 'smtp' transport agent.  My fix for it is in the patch below, but no
doubt there is a better fix since I noticed a test for 'gai_strerror' in
'configure'.

There was also a minor problem under Solaris 2.6 and gcc 2.7.2.3.  I had
to comment out the declarations for 'inet_ntop()' and 'inet_pton()' at
line 55-56 of 'include/libc.h' because Sun's header files already had
declarations.  Should I be using 'configure --with-ipv6-replacement-libc'
here? 

Finally, I have a much improved and cleaned up version of my Python
spamlist generator.  Also, I have implemented the '-n' option in
'policy-builder.sh' to allow easy use of another spamlist generating
script.

diff -rNu zmailer-2.99.49p9/libc/Makefile.in zmailer-2.99.49p9.new/libc/Makefile.in
--- zmailer-2.99.49p9/libc/Makefile.in	Fri Nov 14 19:33:19 1997
+++ zmailer-2.99.49p9.new/libc/Makefile.in	Tue Dec  2 13:08:20 1997
@@ -34,10 +34,10 @@
 OBJS	= mail.o mail_alloc.o mail_host.o fullname.o getopt.o myhostname.o \
 	getzenv.o whathost.o getdtblsiz.o __fopen.o setreuid.o
 LIBMAILERobjs = mail.o mail_alloc.o getzenv.o mail_host.o myhostname.o	\
-	fullname.o whathost.o strerror.o
+	fullname.o whathost.o strerror.o gai_strerror.o
 SOURCE	= mail.c mail_alloc.c mail_host.c fullname.c getopt.c myhostname.c \
 	getzenv.c setvbuf.c strchr.c strrchr.c strsignal.c strerror.c \
-	whathost.c getdtblsiz.c setreuid.c __fopen.c setreuid.c
+	gai_strerror.c whathost.c getdtblsiz.c setreuid.c __fopen.c setreuid.c
 
 $(LIBNAME).a: $(TOPDIR)/libs/$(LIBNAME).a $(TOPDIR)/libs/libzmailer.a
 
diff -rNu zmailer-2.99.49p9/transports/smtp/Makefile.in zmailer-2.99.49p9.new/transports/smtp/Makefile.in
--- zmailer-2.99.49p9/transports/smtp/Makefile.in	Mon Aug  4 11:09:27 1997
+++ zmailer-2.99.49p9.new/transports/smtp/Makefile.in	Tue Dec  2 13:09:16 1997
@@ -23,7 +23,7 @@
 LIBMALLOC=	@LIBMALLOC@
 LIBMALLOCDEB=	$(TOPDIR)/libmalloc/libmalloc_d.a
 #
-LIB=		-L$(TOPDIR)/libs -lta -lz -lzc $(LIBMALLOC)
+LIB=		-L$(TOPDIR)/libs -lta -lz -lzc -lzmailer $(LIBMALLOC)
 LIBDEB=		$(TOPDIR)/libs/libtag
 LINTLIB=	../libta/llib-llibta.ln
 
diff -rNu zmailer-2.99.49p9/utils/policy-builder.sh.in zmailer-2.99.49p9.new/utils/policy-builder.sh.in
--- zmailer-2.99.49p9/utils/policy-builder.sh.in	Mon Nov 17 14:05:31 1997
+++ zmailer-2.99.49p9.new/utils/policy-builder.sh.in	Tue Dec  2 14:41:43 1997
@@ -13,6 +13,15 @@
 # The produced database retains the first instance of any given key.
 #
 
+FLAG=
+while getopts n c; do
+  case $c in
+    n)       FLAG=$c;;
+    ?)       exit 2;;
+  esac
+done
+shift `expr $OPTIND - 1`
+
 ZCONFIG=@ZMAILERCFGFILE@
 . $ZCONFIG
 
@@ -26,8 +35,10 @@
 fi
 
 
-lynx -source http://www.webeasy.com:8080/spam/spam_download_table \
+if [ f$FLAG != fn ]; then
+  lynx -source http://www.webeasy.com:8080/spam/spam_download_table \
    > smtp-policy.spam.new && mv smtp-policy.spam.new smtp-policy.spam
+fi
 
 # Fork off a subshell to do it all...
 (
diff -rNu zmailer-2.99.49p9/utils/spamlist.py zmailer-2.99.49p9.new/utils/spamlist.py
--- zmailer-2.99.49p9/utils/spamlist.py	Wed Dec 31 18:00:00 1969
+++ zmailer-2.99.49p9.new/utils/spamlist.py	Tue Dec  2 14:28:48 1997
@@ -0,0 +1,124 @@
+#!/usr/bin/python
+# Emacs: use -*-Python-*- mode.
+#
+# Z-mailer spam list maker
+#
+# Roy Bixler
+# rcb@press-gopher.uchicago.edu
+# 1 Dec. 1997
+#
+
+import ftplib, httplib, string, sys
+from urlparse import *
+
+# returns the contents at the given URL (must be either of type "http" or
+# "ftp") as a list
+def get_url_contents(url):
+    global lns
+    lns = []
+    url_comps = urlparse(url)
+    if (url_comps[0] == "ftp"):
+	def ftp_line(ln):
+	    lns.append(ln)
+	h = ftplib.FTP(url_comps[1])
+	h.login()
+	i = string.rfind(url_comps[2], '/')
+	if (i >= 0):
+	    h.cwd(url_comps[2][:i])
+	    h.retrlines("RETR "+url_comps[2][i+1:], ftp_line)
+	else:
+	    h.retrlines("RETR "+url_comps[2], ftp_line)
+	h.close()
+    elif (url_comps[0] == "http"):
+	h = httplib.HTTP(url_comps[1])
+	h.putrequest('GET', url_comps[2])
+	h.putheader('Accept', 'text/html')
+	h.putheader('Accept', 'text/plain')
+	h.endheaders()
+	errcode, errmsg, headers = h.getreply()
+	# HTTP/1.1 replies seem to generate an errorcode of -1, so try
+	# to handle this case.  This may simply be a manifestation of
+	# a broken Python 1.4 httplib module.
+	if (errcode == -1):
+	    try:
+		real_errcode = string.atoi(string.split(errmsg)[1])
+	    except ValueError:
+		real_errcode = -1 # yes, it really is bogus :-/
+	    sys.stderr.write("%d" % (real_errcode)) # Should be 200
+	else:
+	    sys.stderr.write("%d" % (errcode)) # Should be 200
+	f = h.getfile()
+	ln = f.readline()
+	# once again, try to compensate for broken behavior on HTTP/1.1
+	# by eating the header lines which would otherwise show up in
+	# the data
+	if ((errcode == -1) and (real_errcode <> -1)):
+	    while ((ln) and
+		   ((len(ln) > 2) or
+		    (ln[0] <> "\r") or (ln[-1] <> "\n"))):
+		ln = f.readline()
+	while ln:
+	    lns.append(string.rstrip(ln)) # Get the raw HTML
+	    ln = f.readline()
+	f.close()
+    return lns
+
+# if there is not @-sign found, insert at beginning of string
+def atify(dom):
+    if (string.find(dom, '@') == -1):
+	return '@'+dom
+    else:
+	return dom
+
+# add the information found at 'svc_url' to a list of junk e-mailers.
+# The list consists of the dictionary 'jdict'.  'svc_name' is merely used
+# for the cosmetic purpose of progress reporting.  'start_after' specifies
+# a string which marks the beginning of the list and 'end_before' similarly
+# specifies a marker which tells when to stop reading the list.  These are
+# both optional parameters.
+def add_to_junkers_dict(jdict, svc_name, svc_url, start_after='',
+			end_before=''):
+    sys.stderr.write("%s: (status = " % (svc_name))
+    tdict = get_url_contents(svc_url)
+    sys.stderr.write(") - done\n")
+    i = 0
+    if (start_after):
+	while ((i < len(tdict)) and
+	       (tdict[i][0:len(start_after)] <> start_after)):
+	    i = i+1
+	i = i+1
+    while (i < len(tdict)):
+	if ((end_before) and (tdict[i][0:len(end_before)] == end_before)):
+	    break
+	if ((tdict[i]) and (tdict[i][0] <> "#")):
+	    jdict[atify(tdict[i])] = svc_name
+	i = i+1
+
+# and now for the main program
+
+# start with an empty junk list
+sl = {}
+
+add_to_junkers_dict(sl, "Hilotek",
+		    "http://www.hilotek.com/Mail/SpamDomains.html",
+		    "<h3", "<P")
+add_to_junkers_dict(sl, "Taz 1",
+		    "http://www.taz.net.au/Mail/SpamDomains",
+		    "", "")
+add_to_junkers_dict(sl, "Taz 2",
+		    "http://www.taz.net.au/Mail/Spammers",
+		    "", "")
+add_to_junkers_dict(sl, "Webeasy",
+		    "http://www.webeasy.com:8080/spam/spam_download_table",
+		    "", "")
+add_to_junkers_dict(sl, "Znet",
+		    "http://www.znet.com/spammers.txt",
+		    "", "")
+
+# we only really care about the dictionary keys
+ksl = sl.keys()
+
+# output the sorted dictionary keys
+ksl.sort()
+for i in ksl:
+    print i


---
Roy Bixler
The University of Chicago Press
rcb@press-gopher.uchicago.edu

