[PATCH pglister] Add Archived-At header to delivered messages

This changeset adds a new header, Archived-At, to delivered messages.
This header field contains a direct link to the message in the web
archive. The Archived-At header is defined by RFC 5064:

https://tools.ietf.org/html/rfc5064
---

Notes:
To build the link, I used the urlpattern field from archive_server table
and replace '/list/' by '/message-id/':

https://www.postgresql.org/list/% -> https://www.postgresql.org/message-id/

The urlpattern field is now retrieved in 'mailinglists' view, hence the
migration.

That's not very elegant, but I did not find a better way to achieve this.
Perhaps adding a 'messageidpattern' column to 'lists_archiveserver' table
would be better? Or maybe I missed something?

Also, this is untested...

lib/baselib/lists.py | 13 ++++--
lib/handlers/mailhandler.py | 2 +-
.../lists/migrations/0053_add_archivedat.py | 41 +++++++++++++++++++
3 files changed, 52 insertions(+), 4 deletions(-)
create mode 100644 web/pglister/lists/migrations/0053_add_archivedat.py

diff --git a/lib/baselib/lists.py b/lib/baselib/lists.py
index 76f8fc8..18bf4ae 100644
--- a/lib/baselib/lists.py
+++ b/lib/baselib/lists.py
@@ -1,4 +1,5 @@
import re
+import urllib.parse
import requests
import psycopg2.extras

@@ -130,7 +131,7 @@ class ModerationReason(object):


class MailingList(object):
- def __init__(self, conn, id, address, name, domain, moderation_level, moderation_regex, spamscore_threshold, maxsize, maxsizedrop, archive_server, archive_address, helppage, archive_domain, blocklist_regex, whitelist_regex, tagged_delivery, tagkey, taglistsource, send_moderation_notices, cc_policy, bcc_policy, ignore_global_mod, reject_to_cc):
+ def __init__(self, conn, id, address, name, domain, moderation_level, moderation_regex, spamscore_threshold, maxsize, maxsizedrop, archive_server, archive_address, archive_urlpattern, helppage, archive_domain, blocklist_regex, whitelist_regex, tagged_delivery, tagkey, taglistsource, send_moderation_notices, cc_policy, bcc_policy, ignore_global_mod, reject_to_cc):
"""
Internal, do not call directly! Instead use constructor methods:
MailingList.get_by_address(address)
@@ -148,6 +149,7 @@ class MailingList(object):
self.maxsizedrop = maxsizedrop
self.archive_server = archive_server
self.archive_address = archive_address
+ self.archive_urlpattern = archive_urlpattern
self.helppage = helppage
self.archive_domain = archive_domain
self.blocklist_regexes = [re.compile(r, re.MULTILINE) for r in blocklist_regex.splitlines()]
@@ -171,7 +173,7 @@ class MailingList(object):
if not maxsizedrop:
self.maxsizedrop = int(config.get('defaults', 'size_drop'))

- _SELECTFIELDS = "id, address, name, domain, moderation_level, moderation_regex, spamscore_threshold, maxsizemoderate, maxsizedrop, archiveserver, archiveaddress, helppage, archivedomain, blocklist_regex, whitelist_regex, tagged_delivery, tagkey, taglistsource, send_moderation_notices, cc_policy, bcc_policy, ignore_global_mod_regex, reject_to_cc"
+ _SELECTFIELDS = "id, address, name, domain, moderation_level, moderation_regex, spamscore_threshold, maxsizemoderate, maxsizedrop, archiveserver, archiveaddress, archiveurlpattern, helppage, archivedomain, blocklist_regex, whitelist_regex, tagged_delivery, tagkey, taglistsource, send_moderation_notices, cc_policy, bcc_policy, ignore_global_mod_regex, reject_to_cc"

@staticmethod
def get_by_address(conn, address):
@@ -279,7 +281,7 @@ OR EXISTS (SELECT 1 FROM mailinglist_whitelist WHERE listid=%(id)s AND email=%(e
})
return curs.fetchall()

- def writeheaders(self, buf):
+ def writeheaders(self, buf, messageid):
"""
Write the appropriate RFC2369 and RFC2919 headers to the
buffer at the current location.
@@ -291,6 +293,7 @@ OR EXISTS (SELECT 1 FROM mailinglist_whitelist WHERE listid=%(id)s AND email=%(e
buf.write("List-Owner: <mailto:{0}>\r\n".format(self.owner_address()).encode('utf8'))
if self.archive_address:
buf.write("List-Archive: <{0}>\r\n".format(self.archive_address).encode('utf8'))
+ buf.write("Archived-At: <{0}>\r\n".format(self.archived_at(messageid)).encode('utf-8'))
buf.write("Precedence: bulk\r\n".encode('utf8'))

def owner_address(self):
@@ -320,6 +323,10 @@ OR EXISTS (SELECT 1 FROM mailinglist_whitelist WHERE listid=%(id)s AND email=%(e
else:
return None

+ def archived_at(self, messageid):
+ msgid = urllib.parse.quote(messageid)
+ return self.archive_urlpattern.replace("/list/", "/message-id/").replace("%", msgid)
+
def moderator_notice_address(self):
"""
Return address used to send moderation notices. For now, we just
diff --git a/lib/handlers/mailhandler.py b/lib/handlers/mailhandler.py
index 863cfba..ae92cbb 100644
--- a/lib/handlers/mailhandler.py
+++ b/lib/handlers/mailhandler.py
@@ -759,7 +759,7 @@ ORDER BY 1""",
if l.rstrip(b"\r\n") == b'':
# Empty line means we've hit the header boundary. Write
# our own custom headers and then switch to body mode.
- self.mlist.writeheaders(targethdr)
+ self.mlist.writeheaders(targethdr, self.messageid)
break
elif l == b'':
raise Exception("Reached end of input without finding end of headers")
diff --git a/web/pglister/lists/migrations/0053_add_archivedat.py b/web/pglister/lists/migrations/0053_add_archivedat.py
new file mode 100644
index 0000000..4266e3b
--- /dev/null
+++ b/web/pglister/lists/migrations/0053_add_archivedat.py
@@ -0,0 +1,41 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('lists', '0052_cannedunsubscriptionnotice'),
+ ]
+
+ operations = [
+ migrations.RunSQL("""
+CREATE OR REPLACE VIEW mailinglists AS
+ SELECT l.id AS id,
+ l.name || '@' || d.name AS address,
+ l.name AS name,
+ d.name AS domain,
+ l.moderation_level AS moderation_level,
+ l.moderation_regex AS moderation_regex,
+ l.spamscore_threshold AS spamscore_threshold,
+ l.maxsizemoderate AS maxsizemoderate,
+ l.maxsizedrop AS maxsizedrop,
+ a.name AS archiveserver,
+ replace(a.urlpattern, '%', l.name) AS archiveaddress,
+ a.urlpattern AS archiveurlpattern,
+ replace(d.lists_helppage, '%', l.name) AS helppage,
+ a.maildomain AS archivedomain,
+ l.blocklist_regex AS blocklist_regex,
+ l.whitelist_regex AS whitelist_regex,
+ l.tagged_delivery AS tagged_delivery,
+ l.tagkey AS tagkey,
+ l.taglistsource AS taglistsource,
+ l.send_moderation_notices AS send_moderation_notices,
+ l.cc_policy AS cc_policy,
+ l.bcc_policy AS bcc_policy,
+ l.ignore_global_mod_regex AS ignore_global_mod_regex,
+ l.reject_to_cc AS reject_to_cc
+ FROM lists_list l
+ INNER JOIN lists_domain d ON d.id=l.domain_id
+ LEFT JOIN lists_archiveserver a ON a.id=l.archivedat_id
+"""),
+ ]
--
2.20.1