[pkgsrc/trunk]: pkgsrc doc/guide: fill in the MASTER_SITE variables automatic...

[rillig <rillig%pkgsrc.org@localhost>]

details:   https://anonhg.NetBSD.org/pkgsrc/rev/a44803891373
branches:  trunk
changeset: 323151:a44803891373
user:      rillig <rillig%pkgsrc.org@localhost>
date:      Sun May 05 18:36:05 2019 +0000

description:
doc/guide: fill in the MASTER_SITE variables automatically

Keeping these two lists in sync is not something that humans should do.

diffstat:

 doc/guide/Makefile                   |   4 +-
 doc/guide/files/build.xml            |  38 +--------------
 doc/guide/files/fill-placeholders.py |  88 ++++++++++++++++++++++++++++++++++++
 doc/guide/files/help-topics.gen.py   |  31 ------------
 doc/guide/files/help-topics.tmpl.xml |  15 ------
 doc/guide/files/help-topics.xml      |  13 +++++
 mk/fetch/sites.mk                    |   5 +-
 7 files changed, 107 insertions(+), 87 deletions(-)

diffs (249 lines):

diff -r 9e41eef191f9 -r a44803891373 doc/guide/Makefile
--- a/doc/guide/Makefile        Sun May 05 17:55:34 2019 +0000
+++ b/doc/guide/Makefile        Sun May 05 18:36:05 2019 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile,v 1.54 2019/04/29 16:18:41 rillig Exp $
+# $NetBSD: Makefile,v 1.55 2019/05/05 18:36:05 rillig Exp $
 
 DISTNAME=              pkgsrc-guide-${PKGVERSION}
 CATEGORIES=            # empty
@@ -66,7 +66,7 @@
 pre-configure: generate-help-topics
 generate-help-topics: .PHONY
        ${RUN} ${MAKE} help topic=:index > ${WRKSRC}/help-topics.data
-       ${RUN} cd ${WRKSRC} && python help-topics.gen.py
+       ${RUN} cd ${WRKSRC} && env PKGSRCDIR=${PKGSRCDIR} python fill-placeholders.py *.xml
 
 do-build:
 .for _output_ in ${OUTPUTS}
diff -r 9e41eef191f9 -r a44803891373 doc/guide/files/build.xml
--- a/doc/guide/files/build.xml Sun May 05 17:55:34 2019 +0000
+++ b/doc/guide/files/build.xml Sun May 05 18:36:05 2019 +0000
@@ -1,4 +1,4 @@
-<!-- $NetBSD: build.xml,v 1.78 2017/05/26 17:59:37 leot Exp $ -->
+<!-- $NetBSD: build.xml,v 1.79 2019/05/05 18:36:05 rillig Exp $ -->
 
 <chapter id="build">
   <title>The build process</title>
@@ -249,41 +249,7 @@
       packages.  The names of the variables should speak for
       themselves.</para>
 
-      <!-- sort mk/fetch/sites.mk | sed -n 's/\(^MA[A-Z_]*\).*/    ${\1}/p' -->
-
-<programlisting>
-${MASTER_SITE_APACHE}
-${MASTER_SITE_BACKUP}
-${MASTER_SITE_CYGWIN}
-${MASTER_SITE_DEBIAN}
-${MASTER_SITE_FREEBSD}
-${MASTER_SITE_FREEBSD_LOCAL}
-${MASTER_SITE_GENTOO}
-${MASTER_SITE_GNOME}
-${MASTER_SITE_GNU}
-${MASTER_SITE_GNUSTEP}
-${MASTER_SITE_HASKELL_HACKAGE}
-${MASTER_SITE_IFARCHIVE}
-${MASTER_SITE_KDE}
-${MASTER_SITE_MOZILLA}
-${MASTER_SITE_MOZILLA_ALL}
-${MASTER_SITE_MOZILLA_ESR}
-${MASTER_SITE_MYSQL}
-${MASTER_SITE_NETLIB}
-${MASTER_SITE_OPENOFFICE}
-${MASTER_SITE_OSDN}
-${MASTER_SITE_PERL_CPAN}
-${MASTER_SITE_PGSQL}
-${MASTER_SITE_RUBYGEMS}
-${MASTER_SITE_R_CRAN}
-${MASTER_SITE_SOURCEFORGE}
-${MASTER_SITE_SUNSITE}
-${MASTER_SITE_SUSE}
-${MASTER_SITE_TEX_CTAN}
-${MASTER_SITE_XCONTRIB}
-${MASTER_SITE_XEMACS}
-${MASTER_SITE_XORG}
-</programlisting>
+@master_sites@
 
       <para>Some explanations for the less self-explaining ones:
       <varname>MASTER_SITE_BACKUP</varname> contains backup sites
diff -r 9e41eef191f9 -r a44803891373 doc/guide/files/fill-placeholders.py
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/doc/guide/files/fill-placeholders.py      Sun May 05 18:36:05 2019 +0000
@@ -0,0 +1,88 @@
+#! python
+# $NetBSD: fill-placeholders.py,v 1.1 2019/05/05 18:36:05 rillig Exp $
+
+"""
+Fills in some sections of data that are determined directly from the
+pkgsrc code, such as variable names or help topics.
+"""
+
+import filecmp
+import os
+import re
+import sys
+from typing import List, Match
+from xml.sax.saxutils import escape as to_xml
+
+pkgsrcdir = os.environ['PKGSRCDIR']
+
+
+def read_lines(filename: str) -> List[str]:
+    with open(filename, 'r') as f:
+        return f.readlines()
+
+
+def help_topics() -> str:
+    # type="vert" would spread the columns over several pages,
+    # starting with aaaa | pppp on the first page, which is
+    # unintuitive in both the PDF and the HTML version.
+    out = ['<simplelist type="horiz" columns="2">']
+    for line in read_lines('help-topics.data')[2:]:
+        topic = line.strip()
+        out.append(f'<member>{to_xml(topic)}</member>')
+    out.append('</simplelist>')
+
+    return '\n'.join(out)
+
+
+def master_sites() -> str:
+    sites = []
+
+    for line in read_lines(f'{pkgsrcdir}/mk/fetch/sites.mk'):
+        m = re.match(r'^(MAS\w+)', line)
+        if m:
+            sites.append(m[1])
+
+    out = ['<simplelist type="horiz" columns="2">']
+    for site in sorted(sites):
+        out.append(f'<member>{to_xml(site)}</member>')
+    out.append('</simplelist>')
+
+    return '\n'.join(out)
+
+
+def process(filename: str, placeholders: set):
+    tmpl_filename = f'{filename}.tmpl'
+    tmp_filename = f'{filename}.tmp'
+    in_filename = tmpl_filename if os.path.isfile(tmpl_filename) else filename
+    out_filename = filename
+
+    phs = dict()
+    for p in placeholders:
+        phs[p.__name__] = p
+
+    def repl(m: Match):
+        return phs[m[1]]() if m[1] in phs else m[0]
+
+    out = []
+    for line in read_lines(in_filename):
+        out.append(re.sub(r'@(\w+)@', repl, line))
+
+    with open(tmp_filename, 'w') as f:
+        f.writelines(out)
+
+    if filecmp.cmp(tmp_filename, out_filename):
+        os.remove(tmp_filename)
+    elif in_filename == tmpl_filename:
+        os.replace(tmp_filename, out_filename)
+    else:
+        os.rename(out_filename, tmpl_filename)
+        os.rename(tmp_filename, out_filename)
+
+
+def main():
+    for filename in sys.argv[1:]:
+        process(filename, {help_topics, master_sites})
+
+
+if __name__ == '__main__':
+    main()
diff -r 9e41eef191f9 -r a44803891373 doc/guide/files/help-topics.gen.py
--- a/doc/guide/files/help-topics.gen.py        Sun May 05 17:55:34 2019 +0000
+++ /dev/null   Thu Jan 01 00:00:00 1970 +0000
@@ -1,31 +0,0 @@
-#! python
-# $NetBSD: help-topics.gen.py,v 1.1 2019/04/29 16:18:41 rillig Exp $
-
-import os
-
-tmpl_file = "help-topics.tmpl.xml"
-data_file = "help-topics.data"
-out_file = "help-topics.xml"
-
-
-def merge():
-    def read_lines(filename):
-        with open(filename) as f:
-            return f.readlines()
-
-    out = []
-    for tmpl_line in read_lines(tmpl_file):
-        if '@topic@' in tmpl_line:
-            for topic in read_lines(data_file)[2:]:
-                xml_topic = topic.replace('&', '%amp;').replace('<', '&lt;')
-                out.append(tmpl_line.replace('@topic@', xml_topic))
-        else:
-            out.append(tmpl_line)
-
-    with open(f'{out_file}.tmp', 'w') as f:
-        f.writelines(out)
-    os.rename(f'{out_file}.tmp', out_file)
-
-
-if __name__ == '__main__':
-    merge()
diff -r 9e41eef191f9 -r a44803891373 doc/guide/files/help-topics.tmpl.xml
--- a/doc/guide/files/help-topics.tmpl.xml      Sun May 05 17:55:34 2019 +0000
+++ /dev/null   Thu Jan 01 00:00:00 1970 +0000
@@ -1,15 +0,0 @@
-<!-- $NetBSD: help-topics.tmpl.xml,v 1.1 2019/04/28 15:22:24 rillig Exp $ -->
-
-<appendix id="help-topics">
-<title>Help topics</title>
-
-<para>
-       The following list contains all help topics that are available
-       when running <command>bmake help topic=:index</command>.
-</para>
-
-<itemizedlist>
-       <listitem><para>@topic@</para></listitem>
-</itemizedlist>
-
-</appendix>
diff -r 9e41eef191f9 -r a44803891373 doc/guide/files/help-topics.xml
--- /dev/null   Thu Jan 01 00:00:00 1970 +0000
+++ b/doc/guide/files/help-topics.xml   Sun May 05 18:36:05 2019 +0000
@@ -0,0 +1,13 @@
+<!-- $NetBSD: help-topics.xml,v 1.1 2019/05/05 18:36:05 rillig Exp $ -->
+
+<appendix id="help-topics">
+<title>Help topics</title>
+
+<para>
+       The following list contains all help topics that are available
+       when running <command>bmake help topic=:index</command>.
+</para>
+
+@help_topics@
+
+</appendix>
diff -r 9e41eef191f9 -r a44803891373 mk/fetch/sites.mk
--- a/mk/fetch/sites.mk Sun May 05 17:55:34 2019 +0000
+++ b/mk/fetch/sites.mk Sun May 05 18:36:05 2019 +0000
@@ -1,10 +1,9 @@
-# $NetBSD: sites.mk,v 1.162 2019/02/04 09:36:41 wiz Exp $
+# $NetBSD: sites.mk,v 1.163 2019/05/05 18:36:05 rillig Exp $
 #
 # This Makefile fragment defines read-only MASTER_SITE_* variables
 # representing some well-known master distribution sites for software.
 #
-# When variables are added or removed, the pkgsrc guide (build.xml)
-# should also be updated.
+# These variables are also listed in the pkgsrc guide.
 
 MASTER_SITE_XCONTRIB+=  \
        ftp://ftp.gwdg.de/pub/x11/x.org/contrib/ \