pkgsrc-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[pkgsrc/trunk]: pkgsrc/pkgtools/url2pkg/files pkgtools/url2pkg: reorganize th...



details:   https://anonhg.NetBSD.org/pkgsrc/rev/0d5c2409820b
branches:  trunk
changeset: 340593:0d5c2409820b
user:      rillig <rillig%pkgsrc.org@localhost>
date:      Fri Oct 04 22:26:34 2019 +0000

description:
pkgtools/url2pkg: reorganize the Python implementation

The previous version of the code was largely work in progress. Now the code
has been grouped and sorted. A few bugs have been fixed on the way:

* If a PKGNAME had been added in the interactive editor session, it had
  been overwritten before. This was because of a typo.

* The whole code has been grouped into classes, to clearly show the
  dependencies between the parts.

* Generation of the initial Makefile has been split into smaller methods,
  to make them individually testable and to reduce the scope of the local
  variables.

* When creating a package in a directory pkgsrc/local/*, "local" is not
  used as the primary category of the package.

* GNU configure and other configure scripts are also detected if they are
  not placed directly in WRKSRC.

* Packages that contain *.po files will have USE_PKGLOCALEDIR=yes in the
  package Makefile. Previously, only *.mo or *.gmo files triggered this
  variable.

* When PKGNAME is based on DISTNAME, it is only written to the package
  Makefile if there is an actual prefix or transformation.

diffstat:

 pkgtools/url2pkg/files/url2pkg.py      |  668 +++++++++++++++++---------------
 pkgtools/url2pkg/files/url2pkg_test.py |  335 ++++++++++++---
 2 files changed, 603 insertions(+), 400 deletions(-)

diffs (truncated from 1610 to 300 lines):

diff -r 234e515140e9 -r 0d5c2409820b pkgtools/url2pkg/files/url2pkg.py
--- a/pkgtools/url2pkg/files/url2pkg.py Fri Oct 04 22:19:17 2019 +0000
+++ b/pkgtools/url2pkg/files/url2pkg.py Fri Oct 04 22:26:34 2019 +0000
@@ -1,5 +1,5 @@
 #! @PYTHONBIN@
-# $NetBSD: url2pkg.py,v 1.7 2019/10/03 23:02:59 rillig Exp $
+# $NetBSD: url2pkg.py,v 1.8 2019/10/04 22:26:34 rillig Exp $
 
 # Copyright (c) 2019 The NetBSD Foundation, Inc.
 # All rights reserved.
@@ -28,214 +28,30 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
+
+# Overview
+#
+# The url2pkg program gets a URL and produces a pkgsrc package from it,
+# filling in several defaults in order to reduce the amount of manual work.
+#
+# This happens in two phases. Phase 1 is done by the Generator and generates
+# an initial Makefile, just enough to download the distfile. Phase 2 then
+# takes the generated Makefile and applies various adjustments for different
+# types of packages, such as Perl or Python modules. This is done by the
+# Adjuster class.
+
+
 import getopt
 import glob
 import os
 import re
 import subprocess
 import sys
-from os.path import isfile
-from typing import Callable, Dict, Iterator, List, Optional, Sequence, Union
-
-
-class Config:
-
-    def __init__(self):
-        self.make = '@MAKE@'
-        self.libdir = '@LIBDIR@'
-        self.perl5 = '@PERL5@'
-        self.pkgsrcdir = '@PKGSRCDIR@'
-        self.pythonbin = '@PYTHONBIN@'
-        self.pkgdir = '.'  # only overridable for tests
-        self.verbose = False
-
-
-config = Config()
-distname = ''
-
-
-def debug(fmt: str, *args):
-    if config.verbose:
-        msg = fmt.format(*map(repr, args)) if len(args) else fmt
-        sys.stderr.write(f'url2pkg: {msg}\n')
-
-
-def run_editor(fname: str, lineno: int):
-    editor = os.getenv('PKGEDITOR') or os.getenv('EDITOR') or 'vi'
-
-    args: List[str] = [editor]
-    if re.search(editor, r'(^|/)(mcedit|nano|pico|vi|vim)$'):
-        args.append(f'+{lineno}')
-    args.append(fname)
-
-    subprocess.check_call(args)
-
-
-def generate_initial_package_Makefile_lines(url):
-    global distname
-
-    master_site = ''
-    master_sites = ''
-    distfile = ''
-    homepage = ''
-    extract_sufx: str
-    categories: str
-    github_project = ''
-    github_release = ''
-    dist_subdir = ''
-    pkgname_prefix = ''
-    pkgname_transform = ''
-
-    with open('../../mk/fetch/sites.mk') as sites_mk:
-        for line in sites_mk:
-            m = re.search(r'^(MASTER_SITE_.*)\+=', line)
-            if m:
-                master_site = m[1]
-                continue
-
-            m = re.search(r'^\t(.*?)(?:\s+\\)?$', line)
-            if not m:
-                continue
-
-            site = m[1]
-            if not url.startswith(site):
-                continue
-
-            rest = url[len(site):]
-            m = re.search(r'^(.+)/([^/]+)$', rest)
-            if not m:
-                master_sites = f"${{{master_site}}}"
-                continue
-
-            subdir, distfile = m.groups()
-
-            master_sites = f'${{{master_site}:={subdir}/}}'
-            if master_site == 'MASTER_SITE_SOURCEFORGE':
-                homepage = f'https://{subdir}.sourceforge.net/'
-            elif master_site == 'MASTER_SITE_GNU':
-                homepage = f'https://www.gnu.org/software/{subdir}/'
-            else:
-                homepage = url[:-len(distfile)]
-
-    m = re.search(r'^https://downloads\.sourceforge\.net/project/([^/?]+)/[^?]+/([^/?]+)(?:[?].*)?$', url)
-    if m:
-        project, filename = m.groups()
-
-        master_sites = f'${{MASTER_SITE_SOURCEFORGE:={project}/}}'
-        homepage = f'https://{project}.sourceforge.net/'
-        distfile = filename
-
-    m = re.search(r'^https://github\.com/(.+)/(.+)/archive/(.+)(\.tar\.gz|\.zip)$', url)
-    if m:
-        org, proj, tag, ext = m.groups()
-
-        github_project = proj
-        master_sites = f'${{MASTER_SITE_GITHUB:={org}/}}'
-        homepage = f'https://github.com/{org}/{proj}/'
-        if github_project not in tag:
-            pkgname_prefix = '${GITHUB_PROJECT}-'
-            dist_subdir = '${GITHUB_PROJECT}'
-        distfile = tag + ext
-
-    m = re.search(r'^https://github\.com/(.+)/(.+)/releases/download/(.+)/(.+)(\.tar\.gz|\.zip)$', url)
-    if m:
-        org, proj, tag, base, ext = m.groups()
-
-        github_project = proj
-        master_sites = f'${{MASTER_SITE_GITHUB:={org}/}}'
-        homepage = f'https://github.com/{org}/{proj}/'
-        if proj not in base:
-            github_project = proj
-            dist_subdir = '${GITHUB_PROJECT}'
-        github_release = '${DISTNAME}' if tag == base else tag
-        distfile = base + ext
-
-    if master_sites == '':
-        m = re.search(r'^(.*/)(.*)$', url)
-        if not m:
-            sys.exit(f'error: URL "{url}" must have at least one slash')
-        master_sites = m[1]
-        distfile = m[2]
-        homepage = master_sites
-
-    m = re.search(r'^(.*?)((?:\.tar)?\.\w+)$', distfile)
-    if m:
-        distname, extract_sufx = m.groups()
-    else:
-        distname, extract_sufx = distfile, '# none'
-
-    m = re.search(r'^v\d', distname)
-    if m:
-        pkgname_transform = ':S,^v,,'
-    elif re.search(r'-v\d', distname) and not re.search(r'-v.*-v\d', distname):
-        pkgname_transform = ':S,-v,-,'
-
-    main_category = re.search(r'.*/([^/]+)/[^/]+$', os.getcwd())[1]
-    categories = main_category if main_category != 'wip' else '# TODO: add primary category'
-
-    if extract_sufx == '.tar.gz' or extract_sufx == '.gem':
-        extract_sufx = ''
-
-    pkgname = '' if pkgname_prefix == '' and pkgname_transform == '' \
-        else f'{pkgname_prefix}${"{"}DISTNAME{pkgname_transform}{"}"}'
-
-    maintainer = \
-        os.getenv('PKGMAINTAINER') or os.getenv('REPLYTO') \
-        or 'INSERT_YOUR_MAIL_ADDRESS_HERE'
-
-    lines = Lines()
-    lines.add('# $''NetBSD$')
-    lines.add('')
-
-    lines.add_vars(
-        Var('GITHUB_PROJECT', '=', github_project),
-        Var('DISTNAME', '=', distname),
-        Var('PKGNAME', '=', pkgname),
-        Var('CATEGORIES', '=', categories),
-        Var('MASTER_SITES', '=', master_sites),
-        Var('GITHUB_RELEASE', '=', github_release),
-        Var('EXTRACT_SUFX', '=', extract_sufx),
-        Var('DIST_SUBDIR', '=', dist_subdir),
-    )
-
-    lines.add_vars(
-        Var('MAINTAINER', '=', maintainer),
-        Var('HOMEPAGE', '=', homepage),
-        Var('COMMENT', '=', 'TODO: Short description of the package'),
-        Var('#LICENSE', '=', '# TODO: (see mk/license.mk)'),
-    )
-
-    lines.add('# url2pkg-marker (please do not remove this line.)')
-    lines.add('.include "../../mk/bsd.pkg.mk"')
-
-    return lines
-
-
-def generate_initial_package(url):
-    pkgdir = config.pkgdir
-    makefile = f'{pkgdir}/Makefile'
-    descr = f'{pkgdir}/DESCR'
-    plist = f'{pkgdir}/PLIST'
-
-    try:
-        os.rename(makefile, f'{makefile}.url2pkg~')
-    except OSError:
-        pass
-
-    generate_initial_package_Makefile_lines(url).write_to(makefile)
-    if not isfile(plist):
-        Lines('@comment $''NetBSD$').write_to(plist)
-    if not isfile(descr):
-        Lines().write_to(descr)
-
-    run_editor(makefile, 5)
-
-    bmake('distinfo')
-    bmake('extract')
+from typing import Callable, Dict, Iterator, List, Optional, Sequence, Union, Tuple
 
 
 class Var:
-    """ A variable assignment for the package Makefile """
+    """ An abstract variable assignment for the package Makefile. """
 
     def __init__(self, name: str, op: str, value: str):
         self.name = name
@@ -243,26 +59,8 @@
         self.value = value
 
 
-def aligned(vars: List[Var]) -> List[str]:
-    relevant = list(filter(lambda v: v.value != '', vars))
-    if not relevant:
-        return []
-
-    def tabwidth(var: Var) -> int:
-        return (len(var.name) + len(var.op) + len('\t') + 7) // 8 * 8
-
-    width = max(map(tabwidth, relevant), default=0)
-
-    aligned_lines = []
-    for var in relevant:
-        tabs = '\t' * ((width - len(var.name) - len(var.op) + 7) // 8)
-        aligned_lines.append(var.name + var.op + tabs + var.value)
-
-    aligned_lines.append('')
-    return aligned_lines
-
-
 class Varassign:
+    """ A variable assignment including layout information. """
 
     def __init__(self, index: int, varname: str, op: str, indent: str,
                  value: str, space_after_value: str, comment: str):
@@ -275,25 +73,45 @@
         self.comment = comment
 
 
-def find_package(pkgbase: str) -> str:
-    candidates = glob.glob(config.pkgsrcdir + '/*/' + pkgbase)
-    debug('candidates for package {0} are {1}', pkgbase, candidates)
-    if len(candidates) != 1:
-        return ''
-    return candidates[0].replace(config.pkgsrcdir, '../..')
+class Url2Pkg:
 
+    def __init__(self):
+        self.make = '@MAKE@'
+        self.libdir = '@LIBDIR@'
+        self.perl5 = '@PERL5@'
+        self.pkgsrcdir = '@PKGSRCDIR@'
+        self.pythonbin = '@PYTHONBIN@'
+        self.pkgdir = '.'  # only overridable for tests
+        self.verbose = False
+        self.out = sys.stdout
+        self.err = sys.stderr
 
-def bmake(*args: str) -> None:
-    debug('running bmake {0}', args)
-    subprocess.check_call([config.make] + list(args), cwd=config.pkgdir)
+    def debug(self, fmt: str, *args):
+        if self.verbose:
+            msg = fmt.format(*map(repr, args)) if len(args) else fmt
+            self.err.write(f'url2pkg: {msg}\n')
 
+    def find_package(self, pkgbase: str) -> str:



Home | Main Index | Thread Index | Old Index