pkgsrc-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[pkgsrc/trunk]: pkgsrc/pkgtools/url2pkg/files pkgtools/url2pkg: reorganize th...
details: https://anonhg.NetBSD.org/pkgsrc/rev/729249e3412d
branches: trunk
changeset: 402171:729249e3412d
user: rillig <rillig%pkgsrc.org@localhost>
date: Fri Oct 04 22:26:34 2019 +0000
description:
pkgtools/url2pkg: reorganize the Python implementation
The previous version of the code was largely work in progress. Now the code
has been grouped and sorted. A few bugs have been fixed on the way:
* If a PKGNAME had been added in the interactive editor session, it had
been overwritten before. This was because of a typo.
* The whole code has been grouped into classes, to clearly show the
dependencies between the parts.
* Generation of the initial Makefile has been split into smaller methods,
to make them individually testable and to reduce the scope of the local
variables.
* When creating a package in a directory pkgsrc/local/*, "local" is not
used as the primary category of the package.
* GNU configure and other configure scripts are also detected if they are
not placed directly in WRKSRC.
* Packages that contain *.po files will have USE_PKGLOCALEDIR=yes in the
package Makefile. Previously, only *.mo or *.gmo files triggered this
variable.
* When PKGNAME is based on DISTNAME, it is only written to the package
Makefile if there is an actual prefix or transformation.
diffstat:
pkgtools/url2pkg/files/url2pkg.py | 668 +++++++++++++++++---------------
pkgtools/url2pkg/files/url2pkg_test.py | 335 ++++++++++++---
2 files changed, 603 insertions(+), 400 deletions(-)
diffs (truncated from 1610 to 300 lines):
diff -r 58fde14efe23 -r 729249e3412d pkgtools/url2pkg/files/url2pkg.py
--- a/pkgtools/url2pkg/files/url2pkg.py Fri Oct 04 22:19:17 2019 +0000
+++ b/pkgtools/url2pkg/files/url2pkg.py Fri Oct 04 22:26:34 2019 +0000
@@ -1,5 +1,5 @@
#! @PYTHONBIN@
-# $NetBSD: url2pkg.py,v 1.7 2019/10/03 23:02:59 rillig Exp $
+# $NetBSD: url2pkg.py,v 1.8 2019/10/04 22:26:34 rillig Exp $
# Copyright (c) 2019 The NetBSD Foundation, Inc.
# All rights reserved.
@@ -28,214 +28,30 @@
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
+
+# Overview
+#
+# The url2pkg program gets a URL and produces a pkgsrc package from it,
+# filling in several defaults in order to reduce the amount of manual work.
+#
+# This happens in two phases. Phase 1 is done by the Generator and generates
+# an initial Makefile, just enough to download the distfile. Phase 2 then
+# takes the generated Makefile and applies various adjustments for different
+# types of packages, such as Perl or Python modules. This is done by the
+# Adjuster class.
+
+
import getopt
import glob
import os
import re
import subprocess
import sys
-from os.path import isfile
-from typing import Callable, Dict, Iterator, List, Optional, Sequence, Union
-
-
-class Config:
-
- def __init__(self):
- self.make = '@MAKE@'
- self.libdir = '@LIBDIR@'
- self.perl5 = '@PERL5@'
- self.pkgsrcdir = '@PKGSRCDIR@'
- self.pythonbin = '@PYTHONBIN@'
- self.pkgdir = '.' # only overridable for tests
- self.verbose = False
-
-
-config = Config()
-distname = ''
-
-
-def debug(fmt: str, *args):
- if config.verbose:
- msg = fmt.format(*map(repr, args)) if len(args) else fmt
- sys.stderr.write(f'url2pkg: {msg}\n')
-
-
-def run_editor(fname: str, lineno: int):
- editor = os.getenv('PKGEDITOR') or os.getenv('EDITOR') or 'vi'
-
- args: List[str] = [editor]
- if re.search(editor, r'(^|/)(mcedit|nano|pico|vi|vim)$'):
- args.append(f'+{lineno}')
- args.append(fname)
-
- subprocess.check_call(args)
-
-
-def generate_initial_package_Makefile_lines(url):
- global distname
-
- master_site = ''
- master_sites = ''
- distfile = ''
- homepage = ''
- extract_sufx: str
- categories: str
- github_project = ''
- github_release = ''
- dist_subdir = ''
- pkgname_prefix = ''
- pkgname_transform = ''
-
- with open('../../mk/fetch/sites.mk') as sites_mk:
- for line in sites_mk:
- m = re.search(r'^(MASTER_SITE_.*)\+=', line)
- if m:
- master_site = m[1]
- continue
-
- m = re.search(r'^\t(.*?)(?:\s+\\)?$', line)
- if not m:
- continue
-
- site = m[1]
- if not url.startswith(site):
- continue
-
- rest = url[len(site):]
- m = re.search(r'^(.+)/([^/]+)$', rest)
- if not m:
- master_sites = f"${{{master_site}}}"
- continue
-
- subdir, distfile = m.groups()
-
- master_sites = f'${{{master_site}:={subdir}/}}'
- if master_site == 'MASTER_SITE_SOURCEFORGE':
- homepage = f'https://{subdir}.sourceforge.net/'
- elif master_site == 'MASTER_SITE_GNU':
- homepage = f'https://www.gnu.org/software/{subdir}/'
- else:
- homepage = url[:-len(distfile)]
-
- m = re.search(r'^https://downloads\.sourceforge\.net/project/([^/?]+)/[^?]+/([^/?]+)(?:[?].*)?$', url)
- if m:
- project, filename = m.groups()
-
- master_sites = f'${{MASTER_SITE_SOURCEFORGE:={project}/}}'
- homepage = f'https://{project}.sourceforge.net/'
- distfile = filename
-
- m = re.search(r'^https://github\.com/(.+)/(.+)/archive/(.+)(\.tar\.gz|\.zip)$', url)
- if m:
- org, proj, tag, ext = m.groups()
-
- github_project = proj
- master_sites = f'${{MASTER_SITE_GITHUB:={org}/}}'
- homepage = f'https://github.com/{org}/{proj}/'
- if github_project not in tag:
- pkgname_prefix = '${GITHUB_PROJECT}-'
- dist_subdir = '${GITHUB_PROJECT}'
- distfile = tag + ext
-
- m = re.search(r'^https://github\.com/(.+)/(.+)/releases/download/(.+)/(.+)(\.tar\.gz|\.zip)$', url)
- if m:
- org, proj, tag, base, ext = m.groups()
-
- github_project = proj
- master_sites = f'${{MASTER_SITE_GITHUB:={org}/}}'
- homepage = f'https://github.com/{org}/{proj}/'
- if proj not in base:
- github_project = proj
- dist_subdir = '${GITHUB_PROJECT}'
- github_release = '${DISTNAME}' if tag == base else tag
- distfile = base + ext
-
- if master_sites == '':
- m = re.search(r'^(.*/)(.*)$', url)
- if not m:
- sys.exit(f'error: URL "{url}" must have at least one slash')
- master_sites = m[1]
- distfile = m[2]
- homepage = master_sites
-
- m = re.search(r'^(.*?)((?:\.tar)?\.\w+)$', distfile)
- if m:
- distname, extract_sufx = m.groups()
- else:
- distname, extract_sufx = distfile, '# none'
-
- m = re.search(r'^v\d', distname)
- if m:
- pkgname_transform = ':S,^v,,'
- elif re.search(r'-v\d', distname) and not re.search(r'-v.*-v\d', distname):
- pkgname_transform = ':S,-v,-,'
-
- main_category = re.search(r'.*/([^/]+)/[^/]+$', os.getcwd())[1]
- categories = main_category if main_category != 'wip' else '# TODO: add primary category'
-
- if extract_sufx == '.tar.gz' or extract_sufx == '.gem':
- extract_sufx = ''
-
- pkgname = '' if pkgname_prefix == '' and pkgname_transform == '' \
- else f'{pkgname_prefix}${"{"}DISTNAME{pkgname_transform}{"}"}'
-
- maintainer = \
- os.getenv('PKGMAINTAINER') or os.getenv('REPLYTO') \
- or 'INSERT_YOUR_MAIL_ADDRESS_HERE'
-
- lines = Lines()
- lines.add('# $''NetBSD$')
- lines.add('')
-
- lines.add_vars(
- Var('GITHUB_PROJECT', '=', github_project),
- Var('DISTNAME', '=', distname),
- Var('PKGNAME', '=', pkgname),
- Var('CATEGORIES', '=', categories),
- Var('MASTER_SITES', '=', master_sites),
- Var('GITHUB_RELEASE', '=', github_release),
- Var('EXTRACT_SUFX', '=', extract_sufx),
- Var('DIST_SUBDIR', '=', dist_subdir),
- )
-
- lines.add_vars(
- Var('MAINTAINER', '=', maintainer),
- Var('HOMEPAGE', '=', homepage),
- Var('COMMENT', '=', 'TODO: Short description of the package'),
- Var('#LICENSE', '=', '# TODO: (see mk/license.mk)'),
- )
-
- lines.add('# url2pkg-marker (please do not remove this line.)')
- lines.add('.include "../../mk/bsd.pkg.mk"')
-
- return lines
-
-
-def generate_initial_package(url):
- pkgdir = config.pkgdir
- makefile = f'{pkgdir}/Makefile'
- descr = f'{pkgdir}/DESCR'
- plist = f'{pkgdir}/PLIST'
-
- try:
- os.rename(makefile, f'{makefile}.url2pkg~')
- except OSError:
- pass
-
- generate_initial_package_Makefile_lines(url).write_to(makefile)
- if not isfile(plist):
- Lines('@comment $''NetBSD$').write_to(plist)
- if not isfile(descr):
- Lines().write_to(descr)
-
- run_editor(makefile, 5)
-
- bmake('distinfo')
- bmake('extract')
+from typing import Callable, Dict, Iterator, List, Optional, Sequence, Union, Tuple
class Var:
- """ A variable assignment for the package Makefile """
+ """ An abstract variable assignment for the package Makefile. """
def __init__(self, name: str, op: str, value: str):
self.name = name
@@ -243,26 +59,8 @@
self.value = value
-def aligned(vars: List[Var]) -> List[str]:
- relevant = list(filter(lambda v: v.value != '', vars))
- if not relevant:
- return []
-
- def tabwidth(var: Var) -> int:
- return (len(var.name) + len(var.op) + len('\t') + 7) // 8 * 8
-
- width = max(map(tabwidth, relevant), default=0)
-
- aligned_lines = []
- for var in relevant:
- tabs = '\t' * ((width - len(var.name) - len(var.op) + 7) // 8)
- aligned_lines.append(var.name + var.op + tabs + var.value)
-
- aligned_lines.append('')
- return aligned_lines
-
-
class Varassign:
+ """ A variable assignment including layout information. """
def __init__(self, index: int, varname: str, op: str, indent: str,
value: str, space_after_value: str, comment: str):
@@ -275,25 +73,45 @@
self.comment = comment
-def find_package(pkgbase: str) -> str:
- candidates = glob.glob(config.pkgsrcdir + '/*/' + pkgbase)
- debug('candidates for package {0} are {1}', pkgbase, candidates)
- if len(candidates) != 1:
- return ''
- return candidates[0].replace(config.pkgsrcdir, '../..')
+class Url2Pkg:
+ def __init__(self):
+ self.make = '@MAKE@'
+ self.libdir = '@LIBDIR@'
+ self.perl5 = '@PERL5@'
+ self.pkgsrcdir = '@PKGSRCDIR@'
+ self.pythonbin = '@PYTHONBIN@'
+ self.pkgdir = '.' # only overridable for tests
+ self.verbose = False
+ self.out = sys.stdout
+ self.err = sys.stderr
-def bmake(*args: str) -> None:
- debug('running bmake {0}', args)
- subprocess.check_call([config.make] + list(args), cwd=config.pkgdir)
+ def debug(self, fmt: str, *args):
+ if self.verbose:
+ msg = fmt.format(*map(repr, args)) if len(args) else fmt
+ self.err.write(f'url2pkg: {msg}\n')
+ def find_package(self, pkgbase: str) -> str:
Home |
Main Index |
Thread Index |
Old Index