pkgsrc-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[pkgsrc/trunk]: pkgsrc/pkgtools/url2pkg/files pkgtools/url2pkg: fixed migrati...
details: https://anonhg.NetBSD.org/pkgsrc/rev/7d72db51a3e0
branches: trunk
changeset: 402228:7d72db51a3e0
user: rillig <rillig%pkgsrc.org@localhost>
date: Sat Oct 05 18:00:09 2019 +0000
description:
pkgtools/url2pkg: fixed migration of Python packages from GitHub to PyPI
diffstat:
pkgtools/url2pkg/files/url2pkg.py | 165 +++++++++++--------
pkgtools/url2pkg/files/url2pkg_test.py | 273 ++++++++++++++++++++++++++++++--
2 files changed, 352 insertions(+), 86 deletions(-)
diffs (truncated from 788 to 300 lines):
diff -r 5bd12b08e91b -r 7d72db51a3e0 pkgtools/url2pkg/files/url2pkg.py
--- a/pkgtools/url2pkg/files/url2pkg.py Sat Oct 05 17:39:55 2019 +0000
+++ b/pkgtools/url2pkg/files/url2pkg.py Sat Oct 05 18:00:09 2019 +0000
@@ -1,5 +1,5 @@
#! @PYTHONBIN@
-# $NetBSD: url2pkg.py,v 1.10 2019/10/05 12:22:51 rillig Exp $
+# $NetBSD: url2pkg.py,v 1.11 2019/10/05 18:00:09 rillig Exp $
# Copyright (c) 2019 The NetBSD Foundation, Inc.
# All rights reserved.
@@ -44,6 +44,7 @@
import getopt
import glob
import os
+import pathlib
import re
import subprocess
import sys
@@ -81,10 +82,11 @@
self.perl5 = '@PERL5@'
self.pkgsrcdir = '@PKGSRCDIR@'
self.pythonbin = '@PYTHONBIN@'
+ self.editor = os.getenv('PKGEDITOR') or os.getenv('EDITOR') or 'vi'
self.verbose = False
- self.pkgdir = '.' # only overridable for tests
+ self.pkgdir = pathlib.Path('.') # only overridable for tests
self.out = sys.stdout # only overridable for tests
self.err = sys.stderr # only overridable for tests
@@ -121,30 +123,32 @@
self.add(line)
@classmethod
- def read_from(cls, filename: str) -> 'Lines':
- pass
-
- lines = Lines()
- with open(filename) as f:
- for line in f:
- lines.add(line.rstrip('\n'))
- return lines
+ def read_from(cls, filename: Union[str, pathlib.Path]) -> 'Lines':
+ return Lines(*pathlib.Path(filename).read_text().splitlines())
- def write_to(self, filename: str):
- with open(f'{filename}.tmp', 'w') as f:
+ def write_to(self, filename: Union[str, pathlib.Path]):
+ target = pathlib.Path(filename)
+ tmp = target.with_name(f'{target.name}.tmp')
+ with tmp.open('w') as f:
f.writelines(line + '\n' for line in self.lines)
- try:
- os.remove(filename)
- except OSError:
- pass
- os.rename(f'{filename}.tmp', filename)
+ tmp.replace(target)
def all_varassigns(self, varname: str) -> Sequence[Varassign]:
varassigns = []
for (i, line) in enumerate(self.lines):
- m = re.search(r'^(#?[\w+\-]+?)([!+:?]?=)([ \t]*)([^#\\]*?)(\s*)(#.*|)$', line)
+ pattern = r'''(?x)
+ ^
+ ([#]?[\w+\-]+?) # varname
+ ([!+:?]?=) # op
+ ([ \t]*) # indent
+ ([^#\\]*?) # value
+ (\s*) # space_after_value
+ ([#].*|) # comment
+ $
+ '''
+ m = re.search(pattern, line)
if m and m[1].lstrip('#') == varname:
- varassigns.append(Varassign(i, m[1], m[2], m[3], m[4], m[5], m[6]))
+ varassigns.append(Varassign(i, *m.groups()))
return varassigns
def unique_varassign(self, varname: str) -> Optional[Varassign]:
@@ -239,6 +243,7 @@
self.extract_sufx = ''
self.categories = ''
self.github_project = ''
+ self.github_tag = ''
self.github_release = ''
self.dist_subdir = ''
self.pkgname_prefix = ''
@@ -290,12 +295,15 @@
self.homepage = self.url[:-len(self.distfile)] + ' # TODO: check'
def adjust_site_SourceForge(self):
- pattern = r'^https?://downloads\.sourceforge\.net/' \
- r'(?:project|sourceforge)/' \
- r'([^/?]+)/' \
- r'((?:[^/?]+/)*)' \
- r'([^/?]+)' \
- r'(?:\?.*)?$'
+ pattern = r'''(?x)
+ ^
+ https?://downloads\.sourceforge\.net/(?:project|sourceforge)/
+ ([^/?]+)/ # project name
+ ((?:[^/?]+/)*) # subdirectories
+ ([^/?]+) # filename
+ (?:\?.*)? # query parameters
+ $
+ '''
m = re.search(pattern, self.url)
if not m:
return
@@ -306,11 +314,15 @@
self.distfile = filename
def adjust_site_GitHub_archive(self):
- pattern = r'^https://github\.com/' \
- r'(.+)/' \
- r'(.+)/archive/' \
- r'(.+)' \
- r'(\.tar\.gz|\.zip)$'
+ pattern = r'''(?x)
+ ^
+ https://github\.com/
+ (.+)/ # org
+ (.+)/archive/ # proj
+ (.+) # tag
+ (\.tar\.gz|\.zip) # ext
+ $
+ '''
m = re.search(pattern, self.url)
if not m:
return
@@ -318,6 +330,7 @@
org, proj, tag, ext = m.groups()
self.github_project = proj
+ self.github_tag = tag
self.master_sites = f'${{MASTER_SITE_GITHUB:={org}/}}'
self.homepage = f'https://github.com/{org}/{proj}/'
if proj not in tag:
@@ -326,12 +339,15 @@
self.distfile = tag + ext
def adjust_site_GitHub_release(self):
- pattern = r'^https://github\.com/' \
- r'(.+)/' \
- r'(.+)/releases/download/' \
- r'(.+)/' \
- r'(.+)' \
- r'(\.tar\.gz|\.zip)$'
+ pattern = r'''(?x)
+ ^https://github\.com/
+ (.+)/ # org
+ (.+)/ # proj
+ releases/download/
+ (.+)/ # tag
+ (.+) # base
+ (\.tar\.gz|\.zip)$ # ext
+ '''
m = re.search(pattern, self.url)
if not m:
return
@@ -395,6 +411,7 @@
lines.add_vars(
Var('GITHUB_PROJECT', '=', self.github_project),
+ Var('GITHUB_TAG', '=', self.github_tag),
Var('DISTNAME', '=', self.distname),
Var('PKGNAME', '=', self.pkgname),
Var('CATEGORIES', '=', self.categories),
@@ -416,7 +433,7 @@
return lines
- def generate_Makefile(self):
+ def generate_Makefile(self) -> Lines:
self.adjust_site_SourceForge()
self.adjust_site_GitHub_archive()
self.adjust_site_GitHub_release()
@@ -427,14 +444,14 @@
def generate_package(self, up: Url2Pkg) -> Lines:
pkgdir = up.pkgdir
- makefile = f'{pkgdir}/Makefile'
- descr = f'{pkgdir}/DESCR'
- plist = f'{pkgdir}/PLIST'
+ makefile = pkgdir / 'Makefile'
+ descr = pkgdir / 'DESCR'
+ plist = pkgdir / 'PLIST'
initial_lines = self.generate_Makefile()
try:
- os.rename(makefile, f'{makefile}.url2pkg~')
+ makefile.replace(f'{makefile}.url2pkg~')
except OSError:
pass
initial_lines.write_to(makefile)
@@ -443,11 +460,9 @@
if not os.path.isfile(descr):
Lines().write_to(descr)
- editor = os.getenv('PKGEDITOR') or os.getenv('EDITOR') or 'vi'
- subprocess.check_call([editor, makefile])
+ subprocess.check_call([up.editor, makefile])
- up.bmake('distinfo')
- up.bmake('extract')
+ up.bmake('distinfo', 'extract')
return initial_lines
@@ -568,10 +583,10 @@
effective_env.update(env)
self.up.debug('reading dependencies: cd {0} && env {1} {2}', cwd, env, cmd)
- output = subprocess.check_output(args=cmd, shell=True, env=effective_env, cwd=cwd)
+ output: bytes = subprocess.check_output(args=cmd, shell=True, env=effective_env, cwd=cwd)
dep_lines: List[Tuple[str, str, str, str]] = []
- for line in output.decode('utf-8').split('\n'):
+ for line in output.decode('utf-8').splitlines():
# example: DEPENDS pkgbase>=1.2.3:../../category/pkgbase
m = re.search(r'^(\w+)\t([^\s:>]+)(>[^\s:]+|)(?::(\.\./\.\./\S+))?$', line)
if m:
@@ -604,7 +619,6 @@
def wrksrc_find(self, what: Union[str, Callable[[str], bool]]) -> Iterator[str]:
def search(f):
- print('search', f)
return re.search(what, f) if type(what) == str else what(f)
return list(sorted(filter(search, self.wrksrc_files)))
@@ -706,7 +720,7 @@
self.adjust_perl_module_homepage()
try:
- os.unlink('PLIST')
+ (self.up.pkgdir / 'PLIST').unlink()
except OSError:
pass
@@ -801,36 +815,51 @@
def adjust_lines_python_module(self, lines: Lines):
- initial_lines = self.initial_lines
- current_lines = self.makefile_lines
+ initial_lines = self.initial_lines # as generated by url2pkg
+ edited_lines = self.makefile_lines # as edited by the package developer
- if 'python' not in initial_lines.get('CATEGORIES'):
+ if 'python' not in lines.get('CATEGORIES'):
+ return
+ if lines.get('GITHUB_PROJECT') == '':
return
+
+ # don't risk to overwrite any changes made by the package developer.
+ if edited_lines.lines != initial_lines.lines:
+ lines.lines.insert(-2, '# TODO: Migrate MASTER_SITES to MASTER_SITE_PYPI')
+ return
+
pkgbase = initial_lines.get('GITHUB_PROJECT')
- if pkgbase == '':
- return
- pkgbase1 = pkgbase[:1]
+ pkgbase1 = pkgbase[:1] if pkgbase != '' else ''
pkgversion_norev = re.sub(r'^v', '', initial_lines.get('DISTNAME'))
- # don't risk to overwrite any changes made by the package developer.
- if '\n'.join(current_lines.lines) != '\n'.join(initial_lines.lines):
- lines.lines.insert(-2, '# TODO: Migrate MASTER_SITES to PYPI')
- return
-
tx_lines = Lines(*self.makefile_lines.lines)
- if (tx_lines.remove('GITHUB_PROJECT')
+ if not (tx_lines.remove('GITHUB_PROJECT')
and tx_lines.set('DISTNAME', f'{pkgbase}-{pkgversion_norev}')
and tx_lines.set('PKGNAME', '${PYPKGPREFIX}-${DISTNAME}')
and tx_lines.set('MASTER_SITES', f'${{MASTER_SITE_PYPI:={pkgbase1}/{pkgbase}/}}')
and tx_lines.remove('DIST_SUBDIR')):
- tx_lines.remove_if('EXTRACT_SUFX', '.zip')
- self.makefile_lines = tx_lines
- self.regenerate_distinfo = True
+ return
+
+ tx_lines.remove_if('GITHUB_TAG', initial_lines.get('DISTNAME'))
+ tx_lines.remove_if('EXTRACT_SUFX', '.zip')
+
+ up = self.up
+ try_mk = up.pkgdir / 'try-pypi.mk'
+ tx_lines.write_to(try_mk)
+ args = [up.make, '-f', str(try_mk), 'distinfo']
+ up.debug('running {0} to try PyPI', args)
+ fetch_ok = subprocess.call(args, cwd=up.pkgdir) == 0
+ try_mk.unlink()
+ if not fetch_ok:
+ return
+
+ lines.lines = tx_lines.lines
Home |
Main Index |
Thread Index |
Old Index