pkgsrc-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[pkgsrc/trunk]: pkgsrc/pkgtools/url2pkg/files pkgtools/url2pkg: fixed migrati...



details:   https://anonhg.NetBSD.org/pkgsrc/rev/b024ff9fa05f
branches:  trunk
changeset: 340651:b024ff9fa05f
user:      rillig <rillig%pkgsrc.org@localhost>
date:      Sat Oct 05 18:00:09 2019 +0000

description:
pkgtools/url2pkg: fixed migration of Python packages from GitHub to PyPI

diffstat:

 pkgtools/url2pkg/files/url2pkg.py      |  165 +++++++++++--------
 pkgtools/url2pkg/files/url2pkg_test.py |  273 ++++++++++++++++++++++++++++++--
 2 files changed, 352 insertions(+), 86 deletions(-)

diffs (truncated from 788 to 300 lines):

diff -r c66079bfcf6b -r b024ff9fa05f pkgtools/url2pkg/files/url2pkg.py
--- a/pkgtools/url2pkg/files/url2pkg.py Sat Oct 05 17:39:55 2019 +0000
+++ b/pkgtools/url2pkg/files/url2pkg.py Sat Oct 05 18:00:09 2019 +0000
@@ -1,5 +1,5 @@
 #! @PYTHONBIN@
-# $NetBSD: url2pkg.py,v 1.10 2019/10/05 12:22:51 rillig Exp $
+# $NetBSD: url2pkg.py,v 1.11 2019/10/05 18:00:09 rillig Exp $
 
 # Copyright (c) 2019 The NetBSD Foundation, Inc.
 # All rights reserved.
@@ -44,6 +44,7 @@
 import getopt
 import glob
 import os
+import pathlib
 import re
 import subprocess
 import sys
@@ -81,10 +82,11 @@
         self.perl5 = '@PERL5@'
         self.pkgsrcdir = '@PKGSRCDIR@'
         self.pythonbin = '@PYTHONBIN@'
+        self.editor = os.getenv('PKGEDITOR') or os.getenv('EDITOR') or 'vi'
 
         self.verbose = False
 
-        self.pkgdir = '.'  # only overridable for tests
+        self.pkgdir = pathlib.Path('.')  # only overridable for tests
         self.out = sys.stdout  # only overridable for tests
         self.err = sys.stderr  # only overridable for tests
 
@@ -121,30 +123,32 @@
             self.add(line)
 
     @classmethod
-    def read_from(cls, filename: str) -> 'Lines':
-        pass
-
-        lines = Lines()
-        with open(filename) as f:
-            for line in f:
-                lines.add(line.rstrip('\n'))
-        return lines
+    def read_from(cls, filename: Union[str, pathlib.Path]) -> 'Lines':
+        return Lines(*pathlib.Path(filename).read_text().splitlines())
 
-    def write_to(self, filename: str):
-        with open(f'{filename}.tmp', 'w') as f:
+    def write_to(self, filename: Union[str, pathlib.Path]):
+        target = pathlib.Path(filename)
+        tmp = target.with_name(f'{target.name}.tmp')
+        with tmp.open('w') as f:
             f.writelines(line + '\n' for line in self.lines)
-        try:
-            os.remove(filename)
-        except OSError:
-            pass
-        os.rename(f'{filename}.tmp', filename)
+        tmp.replace(target)
 
     def all_varassigns(self, varname: str) -> Sequence[Varassign]:
         varassigns = []
         for (i, line) in enumerate(self.lines):
-            m = re.search(r'^(#?[\w+\-]+?)([!+:?]?=)([ \t]*)([^#\\]*?)(\s*)(#.*|)$', line)
+            pattern = r'''(?x)
+                ^
+                ([#]?[\w+\-]+?)  # varname
+                ([!+:?]?=)       # op
+                ([ \t]*)         # indent
+                ([^#\\]*?)       # value
+                (\s*)            # space_after_value
+                ([#].*|)         # comment
+                $
+                '''
+            m = re.search(pattern, line)
             if m and m[1].lstrip('#') == varname:
-                varassigns.append(Varassign(i, m[1], m[2], m[3], m[4], m[5], m[6]))
+                varassigns.append(Varassign(i, *m.groups()))
         return varassigns
 
     def unique_varassign(self, varname: str) -> Optional[Varassign]:
@@ -239,6 +243,7 @@
         self.extract_sufx = ''
         self.categories = ''
         self.github_project = ''
+        self.github_tag = ''
         self.github_release = ''
         self.dist_subdir = ''
         self.pkgname_prefix = ''
@@ -290,12 +295,15 @@
             self.homepage = self.url[:-len(self.distfile)] + ' # TODO: check'
 
     def adjust_site_SourceForge(self):
-        pattern = r'^https?://downloads\.sourceforge\.net/' \
-                  r'(?:project|sourceforge)/' \
-                  r'([^/?]+)/' \
-                  r'((?:[^/?]+/)*)' \
-                  r'([^/?]+)' \
-                  r'(?:\?.*)?$'
+        pattern = r'''(?x)
+            ^
+            https?://downloads\.sourceforge\.net/(?:project|sourceforge)/
+            ([^/?]+)/       # project name
+            ((?:[^/?]+/)*)  # subdirectories
+            ([^/?]+)        # filename
+            (?:\?.*)?       # query parameters
+            $
+            '''
         m = re.search(pattern, self.url)
         if not m:
             return
@@ -306,11 +314,15 @@
         self.distfile = filename
 
     def adjust_site_GitHub_archive(self):
-        pattern = r'^https://github\.com/' \
-                  r'(.+)/' \
-                  r'(.+)/archive/' \
-                  r'(.+)' \
-                  r'(\.tar\.gz|\.zip)$'
+        pattern = r'''(?x)
+            ^
+            https://github\.com/
+            (.+)/               # org
+            (.+)/archive/       # proj
+            (.+)                # tag
+            (\.tar\.gz|\.zip)   # ext
+            $
+            '''
         m = re.search(pattern, self.url)
         if not m:
             return
@@ -318,6 +330,7 @@
         org, proj, tag, ext = m.groups()
 
         self.github_project = proj
+        self.github_tag = tag
         self.master_sites = f'${{MASTER_SITE_GITHUB:={org}/}}'
         self.homepage = f'https://github.com/{org}/{proj}/'
         if proj not in tag:
@@ -326,12 +339,15 @@
         self.distfile = tag + ext
 
     def adjust_site_GitHub_release(self):
-        pattern = r'^https://github\.com/' \
-                  r'(.+)/' \
-                  r'(.+)/releases/download/' \
-                  r'(.+)/' \
-                  r'(.+)' \
-                  r'(\.tar\.gz|\.zip)$'
+        pattern = r'''(?x)
+            ^https://github\.com/
+            (.+)/               # org
+            (.+)/               # proj   
+            releases/download/
+            (.+)/               # tag
+            (.+)                # base
+            (\.tar\.gz|\.zip)$  # ext
+            '''
         m = re.search(pattern, self.url)
         if not m:
             return
@@ -395,6 +411,7 @@
 
         lines.add_vars(
             Var('GITHUB_PROJECT', '=', self.github_project),
+            Var('GITHUB_TAG', '=', self.github_tag),
             Var('DISTNAME', '=', self.distname),
             Var('PKGNAME', '=', self.pkgname),
             Var('CATEGORIES', '=', self.categories),
@@ -416,7 +433,7 @@
 
         return lines
 
-    def generate_Makefile(self):
+    def generate_Makefile(self) -> Lines:
         self.adjust_site_SourceForge()
         self.adjust_site_GitHub_archive()
         self.adjust_site_GitHub_release()
@@ -427,14 +444,14 @@
 
     def generate_package(self, up: Url2Pkg) -> Lines:
         pkgdir = up.pkgdir
-        makefile = f'{pkgdir}/Makefile'
-        descr = f'{pkgdir}/DESCR'
-        plist = f'{pkgdir}/PLIST'
+        makefile = pkgdir / 'Makefile'
+        descr = pkgdir / 'DESCR'
+        plist = pkgdir / 'PLIST'
 
         initial_lines = self.generate_Makefile()
 
         try:
-            os.rename(makefile, f'{makefile}.url2pkg~')
+            makefile.replace(f'{makefile}.url2pkg~')
         except OSError:
             pass
         initial_lines.write_to(makefile)
@@ -443,11 +460,9 @@
         if not os.path.isfile(descr):
             Lines().write_to(descr)
 
-        editor = os.getenv('PKGEDITOR') or os.getenv('EDITOR') or 'vi'
-        subprocess.check_call([editor, makefile])
+        subprocess.check_call([up.editor, makefile])
 
-        up.bmake('distinfo')
-        up.bmake('extract')
+        up.bmake('distinfo', 'extract')
 
         return initial_lines
 
@@ -568,10 +583,10 @@
         effective_env.update(env)
 
         self.up.debug('reading dependencies: cd {0} && env {1} {2}', cwd, env, cmd)
-        output = subprocess.check_output(args=cmd, shell=True, env=effective_env, cwd=cwd)
+        output: bytes = subprocess.check_output(args=cmd, shell=True, env=effective_env, cwd=cwd)
 
         dep_lines: List[Tuple[str, str, str, str]] = []
-        for line in output.decode('utf-8').split('\n'):
+        for line in output.decode('utf-8').splitlines():
             # example: DEPENDS   pkgbase>=1.2.3:../../category/pkgbase
             m = re.search(r'^(\w+)\t([^\s:>]+)(>[^\s:]+|)(?::(\.\./\.\./\S+))?$', line)
             if m:
@@ -604,7 +619,6 @@
 
     def wrksrc_find(self, what: Union[str, Callable[[str], bool]]) -> Iterator[str]:
         def search(f):
-            print('search', f)
             return re.search(what, f) if type(what) == str else what(f)
 
         return list(sorted(filter(search, self.wrksrc_files)))
@@ -706,7 +720,7 @@
         self.adjust_perl_module_homepage()
 
         try:
-            os.unlink('PLIST')
+            (self.up.pkgdir / 'PLIST').unlink()
         except OSError:
             pass
 
@@ -801,36 +815,51 @@
 
     def adjust_lines_python_module(self, lines: Lines):
 
-        initial_lines = self.initial_lines
-        current_lines = self.makefile_lines
+        initial_lines = self.initial_lines  # as generated by url2pkg
+        edited_lines = self.makefile_lines  # as edited by the package developer
 
-        if 'python' not in initial_lines.get('CATEGORIES'):
+        if 'python' not in lines.get('CATEGORIES'):
+            return
+        if lines.get('GITHUB_PROJECT') == '':
             return
+
+        # don't risk to overwrite any changes made by the package developer.
+        if edited_lines.lines != initial_lines.lines:
+            lines.lines.insert(-2, '# TODO: Migrate MASTER_SITES to MASTER_SITE_PYPI')
+            return
+
         pkgbase = initial_lines.get('GITHUB_PROJECT')
-        if pkgbase == '':
-            return
-        pkgbase1 = pkgbase[:1]
+        pkgbase1 = pkgbase[:1] if pkgbase != '' else ''
         pkgversion_norev = re.sub(r'^v', '', initial_lines.get('DISTNAME'))
 
-        # don't risk to overwrite any changes made by the package developer.
-        if '\n'.join(current_lines.lines) != '\n'.join(initial_lines.lines):
-            lines.lines.insert(-2, '# TODO: Migrate MASTER_SITES to PYPI')
-            return
-
         tx_lines = Lines(*self.makefile_lines.lines)
-        if (tx_lines.remove('GITHUB_PROJECT')
+        if not (tx_lines.remove('GITHUB_PROJECT')
                 and tx_lines.set('DISTNAME', f'{pkgbase}-{pkgversion_norev}')
                 and tx_lines.set('PKGNAME', '${PYPKGPREFIX}-${DISTNAME}')
                 and tx_lines.set('MASTER_SITES', f'${{MASTER_SITE_PYPI:={pkgbase1}/{pkgbase}/}}')
                 and tx_lines.remove('DIST_SUBDIR')):
-            tx_lines.remove_if('EXTRACT_SUFX', '.zip')
-            self.makefile_lines = tx_lines
-            self.regenerate_distinfo = True
+            return
+
+        tx_lines.remove_if('GITHUB_TAG', initial_lines.get('DISTNAME'))
+        tx_lines.remove_if('EXTRACT_SUFX', '.zip')
+
+        up = self.up
+        try_mk = up.pkgdir / 'try-pypi.mk'
+        tx_lines.write_to(try_mk)
+        args = [up.make, '-f', str(try_mk), 'distinfo']
+        up.debug('running {0} to try PyPI', args)
+        fetch_ok = subprocess.call(args, cwd=up.pkgdir) == 0
+        try_mk.unlink()
+        if not fetch_ok:
+            return
+
+        lines.lines = tx_lines.lines



Home | Main Index | Thread Index | Old Index