pkgsrc-Changes-HG archive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]

[pkgsrc/trunk]: pkgsrc/pkgtools/url2pkg/files pkgtools/url2pkg: improve URL h...



details:   https://anonhg.NetBSD.org/pkgsrc/rev/274d38ce8074
branches:  trunk
changeset: 402213:274d38ce8074
user:      rillig <rillig%pkgsrc.org@localhost>
date:      Sat Oct 05 11:02:30 2019 +0000

description:
pkgtools/url2pkg: improve URL handling for SourceForge

* Improved handling of SourceForge URLs
* Fixed wrong HOMEPAGE for URLs composed from MASTER_SITE_*
* Improved the existing tests and added some more

diffstat:

 pkgtools/url2pkg/files/url2pkg.py      |  148 +++--
 pkgtools/url2pkg/files/url2pkg_test.py |  838 ++++++++++++++++++--------------
 2 files changed, 557 insertions(+), 429 deletions(-)

diffs (truncated from 1473 to 300 lines):

diff -r d3a1a7654a2c -r 274d38ce8074 pkgtools/url2pkg/files/url2pkg.py
--- a/pkgtools/url2pkg/files/url2pkg.py Sat Oct 05 08:53:27 2019 +0000
+++ b/pkgtools/url2pkg/files/url2pkg.py Sat Oct 05 11:02:30 2019 +0000
@@ -1,5 +1,5 @@
 #! @PYTHONBIN@
-# $NetBSD: url2pkg.py,v 1.8 2019/10/04 22:26:34 rillig Exp $
+# $NetBSD: url2pkg.py,v 1.9 2019/10/05 11:02:30 rillig Exp $
 
 # Copyright (c) 2019 The NetBSD Foundation, Inc.
 # All rights reserved.
@@ -47,7 +47,7 @@
 import re
 import subprocess
 import sys
-from typing import Callable, Dict, Iterator, List, Optional, Sequence, Union, Tuple
+from typing import Callable, Dict, Iterator, List, Optional, Sequence, Tuple, Union
 
 
 class Var:
@@ -81,10 +81,12 @@
         self.perl5 = '@PERL5@'
         self.pkgsrcdir = '@PKGSRCDIR@'
         self.pythonbin = '@PYTHONBIN@'
-        self.pkgdir = '.'  # only overridable for tests
+
         self.verbose = False
-        self.out = sys.stdout
-        self.err = sys.stderr
+
+        self.pkgdir = '.'  # only overridable for tests
+        self.out = sys.stdout  # only overridable for tests
+        self.err = sys.stderr  # only overridable for tests
 
     def debug(self, fmt: str, *args):
         if self.verbose:
@@ -137,6 +139,33 @@
             pass
         os.rename(f'{filename}.tmp', filename)
 
+    def all_varassigns(self, varname: str) -> Sequence[Varassign]:
+        varassigns = []
+        for (i, line) in enumerate(self.lines):
+            m = re.search(r'^(#?[\w+\-]+?)([!+:?]?=)([ \t]*)([^#\\]*?)(\s*)(#.*|)$', line)
+            if m and m[1].lstrip('#') == varname:
+                varassigns.append(Varassign(i, m[1], m[2], m[3], m[4], m[5], m[6]))
+        return varassigns
+
+    def unique_varassign(self, varname: str) -> Optional[Varassign]:
+        varassigns = self.all_varassigns(varname)
+        return varassigns[0] if len(varassigns) == 1 else None
+
+    def get(self, varname: str) -> str:
+        """
+        Returns the value from the only variable assignment, or an empty
+        string.
+        """
+        varassign = self.unique_varassign(varname)
+        return varassign.value if varassign is not None and varassign.varname == varname else ''
+
+    def index(self, pattern: str) -> int:
+        """ Returns the first index where the pattern is found, or -1. """
+        for (i, line) in enumerate(self.lines):
+            if re.search(pattern, line):
+                return i
+        return -1
+
     def add(self, *lines: Sequence[str]):
         for line in lines:
             assert type(line) == str, type(line)
@@ -162,18 +191,6 @@
             self.add(var.name + var.op + '\t' * tabs + var.value)
         self.add('')
 
-    def unique_varassign(self, varname: str) -> Optional[Varassign]:
-        varassigns = self.all_varassigns(varname)
-        return varassigns[0] if len(varassigns) == 1 else None
-
-    def all_varassigns(self, varname: str) -> Sequence[Varassign]:
-        varassigns = []
-        for (i, line) in enumerate(self.lines):
-            m = re.search(r'^(#?[\w+\-]+?)([!+:?]?=)([ \t]*)([^#\\]*?)(\s*)(#.*|)$', line)
-            if m and m[1].lstrip('#') == varname:
-                varassigns.append(Varassign(i, m[1], m[2], m[3], m[4], m[5], m[6]))
-        return varassigns
-
     def set(self, varname: str, new_value: str) -> bool:
         """ Updates the value of an existing variable in the lines. """
 
@@ -202,14 +219,6 @@
             self.lines.pop(varassign.index)
         return varassign is not None
 
-    def get(self, varname: str) -> str:
-        """
-        Returns the value from the only variable assignment, or an empty
-        string.
-        """
-        varassign = self.unique_varassign(varname)
-        return varassign.value if varassign is not None and varassign.varname == varname else ''
-
     def remove_if(self, varname: str, expected_value: str) -> bool:
         """ Removes a variable assignment if its value is the expected one. """
         for varassign in self.all_varassigns(varname):
@@ -218,13 +227,6 @@
                 return True
         return False
 
-    def index(self, pattern: str) -> int:
-        """ Returns the first index where the pattern is found, or -1. """
-        for (i, line) in enumerate(self.lines):
-            if re.search(pattern, line):
-                return i
-        return -1
-
 
 class Generator:
     """ Generates the initial package Makefile. """
@@ -245,8 +247,10 @@
         self.distname = ''
         self.pkgname = ''
 
-    @staticmethod
-    def foreach_site(action: Callable[[str, str], None]):
+    def foreach_site_from_sites_mk(self, action: Callable[[str, str], None]):
+        if self.master_sites != '':
+            return
+
         varname = ''
         with open('../../mk/fetch/sites.mk') as sites_mk:
             for line in sites_mk:
@@ -262,38 +266,51 @@
                 site_url = m[1]
                 action(varname, site_url)
 
-    def adjust_site(self, varname: str, site_url: str):
+    def adjust_site_from_sites_mk(self, varname: str, site_url: str):
         if not self.url.startswith(site_url):
             return
 
         rest = self.url[len(site_url):]
-        m = re.search(r'^(.+)/([^/]+)$', rest)
-        if not m:
+        if '/' not in rest:
             self.master_sites = f'${{{varname}}}'
+            self.distfile = rest
+            self.homepage = '# TODO'
             return
 
-        subdir, self.distfile = m.groups()
+        subdir, self.distfile = re.search(r'^(.*/)(.*)$', rest).groups()
 
-        self.master_sites = f'${{{varname}:={subdir}/}}'
-        if varname == 'MASTER_SITE_SOURCEFORGE':
-            self.homepage = f'https://{subdir}.sourceforge.net/'
-        elif varname == 'MASTER_SITE_GNU':
-            self.homepage = f'https://www.gnu.org/software/{subdir}/'
+        self.master_sites = f'${{{varname}:={subdir}}}'
+        if varname == 'MASTER_SITE_GNU':
+            self.homepage = f'https://www.gnu.org/software/{subdir}'
         else:
-            self.homepage = site_url[:-len(self.distfile)]
+            print('site_url', site_url)
+            print('distfile', self.distfile)
+            self.homepage = self.url[:-len(self.distfile)] + ' # TODO: check'
+            print('homepage', self.homepage)
 
-    def adjust_site_sourceforge(self):
-        m = re.search(r'^https://downloads\.sourceforge\.net/project/([^/?]+)/[^?]+/([^/?]+)(?:[?].*)?$', self.url)
+    def adjust_site_SourceForge(self):
+        pattern = r'^https?://downloads\.sourceforge\.net/' \
+                  r'(?:project|sourceforge)/' \
+                  r'([^/?]+)/' \
+                  r'((?:[^/?]+/)*)' \
+                  r'([^/?]+)' \
+                  r'(?:\?.*)?$'
+        m = re.search(pattern, self.url)
         if not m:
             return
 
-        project, filename = m.groups()
-        self.master_sites = f'${{MASTER_SITE_SOURCEFORGE:={project}/}}'
+        project, subdir, filename = m.groups()
+        self.master_sites = f'${{MASTER_SITE_SOURCEFORGE:={project}/{subdir}}}'
         self.homepage = f'https://{project}.sourceforge.net/'
         self.distfile = filename
 
     def adjust_site_GitHub_archive(self):
-        m = re.search(r'^https://github\.com/(.+)/(.+)/archive/(.+)(\.tar\.gz|\.zip)$', self.url)
+        pattern = r'^https://github\.com/' \
+                  r'(.+)/' \
+                  r'(.+)/archive/' \
+                  r'(.+)' \
+                  r'(\.tar\.gz|\.zip)$'
+        m = re.search(pattern, self.url)
         if not m:
             return
 
@@ -308,7 +325,13 @@
         self.distfile = tag + ext
 
     def adjust_site_GitHub_release(self):
-        m = re.search(r'^https://github\.com/(.+)/(.+)/releases/download/(.+)/(.+)(\.tar\.gz|\.zip)$', self.url)
+        pattern = r'^https://github\.com/' \
+                  r'(.+)/' \
+                  r'(.+)/releases/download/' \
+                  r'(.+)/' \
+                  r'(.+)' \
+                  r'(\.tar\.gz|\.zip)$'
+        m = re.search(pattern, self.url)
         if not m:
             return
 
@@ -327,15 +350,14 @@
         if self.master_sites != '':
             return
 
-        m = re.search(r'^(.*/)(.*)$', self.url)
-        if not m:
+        if '/' not in self.url:
             sys.exit(f'error: URL "{self.url}" must have at least one slash')
+        base_url, self.distfile = re.search(r'^(.*/)(.*)$', self.url).groups()
 
-        self.master_sites = m[1]
-        self.distfile = m[2]
-        self.homepage = self.master_sites
+        self.master_sites = base_url
+        self.homepage = base_url
 
-    def determine_distname(self):
+    def adjust_everything_else(self):
         m = re.search(r'^(.*?)((?:\.tar)?\.\w+)$', self.distfile)
         if m:
             distname, extract_sufx = m.groups()
@@ -394,12 +416,12 @@
         return lines
 
     def generate_Makefile(self):
-        self.foreach_site(self.adjust_site)
-        self.adjust_site_sourceforge()
+        self.adjust_site_SourceForge()
         self.adjust_site_GitHub_archive()
         self.adjust_site_GitHub_release()
+        self.foreach_site_from_sites_mk(self.adjust_site_from_sites_mk)
         self.adjust_site_other()
-        self.determine_distname()
+        self.adjust_everything_else()
         return self.generate_lines()
 
     def generate_package(self, up: Url2Pkg) -> Lines:
@@ -804,7 +826,7 @@
             self.makefile_lines = tx_lines
             self.regenerate_distinfo = True
 
-    def generate_adjusted_Makefile_lines(self) -> Lines:
+    def generate_lines(self) -> Lines:
         marker_index = self.makefile_lines.index(r'^# url2pkg-marker')
         if marker_index == -1:
             raise Exception('ERROR: didn\'t find the url2pkg marker in the Makefile.')
@@ -847,7 +869,7 @@
 
         return lines
 
-    def adjust_package_from_extracted_distfiles(self):
+    def adjust(self):
 
         def scan(basedir: str, pattern: str) -> List[str]:
             full_paths = glob.glob(f'{basedir}/{pattern}', recursive=True)
@@ -873,7 +895,7 @@
         self.adjust_po()
         self.adjust_use_languages()
 
-        self.generate_adjusted_Makefile_lines().write_to(self.up.pkgdir + '/Makefile')
+        self.generate_lines().write_to(self.up.pkgdir + '/Makefile')
 
         if self.regenerate_distinfo:
             self.up.bmake('distinfo')
@@ -899,7 +921,7 @@
     else:
         initial_lines = Generator(url).generate_lines()
 
-    Adjuster(up, url, initial_lines).adjust_package_from_extracted_distfiles()
+    Adjuster(up, url, initial_lines).adjust()
 
     print('')
     print('Remember to run pkglint when you\'re done.')
diff -r d3a1a7654a2c -r 274d38ce8074 pkgtools/url2pkg/files/url2pkg_test.py
--- a/pkgtools/url2pkg/files/url2pkg_test.py    Sat Oct 05 08:53:27 2019 +0000
+++ b/pkgtools/url2pkg/files/url2pkg_test.py    Sat Oct 05 11:02:30 2019 +0000
@@ -1,7 +1,9 @@
-# $NetBSD: url2pkg_test.py,v 1.7 2019/10/04 22:26:34 rillig Exp $
+# $NetBSD: url2pkg_test.py,v 1.8 2019/10/05 11:02:30 rillig Exp $
 
+import pytest
 from url2pkg import *
 
+mkcvsid = '# $''NetBSD$'
 up: Url2Pkg
 
 
@@ -15,16 +17,57 @@



Home | Main Index | Thread Index | Old Index