diff --git a/vrms_arch/license_finder.py b/vrms_arch/license_finder.py index 6aa59a0..444a321 100644 --- a/vrms_arch/license_finder.py +++ b/vrms_arch/license_finder.py @@ -1,215 +1,188 @@ import pyalpm +import re import sys -AMBIGUOUS_LICENSES = [ +def clean_license_name(license): + license = license.lower() + license = re.sub('(?:^custom:|[,\s_"-])', '', license) + return license + +AMBIGUOUS_LICENSES = [clean_license_name(license) for license in [ "custom", "other", "unknown", - "CUSTOM", # CCPL (Creative Commons) should be specified with one of the # sublicenses (one of /usr/share/licenses/common/CCPL/*) , some of # which are non-free "CCPL", # ['claws-mail-themes', '0ad', '0ad-data', 'archlinux-lxdm-theme', 'mari0', 'performous-freesongs'] "CCPL:cc-by-sa-3.0", -] +]] -FREE_LICENSES = [ +FREE_LICENSES = [clean_license_name(license) for license in [ + 'AFL-3.0', 'AGPL', 'AGPL3', - 'APACHE', 'Apache', 'Apache 2.0', + 'Apache 2.0 with LLVM Exception', + 'Apache 2.0 with LLVM Execption', 'Apache License (2.0)', - 'apache', + 'Apache2', + 'Arphic Public License', 'Artistic', - 'Artistic2.0', + 'Artistic 2.0', + 'Beerware', + 'bitstream-vera', 'Boost', - 'boost', 'BSD', - 'bsd', 'BSD2', - 'BSD-2', + 'BSD-2-clause', 'BSD3', + 'BSD-3-clause', + 'BSD-like', + 'BSD-style', + 'BSL', + 'bzip2', 'CC0', - 'CC BY-SA 4.0', - 'CC-BY-SA 4.0', + 'CC-BY-SA', + 'CC-BY-SA-2.5', + 'CC-BY-SA-3.0', + 'CC BY-SA-4.0', 'CCPL:by-sa', 'CCPL:cc-by-sa', 'CDDL', + 'CeCILL', 'CPL', + 'Creative Commons, Attribution 3.0 Unported', + 'dumb', + 'EDL', 'EPL', 'EPL/1.1', + 'etpan', + 'ex', + 'Expat', 'FDL', 'FDL1.2', + 'FFSL', + 'FIPL', + 'font embedding exception', + 'GD', + 'GFL', 'GPL', - 'GPL-2' - 'GPL-2.0', - 'GPL-2.0+', - 'GPL-3', - 'GPL-3.0', 'GPL2', + 'GPL-2.0+', + 'GPL-2.0', 'GPL3', - 'GPL3-only', + 'GPL-3.0', 'GPL3+GPLv2', + 'GPL3-only', + 'GPL3 or any later version', + 'GPL/BSD', + 'GPL+FE', 'GPLv2', 'GPLv3', - 'GPL3 or any later version', + 'HPND', + 'IBM Public Licence', + 'icu', + 'ImageMagick', + 'Info-ZIP', + 'INN', 'ISC', + 'isc-dhcp', + 'JasPer2.0', + 'Khronos', 'LGPL', 'LGPL2', 'LGPL2.1', + 'LGPL2.1+', 'LGPL3', 'LGPLv3+', + 'libpng', + 'libtiff', + 'libxcomposite', + 'LPPL', + 'lsof', + 'MirOS', 'MIT', + 'MIT/X', + 'MITX11', + 'MIT-style', + 'Modified BSD', 'MPL', 'MPL2', - 'Modified BSD', + 'MPLv2', + 'NCSA', + 'neovim', + 'nfsidmap', + 'NoCopyright', + 'none', + 'OASIS', 'OFL', 'OFL-1.1', - 'PHP', - 'PSF', + 'OPEN DATA LICENSE', + 'OpenLDAP', + 'OpenMPI', + 'OSGPL', 'perl', 'PerlArtistic', 'PerlArtistic2', + 'PHP', + 'pil', + 'PostgreSQL', + 'PSF', 'Public Domain', 'Python', - 'RUBY', + 'Qhull', + 'QPL', + 'QPL-1.0', + 'qwt', 'Ruby', + 'scite', + 'scowl', + 'Sendmail', + 'Sendmail open source license', + 'SGI', 'SIL', - 'SIL OPEN FONT LICENSE Version 1.1', + 'SIL Open Font License', 'SIL Open Font License 1.1 and Bitstream Vera License', + 'SIL Open Font License, Version 1.0', + 'SIL OPEN FONT LICENSE Version 1.1', + 'sip', + 'Sleepycat', + 'tcl', + 'TekHVC', + 'TRADEMARKS', + 'Ubuntu Font Licence 1.0', + 'UCD', + 'Unicode-DFS', + 'University of Illinois/NCSA Open Source License', + 'Unlicense', + 'usermin', + 'vim', + 'voidspace', 'W3C', + 'w3m', + 'webmin', + 'WTF', 'WTFPL', + 'wxWindows', 'X11', - 'ZLIB', + 'X11-DEC', + 'XFREE86', + 'Xiph', 'zlib', + 'zlib/libpng', 'ZPL', - 'custom: Arphic Public_License', - 'custom: BSD', - 'custom: ISC', - 'custom: MIT', - 'custom: OFL', - 'custom: SIL Open Font License', - 'custom: QPL-1.0', - 'custom: public domain', - 'custom:"IBM Public Licence"', - 'custom:"font embedding exception"', - 'custom:"icu"', - 'custom:"pil"', - 'custom:"sip"', - 'custom:Arphic Public License', - 'custom:Arphic_Public_License', - 'custom:Artistic', - 'custom:Artistic 2.0', - 'custom:Artistic-2.0', - 'custom:Apache 2.0 with LLVM Exception', - 'custom:Apache 2.0 with LLVM Execption', - 'custom:BSD', - 'custom:BSD-like', - 'custom:BSD-style', - 'custom:BSD2', - 'custom:BSD3', - 'custom:Boost', - 'custom:CC0', - 'custom:CCBYSA', - 'custom:CCBYSA3.0', - 'custom:CCPL:by-sa', - 'custom:CeCILL', - 'custom:Creative Commons, Attribution 3.0 Unported', - 'custom:EPL', - 'custom:Expat', - 'custom:FFSL', - 'custom:FIPL', - 'custom:GD', - 'custom:GFL', - 'custom:GPL', - 'custom:GPL/BSD', - 'custom:GPL+FE', - 'custom:HPND', - 'custom:INN', - 'custom:Info-ZIP', - 'custom:ISC', - 'custom:JasPer2.0', - 'custom:Khronos', - 'custom:LGPL', - 'custom:LGPL2', - 'custom:MIT', - 'custom:MIT/X', - 'custom:MITX11', - 'custom:MPL2', - 'custom:MPLv2', - 'custom:MirOS', - 'custom:NoCopyright', - 'custom:OASIS', - 'custom:OFL', - 'custom:OPEN DATA LICENSE', - 'custom:OpenLDAP', - 'custom:OpenMPI', - 'custom:OSGPL', - 'custom:PYTHON', - 'custom:PostgreSQL', - 'custom:PSF', - 'custom:Public Domain', - 'custom:Public_Domain', - 'custom:PublicDomain', - 'custom:QPL', - 'custom:Qhull', - 'custom:SGI', - 'custom:SIL', - 'custom:SIL Open Font License, Version 1.0', - 'custom:Sendmail', - 'custom:Sendmail open source license', - 'custom:Sleepycat', - 'custom:TekHVC', - 'custom:TRADEMARKS', - 'custom:Ubuntu Font Licence 1.0', - 'custom:University of Illinois/NCSA Open Source License', - 'custom:Unlicense', - 'custom:WTFPL', - 'custom:X11', - 'custom:X11-DEC', - 'custom:XFREE86', - 'custom:Xiph', - 'custom:ZLIB', - 'custom:artistic', - 'custom:bitstream-vera', - 'custom:bzip2', - 'custom:cc-by-sa-2.5', - 'custom:dumb', - 'custom:etpan', - 'custom:ex', - 'custom:icu', - 'custom:isc-dhcp', - 'custom:libpng', - 'custom:libtiff', - 'custom:libxcomposite', - 'custom:lsof', - 'custom:nfsidmap', - 'custom:neovim', - 'custom:none', - 'custom:public domain', - 'custom:publicdomain', - 'custom:qwt', - 'custom:scite', - 'custom:scowl', - 'custom:tcl', - 'custom:unknown', - 'custom:unlicense', - 'custom:usermin', - 'custom:vim', - 'custom:voidspace', - 'custom:w3m', - 'custom:webmin', - 'custom:wxWindows', - 'custom:zlib', - 'custom:zlib/libpng', -] +]] class LicenseFinder(object): def __init__(self): - # all of the seen license names with counts + # all of the seen (clean) license names with counts self.by_license = {} + # all of the seen (clean) license names with their raw variants + self.license_names = {} + # packages with "custom" license self.unknown_packages = set() @@ -222,19 +195,29 @@ def visit_db(self, db): free_pkgs = [] for pkg in pkgs: + licenses = [] + + # get a list of all licenses on the box for license in pkg.licenses: - # get a list of all licenses on the box - if license not in self.by_license: - self.by_license[license] = [pkg] + clean_license = clean_license_name(license) + licenses.append(clean_license) + + if clean_license not in self.by_license: + self.by_license[clean_license] = [pkg] else: - self.by_license[license].append(pkg) + self.by_license[clean_license].append(pkg) + + if clean_license not in self.license_names: + self.license_names[clean_license] = {} + if license not in self.license_names[clean_license]: + self.license_names[clean_license][license] = 0 + self.license_names[clean_license][license] += 1 - free_licenses = list(filter(lambda x: x in FREE_LICENSES, pkg.licenses)) - amb_licenses = list(filter(lambda x: x in AMBIGUOUS_LICENSES, pkg.licenses)) + free_licenses = list(filter(lambda x: x in FREE_LICENSES, licenses)) + amb_licenses = list(filter(lambda x: x in AMBIGUOUS_LICENSES, licenses)) if len(free_licenses) > 0: free_pkgs.append(pkg) - continue elif len(amb_licenses) > 0: self.unknown_packages.add(pkg) else: @@ -247,7 +230,9 @@ def list_all_licenses_as_python(self): sorted_by_popularity.sort(key=lambda lic : len(self.by_license[lic]), reverse=True) for lic in sorted_by_popularity: pop = len(self.by_license[lic]) - print(" \"%s\",%s" % (lic.replace("\"", "\\\""), " # %s" % [ p.name for p in self.by_license[lic] ] if pop < obscure_license_pop_cutoff else "")) + license_names = self.license_names[lic] + license_name = max(license_names, key=license_names.get) + print(" \"%s\",%s" % (license_name.replace("\"", "\\\""), " # %s" % [ p.name for p in self.by_license[lic] ] if pop < obscure_license_pop_cutoff else "")) def list_all_licenses(self): sorted_by_popularity = list(self.by_license.keys()) diff --git a/vrms_arch/unambiguous_db.py b/vrms_arch/unambiguous_db.py index fe8c49a..b8a2e60 100644 --- a/vrms_arch/unambiguous_db.py +++ b/vrms_arch/unambiguous_db.py @@ -4,90 +4,190 @@ 'adobe-source-code-pro-fonts': ['OFL'], 'adobe-source-sans-pro-fonts': ['OFL'], 'adobe-source-serif-pro-fonts': ['OFL'], + 'amd-ucode': ['custom:non-free'], + 'boost': ['Boost'], + 'boost-libs': ['Boost'], + 'broadcom-wl': ['custom:non-free'], + 'bsd-games': ['BSD'], 'bzip2': ['custom:bzip2'], - 'db': ['custom:Sleepycat'], + 'cmake': ['BSD'], 'cups-filters': ['GPL', 'GPL2', 'GPL3-only', 'GPL3'], + 'db': ['custom:Sleepycat'], + 'discord': ['custom:non-free'], 'expat': ['MIT'], + 'file': ['BSD'], 'flashplugin': ['custom:non-free'], + 'foldingathome': ['custom:non-free'], 'fontconfig': ['custom:HPND'], 'gd': ['custom:GD'], 'ghc': ['BSD3'], 'ghc-libs': ['BSD3'], 'hdf5': ['BSD3'], + 'imagemagick': ['custom:ImageMagick', 'custom:Public Domain', 'MIT'], 'intel-ucode': ['custom:non-free'], + 'ipw2100-fw': ['custom:non-free'], + 'ipw2200-fw': ['custom:non-free'], 'java-openjfx': ['GPL2'], + 'jdk11-openjdk': ['GPL2'], 'jdk8-openjdk': ['GPL2'], + 'jdk': ['custom:non-free'], 'jdk-openjdk': ['GPL2'], + 'jre11-openjdk': ['GPL2'], + 'jre11-openjdk-headless': ['GPL2'], 'jre8-openjdk': ['GPL2'], 'jre8-openjdk-headless': ['GPL2'], + 'jre': ['custom:non-free'], 'jre-openjdk': ['GPL2'], 'jre-openjdk-headless': ['GPL2'], - 'jre11-openjdk': ['GPL2'], - 'jre11-openjdk-headless': ['GPL2'], 'l-smash': ['ISC'], + 'lapack': ['BSD'], 'lib32-sqlite': ['custom:publicdomain'], 'libaec': ['BSD2'], + 'libbsd': ['BSD', 'MIT', 'custom:Public Domain', 'custom:Beerware'], 'libdrm': ['MIT'], 'libfontenc': ['MIT'], 'libice': ['custom:X11'], 'libldap': ['custom:OpenLDAP'], - 'libmodplug': ['custom:publicdomain'], + 'libmagick6': ['custom:ImageMagick', 'custom:Public Domain', 'MIT'], 'libmng': ['ZLIB'], + 'libmodplug': ['custom:publicdomain'], 'libpciaccess': ['MIT', 'ISC', 'custom:X11'], - 'libpng': ['custom:libpng'], 'libpng12': ['custom:libpng'], + 'libpng': ['custom:libpng'], 'libsm': ['MIT', 'custom:X11'], 'libtiff': ['custom:libtiff'], + 'libtommath': ['custom:Public Domain'], 'libuv': ['MIT', 'BSD2', 'BSD3', 'ISC'], + 'libva-mesa-driver': ['MIT', 'custom:Khronos', 'custom:SGI'], + 'libva-utils': ['MIT'], 'libvdpau': ['MIT'], 'libwpe': ['BSD2'], 'libx11': ['MIT', 'custom:X11', 'custom:TekHVC'], 'libxau': ['custom:X11'], + 'libxaw': ['custom:X11', 'custom:HPND', 'MIT'], + 'libxcb': ['custom:X11'], 'libxcomposite': ['custom:libxcomposite', 'MIT'], + 'libxcursor': ['custom:HPND'], + 'libxdamage': ['custom:HPND'], + 'libxdmcp': ['custom:X11'], + 'libxext': ['custom:X11', 'custom:HPND', 'MIT'], 'libxfixes': ['custom:HPND', 'MIT'], + 'libxfont2': ['MIT', 'custom:HPND', 'BSD2', 'custom:X11'], 'libxft': ['custom:HPND'], - 'libxi': ['custom:X11', 'custom:HPND', 'MIT'], 'libxinerama': ['MIT', 'custom:X11'], + 'libxi': ['custom:X11', 'custom:HPND', 'MIT'], + 'libxkbcommon': ['MIT', 'custom:HPND', 'custom:X11'], + 'libxkbcommon-x11': ['MIT', 'custom:HPND', 'custom:X11'], + 'libxkbfile': ['custom:HPND', 'custom:X11'], + 'libxmu': ['custom:X11'], + 'libxpm': ['custom:X11'], + 'libxpresent': ['MIT', 'custom:HPND'], + 'libxrandr': ['custom:HPND'], + 'libxrender': ['custom:HPND'], + 'libxres': ['custom:X11'], + 'libxslt': ['custom:X11'], + 'libxss': ['custom:X11'], + 'libxt': ['MIT', 'custom:HPND', 'custom:X11'], + 'libxtst': ['MIT', 'custom:HPND', 'custom:X11'], 'libxv': ['custom:HPND'], + 'libxxf86vm': ['custom:X11'], + 'lolcat': ['BSD3'], + 'l-smash': ['ISC'], 'lsof': ['custom:lsof'], - 'mesa': ['MIT', 'custom:Khronos', 'custom:SGI'], 'mesa-demos': ['MIT', 'custom:SGI', 'LGPL'], + 'mesa': ['MIT', 'custom:Khronos', 'custom:SGI'], + 'mesa-vdpau': ['MIT', 'custom:Khronos', 'custom:SGI'], 'minizip': ['ZLIB'], + 'mobile-broadband-provider-info': ['custom:Public Domain'], 'ntp': ['custom:HPND', 'BSD2'], - 'nvidia': ['custom:non-free'], + 'nvidia-cg-toolkit': ['custom:non-free'], 'nvidia-dkms': ['custom:non-free'], + 'nvidia': ['custom:non-free'], 'nvidia-utils': ['custom:non-free'], 'openjdk8-src': ['GPL2'], 'orc': ['BSD2', 'BSD3'], 'pepper-flash': ['custom:non-free'], + 'perl-rename': ['GPL', 'PerlArtistic'], 'pigz': ['ZLIB'], 'pixman': ['MIT'], - 'python': ['Python'], + 'png++': ['BSD3'], + 'popt': ['custom:X11'], 'python2': ['Python'], 'python-docutils': ['custom:publicdomain', 'BSD2', 'Python', 'GPL3'], + 'python-magic': ['BSD2'], 'python-numpy': ['BSD3'], - 'popt': ['custom:X11'], + 'python': ['Python'], 'qhull': ['custom:Qhull'], 'rogue': ['BSD3'], 'rpcbind': ['BSD3'], + 'sdl_image': ['zlib'], + 'sound-theme-freedesktop': ['custom:CC-BY-SA', 'GPL2', 'custom:Creative Commons Attribution 3.0 Unported', 'GPL'], + 'spirv-tools': ['Apache'], 'steam': ['custom:non-free'], 'sudo': ['ISC', 'BSD3', 'BSD2', 'ZLIB'], 'tcl': ['custom:tcl'], 'ttf-bitstream-vera': ['custom:bitstream-vera'], 'ttf-dejavu': ['custom:publicdomain', 'custom:bitstream-vera'], 'unrar': ['custom:non-free'], + 'unzip': ['custom:Info-ZIP'], + 'vdpauinfo': ['MIT'], 'vulkan-icd-loader': ['Apache'], + 'vulkan-mesa-layers': ['MIT', 'custom:Khronos', 'custom:SGI'], + 'vulkan-radeon': ['MIT', 'custom:Khronos', 'custom:SGI'], + 'vulkan-tools': ['Apache'], 'w3m': ['custom:w3m'], + 'webkit2gtk': ['BSD', 'MIT', 'GPL2', 'GPL3', 'LGPL2', 'LGPL2.1', 'custom:Unicode-DFS'], 'webrtc-audio-processing': ['BSD3'], - 'xorg-server-xwayland': ['MIT'], + 'xcb-proto': ['custom:X11'], + 'xcb-util-image': ['custom:X11'], + 'xcb-util-keysyms': ['custom:HPND', 'custom:X11'], + 'xcb-util-renderutil': ['custom:HPND', 'custom:X11'], + 'xcb-util-wm': ['custom:HPND', 'custom:X11'], + 'xcb-util': ['custom:X11'], + 'xf86-input-libinput': ['custom:HPND'], + 'xf86-video-amdgpu': ['custom:X11', 'MIT', 'custom:HPND'], + 'xfce4-wavelan-plugin': ['BSD2'], + 'xfce4-xkb-plugin': ['BSD2'], + 'xkeyboard-config': ['MIT', 'custom:HPND', 'custom:X11', 'custom:MIT-style'], + 'xorg-bdftopcf': ['custom:X11'], + 'xorg-fonts-encodings': ['custom:Public Domain'], + 'xorg-font-util': ['custom:UCD', 'MIT', 'BSD2'], + 'xorg-iceauth': ['custom:X11'], + 'xorg-mkfontscale': ['MIT', 'custom:HPND', 'custom:X11'], + 'xorgproto': ['MIT', 'custom:HPND', 'custom:X11'], + 'xorg-server': ['BSD', 'MIT', 'custom:HPND', 'custom:X11'], + 'xorg-server-common': ['BSD', 'MIT', 'custom:HPND', 'custom:X11'], + 'xorg-setxkbmap': ['custom:HPND'], 'xorg-x11perf': ['custom:HPND'], + 'xorg-xauth': ['custom:X11'], + 'xorg-xdpyinfo': ['custom:X11'], + 'xorg-xev': ['custom:X11'], + 'xorg-xeyes': ['custom:X11'], + 'xorg-xfontsel': ['custom:HPND', 'custom:X11'], 'xorg-xhost': ['MIT', 'custom:X11'], + 'xorg-xinit': ['custom:X11'], 'xorg-xinput': ['MIT'], + 'xorg-xkbcomp': ['custom:HPND', 'custom:X11'], 'xorg-xkbevd': ['custom:HPND'], 'xorg-xkbutils': ['custom:HPND', 'custom:X11-DEC', 'MIT'], + 'xorg-xmessage': ['custom:X11'], + 'xorg-xmodmap': ['MIT', 'custom:X11'], + 'xorg-xprop': ['custom:HPND', 'custom:X11'], + 'xorg-xrandr': ['custom:HPND'], + 'xorg-xrdb': ['custom:X11'], + 'xorg-xset': ['custom:X11'], + 'xorg-xwayland': ['MIT'], + 'youtube-dl': ['custom:Unlicense'], + 'zathura-cb': ['zlib'], + 'zathura-djvu': ['zlib'], + 'zathura-pdf-mupdf': ['zlib'], + 'zathura-pdf-poppler': ['zlib'], + 'zathura-ps': ['zlib'], + 'zathura': ['zlib'], 'zip': ['custom:Info-ZIP'], 'zlib': ['ZLIB'], 'zoom': ['custom:non-free'], # AUR + 'zsh': ['custom:MIT-style', 'GPL'], } class Package(object):