diff --git a/Dockerfile b/Dockerfile index 05e8164e..8405a72a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,25 +1,30 @@ FROM debian:stretch ENV LANG C.UTF-8 ADD backports.list /etc/apt/sources.list.d/backports.list RUN apt-get update && apt-get install -y nodejs -t stretch-backports && \ apt-get install -y composer git \ ruby ruby2.3 ruby2.3-dev rubygems-integration \ python-minimal build-essential \ php-ast php-xml php-zip php-gd php-gmp php-mbstring php-curl \ - python3 python3-pip python3-setuptools python3-wheel python3-requests \ + python3 python3-dev python3-pip python3-virtualenv \ --no-install-recommends && rm -rf /var/lib/apt/lists/* RUN gem install --no-rdoc --no-ri jsduck RUN git clone --depth 1 https://gerrit.wikimedia.org/r/p/integration/npm.git /srv/npm \ && rm -rf /srv/npm/.git \ && ln -s /srv/npm/bin/npm-cli.js /usr/bin/npm +# TODO move grr into venv RUN pip3 install grr +RUN python3 -m virtualenv -p python3 /venv +RUN mkdir -p /venv/src/ +COPY setup.py /venv/src/ +COPY ./libup /venv/src/libup +RUN cd /venv/src && /venv/bin/python3 setup.py install RUN git config --global user.name "libraryupgrader" RUN git config --global user.email "tools.libraryupgrader@tools.wmflabs.org" ENV COMPOSER_PROCESS_TIMEOUT 1800 # Shared cache ENV NPM_CONFIG_CACHE=/cache ENV XDG_CACHE_HOME=/cache -COPY ./container /usr/src/myapp -COPY container/ng.py /usr/bin/libup-ng +COPY ./libup /venv/src WORKDIR /usr/src/myapp -CMD [ "python3", "thing.py" ] +ENTRYPOINT [ "/venv/bin/libup-ng" ] diff --git a/container/ng.py b/libup/ng.py similarity index 99% rename from container/ng.py rename to libup/ng.py index 0fd87c41..14e492a6 100755 --- a/container/ng.py +++ b/libup/ng.py @@ -1,160 +1,161 @@ #!/usr/bin/env python3 """ next generation of libraryupgrader Copyright (C) 2019 Kunal Mehta This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . """ import argparse import json import os import subprocess import urllib.parse class LibraryUpgrader: @property def has_npm(self): return os.path.exists('package.json') @property def has_composer(self): return os.path.exists('composer.json') def check_call(self, args: list) -> str: res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) # TODO: log + print(res.stdout.decode()) res.check_returncode() return res.stdout.decode() def gerrit_url(self, repo: str, user=None, pw=None) -> str: host = '' if user: if pw: host = user + ':' + urllib.parse.quote_plus(pw) + '@' else: host = user + '@' host += 'gerrit.wikimedia.org' return 'https://%s/r/%s.git' % (host, repo) def ensure_package_lock(self): if not os.path.exists('package-lock.json'): self.check_call(['npm', 'i', '--package-lock-only']) def npm_deps(self): if not self.has_npm: return None with open('package.json') as f: pkg = json.load(f) return { 'deps': pkg.get('dependencies', {}), 'dev': pkg.get('devDependencies', {}), } def composer_deps(self): if not self.has_composer: return None with open('composer.json') as f: pkg = json.load(f) ret = { 'deps': pkg.get('require', {}), 'dev': pkg.get('require-dev', {}), } if 'phan-taint-check-plugin' in pkg.get('extra', {}): ret['dev']['mediawiki/phan-taint-check-plugin'] \ = pkg['extra']['phan-taint-check-plugin'] return ret def npm_audit(self): if not self.has_npm: return {} self.ensure_package_lock() try: subprocess.check_output(['npm', 'audit', '--json']) # If npm audit didn't fail, there are no vulnerable packages return {} except subprocess.CalledProcessError as e: try: return json.loads(e.output.decode()) except json.decoder.JSONDecodeError: print('Error, invalid JSON from npm audit, skipping') return {'error': e.output.decode()} def npm_test(self): if not self.has_npm: return self.ensure_package_lock() self.check_call(['npm', 'ci']) self.check_call(['npm', 'test']) def composer_test(self): if not self.has_composer: return self.check_call(['composer', 'install']) self.check_call(['composer', 'test']) def git_clean(self): self.check_call(['git', 'clean', '-fdx']) def clone_commands(self, repo): url = self.gerrit_url(repo) self.check_call(['git', 'clone', url, 'repo', '--depth=1']) os.chdir('repo') self.check_call(['grr', 'init']) # Install commit-msg hook def sha1(self): return self.check_call(['git', 'show-ref', 'HEAD']).split(' ')[0] def run(self, repo, output): self.clone_commands(repo) data = { 'repo': repo, 'sha1': self.sha1() } data['npm-audit'] = self.npm_audit() data['npm-deps'] = self.npm_deps() try: self.npm_test() data['npm-test'] = {'result': True} except subprocess.CalledProcessError as e: data['npm-test'] = {'result': False, 'error': e.output.decode()} self.git_clean() data['composer-deps'] = self.composer_deps() try: self.composer_test() data['composer-test'] = {'result': True} except subprocess.CalledProcessError as e: data['composer-test'] = {'result': False, 'error': e.output.decode()} with open(output, 'w') as f: json.dump(data, f) def main(): parser = argparse.ArgumentParser(description='next generation of libraryupgrader') parser.add_argument('repo', help='Git repository') parser.add_argument('output', help='Path to output results to') args = parser.parse_args() libup = LibraryUpgrader() libup.run(args.repo, args.output) if __name__ == '__main__': main() diff --git a/container/ng_test.py b/libup/ng_test.py similarity index 98% rename from container/ng_test.py rename to libup/ng_test.py index 56b46057..5a14f01b 100644 --- a/container/ng_test.py +++ b/libup/ng_test.py @@ -1,76 +1,76 @@ """ Copyright (C) 2019 Kunal Mehta This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . """ import pytest import subprocess -from ng import LibraryUpgrader +from .ng import LibraryUpgrader class MockLibraryUpgrader(LibraryUpgrader): def __init__(self): self.called = [] def check_call(self, args: list) -> str: self.called.append(args) return ' '.join(args) def test_gerrit_url(): libup = LibraryUpgrader() assert libup.gerrit_url('repo/name') \ == 'https://gerrit.wikimedia.org/r/repo/name.git' assert libup.gerrit_url('repo/name', user='foo') \ == 'https://foo@gerrit.wikimedia.org/r/repo/name.git' assert libup.gerrit_url('repo/name', user='foo', pw='bar!!+/') \ == 'https://foo:bar%21%21%2B%2F@gerrit.wikimedia.org/r/repo/name.git' def test_has_npm(fs): libup = LibraryUpgrader() assert libup.has_npm is False fs.create_file('package.json', contents='{}') assert libup.has_npm is True def test_has_composer(fs): libup = LibraryUpgrader() assert libup.has_composer is False fs.create_file('composer.json', contents='{}') assert libup.has_composer is True def test_check_call(): libup = LibraryUpgrader() res = libup.check_call(['echo', 'hi']) assert res == 'hi\n' def test_check_call_fail(): libup = LibraryUpgrader() with pytest.raises(subprocess.CalledProcessError): libup.check_call(['false']) def test_ensure_package_lock(fs): libup = MockLibraryUpgrader() libup.ensure_package_lock() assert libup.called == [['npm', 'i', '--package-lock-only']] libup = MockLibraryUpgrader() fs.create_file('package-lock.json', contents='{}') libup.ensure_package_lock() assert libup.called == [] diff --git a/libup/tasks.py b/libup/tasks.py index 7314537d..4bc580c8 100644 --- a/libup/tasks.py +++ b/libup/tasks.py @@ -1,51 +1,50 @@ """ Copyright (C) 2019 Kunal Mehta This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . """ from celery import Celery import os import random import string from . import docker app = Celery('tasks', broker='amqp://localhost') def _random_string(): return ''.join(random.choice(string.ascii_letters) for _ in range(15)) @app.task def run_check(repo: str, data_root: str, log_dir: str): rand = _random_string() docker.run( name=rand, env={}, background=False, mounts={log_dir: '/out'}, rm=True, extra_args=[repo, '/out/%s.json' % rand], - entrypoint='/usr/bin/libup-ng' ) output = os.path.join(log_dir, '%s.json' % rand) assert os.path.exists(output) # Copy it over to the current directory, # potentially overwriting existing results with open(output) as fr: fname = os.path.join(data_root, 'current', repo.replace('/', '_') + '.json') with open(fname, 'w') as fw: fw.write(fr.read()) diff --git a/setup.py b/setup.py index 3cc054bc..6d6ff3ec 100644 --- a/setup.py +++ b/setup.py @@ -1,30 +1,31 @@ from setuptools import setup setup( name='libup', version='0.0.1', packages=['libup'], url='https://www.mediawiki.org/wiki/Libraryupgrader', license='AGPL-3.0-or-later', author='Kunal Mehta', author_email='legoktm@member.fsf.org', description='semi-automated tool that manages upgrades of libraries', include_package_data=True, install_requires=[ 'requests', 'wikimediaci-utils', - 'flask', + # 'Flask', 'flask-bootstrap', 'gunicorn', 'markdown', 'semver', 'celery', ], entry_points={ 'console_scripts': [ 'libup-run = libup.run:main', 'libup-upgrade = libup.upgrade:main', + 'libup-ng = libup.ng:main', ] } )