commit 8506f5ee8b73dd19d4d691ab867a3fb45743ef1c Author: Lokendra Singh Date: Mon Jul 29 00:04:35 2019 +0530 Improved installation (#56) * Wrapper for apertium-tagger * Merge branch 'master' into swig_wrapper_tagger * Type Fix * added: os based installation support for ubuntu & windows * Fixed: logging * Fix: coverage * change: import location * added: py.typed PEP 561 * Removed: distro * added: installation of wrapper * Build only the wrapper * coverage ignores in .coveragerc * ubuntu installation switch to urllib3 from wget * use existing import statement * pass list instead of string to subprocess.run() * dont build lttoolbox * Rename _lttoolbox.so * Fixed import statement * rename wrappers * rename: install_language_pack -> install_module * build: apertium-core * Implemented changes suggested Move: _download_apertium_windows() -> install_apertium_base() TODO: install_wrapper() remove: Ubuntu.__init__() add: -y flag during installation on ubuntu remove args from _rename_wrappers() rename: get_installer_object() -> get_installer() * Implemented changes suggested Install single module at a time * mock streamparser import apertium in setup.py, causes import error, if apertium-streamparser isnt available during installation * travis: setup.py install before pipenv * ValueError if unsupported target * store value of platform.system() * setup.py: remove mocking replace: install_requires -> setup_requires allows using streamparser * perform test before pipenv install * Rearrange a bit * close file handles of tempfile * Simplify output/input file handling * Fix: tempfile.read() is already byte type * Snip trailing whitespace diff --git a/.appveyor.yml b/.appveyor.yml index adcaf55..dcf9957 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -7,13 +7,13 @@ environment: PYTHON_VERSION: 3.5 PYTHON_ARCH: 32 - PYTHON: C:\\Python35-x64 - PYTHON_VERSION: 3.4 + PYTHON_VERSION: 3.5 PYTHON_ARCH: 64 - PYTHON: C:\\Python36 PYTHON_VERSION: 3.6 PYTHON_ARCH: 32 - PYTHON: C:\\Python36-x64 - PYTHON_VERSION: 3.4 + PYTHON_VERSION: 3.6 PYTHON_ARCH: 64 - PYTHON: C:\\Python37 PYTHON_VERSION: 3.7 diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..7e0e432 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,2 @@ +[run] +omit=apertium/installer.py diff --git a/.flake8 b/.flake8 index 0b34362..93f9451 100644 --- a/.flake8 +++ b/.flake8 @@ -2,5 +2,5 @@ ignore = W504 max-line-length = 160 import-order-style = google -exclude = apertium/swig/, docs/source/conf.py +exclude = docs/source/conf.py application-import-names = apertium diff --git a/.travis.yml b/.travis.yml index 230c37c..dc0ae16 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,14 +12,12 @@ python: - 'nightly' install: - pip install pipenv + - python3 setup.py install - pipenv install --dev --system before_script: - - wget http://apertium.projectjj.com/apt/install-nightly.sh -O - | sudo bash - - sudo apt-get -f --allow-unauthenticated install apertium-all-dev - - sudo apt-get -f --allow-unauthenticated install apertium-nno-nob apertium-es-en apertium-eng - - python3 setup.py test - ./build-swig-wrapper.sh script: + - python3 setup.py test - flake8 --verbose apertium - if [[ $TRAVIS_PYTHON_VERSION != 3.4 ]]; then mypy apertium --strict --any-exprs-report .mypy_coverage --ignore-missing-imports; diff --git a/Pipfile.lock b/Pipfile.lock index 1655fee..094d9c3 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -33,10 +33,10 @@ }, "attrs": { "hashes": [ - "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", - "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" ], - "version": "==18.2.0" + "version": "==19.1.0" }, "babel": { "hashes": [ @@ -133,11 +133,11 @@ }, "flake8": { "hashes": [ - "sha256:859996073f341f2670741b51ec1e67a01da142831aa1fdc6242dbf88dffbe661", - "sha256:a796a115208f5c03b18f332f7c11729812c8c3ded6c46319c59b53efd3819da8" + "sha256:19241c1cbc971b9962473e4438a2ca19749a7dd002dd1a946eaba171b4114548", + "sha256:8e9dfa3cecb2400b3738a42c54c3043e821682b9c840b0448c0503f781130696" ], "index": "pypi", - "version": "==3.7.7" + "version": "==3.7.8" }, "flake8-bugbear": { "hashes": [ @@ -173,12 +173,12 @@ }, "flake8-eradicate": { "hashes": [ - "sha256:0953cd3bcae4bfd04d45075234e0b5fd465ff50ecc56cdcaf0027da751632127", - "sha256:c762fbb5c3e3694c9ba656d38477b2dcca6599b8baeee4984d05d655591a6f83" + "sha256:86804c682f9805a689379307939f350140fe9c015c3e600baff37fb23f7f21cc", + "sha256:cc2c3300a6643f8347988cc828478c347975f7bf9c8fc1f8a7027da41ab9bdbd" ], "index": "pypi", "markers": "python_version > '3.5'", - "version": "==0.2.0" + "version": "==0.2.1" }, "flake8-import-order": { "hashes": [ @@ -265,20 +265,20 @@ }, "mypy": { "hashes": [ - "sha256:12d18bd7fc642c5d54b1bb62dde813a7e2ab79b32ee11ff206ac387c68fc2ad4", - "sha256:23e24bc1683a36f39dee67d8ac74ea414654642eee26d420bada95b8ee8c9095", - "sha256:2b38e64c52a8968df4ebcae0ddba4a54eb94d184695dd4e54e14509a9389b78c", - "sha256:3d4f551466a76e278187ec3a5b26cfb50f72f6760b749aa00ac69a6f9c99898d", - "sha256:53d5dacb8d844e50be698830509aa592b093547e7ab90aee63eb23db61109007", - "sha256:56f981d246010ba21cac6b2455eaecfaf68fc8a5663d865b26c8e579c36f751d", - "sha256:8c57f6f59f1e8479d9fc6e1bf034353e54626ed64e32394c613afc493a441dc1", - "sha256:bbed4a593d87476b592d52867ef86da2155ccd0becf0c4c02e6567d842e43368", - "sha256:d6ff850e2ba18b2db7704897c8f2f1384478e3b75ad292ec06196bf7794f3a40", - "sha256:e13b1bb8785d7f785e0b88873f1c21cda58ceba9ce1153b58cbfa24b09a111d5", - "sha256:e2b9ee6f648ce72d6741925a47c88c2391168ef973b6f74f17969450c5b1ffdd" + "sha256:0107bff4f46a289f0e4081d59b77cef1c48ea43da5a0dbf0005d54748b26df2a", + "sha256:07957f5471b3bb768c61f08690c96d8a09be0912185a27a68700f3ede99184e4", + "sha256:10af62f87b6921eac50271e667cc234162a194e742d8e02fc4ddc121e129a5b0", + "sha256:11fd60d2f69f0cefbe53ce551acf5b1cec1a89e7ce2d47b4e95a84eefb2899ae", + "sha256:15e43d3b1546813669bd1a6ec7e6a11d2888db938e0607f7b5eef6b976671339", + "sha256:352c24ba054a89bb9a35dd064ee95ab9b12903b56c72a8d3863d882e2632dc76", + "sha256:437020a39417e85e22ea8edcb709612903a9924209e10b3ec6d8c9f05b79f498", + "sha256:49925f9da7cee47eebf3420d7c0e00ec662ec6abb2780eb0a16260a7ba25f9c4", + "sha256:6724fcd5777aa6cebfa7e644c526888c9d639bd22edd26b2a8038c674a7c34bd", + "sha256:7a17613f7ea374ab64f39f03257f22b5755335b73251d0d253687a69029701ba", + "sha256:cdc1151ced496ca1496272da7fc356580e95f2682be1d32377c22ddebdf73c91" ], "index": "pypi", - "version": "==0.711" + "version": "==0.720" }, "mypy-extensions": { "hashes": [ @@ -427,6 +427,14 @@ ], "version": "==1.4.0" }, + "typing-extensions": { + "hashes": [ + "sha256:2ed632b30bb54fc3941c382decfd0ee4148f5c591651c9272473fea2c6397d95", + "sha256:b1edbbf0652660e32ae780ac9433f4231e7339c7f9a8057d0f042fcbcea49b87", + "sha256:d8179012ec2c620d3791ca6fe2bf7979d979acdbef1fca0bc56b37411db682ed" + ], + "version": "==3.7.4" + }, "urllib3": { "hashes": [ "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", diff --git a/apertium/__init__.py b/apertium/__init__.py index c462f04..a2b1501 100644 --- a/apertium/__init__.py +++ b/apertium/__init__.py @@ -5,6 +5,7 @@ from typing import Dict, Tuple from apertium.analysis import analyze, Analyzer # noqa: F401 from apertium.generation import generate, Generator # noqa: F401 +from apertium.installer import install_module # noqa: F401 from apertium.mode_search import search_path from apertium.translation import translate, Translator # noqa: F401 diff --git a/apertium/installer.py b/apertium/installer.py new file mode 100644 index 0000000..128b3a2 --- /dev/null +++ b/apertium/installer.py @@ -0,0 +1,173 @@ +from distutils.dir_util import copy_tree +import logging +import os +import platform +import shutil +import subprocess +import tempfile +from typing import Optional +from urllib.request import urlretrieve +from zipfile import ZipFile + + +class Windows: + """Download ApertiumWin64 and Move to %localappdata%""" + base_link = 'http://apertium.projectjj.com/{}' + + def __init__(self) -> None: + self._install_path = os.getenv('LOCALAPPDATA') + self._apertium_path = os.path.join(self._install_path, 'apertium-all-dev') + self._download_path = tempfile.mkdtemp() + logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG) + self._logger = logging.getLogger() + self._logger.setLevel(logging.DEBUG) + + def _download_zip(self, download_files: dict, extract_path: Optional[str]) -> None: + for zip_name, zip_link in download_files.items(): + zip_download_path = os.path.join(self._download_path, zip_name) + urlretrieve(Windows.base_link.format(zip_link), filename=zip_download_path) + self._logger.info('%s download completed', zip_name) + + # Extract the zip + with ZipFile(zip_download_path) as zip_file: + zip_file.extractall(path=extract_path) + self._logger.info('%s Extraction completed', zip_name) + os.remove(zip_download_path) + self._logger.info('%s removed', zip_name) + + def _download_package(self, package: str) -> None: + """Installs Packages to %localappdata%/Apertium""" + + zip_path = 'win32/nightly/data.php?zip=' + package_zip = {package: zip_path + package} + self._download_zip(package_zip, self._download_path) + + # move the extracted files to desired location + lang_data_path = os.path.join(self._download_path, 'usr', 'share', 'apertium') + + self._logger.info('Copying Language Data to Apertium') + for directory in os.listdir(lang_data_path): + source = os.path.join(lang_data_path, directory) + destination = os.path.join(self._apertium_path, 'share', 'apertium', directory) + copy_tree(source, destination) + self._logger.info('%s -> %s', source, destination) + + shutil.rmtree(os.path.join(self._download_path, 'usr')) + + def _edit_modes(self) -> None: + r"""The mode files need to be modified before being used on Windows System + + 1. Replace /usr/share with %localappdata%\apertium-all-dev\share + 2. Replace "/" with "\" to make path compatible with Windows System + """ + + # List of Mode Files + mode_path = os.path.join(self._apertium_path, 'share', 'apertium', 'modes') + for f in os.listdir(mode_path): + if os.path.isfile(os.path.join(mode_path, f)) and f.endswith('.mode'): + self._logger.info('Editing mode %s ', f) + with open(os.path.join(mode_path, f)) as infile: + line = infile.read() + + contents = line.split(' ') + # Editing mode file to be compatible with windows platform + for i, t in enumerate(contents): + if len(t) > 2 and t[0] == "'" and t[1] == '/': + t = t.replace('/', os.sep) + t = t.replace(r'\usr', self._apertium_path) + contents[i] = t + line = ' '.join(contents) + with open(os.path.join(mode_path, f), 'w') as outfile: + outfile.write(line) + outfile.close() + + def install_apertium_base(self) -> None: + """Installs Apertium-all-dev to %localappdata%""" + + apertium_windows = { + 'apertium-all-dev.zip': '/win64/nightly/apertium-all-dev.zip', + } + + self._download_zip(apertium_windows, self._install_path) + + def install_apertium_module(self, language: str) -> None: + self._download_package(language) + self._edit_modes() + + def install_wrapper(self, swig_wrapper: str) -> None: + # TODO: create installer for wrappers on windows + pass + + +class Ubuntu: + @staticmethod + def _install_package_source() -> None: + install_script_url = 'http://apertium.projectjj.com/apt/install-nightly.sh' + with tempfile.NamedTemporaryFile('w') as install_script: + urlretrieve(install_script_url, install_script.name) + execute = subprocess.run(['sudo', 'bash', install_script.name]) + execute.check_returncode() + + @staticmethod + def _download_package(package: str) -> None: + command = ['sudo', 'apt-get', 'install', '-y', package] + execute = subprocess.run(command) + execute.check_returncode() + + @staticmethod + def _rename_wrappers() -> None: + wrapper_name = { + 'python3-apertium': '_apertium_core', + 'python3-apertium-lex-tools': '_lextools', + 'python3-lttoolbox': '_lttoolbox', + } + dist_package = '/usr/lib/python3/dist-packages' + for wrapper in wrapper_name.values(): + for f in os.listdir(dist_package): + if f.startswith(wrapper): + old_name = os.path.join(dist_package, f) + new_name = os.path.join(dist_package, '{}.so'.format(f.split('.')[0])) + if old_name != new_name: + subprocess.run(['sudo', 'mv', old_name, new_name]) + + def install_apertium_module(self, language: str) -> None: + self._download_package(language) + + def install_apertium_base(self) -> None: + self._install_package_source() + self._download_package('apertium-all-dev') + + def install_wrapper(self, swig_wrapper: str) -> None: + self._download_package(swig_wrapper) + self._rename_wrappers() + + +def get_installer(): + system = platform.system() + if system == 'Windows': + return Windows() + elif system == 'Linux': + with open('/etc/os-release') as os_release: + distro_name = os_release.readline().split('=')[-1].strip().replace('"', '') + if distro_name == 'Ubuntu': + return Ubuntu() + else: + raise ValueError('Installation on {} not supported'.format(distro_name)) + else: + raise ValueError('Installation on {} not supported'.format(system)) + + +def install_apertium() -> None: + installer = get_installer() + installer.install_apertium_base() + + +def install_module(module: str) -> None: + apertium_module = 'apertium-{}'.format(module) + installer = get_installer() + installer.install_apertium_module(apertium_module) + + +def install_wrapper(swig_wrapper: str) -> None: + installer = get_installer() + installer.install_wrapper(swig_wrapper) diff --git a/apertium/py.typed b/apertium/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/apertium/utils.py b/apertium/utils.py index 8898770..cd71e80 100644 --- a/apertium/utils.py +++ b/apertium/utils.py @@ -1,9 +1,14 @@ import os +import platform import subprocess +import sys import tempfile from typing import List try: + if platform.system() == 'Linux': + sys.path.append('/usr/lib/python3/dist-packages') + import apertium_core import lextools import lttoolbox @@ -46,14 +51,16 @@ def execute_pipeline(inp: str, commands: List[List[str]]) -> str: # delete=False and manually delete the file. used_wrapper = True if wrappers_available: - input_file = tempfile.NamedTemporaryFile(delete=False) + input_file = tempfile.NamedTemporaryFile(delete=False, mode='w') output_file = tempfile.NamedTemporaryFile(delete=False) + input_file_name, output_file_name = input_file.name, output_file.name + arg = command[1][1] if len(command) >= 3 else '' path = command[-1] - input_file_name, output_file_name = input_file.name, output_file.name - with open(input_file_name, 'w') as input_file: - text = end.decode() - input_file.write(text) + text = end.decode() + input_file.write(text) + input_file.close() + if 'lt-proc' == command[0]: lttoolbox.LtLocale.tryToSetLocale() fst = lttoolbox.FST() @@ -77,16 +84,21 @@ def execute_pipeline(inp: str, commands: List[List[str]]) -> str: elif 'apertium-pretransfer' == command[0]: obj = apertium_core.apertium() obj.pretransfer(arg, input_file.name, output_file.name) + elif 'apertium-tagger' == command[0]: + command += [input_file.name, output_file.name] + apertium_core.tagger(len(command), command) else: used_wrapper = False + if used_wrapper: - with open(output_file_name) as output_file: - end = output_file.read().encode() + output_file.seek(0) + end = output_file.read() + output_file.close() + os.remove(input_file_name) os.remove(output_file_name) if not wrappers_available or not used_wrapper: - if not used_wrapper: - apertium.logger.warning('Calling subprocess %s', command[0]) + apertium.logger.warning('Calling subprocess %s', command[0]) proc = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE) end, _ = proc.communicate(end) return end.decode() diff --git a/build-swig-wrapper.sh b/build-swig-wrapper.sh index 15cd82c..8ad4e55 100755 --- a/build-swig-wrapper.sh +++ b/build-swig-wrapper.sh @@ -3,23 +3,18 @@ set -xe sudo apt-get install -y swig build-essential python3-setuptools -git clone --depth 1 https://github.com/apertium/lttoolbox.git -pushd lttoolbox -./autogen.sh --enable-python-bindings && make -j2 -cd python -python3 setup.py install -popd - git clone --depth 1 https://github.com/apertium/apertium-lex-tools.git pushd apertium-lex-tools -./autogen.sh --enable-python-bindings && make -j2 +./autogen.sh --enable-python-bindings cd python +make -j2 python3 setup.py install popd git clone --depth 1 https://github.com/apertium/apertium.git apertium-core pushd apertium-core -./autogen.sh --enable-python-bindings && make -j2 +./autogen.sh --enable-python-bindings cd python +make -j2 python3 setup.py install popd diff --git a/installer.py b/installer.py deleted file mode 100644 index dbf23d6..0000000 --- a/installer.py +++ /dev/null @@ -1,108 +0,0 @@ -from distutils.dir_util import copy_tree -import logging -import os -import platform -import shutil -import tempfile -from urllib.request import urlretrieve -from zipfile import ZipFile - - -class Installer: - base_link = 'http://apertium.projectjj.com/{}' - - def __init__(self, languages): # type: (Installer, list) -> None - self._install_path = os.getenv('LOCALAPPDATA') - self._apertium_path = os.path.join(self._install_path, 'apertium-all-dev') - self._download_path = tempfile.mkdtemp() - self._languages = languages - logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG) - self._logger = logging.getLogger() - self._logger.setLevel(logging.DEBUG) - - def _download_zips(self, download_files, extract_path): # type: (Installer, dict, str) -> None - for zip_name, zip_link in download_files.items(): - zip_download_path = os.path.join(self._download_path, zip_name) - urlretrieve(Installer.base_link.format(zip_link), filename=zip_download_path) - self._logger.info('%s download completed', zip_name) - - # Extract the zip - with ZipFile(zip_download_path) as zip_file: - zip_file.extractall(path=extract_path) - self._logger.info('%s Extraction completed', zip_name) - os.remove(zip_download_path) - self._logger.info('%s removed', zip_name) - - def _download_apertium_windows(self): # type: (Installer) -> None - """Installs Apertium-all-dev to %localappdata%""" - - apertium_windows = { - 'apertium-all-dev.zip': '/win64/nightly/apertium-all-dev.zip', - } - - self._download_zips(apertium_windows, self._install_path) - - def _download_package(self): # type: (Installer) -> None - """Installs Language Data to Apertium""" - - if platform.system() == 'Windows': - zip_path = 'win32/nightly/data.php?zip=' - else: - raise ValueError('Installation for {} is not supported'.format(platform.system())) - language_zip = {} - for curr_lang in self._languages: - language_zip[curr_lang] = zip_path + curr_lang - - self._download_zips(language_zip, self._download_path) - - # move the extracted files to desired location - lang_data_path = os.path.join(self._download_path, 'usr', 'share', 'apertium') - - self._logger.info('Copying Language Data to Apertium') - for directory in os.listdir(lang_data_path): - source = os.path.join(lang_data_path, directory) - destination = os.path.join(self._apertium_path, 'share', 'apertium', directory) - copy_tree(source, destination) - self._logger.info('%s -> %s', source, destination) - - shutil.rmtree(os.path.join(self._download_path, 'usr')) - - def _edit_modes(self): # type: (Installer) -> None - r"""The mode files need to be modified before being used on Windows System - - 1. Replace /usr/share with %localappdata%\apertium-all-dev\share - 2. Replace "/" with "\" to make path compatible with Windows System - """ - - # List of Mode Files - mode_path = os.path.join(self._apertium_path, 'share', 'apertium', 'modes') - for f in os.listdir(mode_path): - if os.path.isfile(os.path.join(mode_path, f)) and f.endswith('.mode'): - self._logger.info('Editing mode %s ', f) - with open(os.path.join(mode_path, f)) as infile: - line = infile.read() - - contents = line.split(' ') - # Editing mode file to be compatible with windows platform - for i, t in enumerate(contents): - if len(t) > 2 and t[0] == "'" and t[1] == '/': - t = t.replace('/', os.sep) - t = t.replace(r'\usr', self._apertium_path) - contents[i] = t - line = ' '.join(contents) - with open(os.path.join(mode_path, f), 'w') as outfile: - outfile.write(line) - outfile.close() - - def install_windows(self): - self._download_apertium_windows() - self._download_package() - self._edit_modes() - - -def install_apertium_windows(): - """Download ApertiumWin64 and Move to %localappdata%""" - - if platform.system() == 'Windows': - p = Installer(['apertium-eng', 'apertium-en-es']) - p.install_windows() diff --git a/setup.py b/setup.py index 07ed5db..4c96e00 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,8 @@ from atexit import register from os import path - -import installer - from setuptools import find_packages, setup # noqa: I202 from setuptools.command.install import install +import sys class PostInstallCommand(install): @@ -14,8 +12,14 @@ class PostInstallCommand(install): @staticmethod def _post_install(): - installer.install_apertium_windows() + import apertium + apertium.installer.install_apertium() + apertium.installer.install_module('eng') + apertium.installer.install_module('en-es') + apertium.installer.install_wrapper('python3-apertium') + apertium.installer.install_wrapper('python3-apertium-lex-tools') + apertium.installer.install_wrapper('python3-lttoolbox') setup( name='apertium-python', @@ -25,10 +29,11 @@ setup( long_description_content_type='text/markdown; charset=UTF-8', url='https://github.com/apertium/apertium-python', python_requires='>=3.4', - install_requires=[ + setup_requires=[ 'apertium-streamparser==5.0.2', ], test_suite='tests', + package_data={'apertium': ['py.typed']}, packages=find_packages(exclude=['tests']), cmdclass={ 'install': PostInstallCommand,