tox.ini

# Tox (https://tox.readthedocs.io/) is a tool for running tests
# in multiple virtualenvs. This configuration file will run the
# test suite on all supported python versions. To use it, "pip install tox"
# and then run "tox" from this directory.

[tox]
envlist = pre-commit,pylint,typing,py
minversion = 1.7.0

[test-requirements]
deps =
    attrs
    pexpect >= 4.8.0
    pyftpdlib >= 1.5.8
    pygments
    pytest
    pytest-cov==4.0.0
    pytest-xdist
    sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
    testfixtures
    pywin32; sys_platform == "win32"

[testenv]
deps =
    {[test-requirements]deps}

    # mitmproxy does not support PyPy
    mitmproxy; implementation_name != 'pypy'
    # https://github.com/pallets/werkzeug/pull/2768 breaks flask, required by
    # mitmproxy.
    werkzeug < 3; python_version < '3.9' and implementation_name != 'pypy'
passenv =
    S3_TEST_FILE_URI
    AWS_ACCESS_KEY_ID
    AWS_SECRET_ACCESS_KEY
    AWS_SESSION_TOKEN
    GCS_TEST_FILE_URI
    GCS_PROJECT_ID
#allow tox virtualenv to upgrade pip/wheel/setuptools
download = true
commands =
    pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 docs scrapy tests} --doctest-modules
install_command =
    python -I -m pip install -ctests/upper-constraints.txt {opts} {packages}

[testenv:typing]
basepython = python3
deps =
    mypy==1.10.0
    typing-extensions==4.12.1
    types-lxml==2024.4.14
    types-Pygments==2.18.0.20240506
    types-pyOpenSSL==24.1.0.20240425
    types-setuptools==70.0.0.20240524
    botocore-stubs==1.34.94
    boto3-stubs[s3]==1.34.119
    attrs >= 18.2.0
    Pillow >= 10.3.0
    pytest >= 8.2.0
    w3lib >= 2.2.0
commands =
    mypy {posargs: scrapy tests}

[testenv:typing-tests]
basepython = python3.8
deps =
    {[test-requirements]deps}
    {[testenv:typing]deps}
    pytest-mypy-testing==0.1.3
commands =
    pytest {posargs: tests_typing}

[testenv:pre-commit]
basepython = python3
deps =
    pre-commit
commands =
    pre-commit run {posargs:--all-files}

[testenv:pylint]
basepython = python3
deps =
    {[testenv:extra-deps]deps}
    pylint==3.1.0
commands =
    pylint conftest.py docs extras scrapy setup.py tests

[testenv:twinecheck]
basepython = python3
deps =
    twine==4.0.2
    build==1.0.3
commands =
    python -m build --sdist
    twine check dist/*

[pinned]
basepython = python3.8
deps =
    cryptography==36.0.0
    cssselect==0.9.1
    h2==3.0
    itemadapter==0.1.0
    parsel==1.5.0
    Protego==0.1.15
    pyOpenSSL==21.0.0
    queuelib==1.4.2
    service_identity==18.1.0
    Twisted[http2]==18.9.0
    w3lib==1.17.0
    zope.interface==5.1.0
    lxml==4.4.1
    {[test-requirements]deps}

    # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies
    # above, hence we do not install it in pinned environments at the moment
setenv =
    _SCRAPY_PINNED=true
install_command =
    python -I -m pip install {opts} {packages}
commands =
    pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests}

[testenv:pinned]
basepython = {[pinned]basepython}
deps =
    {[pinned]deps}
    PyDispatcher==2.0.5
install_command = {[pinned]install_command}
setenv =
    {[pinned]setenv}
commands = {[pinned]commands}

[testenv:windows-pinned]
basepython = {[pinned]basepython}
deps =
    {[pinned]deps}
    PyDispatcher==2.0.5
install_command = {[pinned]install_command}
setenv =
    {[pinned]setenv}
commands = {[pinned]commands}

[testenv:extra-deps]
basepython = python3
deps =
    {[testenv]deps}
    boto3
    google-cloud-storage
    # Twisted[http2] currently forces old mitmproxy because of h2 version
    # restrictions in their deps, so we need to pin old markupsafe here too.
    markupsafe < 2.1.0
    robotexclusionrulesparser
    Pillow
    Twisted[http2]
    uvloop; platform_system != "Windows"
    bpython  # optional for shell wrapper tests
    brotli; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
    brotlicffi; implementation_name == 'pypy'  # optional for HTTP compress downloader middleware tests
    zstandard; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
    ipython

[testenv:extra-deps-pinned]
basepython = {[pinned]basepython}
deps =
    {[pinned]deps}
    boto3==1.20.0
    google-cloud-storage==1.29.0
    Pillow==7.1.0
    robotexclusionrulesparser==1.6.2
    brotlipy
    uvloop==0.14.0; platform_system != "Windows"
    bpython==0.7.1
    zstandard==0.1; implementation_name != 'pypy'
    ipython==2.0.0
    brotli==0.5.2; implementation_name != 'pypy'
    brotlicffi==0.8.0; implementation_name == 'pypy'
install_command = {[pinned]install_command}
setenv =
    {[pinned]setenv}
commands = {[pinned]commands}

[testenv:asyncio]
commands =
    {[testenv]commands} --reactor=asyncio

[testenv:asyncio-pinned]
basepython = {[pinned]basepython}
deps = {[testenv:pinned]deps}
commands = {[pinned]commands} --reactor=asyncio
install_command = {[pinned]install_command}
setenv =
    {[pinned]setenv}

[testenv:pypy3]
basepython = pypy3
commands =
    pytest {posargs:--durations=10 docs scrapy tests}

[testenv:pypy3-pinned]
basepython = pypy3.8
deps =
    {[pinned]deps}
    PyPyDispatcher==2.1.0
commands =
    pytest {posargs:--durations=10 scrapy tests}
install_command = {[pinned]install_command}
setenv =
    {[pinned]setenv}

[docs]
changedir = docs
deps =
    -rdocs/requirements.txt
setenv =
    READTHEDOCS_PROJECT=scrapy
    READTHEDOCS_VERSION=master

[testenv:docs]
basepython = python3
changedir = {[docs]changedir}
deps = {[docs]deps}
setenv = {[docs]setenv}
commands =
    sphinx-build -W -b html . {envtmpdir}/html

[testenv:docs-coverage]
basepython = python3
changedir = {[docs]changedir}
deps = {[docs]deps}
setenv = {[docs]setenv}
commands =
    sphinx-build -b coverage . {envtmpdir}/coverage

[testenv:docs-links]
basepython = python3
changedir = {[docs]changedir}
deps = {[docs]deps}
setenv = {[docs]setenv}
commands =
    sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck


# Run S3 tests with botocore installed but without boto3.

[testenv:botocore]
deps =
    {[testenv]deps}
    botocore>=1.4.87
commands =
    pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}

[testenv:botocore-pinned]
basepython = {[pinned]basepython}
deps =
    {[pinned]deps}
    botocore==1.4.87
install_command = {[pinned]install_command}
setenv =
    {[pinned]setenv}
commands =
    pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}