Merge pull request #334 from oakkitten/fix-tox-tests

Fix tests by “pinning” Anki dependencies via a time machine

.github/workflows/main.yml

@@ -1,32 +0,0 @@
-name: Tests
-
-on: [push, pull_request, workflow_dispatch]
-
-jobs:
-  run-tests:
-    name: Run tests
-    runs-on: ubuntu-latest
-    steps:
-      - name: Install dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y pyqt5-dev-tools xvfb
-
-      - name: Setup Python 3.8
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
-
-      - name: Setup Python 3.9
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
-      - name: Install tox
-        run: pip install tox
-
-      - name: Checkout repository
-        uses: actions/checkout@v2
-
-      - name: Run tests
-        run: tox -- --forked --verbose

.github/workflows/tests.yml

@@ -0,0 +1,71 @@
+name: Tests
+
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+  run-tests:
+    name: ${{ matrix.name }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - name: Anki 2.1.45
+            python: 3.8
+            environment: py38-anki2.1.45
+          - name: Anki 2.1.46
+            python: 3.8
+            environment: py38-anki2.1.46
+          - name: Anki 2.1.47
+            python: 3.8
+            environment: py38-anki2.1.47
+          - name: Anki 2.1.48
+            python: 3.8
+            environment: py38-anki2.1.48
+          - name: Anki 2.1.49
+            python: 3.8
+            environment: py38-anki2.1.49
+          - name: Anki 2.1.50 (Qt5)
+            python: 3.9
+            environment: py39-anki2.1.50-qt5
+          - name: Anki 2.1.50 (Qt6)
+            python: 3.9
+            environment: py39-anki2.1.50-qt6
+          - name: Anki 2.1.51 (Qt5)
+            python: 3.9
+            environment: py39-anki2.1.51-qt5
+          - name: Anki 2.1.51 (Qt6)
+            python: 3.9
+            environment: py39-anki2.1.51-qt6
+          - name: Anki 2.1.52 (Qt5)
+            python: 3.9
+            environment: py39-anki2.1.52-qt5
+          - name: Anki 2.1.52 (Qt6)
+            python: 3.9
+            environment: py39-anki2.1.52-qt6
+          - name: Anki 2.1.53 (Qt5)
+            python: 3.9
+            environment: py39-anki2.1.53-qt5
+          - name: Anki 2.1.53 (Qt6)
+            python: 3.9
+            environment: py39-anki2.1.53-qt6
+      fail-fast: false
+
+    steps:
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y pyqt5-dev-tools xvfb jq
+
+      - name: Setup Python ${{ matrix.python }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Install tox
+        run: pip install tox
+
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Run tests
+        run: tox -vvve ${{ matrix.environment }} -- --forked --verbose

tox-install-command

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -eux
+trap '[[ -v SERVER_PID ]] && pkill -P $SERVER_PID' EXIT
+print_first_group() { perl -snle 'm/$re/; print $1; exit 0' -- -re="$1"; }
+
+envname="$1"
+toxworkdir="$2"
+packages=("${@:3}")
+
+version=$(print_first_group 'anki([\d\.a-z]+)' <<< "$envname")
+upload_time=$(curl https://pypi.org/pypi/anki/json \
+            | jq --arg v "$version" -r '.releases[$v][0].upload_time_iso_8601')
+cutoff_time=$(date --utc -d "$upload_time +1 hour" '+%Y-%m-%dT%H:%M:%S')
+
+coproc SERVER { "$toxworkdir"/.tox/bin/python -um pypi_timemachine "$cutoff_time"; }
+index_url=$(print_first_group '(http\S+)' <&"${SERVER[0]}")
+
+python -m pip install --index-url "$index_url" "anki==$version" "$AQT==$version"
+python -m pip install "${packages[@]}"

tox.ini

@@ -42,42 +42,44 @@
 #       QT_DEBUG_PLUGINS=1
 #       ANKIDEV=1
 
+# Note: pypi packages anki and aqt do not pin their dependencies.
+# To tests against historically accurate dependencies, we use a “time machine”
+# that prevents pip from using packages that were uploaded after a specified date.
+
 [tox]
 minversion = 3.24
 skipsdist = true
 skip_install = true
-envlist = py38-anki{45,46,47,48,49},py39-anki{50qt5,50qt6}
+requires =
+    pypi-timemachine
+envlist =
+    py38-anki2.1.{45,46,47,48,49}
+    py39-anki2.1.{50,51,52,53}-qt{5,6}
+
+[testenv:.tox]
+install_command =
+    python -m pip install {packages}
 
 [testenv]
+install_command =
+    bash tox-install-command {envname} {toxworkdir} {packages}
+
 commands =
     env HOME={envtmpdir}/home xvfb-run python -m pytest {posargs}
+
 setenv =
-    anki50qt6: DISABLE_QT5_COMPAT=1
+    DISABLE_QT5_COMPAT=1
+
+    !qt{5,6}: AQT=aqt
+    qt5: AQT=aqt[qt5]
+    qt6: AQT=aqt[qt6]
+
 allowlist_externals =
+    bash
     env
     xvfb-run
+
 deps =
     pytest==7.1.1
     pytest-forked==1.4.0
     pytest-anki @ git+https://github.com/oakkitten/pytest-anki.git@a0d27aa5
-
-    anki45: anki==2.1.45
-    anki45: aqt==2.1.45
-
-    anki46: anki==2.1.46
-    anki46: aqt==2.1.46
-
-    anki47: anki==2.1.47
-    anki47: aqt==2.1.47
-
-    anki48: anki==2.1.48
-    anki48: aqt==2.1.48
-
-    anki49: anki==2.1.49
-    anki49: aqt==2.1.49
-
-    anki50qt5: anki==2.1.50
-    anki50qt5: aqt[qt5]==2.1.50
-
-    anki50qt6: anki==2.1.50
-    anki50qt6: aqt[qt6]==2.1.50