diff --git a/.ci/azure-pipelines.yml b/.ci/azure-pipelines.yml
index 25448ed1..a76bfc1d 100644
--- a/.ci/azure-pipelines.yml
+++ b/.ci/azure-pipelines.yml
@@ -7,14 +7,29 @@ trigger:
     include:
       - '*'
 
-steps:
-  # On every PR, build the addon and make it available for download as an artifact
-  - template: build.yml
-    parameters:
-      py_versions: [ 'py2', 'py3' ]
+jobs:
+  - job: 'Validate'
+    pool:
+      vmImage: 'ubuntu-18.04'
+    strategy:
+      matrix:
+        Python27:
+          python.version: '2.7'
+        Python36:
+          python.version: '3.6'
+    steps:
+      # Run tests and linting
+      - template: validate.yml
 
-  # When triggered by a tag, publish the built addon to the repo server
-  - ${{ if startsWith(variables['Build.SourceBranch'], 'refs/tags') }}:
-    - template: publish.yml
-      parameters:
-        py_versions: [ 'py2', 'py3' ]
+  - job: Build
+    steps:
+      # On every PR, build the addon and make it available for download as an artifact
+      - template: build.yml
+        parameters:
+          py_versions: [ 'py2', 'py3' ]
+
+      # When triggered by a tag, publish the built addon to the repo server
+      - ${{ if startsWith(variables['Build.SourceBranch'], 'refs/tags') }}:
+        - template: publish.yml
+          parameters:
+            py_versions: [ 'py2', 'py3' ]
diff --git a/.ci/validate.yml b/.ci/validate.yml
new file mode 100644
index 00000000..0cc73368
--- /dev/null
+++ b/.ci/validate.yml
@@ -0,0 +1,26 @@
+steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '$(python.version)'
+
+  - script: python -m pip install -r requirements-dev.txt
+    displayName: 'Install dev tools'
+
+  - script: |
+      rm flake8.output
+      flake8 . --statistics --output-file=flake8.output && echo "::set-output name=lint::ok" || echo "::set-output name=lint::error"
+      cat flake8.output | awk -F: '{ if ($0 ~ /^\./) { gsub(/^[ \t]+/,"",$4); print "::error file=" $1 ",line=" $2 ",col=" $3 "::" $4 } else { print $0 } }'
+    displayName: 'Run Linter'
+    failOnStderr: false
+
+  - script: |
+      rm ../test_results.xml
+      pytest --junitxml=../test_results.xml
+    displayName: 'Run Tests'
+    failOnStderr: false
+
+  - task: PublishTestResults@2
+    condition: succeededOrFailed()
+    inputs:
+      testResultsFiles: '../test_results.xml'
+      testRunTitle: 'Publish test results for Python $(python.version)'