Release v1.0.2

(cherry picked from commit 1a7612e559)

.appveyor.yml

@@ -5,13 +5,15 @@ cache:
 build: off
 environment:
     PYTHONUNBUFFERED: 1
+    PYTHON: C:\Python36\python.exe
     matrix:
-    - TESTENV: py34
-    - TESTENV: unittests-frozen
+    - TESTENV: py36-pyqt59
     - TESTENV: pylint
 
 install:
-  - C:\Python27\python -u scripts\dev\ci\appveyor_install.py
+  - '%PYTHON% -m pip install -U pip'
+  - '%PYTHON% -m pip install -r misc\requirements\requirements-tox.txt'
+  - 'set PATH=%PATH%;C:\Python36'
 
 test_script:
-  - C:\Python34\Scripts\tox -e %TESTENV%
+  - '%PYTHON% -m tox -e %TESTENV%'

.codecov.yml

@@ -0,0 +1,7 @@
+coverage:
+  status:
+    project: off
+    patch: off
+    changes: off
+
+comment: off

.flake8

@@ -11,6 +11,7 @@ exclude = .*,__pycache__,resources.py
 #       (for pytest's __tracebackhide__)
 # F401: Unused import
 # N802: function name should be lowercase
+# N806: variable in function should be lowercase
 # P101: format string does contain unindexed parameters
 # P102: docstring does contain unindexed parameters
 # P103: other string does contain unindexed parameters
@@ -35,13 +36,15 @@ max-complexity = 12
 putty-auto-ignore = True
 putty-ignore =
     /# pylint: disable=invalid-name/ : +N801,N806
-    /# pylint: disable=wildcard-import/ : +F403
     /# pragma: no mccabe/ : +C901
     tests/*/test_*.py : +D100,D101,D401
-    tests/unit/browser/webkit/test_history.py : +N806
+    tests/conftest.py : +F403
+    tests/unit/browser/test_history.py : +N806
     tests/helpers/fixtures.py : +N806
     tests/unit/browser/webkit/http/test_content_disposition.py : +D400
     scripts/dev/ci/appveyor_install.py : +FI53
+    # FIXME:conf
+    tests/unit/completion/test_models.py : +F821
 copyright-check = True
 copyright-regexp = # Copyright [\d-]+ .*
 copyright-min-file-size = 110

.github/CODEOWNERS

@@ -0,0 +1,8 @@
+qutebrowser/browser/history.py  @rcorre
+qutebrowser/completion/*  @rcorre
+qutebrowser/misc/sql.py  @rcorre
+tests/end2end/features/completion.feature  @rcorre
+tests/end2end/features/test_completion_bdd.py  @rcorre
+tests/unit/browser/test_history.py  @rcorre
+tests/unit/completion/*  @rcorre
+tests/unit/misc/test_sql.py  @rcorre

.github/CONTRIBUTING.asciidoc

@@ -0,0 +1,9 @@
+- Before you start to work on something, please leave a comment on the relevant
+  issue (or open one). This makes sure there is no duplicate work done.
+
+- Either run the testsuite locally, or keep an eye on Travis CI / AppVeyor
+  after pushing changes.
+
+See the full contribution docs for details:
+
+include::../doc/contributing.asciidoc[]

.github/ISSUE_TEMPLATE.md

@@ -1,2 +1,2 @@
 <!-- If this is a bug report, please remember to mention your version info from
-the `qute:version` page or `qutebrowser --version` -->
+`:open qute:version` or `qutebrowser --version` -->

.gitignore

@@ -15,11 +15,8 @@ __pycache__
 /qutebrowser/3rdparty
 /doc/*.html
 /README.html
-/CHANGELOG.html
-/CONTRIBUTING.html
-/FAQ.html
-/INSTALL.html
 /qutebrowser/html/doc/
+/qutebrowser/html/*.html
 /.venv*
 /.coverage
 /htmlcov

.pylintrc

@@ -31,20 +31,23 @@ disable=no-self-use,
         bare-except,
         eval-used,
         exec-used,
-        file-ignored,
-        wrong-import-order,
         ungrouped-imports,
-        redefined-variable-type,
         suppressed-message,
         too-many-return-statements,
         duplicate-code,
-        wrong-import-position
+        wrong-import-position,
+        no-else-return,
+        # https://github.com/PyCQA/pylint/issues/1698
+        unsupported-membership-test,
+        unsupported-assignment-operation,
+        unsubscriptable-object
 
 [BASIC]
 function-rgx=[a-z_][a-z0-9_]{2,50}$
 const-rgx=[A-Za-z_][A-Za-z0-9_]{0,30}$
 method-rgx=[a-z_][A-Za-z0-9_]{1,50}$
 attr-rgx=[a-z_][a-z0-9_]{0,30}$
+class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{1,30}|(__.*__))$
 argument-rgx=[a-z_][a-z0-9_]{0,30}$
 variable-rgx=[a-z_][a-z0-9_]{0,30}$
 docstring-min-length=3
@@ -52,12 +55,9 @@ no-docstring-rgx=(^_|^main$)
 
 [FORMAT]
 max-line-length=79
-ignore-long-lines=(<?https?://|^# Copyright 201\d|# (pylint|flake8): disable=)
+ignore-long-lines=(<?https?://|^# Copyright 201\d)
 expected-line-ending-format=LF
 
-[SIMILARITIES]
-min-similarity-lines=8
-
 [VARIABLES]
 dummy-variables-rgx=_.*
 
@@ -68,12 +68,11 @@ max-args=10
 valid-metaclass-classmethod-first-arg=cls
 
 [TYPECHECK]
-# WORKAROUND for https://github.com/PyCQA/astroid/pull/357
-ignored-modules=pytest
-# MsgType added as WORKAROUND for
-# https://bitbucket.org/logilab/pylint/issues/690/
-# UnsetObject because pylint infers any objreg.get(...) as UnsetObject.
-ignored-classes=qutebrowser.utils.objreg.UnsetObject,
-                qutebrowser.browser.webkit.webelem.WebElementWrapper,
-                scripts.dev.check_coverage.MsgType,
-                qutebrowser.browser.downloads.UnsupportedAttribute
+ignored-modules=PyQt5,PyQt5.QtWebKit
+ignored-classes=_CountingAttr
+
+[IMPORTS]
+# WORKAROUND
+# For some reason, pylint doesn't know about some Python 3 modules on
+# AppVeyor...
+known-standard-library=faulthandler,http,enum,tokenize,posixpath,importlib,types

.pyup.yml

@@ -0,0 +1 @@
+schedule: "every week on monday"

.travis.yml

@@ -1,14 +1,11 @@
-sudo: required
+sudo: false
 dist: trusty
-language: generic
+language: python
+group: edge
+python: 3.6
 
 matrix:
   include:
-    - os: linux
-      env: TESTENV=py34-cov
-    - os: linux
-      env: DOCKER=debian-jessie
-      services: docker
     - os: linux
       env: DOCKER=archlinux
       services: docker
@@ -16,36 +13,32 @@ matrix:
       env: DOCKER=archlinux-webengine QUTE_BDD_WEBENGINE=true
       services: docker
     - os: linux
-      env: DOCKER=archlinux-ng
-      services: docker
-    - os: linux
-      env: DOCKER=ubuntu-xenial
-      services: docker
-    - os: linux
-      language: python
-      python: 3.6
       env: TESTENV=py36-pyqt571
     - os: linux
-      language: python
-      python: 3.5
-      env: TESTENV=py35-pyqt58
-    - os: linux
-      language: python
-      python: 3.6
       env: TESTENV=py36-pyqt58
+    - os: linux
+      python: 3.5
+      env: TESTENV=py35-pyqt59
+    - os: linux
+      env: TESTENV=py36-pyqt59-cov
     - os: osx
-      env: TESTENV=py36 OSX=elcapitan
-      osx_image: xcode7.3
+      env: TESTENV=py36 OSX=sierra
+      osx_image: xcode8.3
+      language: generic
     # https://github.com/qutebrowser/qutebrowser/issues/2013
     # - os: osx
     #   env: TESTENV=py35 OSX=yosemite
     #   osx_image: xcode6.4
     - os: linux
-      env: TESTENV=pylint
+      env: TESTENV=pylint PYTHON=python3.6
     - os: linux
       env: TESTENV=flake8
     - os: linux
       env: TESTENV=docs
+      addons:
+         apt:
+           packages:
+           - asciidoc
     - os: linux
       env: TESTENV=vulture
     - os: linux
@@ -56,10 +49,9 @@ matrix:
       env: TESTENV=check-manifest
     - os: linux
       env: TESTENV=eslint
-  allow_failures:
-    - os: osx
-      env: TESTENV=py36 OSX=elcapitan
-      osx_image: xcode7.3
+      language: node_js
+      python: null
+      node_js: node
   fast_finish: true
 
 cache:
@@ -67,12 +59,9 @@ cache:
     - $HOME/.cache/pip
     - $HOME/build/qutebrowser/qutebrowser/.cache
 
-before_install:
-  # We need to do this so we pick up the system-wide python properly
-  - 'export PATH="/usr/bin:$PATH"'
-
 install:
   - bash scripts/dev/ci/travis_install.sh
+  - ulimit -c unlimited
 
 script:
   - bash scripts/dev/ci/travis_run.sh
@@ -80,6 +69,9 @@ script:
 after_success:
   - '[[ $TESTENV == *-cov ]] && codecov -e TESTENV -X gcov'
 
+after_failure:
+  - bash scripts/dev/ci/travis_backtrace.sh
+
 notifications:
   webhooks:
     - https://buildtimetrend.herokuapp.com/travis

INSTALL.asciidoc

@@ -1,356 +0,0 @@
-Installing qutebrowser
-======================
-
-On Debian / Ubuntu
-------------------
-
-qutebrowser should run on these systems:
-
-* Debian jessie or newer
-* Ubuntu Trusty (14.04 LTS) or newer
-* Any other distribution based on these (e.g. Linux Mint 17+)
-
-Unfortunately there is no Debian package in the official repos yet, but installing qutebrowser is
-still relatively easy!
-
-You can use packages that are built for every release or build it yourself from git.
-
-Using the packages
-~~~~~~~~~~~~~~~~~~
-
-Install the dependencies via apt-get:
-
-----
-# apt-get install python3-lxml python-tox python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python3-sip python3-jinja2 python3-pygments python3-yaml
-----
-
-Get the qutebrowser package from the
-https://github.com/qutebrowser/qutebrowser/releases[release page] and download
-the https://qutebrowser.org/python3-pypeg2_2.15.2-1_all.deb[PyPEG2 package].
-
-Install the packages:
-
-----
-# dpkg -i python3-pypeg2_*_all.deb
-# dpkg -i qutebrowser_*_all.deb
-----
-
-Build it from git
-~~~~~~~~~~~~~~~~~
-
-Install the dependencies via apt-get:
-
-[NOTE]
-==========================
-On Debian, it's recommended to install the Qt packages from the
-https://wiki.debian.org/DebianExperimental[experimental] repository as those
-are a much newer version of Qt which is more stable.
-
-Add the following line to your `/etc/apt/sources.list`:
-
-----
-deb http://ftp.debian.org/debian experimental main
-----
-
-Then install the packages like this:
-
-----
-# apt-get update
-# apt-get install -t experimental python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python3-sip python3-dev
-# apt-get install python-tox
-----
-
-It's also recommended to pin those packages to receive updates by creating a
-file `/etc/apt/preferences.d/qutebrowser` with the following contents:
-
-----
-Package: python3-pyqt5* libqt5*
-Pin: release a=experimental
-Pin-Priority: 800
-----
-==========================
-
-For distributions other than Debian or if you prefer to not use the
-experimental repo:
-
-----
-# apt-get install python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python-tox python3-sip python3-dev
-----
-
-To generate the documentation for the `:help` command, when using the git
-repository (rather than a release):
-
-----
-# apt-get install asciidoc source-highlight
-$ python3 scripts/asciidoc2html.py
-----
-
-If video or sound don't seem to work, try installing the gstreamer plugins:
-
-----
-# apt-get install gstreamer1.0-plugins-{bad,base,good,ugly}
-----
-
-Then <<tox,install qutebrowser via tox>>.
-
-On Fedora
----------
-
-qutebrowser is available in the official repositories for Fedora 22 and newer.
-
-----
-# dnf install qutebrowser
-----
-
-On Archlinux
-------------
-
-qutebrowser is available in the official [community] repository.
-
-----
-# pacman -S qutebrowser
-----
-
-There is also a -git version available in the AUR:
-https://aur.archlinux.org/packages/qutebrowser-git/[qutebrowser-git].
-
-You can install it using `makepkg` like this:
-
-----
-$ git clone https://aur.archlinux.org/qutebrowser-git.git
-$ cd qutebrowser-git
-$ makepkg -si
-$ cd ..
-$ rm -r qutebrowser-git
-----
-
-or you could use an AUR helper, e.g. `yaourt -S qutebrowser-git`.
-
-If video or sound don't seem to work, try installing the gstreamer plugins:
-
-----
-# pacman -S gst-plugins-{base,good,bad,ugly} gst-libav
-----
-
-On Gentoo
----------
-
-qutebrowser is available in the main repository and can be installed with:
-
-----
-# emerge -av qutebrowser
-----
-
-Make sure you have `python3_4` in your `PYTHON_TARGETS`
-(`/etc/portage/make.conf`) and rebuild your system (`emerge -uDNav @world`) if
-necessary.
-
-If video or sound don't seem to work, try installing the gstreamer plugins:
-
-----
-# emerge -av gst-plugins-{base,good,bad,ugly,libav}
-----
-
-
-On Void Linux
--------------
-
-qutebrowser is available in the official repositories and can be installed
-with:
-
-----
-# xbps-install qutebrowser
-----
-
-On NixOS
---------
-
-Nixpkgs collection contains `pkgs.qutebrowser` since June 2015. You can install
-it with:
-
-----
-$ nix-env -i qutebrowser
-----
-
-On openSUSE
------------
-
-There are prebuilt RPMs available for Tumbleweed and Leap 42.1:
-
-http://software.opensuse.org/download.html?project=home%3Aarpraher&package=qutebrowser[One Click Install]
-
-Or add the repo manually:
-
-----
-# zypper addrepo http://download.opensuse.org/repositories/home:arpraher/openSUSE_Tumbleweed/home:arpraher.repo
-# zypper refresh
-# zypper install qutebrowser
-----
-
-On OpenBSD
-----------
-
-qutebrowser is in http://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/www/qutebrowser/[OpenBSD ports].
-
-Manual install:
-
-----
-# cd /usr/ports/www/qutebrowser
-# make install
-----
-
-Or alternatively if you're using `-current` (or OpenBSD 6.1 once it's been released):
-
-----
-# pkg_add qutebrowser
-----
-
-On Windows
-----------
-
-There are different ways to install qutebrowser on Windows:
-
-Prebuilt binaries
-~~~~~~~~~~~~~~~~~
-
-Prebuilt standalone packages and MSI installers
-https://github.com/qutebrowser/qutebrowser/releases[are built] for every
-release.
-
-https://chocolatey.org/packages/qutebrowser[Chocolatey package]
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-* PackageManagement PowerShell module
-----
-PS C:\> Install-Package qutebrowser
-----
-* Chocolatey's client
-----
-C:\> choco install qutebrowser
-----
-
-Manual install
-~~~~~~~~~~~~~~
-
-* Use the installer from http://www.python.org/downloads[python.org] to get
-Python 3 (be sure to install pip).
-* Use the installer from
-http://www.riverbankcomputing.com/software/pyqt/download5[Riverbank computing]
-to get Qt and PyQt5.
-* Install https://testrun.org/tox/latest/index.html[tox] via
-https://pip.pypa.io/en/latest/[pip]:
-
-----
-$ pip install tox
-----
-
-Then <<tox,install qutebrowser via tox>>.
-
-On OS X
--------
-
-Prebuilt binary
-~~~~~~~~~~~~~~~
-
-The easiest way to install qutebrowser on OS X is to use the prebuilt `.app`
-files from the
-https://github.com/qutebrowser/qutebrowser/releases[release page].
-
-This binary is also available through the
-https://caskroom.github.io/[Homebrew Cask] package manager:
-
-----
-$ brew cask install qutebrowser
-----
-
-Manual Install
-~~~~~~~~~~~~~~
-
-Alternatively, you can install the dependencies via a package manager (like
-http://brew.sh/[Homebrew] or https://www.macports.org/[MacPorts]) and run
-qutebrowser from source.
-
-==== Homebrew
-
-----
-$ brew install qt5
-$ pip3 install qutebrowser
-----
-
-Homebrew's builds of Qt and PyQt no longer include QtWebKit - if you need
-QtWebKit support, it is necessary to build from source. The build takes several
-hours on an average laptop.
-
-----
-$ brew install qt5 --with-qtwebkit
-$ brew install -s pyqt5
-$ pip3 install qutebrowser
-----
-
-Packagers
----------
-
-There are example .desktop and icon files provided. They would go in the
-standard location for your distro (`/usr/share/applications` and
-`/usr/share/pixmaps` for example).
-
-The normal `setup.py install` doesn't install these files, so you'll have to do
-it as part of the packaging process.
-
-[[tox]]
-Installing qutebrowser with tox
--------------------------------
-
-First of all, clone the repository using http://git-scm.org/[git] and switch
-into the repository folder:
-
-----
-$ git clone https://github.com/qutebrowser/qutebrowser.git
-$ cd qutebrowser
-----
-
-
-Then run tox inside the qutebrowser repository to set up a
-https://docs.python.org/3/library/venv.html[virtual environment]:
-
-----
-$ tox -e mkvenv
-----
-
-On Windows, run tox with the 'mkvenv-win' option, however make sure that ONLY Python3 is in your PATH before running tox.
-
-----
-$ tox -e mkvenv-win
-----
-
-This installs all needed Python dependencies in a `.venv` subfolder. The
-system-wide Qt5/PyQt5 installations are symlinked into the virtual environment.
-
-You can then create a simple wrapper script to start qutebrowser somewhere in
-your `$PATH` (e.g. `/usr/local/bin/qutebrowser` or `~/bin/qutebrowser`):
-
-----
-#!/bin/bash
-~/path/to/qutebrowser/.venv/bin/python3 -m qutebrowser "$@"
-----
-
-If you are developing on qutebrowser, you may want to redirect it to a local
-config:
-
-----
-#!/bin/bash
-~/path/to/qutebrowser/.venv/bin/python3 -m qutebrowser -c .qutebrowser-local "$@"
-----
-
-Updating
-~~~~~~~~
-
-When you updated your local copy of the code (e.g. by pulling the git repo, or
-extracting a new version), the virtualenv should automatically use the updated
-code. However, if dependencies got added, this won't be reflected in the
-virtualenv. Thus it's recommended to run the following command to recreate the
-virtualenv:
-
-----
-$ tox -r -e mkvenv
-----

MANIFEST.in

@@ -8,15 +8,16 @@ graft icons
 graft doc/img
 graft misc/apparmor
 graft misc/userscripts
-recursive-include scripts *.py
+recursive-include scripts *.py *.sh
 include qutebrowser/utils/testfile
 include qutebrowser/git-commit-id
-include COPYING doc/* README.asciidoc CONTRIBUTING.asciidoc FAQ.asciidoc INSTALL.asciidoc CHANGELOG.asciidoc
-include qutebrowser.desktop
+include LICENSE doc/* README.asciidoc
+include misc/qutebrowser.desktop
 include requirements.txt
 include tox.ini
 include qutebrowser.py
 include misc/cheatsheet.svg
+include qutebrowser/config/configdata.yml
 
 prune www
 prune scripts/dev
@@ -26,24 +27,19 @@ exclude scripts/asciidoc2html.py
 exclude doc/notes
 recursive-exclude doc *.asciidoc
 include doc/qutebrowser.1.asciidoc
+include doc/changelog.asciidoc
 prune tests
 prune qutebrowser/3rdparty
 prune misc/requirements
 prune misc/docker
-exclude .editorconfig
 exclude pytest.ini
 exclude qutebrowser.rcc
-exclude .coveragerc
-exclude .pylintrc
 exclude qutebrowser/javascript/.eslintrc.yaml
 exclude qutebrowser/javascript/.eslintignore
 exclude doc/help
-exclude .appveyor.yml
-exclude .travis.yml
-exclude codecov.yml
-exclude .pydocstylerc
+exclude .*
 exclude misc/appveyor_install.py
 exclude misc/qutebrowser.spec
-exclude .flake8
+exclude misc/qutebrowser.nsi
 
 global-exclude __pycache__ *.pyc *.pyo

README.asciidoc

@@ -9,9 +9,8 @@ qutebrowser
 // QUTE_WEB_HIDE
 image:icons/qutebrowser-64x64.png[qutebrowser logo] *A keyboard-driven, vim-like browser based on PyQt5 and Qt.*
 
-image:https://img.shields.io/pypi/l/qutebrowser.svg?style=flat["license badge",link="https://github.com/qutebrowser/qutebrowser/blob/master/COPYING"]
+image:https://img.shields.io/pypi/l/qutebrowser.svg?style=flat["license badge",link="https://github.com/qutebrowser/qutebrowser/blob/master/LICENSE"]
 image:https://img.shields.io/pypi/v/qutebrowser.svg?style=flat["version badge",link="https://pypi.python.org/pypi/qutebrowser/"]
-image:https://requires.io/github/qutebrowser/qutebrowser/requirements.svg?branch=master["requirements badge",link="https://requires.io/github/qutebrowser/qutebrowser/requirements/?branch=master"]
 image:https://travis-ci.org/qutebrowser/qutebrowser.svg?branch=master["Build Status", link="https://travis-ci.org/qutebrowser/qutebrowser"]
 image:https://ci.appveyor.com/api/projects/status/5pyauww2k68bbow2/branch/master?svg=true["AppVeyor build status", link="https://ci.appveyor.com/project/qutebrowser/qutebrowser"]
 image:https://codecov.io/github/qutebrowser/qutebrowser/coverage.svg?branch=master["coverage badge",link="https://codecov.io/github/qutebrowser/qutebrowser?branch=master"]
@@ -36,11 +35,8 @@ Downloads
 ---------
 
 See the https://github.com/qutebrowser/qutebrowser/releases[github releases
-page] for available downloads (currently a source archive, and standalone
-packages as well as MSI installers for Windows).
-
-See link:INSTALL.asciidoc[INSTALL] for detailed instructions on how to get
-qutebrowser running for various platforms.
+page] for available downloads and the link:doc/install.asciidoc[INSTALL] file for
+detailed instructions on how to get qutebrowser running on various platforms.
 
 Documentation
 -------------
@@ -48,14 +44,15 @@ Documentation
 In addition to the topics mentioned in this README, the following documents are
 available:
 
-* A https://qutebrowser.org/img/cheatsheet-big.png[key binding cheatsheet]: +
+* https://qutebrowser.org/img/cheatsheet-big.png[Key binding cheatsheet]: +
 image:https://qutebrowser.org/img/cheatsheet-small.png["qutebrowser key binding cheatsheet",link="https://qutebrowser.org/img/cheatsheet-big.png"]
 * link:doc/quickstart.asciidoc[Quick start guide]
-* A https://www.shortcutfoo.com/app/dojos/qutebrowser[free training course] to remember those key bindings.
-* link:FAQ.asciidoc[Frequently asked questions]
-* link:CONTRIBUTING.asciidoc[Contributing to qutebrowser]
-* link:INSTALL.asciidoc[INSTALL]
-* link:CHANGELOG.asciidoc[Change Log]
+* https://www.shortcutfoo.com/app/dojos/qutebrowser[Free training course] to remember those key bindings
+* link:doc/faq.asciidoc[Frequently asked questions]
+* link:doc/help/configuring.asciidoc[Configuring qutebrowser]
+* link:doc/contributing.asciidoc[Contributing to qutebrowser]
+* link:doc/install.asciidoc[Installing qutebrowser]
+* link:doc/changelog.asciidoc[Change Log]
 * link:doc/stacktrace.asciidoc[Reporting segfaults]
 * link:doc/userscripts.asciidoc[How to write userscripts]
 
@@ -70,14 +67,18 @@ message to the
 https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser[mailinglist] at
 mailto:qutebrowser@lists.qutebrowser.org[].
 
-There's also a https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser-announce[announce-only mailinglist]
-at mailto:qutebrowser-announce@lists.qutebrowser.org[].
+There's also an https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser-announce[announce-only mailinglist]
+at mailto:qutebrowser-announce@lists.qutebrowser.org[] (the announcements also
+get sent to the general qutebrowser@ list).
+
+If you're a reddit user, there's a
+https://www.reddit.com/r/qutebrowser/[/r/qutebrowser] subreddit there.
 
 Contributions / Bugs
 --------------------
 
 You want to contribute to qutebrowser? Awesome! Please read
-link:CONTRIBUTING.asciidoc[the contribution guidelines] for details and
+link:doc/contributing.asciidoc[the contribution guidelines] for details and
 useful hints.
 
 If you found a bug or have a feature request, you can report it in several
@@ -97,29 +98,36 @@ Requirements
 
 The following software and libraries are required to run qutebrowser:
 
-* http://www.python.org/[Python] 3.4 or newer
-* http://qt.io/[Qt] 5.2.0 or newer (5.5.1 recommended)
-* QtWebKit (old or link:https://github.com/annulen/webkit/wiki[reloaded]/NG) or QtWebEngine
-* http://www.riverbankcomputing.com/software/pyqt/intro[PyQt] 5.2.0 or newer
-(5.5.1 recommended) for Python 3
+* http://www.python.org/[Python] 3.5 or newer (3.6 recommended)
+* http://qt.io/[Qt] 5.7.1 or newer with the following modules:
+  - QtCore / qtbase
+  - QtQuick (part of qtbase in some distributions)
+  - QtSQL (part of qtbase in some distributions)
+  - QtOpenGL
+  - QtWebEngine, or
+  - QtWebKit - only the
+    link:https://github.com/annulen/webkit/wiki[updated fork] (5.212) is
+    supported
+* http://www.riverbankcomputing.com/software/pyqt/intro[PyQt] 5.7.0 or newer
+  (5.9 recommended) for Python 3
 * https://pypi.python.org/pypi/setuptools/[pkg_resources/setuptools]
 * http://fdik.org/pyPEG/[pyPEG2]
 * http://jinja.pocoo.org/[jinja2]
 * http://pygments.org/[pygments]
 * http://pyyaml.org/wiki/PyYAML[PyYAML]
+* http://www.attrs.org/[attrs]
 
-The following libraries are optional and provide a better user experience:
+The following libraries are optional:
 
-* http://cthedot.de/cssutils/[cssutils]
+* http://cthedot.de/cssutils/[cssutils] (for an improved `:download --mhtml`
+  with QtWebKit).
+* On Windows, https://pypi.python.org/pypi/colorama/[colorama] for colored log
+  output.
+* http://asciidoc.org/[asciidoc] to generate the documentation for the `:help`
+  command, when using the git repository (rather than a release).
 
-To generate the documentation for the `:help` command, when using the git
-repository (rather than a release), http://asciidoc.org/[asciidoc] is needed.
-
-On Windows, https://pypi.python.org/pypi/colorama/[colorama] is needed to
-display colored log output.
-
-See link:INSTALL.asciidoc[INSTALL] for directions on how to install qutebrowser
-and its dependencies.
+See link:doc/install.asciidoc[the documentation] for directions on how to
+install qutebrowser and its dependencies.
 
 Donating
 --------
@@ -140,206 +148,62 @@ get in touch!
 Authors
 -------
 
-Contributors, sorted by the number of commits in descending order:
+qutebrowser's primary author is Florian Bruhin (The Compiler), but qutebrowser
+wouldn't be what it is without the help of
+https://github.com/qutebrowser/qutebrowser/graphs/contributors[hundreds of contributors]!
 
-// QUTE_AUTHORS_START
-* Florian Bruhin
-* Daniel Schadt
-* Ryan Roden-Corrent
-* Jan Verbeek
-* Jakub Klinkovský
-* Antoni Boucher
-* Lamar Pavel
-* Marshall Lochbaum
-* Bruno Oliveira
-* Alexander Cogneau
-* Felix Van der Jeugt
-* Daniel Karbach
-* Martin Tournoij
-* Kevin Velghe
-* Raphael Pierzina
-* Joel Torstensson
-* Patric Schmitz
-* Tarcisio Fedrizzi
-* Claude
-* Corentin Julé
-* meles5
-* Philipp Hansch
-* Imran Sobir
-* Panagiotis Ktistakis
-* Artur Shaik
-* Nathan Isom
-* Thorsten Wißmann
-* Austin Anderson
-* Fritz Reichwald
-* Jimmy
-* Niklas Haas
-* Maciej Wołczyk
-* Spreadyy
-* Alexey "Averrin" Nabrodov
-* nanjekyejoannah
-* avk
-* ZDarian
-* Milan Svoboda
-* John ShaggyTwoDope Jenkins
-* Clayton Craft
-* Peter Vilim
-* knaggita
-* Oliver Caldwell
-* Julian Weigt
-* Tomasz Kramkowski
-* Sebastian Frysztak
-* Nikolay Amiantov
-* Julie Engel
-* Jonas Schürmann
-* error800
-* Michael Hoang
-* Liam BEGUIN
-* Daniel Fiser
-* skinnay
-* Zach-Button
-* Samuel Walladge
-* Peter Rice
-* Ismail S
-* Halfwit
-* David Vogt
-* Claire Cavanaugh
-* rikn00
-* kanikaa1234
-* haitaka
-* Nick Ginther
-* Michał Góral
-* Michael Ilsaas
-* Martin Zimmermann
-* Jussi Timperi
-* Cosmin Popescu
-* Brian Jackson
-* thuck
-* sbinix
-* rsteube
-* neeasade
-* jnphilipp
-* Tobias Patzl
-* Stefan Tatschner
-* Samuel Loury
-* Peter Michely
-* Panashe M. Fundira
-* Lucas Hoffmann
-* Link
-* Larry Hynes
-* Kirill A. Shutemov
-* Johannes Altmanninger
-* Jeremy Kaplan
-* Ismail
-* Edgar Hipp
-* Daryl Finlay
-* pkill9
-* arza
-* adam
-* Samir Benmendil
-* Regina Hug
-* Mathias Fussenegger
-* Marcelo Santos
-* Joel Bradshaw
-* Jean-Louis Fuchs
-* Franz Fellner
-* Eric Drechsel
-* zwarag
-* xd1le
-* rmortens
-* oniondreams
-* issue
-* haxwithaxe
-* evan
-* dylan araps
-* addictedtoflames
-* Xitian9
-* Tomas Orsava
-* Tom Janson
-* Tobias Werth
-* Tim Harder
-* Thiago Barroso Perrotta
-* Sorokin Alexei
-* Simon Désaulniers
-* Rok Mandeljc
-* Noah Huesser
-* Moez Bouhlel
-* Matthias Lisin
-* Marcel Schilling
-* Lazlow Carmichael
-* Kevin Wang
-* Ján Kobezda
-* Johannes Martinsson
-* Jean-Christophe Petkovich
-* Jay Kamat
-* Helen Sherwood-Taylor
-* HalosGhost
-* Gregor Pohl
-* Eivind Uggedal
-* Dietrich Daroch
-* Derek Sivers
-* Daniel Lu
-* Arseniy Seroka
-* Andy Balaam
-* Andreas Fischer
-* Akselmo
-// QUTE_AUTHORS_END
-
-The following people have contributed graphics:
+Additionally, the following people have contributed graphics:
 
 * Jad/link:http://yelostudio.com[yelo] (new icon)
 * WOFall (original icon)
 * regines (key binding cheatsheet)
 
-Thanks / Similar projects
--------------------------
+Also, thanks to everyone who contributed to one of qutebrowser's
+link:doc/backers.asciidoc[crowdfunding campaigns]!
 
-Many projects with a similar goal as qutebrowser exist:
-
-* http://portix.bitbucket.org/dwb/[dwb] (C, GTK+ with WebKit1, currently
-http://www.reddit.com/r/linux/comments/2huqbc/dwb_abandoned/[unmaintained] -
-main inspiration for qutebrowser)
-* https://github.com/fanglingsu/vimb[vimb] (C, GTK+ with WebKit1, active)
-* http://sourceforge.net/p/vimprobable/wiki/Home/[vimprobable] (C, GTK+ with
-WebKit1, dead)
-* http://surf.suckless.org/[surf] (C, GTK+ with WebKit1, active)
-* https://mason-larobina.github.io/luakit/[luakit] (C/Lua, GTK+ with
-WebKit1, not very active)
-* http://pwmt.org/projects/jumanji/[jumanji] (C, GTK+ with WebKit1, not very
-active)
-* http://www.uzbl.org/[uzbl] (C, GTK+ with WebKit1/WebKit2, active)
-* http://conkeror.org/[conkeror] (Javascript, Emacs-like, XULRunner/Gecko,
-active)
-* https://github.com/AeroNotix/lispkit[lispkit] (quite new, lisp, GTK+ with
-WebKit, active)
-* http://www.vimperator.org/[Vimperator] (Firefox addon)
-* http://5digits.org/pentadactyl/[Pentadactyl] (Firefox addon)
-* https://github.com/akhodakivskiy/VimFx[VimFx] (Firefox addon)
-* https://github.com/1995eaton/chromium-vim[cVim] (Chrome/Chromium addon)
-* http://vimium.github.io/[vimium] (Chrome/Chromium addon)
-* https://chrome.google.com/webstore/detail/vichrome/gghkfhpblkcmlkmpcpgaajbbiikbhpdi?hl=en[ViChrome] (Chrome/Chromium addon)
-* https://github.com/jinzhu/vrome[Vrome] (Chrome/Chromium addon)
+Similar projects
+----------------
 
+Many projects with a similar goal as qutebrowser exist.
 Most of them were inspirations for qutebrowser in some way, thanks for that!
 
-Thanks as well to the following projects and people for helping me with
-problems and helpful hints:
+Active
+~~~~~~
 
-* http://eric-ide.python-projects.org/[eric5] / Detlev Offenbach
-* https://code.google.com/p/devicenzo/[devicenzo]
-* portix
-* seir
-* nitroxleecher
+* https://fanglingsu.github.io/vimb/[vimb] (C, GTK+ with WebKit2)
+* https://luakit.github.io/luakit/[luakit] (C/Lua, GTK+ with WebKit2)
+* http://surf.suckless.org/[surf] (C, GTK+ with WebKit1/WebKit2)
+* http://www.uzbl.org/[uzbl] (C, GTK+ with WebKit1/WebKit2)
+* Chrome/Chromium addons:
+  https://github.com/1995eaton/chromium-vim[cVim],
+  http://vimium.github.io/[Vimium],
+  https://github.com/brookhong/Surfingkeys[Surfingkeys],
+  https://key.saka.io/[Saka Key]
+* Firefox addons (based on WebExtensions):
+  https://addons.mozilla.org/en-GB/firefox/addon/vimium-ff/[Vimium-FF] (experimental),
+  https://key.saka.io[Saka Key],
+  https://github.com/cmcaine/tridactyl[Tridactyl] (in early development, working
+  on a https://bugzilla.mozilla.org/show_bug.cgi?id=1215061[better API] for
+  keyboard integration in Firefox).
 
-Also, thanks to:
+Inactive
+~~~~~~~~
 
-* Everyone contributing to the link:doc/backers.asciidoc[crowdfunding].
-* Everyone who had the patience to test qutebrowser before v0.1.
-* Everyone triaging/fixing my bugs in the
-https://bugreports.qt.io/secure/Dashboard.jspa[Qt bugtracker]
-* Everyone answering my questions on http://stackoverflow.com/[Stack Overflow]
-and in IRC.
-* All the projects which were a great help while developing qutebrowser.
+* https://bitbucket.org/portix/dwb[dwb] (C, GTK+ with WebKit1,
+https://bitbucket.org/portix/dwb/pull-requests/22/several-cleanups-to-increase-portability/diff[unmaintained] -
+main inspiration for qutebrowser)
+* http://sourceforge.net/p/vimprobable/wiki/Home/[vimprobable] (C, GTK+ with
+  WebKit1)
+* http://pwmt.org/projects/jumanji/[jumanji] (C, GTK+ with WebKit1)
+* http://conkeror.org/[conkeror] (Javascript, Emacs-like, XULRunner/Gecko)
+* Firefox addons (not based on WebExtensions or no recent activity):
+  http://www.vimperator.org/[Vimperator],
+  http://5digits.org/pentadactyl/[Pentadactyl],
+  https://github.com/akhodakivskiy/VimFx[VimFx],
+  https://github.com/shinglyu/QuantumVim[QuantumVim]
+* Chrome/Chromium addons:
+  https://chrome.google.com/webstore/detail/vichrome/gghkfhpblkcmlkmpcpgaajbbiikbhpdi?hl=en[ViChrome],
+  https://github.com/jinzhu/vrome[Vrome]
 
 License
 -------
@@ -355,7 +219,7 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.
 
 You should have received a copy of the GNU General Public License
-along with this program.  If not, see <http://www.gnu.org/licenses/>.
+along with this program.  If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>.
 
 pdf.js
 ------

codecov.yml

@@ -1,9 +0,0 @@
-status:
-  project:
-    enabled: no
-  patch:
-    enabled: no
-  changes:
-    enabled: no
-
-comment: off

doc/backers.asciidoc

@@ -1,13 +1,135 @@
 Crowdfunding backers
 ====================
 
+2017
+----
+
+Mid-2017, qutebrowser had its
+https://www.kickstarter.com/projects/the-compiler/qutebrowser-v10-with-per-domain-settings[second crowdfunding]
+with the goal of implementing the new config system and releasing v1.0.
+
+Thanks a lot to the following people who contributed to it:
+
+Gold sponsors
+~~~~~~~~~~~~~
+
+TODO
+
+Silver sponsors
+~~~~~~~~~~~~~~~
+
+TODO
+
+Other sponsors
+~~~~~~~~~~~~~~
+
+TODO: people with t-shirts or higher pledge levels
+
+- 7scan
+- Alex Suykov
+- Alexey Zhikhartsev
+- Allan Nordhøy
+- Anirudh Sanjeev
+- Anssi Puustinen
+- Benedikt Steindorf
+- Bernardo Kuri
+- Blaise Duszynski
+- Bostan
+- Bruno Oliveira
+- Colin Jacobs
+- Daniel Andersson
+- Danilo
+- David Beley
+- David Hollings
+- David Parrish
+- Derin Yarsuvat
+- Dmytro Kostiuchenko
+- Frederik Thorøe
+- G4v4g4i
+- Gyula Teleki
+- H
+- Hosaka
+- Iordanis Grigoriou
+- Isaac Sandaljian
+- Jakub Podeszwik
+- Jamie Anderson
+- Jasper Woudenberg
+- Jens Højgaard
+- Johannes
+- John Baber-Lucero
+- Jonas Schürmann
+- Kenichiro Ito
+- Kenny Low
+- Lars Ivar Igesund
+- Lucas Aride Moulin
+- Ludovic Chabant
+- Lukas Gierth
+- Marulkan
+- Matthew Chun-Lum
+- Matthew Cronen
+- Matthew Quigley
+- Michael Schönwälder
+- Mika Kutila
+- Mitchell Stokes
+- Nathan Howell
+- Nathan Schlehlein
+- Noël Zindel
+- Obri
+- Patrik Peng
+- Peter DiMarco
+- Peter Rice
+- Philipp Middendorf
+- Pkill9
+- Prescott
+- Robotichead
+- Roshless
+- Ryan Ellis
+- Ryan P Deslandes
+- Sam Doshi
+- Sam Stone
+- Sean Herman
+- Sebastian Frysztak
+- Shelby Cruver
+- SirCmpwn
+- Soham Pal
+- Stewart Webb
+- Sven Reinecke
+- Tom Bass
+- Tomas Slusny
+- Tomasz Kramkowski
+- Tommy Thomas
+- Vasilij Schneidermann
+- Vlaaaaaaad
+- beanieuptop
+- demure
+- evenorbert
+- fishss
+- gsnewmark
+- guillermohs9
+- hubcaps
+- lobachevsky
+- neodarz
+- nihlaeth
+- notbenh
+- patrick suwanvithaya
+- pyratebeard
+- randm_dave
+- sabreman
+- toml
+- vimja
+- wiz
+- 43 Anonymous
+
+2016
+----
+
 Mid-2016, qutebrowser did run a http://igg.me/at/qutebrowser[crowdfunding] for
 QtWebEngine support in qutebrowser.
 
 Thanks a lot to the following people who contributed to it:
 
 Gold sponsors
--------------
+~~~~~~~~~~~~~
 
 - Chris Salzberg
 - Clayton Craft
@@ -16,7 +138,7 @@ Gold sponsors
 - 1 Anonymous
 
 Day sponsors
-------------
+~~~~~~~~~~~~
 
 - Agent 42
 - Iggy Jackson
@@ -28,7 +150,7 @@ Day sponsors
 - 4 Anonymous
 
 Other sponsors
---------------
+~~~~~~~~~~~~~~
 
 - AP M
 - Alessandro Balzano

doc/changelog.asciidoc

@@ -4,7 +4,8 @@ Change Log
 // http://keepachangelog.com/
 
 All notable changes to this project will be documented in this file.
-This project adheres to http://semver.org/[Semantic Versioning].
+This project adheres to http://semver.org/[Semantic Versioning], though minor
+breaking changes (such as renamed commands) can happen in minor releases.
 
 // tags:
 // `Added` for new features.
@@ -14,6 +15,282 @@ This project adheres to http://semver.org/[Semantic Versioning].
 // `Fixed` for any bug fixes.
 // `Security` to invite users to upgrade in case of vulnerabilities.
 
+v1.0.2
+------
+
+Fixes
+~~~~~
+
+- Fix workaround for black screens or crashes with Nvidia cards
+- Handle a filesystem going read-only gracefully
+- Fix crash when setting `fonts.monospace`
+- Fix list options not being modifyable via `.append()` in `config.py`
+- Mark the content.notifications setting as QtWebKit only correctly
+- Fix wrong rendering of keys like `<back>` in the completion
+
+Changed
+~~~~~~~
+
+- Nicer error messages and other minor improvements
+
+v1.0.1
+------
+
+Fixes
+~~~~~
+
+- Fixed starting after customizing `fonts.tabs` or `fonts.debug_console`.
+- Fixed starting with old PyQt versions compiled against newer Qt versions.
+- Fixed check for PyQt version to correctly enforce 5.7 (not 5.2).
+
+v1.0.0
+------
+
+Major changes
+~~~~~~~~~~~~~
+
+- Dependency changes:
+  * Support for legacy QtWebKit (before 5.212 which is
+    https://github.com/annulen/webkit/wiki[distributed independently from Qt])
+    is dropped.
+  * Support for Python 3.4 is dropped.
+  * Support for Qt before 5.7.1 and PyQt before 5.7 is dropped.
+  * New dependency on the QtSql module and Qt sqlite support.
+  * New dependency on the http://www.attrs.org/[attrs] project (packaged as
+    `python-attr` in some distributions).
+  * The depedency on PyOpenGL (when using QtWebEngine) got removed. Note
+    that PyQt5.QtOpenGL is still a dependency.
+  * PyQt5.QtOpenGL is now always required, even with QtWebKit.
+- The QtWebEngine backend is now used by default. Note this means that
+  QtWebEngine now should be a required dependency, and QtWebKit (if new enough)
+  should be changed to an optional dependency.
+- Completely rewritten configuration system which ignores the old config file.
+  See link:qute://help/configuring.html[] for details.
+- Various documentation files got moved to the doc/ subfolder;
+ `qutebrowser.desktop` got moved to misc/.
+- `:set` now doesn't support toggling/cycling values anymore, that functionality
+  got moved to `:config-cycle`.
+- New completion engine based on sqlite, which allows to complete
+  the entire browsing history. The default for
+  `completion.web_history_max_items` got changed to `-1` (unlimited). If the
+  completion is too slow on your machine, try setting it to a few 1000 items.
+
+Added
+~~~~~
+
+- QtWebEngine: Spell checking support, see the `spellcheck.languages` setting.
+- New `qt.args` setting to pass additional arguments to Qt/Chromium.
+- New `backend` setting to select the backend to use.
+  Together with the previous setting, this should make most wrapper scripts
+  unnecessary.
+- qutebrowser can now be set as the default browser on macOS.
+- New config commands:
+  * `:config-cycle` to cycle an option between multiple values.
+  * `:config-unset` to remove a configured option.
+  * `:config-clear` to remove all configured options.
+  * `:config-source` to (re-)read a `config.py` file.
+  * `:config-edit` to open the `config.py` file in an editor.
+  * `:config-write-py` to write a `config.py` template file.
+- New `:version` command which opens `qute://version`.
+- New back/forward indicator in the statusbar.
+- New `bindings.key_mappings` setting to map keys to other keys.
+- QtWebEngine: Support for proxy authentication.
+
+Changed
+~~~~~~~
+
+- Using `:download` now uses the page's title as filename.
+- Using `:back` or `:forward` with a count now skips intermediate pages.
+- When there are multiple messages shown, the timeout is increased.
+- `:search` now only clears the search if one was displayed before, so pressing
+  `<Escape>` doesn't un-focus inputs anymore.
+- Pinned tabs now adjust to their text's width, so the `tabs.width.pinned`
+  setting got removed.
+- `:set-cmd-text` now has a `--run-on-count` argument to run the underlying
+  command directly if a count was given.
+- `:scroll-perc` got renamed to `:scroll-to-perc`.
+
+Removed
+~~~~~~~
+
+- Migrating QtWebEngine data written by versions before 2016-11-15 (before
+  v0.9.0) is now not supported anymore.
+- Upgrading qutebrowser with a version older than v0.4.0 still running now won't
+  work properly anymore.
+- The `--harfbuzz` and `--relaxed-config` commandline arguments got dropped.
+
+Fixes
+~~~~~
+
+- Exiting fullscreen via `:fullscreen` or buttons on a page now
+  restores the correct previous window state (maximized/fullscreen).
+- When `input.insert_mode.auto_load` is set, background tabs now don't enter
+  insert mode anymore.
+- The keybinding help widget now works correctly when using keybindings with a
+  count.
+- The `window.hide_wayland_decoration` setting now works correctly again.
+
+v0.11.1
+-------
+
+Fixes
+~~~~~
+
+- Fixed empty space being shown after tabs in the tabbar in some cases.
+- Fixed `:restart` in private browsing mode.
+- Fixed printing on macOS.
+- Closing a pinned tab via mouse now also prompts for confirmation.
+- The "try again" button on error pages works correctly again.
+- :spawn -u -d is now disallowed.
+- :spawn -d shows error messages correctly now.
+
+v0.11.0
+-------
+
+New dependencies
+~~~~~~~~~~~~~~~~
+
+- New dependency on `PyQt5.QtOpenGL` if QtWebEngine is used. QtWebEngine depends
+  on QtOpenGL already, but on distributions packaging split PyQt5 wrappers, the
+  wrappers for QtOpenGL are now required.
+- New dependency on `PyOpenGL` if QtWebEngine is used.
+
+Added
+~~~~~
+
+- Private browsing is now implemented for QtWebEngine, *and changed its
+  behavior*: The `general -> private-browsing` setting now only applies to newly
+  opened windows, and you can use the `-p` flag to `:open` to open a private
+  window.
+- New "pinned tabs" feature, with a new `:tab-pin` command (bound
+  to `<Ctrl-p>` by default).
+- (QtWebEngine) Implemented `:follow-selected`.
+- New `:clear-messages` command to clear shown messages.
+- New `ui -> keyhint-delay` setting to configure the delay until
+  the keyhint overlay pops up.
+- New `-s` option for `:open` to force a HTTPS scheme.
+- `:debug-log-filter` now accepts `none` as an argument to clear any log
+  filters.
+- New `--debug-flag` argument which replaces `--debug-exit` and
+  `--pdb-postmortem`.
+- New `tabs -> favicon-scale` option to scale up/down favicons.
+- `colors -> statusbar.bg/fg.private` and `.command.private` to
+  customize statusbar colors for private windows.
+- New `{private}` field displaying `[Private Mode]` for
+  `ui -> window-title-format` and `tabs -> title-format`.
+- (QtWebEngine) Proxy support with Qt 5.7.1 (already was supported for 5.8 and
+  newer)
+
+Changed
+~~~~~~~
+
+- To prevent elaborate phishing attacks, the Punycode version (`xn--*`) is now
+  shown in addition to the decoded version for international domain names
+  (IDN).
+- Starting with legacy QtWebKit now shows a warning message.
+  *With the next release, support for it will be removed.*
+- The Windows releases are redone from scratch, which means:
+  * They now use the new QtWebEngine backend
+  * The bundled Qt is updated from 5.5 to 5.9
+  * The bundled Python is updated from 3.4 to 3.6
+  * They are now generated with PyInstaller instead of cx_Freeze
+  * The installer is now generated using NSIS instead of being a MSI
+- Improved `qute://history` page (with lazy loading)
+- Crash reports are not public anymore.
+- Paths like `C:` are now treated as absolute paths on Windows for downloads,
+  and invalid paths are handled properly.
+- Comments in the config file are now placed before the individual options
+  instead of being before sections.
+- Messages are now hidden when clicked.
+- stdin is now closed immediately for processes spawned from qutebrowser.
+- When `ui -> message-timeout` is set to 0, messages are now never cleared.
+- Middle/right-clicking the blank parts of the tab bar (when vertical) now
+  closes the current tab.
+- The adblocker now also blocks non-GET requests (e.g. POST).
+- `javascript:` links can now be hinted.
+- `:view-source`, `:tab-clone` and `:navigate --tab` now don't open the tab as
+  "explicit" anymore, i.e. (with the default settings) open it next to the
+  active tab.
+- `qute:*` pages now use `qute://*` instead (e.g. `qute://version` instead of
+  `qute:version`), but the old versions are automatically redirected.
+- Texts in prompts are now selectable.
+- The default level for `:messages` is now `info`, not `error`
+- Trying to focus the currently focused tab with `:tab-focus` now focuses the
+  last viewed tab.
+- (QtWebEngine) With Qt 5.9, `content -> cookies-store` can now be set without
+  a restart.
+- (QtWebEngine) With Qt 5.9, better error messages are now shown for failed
+  downloads.
+- (QtWebEngine) The underlying Chromium version is now shown in the version
+  info.
+- (QtWebKit) Renderer process crashes now show an error page on Qt 5.9 or newer.
+- (QtWebKit) storage -> offline-web-application-storage` got renamed to `...-cache`
+- (QtWebKit) PAC now supports SOCKS5 as type.
+
+Fixed
+~~~~~
+
+- The macOS .dmg is now built against Qt 5.9 which fixes various
+  important issues (such as not being able to type dead keys).
+- Fixed crash with `:download` on PyQt 5.9.
+- Cloning a page without history doesn't crash anymore.
+- When a download results in a HTTP error, it now shows the error correctly
+  instead of crashing.
+- Pressing ctrl-c while a config error is shown works as intended now.
+- When the key config isn't writable, we now show an error instead of crashing.
+- Fixed crash when unbinding an unbound key in the key config.
+- Fixed crash when using `:debug-log-filter` when `--filter` wasn't given on startup.
+- Fixed crash with some invalid setting values.
+- Continuing a search after clearing it now works correctly.
+- The tabbar and completion should now be more consistently and correctly
+  styled with various system styles.
+- Applying styiles in `qt5ct` now shouldn't crash anymore.
+- The validation for colors in stylesheets is now less strict,
+  allowing for all valid Qt values.
+- `data:` URLs now aren't added to the history anymore.
+- Accidentally starting with Python 2 now shows a proper error message again.
+- For some people, running some userscripts crashed - this should now be fixed.
+- Various other rare crashes should now be fixed.
+- The settings documentation was truncated with v0.10.1 which should now be
+  fixed.
+- Scrolling to an anchor in a background tab now works correctly, and javascript
+  gets the correct window size for background tabs.
+- (QtWebEngine) Added a workaround for a black screen with some setups
+- (QtWebEngine) Starting with Nouveau graphics now shows an error message
+  instead of crashing in Qt.
+- (QtWebEngine) Retrying downloads now shows an error instead of crashing.
+- (QtWebEngine) Cloning a view-source tab now doesn't crash anymore.
+- (QtWebEngine) `window.navigator.userAgent` is now set correctly when
+  customizing the user agent.
+- (QtWebEngine) HTML fullscreen is now tracked for each tab separately, which
+  means it's not possible anymore to accidentally get stuck in fullscreen state
+  by closing a tab with a fullscreen video.
+- (QtWebEngine) `:scroll-page` with `--bottom-navigate` now works correctly.
+- (QtWebKit) The HTTP cache is disabled on Qt 5.7.1 and 5.8 now as it leads to
+  frequent crashes due to a Qt bug.
+- (QtWebKit) Fixed Crash when a PAC file returns an invalid value.
+
+v0.10.1
+-------
+
+Changed
+~~~~~~~
+
+- `--qt-arg` and `--qt-flag` can now also be used to pass arguments to Chromium when using QtWebEngine.
+
+Fixed
+~~~~~
+
+- URLs are now redacted properly (username/password, and path/query for HTTPS) when using Proxy Autoconfig with QtWebKit
+- Crash when updating adblock lists with invalid UTF8-chars in them
+- Fixed the web inspector with QtWebEngine
+- Version checks when starting qutebrowser now also take the Qt version PyQt was compiled against into account
+- Hinting a input now doesn't select existing text anymore with QtWebKit
+- The cursor now moves to the end when input elements are selected with QtWebEngine
+- Download suffixes like (1) are now correctly stripped with QtWebEngine
+- Crash when trying to print a tab which was closed in the meantime
+- Crash when trying to open a file twice on Windows
+
 v0.10.0
 -------
 
@@ -36,7 +313,7 @@ Added
 - Open tabs are now auto-saved on each successful load and restored in case of a crash
 - `:jseval` now has a `--file` flag so you can pass a javascript file
 - `:session-save` now has a `--only-active-window` flag to only save the active window
-- OS X builds are back, and built with QtWebEngine
+- macOS builds are back, and built with QtWebEngine
 
 Changed
 ~~~~~~~
@@ -52,6 +329,18 @@ Changed
 - `network -> proxy` can also be set to `pac+file://...` now to
   use a local proxy autoconfig file (on QtWebKit)
 
+Removed
+~~~~~~~
+
+- (QtWebKit) Various rarely customized settings were removed:
+  * `ui -> css-media-type` (defaults to desktop)
+  * `general -> site-specific-quirks` (now always turned on)
+  * `storage -> offline-storage-default-quota` (defaults to 5MB)
+  * `storage -> offline-web-application-cache-quota` (defaults to no quota)
+  * `storage -> object-cache-capacities` (default depends on disk space)
+  * `content -> css-regions` (now always turned off)
+  * `storage -> offline-storage-database` (merged into `storage -> local-storage`)
+
 Fixed
 ~~~~~
 
@@ -326,7 +615,7 @@ Fixed
 - Fix crash when pressing enter without a command
 - Adjust error message to point out QtWebEngine is unsupported with the OS
   X .app currently.
-- Hide Harfbuzz warning with the OS X .app
+- Hide Harfbuzz warning with the macOS .app
 
 v0.8.0
 ------
@@ -689,7 +978,7 @@ Fixed
 - Fixed scrolling to the very left/right with `:scroll-perc`.
 - Using an external editor should now work correctly with some funny chars
   (U+2028/U+2029/BOM).
-- Movements in caret mode now should work correctly on OS X and Windows.
+- Movements in caret mode now should work correctly on macOS and Windows.
 - Fixed upgrade from earlier config versions.
 - Fixed crash when killing a running userscript.
 - Fixed characters being passed through when shifted with
@@ -764,7 +1053,7 @@ Changed
 - The completion widget doesn't show a border anymore.
 - The tabbar doesn't display ugly arrows anymore if there isn't enough space
   for all tabs.
-- Some insignificant Qt warnings which were printed on OS X are now hidden.
+- Some insignificant Qt warnings which were printed on macOS are now hidden.
 - Better support for Qt 5.5 and Python 3.5.
 
 Fixed
@@ -875,7 +1164,7 @@ Fixed
 - Fixed AssertionError when closing many windows quickly.
 - Various fixes for deprecated key bindings and auto-migrations.
 - Workaround for qutebrowser not starting when there are NUL-bytes in the history (because of a currently unknown bug).
-- Fixed handling of keybindings containing Ctrl/Meta on OS X.
+- Fixed handling of keybindings containing Ctrl/Meta on macOS.
 - Fixed crash when downloading a URL without filename (e.g. magnet links) via "Save as...".
 - Fixed exception when starting qutebrowser with `:set` as argument.
 - Fixed horrible completion performance when the `shrink` option was set.
@@ -973,7 +1262,7 @@ Changed
 - Add a `:search` command in addition to `/foo` so it's more visible and can be used from scripts.
 - Various improvements to documentation, logging, and the crash reporter.
 - Expand `~` to the users home directory with `:run-userscript`.
-- Improve the userscript runner on Linux/OS X by using `QSocketNotifier`.
+- Improve the userscript runner on Linux/macOS by using `QSocketNotifier`.
 - Add luakit-like `gt`/`gT` keybindings to cycle through tabs.
 - Show default value for config values in the completion.
 - Clone tab icon, tab text and zoom level when cloning tabs.
@@ -993,7 +1282,7 @@ Changed
     * `init_venv.py` and `run_checks.py` have been replaced by http://tox.readthedocs.org/[tox]. Install tox and run `tox -e mkvenv` instead.
     * The tests now use http://pytest.org/[pytest]
     * Many new tests added
-    * Mac Mini buildbot to run the tests on OS X.
+    * Mac Mini buildbot to run the tests on macOS.
     * Coverage recording via http://nedbatchelder.com/code/coverage/[coverage.py].
     * New `--pdb-postmortem argument` to drop into the pdb debugger on exceptions.
     * Use https://github.com/ionelmc/python-hunter[hunter] for line tracing instead of a selfmade solution.
@@ -1129,7 +1418,7 @@ Fixed
 
 * Fix rare exception when a key is pressed shortly after opening a window
 * Fix exception with certain invalid URLs like `http:foo:0`
-* Work around Qt bug which renders checkboxes on OS X unusable
+* Work around Qt bug which renders checkboxes on macOS unusable
 * Fix exception when a local files can't be read in `:adblock-update`
 * Hide 2 more Qt warnings.
 * Add `!important` to hint CSS so websites don't override the hint look
@@ -1165,7 +1454,7 @@ Changes
 * Set zoom to default instead of 100% with `:zoom`/`=`.
 * Adjust page zoom if default zoom changed.
 * Force tabs to be focused on `:undo`.
-* Replace manual installation instructions on OS X with homebrew/macports.
+* Replace manual installation instructions on macOS with homebrew/macports.
 * Allow min-/maximizing of print preview on Windows.
 * Various documentation improvements.
 * Various other small improvements and cleanups.

doc/contributing.asciidoc

@@ -39,8 +39,13 @@ pointers:
 
 * https://github.com/qutebrowser/qutebrowser/labels/easy[Issues which should
 be easy to solve]
-* https://github.com/qutebrowser/qutebrowser/labels/not%20code[Issues which
-require little/no coding]
+* https://github.com/qutebrowser/qutebrowser/labels/component%3A%20docs[Documentation issues which require little/no coding]
+
+If you prefer C++ or Javascript to Python, see the relevant issues which involve
+work in those languages:
+
+* https://github.com/qutebrowser/qutebrowser/issues?utf8=%E2%9C%93&q=is%3Aopen%20is%3Aissue%20label%3Ac%2B%2B[C++] (mostly work on Qt, the library behind qutebrowser)
+* https://github.com/qutebrowser/qutebrowser/issues?q=is%3Aopen+is%3Aissue+label%3Ajavascript[JavaScript]
 
 There are also some things to do if you don't want to write code:
 
@@ -92,25 +97,24 @@ unittests and several linters/checkers.
 Currently, the following tox environments are available:
 
 * Tests using https://www.pytest.org[pytest]:
-  - `py34`: Run pytest for python-3.4.
-  - `py35`: Run pytest for python-3.5.
-  - `py34-cov`: Run pytest for python-3.4 with code coverage report.
-  - `py35-cov`: Run pytest for python-3.5 with code coverage report.
+  - `py35`, `py36`: Run pytest for python 3.5/3.6 with the system-wide PyQt.
+  - `py36-pyqt57`, ..., `py36-pyqt59`: Run pytest with the given PyQt version (`py35-*` also works).
+  - `py36-pyqt59-cov`: Run with coverage support (other Python/PyQt versions work too).
 * `flake8`: Run https://pypi.python.org/pypi/flake8[flake8] checks:
                 https://pypi.python.org/pypi/pyflakes[pyflakes],
                 https://pypi.python.org/pypi/pep8[pep8],
-                https://pypi.python.org/pypi/mccabe[mccabe]
+                https://pypi.python.org/pypi/mccabe[mccabe].
 * `vulture`: Run https://pypi.python.org/pypi/vulture[vulture] to find
   unused code portions.
 * `pylint`: Run http://pylint.org/[pylint] static code analysis.
 * `pydocstyle`: Check
   https://www.python.org/dev/peps/pep-0257/[PEP257] compliance with
-  https://github.com/PyCQA/pydocstyle[pydocstyle]
+  https://github.com/PyCQA/pydocstyle[pydocstyle].
 * `pyroma`: Check packaging practices with
-  https://pypi.python.org/pypi/pyroma/[pyroma]
+  https://pypi.python.org/pypi/pyroma/[pyroma].
 * `eslint`: Run http://eslint.org/[ESLint] javascript checker.
 * `check-manifest`: Check MANIFEST.in completeness with
-  https://github.com/mgedmin/check-manifest[check-manifest]
+  https://github.com/mgedmin/check-manifest[check-manifest].
 * `mkvenv`: Bootstrap a virtualenv for testing.
 * `misc`: Run `scripts/misc_checks.py` to check for:
     - untracked git files
@@ -127,7 +131,7 @@ techniques are useful to handle these:
 
 * Use `_foo` for unused parameters, with `foo` being a descriptive name. Using
 `_` is discouraged.
-* If you think you have a good reason to suppress a message, then add the 
+* If you think you have a good reason to suppress a message, then add the
 following comment:
 +
 ----
@@ -175,7 +179,7 @@ In the _scripts/_ subfolder there's a `run_profile.py` which profiles the code
 and shows a graphical representation of what takes how much time.
 
 It uses the built-in Python
-https://docs.python.org/3.4/library/profile.html[cProfile] module and can show
+https://docs.python.org/3.6/library/profile.html[cProfile] module and can show
 the output in four different ways:
 
 * Raw profile file (`--profile-tool=none`)
@@ -214,7 +218,7 @@ Documentation of used Python libraries:
 * http://pygments.org/docs/[pygments]
 * http://fdik.org/pyPEG/index.html[pyPEG2]
 * http://pythonhosted.org/setuptools/[setuptools]
-* http://cx-freeze.readthedocs.org/en/latest/overview.html[cx_Freeze]
+* http://www.pyinstaller.org/[PyInstaller]
 * https://pypi.python.org/pypi/colorama[colorama]
 
 Related RFCs and standards:
@@ -286,7 +290,7 @@ There are some exceptions to that:
 
 * `QThread` is used instead of Python threads because it provides signals and
 slots.
-* `QProcess` is used instead of Python's `subprocess`
+* `QProcess` is used instead of Python's `subprocess`.
 * `QUrl` is used instead of storing URLs as string, see the
 <<handling-urls,handling URLs>> section for details.
 
@@ -307,7 +311,7 @@ carefully be checked.
 * Methods of Qt objects have certain maximum values based on their
 underlying C++ types.
 +
-To avoid passing too large of a numeric parameter to a Qt function, all 
+To avoid passing too large of a numeric parameter to a Qt function, all
 numbers should be range-checked using `qutebrowser.qtutils.check_overflow`,
 or by other means (e.g. by setting a maximum value for a config object).
 
@@ -321,7 +325,7 @@ dictionaries which map object names to the actual long-living objects.
 There are currently these object registries, also called 'scopes':
 
 * The `global` scope, with objects which are used globally (`config`,
-`cookie-jar`, etc.)
+`cookie-jar`, etc.).
 * The `tab` scope with objects which are per-tab (`hintmanager`, `webview`,
 etc.). Passing this scope to `objreg.get()` selects the object in the currently
 focused tab by default. A tab can be explicitly selected by passing
@@ -430,14 +434,14 @@ def foo(bar: int, baz=True):
 ----
 
 Possible values:
-    - A callable (`int`, `float`, etc.): Gets called to validate/convert the
-      value.
-    - A python enum type: All members of the enum are possible values.
-    - A `typing.Union` of multiple types above: Any of these types are valid
-      values, e.g., `typing.Union[str, int]`
+
+- A callable (`int`, `float`, etc.): Gets called to validate/convert the value.
+- A python enum type: All members of the enum are possible values.
+- A `typing.Union` of multiple types above: Any of these types are valid
+  values, e.g., `typing.Union[str, int]`.
 
 You can customize how an argument is handled using the `@cmdutils.argument`
-decorator *after* `@cmdutils.register`. This can, for example, be used to 
+decorator *after* `@cmdutils.register`. This can, for example, be used to
 customize the flag an argument should get:
 
 [source,python]
@@ -464,10 +468,11 @@ For `typing.Union` types, the given `choices` are only checked if other types
 The following arguments are supported for `@cmdutils.argument`:
 
 - `flag`: Customize the short flag (`-x`) the argument will get.
-- `win_id=True`: Mark the argument as special window ID argument
-- `count=True`: Mark the argument as special count argument
-- `hide=True`: Hide the argument from the documentation
-- `completion`: A `usertypes.Completion` member to use as completion.
+- `win_id=True`: Mark the argument as special window ID argument.
+- `count=True`: Mark the argument as special count argument.
+- `hide=True`: Hide the argument from the documentation.
+- `completion`: A completion function (see `qutebrowser.completions.models.*`)
+  to use when completing arguments for the given command.
 - `choices`: The allowed string choices for the argument.
 
 The name of an argument will always be the parameter name, with any trailing
@@ -529,12 +534,12 @@ ____
 Setting up a Windows Development Environment
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-* Install https://www.python.org/downloads/release/python-344/[Python 3.4]
-* Install https://sourceforge.net/projects/pyqt/files/PyQt5/PyQt-5.5.1/[PyQt 5.5]
-* Create a file at `C:\Windows\system32\python3.bat` with the following content:
-  `@C:\Python34\python %*`
-  This will make the Python 3.4 interpreter available as `python3`, which is used by various development scripts.
-* Install git from the https://git-scm.com/download/win[git-scm downloads page]
+* Install https://www.python.org/downloads/release/python-362/[Python 3.6].
+* Install PyQt via `pip install PyQt5`.
+* Create a file at `C:\Windows\system32\python3.bat` with the following content (adjust the path as necessary):
+  `@C:\Python36\python %*`.
+  This will make the Python 3.6 interpreter available as `python3`, which is used by various development scripts.
+* Install git from the https://git-scm.com/download/win[git-scm downloads page].
   Try not to enable `core.autocrlf`, since that will cause `flake8` to complain a lot. Use an editor that can deal with plain line feeds instead.
 * Clone your favourite qutebrowser repository.
 * To install tox, open an elevated cmd, enter your working directory and run `pip install -rmisc/requirements/requirements-tox.txt`.
@@ -546,6 +551,28 @@ Rebuilding the website
 
 If you want to rebuild the website, run `./scripts/asciidoc2html.py --website <outputdir>`.
 
+Chrome URLs
+~~~~~~~~~~~
+
+With the QtWebEngine backend, qutebrowser supports several chrome:// urls which
+can be useful for debugging:
+
+- chrome://appcache-internals/
+- chrome://blob-internals/
+- chrome://gpu/
+- chrome://histograms/
+- chrome://indexeddb-internals/
+- chrome://media-internals/
+- chrome://network-errors/
+- chrome://serviceworker-internals/
+- chrome://webrtc-internals/
+- chrome://crash/ (crashes the current renderer process!)
+- chrome://kill/ (kills the current renderer process!)
+- chrome://gpucrash/ (crashes qutebrowser!)
+- chrome://gpuhang/ (hangs qutebrowser!)
+- chrome://gpuclean/ (crashes the current renderer process!)
+- chrome://ppapiflashcrash/
+- chrome://ppapiflashhang/
 
 Style conventions
 -----------------
@@ -631,7 +658,7 @@ New Qt release
 https://bugreports.qt.io/issues/?jql=reporter%20%3D%20%22The%20Compiler%22%20ORDER%20BY%20fixVersion%20ASC[Qt bugtracker]
 and make sure all bugs marked as resolved are actually fixed.
 * Update own PKGBUILDs based on upstream Archlinux updates and rebuild.
-* Update recommended Qt version in `README`
+* Update recommended Qt version in `README`.
 * Grep for `WORKAROUND` in the code and test if fixed stuff works without the
 workaround.
 * Check relevant
@@ -641,37 +668,32 @@ bugs] and check if they're fixed.
 New PyQt release
 ~~~~~~~~~~~~~~~~
 
-* See above
-* Install new PyQt in Windows VM (32- and 64-bit)
+* See above.
+* Install new PyQt in Windows VM (32- and 64-bit).
 * Download new installer and update PyQt installer path in `ci_install.py`.
-* Update `tox.ini`/`.travis.yml`/`.appveyor.yml` to test new versions
+* Update `tox.ini`/`.travis.yml`/`.appveyor.yml` to test new versions.
 
 qutebrowser release
 ~~~~~~~~~~~~~~~~~~~
 
 * Make sure there are no unstaged changes and the tests are green.
-* Run `x=... y=...` to set the respective shell variables
+* Run `x=... y=...` to set the respective shell variables.
 
-* Add newest config to `tests/unit/config/old_configs` and update `test_upgrade_version`
-    - `python -m qutebrowser --basedir conf :quit`
-    - `sed '/^#/d'  conf/config/qutebrowser.conf > tests/unit/config/old_configs/qutebrowser-v0.x.y.conf`
-    - `rm -r conf`
-    - commit
 * Adjust `__version_info__` in `qutebrowser/__init__.py`.
-* Update changelog (remove *(unreleased)*)
-* Run tests again
-* Commit
+* Update changelog (remove *(unreleased)*).
+* Run tests again.
+* Commit.
 
-* Create annotated git tag (`git tag -s "v0.$x.$y" -m "Release v0.$x.$y"`)
-* `git push origin`; `git push origin v0.$x.$y`
+* Create annotated git tag (`git tag -s "v1.$x.$y" -m "Release v1.$x.$y"`).
+* `git push origin`; `git push origin v1.$x.$y`.
 * If committing on minor branch, cherry-pick release commit to master.
-* Create release on github
+* Create release on github.
 * Mark the milestone at https://github.com/qutebrowser/qutebrowser/milestones
 as closed.
 
-* Linux: Run `python3 scripts/dev/build_release.py --upload v0.$x.$y`
-* Windows: Run `C:\Python34_x32\python scripts\dev\build_release.py --asciidoc C:\Python27\python C:\asciidoc-8.6.9\asciidoc.py --upload v0.X.Y` (replace X/Y by hand)
-* OS X: Run `python3 scripts/dev/build_release.py --upload v0.X.Y` (replace X/Y by hand)
-* On server: Run `python3 scripts/dev/download_release.py v0.X.Y` (replace X/Y by hand)
-* Update `qutebrowser-git` PKGBUILD if dependencies/install changed
-* Announce to qutebrowser and qutebrowser-announce mailinglist
+* Linux: Run `python3 scripts/dev/build_release.py --upload v1.$x.$y`.
+* Windows: Run `C:\Python36-32\python scripts\dev\build_release.py --asciidoc C:\Python27\python C:\asciidoc-8.6.9\asciidoc.py --upload v1.X.Y` (replace X/Y by hand).
+* macOS: Run `python3 scripts/dev/build_release.py --upload v1.X.Y` (replace X/Y by hand).
+* On server: Run `python3 scripts/dev/download_release.py v1.X.Y` (replace X/Y by hand).
+* Update `qutebrowser-git` PKGBUILD if dependencies/install changed.
+* Announce to qutebrowser and qutebrowser-announce mailinglist.

doc/faq.asciidoc

@@ -72,8 +72,8 @@ Is there an adblocker?::
     http://www.reddit.com/r/programming/comments/25j41u/adblock_pluss_effect_on_firefoxs_memory_usage/chhpomw[big
     impact] on browsing speed and
     https://blog.mozilla.org/nnethercote/2014/05/14/adblock-pluss-effect-on-firefoxs-memory-usage/[RAM
-    usage], so implementing it properly might take some time and won't be done
-    for v0.1 if at all.
+    usage], so implementing support for AdBlockPlus-like lists is currently not
+    a priority.
 
 How do I play Youtube videos with mpv?::
     You can easily add a key binding to play youtube videos inside a real video
@@ -124,6 +124,66 @@ When using quickmark, you can give them all names, like
 `:open foodrecipes`, you will see a list of all the food recipe sites,
 without having to remember the exact website title or address.
 
+How do I use spell checking?::
+	Configuring spell checking in qutebrowser depends on the backend in use
+    (see https://github.com/qutebrowser/qutebrowser/issues/700[#700] for
+	a more detailed discussion).
++
+For QtWebKit:
+
+. Install https://github.com/QupZilla/qtwebkit-plugins[qtwebkit-plugins].
+  . Note: with QtWebKit reloaded you may experience some issues. See
+    https://github.com/QupZilla/qtwebkit-plugins/issues/10[#10].
+. The dictionary to use is taken from the `DICTIONARY` environment variable.
+  The default is `en_US`. For example to use Dutch spell check set `DICTIONARY`
+  to `nl_NL`; you can't use multiple dictionaries or change them at runtime at
+  the moment.
+  (also see the README file for `qtwebkit-plugins`).
+. Remember to install the hunspell dictionaries if you don't have them already
+  (most distros should have packages for this).
+
++
+For QtWebEngine:
+
+. Make sure your versions of PyQt and Qt are 5.8 or higher.
+. Use `install_dict.py` script to install dictionaries.
+  Run the script with `-h` for the parameter description.
+. Set `spellcheck.languages` to the desired list of languages, e.g.:
+  `:set spellcheck.languages "['en-US', 'pl-PL']"`
+
+How do I use Tor with qutebrowser?::
+    Start tor on your machine, and do `:set network proxy socks://localhost:9050/`
+    in qutebrowser. Note this won't give you the same amount of fingerprinting
+    protection that the Tor Browser does, but it's useful to be able to access
+    `.onion` sites.
+
+Why does J move to the next (right) tab, and K to the previous (left) one?::
+    One reason is because https://bitbucket.org/portix/dwb[dwb] did it that way,
+    and qutebrowser's keybindings are designed to be compatible with dwb's.
+    The rationale behind it is that J is "down" in vim, and K is "up", which
+    corresponds nicely to "next"/"previous". It also makes much more sense with
+    vertical tabs (e.g. `:set tabs position left`).
+
+What's the difference between insert and passthrough mode?::
+    They are quite similar, but insert mode has some bindings (like `Ctrl-e` to
+    open an editor) while passthrough mode only has escape bound. It might also
+    be useful to rebind escape to something else in passthrough mode only, to be
+    able to send an escape keypress to the website.
+
+Why takes it longer to open an URL in qutebrowser than in chromium?::
+    When opening an URL in an existing instance the normal qutebrowser
+    Python script is started and a few PyQt libraries need to be
+    loaded until it is detected that there is an instance running
+    where the URL is then passed to. This takes some time.
+    One workaround is to use this
+    https://github.com/qutebrowser/qutebrowser/blob/master/scripts/open_url_in_instance.sh[script]
+    and place it in your $PATH with the name "qutebrowser". This
+    script passes the URL via an unix socket to qutebrowser (if its
+    running already) using socat which is much faster and starts a new
+    qutebrowser if it is not running already. Also check if you want
+    to use webengine as backend in line 17 and change it to your
+    needs.
+    
 == Troubleshooting
 
 Configuration not saved after modifying config.::
@@ -132,7 +192,7 @@ Configuration not saved after modifying config.::
 
 Unable to view flash content.::
     If you have flash installed for on your system, it's necessary to enable plugins
-    to use the flash plugin. Using the command `:set content allow-plugins true`
+    to use the flash plugin. Using the command `:set content.plugins true`
     in qutebrowser will enable plugins. Packages for flash should
     be provided for your platform or it can be obtained from
     http://get.adobe.com/flashplayer/[Adobe].
@@ -144,25 +204,17 @@ Experiencing freezing on sites like duckduckgo and youtube.::
     See https://github.com/qutebrowser/qutebrowser/issues/357[Issue #357]
     for more details.
 
-Experiencing segfaults (crashes) on Debian systems.::
-    For Debian it's highly recommended to install the `gstreamer0.10-plugins-base` package.
-    This is a workaround for a bug in Qt, it has been fixed upstream in Qt 5.4
-    More details can be found
-    https://bugs.webkit.org/show_bug.cgi?id=119951[here].
-
-Segfaults on Facebook, Medium, Amazon, ...::
-    If you are on a Debian or Ubuntu based system, you might experience some crashes
-    visiting these sites. This is caused by various bugs in Qt which have been
-    fixed in Qt 5.4. However Debian and Ubuntu are slow to adopt or upgrade
-    some packages. On Debian Jessie, it's recommended to use the experimental
-    repos as described in https://github.com/qutebrowser/qutebrowser/blob/master/INSTALL.asciidoc#on-debian--ubuntu[INSTALL].
+When using QtWebEngine, qutebrowser reports "Render Process Crashed" and the console prints a traceback on Gentoo Linux or another Source-Based Distro::
+    As stated in https://gcc.gnu.org/gcc-6/changes.html[GCC's Website] GCC 6 has introduced some optimizations that could break non-conforming codebases, like QtWebEngine. +
+    As a workaround, you can disable the nullpointer check optimization by adding the -fno-delete-null-pointer-checks flag while compiling. +
+    On gentoo, you just need to add it into your make.conf, like this: +
+    
+    CFLAGS="... -fno-delete-null-pointer-checks"
+    CXXFLAGS="... -fno-delete-null-pointer-checks"
 +
-Since Ubuntu Trusty (using Qt 5.2.1),
-https://bugreports.qt.io/browse/QTBUG-42417?jql=component%20%3D%20WebKit%20and%20resolution%20%3D%20Done%20and%20fixVersion%20in%20(5.3.0%2C%20%225.3.0%20Alpha%22%2C%20%225.3.0%20Beta1%22%2C%20%225.3.0%20RC1%22%2C%205.3.1%2C%205.3.2%2C%205.4.0%2C%20%225.4.0%20Alpha%22%2C%20%225.4.0%20Beta%22%2C%20%225.4.0%20RC%22)%20and%20priority%20in%20(%22P2%3A%20Important%22%2C%20%22P1%3A%20Critical%22%2C%20%22P0%3A%20Blocker%22)[over
-70 important bugs] have been fixed in QtWebKit. For Debian Jessie (using Qt 5.3.2)
-it's still
-https://bugreports.qt.io/browse/QTBUG-42417?jql=component%20%3D%20WebKit%20and%20resolution%20%3D%20Done%20and%20fixVersion%20in%20(5.4.0%2C%20%225.4.0%20Alpha%22%2C%20%225.4.0%20Beta%22%2C%20%225.4.0%20RC%22)%20and%20priority%20in%20(%22P2%3A%20Important%22%2C%20%22P1%3A%20Critical%22%2C%20%22P0%3A%20Blocker%22)[nearly
-20 important bugs].
+And then re-emerging qtwebengine with: +
+    
+    emerge -1 qtwebengine
 
 My issue is not listed.::
     If you experience any segfaults or crashes, you can report the issue in

doc/help/commands.asciidoc

@@ -1,5 +1,5 @@
 // DO NOT EDIT THIS FILE DIRECTLY!
-// It is autogenerated from docstrings by running:
+// It is autogenerated by running:
 //   $ python3 scripts/dev/src2asciidoc.py
 
 = Commands
@@ -31,6 +31,12 @@ It is possible to run or bind multiple commands by separating them with `;;`.
 |<<bookmark-load,bookmark-load>>|Load a bookmark.
 |<<buffer,buffer>>|Select tab by index or url/title best match.
 |<<close,close>>|Close the current window.
+|<<config-clear,config-clear>>|Set all settings back to their default.
+|<<config-cycle,config-cycle>>|Cycle an option between multiple values.
+|<<config-edit,config-edit>>|Open the config.py file in the editor.
+|<<config-source,config-source>>|Read a config.py file.
+|<<config-unset,config-unset>>|Unset an option.
+|<<config-write-py,config-write-py>>|Write the current configuration to a config.py file.
 |<<download,download>>|Download a given URL, or current page if no URL given.
 |<<download-cancel,download-cancel>>|Cancel the last/[count]th download.
 |<<download-clear,download-clear>>|Remove all finished downloads from the list.
@@ -82,12 +88,13 @@ It is possible to run or bind multiple commands by separating them with `;;`.
 |<<tab-move,tab-move>>|Move the current tab according to the argument and [count].
 |<<tab-next,tab-next>>|Switch to the next tab, or switch [count] tabs forward.
 |<<tab-only,tab-only>>|Close all tabs except for the current one.
+|<<tab-pin,tab-pin>>|Pin/Unpin the current/[count]th tab.
 |<<tab-prev,tab-prev>>|Switch to the previous tab, or switch [count] tabs back.
 |<<unbind,unbind>>|Unbind a keychain.
-|<<undo,undo>>|Re-open a closed tab (optionally skipping [count] closed tabs).
+|<<undo,undo>>|Re-open a closed tab.
+|<<version,version>>|Show version information.
 |<<view-source,view-source>>|Show the source of the current page in a new tab.
 |<<window-only,window-only>>|Close all windows except for the current one.
-|<<wq,wq>>|Save open pages and quit.
 |<<yank,yank>>|Yank something to the clipboard or primary selection.
 |<<zoom,zoom>>|Set the zoom level for the current tab.
 |<<zoom-in,zoom-in>>|Increase the zoom level for the current tab.
@@ -115,7 +122,7 @@ How many pages to go back.
 
 [[bind]]
 === bind
-Syntax: +:bind [*--mode* 'mode'] [*--force*] 'key' ['command']+
+Syntax: +:bind [*--mode* 'mode'] [*--default*] 'key' ['command']+
 
 Bind a key to a command.
 
@@ -125,9 +132,10 @@ Bind a key to a command.
 
 
 ==== optional arguments
-* +*-m*+, +*--mode*+: A comma-separated list of modes to bind the key in (default: `normal`).
+* +*-m*+, +*--mode*+: A comma-separated list of modes to bind the key in (default: `normal`). See `:help bindings.commands` for the
+ available modes.
 
-* +*-f*+, +*--force*+: Rebind the key if it is already bound.
+* +*-d*+, +*--default*+: If given, restore a default binding.
 
 ==== note
 * This command does not split arguments after the last argument and handles quotes literally.
@@ -183,20 +191,96 @@ Load a bookmark.
 
 [[buffer]]
 === buffer
-Syntax: +:buffer 'index'+
+Syntax: +:buffer ['index']+
 
 Select tab by index or url/title best match.
 
-Focuses window if necessary.
+Focuses window if necessary when index is given. If both index and count are given, use count.
 
 ==== positional arguments
 * +'index'+: The [win_id/]index of the tab to focus. Or a substring in which case the closest match will be focused.
 
 
+==== count
+The tab index to focus, starting with 1.
+
 [[close]]
 === close
 Close the current window.
 
+[[config-clear]]
+=== config-clear
+Syntax: +:config-clear [*--save*]+
+
+Set all settings back to their default.
+
+==== optional arguments
+* +*-s*+, +*--save*+: If given, all configuration in autoconfig.yml is also removed.
+
+
+[[config-cycle]]
+=== config-cycle
+Syntax: +:config-cycle [*--temp*] [*--print*] 'option' ['values' ['values' ...]]+
+
+Cycle an option between multiple values.
+
+==== positional arguments
+* +'option'+: The name of the option.
+* +'values'+: The values to cycle through.
+
+==== optional arguments
+* +*-t*+, +*--temp*+: Set value temporarily until qutebrowser is closed.
+* +*-p*+, +*--print*+: Print the value after setting.
+
+[[config-edit]]
+=== config-edit
+Syntax: +:config-edit [*--no-source*]+
+
+Open the config.py file in the editor.
+
+==== optional arguments
+* +*-n*+, +*--no-source*+: Don't re-source the config file after editing.
+
+[[config-source]]
+=== config-source
+Syntax: +:config-source [*--clear*] ['filename']+
+
+Read a config.py file.
+
+==== positional arguments
+* +'filename'+: The file to load. If not given, loads the default config.py.
+
+
+==== optional arguments
+* +*-c*+, +*--clear*+: Clear current settings first.
+
+[[config-unset]]
+=== config-unset
+Syntax: +:config-unset [*--temp*] 'option'+
+
+Unset an option.
+
+This sets an option back to its default and removes it from autoconfig.yml.
+
+==== positional arguments
+* +'option'+: The name of the option.
+
+==== optional arguments
+* +*-t*+, +*--temp*+: Don't touch autoconfig.yml.
+
+[[config-write-py]]
+=== config-write-py
+Syntax: +:config-write-py [*--force*] [*--defaults*] ['filename']+
+
+Write the current configuration to a config.py file.
+
+==== positional arguments
+* +'filename'+: The file to write to, or not given for the default config.py.
+
+==== optional arguments
+* +*-f*+, +*--force*+: Force overwriting existing files.
+* +*-d*+, +*--defaults*+: Write the defaults instead of values configured via :set.
+
 [[download]]
 === download
 Syntax: +:download [*--mhtml*] [*--dest* 'dest'] ['url'] ['dest-old']+
@@ -280,7 +364,7 @@ Syntax: +:edit-url [*--bg*] [*--tab*] [*--window*] ['url']+
 
 Navigate to a url formed in an external editor.
 
-The editor which should be launched can be configured via the `general -> editor` config option.
+The editor which should be launched can be configured via the `editor.command` config option.
 
 ==== positional arguments
 * +'url'+: URL to edit; defaults to the current page url.
@@ -337,7 +421,7 @@ Show help about a command or setting.
 * +'topic'+: The topic to show help for. 
 
  - :__command__ for commands.
- - __section__\->__option__ for settings.
+ - __section__.__option__ for settings.
 
 
 ==== optional arguments
@@ -367,7 +451,7 @@ Start hinting.
  - `normal`: Open the link.
  - `current`: Open the link in the current tab.
  - `tab`: Open the link in a new tab (honoring the
- background-tabs setting).
+ `tabs.background_tabs` setting).
  - `tab-fg`: Open the link in a new foreground tab.
  - `tab-bg`: Open the link in a new background tab.
  - `window`: Open the link in a new window.
@@ -401,13 +485,13 @@ Start hinting.
 
 
 ==== optional arguments
-* +*-r*+, +*--rapid*+: Whether to do rapid hinting. This is only possible with targets `tab` (with background-tabs=true), `tab-bg`,
+* +*-r*+, +*--rapid*+: Whether to do rapid hinting. This is only possible with targets `tab` (with `tabs.background_tabs=true`), `tab-bg`,
  `window`, `run`, `hover`, `userscript` and `spawn`.
 
 * +*-m*+, +*--mode*+: The hinting mode to use. 
 
  - `number`: Use numeric hints.
- - `letter`: Use the chars in the hints->chars settings.
+ - `letter`: Use the chars in the hints.chars setting.
  - `word`: Use hint words based on the html elements and the
  extra words.
  
@@ -544,7 +628,8 @@ For `increment` and `decrement`, the number to change the URL by. For `up`, the
 
 [[open]]
 === open
-Syntax: +:open [*--implicit*] [*--bg*] [*--tab*] [*--window*] ['url']+
+Syntax: +:open [*--related*] [*--bg*] [*--tab*] [*--window*] [*--secure*] [*--private*]
+     ['url']+
 
 Open a URL in the current/[count]th tab.
 
@@ -554,11 +639,13 @@ If the URL contains newlines, each line gets opened in its own tab.
 * +'url'+: The URL to open.
 
 ==== optional arguments
-* +*-i*+, +*--implicit*+: If opening a new tab, treat the tab as implicit (like clicking on a link).
+* +*-r*+, +*--related*+: If opening a new tab, position the tab as related to the current one (like clicking on a link).
 
 * +*-b*+, +*--bg*+: Open in a new background tab.
 * +*-t*+, +*--tab*+: Open in a new tab.
 * +*-w*+, +*--window*+: Open in a new window.
+* +*-s*+, +*--secure*+: Force HTTPS.
+* +*-p*+, +*--private*+: Open a new window in private browsing mode.
 
 ==== count
 The tab index to open the URL in.
@@ -628,8 +715,17 @@ Save the current page as a quickmark.
 
 [[quit]]
 === quit
+Syntax: +:quit [*--save*] ['session']+
+
 Quit qutebrowser.
 
+==== positional arguments
+* +'session'+: The name of the session to save.
+
+==== optional arguments
+* +*-s*+, +*--save*+: When given, save the open windows even if auto_save.session is turned off.
+
+
 [[record-macro]]
 === record-macro
 Syntax: +:record-macro ['register']+
@@ -742,12 +838,13 @@ Load a session.
 [[session-save]]
 === session-save
 Syntax: +:session-save [*--current*] [*--quiet*] [*--force*] [*--only-active-window*]
+             [*--with-private*]
              ['name']+
 
 Save a session.
 
 ==== positional arguments
-* +'name'+: The name of the session. If not given, the session configured in general -> session-default-name is saved.
+* +'name'+: The name of the session. If not given, the session configured in session_default_name is saved.
 
 
 ==== optional arguments
@@ -755,27 +852,27 @@ Save a session.
 * +*-q*+, +*--quiet*+: Don't show confirmation message.
 * +*-f*+, +*--force*+: Force saving internal sessions (starting with an underline).
 * +*-o*+, +*--only-active-window*+: Saves only tabs of the currently active window.
+* +*-p*+, +*--with-private*+: Include private windows.
 
 [[set]]
 === set
-Syntax: +:set [*--temp*] [*--print*] ['section'] ['option'] ['values' ['values' ...]]+
+Syntax: +:set [*--temp*] [*--print*] ['option'] ['value']+
 
 Set an option.
 
-If the option name ends with '?', the value of the option is shown instead. If the option name ends with '!' and it is a boolean value, toggle it.
+If the option name ends with '?', the value of the option is shown instead.
 
 ==== positional arguments
-* +'section'+: The section where the option is in.
 * +'option'+: The name of the option.
-* +'values'+: The value to set, or the values to cycle through.
+* +'value'+: The value to set.
 
 ==== optional arguments
-* +*-t*+, +*--temp*+: Set value temporarily.
+* +*-t*+, +*--temp*+: Set value temporarily until qutebrowser is closed.
 * +*-p*+, +*--print*+: Print the value after setting.
 
 [[set-cmd-text]]
 === set-cmd-text
-Syntax: +:set-cmd-text [*--space*] [*--append*] 'text'+
+Syntax: +:set-cmd-text [*--space*] [*--append*] [*--run-on-count*] 'text'+
 
 Preset the statusbar to some text.
 
@@ -785,6 +882,11 @@ Preset the statusbar to some text.
 ==== optional arguments
 * +*-s*+, +*--space*+: If given, a space is added to the end.
 * +*-a*+, +*--append*+: If given, the text is appended to the current text.
+* +*-r*+, +*--run-on-count*+: If given with a count, the command is run with the given count rather than setting the command text.
+
+
+==== count
+The count if given.
 
 ==== note
 * This command does not split arguments after the last argument and handles quotes literally.
@@ -830,15 +932,16 @@ Duplicate the current tab.
 
 [[tab-close]]
 === tab-close
-Syntax: +:tab-close [*--prev*] [*--next*] [*--opposite*]+
+Syntax: +:tab-close [*--prev*] [*--next*] [*--opposite*] [*--force*]+
 
 Close the current/[count]th tab.
 
 ==== optional arguments
 * +*-p*+, +*--prev*+: Force selecting the tab before the current tab.
 * +*-n*+, +*--next*+: Force selecting the tab after the current tab.
-* +*-o*+, +*--opposite*+: Force selecting the tab in the opposite direction of what's configured in 'tabs->select-on-remove'.
+* +*-o*+, +*--opposite*+: Force selecting the tab in the opposite direction of what's configured in 'tabs.select_on_remove'.
 
+* +*-f*+, +*--force*+: Avoid confirmation for pinned tabs.
 
 ==== count
 The tab index to close
@@ -891,13 +994,23 @@ How many tabs to switch forward.
 
 [[tab-only]]
 === tab-only
-Syntax: +:tab-only [*--prev*] [*--next*]+
+Syntax: +:tab-only [*--prev*] [*--next*] [*--force*]+
 
 Close all tabs except for the current one.
 
 ==== optional arguments
 * +*-p*+, +*--prev*+: Keep tabs before the current.
 * +*-n*+, +*--next*+: Keep tabs after the current.
+* +*-f*+, +*--force*+: Avoid confirmation for pinned tabs.
+
+[[tab-pin]]
+=== tab-pin
+Pin/Unpin the current/[count]th tab.
+
+Pinning a tab shrinks it to the size of its title text. Attempting to close a pinned tab will cause a confirmation, unless --force is passed.
+
+==== count
+The tab index to pin or unpin
 
 [[tab-prev]]
 === tab-prev
@@ -908,18 +1021,24 @@ How many tabs to switch back.
 
 [[unbind]]
 === unbind
-Syntax: +:unbind 'key' ['mode']+
+Syntax: +:unbind [*--mode* 'mode'] 'key'+
 
 Unbind a keychain.
 
 ==== positional arguments
 * +'key'+: The keychain or special key (inside <...>) to unbind.
-* +'mode'+: A comma-separated list of modes to unbind the key in (default: `normal`).
+
+==== optional arguments
+* +*-m*+, +*--mode*+: A mode to unbind the key in (default: `normal`). See `:help bindings.commands` for the available modes.
 
 
 [[undo]]
 === undo
-Re-open a closed tab (optionally skipping [count] closed tabs).
+Re-open a closed tab.
+
+[[version]]
+=== version
+Show version information.
 
 [[view-source]]
 === view-source
@@ -929,15 +1048,6 @@ Show the source of the current page in a new tab.
 === window-only
 Close all windows except for the current one.
 
-[[wq]]
-=== wq
-Syntax: +:wq ['name']+
-
-Save open pages and quit.
-
-==== positional arguments
-* +'name'+: The name of the session.
-
 [[yank]]
 === yank
 Syntax: +:yank [*--sel*] [*--keep*] ['what']+
@@ -995,6 +1105,7 @@ How many steps to zoom out.
 |==============
 |Command|Description
 |<<clear-keychain,clear-keychain>>|Clear the currently entered key chain.
+|<<clear-messages,clear-messages>>|Clear all message notifications.
 |<<click-element,click-element>>|Click the element matching the given filter.
 |<<command-accept,command-accept>>|Execute the command currently in the commandline.
 |<<command-history-next,command-history-next>>|Go forward in the commandline history.
@@ -1025,6 +1136,7 @@ How many steps to zoom out.
 |<<move-to-start-of-line,move-to-start-of-line>>|Move the cursor or selection to the start of the line.
 |<<move-to-start-of-next-block,move-to-start-of-next-block>>|Move the cursor or selection to the start of next block.
 |<<move-to-start-of-prev-block,move-to-start-of-prev-block>>|Move the cursor or selection to the start of previous block.
+|<<nop,nop>>|Do nothing.
 |<<open-editor,open-editor>>|Open an external editor with the currently selected form field.
 |<<prompt-accept,prompt-accept>>|Accept the current prompt.
 |<<prompt-item-focus,prompt-item-focus>>|Shift the focus of the prompt file completion menu to another item.
@@ -1048,8 +1160,8 @@ How many steps to zoom out.
 |<<run-with-count,run-with-count>>|Run a command with the given count.
 |<<scroll,scroll>>|Scroll the current tab in the given direction.
 |<<scroll-page,scroll-page>>|Scroll the frame page-wise.
-|<<scroll-perc,scroll-perc>>|Scroll to a specific percentage of the page.
 |<<scroll-px,scroll-px>>|Scroll the current tab by 'count * dx/dy' pixels.
+|<<scroll-to-perc,scroll-to-perc>>|Scroll to a specific percentage of the page.
 |<<search-next,search-next>>|Continue the search to the ([count]th) next term.
 |<<search-prev,search-prev>>|Continue the search to the ([count]th) previous term.
 |<<set-mark,set-mark>>|Set a mark at the current scroll position in the current tab.
@@ -1059,6 +1171,10 @@ How many steps to zoom out.
 === clear-keychain
 Clear the currently entered key chain.
 
+[[clear-messages]]
+=== clear-messages
+Clear all message notifications.
+
 [[click-element]]
 === click-element
 Syntax: +:click-element [*--target* 'target'] [*--force-event*] 'filter' 'value'+
@@ -1268,11 +1384,15 @@ Move the cursor or selection to the start of previous block.
 ==== count
 How many blocks to move.
 
+[[nop]]
+=== nop
+Do nothing.
+
 [[open-editor]]
 === open-editor
 Open an external editor with the currently selected form field.
 
-The editor which should be launched can be configured via the `general -> editor` config option.
+The editor which should be launched can be configured via the `editor.command` config option.
 
 [[prompt-accept]]
 === prompt-accept
@@ -1433,6 +1553,8 @@ Syntax: +:scroll 'direction'+
 
 Scroll the current tab in the given direction.
 
+Note you can use `:run-with-count` to have a keybinding with a bigger scroll increment.
+
 ==== positional arguments
 * +'direction'+: In which direction to scroll (up/down/left/right/top/bottom).
 
@@ -1459,9 +1581,22 @@ Scroll the frame page-wise.
 ==== count
 multiplier
 
-[[scroll-perc]]
-=== scroll-perc
-Syntax: +:scroll-perc [*--horizontal*] ['perc']+
+[[scroll-px]]
+=== scroll-px
+Syntax: +:scroll-px 'dx' 'dy'+
+
+Scroll the current tab by 'count * dx/dy' pixels.
+
+==== positional arguments
+* +'dx'+: How much to scroll in x-direction.
+* +'dy'+: How much to scroll in y-direction.
+
+==== count
+multiplier
+
+[[scroll-to-perc]]
+=== scroll-to-perc
+Syntax: +:scroll-to-perc [*--horizontal*] ['perc']+
 
 Scroll to a specific percentage of the page.
 
@@ -1476,19 +1611,6 @@ The percentage can be given either as argument or as count. If no percentage is
 ==== count
 Percentage to scroll.
 
-[[scroll-px]]
-=== scroll-px
-Syntax: +:scroll-px 'dx' 'dy'+
-
-Scroll the current tab by 'count * dx/dy' pixels.
-
-==== positional arguments
-* +'dx'+: How much to scroll in x-direction.
-* +'dy'+: How much to scroll in y-direction.
-
-==== count
-multiplier
-
 [[search-next]]
 === search-next
 Continue the search to the ([count]th) next term.
@@ -1529,6 +1651,7 @@ These commands are mainly intended for debugging. They are hidden if qutebrowser
 |<<debug-clear-ssl-errors,debug-clear-ssl-errors>>|Clear remembered SSL error answers.
 |<<debug-console,debug-console>>|Show the debugging console.
 |<<debug-crash,debug-crash>>|Crash for debugging purposes.
+|<<debug-dump-history,debug-dump-history>>|Dump the history to a file in the old pre-SQL format.
 |<<debug-dump-page,debug-dump-page>>|Dump the current page's content to a file.
 |<<debug-log-capacity,debug-log-capacity>>|Change the number of log lines to be stored in RAM.
 |<<debug-log-filter,debug-log-filter>>|Change the log filter for console logging.
@@ -1563,6 +1686,15 @@ Crash for debugging purposes.
 ==== positional arguments
 * +'typ'+: either 'exception' or 'segfault'.
 
+[[debug-dump-history]]
+=== debug-dump-history
+Syntax: +:debug-dump-history 'dest'+
+
+Dump the history to a file in the old pre-SQL format.
+
+==== positional arguments
+* +'dest'+: Where to write the file to.
+
 [[debug-dump-page]]
 === debug-dump-page
 Syntax: +:debug-dump-page [*--plain*] 'dest'+
@@ -1591,7 +1723,8 @@ Syntax: +:debug-log-filter 'filters'+
 Change the log filter for console logging.
 
 ==== positional arguments
-* +'filters'+: A comma separated list of logger names.
+* +'filters'+: A comma separated list of logger names. Can also be "none" to clear any existing filters.
+
 
 [[debug-log-level]]
 === debug-log-level
@@ -1646,7 +1779,7 @@ Syntax: +:debug-webaction 'action'+
 
 Execute a webaction.
 
-See http://doc.qt.io/qt-5/qwebpage.html#WebAction-enum for the available actions.
+Available actions: http://doc.qt.io/archives/qt-5.5/qwebpage.html#WebAction-enum (WebKit) http://doc.qt.io/qt-5/qwebenginepage.html#WebAction-enum (WebEngine)
 
 ==== positional arguments
 * +'action'+: The action to execute, e.g. MoveToNextChar.

doc/help/configuring.asciidoc

@@ -0,0 +1,369 @@
+Configuring qutebrowser
+=======================
+
+IMPORTANT: qutebrowser's configuration system was completely rewritten in
+September 2017. This information is not applicable to older releases, and older
+information elsewhere might be outdated. **If you had an old configuration
+around and upgraded, this page will automatically open once**. To view it at a
+later time, use the `:help` command.
+
+Migrating older configurations
+------------------------------
+
+qutebrowser does no automatic migration for the new configuration. However,
+there's a special link:qute://configdiff/old[configdiff] page in qutebrowser,
+which will show you the changes you did in your old configuration, compared to
+the old defaults.
+
+Other changes in default settings:
+
+- `<Up>` and `<Down>` in the completion now navigate through command history
+  instead of selecting completion items. You can get back the old behavior by
+  doing:
++
+----
+:bind -f -m command <Up> completion-item-focus prev
+:bind -f -m command <Down> completion-item-focus next
+----
+
+- The default for `completion.web_history_max_items` is now set to `-1`, showing
+  an unlimited number of items in the completion for `:open` as the new
+  sqlite-based completion is much faster. If the `:open` completion is too slow
+  on your machine, set an appropriate limit again.
+
+Configuring qutebrowser via the user interface
+----------------------------------------------
+
+The easy (but less flexible) way to configure qutebrowser is using its user
+interface or command line. Changes you make this way are immediately active
+(with the exception of a few settings, where this is pointed out in the
+documentation) and are persisted in an `autoconfig.yml` file.
+
+The `autoconfig.yml` file is located in the "config" folder listed on the
+link:qute://version[] page. On macOS, the "auto config" folder is used, which is
+different from where hand-written config files are kept.
+
+However, **do not** edit `autoconfig.yml` by hand. Instead, see the next
+section.
+
+If you want to customize many settings, you can open the link:qute://settings[]
+page by running `:set` without any arguments, where all settings are listed and
+customizable.
+
+Using the link:commands.html#set[`:set`] command and command completion, you
+can quickly set settings interactively, for example `:set tabs.position left`.
+
+To get more help about a setting, use e.g. `:help tabs.position`.
+
+To bind and unbind keys, you can use the link:commands.html#bind[`:bind`] and
+link:commands.html#unbind[`:unbind`] commands:
+
+- Binding the key chain `,v` to the `:spawn mpv {url}` command:
+  `:bind ,v spawn mpv {url}`
+- Unbinding the same key chain: `:unbind ,v`
+
+Key chains starting with a comma are ideal for custom bindings, as the comma key
+will never be used in a default keybinding.
+
+See the help pages linked above (or `:help :bind`, `:help :unbind`) for more
+information.
+
+Other useful commands for config manipulation are
+link:commands.html#config-unset[`:config-unset`] to reset a value to its default,
+link:commands.html#config-clear[`:config-clear`] to reset the entire configuration,
+and link:commands.html#config-cycle[`:config-cycle`] to cycle a setting between
+different values.
+
+Configuring qutebrowser via config.py
+-------------------------------------
+
+For more powerful configuration possibilities, you can create a `config.py`
+file. Since it's a Python file, you have much more flexibility for
+configuration. Note that qutebrowser will never touch this file - this means
+you'll be responsible for updating it when upgrading to a newer qutebrowser
+version.
+
+You can run `:config-edit` inside qutebrowser to open the file in your editor,
+`:config-source` to reload the file (`:config-edit` does this automatically), or
+`:config-write-py --defaults` to write a template file to work with.
+
+The file should be located in the "config" location listed on
+link:qute://version[], which is typically `~/.config/qutebrowser/config.py` on
+Linux, `~/.qutebrowser/config.py` on macOS, and
+`%APPDATA%/qutebrowser/config.py` on Windows.
+
+Two global objects are pre-defined when running `config.py`: `c` and `config`.
+
+Changing settings
+~~~~~~~~~~~~~~~~~
+
+While you can set settings using the `config.set()` method (which is explained
+in the next section), it's easier to use the `c` shorthand object to easily set
+settings like this:
+
+.config.py:
+[source,python]
+----
+c.tabs.position = "left"
+c.completion.shrink = True
+----
+
+Note that qutebrowser does some Python magic so it's able to warn you about
+mistyped config settings. As an example, if you do `c.tabs.possition = "left"`,
+you'll get an error when starting.
+
+See the link:settings.html[settings help page] for all available settings. The
+accepted values depend on the type of the option. Commonly used are:
+
+- Strings: `c.tabs.position = "left"`
+- Booleans: `c.completion.shrink = True`
+- Integers: `c.messages.timeout = 5000`
+- Dictionaries:
+  * `c.headers.custom = {'X-Hello': 'World', 'X-Awesome': 'yes'}` to override
+    any other values in the dictionary.
+  * `c.aliases['foo'] = 'message-info foo'` to add a single value.
+- Lists:
+  * `c.url.start_pages = ["https://www.qutebrowser.org/"]` to override any
+    previous elements.
+  * `c.url.start_pages.append("https://www.python.org/")` to add a new value.
+
+Any other config types (e.g. a color) are specified as a string. The only
+exception is the `Regex` type, which can take either a string (with an `r`
+prefix to preserve backslashes) or a Python regex object:
+
+- `c.hints.next_regexes.append(r'\bvor\b')`
+- `c.hints.prev_regexes.append(re.compile(r'\bzurück\b'))`
+
+If you want to read a setting, you can use the `c` object to do so as well:
+`c.colors.tabs.even.bg = c.colors.tabs.odd.bg`.
+
+
+Using strings for setting names
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you want to set settings based on their name as a string, use the
+`config.set` method:
+
+.config.py:
+[source,python]
+----
+# Equivalent to:
+# c.content.javascript.enabled = False
+config.set('content.javascript.enabled', False)
+----
+
+To read a setting, use the `config.get` method:
+
+[source,python]
+----
+# Equivalent to:
+# color = c.colors.completion.fg
+color = config.get('colors.completion.fg')
+----
+
+Binding keys
+~~~~~~~~~~~~
+
+While it's possible to change the `bindings.commands` setting to bind keys, it's
+preferred to use the `config.bind` command. Doing so ensures the commands are
+valid and normalizes different expressions which map to the same key.
+
+For details on how to specify keys and the available modes, see the
+link:settings.html#bindings.commands[documentation] for the `bindings.commands`
+setting.
+
+To bind a key:
+
+.config.py:
+[source,python]
+----
+config.bind('<Ctrl-v>', 'spawn mpv {url}')
+----
+
+To bind a key in a mode other than `'normal'`, add a `mode` argument:
+
+[source,python]
+----
+config.bind('<Ctrl-y>', 'prompt-yes', mode='prompt')
+----
+
+To unbind a key (either a key which has been bound before, or a default binding):
+
+[source,python]
+----
+config.unbind('<Ctrl-v>', mode='normal')
+----
+
+To bind keys without modifiers, specify a key chain to bind as a string. Key
+chains starting with a comma are ideal for custom bindings, as the comma key
+will never be used in a default keybinding.
+
+[source,python]
+----
+config.bind(',v', 'spawn mpv {url}')
+----
+
+To suppress loading of any default keybindings, you can set
+`c.bindings.default = {}`.
+
+Loading `autoconfig.yml`
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+By default, all customization done via `:set`, `:bind` and `:unbind` is
+temporary as soon as a `config.py` exists. The settings done that way are always
+saved in the `autoconfig.yml` file, but you'll need to explicitly load it in
+your `config.py` by doing:
+
+.config.py:
+[source,python]
+----
+config.load_autoconfig()
+----
+
+If you do so at the top of your file, your `config.py` settings will take
+precedence as they overwrite the settings done in `autoconfig.yml`.
+
+Importing other modules
+~~~~~~~~~~~~~~~~~~~~~~~
+
+You can import any module from the
+https://docs.python.org/3/library/index.html[Python standard library] (e.g.
+`import os.path`), as well as any module installed in the environment
+qutebrowser is run with.
+
+If you have an `utils.py` file in your qutebrowser config folder, you can import
+that via `import utils` as well.
+
+While it's in some cases possible to import code from the qutebrowser
+installation, doing so is unsupported and discouraged.
+
+Getting the config directory
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you need to get the qutebrowser config directory, you can do so by reading
+`config.configdir`. Similarily, you can get the qutebrowser data directory via
+`config.datadir`.
+
+This gives you a https://docs.python.org/3/library/pathlib.html[`pathlib.Path`
+object], on which you can use `/` to add more directory parts, or `str(...)` to
+get a string:
+
+.config.py:
+[source,python]
+----
+print(str(config.configdir / 'config.py')
+----
+
+Handling errors
+~~~~~~~~~~~~~~~
+
+If there are errors in your `config.py`, qutebrowser will try to apply as much
+of it as possible, and show an error dialog before starting.
+
+qutebrowser tries to display errors which are easy to understand even for people
+who are not used to writing Python. If you see a config error which you find
+confusing or you think qutebrowser could handle better, please
+https://github.com/qutebrowser/qutebrowser/issues[open an issue]!
+
+Recipes
+~~~~~~~
+
+Reading a YAML file
+^^^^^^^^^^^^^^^^^^^
+
+To read a YAML config like this:
+
+.config.yml:
+----
+tabs.position: left
+tabs.show: switching
+----
+
+You can use:
+
+.config.py:
+[source,python]
+----
+import yaml
+
+with (config.configdir / 'config.yml').open() as f:
+    yaml_data = yaml.load(f)
+
+for k, v in yaml_data.items():
+    config.set(k, v)
+----
+
+Reading a nested YAML file
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To read a YAML file with nested values like this:
+
+.colors.yml:
+----
+colors:
+  statusbar:
+    normal:
+      bg: lime
+      fg: black
+    url:
+      fg: red
+----
+
+You can use:
+
+.config.py:
+[source,python]
+----
+import yaml
+
+with (config.configdir / 'colors.yml').open() as f:
+    yaml_data = yaml.load(f)
+
+def dict_attrs(obj, path=''):
+    if isinstance(obj, dict):
+        for k, v in obj.items():
+            yield from dict_attrs(v, '{}.{}'.format(path, k) if path else k)
+    else:
+        yield path, obj
+
+for k, v in dict_attrs(yaml_data):
+    config.set(k, v)
+----
+
+Note that this won't work for values which are dictionaries.
+
+Binding chained commands
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you have a lot of chained commands you want to bind, you can write a helper
+to do so:
+
+[source,python]
+----
+def bind_chained(key, *commands):
+    config.bind(key, ' ;; '.join(commands))
+
+bind_chained('<Escape>', 'clear-keychain', 'search')
+----
+
+Avoiding flake8 errors
+^^^^^^^^^^^^^^^^^^^^^^
+
+If you use an editor with flake8 integration which complains about `c` and `config` being undefined, you can use:
+
+[source,python]
+----
+c = c  # noqa: F821
+config = config  # noqa: F821
+----
+
+For type annotation support (note that those imports aren't guaranteed to be
+stable across qutebrowser versions):
+
+[source,python]
+----
+from qutebrowser.config.configfiles import ConfigAPI  # noqa: F401
+from qutebrowser.config.config import ConfigContainer  # noqa: F401
+config = config  # type: ConfigAPI # noqa: F821
+c = c  # type: ConfigContainer # noqa: F821
+----

doc/help/index.asciidoc

@@ -7,12 +7,13 @@ Documentation
 The following help pages are currently available:
 
 * link:../quickstart.html[Quick start guide]
-* link:../../FAQ.html[Frequently asked questions]
-* link:../../CHANGELOG.html[Change Log]
+* link:../faq.html[Frequently asked questions]
+* link:../changelog.html[Change Log]
 * link:commands.html[Documentation of commands]
+* link:configuring.html[Configuring qutebrowser]
 * link:settings.html[Documentation of settings]
 * link:../userscripts.html[How to write userscripts]
-* link:../../CONTRIBUTING.html[Contributing to qutebrowser]
+* link:../contributing.html[Contributing to qutebrowser]
 
 Getting help
 ------------

doc/install.asciidoc

@@ -0,0 +1,380 @@
+Installing qutebrowser
+======================
+
+toc::[]
+
+NOTE: qutebrowser recently had some bigger dependency changes for v1.0.0, which
+means those instructions might be out of date in some places.
+https://github.com/qutebrowser/qutebrowser/blob/master/doc/contributing.asciidoc[Please help]
+updating them if you notice something being broken!
+
+On Debian / Ubuntu
+------------------
+
+How to install qutebrowser depends a lot on the version of Debian/Ubuntu you're
+running.
+
+Debian Jessie / Ubuntu 14.04 LTS / Linux Mint < 18
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Those distributions only have Python 3.4 and a too old Qt version available,
+while qutebrowser requires Python 3.5 and Qt 5.7.1 or newer.
+
+It should be possible to install Python 3.5 e.g. from the
+https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa[deadsnakes PPA] or via_ipca
+https://github.com/pyenv/pyenv[pyenv], but nobody tried that yet.
+
+If you get qutebrowser running on those distributions, please
+https://github.com/qutebrowser/qutebrowser/blob/master/doc/contributing.asciidoc[contribute]
+to update this documentation!
+
+Ubuntu 16.04 LTS / Linux Mint 18
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Ubuntu 16.04 doesn't come with an up-to-date engine (a new enough QtWebKit, or
+QtWebEngine). However, it comes with Python 3.5, so you can
+<<tox,install qutebrowser via tox>>.
+
+Debian Stretch / Ubuntu 17.04 and newer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Those versions come with QtWebEngine in the repositories. This makes it possible
+to install qutebrowser via the Debian package.
+
+Install the dependencies via apt-get:
+
+----
+# apt install python-tox python3-{lxml,pyqt5,sip,jinja2,pygments,yaml,attr} python3-pyqt5.qt{webengine,quick,opengl,sql} libqt5sql5-sqlite
+----
+
+Get the qutebrowser package from the
+https://github.com/qutebrowser/qutebrowser/releases[release page] and download
+the https://qutebrowser.org/python3-pypeg2_2.15.2-1_all.deb[PyPEG2 package].
+
+Install the packages:
+
+----
+# dpkg -i python3-pypeg2_*_all.deb
+# dpkg -i qutebrowser_*_all.deb
+----
+
+Some additional hints:
+
+- Alternatively, you can <<tox,install qutebrowser via tox>> to get a newer
+  QtWebEngine version.
+- If running from git, run the following to generate the documentation for the
+  `:help` command:
++
+----
+# apt-get install --no-install-recommends asciidoc source-highlight
+$ python3 scripts/asciidoc2html.py
+----
+
+- If you prefer using QtWebKit, there's an up-to-date version available in
+  Debian experimental, or from http://repo.paretje.be/unstable/[this repository]
+  for Debian Stretch.
+- If video or sound don't work with QtWebKit, try installing the gstreamer plugins:
++
+----
+# apt-get install gstreamer1.0-plugins-{bad,base,good,ugly}
+----
+
+On Fedora
+---------
+
+qutebrowser is available in the official repositories for Fedora 22 and newer.
+
+----
+# dnf install qutebrowser
+----
+
+It's also recommended to install `python3-qt5-webengine` and start with `--backend
+webengine` to use the new backend. v1.0.0 (which is not in the Fedora repos
+currently) uses QtWebEngine by default.
+
+On Archlinux
+------------
+
+qutebrowser is available in the official [community] repository.
+
+----
+# pacman -S qutebrowser
+----
+
+There is also a -git version available in the AUR:
+https://aur.archlinux.org/packages/qutebrowser-git/[qutebrowser-git].
+
+You can install it using `makepkg` like this:
+
+----
+$ git clone https://aur.archlinux.org/qutebrowser-git.git
+$ cd qutebrowser-git
+$ makepkg -si
+$ cd ..
+$ rm -r qutebrowser-git
+----
+
+or you could use an AUR helper, e.g. `yaourt -S qutebrowser-git`.
+
+If video or sound don't work with QtWebKit, try installing the gstreamer plugins:
+
+----
+# pacman -S gst-plugins-{base,good,bad,ugly} gst-libav
+----
+
+On Gentoo
+---------
+
+The Gentoo packages (even the live version) are lagging behind a lot and are
+effectively unmaintained. If you want to create and maintain an official
+qutebrowser overlay for Gentoo, please mailto:mail@qutebrowser.org[get in
+touch.]
+
+It's recommended to <<tox,install qutebrowser via tox>> instead.
+
+To get an up-to-date QtWebKit, you can use
+https://gist.github.com/annulen/309569fb61e5d64a703c055c1e726f71[this ebuild].
+
+If video or sound don't work with QtWebKit, try installing the gstreamer
+plugins:
+
+----
+# emerge -av gst-plugins-{base,good,bad,ugly,libav}
+----
+
+To be able to play videos with proprietary codecs with QtWebEngine, you will
+need to turn off the `bindist` flag for `dev-qt/qtwebengine`.
+
+See the https://wiki.gentoo.org/wiki/Qutebrowser#USE_flags[Gentoo Wiki] for
+more information.
+
+On Void Linux
+-------------
+
+qutebrowser is available in the official repositories and can be installed
+with:
+
+----
+# xbps-install qutebrowser
+----
+
+It's currently recommended to install `python3-PyQt5-webengine` and
+`python3-PyQt5-opengl`, then start with `--backend webengine` to use the new
+backend.
+
+Since the v1.0 release, qutebrowser uses QtWebEngine by default.
+
+On NixOS
+--------
+
+Nixpkgs collection contains `pkgs.qutebrowser` since June 2015. You can install
+it with:
+
+----
+$ nix-env -i qutebrowser
+----
+
+It's recommended to install `qt5.qtwebengine` and start with
+`--backend webengine` to use the new backend.
+
+Since the v1.0 release, qutebrowser uses QtWebEngine by default.
+
+On openSUSE
+-----------
+
+There are prebuilt RPMs available at https://software.opensuse.org/download.html?project=network&package=qutebrowser[OBS].
+
+To use the QtWebEngine backend, install `libqt5-qtwebengine`.
+
+On OpenBSD
+----------
+
+qutebrowser is in http://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/www/qutebrowser/[OpenBSD ports].
+
+Install the package:
+
+----
+# pkg_add qutebrowser
+----
+
+Or alternatively, use the ports system :
+
+----
+# cd /usr/ports/www/qutebrowser
+# make install
+----
+
+On Windows
+----------
+
+There are different ways to install qutebrowser on Windows:
+
+Prebuilt binaries
+~~~~~~~~~~~~~~~~~
+
+Prebuilt standalone packages and installers
+https://github.com/qutebrowser/qutebrowser/releases[are built] for every
+release.
+
+Note that you'll need to upgrade to new versions manually (subscribe to the
+https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser-announce[qutebrowser-announce
+mailinglist] to get notified on new releases). You can install a newer version
+without uninstalling the older one.
+
+https://chocolatey.org/packages/qutebrowser[Chocolatey package]
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* PackageManagement PowerShell module
+----
+PS C:\> Install-Package qutebrowser
+----
+* Chocolatey's client
+----
+C:\> choco install qutebrowser
+----
+
+Manual install
+~~~~~~~~~~~~~~
+
+* Use the installer from http://www.python.org/downloads[python.org] to get
+Python 3 (be sure to install pip).
+* Install https://testrun.org/tox/latest/index.html[tox] via
+https://pip.pypa.io/en/latest/[pip]:
+
+----
+$ pip install tox
+----
+
+Then <<tox,install qutebrowser via tox>>.
+
+On macOS
+--------
+
+Prebuilt binary
+~~~~~~~~~~~~~~~
+
+The easiest way to install qutebrowser on macOS is to use the prebuilt `.app`
+files from the
+https://github.com/qutebrowser/qutebrowser/releases[release page].
+
+Note that you'll need to upgrade to new versions manually (subscribe to the
+https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser-announce[qutebrowser-announce
+mailinglist] to get notified on new releases).
+
+This binary is also available through the
+https://caskroom.github.io/[Homebrew Cask] package manager:
+
+----
+$ brew cask install qutebrowser
+----
+
+Manual Install
+~~~~~~~~~~~~~~
+
+Alternatively, you can install the dependencies via a package manager (like
+http://brew.sh/[Homebrew] or https://www.macports.org/[MacPorts]) and run
+qutebrowser from source.
+
+==== Homebrew
+
+----
+$ brew install qt5
+$ pip3 install qutebrowser
+----
+
+Since the v1.0 release, qutebrowser uses QtWebEngine by default.
+
+Homebrew's builds of Qt and PyQt don't come with QtWebKit (and `--with-qtwebkit`
+uses an old version of QtWebKit which qutebrowser doesn't support anymore). If
+you want QtWebKit support, you'll need to build an up-to-date QtWebKit
+https://github.com/annulen/webkit/wiki/Building-QtWebKit-on-OS-X[manually].
+
+Packagers
+---------
+
+There are example .desktop and icon files provided. They would go in the
+standard location for your distro (`/usr/share/applications` and
+`/usr/share/pixmaps` for example).
+
+The normal `setup.py install` doesn't install these files, so you'll have to do
+it as part of the packaging process.
+
+[[tox]]
+Installing qutebrowser with tox
+-------------------------------
+
+Getting the repository
+~~~~~~~~~~~~~~~~~~~~~~
+
+First of all, clone the repository using http://git-scm.org/[git] and switch
+into the repository folder:
+
+----
+$ git clone https://github.com/qutebrowser/qutebrowser.git
+$ cd qutebrowser
+----
+
+Installing depdendencies (including Qt)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Then run tox inside the qutebrowser repository to set up a
+https://docs.python.org/3/library/venv.html[virtual environment]:
+
+----
+$ tox -e mkvenv-pypi
+----
+
+This installs all needed Python dependencies in a `.venv` subfolder.
+
+This comes with an up-to-date Qt/PyQt including QtWebEngine, but has a few
+caveats:
+
+- Make sure your `python3` is Python 3.5 or newer, otherwise you'll get a "No
+  matching distribution found" error. Note that qutebrowser itself also requires
+  this.
+- It only works on 64-bit x86 systems, with other architectures you'll get the
+  same error.
+- If your distribution uses OpenSSL 1.1 (like Debian Stretch or Archlinux),
+  you'll need to set `LD_LIBRARY_PATH` to the OpenSSL 1.0 directory
+  (`export LD_LIBRARY_PATH=/usr/lib/openssl-1.0` on Archlinux) before starting
+  qutebrowser if you want SSL to work in certain downloads (e.g. for
+  `:adblock-update` or `:download`).
+- It comes with a QtWebEngine compiled without proprietary codec support (such
+  as h.264).
+
+See the next section for an alternative.
+
+Installing dependencies (system-wide Qt)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Alternatively, you can use `tox -e mkvenv` (without `-pypi`) to symlink your
+local Qt install instead of installing PyQt in the virtualenv. However, unless
+you have a new QtWebKit or QtWebEngine available, qutebrowser will not work. It
+also typically means you'll be using an older release of QtWebEngine.
+
+On Windows, run `tox -e 'mkvenv-win' instead, however make sure that ONLY
+Python3 is in your PATH before running tox.
+
+Creating a wrapper script
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can then create a simple wrapper script to start qutebrowser somewhere in
+your `$PATH` (e.g. `/usr/local/bin/qutebrowser` or `~/bin/qutebrowser`):
+
+----
+#!/bin/bash
+~/path/to/qutebrowser/.venv/bin/python3 -m qutebrowser "$@"
+----
+
+Updating
+~~~~~~~~
+
+When you updated your local copy of the code (e.g. by pulling the git repo, or
+extracting a new version), the virtualenv should automatically use the updated
+code. However, if dependencies got added, this won't be reflected in the
+virtualenv. Thus it's recommended to run the following command to recreate the
+virtualenv:
+
+----
+$ tox -r -e mkvenv-pypi
+----

doc/notes

@@ -1,196 +0,0 @@
-henk's thoughts
-===============
-
-1. Power to the user! Protect privacy!
-Things the browser should only do with explicit consent from the user, if
-applicable the user should be able to choose which protocol/host/port triplets
-to white/blacklist:
-
-- load/run executable code, like js, flash, java applets, ... (think NoScript)
-- requests to other domains, ports or using a different protocol than what the
-  user requested (think RequestPolicy)
-- accept cookies
-- storing/saving/caching things, e.g. open tabs ("session"), cookies, page
-  contents, browsing/download history, form data, ...
-- send referrer
-- disclose any (presence, type, version, settings, capabilities, etc.)
-  information about OS, browser, installed fonts, plugins, addons, etc.
-
-2. Be efficient!
-I tend to leave a lot of tabs open and nobody can deny that some websites
-simply suck, so the browser should, unless told otherwise by the user:
-
-- load tabs only when needed
-- run code in tabs only when needed, i.e. when the tab is currently being
-  used/viewed (background tabs doing some JS magic even when they are not being
-  used can create a lot of unnecessary load on the machine)
-- finish requests to the domain the user requested (e.g. www.example.org)
-  before doing any requests to other subdomains (e.g. images.example.org) and
-  finish those before doing requests to thirdparty domains (e.g. example.com)
-
-3. Be stable!
-- one site should not make the complete browser crash, only that site's tab
-
-
-Upstream Bugs
-=============
-
-- Web inspector is blank unless .hide()/.show() is called.
-    Asked on SO: http://stackoverflow.com/q/23499159/2085149
-    TODO: Report to PyQt/Qt
-
-- Report some other crashes
-
-
-/u/angelic_sedition's thoughts
-==============================
-
-Well support for greasemonkey scripts and bookmarklets/js (which was mentioned
-in the arch forum post) would be a big addition. What I've usually missed when
-using other vim-like browsers is things that allow for different settings and
-key bindings for different contexts. With that implemented I think I could
-switch to a lightweight browser (and believe me, I'd like to) for the most part
-and only use firefox when I needed downthemall or something.
-
-For example, I have different bindings based on tab position that are reloaded
-with a pentadactyl autocmd so that <space><homerow keys> will take me to tab
-1-10 if I'm in that range or 2-20 if I'm in that range. I have an autocmd that
-will run on completed downloads that passes the file path to a script that will
-open ranger in a floating window with that file cut (this is basically like
-using ranger to save files instead of the crappy gui popup).
-
-I also have a few bindings based on tabgroups. Tabgroups are a firefox feature,
-but I find them very useful for sorting things by topic so that only the tabs
-I'm interested at the moment are visible.
-
-Pentadactyl has a feature it calls groups. You can create a group that will
-activate for sites/urls that match a pattern with some regex support. This
-allows me, for example, to set up different (more convenient) bindings for
-zooming only on images. I'll never need use the equivalent of vim n (next text
-search match), so I can bind that to zoom. This allows setting up custom
-quickmarks/gotos using the same keys for different websites. For example, on
-reddit I have different g(some key) bindings to go to different subreddits.
-This can also be used to pass certain keys directly to the site (e.g. for use
-with RES). For sites that don't have modifiable bindings, I can use this with
-pentadactyl's feedkeys or xdotool to create my own custom bindings. I even have
-a binding that will call out to bash script with different arguments depending
-on the site to download an image or an image gallery depending on the site (in
-some cases passing the url to some cli program).
-
-I've also noticed the lack of completion. For example, on "o" pentadactyl will
-show sites (e.g. from history) that can be completed. I think I've been spoiled
-by pentadactyl having completion for just about everything.
-
-
-suckless surf ML post
-=====================
-
-From: Ben Woolley <tautolog_AT_gmail.com>
-Date: Wed, 7 Jan 2015 18:29:25 -0800
-
-Hi all,
-
-This patch is a bit of a beast for surf. It is intended to be applied after
-the disk cache patch. It breaks some internal interfaces, so it could
-conflict with other patches.
-
-I have been wanting a browser to implement a complete same-origin policy,
-and have been investigating how to do this in various browsers for many
-months. When I saw how surf opened new windows in a separate process, and
-was so simple, I knew I could do it quickly. Over the last two weeks, I
-have been developing this implementation on surf.
-
-The basic idea is to prevent browser-based tracking as you browse from site
-to site, or origin to origin. By "origin" domain, I mean the "first-party"
-domain, the domain normally in the location bar (of the typical browser
-interface). Each origin domain effectively gets its own browser profile,
-and a browser process only ever deals with one origin domain at a time.
-This isolates origins vertically, preventing cookies, disk cache, memory
-cache, and window.name vulnerabilities. Basically, all known
-vulnerabilities that google and Mozilla cite as counter-examples when they
-explain why they haven't disabled third-party cookies yet.
-
-When you are on msnbc.com, the tracking pixels will be stored in a cookie
-file for msnbc.com. When you go to cnn.com, the tracking pixels will be
-stored in a cookie file for cnn.com. You will not be tracked between them.
-However, third-party cookies, and the caching of third party resources will
-still work, but they will be isolated between origin domains. Instead of
-blocking cookies and cache entries, they are "double-keyed", or *also*
-keyed by origin.
-
-There is a unidirectional communication channel, however, from one origin
-to the next, through navigation from one origin to the next. That is, the
-query string is passed from one origin to the next, and may embed
-identifiers. One example is an affiliate link that identifies where the
-lead came from. I have implemented what I call "horizontal isolation", in
-the form of an "Origin Crossing Gate".
-
-Whenever you follow a link to a new domain, or even are just redirected to
-a new domain, a new window/tab is opened, and passed the referring origin
-via -R. The page passed to -O, for example -O originprompt.html, is an HTML
-page that is loaded in the new origin's context. That page tells you the
-origin you were on, the new origin, and the full link, and you can decide
-to go just to the new origin, or go to the full URL, after reviewing it for
-tracking data.
-
-Also, you may click links that store your trust of that relationship with
-various expiration times, the same way you would trust geolocation requests
-for a particular origin for a period of time. The database used is actually
-the new origin's cookie file. Since the origin prompt is loaded in the new
-origin's context, I can set a cookie on behalf of the new origin. The
-expiration time of the trust is the expiration time of the cookie. The
-cookie implementation in webkit automatically expires the trust as part of
-how cookies work. Each time you cross an origin, the origin crossing page
-checks the cookie to see if trust is still established. If so, it will use
-window.location.replace() to continue on automatically. The initial page
-renders blank until the trust is invalidated, in which case the content of
-the gate is made visible.
-
-However, the new origin is technically able to mess with those cookies, so
-a website could set trust for an origin crossing. I have addressed that by
-hashing the key with a salt, and setting the real expiration time as the
-value, along with an HMAC to verify the contents of the value. If the
-cookie is messed with in any way, the trust will be disabled, and the
-prompt will appear again. So it has a fail-safe function.
-
-I know it seems a bit convoluted, but it just started out as a nice little
-rabbit hole, and I just wanted to get something workable. At first I
-thought using the cookie expiration time was convenient, but then when I
-realized that I needed to protect the cookie, things got a bit hairy. But
-it works.
-
-Each profile is, by default, stored in ~/.surf/origins/$origin/
-The interesting side effect is that if there is a problem where a website
-relies on the cross-site cookie vulnerability to make a connection, you can
-simply make a symbolic link from one origin folder to another, and they
-will share the same profile. And if you want to delete cookies and/or cache
-for a particular origin, you just rm -rf the origin's profile folder, and
-don't have to interfere with your other sites that are working just fine.
-
-One thing I don't handle are cross-origins POSTs. They just end up as GET
-requests right now. I intend to do something about that, but I haven't
-figured that out yet.
-
-I have only been using this functionality for a few days myself, so I have
-absolutely no feedback yet. I wanted to provide the first implementation of
-the management of identity as a system resource the same way that things
-like geolocation, camera, and microphone resources are managed in browsers
-and mobile apps.
-
-Currently, Mozilla and Tor have are working on third-party tracking issues
-in Firefox.
-https://blog.mozilla.org/privacy/2014/11/10/introducing-polaris-privacy-initiative-to-accelerate-user-focused-privacy-online/
-
-Up to this point, Tor has provided a patch that double-keys cookies with
-the origin domain, but no other progress is visible. I have seen no
-discussion of how horizontal isolation is supposed to happen, and I wanted
-to show people that it can be done, and this is one way it can be done, and
-to compel the other browser makers to catch up, and hopefully the community
-can work toward a standard *without* the tracking loopholes, by showing
-people what a *complete* solution looks like.
-
-Thank you,
-
-Ben Woolley
-
-Patch: http://lists.suckless.org/dev/att-25070/0005-same-origin-policy.patch

doc/quickstart.asciidoc

@@ -31,7 +31,7 @@ image:http://qutebrowser.org/img/cheatsheet-small.png["qutebrowser key binding c
 * Run `:adblock-update` to download adblock lists and activate adblocking.
 * If you just cloned the repository, you'll need to run
 `scripts/asciidoc2html.py` to generate the documentation.
-* Go to the link:qute://settings[settings page] to set up qutebrowser the way you want it. (Currently not available with the QtWebEngine backend and on the OS X build - use the `:set` command instead)
+* Go to the link:qute://settings[settings page] to set up qutebrowser the way you want it.
 * Subscribe to
 https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser[the mailinglist] or
 https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser-announce[the announce-only mailinglist].

doc/qutebrowser.1.asciidoc

@@ -17,7 +17,7 @@ qutebrowser - a keyboard-driven, vim-like browser based on PyQt5.
 
 == DESCRIPTION
 qutebrowser is a keyboard-focused browser with a minimal GUI. It's based
-on Python, PyQt5 and QtWebKit and free software, licensed under the GPL.
+on Python and Qt5 and is free software, licensed under the GPL.
 
 It was inspired by other browsers/addons like dwb and Vimperator/Pentadactyl.
 
@@ -44,7 +44,7 @@ show it.
 *-V*, *--version*::
     Show version and quit.
 
-*-s* 'SECTION' 'OPTION' 'VALUE', *--set* 'SECTION' 'OPTION' 'VALUE'::
+*-s* 'OPTION' 'VALUE', *--set* 'OPTION' 'VALUE'::
     Set a temporary setting for this session.
 
 *-r* 'SESSION', *--restore* 'SESSION'::
@@ -57,7 +57,7 @@ show it.
     How URLs should be opened if there is already a qutebrowser instance running.
 
 *--backend* '{webkit,webengine}'::
-    Which backend to use (webengine backend is EXPERIMENTAL!).
+    Which backend to use.
 
 *--enable-webengine-inspector*::
     Enable the web inspector for QtWebEngine. Note that this is a SECURITY RISK and you should not visit untrusted websites with the inspector turned on. See https://bugreports.qt.io/browse/QTBUG-50725 for more details.
@@ -84,21 +84,9 @@ show it.
 *--force-color*::
     Force colored logging
 
-*--harfbuzz* '{old,new,system,auto}'::
-    HarfBuzz engine version to use. Default: auto.
-
-*--relaxed-config*::
-    Silently remove unknown config options.
-
 *--nowindow*::
     Don't show the main window.
 
-*--debug-exit*::
-    Turn on debugging of late exit.
-
-*--pdb-postmortem*::
-    Drop into pdb on exceptions.
-
 *--temp-basedir*::
     Use a temporary basedir.
 
@@ -110,13 +98,16 @@ show it.
 
 *--qt-flag* 'QT_FLAG'::
     Pass an argument to Qt as flag.
+
+*--debug-flag* 'DEBUG_FLAGS'::
+    Pass name of debugging feature to be turned on.
 // QUTE_OPTIONS_END
 
 == FILES
 
-- '~/.config/qutebrowser/qutebrowser.conf': Main config file.
+- '~/.config/qutebrowser/config.py': Configuration file.
+- '~/.config/qutebrowser/autoconfig.yml': Configuration done via the GUI.
 - '~/.config/qutebrowser/quickmarks': Saved quickmarks.
-- '~/.config/qutebrowser/keys.conf': Defined key bindings.
 - '~/.local/share/qutebrowser/': Various state information.
 - '~/.cache/qutebrowser/': Temporary data.
 

doc/userscripts.asciidoc

@@ -60,7 +60,7 @@ Sending commands
 Normal qutebrowser commands can be written to `$QUTE_FIFO` and will be
 executed.
 
-On Unix/OS X, this is a named pipe and commands written to it will get executed
+On Unix/macOS, this is a named pipe and commands written to it will get executed
 immediately.
 
 On Windows, this is a regular file, and the commands in it will be executed as
@@ -78,3 +78,11 @@ Opening the currently selected word on http://www.dict.cc/[dict.cc]:
 
 echo "open -t http://www.dict.cc/?s=$QUTE_SELECTED_TEXT" >> "$QUTE_FIFO"
 ----
+
+Libraries
+---------
+
+Some third-party libraries are available to make writing userscripts easier:
+
+- Python: https://github.com/hiway/python-qutescript[python-qutescript]
+- Node.js: https://www.npmjs.com/package/qutejs[qutejs]

misc/qutebrowser.nsi

@@ -0,0 +1,77 @@
+Name "qutebrowser"
+
+Unicode true
+RequestExecutionLevel admin
+SetCompressor /solid lzma
+
+!ifdef X64
+  OutFile "..\dist\qutebrowser-${VERSION}-amd64.exe"
+  InstallDir "$ProgramFiles64\qutebrowser"
+!else
+  OutFile "..\dist\qutebrowser-${VERSION}-win32.exe"
+  InstallDir "$ProgramFiles\qutebrowser"
+!endif
+
+;Default installation folder
+  
+!include "MUI2.nsh"
+;!include "MultiUser.nsh"
+
+!define MUI_ABORTWARNING
+;!define MULTIUSER_MUI
+;!define MULTIUSER_INSTALLMODE_COMMANDLINE
+!define MUI_ICON "../icons/qutebrowser.ico"
+!define MUI_UNICON "../icons/qutebrowser.ico"
+
+!insertmacro MUI_PAGE_LICENSE "..\LICENSE"
+!insertmacro MUI_PAGE_DIRECTORY
+!insertmacro MUI_PAGE_INSTFILES
+!insertmacro MUI_UNPAGE_CONFIRM
+!insertmacro MUI_UNPAGE_INSTFILES
+
+!insertmacro MUI_LANGUAGE "English"
+
+; depends on admin status
+;SetShellVarContext current
+
+
+Section "Install"
+
+  ; Uninstall old versions
+  ExecWait 'MsiExec.exe /quiet /qn /norestart /X{633F41F9-FE9B-42D1-9CC4-718CBD01EE11}'
+  ExecWait 'MsiExec.exe /quiet /qn /norestart /X{9331D947-AC86-4542-A755-A833429C6E69}'
+
+  SetOutPath "$INSTDIR"
+  
+  !ifdef X64
+	file /r "..\dist\qutebrowser-${VERSION}-x64\*.*"
+  !else
+	file /r "..\dist\qutebrowser-${VERSION}-x86\*.*"
+  !endif
+
+  SetShellVarContext all
+  CreateShortCut "$SMPROGRAMS\qutebrowser.lnk" "$INSTDIR\qutebrowser.exe"
+  
+  ;Create uninstaller
+  WriteUninstaller "$INSTDIR\uninst.exe"
+
+  WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\qutebrowser" "DisplayName" "qutebrowser"
+  WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\qutebrowser" "UninstallString" '"$INSTDIR\uninst.exe"'
+  WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\qutebrowser" "QuietUninstallString" '"$INSTDIR\uninst.exe" /S'
+
+SectionEnd
+
+;--------------------------------
+;Uninstaller Section
+
+Section "Uninstall"
+
+  SetShellVarContext all
+  Delete "$SMPROGRAMS\qutebrowser.lnk"
+
+  RMDir /r "$INSTDIR\*.*"
+  RMDir "$INSTDIR"
+
+  DeleteRegKey HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\qutebrowser"
+
+SectionEnd

misc/qutebrowser.spec

@@ -15,13 +15,14 @@ def get_data_files():
         ('../qutebrowser/img', 'img'),
         ('../qutebrowser/javascript', 'javascript'),
         ('../qutebrowser/html/doc', 'html/doc'),
-        ('../qutebrowser/git-commit-id', '')
+        ('../qutebrowser/git-commit-id', ''),
+        ('../qutebrowser/config/configdata.yml', 'config'),
     ]
 
-    if os.path.exists(os.path.join('qutebrowser', '3rdparty', 'pdfjs')):
-        data_files.append(('../qutebrowser/3rdparty/pdfjs', '3rdparty/pdfjs'))
-    else:
-        print("Warning: excluding pdfjs as it's not present!")
+    # if os.path.exists(os.path.join('qutebrowser', '3rdparty', 'pdfjs')):
+    #     data_files.append(('../qutebrowser/3rdparty/pdfjs', '3rdparty/pdfjs'))
+    # else:
+    #     print("Warning: excluding pdfjs as it's not present!")
 
     return data_files
 
@@ -41,10 +42,10 @@ a = Analysis(['../qutebrowser/__main__.py'],
              pathex=['misc'],
              binaries=None,
              datas=get_data_files(),
-             hiddenimports=[],
+             hiddenimports=['PyQt5.QtOpenGL', 'PyQt5._QOpenGLFunctions_2_0'],
              hookspath=[],
              runtime_hooks=[],
-             excludes=[],
+             excludes=['tkinter'],
              win_no_prefer_redirects=False,
              win_private_assemblies=False,
              cipher=block_cipher)
@@ -70,9 +71,5 @@ coll = COLLECT(exe,
 app = BUNDLE(coll,
              name='qutebrowser.app',
              icon=icon,
-             info_plist={
-                 'NSHighResolutionCapable': 'True',
-                 'NSSupportsAutomaticGraphicsSwitching': 'True',
-             },
              # https://github.com/pyinstaller/pyinstaller/blob/b78bfe530cdc2904f65ce098bdf2de08c9037abb/PyInstaller/hooks/hook-PyQt5.QtWebEngineWidgets.py#L24
              bundle_identifier='org.qt-project.Qt.QtWebEngineCore')

misc/requirements/requirements-codecov.txt

@@ -1,5 +1,9 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-codecov==2.0.5
-coverage==4.3.4
-requests==2.13.0
+certifi==2017.7.27.1
+chardet==3.0.4
+codecov==2.0.9
+coverage==4.4.1
+idna==2.6
+requests==2.18.4
+urllib3==1.22

misc/requirements/requirements-cxfreeze.txt-raw

@@ -1,5 +0,0 @@
-cx-Freeze < 5.0.0
-
-# We'll probably switch to PyInstaller soon, and 5.x doesn't install without a
-# compiler?
-#@ filter: cx-Freeze < 5.0.0

misc/requirements/requirements-flake8.txt

@@ -3,18 +3,21 @@
 flake8==2.6.2  # rq.filter: < 3.0.0
 flake8-copyright==0.2.0
 flake8-debugger==1.4.0  # rq.filter: != 2.0.0
-flake8-deprecated==1.1
-flake8-docstrings==1.0.3
+flake8-deprecated==1.2.1
+flake8-docstrings==1.0.3  # rq.filter: < 1.1.0
 flake8-future-import==0.4.3
 flake8-mock==0.3
-flake8-pep3101==1.0
+flake8-pep3101==1.0  # rq.filter: < 1.1
 flake8-polyfill==1.0.1
 flake8-putty==0.4.0
 flake8-string-format==0.2.3
-flake8-tidy-imports==1.0.5
-flake8-tuple==0.2.12
+flake8-tidy-imports==1.1.0
+flake8-tuple==0.2.13
 mccabe==0.6.1
+packaging==16.8
 pep8-naming==0.4.1
 pycodestyle==2.3.1
-pydocstyle==1.1.1
-pyflakes==1.5.0
+pydocstyle==1.1.1  # rq.filter: < 2.0.0
+pyflakes==1.6.0
+pyparsing==2.2.0
+six==1.11.0

misc/requirements/requirements-flake8.txt-raw

@@ -2,16 +2,16 @@ flake8<3.0.0
 flake8-copyright
 flake8-debugger!=2.0.0
 flake8-deprecated
-flake8-docstrings
+flake8-docstrings<1.1.0
 flake8-future-import
 flake8-mock
-flake8-pep3101
+flake8-pep3101<1.1
 flake8-putty
 flake8-string-format
 flake8-tidy-imports
 flake8-tuple
 pep8-naming
-pydocstyle
+pydocstyle<2.0.0
 pyflakes
 
 # Pinned to 2.0.0 otherwise
@@ -21,6 +21,9 @@ mccabe==0.6.1
 
 # Waiting until flake8-putty updated
 #@ filter: flake8 < 3.0.0
+#@ filter: pydocstyle < 2.0.0
+#@ filter: flake8-docstrings < 1.1.0
+#@ filter: flake8-pep3101 < 1.1
 
 # https://github.com/JBKahn/flake8-debugger/issues/5
 #@ filter: flake8-debugger != 2.0.0

misc/requirements/requirements-pip.txt

@@ -1,8 +1,8 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-appdirs==1.4.2
+appdirs==1.4.3
 packaging==16.8
-pyparsing==2.1.10
-setuptools==34.3.0
-six==1.10.0
-wheel==0.29.0
+pyparsing==2.2.0
+setuptools==36.5.0
+six==1.11.0
+wheel==0.30.0

misc/requirements/requirements-pyinstaller.txt

@@ -1,5 +1,7 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
+altgraph==0.14
+future==0.16.0
+macholib==1.8
+pefile==2017.9.3
 -e git+https://github.com/pyinstaller/pyinstaller.git@develop#egg=PyInstaller
-PyQt5==5.8
-sip==4.19.1

misc/requirements/requirements-pyinstaller.txt-raw

@@ -1,5 +1,4 @@
 -e git+https://github.com/pyinstaller/pyinstaller.git@develop#egg=PyInstaller
-PyQt5
 
 # remove @commit-id for scm installs
-#@ replace: @.*# @develop#
+#@ replace: @.*# @develop#

misc/requirements/requirements-pylint-master.txt

@@ -1,11 +1,18 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
 -e git+https://github.com/PyCQA/astroid.git#egg=astroid
-editdistance==0.3.1
-isort==4.2.5
-lazy-object-proxy==1.2.2
+certifi==2017.7.27.1
+chardet==3.0.4
+github3.py==0.9.6
+idna==2.6
+isort==4.2.15
+lazy-object-proxy==1.3.1
 mccabe==0.6.1
 -e git+https://github.com/PyCQA/pylint.git#egg=pylint
 ./scripts/dev/pylint_checkers
-requests==2.13.0
-wrapt==1.10.8
+requests==2.18.4
+six==1.11.0
+uritemplate==3.0.0
+uritemplate.py==3.0.2
+urllib3==1.22
+wrapt==1.10.11

misc/requirements/requirements-pylint-master.txt-raw

@@ -2,6 +2,7 @@
 -e git+https://github.com/PyCQA/pylint.git#egg=pylint
 ./scripts/dev/pylint_checkers
 requests
+github3.py
 
 # remove @commit-id for scm installs
 #@ replace: @.*# #

misc/requirements/requirements-pylint.txt

@@ -1,13 +1,18 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-astroid==1.4.9
+astroid==1.5.3
+certifi==2017.7.27.1
+chardet==3.0.4
 github3.py==0.9.6
-isort==4.2.5
-lazy-object-proxy==1.2.2
+idna==2.6
+isort==4.2.15
+lazy-object-proxy==1.3.1
 mccabe==0.6.1
-pylint==1.6.5
+pylint==1.7.4
 ./scripts/dev/pylint_checkers
-requests==2.13.0
+requests==2.18.4
+six==1.11.0
 uritemplate==3.0.0
 uritemplate.py==3.0.2
-wrapt==1.10.8
+urllib3==1.22
+wrapt==1.10.11

misc/requirements/requirements-pyqt.txt

@@ -1,3 +1,4 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-cx-Freeze==4.3.4  # rq.filter: < 5.0.0
+PyQt5==5.9
+sip==4.19.3

misc/requirements/requirements-pyqt.txt-raw

@@ -0,0 +1 @@
+PyQt5

misc/requirements/requirements-pyroma.txt

@@ -1,4 +1,4 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-docutils==0.13.1
+docutils==0.14
 pyroma==2.2

misc/requirements/requirements-qutebrowser.txt-raw

@@ -4,3 +4,4 @@ pyPEG2
 PyYAML
 colorama
 cssutils
+attrs

misc/requirements/requirements-tests-git.txt

@@ -4,7 +4,6 @@ hg+https://bitbucket.org/ned/coveragepy
 git+https://github.com/micheles/decorator.git
 git+https://github.com/pallets/flask.git
 git+https://github.com/miracle2k/python-glob2.git
-git+https://github.com/Runscope/httpbin.git
 git+https://github.com/HypothesisWorks/hypothesis-python.git
 git+https://github.com/pallets/itsdangerous.git
 git+https://bitbucket.org/zzzeek/mako.git
@@ -27,7 +26,6 @@ git+https://github.com/pytest-dev/pytest-qt.git
 git+https://github.com/pytest-dev/pytest-repeat.git
 git+https://github.com/pytest-dev/pytest-rerunfailures.git
 git+https://github.com/abusalimov/pytest-travis-fold.git
-git+https://github.com/fschulze/pytest-warnings.git
 git+https://github.com/The-Compiler/pytest-xvfb.git
 hg+https://bitbucket.org/gutworth/six
 hg+https://bitbucket.org/jendrikseipp/vulture
@@ -42,6 +40,7 @@ git+https://github.com/pallets/jinja.git
 git+https://github.com/pallets/markupsafe.git
 hg+http://bitbucket.org/birkenfeld/pygments-main
 hg+https://bitbucket.org/fdik/pypeg
+git+https://github.com/python-attrs/attrs.git
 
 # Fails to build:
 # gcc: error: ext/_yaml.c: No such file or directory

misc/requirements/requirements-tests.txt

@@ -1,36 +1,39 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-beautifulsoup4==4.5.3
-cheroot==5.1.0
+attrs==17.2.0
+beautifulsoup4==4.6.0
+cheroot==5.8.3
 click==6.7
-coverage==4.3.4
-decorator==4.0.11
+# colorama==0.3.9
+coverage==4.4.1
 EasyProcess==0.2.3
-Flask==0.12
-glob2==0.5
-httpbin==0.5.0
-hypothesis==3.6.1
+fields==5.0.0
+Flask==0.12.2
+glob2==0.6
+hunter==2.0.1
+hypothesis==3.32.0
 itsdangerous==0.24
-# Jinja2==2.9.5
-Mako==1.0.6
-# MarkupSafe==0.23
-parse==1.6.6
-parse-type==0.3.4
-py==1.4.32
-pytest==3.0.6
-pytest-bdd==2.18.1
-pytest-benchmark==3.0.0
+# Jinja2==2.9.6
+Mako==1.0.7
+# MarkupSafe==1.0
+parse==1.8.2
+parse-type==0.4.2
+py==1.4.34
+py-cpuinfo==3.3.0
+pytest==3.2.3
+pytest-bdd==2.18.2
+pytest-benchmark==3.1.1
 pytest-catchlog==1.2.2
-pytest-cov==2.4.0
+pytest-cov==2.5.1
 pytest-faulthandler==1.3.1
 pytest-instafail==0.3.0
-pytest-mock==1.5.0
-pytest-qt==2.1.0
+pytest-mock==1.6.3
+pytest-qt==2.2.1
 pytest-repeat==0.4.1
-pytest-rerunfailures==2.1.0
+pytest-rerunfailures==3.1
 pytest-travis-fold==1.2.0
-pytest-warnings==0.2.0
 pytest-xvfb==1.0.0
 PyVirtualDisplay==0.2.1
-vulture==0.12
-Werkzeug==0.11.15
+six==1.11.0
+vulture==0.26
+Werkzeug==0.12.2

misc/requirements/requirements-tests.txt-raw

@@ -2,7 +2,7 @@ beautifulsoup4
 cheroot
 coverage
 Flask
-httpbin
+hunter
 hypothesis
 pytest
 pytest-bdd
@@ -16,8 +16,7 @@ pytest-qt
 pytest-repeat
 pytest-rerunfailures
 pytest-travis-fold
-pytest-warnings
 pytest-xvfb
 vulture
 
-#@ ignore: Jinja2, MarkupSafe
+#@ ignore: Jinja2, MarkupSafe, colorama

misc/requirements/requirements-tox.txt

@@ -1,6 +1,6 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-pluggy==0.4.0
-py==1.4.32
-tox==2.6.0
+pluggy==0.5.2
+py==1.4.34
+tox==2.9.1
 virtualenv==15.1.0

misc/requirements/requirements-vulture.txt

@@ -1,3 +1,3 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-vulture==0.12
+vulture==0.26

misc/userscripts/dmenu_qutebrowser

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 # Copyright 2015 Zach-Button <zachrey.button@gmail.com>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -22,7 +23,7 @@
 # If run from qutebrowser as a userscript, it runs :open on the URL
 # If not, it opens a new qutebrowser window at the URL
 #
-# Ideal for use with tabs-are-windows. Set a hotkey to launch this script, then:
+# Ideal for use with tabs_are_windows. Set a hotkey to launch this script, then:
 #	:bind o spawn --userscript dmenu_qutebrowser
 #
 # Use the hotkey to open in new tab/window, press 'o' to open URL in current tab/window

misc/userscripts/format_json

@@ -0,0 +1,47 @@
+#!/bin/sh
+#
+# Behavior:
+#   Userscript for qutebrowser which will take the raw JSON text of the current
+#   page, format it using `jq`, will add syntax highlighting using `pygments`,
+#   and open the syntax highlighted pretty printed html in a new tab. If the file
+#   is larger than 10MB then this script will only indent the json and will forego
+#   syntax highlighting using pygments.
+#
+#   In order to use this script, just start it using `spawn --userscript` from
+#   qutebrowser. I recommend using an alias, e.g. put this in the
+#   [alias]-section of qutebrowser.conf:
+#
+#     json = spawn --userscript /path/to/json_format
+#
+#   Note that the color style defaults to monokai, but a different pygments style
+#   can be passed as the first parameter to the script. A full list of the pygments
+#   styles can be found at: https://help.farbox.com/pygments.html
+#
+# Bryan Gilbert, 2017
+
+# default style to monokai if none is provided
+STYLE=${1:-monokai}
+# format json using jq
+FORMATTED_JSON="$(cat "$QUTE_TEXT" | jq '.')"
+
+# if jq command failed or formatted json is empty, assume failure and terminate
+if [ $? -ne 0 ] || [ -z "$FORMATTED_JSON" ]; then
+    echo "Invalid json, aborting..."
+    exit 1
+fi
+
+# calculate the filesize of the json document
+FILE_SIZE=$(ls -s --block-size=1048576 "$QUTE_TEXT" | cut -d' ' -f1)
+
+# use pygments to pretty-up the json (syntax highlight) if file is less than 10MB
+if [ "$FILE_SIZE" -lt "10" ]; then
+    FORMATTED_JSON="$(echo "$FORMATTED_JSON" | pygmentize -l json -f html -O full,style=$STYLE)"
+fi
+
+# create a temp file and write the formatted json to that file
+TEMP_FILE="$(mktemp --suffix '.html')"
+echo "$FORMATTED_JSON" > $TEMP_FILE
+
+
+# send the command to qutebrowser to open the new file containing the formatted json
+echo "open -t file://$TEMP_FILE" >> "$QUTE_FIFO"

misc/userscripts/open_download

@@ -12,7 +12,7 @@
 #  - rofi (in a recent version)
 #  - xdg-open and xdg-mime
 #  - You should configure qutebrowser to download files to a single directory
-#  - It comes in handy if you enable remove-finished-downloads. If you want to
+#  - It comes in handy if you enable downloads.remove_finished. If you want to
 #    see the recent downloads, just press "sd".
 #
 # Thorsten Wißmann, 2015 (thorsten` on freenode)

misc/userscripts/openfeeds

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 # Copyright 2015 jnphilipp <me@jnphilipp.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -20,7 +21,7 @@
 
 # Opens all links to feeds defined in the head of a site
 #
-# Ideal for use with tabs-are-windows. Set a hotkey to launch this script, then:
+# Ideal for use with tabs_are_windows. Set a hotkey to launch this script, then:
 #   :bind gF spawn --userscript openfeeds
 #
 # Use the hotkey to open the feeds in new tab/window, press 'gF' to open

misc/userscripts/password_fill

@@ -9,7 +9,7 @@ directly ask me via IRC (nickname thorsten\`) in #qutebrowser on freenode.
 
   $blink!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!$reset
   WARNING: the passwords are stored in qutebrowser's
-           debug log reachable via the url qute:log
+           debug log reachable via the url qute://log
   $blink!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!$reset
 
 Usage: run as a userscript form qutebrowser, e.g.:
@@ -189,7 +189,7 @@ choose_entry_rofi() {
 }
 
 choose_entry_zenity() {
-    MENU_COMMAND=( zenity --list --title "Qutebrowser password fill"
+    MENU_COMMAND=( zenity --list --title "qutebrowser password fill"
                           --text "Pick the password entry:"
                           --column "Name" )
     choose_entry_menu || true
@@ -199,7 +199,7 @@ choose_entry_zenity_radio() {
     zenity_helper() {
         awk '{ print $0 ; print $0 }'                   \
         | zenity --list --radiolist                     \
-                 --title "Qutebrowser password fill"    \
+                 --title "qutebrowser password fill"    \
                  --text "Pick the password entry:"      \
                  --column " " --column "Name"
     }
@@ -327,6 +327,17 @@ open_entry "$file"
 
 js() {
 cat <<EOF
+    function isVisible(elem) {
+        var style = elem.ownerDocument.defaultView.getComputedStyle(elem, null);
+
+        if (style.getPropertyValue("visibility") !== "visible" ||
+            style.getPropertyValue("display") === "none" ||
+            style.getPropertyValue("opacity") === "0") {
+            return false;
+        }
+
+        return elem.offsetWidth > 0 && elem.offsetHeight > 0;
+    };
     function hasPasswordField(form) {
         var inputs = form.getElementsByTagName("input");
         for (var j = 0; j < inputs.length; j++) {
@@ -341,7 +352,7 @@ cat <<EOF
         var inputs = form.getElementsByTagName("input");
         for (var j = 0; j < inputs.length; j++) {
             var input = inputs[j];
-            if (input.type == "text" || input.type == "email") {
+            if (isVisible(input) && (input.type == "text" || input.type == "email")) {
                 input.focus();
                 input.value = "$(javascript_escape "${username}")";
                 input.blur();

misc/userscripts/qutebrowser_viewsource

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 # Copyright 2015 Zach-Button <zachrey.button@gmail.com>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

misc/userscripts/readability

@@ -1,21 +1,33 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 #
 # Executes python-readability on current page and opens the summary as new tab.
 #
+# Depends on the python-readability package, or its fork:
+#
+#   - https://github.com/buriy/python-readability
+#   - https://github.com/bookieio/breadability
+#
 # Usage:
 #   :spawn --userscript readability
 #
 from __future__ import absolute_import
 import codecs, os
-from readability.readability import Document
 
 tmpfile=os.path.expanduser('~/.local/share/qutebrowser/userscripts/readability.html')
 if not os.path.exists(os.path.dirname(tmpfile)):
     os.makedirs(os.path.dirname(tmpfile))
 
 with codecs.open(os.environ['QUTE_HTML'], 'r', 'utf-8') as source:
-    doc = Document(source.read())
-    content = doc.summary().replace('<html>', '<html><head><title>%s</title></head>' % doc.title())
+    data = source.read()
+
+    try:
+        from breadability.readable import Article as reader
+        doc = reader(data)
+        content = doc.readable
+    except ImportError:
+        from readability import Document
+        doc = Document(data)
+        content = doc.summary().replace('<html>', '<html><head><title>%s</title></head>' % doc.title())
 
     with codecs.open(tmpfile, 'w', 'utf-8') as target:
         target.write('<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />')

misc/userscripts/ripbang

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 #
 # Adds DuckDuckGo bang as searchengine.
 #
@@ -8,14 +8,21 @@
 # Example:
 #   :spawn --userscript ripbang amazon maps
 #
-import os, re, requests, sys, urllib
+
+from __future__ import print_function
+import os, re, requests, sys
+
+try:
+    from urllib.parse import unquote
+except ImportError:
+    from urllib import unquote
 
 for argument in sys.argv[1:]:
     bang = '!' + argument
     r = requests.get('https://duckduckgo.com/',
                      params={'q': bang + ' SEARCHTEXT'})
 
-    searchengine = urllib.unquote(re.search("url=[^']+", r.text).group(0))
+    searchengine = unquote(re.search("url=[^']+", r.text).group(0))
     searchengine = searchengine.replace('url=', '')
     searchengine = searchengine.replace('/l/?kh=-1&uddg=', '')
     searchengine = searchengine.replace('SEARCHTEXT', '{}')
@@ -24,4 +31,4 @@ for argument in sys.argv[1:]:
         with open(os.environ['QUTE_FIFO'], 'w') as fifo:
             fifo.write('set searchengines %s %s' % (bang, searchengine))
     else:
-        print '%s %s' % (bang, searchengine)
+        print('%s %s' % (bang, searchengine))

misc/userscripts/view_in_mpv

@@ -49,7 +49,7 @@ msg() {
 
 MPV_COMMAND=${MPV_COMMAND:-mpv}
 # Warning: spaces in single flags are not supported
-MPV_FLAGS=${MPV_FLAGS:- --force-window --no-terminal --keep-open=yes --ytdl }
+MPV_FLAGS=${MPV_FLAGS:- --force-window --no-terminal --keep-open=yes --ytdl --ytdl-raw-options=yes-playlist=}
 video_command=( "$MPV_COMMAND"  $MPV_FLAGS )
 
 js() {
@@ -140,4 +140,4 @@ printjs() {
 echo "jseval -q $(printjs)" >> "$QUTE_FIFO"
 
 msg info "Opening $QUTE_URL with mpv"
-"${video_command[@]}" "$QUTE_URL"
+"${video_command[@]}" "$@" "$QUTE_URL"

pytest.ini

@@ -1,12 +1,13 @@
 [pytest]
-addopts = --strict -rfEw --faulthandler-timeout=70 --instafail --pythonwarnings error
+addopts = --strict -rfEw --faulthandler-timeout=90 --instafail --pythonwarnings error --benchmark-columns=Min,Max,Median
+testpaths = tests
 markers =
     gui: Tests using the GUI (e.g. spawning widgets)
     posix: Tests which only can run on a POSIX OS.
     windows: Tests which only can run on Windows.
     linux: Tests which only can run on Linux.
-    osx: Tests which only can run on OS X.
-    not_osx: Tests which can not run on OS X.
+    mac: Tests which only can run on macOS.
+    not_mac: Tests which can not run on macOS.
     not_frozen: Tests which can't be run if sys.frozen is True.
     no_xvfb: Tests which can't be run with Xvfb.
     frozen: Tests which can only be run if sys.frozen is True.
@@ -14,21 +15,22 @@ markers =
     end2end: End to end tests which run qutebrowser as subprocess
     xfail_norun: xfail the test with out running it
     ci: Tests which should only run on CI.
+    no_ci: Tests which should not run on CI.
     qtwebengine_todo: Features still missing with QtWebEngine
     qtwebengine_skip: Tests not applicable with QtWebEngine
     qtwebkit_skip: Tests not applicable with QtWebKit
-    qtwebkit_ng_xfail: Tests failing with QtWebKit-NG
-    qtwebkit_ng_skip: Tests skipped with QtWebKit-NG
     qtwebengine_flaky: Tests which are flaky (and currently skipped) with QtWebEngine
-    qtwebengine_osx_xfail: Tests which fail on OS X with QtWebEngine
+    qtwebengine_mac_xfail: Tests which fail on macOS with QtWebEngine
     js_prompt: Tests needing to display a javascript prompt
     this: Used to mark tests during development
     no_invalid_lines: Don't fail on unparseable lines in end2end tests
+    issue2478: Tests which are broken on Windows with QtWebEngine, https://github.com/qutebrowser/qutebrowser/issues/2478
+    fake_os: Fake utils.is_* to a fake operating system
 qt_log_level_fail = WARNING
 qt_log_ignore =
     ^SpellCheck: .*
     ^SetProcessDpiAwareness failed: .*
-    ^QWindowsWindow::setGeometryDp: Unable to set geometry .*
+    ^QWindowsWindow::setGeometry(Dp)?: Unable to set geometry .*
     ^QProcess: Destroyed while process .* is still running\.
     ^"Method "GetAll" with signature "s" on interface "org\.freedesktop\.DBus\.Properties" doesn't exist
     ^"Method \\"GetAll\\" with signature \\"s\\" on interface \\"org\.freedesktop\.DBus\.Properties\\" doesn't exist\\n"
@@ -42,13 +44,15 @@ qt_log_ignore =
     ^QWaitCondition: Destroyed while threads are still waiting
     ^QXcbXSettings::QXcbXSettings\(QXcbScreen\*\) Failed to get selection owner for XSETTINGS_S atom
     ^QStandardPaths: XDG_RUNTIME_DIR not set, defaulting to .*
-    ^QGeoclueMaster error creating GeoclueMasterClient\.
-    ^Geoclue error: Process org\.freedesktop\.Geoclue\.Master exited with status 127
     ^QObject::connect: Cannot connect \(null\)::stateChanged\(QNetworkSession::State\) to QNetworkReplyHttpImpl::_q_networkSessionStateChanged\(QNetworkSession::State\)
     ^QXcbClipboard: Cannot transfer data, no data available
     ^load glyph failed
     ^Error when parsing the netrc file
     ^Image of format '' blocked because it is not considered safe. If you are sure it is safe to do so, you can white-list the format by setting the environment variable QTWEBKIT_IMAGEFORMAT_WHITELIST=
     ^QPainter::end: Painter ended with \d+ saved states
-    ^QSslSocket: cannot resolve SSLv[23]_(client|server)_method
+    ^QSslSocket: cannot resolve *
+    ^Incompatible version of OpenSSL
+    ^QQuickWidget::invalidateRenderControl could not make context current
+    ^libpng warning: iCCP: known incorrect sRGB profile
+    ^inotify_add_watch(".*") failed: "No space left on device"
 xfail_strict = true

qutebrowser/__init__.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -22,11 +22,11 @@
 import os.path
 
 __author__ = "Florian Bruhin"
-__copyright__ = "Copyright 2014-2016 Florian Bruhin (The Compiler)"
+__copyright__ = "Copyright 2014-2017 Florian Bruhin (The Compiler)"
 __license__ = "GPL"
 __maintainer__ = __author__
 __email__ = "mail@qutebrowser.org"
-__version_info__ = (0, 10, 0)
+__version_info__ = (1, 0, 2)
 __version__ = '.'.join(str(e) for e in __version_info__)
 __description__ = "A keyboard-driven, vim-like browser based on PyQt5."
 

qutebrowser/__main__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/app.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -17,12 +17,29 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Initialization of qutebrowser and application-wide things."""
+"""Initialization of qutebrowser and application-wide things.
+
+The run() function will get called once early initialization (in
+qutebrowser.py/earlyinit.py) is done. See the qutebrowser.py docstring for
+details about early initialization.
+
+As we need to access the config before the QApplication is created, we
+initialize everything the config needs before the QApplication is created, and
+then leave it in a partially initialized state (no saving, no config errors
+shown yet).
+
+We then set up the QApplication object and initialize a few more low-level
+things.
+
+After that, init() and _init_modules() take over and initialize the rest.
+
+After all initialization is done, the qt_mainloop() function is called, which
+blocks and spins the Qt mainloop.
+"""
 
 import os
 import sys
 import subprocess
-import configparser
 import functools
 import json
 import shutil
@@ -42,9 +59,10 @@ except ImportError:
 
 import qutebrowser
 import qutebrowser.resources
-from qutebrowser.completion.models import instances as completionmodels
+from qutebrowser.completion import completiondelegate
+from qutebrowser.completion.models import miscmodels
 from qutebrowser.commands import cmdutils, runners, cmdexc
-from qutebrowser.config import style, config, websettings, configexc
+from qutebrowser.config import config, websettings, configfiles, configinit
 from qutebrowser.browser import (urlmarks, adblock, history, browsertab,
                                  downloads)
 from qutebrowser.browser.network import proxy
@@ -53,11 +71,15 @@ from qutebrowser.browser.webkit.network import networkmanager
 from qutebrowser.keyinput import macros
 from qutebrowser.mainwindow import mainwindow, prompt
 from qutebrowser.misc import (readline, ipc, savemanager, sessions,
-                              crashsignal, earlyinit)
-from qutebrowser.misc import utilcmds  # pylint: disable=unused-import
-from qutebrowser.utils import (log, version, message, utils, qtutils, urlutils,
-                               objreg, usertypes, standarddir, error, debug)
-# We import utilcmds to run the cmdutils.register decorators.
+                              crashsignal, earlyinit, sql, cmdhistory,
+                              backendproblem)
+from qutebrowser.utils import (log, version, message, utils, urlutils, objreg,
+                               usertypes, standarddir, error)
+# pylint: disable=unused-import
+# We import those to run the cmdutils.register decorators.
+from qutebrowser.mainwindow.statusbar import command
+from qutebrowser.misc import utilcmds
+# pylint: enable=unused-import
 
 
 qApp = None
@@ -71,6 +93,12 @@ def run(args):
     quitter = Quitter(args)
     objreg.register('quitter', quitter)
 
+    log.init.debug("Initializing directories...")
+    standarddir.init(args)
+
+    log.init.debug("Initializing config...")
+    configinit.early_init(args)
+
     global qApp
     qApp = Application(args)
     qApp.setOrganizationName("qutebrowser")
@@ -78,9 +106,6 @@ def run(args):
     qApp.setApplicationVersion(qutebrowser.__version__)
     qApp.lastWindowClosed.connect(quitter.on_last_window_closed)
 
-    log.init.debug("Initializing directories...")
-    standarddir.init(args)
-
     if args.version:
         print(version.version())
         sys.exit(usertypes.Exit.ok)
@@ -133,33 +158,29 @@ def init(args, crash_handler):
     log.init.debug("Starting init...")
     qApp.setQuitOnLastWindowClosed(False)
     _init_icon()
-    utils.actute_warning()
 
     try:
         _init_modules(args, crash_handler)
-    except (OSError, UnicodeDecodeError) as e:
+    except (OSError, UnicodeDecodeError, browsertab.WebTabError) as e:
         error.handle_fatal_exc(e, args, "Error while initializing!",
                                pre_text="Error while initializing")
         sys.exit(usertypes.Exit.err_init)
 
-    QTimer.singleShot(0, functools.partial(_process_args, args))
-    QTimer.singleShot(10, functools.partial(_init_late_modules, args))
-
     log.init.debug("Initializing eventfilter...")
     event_filter = EventFilter(qApp)
     qApp.installEventFilter(event_filter)
     objreg.register('event-filter', event_filter)
 
     log.init.debug("Connecting signals...")
-    config_obj = objreg.get('config')
-    config_obj.style_changed.connect(style.get_stylesheet.cache_clear)
     qApp.focusChanged.connect(on_focus_changed)
 
+    _process_args(args)
+
     QDesktopServices.setUrlHandler('http', open_desktopservices_url)
     QDesktopServices.setUrlHandler('https', open_desktopservices_url)
     QDesktopServices.setUrlHandler('qute', open_desktopservices_url)
 
-    macros.init()
+    objreg.get('web-history').import_txt()
 
     log.init.debug("Init done!")
     crash_handler.raise_crashdlg()
@@ -172,36 +193,32 @@ def _init_icon():
     for size in [16, 24, 32, 48, 64, 96, 128, 256, 512]:
         filename = ':/icons/qutebrowser-{}x{}.png'.format(size, size)
         pixmap = QPixmap(filename)
-        qtutils.ensure_not_null(pixmap)
-        fallback_icon.addPixmap(pixmap)
-    qtutils.ensure_not_null(fallback_icon)
+        if pixmap.isNull():
+            log.init.warning("Failed to load {}".format(filename))
+        else:
+            fallback_icon.addPixmap(pixmap)
     icon = QIcon.fromTheme('qutebrowser', fallback_icon)
-    qtutils.ensure_not_null(icon)
-    qApp.setWindowIcon(icon)
+    if icon.isNull():
+        log.init.warning("Failed to load icon")
+    else:
+        qApp.setWindowIcon(icon)
 
 
 def _process_args(args):
     """Open startpage etc. and process commandline args."""
-    config_obj = objreg.get('config')
-    for sect, opt, val in args.temp_settings:
-        try:
-            config_obj.set('temp', sect, opt, val)
-        except (configexc.Error, configparser.Error) as e:
-            message.error("set: {} - {}".format(e.__class__.__name__, e))
-
     if not args.override_restore:
         _load_session(args.session)
     session_manager = objreg.get('session-manager')
     if not session_manager.did_load:
         log.init.debug("Initializing main window...")
-        window = mainwindow.MainWindow()
+        window = mainwindow.MainWindow(private=None)
         if not args.nowindow:
             window.show()
         qApp.setActiveWindow(window)
 
     process_pos_args(args.command)
     _open_startpage()
-    _open_quickstart(args)
+    _open_special_pages(args)
 
     delta = datetime.datetime.now() - earlyinit.START_TIME
     log.init.debug("Init finished after {}s".format(delta.total_seconds()))
@@ -213,13 +230,12 @@ def _load_session(name):
     Args:
         name: The name of the session to load, or None to read state file.
     """
-    state_config = objreg.get('state-config')
     session_manager = objreg.get('session-manager')
     if name is None and session_manager.exists('_autosave'):
         name = '_autosave'
     elif name is None:
         try:
-            name = state_config['general']['session']
+            name = configfiles.state['general']['session']
         except KeyError:
             # No session given as argument and none in the session file ->
             # start without loading a session
@@ -232,7 +248,7 @@ def _load_session(name):
     except sessions.SessionError as e:
         message.error("Failed to load session {}: {}".format(name, e))
     try:
-        del state_config['general']['session']
+        del configfiles.state['general']['session']
     except KeyError:
         pass
     # If this was a _restart session, delete it.
@@ -264,7 +280,7 @@ def process_pos_args(args, via_ipc=False, cwd=None, target_arg=None):
                 win_id = mainwindow.get_window(via_ipc, force_tab=True)
             log.init.debug("Startup cmd {!r}".format(cmd))
             commandrunner = runners.CommandRunner(win_id)
-            commandrunner.run_safely_init(cmd[1:])
+            commandrunner.run_safely(cmd[1:])
         elif not cmd:
             log.init.debug("Empty argument")
             win_id = mainwindow.get_window(via_ipc, force_window=True)
@@ -272,11 +288,7 @@ def process_pos_args(args, via_ipc=False, cwd=None, target_arg=None):
             if via_ipc and target_arg and target_arg != 'auto':
                 open_target = target_arg
             else:
-                open_target = config.get('general', 'new-instance-open-target')
-            win_id = mainwindow.get_window(via_ipc, force_target=open_target)
-            tabbed_browser = objreg.get('tabbed-browser', scope='window',
-                                        window=win_id)
-            log.init.debug("Startup URL {}".format(cmd))
+                open_target = None
             if not cwd:  # could also be an empty string due to the PyQt signal
                 cwd = None
             try:
@@ -285,9 +297,30 @@ def process_pos_args(args, via_ipc=False, cwd=None, target_arg=None):
                 message.error("Error in startup argument '{}': {}".format(
                     cmd, e))
             else:
-                background = open_target in ['tab-bg', 'tab-bg-silent']
-                tabbed_browser.tabopen(url, background=background,
-                                       explicit=True)
+                win_id = open_url(url, target=open_target, via_ipc=via_ipc)
+
+
+def open_url(url, target=None, no_raise=False, via_ipc=True):
+    """Open an URL in new window/tab.
+
+    Args:
+        url: An URL to open.
+        target: same as new_instance_open_target (used as a default).
+        no_raise: suppress target window raising.
+        via_ipc: Whether the arguments were transmitted over IPC.
+
+    Return:
+        ID of a window that was used to open URL
+    """
+    target = target or config.val.new_instance_open_target
+    background = target in {'tab-bg', 'tab-bg-silent'}
+    win_id = mainwindow.get_window(via_ipc, force_target=target,
+                                   no_raise=no_raise)
+    tabbed_browser = objreg.get('tabbed-browser', scope='window',
+                                window=win_id)
+    log.init.debug("About to open URL: {}".format(url.toDisplayString()))
+    tabbed_browser.tabopen(url, background=background, related=False)
+    return win_id
 
 
 def _open_startpage(win_id=None):
@@ -303,47 +336,52 @@ def _open_startpage(win_id=None):
         window_ids = [win_id]
     else:
         window_ids = objreg.window_registry
-    for cur_win_id in window_ids:
+    for cur_win_id in list(window_ids):  # Copying as the dict could change
         tabbed_browser = objreg.get('tabbed-browser', scope='window',
                                     window=cur_win_id)
         if tabbed_browser.count() == 0:
-            log.init.debug("Opening startpage")
-            for urlstr in config.get('general', 'startpage'):
-                try:
-                    url = urlutils.fuzzy_url(urlstr, do_search=False)
-                except urlutils.InvalidUrlError as e:
-                    message.error("Error when opening startpage: {}".format(e))
-                    tabbed_browser.tabopen(QUrl('about:blank'))
-                else:
-                    tabbed_browser.tabopen(url)
+            log.init.debug("Opening start pages")
+            for url in config.val.url.start_pages:
+                tabbed_browser.tabopen(url)
 
 
-def _open_quickstart(args):
-    """Open quickstart if it's the first start.
+def _open_special_pages(args):
+    """Open special notification pages which are only shown once.
+
+    Currently this is:
+      - Quickstart page if it's the first start.
+      - Legacy QtWebKit warning if needed.
 
     Args:
         args: The argparse namespace.
     """
     if args.basedir is not None:
-        # With --basedir given, don't open quickstart.
+        # With --basedir given, don't open anything.
         return
-    state_config = objreg.get('state-config')
-    try:
-        quickstart_done = state_config['general']['quickstart-done'] == '1'
-    except KeyError:
-        quickstart_done = False
+
+    general_sect = configfiles.state['general']
+    tabbed_browser = objreg.get('tabbed-browser', scope='window',
+                                window='last-focused')
+
+    # Quickstart page
+
+    quickstart_done = general_sect.get('quickstart-done') == '1'
+
     if not quickstart_done:
-        tabbed_browser = objreg.get('tabbed-browser', scope='window',
-                                    window='last-focused')
         tabbed_browser.tabopen(
             QUrl('https://www.qutebrowser.org/quickstart.html'))
-        state_config['general']['quickstart-done'] = '1'
+        general_sect['quickstart-done'] = '1'
 
+    # Setting migration page
 
-def _save_version():
-    """Save the current version to the state config."""
-    state_config = objreg.get('state-config')
-    state_config['general']['version'] = qutebrowser.__version__
+    needs_migration = os.path.exists(
+        os.path.join(standarddir.config(), 'qutebrowser.conf'))
+    migration_shown = general_sect.get('config-migration-shown') == '1'
+
+    if needs_migration and not migration_shown:
+        tabbed_browser.tabopen(QUrl('qute://help/configuring.html'),
+                               background=False)
+        general_sect['config-migration-shown'] = '1'
 
 
 def on_focus_changed(_old, new):
@@ -380,32 +418,46 @@ def _init_modules(args, crash_handler):
         crash_handler: The CrashHandler instance.
     """
     # pylint: disable=too-many-statements
-    log.init.debug("Initializing prompts...")
-    prompt.init()
-
     log.init.debug("Initializing save manager...")
     save_manager = savemanager.SaveManager(qApp)
     objreg.register('save-manager', save_manager)
-    save_manager.add_saveable('version', _save_version)
+    configinit.late_init(save_manager)
+
+    log.init.debug("Checking backend requirements...")
+    backendproblem.init()
+
+    log.init.debug("Initializing prompts...")
+    prompt.init()
 
     log.init.debug("Initializing network...")
     networkmanager.init()
 
-    if qtutils.version_check('5.8'):
-        # Otherwise we can only initialize it for QtWebKit because of crashes
-        log.init.debug("Initializing proxy...")
-        proxy.init()
+    log.init.debug("Initializing proxy...")
+    proxy.init()
 
     log.init.debug("Initializing readline-bridge...")
     readline_bridge = readline.ReadlineBridge()
     objreg.register('readline-bridge', readline_bridge)
 
-    log.init.debug("Initializing config...")
-    config.init(qApp)
-    save_manager.init_autosave()
+    try:
+        log.init.debug("Initializing sql...")
+        sql.init(os.path.join(standarddir.data(), 'history.sqlite'))
 
-    log.init.debug("Initializing web history...")
-    history.init(qApp)
+        log.init.debug("Initializing web history...")
+        history.init(qApp)
+    except sql.SqlError as e:
+        if e.environmental:
+            error.handle_fatal_exc(e, args, 'Error initializing SQL',
+                                   pre_text='Error initializing SQL')
+            sys.exit(usertypes.Exit.err_init)
+        else:
+            raise
+
+    log.init.debug("Initializing completion...")
+    completiondelegate.init()
+
+    log.init.debug("Initializing command history...")
+    cmdhistory.init()
 
     log.init.debug("Initializing crashlog...")
     if not args.no_err_windows:
@@ -440,35 +492,12 @@ def _init_modules(args, crash_handler):
     diskcache = cache.DiskCache(standarddir.cache(), parent=qApp)
     objreg.register('cache', diskcache)
 
-    log.init.debug("Initializing completions...")
-    completionmodels.init()
-
     log.init.debug("Misc initialization...")
-    if config.get('ui', 'hide-wayland-decoration'):
-        os.environ['QT_WAYLAND_DISABLE_WINDOWDECORATION'] = '1'
-    else:
-        os.environ.pop('QT_WAYLAND_DISABLE_WINDOWDECORATION', None)
+    macros.init()
     # Init backend-specific stuff
     browsertab.init()
 
 
-def _init_late_modules(args):
-    """Initialize modules which can be inited after the window is shown."""
-    log.init.debug("Reading web history...")
-    reader = objreg.get('web-history').async_read()
-    with debug.log_time(log.init, 'Reading history'):
-        while True:
-            QApplication.processEvents()
-            try:
-                next(reader)
-            except StopIteration:
-                break
-            except (OSError, UnicodeDecodeError) as e:
-                error.handle_fatal_exc(e, args, "Error while initializing!",
-                                       pre_text="Error while initializing")
-                sys.exit(usertypes.Exit.err_init)
-
-
 class Quitter:
 
     """Utility class to quit/restart the QApplication.
@@ -511,12 +540,13 @@ class Quitter:
                     with tokenize.open(os.path.join(dirpath, fn)) as f:
                         compile(f.read(), fn, 'exec')
 
-    def _get_restart_args(self, pages=(), session=None):
+    def _get_restart_args(self, pages=(), session=None, override_args=None):
         """Get the current working directory and args to relaunch qutebrowser.
 
         Args:
             pages: The pages to re-open.
             session: The session to load, or None.
+            override_args: Argument overrides as a dict.
 
         Return:
             An (args, cwd) tuple.
@@ -567,6 +597,9 @@ class Quitter:
             argdict['temp_basedir'] = False
             argdict['temp_basedir_restarted'] = True
 
+        if override_args is not None:
+            argdict.update(override_args)
+
         # Dump the data
         data = json.dumps(argdict)
         args += ['--json-args', data]
@@ -591,7 +624,7 @@ class Quitter:
         if ok:
             self.shutdown(restart=True)
 
-    def restart(self, pages=(), session=None):
+    def restart(self, pages=(), session=None, override_args=None):
         """Inner logic to restart qutebrowser.
 
         The "better" way to restart is to pass a session (_restart usually) as
@@ -604,6 +637,7 @@ class Quitter:
         Args:
             pages: A list of URLs to open.
             session: The session to load, or None.
+            override_args: Argument overrides as a dict.
 
         Return:
             True if the restart succeeded, False otherwise.
@@ -613,13 +647,19 @@ class Quitter:
         log.destroy.debug("sys.path: {}".format(sys.path))
         log.destroy.debug("sys.argv: {}".format(sys.argv))
         log.destroy.debug("frozen: {}".format(hasattr(sys, 'frozen')))
+
         # Save the session if one is given.
         if session is not None:
             session_manager = objreg.get('session-manager')
-            session_manager.save(session)
+            session_manager.save(session, with_private=True)
+
+        # Make sure we're not accepting a connection from the new process
+        # before we fully exited.
+        ipc.server.shutdown()
+
         # Open a new process and immediately shutdown the existing one
         try:
-            args, cwd = self._get_restart_args(pages, session)
+            args, cwd = self._get_restart_args(pages, session, override_args)
             if cwd is None:
                 subprocess.Popen(args)
             else:
@@ -630,8 +670,25 @@ class Quitter:
         else:
             return True
 
-    @cmdutils.register(instance='quitter', name=['quit', 'q'],
-                       ignore_args=True)
+    @cmdutils.register(instance='quitter', name='quit')
+    @cmdutils.argument('session', completion=miscmodels.session)
+    def quit(self, save=False, session=None):
+        """Quit qutebrowser.
+
+        Args:
+            save: When given, save the open windows even if auto_save.session
+                  is turned off.
+            session: The name of the session to save.
+        """
+        if session is not None and not save:
+            raise cmdexc.CommandError("Session name given without --save!")
+        if save:
+            if session is None:
+                session = sessions.default
+            self.shutdown(session=session)
+        else:
+            self.shutdown()
+
     def shutdown(self, status=0, session=None, last_window=False,
                  restart=False):
         """Quit qutebrowser.
@@ -648,14 +705,14 @@ class Quitter:
         self._shutting_down = True
         log.destroy.debug("Shutting down with status {}, session {}...".format(
             status, session))
-
-        session_manager = objreg.get('session-manager')
-        if session is not None:
-            session_manager.save(session, last_window=last_window,
-                                 load_next_time=True)
-        elif config.get('general', 'save-session'):
-            session_manager.save(sessions.default, last_window=last_window,
-                                 load_next_time=True)
+        session_manager = objreg.get('session-manager', None)
+        if session_manager is not None:
+            if session is not None:
+                session_manager.save(session, last_window=last_window,
+                                     load_next_time=True)
+            elif config.val.auto_save.session:
+                session_manager.save(sessions.default, last_window=last_window,
+                                     load_next_time=True)
 
         if prompt.prompt_queue.shutdown():
             # If shutdown was called while we were asking a question, we're in
@@ -672,7 +729,7 @@ class Quitter:
             # event loop, so we can shut down immediately.
             self._shutdown(status, restart=restart)
 
-    def _shutdown(self, status, restart):
+    def _shutdown(self, status, restart):  # noqa
         """Second stage of shutdown."""
         log.destroy.debug("Stage 2 of shutting down...")
         if qApp is None:
@@ -681,14 +738,16 @@ class Quitter:
         # Remove eventfilter
         try:
             log.destroy.debug("Removing eventfilter...")
-            qApp.removeEventFilter(objreg.get('event-filter'))
+            event_filter = objreg.get('event-filter', None)
+            if event_filter is not None:
+                qApp.removeEventFilter(event_filter)
         except AttributeError:
             pass
         # Close all windows
         QApplication.closeAllWindows()
         # Shut down IPC
         try:
-            objreg.get('ipc-server').shutdown()
+            ipc.server.shutdown()
         except KeyError:
             pass
         # Save everything
@@ -723,21 +782,13 @@ class Quitter:
         # Now we can hopefully quit without segfaults
         log.destroy.debug("Deferring QApplication::exit...")
         objreg.get('signal-handler').deactivate()
-        objreg.get('session-manager').delete_autosave()
+        session_manager = objreg.get('session-manager', None)
+        if session_manager is not None:
+            session_manager.delete_autosave()
         # We use a singleshot timer to exit here to minimize the likelihood of
         # segfaults.
         QTimer.singleShot(0, functools.partial(qApp.exit, status))
 
-    @cmdutils.register(instance='quitter', name='wq')
-    @cmdutils.argument('name', completion=usertypes.Completion.sessions)
-    def save_and_quit(self, name=sessions.default):
-        """Save open pages and quit.
-
-        Args:
-            name: The name of the session.
-        """
-        self.shutdown(session=name)
-
 
 class Application(QApplication):
 
@@ -758,7 +809,7 @@ class Application(QApplication):
         """
         self._last_focus_object = None
 
-        qt_args = qtutils.get_args(args)
+        qt_args = configinit.qt_args(args)
         log.init.debug("Qt arguments: {}, based on {}".format(qt_args, args))
         super().__init__(qt_args)
 
@@ -779,13 +830,22 @@ class Application(QApplication):
             log.misc.debug("Focus object changed: {}".format(output))
         self._last_focus_object = output
 
+    def event(self, e):
+        """Handle macOS FileOpen events."""
+        if e.type() == QEvent.FileOpen:
+            open_url(e.url(), no_raise=True)
+        else:
+            return super().event(e)
+
+        return True
+
     def __repr__(self):
         return utils.get_repr(self)
 
     def exit(self, status):
         """Extend QApplication::exit to log the event."""
         log.destroy.debug("Now calling QApplication::exit.")
-        if self._args.debug_exit:
+        if 'debug-exit' in self._args.debug_flags:
             if hunter is None:
                 print("Not logging late shutdown because hunter could not be "
                       "imported!", file=sys.stderr)

qutebrowser/browser/__init__.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/adblock.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -58,7 +58,7 @@ def get_fileobj(byte_io):
         byte_io = zf.open(filename, mode='r')
     else:
         byte_io.seek(0)  # rewind what zipfile.is_zipfile did
-    return io.TextIOWrapper(byte_io, encoding='utf-8')
+    return byte_io
 
 
 def is_whitelisted_host(host):
@@ -67,11 +67,7 @@ def is_whitelisted_host(host):
     Args:
         host: The host of the request as string.
     """
-    whitelist = config.get('content', 'host-blocking-whitelist')
-    if whitelist is None:
-        return False
-
-    for pattern in whitelist:
+    for pattern in config.val.content.host_blocking.whitelist:
         if fnmatch.fnmatch(host, pattern.lower()):
             return True
     return False
@@ -114,16 +110,16 @@ class HostBlocker:
 
         data_dir = standarddir.data()
         self._local_hosts_file = os.path.join(data_dir, 'blocked-hosts')
-        self.on_config_changed()
+        self._update_files()
 
         config_dir = standarddir.config()
         self._config_hosts_file = os.path.join(config_dir, 'blocked-hosts')
 
-        objreg.get('config').changed.connect(self.on_config_changed)
+        config.instance.changed.connect(self._update_files)
 
     def is_blocked(self, url):
         """Check if the given URL (as QUrl) is blocked."""
-        if not config.get('content', 'host-blocking-enabled'):
+        if not config.val.content.host_blocking.enabled:
             return False
         host = url.host()
         return ((host in self._blocked_hosts or
@@ -147,7 +143,7 @@ class HostBlocker:
             with open(filename, 'r', encoding='utf-8') as f:
                 for line in f:
                     target.add(line.strip())
-        except OSError:
+        except (OSError, UnicodeDecodeError):
             log.misc.exception("Failed to read host blocklist!")
 
         return True
@@ -164,8 +160,9 @@ class HostBlocker:
 
         if not found:
             args = objreg.get('args')
-            if (config.get('content', 'host-block-lists') is not None and
-                    args.basedir is None):
+            if (config.val.content.host_blocking.lists and
+                    args.basedir is None and
+                    config.val.content.host_blocking.enabled):
                 message.info("Run :adblock-update to get adblock lists.")
 
     @cmdutils.register(instance='host-blocker')
@@ -179,18 +176,16 @@ class HostBlocker:
                               self._config_blocked_hosts)
         self._blocked_hosts = set()
         self._done_count = 0
-        urls = config.get('content', 'host-block-lists')
         download_manager = objreg.get('qtnetwork-download-manager',
                                       scope='window', window='last-focused')
-        if urls is None:
-            return
-        for url in urls:
+        for url in config.val.content.host_blocking.lists:
             if url.scheme() == 'file':
+                filename = url.toLocalFile()
                 try:
-                    fileobj = open(url.path(), 'rb')
+                    fileobj = open(filename, 'rb')
                 except OSError as e:
                     message.error("adblock: Error while reading {}: {}".format(
-                        url.path(), e.strerror))
+                        filename, e.strerror))
                     continue
                 download = FakeDownload(fileobj)
                 self._in_progress.append(download)
@@ -205,6 +200,54 @@ class HostBlocker:
                 download.finished.connect(
                     functools.partial(self.on_download_finished, download))
 
+    def _parse_line(self, line):
+        """Parse a line from a host file.
+
+        Args:
+            line: The bytes object to parse.
+
+        Returns:
+            True if parsing succeeded, False otherwise.
+        """
+        if line.startswith(b'#'):
+            # Ignoring comments early so we don't have to care about
+            # encoding errors in them.
+            return True
+
+        try:
+            line = line.decode('utf-8')
+        except UnicodeDecodeError:
+            log.misc.error("Failed to decode: {!r}".format(line))
+            return False
+
+        # Remove comments
+        try:
+            hash_idx = line.index('#')
+            line = line[:hash_idx]
+        except ValueError:
+            pass
+
+        line = line.strip()
+        # Skip empty lines
+        if not line:
+            return True
+
+        parts = line.split()
+        if len(parts) == 1:
+            # "one host per line" format
+            host = parts[0]
+        elif len(parts) == 2:
+            # /etc/hosts format
+            host = parts[1]
+        else:
+            log.misc.error("Failed to parse: {!r}".format(line))
+            return False
+
+        if host not in self.WHITELISTED:
+            self._blocked_hosts.add(host)
+
+        return True
+
     def _merge_file(self, byte_io):
         """Read and merge host files.
 
@@ -218,35 +261,18 @@ class HostBlocker:
         line_count = 0
         try:
             f = get_fileobj(byte_io)
-        except (OSError, UnicodeDecodeError, zipfile.BadZipFile,
-                zipfile.LargeZipFile, LookupError) as e:
+        except (OSError, zipfile.BadZipFile, zipfile.LargeZipFile,
+                LookupError) as e:
             message.error("adblock: Error while reading {}: {} - {}".format(
                 byte_io.name, e.__class__.__name__, e))
             return
+
         for line in f:
             line_count += 1
-            # Remove comments
-            try:
-                hash_idx = line.index('#')
-                line = line[:hash_idx]
-            except ValueError:
-                pass
-            line = line.strip()
-            # Skip empty lines
-            if not line:
-                continue
-            parts = line.split()
-            if len(parts) == 1:
-                # "one host per line" format
-                host = parts[0]
-            elif len(parts) == 2:
-                # /etc/hosts format
-                host = parts[1]
-            else:
+            ok = self._parse_line(line)
+            if not ok:
                 error_count += 1
-                continue
-            if host not in self.WHITELISTED:
-                self._blocked_hosts.add(host)
+
         log.misc.debug("{}: read {} lines".format(byte_io.name, line_count))
         if error_count > 0:
             message.error("adblock: {} read errors for {}".format(
@@ -260,11 +286,10 @@ class HostBlocker:
             message.info("adblock: Read {} hosts from {} sources.".format(
                 len(self._blocked_hosts), self._done_count))
 
-    @config.change_filter('content', 'host-block-lists')
-    def on_config_changed(self):
+    @config.change_filter('content.host_blocking.lists')
+    def _update_files(self):
         """Update files when the config changed."""
-        urls = config.get('content', 'host-block-lists')
-        if urls is None:
+        if not config.val.content.host_blocking.lists:
             try:
                 os.remove(self._local_hosts_file)
             except FileNotFoundError:

qutebrowser/browser/browsertab.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -21,7 +21,8 @@
 
 import itertools
 
-from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, QObject, QSizeF
+import attr
+from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, QObject, QSizeF, Qt
 from PyQt5.QtGui import QIcon
 from PyQt5.QtWidgets import QWidget, QApplication
 
@@ -35,11 +36,12 @@ from qutebrowser.browser import mouse, hints
 tab_id_gen = itertools.count(0)
 
 
-def create(win_id, parent=None):
+def create(win_id, private, parent=None):
     """Get a QtWebKit/QtWebEngine tab object.
 
     Args:
         win_id: The window ID where the tab will be shown.
+        private: Whether the tab is a private/off the record tab.
         parent: The Qt parent to set.
     """
     # Importing modules here so we don't depend on QtWebEngine without the
@@ -51,7 +53,8 @@ def create(win_id, parent=None):
     else:
         from qutebrowser.browser.webkit import webkittab
         tab_class = webkittab.WebKitTab
-    return tab_class(win_id=win_id, mode_manager=mode_manager, parent=parent)
+    return tab_class(win_id=win_id, mode_manager=mode_manager, private=private,
+                     parent=parent)
 
 
 def init():
@@ -59,9 +62,6 @@ def init():
     if objects.backend == usertypes.Backend.QtWebEngine:
         from qutebrowser.browser.webengine import webenginetab
         webenginetab.init()
-    else:
-        from qutebrowser.browser.webkit import webkittab
-        webkittab.init()
 
 
 class WebTabError(Exception):
@@ -83,6 +83,7 @@ TerminationStatus = usertypes.enum('TerminationStatus', [
 ])
 
 
+@attr.s
 class TabData:
 
     """A simple namespace with a fixed set of attributes.
@@ -94,18 +95,29 @@ class TabData:
         viewing_source: Set if we're currently showing a source view.
         override_target: Override for open_target for fake clicks (like hints).
                          Only used for QtWebKit.
+        pinned: Flag to pin the tab.
+        fullscreen: Whether the tab has a video shown fullscreen currently.
     """
 
-    def __init__(self):
-        self.keep_icon = False
-        self.viewing_source = False
-        self.inspector = None
-        self.override_target = None
+    keep_icon = attr.ib(False)
+    viewing_source = attr.ib(False)
+    inspector = attr.ib(None)
+    override_target = attr.ib(None)
+    pinned = attr.ib(False)
+    fullscreen = attr.ib(False)
 
 
 class AbstractAction:
 
-    """Attribute of AbstractTab for Qt WebActions."""
+    """Attribute of AbstractTab for Qt WebActions.
+
+    Class attributes (overridden by subclasses):
+        action_class: The class actions are defined on (QWeb{Engine,}Page)
+        action_base: The type of the actions (QWeb{Engine,}Page.WebAction)
+    """
+
+    action_class = None
+    action_base = None
 
     def __init__(self):
         self._widget = None
@@ -118,6 +130,13 @@ class AbstractAction:
         """Save the current page."""
         raise NotImplementedError
 
+    def run_string(self, name):
+        """Run a webaction based on its name."""
+        member = getattr(self.action_class, name, None)
+        if not isinstance(member, self.action_base):
+            raise WebTabError("{} is not a valid web action!".format(name))
+        self._widget.triggerPageAction(member)
+
 
 class AbstractPrinting:
 
@@ -155,6 +174,8 @@ class AbstractSearch(QObject):
 
     Attributes:
         text: The last thing this view was searched for.
+        search_displayed: Whether we're currently displaying search results in
+                          this view.
         _flags: The flags of the last search (needs to be set by subclasses).
         _widget: The underlying WebView widget.
     """
@@ -163,14 +184,30 @@ class AbstractSearch(QObject):
         super().__init__(parent)
         self._widget = None
         self.text = None
+        self.search_displayed = False
 
-    def search(self, text, *, ignore_case=False, reverse=False,
+    def _is_case_sensitive(self, ignore_case):
+        """Check if case-sensitivity should be used.
+
+        This assumes self.text is already set properly.
+
+        Arguments:
+            ignore_case: The ignore_case value from the config.
+        """
+        mapping = {
+            'smart': not self.text.islower(),
+            'never': True,
+            'always': False,
+        }
+        return mapping[ignore_case]
+
+    def search(self, text, *, ignore_case='never', reverse=False,
                result_cb=None):
         """Find the given text on the page.
 
         Args:
             text: The text to search for.
-            ignore_case: Search case-insensitively. (True/False/'smart')
+            ignore_case: Search case-insensitively. ('always'/'never/'smart')
             reverse: Reverse search direction.
             result_cb: Called with a bool indicating whether a match was found.
         """
@@ -212,7 +249,8 @@ class AbstractZoom(QObject):
         self._win_id = win_id
         self._default_zoom_changed = False
         self._init_neighborlist()
-        objreg.get('config').changed.connect(self._on_config_changed)
+        config.instance.changed.connect(self._on_config_changed)
+        self._zoom_factor = float(config.val.zoom.default) / 100
 
         # # FIXME:qtwebengine is this needed?
         # # For some reason, this signal doesn't get disconnected automatically
@@ -221,21 +259,20 @@ class AbstractZoom(QObject):
         # self.destroyed.connect(functools.partial(
         #     cfg.changed.disconnect, self.init_neighborlist))
 
-    @pyqtSlot(str, str)
-    def _on_config_changed(self, section, option):
-        if section == 'ui' and option in ['zoom-levels', 'default-zoom']:
+    @pyqtSlot(str)
+    def _on_config_changed(self, option):
+        if option in ['zoom.levels', 'zoom.default']:
             if not self._default_zoom_changed:
-                factor = float(config.get('ui', 'default-zoom')) / 100
-                self._set_factor_internal(factor)
-            self._default_zoom_changed = False
+                factor = float(config.val.zoom.default) / 100
+                self.set_factor(factor)
             self._init_neighborlist()
 
     def _init_neighborlist(self):
         """Initialize self._neighborlist."""
-        levels = config.get('ui', 'zoom-levels')
+        levels = config.val.zoom.levels
         self._neighborlist = usertypes.NeighborList(
             levels, mode=usertypes.NeighborList.Modes.edge)
-        self._neighborlist.fuzzyval = config.get('ui', 'default-zoom')
+        self._neighborlist.fuzzyval = config.val.zoom.default
 
     def offset(self, offset):
         """Increase/Decrease the zoom level by the given offset.
@@ -264,15 +301,21 @@ class AbstractZoom(QObject):
             self._neighborlist.fuzzyval = int(factor * 100)
         if factor < 0:
             raise ValueError("Can't zoom to factor {}!".format(factor))
-        self._default_zoom_changed = True
+
+        default_zoom_factor = float(config.val.zoom.default) / 100
+        self._default_zoom_changed = (factor != default_zoom_factor)
+
+        self._zoom_factor = factor
         self._set_factor_internal(factor)
 
     def factor(self):
-        raise NotImplementedError
+        return self._zoom_factor
 
     def set_default(self):
-        default_zoom = config.get('ui', 'default-zoom')
-        self._set_factor_internal(float(default_zoom) / 100)
+        self._set_factor_internal(float(config.val.zoom.default) / 100)
+
+    def set_current(self):
+        self._set_factor_internal(self._zoom_factor)
 
 
 class AbstractCaret(QObject):
@@ -441,11 +484,21 @@ class AbstractHistory:
     def current_idx(self):
         raise NotImplementedError
 
-    def back(self):
-        raise NotImplementedError
+    def back(self, count=1):
+        idx = self.current_idx() - count
+        if idx >= 0:
+            self._go_to_item(self._item_at(idx))
+        else:
+            self._go_to_item(self._item_at(0))
+            raise WebTabError("At beginning of history.")
 
-    def forward(self):
-        raise NotImplementedError
+    def forward(self, count=1):
+        idx = self.current_idx() + count
+        if idx < len(self):
+            self._go_to_item(self._item_at(idx))
+        else:
+            self._go_to_item(self._item_at(len(self) - 1))
+            raise WebTabError("At end of history.")
 
     def can_go_back(self):
         raise NotImplementedError
@@ -453,6 +506,12 @@ class AbstractHistory:
     def can_go_forward(self):
         raise NotImplementedError
 
+    def _item_at(self, i):
+        raise NotImplementedError
+
+    def _go_to_item(self, item):
+        raise NotImplementedError
+
     def serialize(self):
         """Serialize into an opaque format understood by self.deserialize."""
         raise NotImplementedError
@@ -524,6 +583,7 @@ class AbstractTab(QWidget):
     Attributes:
         history: The AbstractHistory for the current tab.
         registry: The ObjectRegistry associated with this tab.
+        private: Whether private browsing is turned on for this tab.
 
         _load_status: loading status of this page
                       Accessible via load_status() method.
@@ -563,7 +623,8 @@ class AbstractTab(QWidget):
     fullscreen_requested = pyqtSignal(bool)
     renderer_process_terminated = pyqtSignal(TerminationStatus, int)
 
-    def __init__(self, win_id, mode_manager, parent=None):
+    def __init__(self, *, win_id, mode_manager, private, parent=None):
+        self.private = private
         self.win_id = win_id
         self.tab_id = next(tab_id_gen)
         super().__init__(parent)
@@ -662,9 +723,9 @@ class AbstractTab(QWidget):
         self._set_load_status(usertypes.LoadStatus.loading)
         self.load_started.emit()
 
-    def _handle_auto_insert_mode(self, ok):
-        """Handle auto-insert-mode after loading finished."""
-        if not config.get('input', 'auto-insert-mode') or not ok:
+    def handle_auto_insert_mode(self, ok):
+        """Handle `input.insert_mode.auto_load` after loading finished."""
+        if not config.val.input.insert_mode.auto_load or not ok:
             return
 
         cur_mode = self._mode_manager.mode
@@ -699,7 +760,8 @@ class AbstractTab(QWidget):
         self.load_finished.emit(ok)
         if not self.title():
             self.title_changed.emit(self.url().toDisplayString())
-        self._handle_auto_insert_mode(ok)
+
+        self.zoom.set_current()
 
     @pyqtSlot()
     def _on_history_trigger(self):
@@ -740,6 +802,10 @@ class AbstractTab(QWidget):
     def clear_ssl_errors(self):
         raise NotImplementedError
 
+    def key_press(self, key, modifier=Qt.NoModifier):
+        """Send a fake key event to this tab."""
+        raise NotImplementedError
+
     def dump_async(self, callback, *, plain=False):
         """Dump the current page to a file ascync.
 
@@ -771,7 +837,7 @@ class AbstractTab(QWidget):
     def icon(self):
         raise NotImplementedError
 
-    def set_html(self, html, base_url):
+    def set_html(self, html, base_url=QUrl()):
         raise NotImplementedError
 
     def networkaccessmanager(self):
@@ -794,6 +860,6 @@ class AbstractTab(QWidget):
         try:
             url = utils.elide(self.url().toDisplayString(QUrl.EncodeUnicode),
                               100)
-        except AttributeError:
-            url = '<AttributeError>'
+        except (AttributeError, RuntimeError) as exc:
+            url = '<{}>'.format(exc.__class__.__name__)
         return utils.get_repr(self, tab_id=self.tab_id, url=url)

qutebrowser/browser/commands.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -23,33 +23,27 @@ import os
 import os.path
 import shlex
 import functools
+import typing
 
-from PyQt5.QtWidgets import QApplication, QTabBar
+from PyQt5.QtWidgets import QApplication, QTabBar, QDialog
 from PyQt5.QtCore import Qt, QUrl, QEvent, QUrlQuery
 from PyQt5.QtGui import QKeyEvent
 from PyQt5.QtPrintSupport import QPrintDialog, QPrintPreviewDialog
-try:
-    from PyQt5.QtWebKitWidgets import QWebPage
-except ImportError:
-    QWebPage = None
-try:
-    from PyQt5.QtWebEngineWidgets import QWebEnginePage
-except ImportError:
-    QWebEnginePage = None
 import pygments
 import pygments.lexers
 import pygments.formatters
 
 from qutebrowser.commands import userscripts, cmdexc, cmdutils, runners
-from qutebrowser.config import config, configexc
+from qutebrowser.config import config, configdata
 from qutebrowser.browser import (urlmarks, browsertab, inspector, navigate,
                                  webelem, downloads)
 from qutebrowser.keyinput import modeman
 from qutebrowser.utils import (message, usertypes, log, qtutils, urlutils,
-                               objreg, utils, typing)
+                               objreg, utils, debug)
 from qutebrowser.utils.usertypes import KeyMode
 from qutebrowser.misc import editor, guiprocess
-from qutebrowser.completion.models import instances, sortfilter
+from qutebrowser.completion.models import urlmodel, miscmodels
+from qutebrowser.mainwindow import mainwindow
 
 
 class CommandDispatcher:
@@ -75,10 +69,9 @@ class CommandDispatcher:
     def __repr__(self):
         return utils.get_repr(self)
 
-    def _new_tabbed_browser(self):
+    def _new_tabbed_browser(self, private):
         """Get a tabbed-browser from a new window."""
-        from qutebrowser.mainwindow import mainwindow
-        new_window = mainwindow.MainWindow()
+        new_window = mainwindow.MainWindow(private=private)
         new_window.show()
         return new_window.tabbed_browser
 
@@ -118,7 +111,7 @@ class CommandDispatcher:
         return widget
 
     def _open(self, url, tab=False, background=False, window=False,
-              explicit=True):
+              related=False, private=None):
         """Helper function to open a page.
 
         Args:
@@ -126,17 +119,22 @@ class CommandDispatcher:
             tab: Whether to open in a new tab.
             background: Whether to open in the background.
             window: Whether to open in a new window
+            private: If opening a new window, open it in private browsing mode.
+                     If not given, inherit the current window's mode.
         """
         urlutils.raise_cmdexc_if_invalid(url)
         tabbed_browser = self._tabbed_browser
-        cmdutils.check_exclusive((tab, background, window), 'tbw')
-        if window:
-            tabbed_browser = self._new_tabbed_browser()
+        cmdutils.check_exclusive((tab, background, window, private), 'tbwp')
+        if window and private is None:
+            private = self._tabbed_browser.private
+
+        if window or private:
+            tabbed_browser = self._new_tabbed_browser(private)
             tabbed_browser.tabopen(url)
         elif tab:
-            tabbed_browser.tabopen(url, background=False, explicit=explicit)
+            tabbed_browser.tabopen(url, background=False, related=related)
         elif background:
-            tabbed_browser.tabopen(url, background=True, explicit=explicit)
+            tabbed_browser.tabopen(url, background=True, related=related)
         else:
             widget = self._current_widget()
             widget.openurl(url)
@@ -160,12 +158,14 @@ class CommandDispatcher:
         else:
             return None
 
-    def _tab_focus_last(self):
+    def _tab_focus_last(self, *, show_error=True):
         """Select the tab which was last focused."""
         try:
             tab = objreg.get('last-focused-tab', scope='window',
                              window=self._win_id)
         except KeyError:
+            if not show_error:
+                return
             raise cmdexc.CommandError("No last focused tab!")
         idx = self._tabbed_browser.indexOf(tab)
         if idx == -1:
@@ -179,7 +179,7 @@ class CommandDispatcher:
             prev: Force selecting the tab before the current tab.
             next_: Force selecting the tab after the current tab.
             opposite: Force selecting the tab in the opposite direction of
-                      what's configured in 'tabs->select-on-remove'.
+                      what's configured in 'tabs.select_on_remove'.
 
         Return:
             QTabBar.SelectLeftTab, QTabBar.SelectRightTab, or None if no change
@@ -191,38 +191,35 @@ class CommandDispatcher:
         elif next_:
             return QTabBar.SelectRightTab
         elif opposite:
-            conf_selection = config.get('tabs', 'select-on-remove')
+            conf_selection = config.val.tabs.select_on_remove
             if conf_selection == QTabBar.SelectLeftTab:
                 return QTabBar.SelectRightTab
             elif conf_selection == QTabBar.SelectRightTab:
                 return QTabBar.SelectLeftTab
             elif conf_selection == QTabBar.SelectPreviousTab:
                 raise cmdexc.CommandError(
-                    "-o is not supported with 'tabs->select-on-remove' set to "
+                    "-o is not supported with 'tabs.select_on_remove' set to "
                     "'last-used'!")
             else:  # pragma: no cover
-                raise ValueError("Invalid select-on-remove value "
+                raise ValueError("Invalid select_on_remove value "
                                  "{!r}!".format(conf_selection))
         return None
 
-    @cmdutils.register(instance='command-dispatcher', scope='window')
-    @cmdutils.argument('count', count=True)
-    def tab_close(self, prev=False, next_=False, opposite=False, count=None):
-        """Close the current/[count]th tab.
+    def _tab_close(self, tab, prev=False, next_=False, opposite=False):
+        """Helper function for tab_close be able to handle message.async.
 
         Args:
+            tab: Tab object to select be closed.
             prev: Force selecting the tab before the current tab.
             next_: Force selecting the tab after the current tab.
             opposite: Force selecting the tab in the opposite direction of
-                      what's configured in 'tabs->select-on-remove'.
+                      what's configured in 'tabs.select_on_remove'.
             count: The tab index to close, or None
         """
-        tab = self._cntwidget(count)
-        if tab is None:
-            return
         tabbar = self._tabbed_browser.tabBar()
         selection_override = self._get_selection_override(prev, next_,
                                                           opposite)
+
         if selection_override is None:
             self._tabbed_browser.close_tab(tab)
         else:
@@ -231,12 +228,55 @@ class CommandDispatcher:
             self._tabbed_browser.close_tab(tab)
             tabbar.setSelectionBehaviorOnRemove(old_selection_behavior)
 
+    @cmdutils.register(instance='command-dispatcher', scope='window')
+    @cmdutils.argument('count', count=True)
+    def tab_close(self, prev=False, next_=False, opposite=False,
+                  force=False, count=None):
+        """Close the current/[count]th tab.
+
+        Args:
+            prev: Force selecting the tab before the current tab.
+            next_: Force selecting the tab after the current tab.
+            opposite: Force selecting the tab in the opposite direction of
+                      what's configured in 'tabs.select_on_remove'.
+            force: Avoid confirmation for pinned tabs.
+            count: The tab index to close, or None
+        """
+        tab = self._cntwidget(count)
+        if tab is None:
+            return
+        close = functools.partial(self._tab_close, tab, prev,
+                                  next_, opposite)
+
+        self._tabbed_browser.tab_close_prompt_if_pinned(tab, force, close)
+
+    @cmdutils.register(instance='command-dispatcher', scope='window',
+                       name='tab-pin')
+    @cmdutils.argument('count', count=True)
+    def tab_pin(self, count=None):
+        """Pin/Unpin the current/[count]th tab.
+
+        Pinning a tab shrinks it to the size of its title text.
+        Attempting to close a pinned tab will cause a confirmation,
+        unless --force is passed.
+
+        Args:
+            count: The tab index to pin or unpin, or None
+        """
+        tab = self._cntwidget(count)
+        if tab is None:
+            return
+
+        to_pin = not tab.data.pinned
+        self._tabbed_browser.set_tab_pinned(tab, to_pin)
+
     @cmdutils.register(instance='command-dispatcher', name='open',
                        maxsplit=0, scope='window')
-    @cmdutils.argument('url', completion=usertypes.Completion.url)
+    @cmdutils.argument('url', completion=urlmodel.url)
     @cmdutils.argument('count', count=True)
-    def openurl(self, url=None, implicit=False,
-                bg=False, tab=False, window=False, count=None):
+    def openurl(self, url=None, related=False,
+                bg=False, tab=False, window=False, count=None, secure=False,
+                private=False):
         """Open a URL in the current/[count]th tab.
 
         If the URL contains newlines, each line gets opened in its own tab.
@@ -246,21 +286,27 @@ class CommandDispatcher:
             bg: Open in a new background tab.
             tab: Open in a new tab.
             window: Open in a new window.
-            implicit: If opening a new tab, treat the tab as implicit (like
-                      clicking on a link).
+            related: If opening a new tab, position the tab as related to the
+                     current one (like clicking on a link).
             count: The tab index to open the URL in, or None.
+            secure: Force HTTPS.
+            private: Open a new window in private browsing mode.
         """
         if url is None:
-            urls = [config.get('general', 'default-page')]
+            urls = [config.val.url.default_page]
         else:
             urls = self._parse_url_input(url)
 
         for i, cur_url in enumerate(urls):
+            if secure:
+                cur_url.setScheme('https')
             if not window and i > 0:
                 tab = False
                 bg = True
-            if tab or bg or window:
-                self._open(cur_url, tab, bg, window, not implicit)
+
+            if tab or bg or window or private:
+                self._open(cur_url, tab, bg, window, related=related,
+                           private=private)
             else:
                 curtab = self._cntwidget(count)
                 if curtab is None:
@@ -271,6 +317,8 @@ class CommandDispatcher:
                     else:
                         # Explicit count with a tab that doesn't exist.
                         return
+                elif curtab.data.pinned:
+                    message.info("Tab is pinned!")
                 else:
                     curtab.openurl(cur_url)
 
@@ -350,7 +398,7 @@ class CommandDispatcher:
                 message.error("Printing failed!")
 
         tab.printing.check_preview_support()
-        diag = QPrintPreviewDialog()
+        diag = QPrintPreviewDialog(tab)
         diag.setAttribute(Qt.WA_DeleteOnClose)
         diag.setWindowFlags(diag.windowFlags() | Qt.WindowMaximizeButtonHint |
                             Qt.WindowMinimizeButtonHint)
@@ -376,9 +424,18 @@ class CommandDispatcher:
                 message.error("Printing failed!")
             diag.deleteLater()
 
-        diag = QPrintDialog()
-        diag.open(lambda: tab.printing.to_printer(diag.printer(),
-                                                  print_callback))
+        def do_print():
+            """Called when the dialog was closed."""
+            tab.printing.to_printer(diag.printer(), print_callback)
+
+        diag = QPrintDialog(tab)
+        if utils.is_mac:
+            # For some reason we get a segfault when using open() on macOS
+            ret = diag.exec_()
+            if ret == QDialog.Accepted:
+                do_print()
+        else:
+            diag.open(do_print)
 
     @cmdutils.register(instance='command-dispatcher', name='print',
                        scope='window')
@@ -433,25 +490,27 @@ class CommandDispatcher:
             raise cmdexc.CommandError(e)
 
         # The new tab could be in a new tabbed_browser (e.g. because of
-        # tabs-are-windows being set)
+        # tabs.tabs_are_windows being set)
         if window:
-            new_tabbed_browser = self._new_tabbed_browser()
+            new_tabbed_browser = self._new_tabbed_browser(
+                private=self._tabbed_browser.private)
         else:
             new_tabbed_browser = self._tabbed_browser
-        newtab = new_tabbed_browser.tabopen(background=bg, explicit=True)
+        newtab = new_tabbed_browser.tabopen(background=bg)
         new_tabbed_browser = objreg.get('tabbed-browser', scope='window',
                                         window=newtab.win_id)
         idx = new_tabbed_browser.indexOf(newtab)
 
         new_tabbed_browser.set_page_title(idx, cur_title)
-        if config.get('tabs', 'show-favicons'):
+        if config.val.tabs.favicons.show:
             new_tabbed_browser.setTabIcon(idx, curtab.icon())
-            if config.get('tabs', 'tabs-are-windows'):
+            if config.val.tabs.tabs_are_windows:
                 new_tabbed_browser.window().setWindowIcon(curtab.icon())
 
         newtab.data.keep_icon = True
         newtab.history.deserialize(history)
         newtab.zoom.set_factor(curtab.zoom.factor())
+        new_tabbed_browser.set_tab_pinned(newtab, curtab.data.pinned)
         return newtab
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
@@ -478,15 +537,13 @@ class CommandDispatcher:
         else:
             widget = self._current_widget()
 
-        for _ in range(count):
+        try:
             if forward:
-                if not widget.history.can_go_forward():
-                    raise cmdexc.CommandError("At end of history.")
-                widget.history.forward()
+                widget.history.forward(count)
             else:
-                if not widget.history.can_go_back():
-                    raise cmdexc.CommandError("At beginning of history.")
-                widget.history.back()
+                widget.history.back(count)
+        except browsertab.WebTabError as e:
+            raise cmdexc.CommandError(e)
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
     @cmdutils.argument('count', count=True)
@@ -565,7 +622,7 @@ class CommandDispatcher:
                         tab=tab, background=bg, window=window)
             elif where in ['up', 'increment', 'decrement']:
                 new_url = handlers[where](url, count)
-                self._open(new_url, tab, bg, window)
+                self._open(new_url, tab, bg, window, related=True)
             else:  # pragma: no cover
                 raise ValueError("Got called with invalid value {} for "
                                 "`where'.".format(where))
@@ -595,6 +652,9 @@ class CommandDispatcher:
     def scroll(self, direction: typing.Union[str, int], count=1):
         """Scroll the current tab in the given direction.
 
+        Note you can use `:run-with-count` to have a keybinding with a bigger
+        scroll increment.
+
         Args:
             direction: In which direction to scroll
                        (up/down/left/right/top/bottom).
@@ -628,7 +688,7 @@ class CommandDispatcher:
                        scope='window')
     @cmdutils.argument('count', count=True)
     @cmdutils.argument('horizontal', flag='x')
-    def scroll_perc(self, perc: float=None, horizontal=False, count=None):
+    def scroll_to_perc(self, perc: float = None, horizontal=False, count=None):
         """Scroll to a specific percentage of the page.
 
         The percentage can be given either as argument or as count.
@@ -664,7 +724,7 @@ class CommandDispatcher:
     @cmdutils.argument('bottom_navigate', metavar='ACTION',
                        choices=('next', 'increment'))
     def scroll_page(self, x: float, y: float, *,
-                    top_navigate: str=None, bottom_navigate: str=None,
+                    top_navigate: str = None, bottom_navigate: str = None,
                     count=1):
         """Scroll the frame page-wise.
 
@@ -711,7 +771,7 @@ class CommandDispatcher:
             url_query.setQueryDelimiters('=', ';')
         url_query.setQuery(url_query_str)
         for key in dict(url_query.queryItems()):
-            if key in config.get('general', 'yank-ignored-url-parameters'):
+            if key in config.val.url.yank_ignored_parameters:
                 url_query.removeQueryItem(key)
         url.setQuery(url_query)
         return url.toString(flags)
@@ -782,7 +842,7 @@ class CommandDispatcher:
             perc = tab.zoom.offset(count)
         except ValueError as e:
             raise cmdexc.CommandError(e)
-        message.info("Zoom level: {}%".format(perc), replace=True)
+        message.info("Zoom level: {}%".format(int(perc)), replace=True)
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
     @cmdutils.argument('count', count=True)
@@ -797,11 +857,11 @@ class CommandDispatcher:
             perc = tab.zoom.offset(-count)
         except ValueError as e:
             raise cmdexc.CommandError(e)
-        message.info("Zoom level: {}%".format(perc), replace=True)
+        message.info("Zoom level: {}%".format(int(perc)), replace=True)
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
     @cmdutils.argument('count', count=True)
-    def zoom(self, zoom: int=None, count=None):
+    def zoom(self, zoom=None, count=None):
         """Set the zoom level for the current tab.
 
         The zoom can be given as argument or as [count]. If neither is
@@ -812,39 +872,61 @@ class CommandDispatcher:
             zoom: The zoom percentage to set.
             count: The zoom percentage to set.
         """
+        if zoom is not None:
+            try:
+                zoom = int(zoom.rstrip('%'))
+            except ValueError:
+                raise cmdexc.CommandError("zoom: Invalid int value {}"
+                                          .format(zoom))
+
         level = count if count is not None else zoom
         if level is None:
-            level = config.get('ui', 'default-zoom')
+            level = config.val.zoom.default
         tab = self._current_widget()
 
         try:
             tab.zoom.set_factor(float(level) / 100)
         except ValueError:
             raise cmdexc.CommandError("Can't zoom {}%!".format(level))
-        message.info("Zoom level: {}%".format(level), replace=True)
+        message.info("Zoom level: {}%".format(int(level)), replace=True)
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
-    def tab_only(self, prev=False, next_=False):
+    def tab_only(self, prev=False, next_=False, force=False):
         """Close all tabs except for the current one.
 
         Args:
             prev: Keep tabs before the current.
             next_: Keep tabs after the current.
+            force: Avoid confirmation for pinned tabs.
         """
         cmdutils.check_exclusive((prev, next_), 'pn')
         cur_idx = self._tabbed_browser.currentIndex()
         assert cur_idx != -1
 
+        def _to_close(i):
+            """Helper method to check if a tab should be closed or not."""
+            return not (i == cur_idx or
+                        (prev and i < cur_idx) or
+                        (next_ and i > cur_idx))
+
+        # Check to see if we are closing any pinned tabs
+        if not force:
+            for i, tab in enumerate(self._tabbed_browser.widgets()):
+                if _to_close(i) and tab.data.pinned:
+                    self._tabbed_browser.tab_close_prompt_if_pinned(
+                        tab,
+                        force,
+                        lambda: self.tab_only(
+                            prev=prev, next_=next_, force=True))
+                    return
+
         for i, tab in enumerate(self._tabbed_browser.widgets()):
-            if (i == cur_idx or (prev and i < cur_idx) or
-                    (next_ and i > cur_idx)):
-                continue
-            else:
+            if _to_close(i):
                 self._tabbed_browser.close_tab(tab)
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
     def undo(self):
-        """Re-open a closed tab (optionally skipping [count] closed tabs)."""
+        """Re-open a closed tab."""
         try:
             self._tabbed_browser.undo()
         except IndexError:
@@ -865,7 +947,7 @@ class CommandDispatcher:
         newidx = self._current_index() - count
         if newidx >= 0:
             self._set_current_index(newidx)
-        elif config.get('tabs', 'wrap'):
+        elif config.val.tabs.wrap:
             self._set_current_index(newidx % self._count())
         else:
             raise cmdexc.CommandError("First tab")
@@ -885,7 +967,7 @@ class CommandDispatcher:
         newidx = self._current_index() + count
         if newidx < self._count():
             self._set_current_index(newidx)
-        elif config.get('tabs', 'wrap'):
+        elif config.val.tabs.wrap:
             self._set_current_index(newidx % self._count())
         else:
             raise cmdexc.CommandError("Last tab")
@@ -928,31 +1010,39 @@ class CommandDispatcher:
             self._open(url, tab, bg, window)
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
-    @cmdutils.argument('index', completion=usertypes.Completion.tab)
-    def buffer(self, index):
+    @cmdutils.argument('index', completion=miscmodels.buffer)
+    @cmdutils.argument('count', count=True)
+    def buffer(self, index=None, count=None):
         """Select tab by index or url/title best match.
 
-        Focuses window if necessary.
+        Focuses window if necessary when index is given. If both index and
+        count are given, use count.
 
         Args:
             index: The [win_id/]index of the tab to focus. Or a substring
                    in which case the closest match will be focused.
+            count: The tab index to focus, starting with 1.
         """
-        index_parts = index.split('/', 1)
+        if count is not None:
+            index_parts = [count]
+        elif index is None:
+            raise cmdexc.CommandError("buffer: Either a count or the argument "
+                                      "index must be specified.")
+        else:
+            index_parts = index.split('/', 1)
 
-        try:
-            for part in index_parts:
-                int(part)
-        except ValueError:
-            model = instances.get(usertypes.Completion.tab)
-            sf = sortfilter.CompletionFilterModel(source=model)
-            sf.set_pattern(index)
-            if sf.count() > 0:
-                index = sf.data(sf.first_item())
-                index_parts = index.split('/', 1)
-            else:
-                raise cmdexc.CommandError(
-                    "No matching tab for: {}".format(index))
+            try:
+                for part in index_parts:
+                    int(part)
+            except ValueError:
+                model = miscmodels.buffer()
+                model.set_pattern(index)
+                if model.count() > 0:
+                    index = model.data(model.first_item())
+                    index_parts = index.split('/', 1)
+                else:
+                    raise cmdexc.CommandError(
+                        "No matching tab for: {}".format(index))
 
         if len(index_parts) == 2:
             win_id = int(index_parts[0])
@@ -984,7 +1074,7 @@ class CommandDispatcher:
     @cmdutils.register(instance='command-dispatcher', scope='window')
     @cmdutils.argument('index', choices=['last'])
     @cmdutils.argument('count', count=True)
-    def tab_focus(self, index: typing.Union[str, int]=None, count=None):
+    def tab_focus(self, index: typing.Union[str, int] = None, count=None):
         """Select the tab given as argument/[count].
 
         If neither count nor index are given, it behaves like tab-next.
@@ -997,12 +1087,15 @@ class CommandDispatcher:
                    last tab.
             count: The tab index to focus, starting with 1.
         """
+        index = count if count is not None else index
+
         if index == 'last':
             self._tab_focus_last()
             return
-        index = count if count is not None else index
-
-        if index is None:
+        elif index == self._current_index() + 1:
+            self._tab_focus_last(show_error=False)
+            return
+        elif index is None:
             self.tab_next()
             return
 
@@ -1018,7 +1111,7 @@ class CommandDispatcher:
     @cmdutils.register(instance='command-dispatcher', scope='window')
     @cmdutils.argument('index', choices=['+', '-'])
     @cmdutils.argument('count', count=True)
-    def tab_move(self, index: typing.Union[str, int]=None, count=None):
+    def tab_move(self, index: typing.Union[str, int] = None, count=None):
         """Move the current tab according to the argument and [count].
 
         If neither is given, move it to the first position.
@@ -1040,7 +1133,7 @@ class CommandDispatcher:
             elif index == '+':  # pragma: no branch
                 new_idx += delta
 
-            if config.get('tabs', 'wrap'):
+            if config.val.tabs.wrap:
                 new_idx %= self._count()
         else:
             # absolute moving
@@ -1076,6 +1169,7 @@ class CommandDispatcher:
             detach: Whether the command should be detached from qutebrowser.
             cmdline: The commandline to execute.
         """
+        cmdutils.check_exclusive((userscript, detach), 'ud')
         try:
             cmd, *args = shlex.split(cmdline)
         except ValueError as e:
@@ -1101,7 +1195,7 @@ class CommandDispatcher:
     @cmdutils.register(instance='command-dispatcher', scope='window')
     def home(self):
         """Open main startpage in current tab."""
-        self.openurl(config.get('general', 'startpage')[0])
+        self._current_widget().openurl(config.val.url.start_pages[0])
 
     def _run_userscript(self, cmd, *args, verbose=False):
         """Run a userscript given as argument.
@@ -1150,8 +1244,7 @@ class CommandDispatcher:
 
     @cmdutils.register(instance='command-dispatcher', scope='window',
                        maxsplit=0)
-    @cmdutils.argument('name',
-                       completion=usertypes.Completion.quickmark_by_name)
+    @cmdutils.argument('name', completion=miscmodels.quickmark)
     def quickmark_load(self, name, tab=False, bg=False, window=False):
         """Load a quickmark.
 
@@ -1169,8 +1262,7 @@ class CommandDispatcher:
 
     @cmdutils.register(instance='command-dispatcher', scope='window',
                        maxsplit=0)
-    @cmdutils.argument('name',
-                       completion=usertypes.Completion.quickmark_by_name)
+    @cmdutils.argument('name', completion=miscmodels.quickmark)
     def quickmark_del(self, name=None):
         """Delete a quickmark.
 
@@ -1227,12 +1319,12 @@ class CommandDispatcher:
         except urlmarks.Error as e:
             raise cmdexc.CommandError(str(e))
         else:
-            msg = "Bookmarked {}!" if was_added else "Removed bookmark {}!"
+            msg = "Bookmarked {}" if was_added else "Removed bookmark {}"
             message.info(msg.format(url.toDisplayString()))
 
     @cmdutils.register(instance='command-dispatcher', scope='window',
                        maxsplit=0)
-    @cmdutils.argument('url', completion=usertypes.Completion.bookmark_by_url)
+    @cmdutils.argument('url', completion=miscmodels.bookmark)
     def bookmark_load(self, url, tab=False, bg=False, window=False,
                       delete=False):
         """Load a bookmark.
@@ -1254,7 +1346,7 @@ class CommandDispatcher:
 
     @cmdutils.register(instance='command-dispatcher', scope='window',
                        maxsplit=0)
-    @cmdutils.argument('url', completion=usertypes.Completion.bookmark_by_url)
+    @cmdutils.argument('url', completion=miscmodels.bookmark)
     def bookmark_del(self, url=None):
         """Delete a bookmark.
 
@@ -1331,6 +1423,9 @@ class CommandDispatcher:
                                       scope='window', window=self._win_id)
         target = None
         if dest is not None:
+            dest = downloads.transform_path(dest)
+            if dest is None:
+                raise cmdexc.CommandError("Invalid target filename")
             target = downloads.FileDownloadTarget(dest)
 
         tab = self._current_widget()
@@ -1356,8 +1451,18 @@ class CommandDispatcher:
                 download_manager.get_mhtml(tab, target)
         else:
             qnam = tab.networkaccessmanager()
-            download_manager.get(self._current_url(), user_agent=user_agent,
-                                 qnam=qnam, target=target)
+
+            suggested_fn = downloads.suggested_fn_from_title(
+                self._current_url().path(), tab.title()
+            )
+
+            download_manager.get(
+                self._current_url(),
+                user_agent=user_agent,
+                qnam=qnam,
+                target=target,
+                suggested_fn=suggested_fn
+            )
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
     def view_source(self):
@@ -1368,19 +1473,22 @@ class CommandDispatcher:
         if tab.data.viewing_source:
             raise cmdexc.CommandError("Already viewing source!")
 
+        try:
+            current_url = self._current_url()
+        except cmdexc.CommandError as e:
+            message.error(str(e))
+            return
+
         def show_source_cb(source):
             """Show source as soon as it's ready."""
             lexer = pygments.lexers.HtmlLexer()
-            formatter = pygments.formatters.HtmlFormatter(full=True,
-                                                          linenos='table')
+            formatter = pygments.formatters.HtmlFormatter(
+                full=True, linenos='table',
+                title='Source for {}'.format(current_url.toDisplayString()))
             highlighted = pygments.highlight(source, lexer, formatter)
-            try:
-                current_url = self._current_url()
-            except cmdexc.CommandError as e:
-                message.error(str(e))
-                return
-            new_tab = self._tabbed_browser.tabopen(explicit=True)
-            new_tab.set_html(highlighted, current_url)
+
+            new_tab = self._tabbed_browser.tabopen()
+            new_tab.set_html(highlighted)
             new_tab.data.viewing_source = True
 
         tab.dump_async(show_source_cb)
@@ -1422,7 +1530,7 @@ class CommandDispatcher:
 
     @cmdutils.register(instance='command-dispatcher', name='help',
                        scope='window')
-    @cmdutils.argument('topic', completion=usertypes.Completion.helptopic)
+    @cmdutils.argument('topic', completion=miscmodels.helptopic)
     def show_help(self, tab=False, bg=False, window=False, topic=None):
         r"""Show help about a command or setting.
 
@@ -1433,7 +1541,7 @@ class CommandDispatcher:
             topic: The topic to show help for.
 
                    - :__command__ for commands.
-                   - __section__\->__option__ for settings.
+                   - __section__.__option__ for settings.
         """
         if topic is None:
             path = 'index.html'
@@ -1443,27 +1551,15 @@ class CommandDispatcher:
                 raise cmdexc.CommandError("Invalid command {}!".format(
                     command))
             path = 'commands.html#{}'.format(command)
-        elif '->' in topic:
-            parts = topic.split('->')
-            if len(parts) != 2:
-                raise cmdexc.CommandError("Invalid help topic {}!".format(
-                    topic))
-            try:
-                config.get(*parts)
-            except configexc.NoSectionError:
-                raise cmdexc.CommandError("Invalid section {}!".format(
-                    parts[0]))
-            except configexc.NoOptionError:
-                raise cmdexc.CommandError("Invalid option {}!".format(
-                    parts[1]))
-            path = 'settings.html#{}'.format(topic.replace('->', '-'))
+        elif topic in configdata.DATA:
+            path = 'settings.html#{}'.format(topic)
         else:
             raise cmdexc.CommandError("Invalid help topic {}!".format(topic))
         url = QUrl('qute://help/{}'.format(path))
         self._open(url, tab, bg, window)
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
-    def messages(self, level='error', plain=False, tab=False, bg=False,
+    def messages(self, level='info', plain=False, tab=False, bg=False,
                  window=False):
         """Show a log of past messages.
 
@@ -1496,6 +1592,7 @@ class CommandDispatcher:
         if text is None:
             message.error("Could not get text from the focused element.")
             return
+        assert isinstance(text, str), text
 
         ed = editor.ExternalEditor(self._tabbed_browser)
         ed.editing_finished.connect(functools.partial(
@@ -1508,7 +1605,7 @@ class CommandDispatcher:
         """Open an external editor with the currently selected form field.
 
         The editor which should be launched can be configured via the
-        `general -> editor` config option.
+        `editor.command` config option.
         """
         tab = self._current_widget()
         tab.elements.find_focused(self._open_editor_cb)
@@ -1533,10 +1630,7 @@ class CommandDispatcher:
                        backend=usertypes.Backend.QtWebKit)
     def paste_primary(self):
         """Paste the primary selection at cursor position."""
-        try:
-            self.insert_text(utils.get_clipboard(selection=True))
-        except utils.SelectionUnsupportedError:
-            self.insert_text(utils.get_clipboard())
+        self.insert_text(utils.get_clipboard(selection=True, fallback=True))
 
     @cmdutils.register(instance='command-dispatcher', maxsplit=0,
                        scope='window')
@@ -1564,7 +1658,7 @@ class CommandDispatcher:
                        hide=True)
     @cmdutils.argument('filter_', choices=['id'])
     def click_element(self, filter_: str, value, *,
-                      target: usertypes.ClickTarget=
+                      target: usertypes.ClickTarget =
                       usertypes.ClickTarget.normal,
                       force_event=False):
         """Click the element matching the given filter.
@@ -1645,23 +1739,25 @@ class CommandDispatcher:
         """
         self.set_mark("'")
         tab = self._current_widget()
-        tab.search.clear()
+        if tab.search.search_displayed:
+            tab.search.clear()
+
+        if not text:
+            return
 
         options = {
-            'ignore_case': config.get('general', 'ignore-case'),
+            'ignore_case': config.val.ignore_case,
             'reverse': reverse,
         }
+
         self._tabbed_browser.search_text = text
         self._tabbed_browser.search_options = dict(options)
 
-        if text:
-            cb = functools.partial(self._search_cb, tab=tab,
-                                   old_scroll_pos=tab.scroller.pos_px(),
-                                   options=options, text=text, prev=False)
-        else:
-            cb = None
-
+        cb = functools.partial(self._search_cb, tab=tab,
+                               old_scroll_pos=tab.scroller.pos_px(),
+                               options=options, text=text, prev=False)
         options['result_cb'] = cb
+
         tab.search.search(text, **options)
 
     @cmdutils.register(instance='command-dispatcher', hide=True,
@@ -1897,38 +1993,25 @@ class CommandDispatcher:
     def debug_webaction(self, action, count=1):
         """Execute a webaction.
 
-        See http://doc.qt.io/qt-5/qwebpage.html#WebAction-enum for the
-        available actions.
+        Available actions:
+        http://doc.qt.io/archives/qt-5.5/qwebpage.html#WebAction-enum (WebKit)
+        http://doc.qt.io/qt-5/qwebenginepage.html#WebAction-enum (WebEngine)
 
         Args:
             action: The action to execute, e.g. MoveToNextChar.
             count: How many times to repeat the action.
         """
         tab = self._current_widget()
-
-        if tab.backend == usertypes.Backend.QtWebKit:
-            assert QWebPage is not None
-            member = getattr(QWebPage, action, None)
-            base = QWebPage.WebAction
-        elif tab.backend == usertypes.Backend.QtWebEngine:
-            assert QWebEnginePage is not None
-            member = getattr(QWebEnginePage, action, None)
-            base = QWebEnginePage.WebAction
-
-        if not isinstance(member, base):
-            raise cmdexc.CommandError("{} is not a valid web action!".format(
-                action))
-
         for _ in range(count):
-            # This whole command is backend-specific anyways, so it makes no
-            # sense to introduce some API for this.
-            # pylint: disable=protected-access
-            tab._widget.triggerPageAction(member)
+            try:
+                tab.action.run_string(action)
+            except browsertab.WebTabError as e:
+                raise cmdexc.CommandError(str(e))
 
     @cmdutils.register(instance='command-dispatcher', scope='window',
                        maxsplit=0, no_cmd_split=True)
     def jseval(self, js_code, file=False, quiet=False, *,
-               world: typing.Union[usertypes.JsWorld, int]=None):
+               world: typing.Union[usertypes.JsWorld, int] = None):
         """Evaluate a JavaScript string.
 
         Args:
@@ -1962,8 +2045,9 @@ class CommandDispatcher:
                     message.info(out)
 
         if file:
+            path = os.path.expanduser(js_code)
             try:
-                with open(js_code, 'r', encoding='utf-8') as f:
+                with open(path, 'r', encoding='utf-8') as f:
                     js_code = f.read()
             except OSError as e:
                 raise cmdexc.CommandError(str(e))
@@ -2016,7 +2100,7 @@ class CommandDispatcher:
         """Navigate to a url formed in an external editor.
 
         The editor which should be launched can be configured via the
-        `general -> editor` config option.
+        `editor.command` config option.
 
         Args:
             url: URL to edit; defaults to the current page url.
@@ -2088,6 +2172,10 @@ class CommandDispatcher:
 
         window = self._tabbed_browser.window()
         if window.isFullScreen():
-            window.showNormal()
+            window.setWindowState(
+                window.state_before_fullscreen & ~Qt.WindowFullScreen)
         else:
+            window.state_before_fullscreen = window.windowState()
             window.showFullScreen()
+        log.misc.debug('state before fullscreen: {}'.format(
+            debug.qflags_key(Qt, window.state_before_fullscreen)))

qutebrowser/browser/downloads.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -19,11 +19,13 @@
 
 """Shared QtWebKit/QtWebEngine code for downloads."""
 
+import re
 import sys
 import html
 import os.path
 import collections
 import functools
+import pathlib
 import tempfile
 
 import sip
@@ -67,15 +69,22 @@ class UnsupportedOperationError(Exception):
 
 def download_dir():
     """Get the download directory to use."""
-    directory = config.get('storage', 'download-directory')
-    remember_dir = config.get('storage', 'remember-download-directory')
+    directory = config.val.downloads.location.directory
+    remember_dir = config.val.downloads.location.remember
 
     if remember_dir and last_used_directory is not None:
-        return last_used_directory
+        ddir = last_used_directory
     elif directory is None:
-        return standarddir.download()
+        ddir = standarddir.download()
     else:
-        return directory
+        ddir = directory
+
+    try:
+        os.makedirs(ddir)
+    except FileExistsError:
+        pass
+
+    return ddir
 
 
 def immediate_download_path(prompt_download_directory=None):
@@ -86,11 +95,10 @@ def immediate_download_path(prompt_download_directory=None):
     Args:
         prompt_download_directory: If this is something else than None, it
                                    will overwrite the
-                                   storage->prompt-download-directory setting.
+                                   downloads.location.prompt setting.
     """
     if prompt_download_directory is None:
-        prompt_download_directory = config.get('storage',
-                                               'prompt-download-directory')
+        prompt_download_directory = config.val.downloads.location.prompt
 
     if not prompt_download_directory:
         return download_dir()
@@ -102,7 +110,7 @@ def _path_suggestion(filename):
     Args:
         filename: The filename to use if included in the suggestion.
     """
-    suggestion = config.get('completion', 'download-path-suggestion')
+    suggestion = config.val.downloads.location.suggestion
     if suggestion == 'path':
         # add trailing '/' if not present
         return os.path.join(download_dir(), '')
@@ -161,6 +169,47 @@ def get_filename_question(*, suggested_filename, url, parent=None):
     return q
 
 
+def transform_path(path):
+    r"""Do platform-specific transformations, like changing E: to E:\.
+
+    Returns None if the path is invalid on the current platform.
+    """
+    if not utils.is_windows:
+        return path
+    path = utils.expand_windows_drive(path)
+    # Drive dependent working directories are not supported, e.g.
+    # E:filename is invalid
+    if re.match(r'[A-Z]:[^\\]', path, re.IGNORECASE):
+        return None
+    # Paths like COM1, ...
+    # See https://github.com/qutebrowser/qutebrowser/issues/82
+    if pathlib.Path(path).is_reserved():
+        return None
+    return path
+
+
+def suggested_fn_from_title(url_path, title=None):
+    """Suggest a filename depending on the URL extension and page title.
+
+    Args:
+        url_path: a string with the URL path
+        title: the page title string
+
+    Return:
+        The download filename based on the title, or None if the extension is
+        not found in the whitelist (or if there is no page title).
+    """
+    ext_whitelist = [".html", ".htm", ".php", ""]
+    _, ext = os.path.splitext(url_path)
+    if ext.lower() in ext_whitelist and title:
+        suggested_fn = utils.sanitize_filename(title)
+        if not suggested_fn.lower().endswith((".html", ".htm")):
+            suggested_fn += ".html"
+    else:
+        suggested_fn = None
+    return suggested_fn
+
+
 class NoFilenameError(Exception):
 
     """Raised when we can't find out a filename in DownloadTarget."""
@@ -451,13 +500,13 @@ class AbstractDownloadItem(QObject):
         Args:
             position: The color type requested, can be 'fg' or 'bg'.
         """
-        # pylint: disable=bad-config-call
-        # WORKAROUND for https://bitbucket.org/logilab/astroid/issue/104/
         assert position in ["fg", "bg"]
-        start = config.get('colors', 'downloads.{}.start'.format(position))
-        stop = config.get('colors', 'downloads.{}.stop'.format(position))
-        system = config.get('colors', 'downloads.{}.system'.format(position))
-        error = config.get('colors', 'downloads.{}.error'.format(position))
+        # pylint: disable=bad-config-option
+        start = getattr(config.val.colors.downloads.start, position)
+        stop = getattr(config.val.colors.downloads.stop, position)
+        system = getattr(config.val.colors.downloads.system, position)
+        error = getattr(config.val.colors.downloads.error, position)
+        # pylint: enable=bad-config-option
         if self.error_msg is not None:
             assert not self.successful
             return error
@@ -507,6 +556,14 @@ class AbstractDownloadItem(QObject):
         """Retry a failed download."""
         raise NotImplementedError
 
+    @pyqtSlot()
+    def try_retry(self):
+        """Try to retry a download and show an error if it's unsupported."""
+        try:
+            self.retry()
+        except UnsupportedOperationError as e:
+            message.error(str(e))
+
     def _get_open_filename(self):
         """Get the filename to open a download.
 
@@ -521,7 +578,7 @@ class AbstractDownloadItem(QObject):
         Args:
             cmdline: The command to use as string. A `{}` is expanded to the
                      filename. None means to use the system's default
-                     application or `default-open-dispatcher` if set. If no
+                     application or `downloads.open_dispatcher` if set. If no
                      `{}` is found, the filename is appended to the cmdline.
         """
         assert self.successful
@@ -529,7 +586,11 @@ class AbstractDownloadItem(QObject):
         if filename is None:  # pragma: no cover
             log.downloads.error("No filename to open the download!")
             return
-        utils.open_file(filename, cmdline)
+        # By using a singleshot timer, we ensure that we return fast. This
+        # is important on systems where process creation takes long, as
+        # otherwise the prompt might hang around and cause bugs
+        # (see issue #2296)
+        QTimer.singleShot(0, lambda: utils.open_file(filename, cmdline))
 
     def _ensure_can_set_filename(self, filename):
         """Make sure we can still set a filename."""
@@ -702,7 +763,7 @@ class AbstractDownloadManager(QObject):
         download.remove_requested.connect(functools.partial(
             self._remove_item, download))
 
-        delay = config.get('ui', 'remove-finished-downloads')
+        delay = config.val.downloads.remove_finished
         if delay > -1:
             download.finished.connect(
                 lambda: QTimer.singleShot(delay, download.remove))
@@ -919,7 +980,7 @@ class DownloadModel(QAbstractListModel):
 
     @cmdutils.register(instance='download-model', scope='window', maxsplit=0)
     @cmdutils.argument('count', count=True)
-    def download_open(self, cmdline: str=None, count=0):
+    def download_open(self, cmdline: str = None, count=0):
         """Open the last/[count]th download.
 
         If no specific command is given, this will use the system's default
@@ -964,7 +1025,7 @@ class DownloadModel(QAbstractListModel):
                 raise cmdexc.CommandError("No failed downloads!")
             else:
                 download = to_retry[0]
-        download.retry()
+        download.try_retry()
 
     def can_clear(self):
         """Check if there are finished downloads to clear."""

qutebrowser/browser/downloadview.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -23,10 +23,10 @@ import functools
 
 import sip
 from PyQt5.QtCore import pyqtSlot, QSize, Qt, QTimer
-from PyQt5.QtWidgets import QListView, QSizePolicy, QMenu
+from PyQt5.QtWidgets import QListView, QSizePolicy, QMenu, QStyleFactory
 
 from qutebrowser.browser import downloads
-from qutebrowser.config import style
+from qutebrowser.config import config
 from qutebrowser.utils import qtutils, utils, objreg
 
 
@@ -64,8 +64,8 @@ class DownloadView(QListView):
 
     STYLESHEET = """
         QListView {
-            background-color: {{ color['downloads.bg.bar'] }};
-            font: {{ font['downloads'] }};
+            background-color: {{ conf.colors.downloads.bar.bg }};
+            font: {{ conf.fonts.downloads }};
         }
 
         QListView::item {
@@ -75,7 +75,8 @@ class DownloadView(QListView):
 
     def __init__(self, win_id, parent=None):
         super().__init__(parent)
-        style.set_register_stylesheet(self)
+        self.setStyle(QStyleFactory.create('Fusion'))
+        config.set_register_stylesheet(self)
         self.setResizeMode(QListView.Adjust)
         self.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
         self.setSizePolicy(QSizePolicy.MinimumExpanding, QSizePolicy.Fixed)
@@ -134,7 +135,7 @@ class DownloadView(QListView):
             if item.successful:
                 actions.append(("Open", item.open_file))
             else:
-                actions.append(("Retry", item.retry))
+                actions.append(("Retry", item.try_retry))
             actions.append(("Remove", item.remove))
         else:
             actions.append(("Cancel", item.cancel))

qutebrowser/browser/hints.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -26,10 +26,11 @@ import re
 import html
 from string import ascii_lowercase
 
+import attr
 from PyQt5.QtCore import pyqtSlot, QObject, Qt, QUrl
 from PyQt5.QtWidgets import QLabel
 
-from qutebrowser.config import config, style
+from qutebrowser.config import config
 from qutebrowser.keyinput import modeman, modeparsers
 from qutebrowser.browser import webelem
 from qutebrowser.commands import userscripts, cmdexc, cmdutils, runners
@@ -65,10 +66,10 @@ class HintLabel(QLabel):
 
     STYLESHEET = """
         QLabel {
-            background-color: {{ color['hints.bg'] }};
-            color: {{ color['hints.fg'] }};
-            font: {{ font['hints'] }};
-            border: {{ config.get('hints', 'border') }};
+            background-color: {{ conf.colors.hints.bg }};
+            color: {{ conf.colors.hints.fg }};
+            font: {{ conf.fonts.hints }};
+            border: {{ conf.hints.border }};
             padding-left: -3px;
             padding-right: -3px;
         }
@@ -80,7 +81,7 @@ class HintLabel(QLabel):
         self.elem = elem
 
         self.setAttribute(Qt.WA_StyledBackground, True)
-        style.set_register_stylesheet(self)
+        config.set_register_stylesheet(self)
 
         self._context.tab.contents_size_changed.connect(self._move_to_elem)
         self._move_to_elem()
@@ -100,7 +101,7 @@ class HintLabel(QLabel):
             matched: The part of the text which was typed.
             unmatched: The part of the text which was not typed yet.
         """
-        if (config.get('hints', 'uppercase') and
+        if (config.val.hints.uppercase and
                 self._context.hint_mode in ['letter', 'word']):
             matched = html.escape(matched.upper())
             unmatched = html.escape(unmatched.upper())
@@ -108,7 +109,7 @@ class HintLabel(QLabel):
             matched = html.escape(matched)
             unmatched = html.escape(unmatched)
 
-        match_color = html.escape(config.get('colors', 'hints.fg.match'))
+        match_color = html.escape(config.val.colors.hints.match.fg)
         self.setText('<font color="{}">{}</font>{}'.format(
             match_color, matched, unmatched))
         self.adjustSize()
@@ -121,7 +122,7 @@ class HintLabel(QLabel):
             log.hints.debug("Frame for {!r} vanished!".format(self))
             self.hide()
             return
-        no_js = config.get('hints', 'find-implementation') != 'javascript'
+        no_js = config.val.hints.find_implementation != 'javascript'
         rect = self.elem.rect_on_view(no_js=no_js)
         self.move(rect.x(), rect.y())
 
@@ -131,6 +132,7 @@ class HintLabel(QLabel):
         self.deleteLater()
 
 
+@attr.s
 class HintContext:
 
     """Context namespace used for hinting.
@@ -158,19 +160,18 @@ class HintContext:
         group: The group of web elements to hint.
     """
 
-    def __init__(self):
-        self.all_labels = []
-        self.labels = {}
-        self.target = None
-        self.baseurl = None
-        self.to_follow = None
-        self.rapid = False
-        self.add_history = False
-        self.filterstr = None
-        self.args = []
-        self.tab = None
-        self.group = None
-        self.hint_mode = None
+    all_labels = attr.ib(attr.Factory(list))
+    labels = attr.ib(attr.Factory(dict))
+    target = attr.ib(None)
+    baseurl = attr.ib(None)
+    to_follow = attr.ib(None)
+    rapid = attr.ib(False)
+    add_history = attr.ib(False)
+    filterstr = attr.ib(None)
+    args = attr.ib(attr.Factory(list))
+    tab = attr.ib(None)
+    group = attr.ib(None)
+    hint_mode = attr.ib(None)
 
     def get_args(self, urlstr):
         """Get the arguments, with {hint-url} replaced by the given URL."""
@@ -203,7 +204,7 @@ class HintActions:
             Target.window: usertypes.ClickTarget.window,
             Target.hover: usertypes.ClickTarget.normal,
         }
-        if config.get('tabs', 'background-tabs'):
+        if config.val.tabs.background:
             target_mapping[Target.tab] = usertypes.ClickTarget.tab_bg
         else:
             target_mapping[Target.tab] = usertypes.ClickTarget.tab
@@ -389,6 +390,7 @@ class HintManager(QObject):
 
     def _cleanup(self):
         """Clean up after hinting."""
+        # pylint: disable=not-an-iterable
         for label in self._context.all_labels:
             label.cleanup()
 
@@ -421,9 +423,9 @@ class HintManager(QObject):
         if hint_mode == 'number':
             chars = '0123456789'
         else:
-            chars = config.get('hints', 'chars')
-        min_chars = config.get('hints', 'min-chars')
-        if config.get('hints', 'scatter') and hint_mode != 'number':
+            chars = config.val.hints.chars
+        min_chars = config.val.hints.min_chars
+        if config.val.hints.scatter and hint_mode != 'number':
             return self._hint_scattered(min_chars, chars, elems)
         else:
             return self._hint_linear(min_chars, chars, elems)
@@ -579,12 +581,10 @@ class HintManager(QObject):
         if elems is None:
             message.error("There was an error while getting hint elements")
             return
-
-        filterfunc = webelem.FILTERS.get(self._context.group, lambda e: True)
-        elems = [e for e in elems if filterfunc(e)]
         if not elems:
             message.error("No elements found.")
             return
+
         strings = self._hint_strings(elems)
         log.hints.debug("hints: {}".format(', '.join(strings)))
 
@@ -605,7 +605,7 @@ class HintManager(QObject):
         modeman.enter(self._win_id, usertypes.KeyMode.hint,
                       'HintManager.start')
 
-        # to make auto-follow == 'always' work
+        # to make auto_follow == 'always' work
         self._handle_auto_follow()
 
     @cmdutils.register(instance='hintmanager', scope='tab', name='hint',
@@ -617,7 +617,7 @@ class HintManager(QObject):
 
         Args:
             rapid: Whether to do rapid hinting. This is only possible with
-                   targets `tab` (with background-tabs=true), `tab-bg`,
+                   targets `tab` (with `tabs.background_tabs=true`), `tab-bg`,
                    `window`, `run`, `hover`, `userscript` and `spawn`.
             add_history: Whether to add the spawned or yanked link to the
                          browsing history.
@@ -633,7 +633,7 @@ class HintManager(QObject):
                 - `normal`: Open the link.
                 - `current`: Open the link in the current tab.
                 - `tab`: Open the link in a new tab (honoring the
-                         background-tabs setting).
+                         `tabs.background_tabs` setting).
                 - `tab-fg`: Open the link in a new foreground tab.
                 - `tab-bg`: Open the link in a new background tab.
                 - `window`: Open the link in a new window.
@@ -651,7 +651,7 @@ class HintManager(QObject):
             mode: The hinting mode to use.
 
                 - `number`: Use numeric hints.
-                - `letter`: Use the chars in the hints->chars settings.
+                - `letter`: Use the chars in the hints.chars setting.
                 - `word`: Use hint words based on the html elements and the
                           extra words.
 
@@ -686,8 +686,7 @@ class HintManager(QObject):
                           Target.hover, Target.userscript, Target.spawn,
                           Target.download, Target.normal, Target.current]:
                 pass
-            elif (target == Target.tab and
-                  config.get('tabs', 'background-tabs')):
+            elif target == Target.tab and config.val.tabs.background:
                 pass
             else:
                 name = target.name.replace('_', '-')
@@ -695,7 +694,7 @@ class HintManager(QObject):
                                           "target {}!".format(name))
 
         if mode is None:
-            mode = config.get('hints', 'mode')
+            mode = config.val.hints.mode
 
         self._check_args(target, *args)
         self._context = HintContext()
@@ -722,7 +721,7 @@ class HintManager(QObject):
         return self._context.hint_mode
 
     def _handle_auto_follow(self, keystr="", filterstr="", visible=None):
-        """Handle the auto-follow option."""
+        """Handle the auto_follow option."""
         if visible is None:
             visible = {string: label
                        for string, label in self._context.labels.items()
@@ -731,7 +730,7 @@ class HintManager(QObject):
         if len(visible) != 1:
             return
 
-        auto_follow = config.get('hints', 'auto-follow')
+        auto_follow = config.val.hints.auto_follow
 
         if auto_follow == "always":
             follow = True
@@ -748,8 +747,8 @@ class HintManager(QObject):
             self._context.to_follow = list(visible.keys())[0]
 
         if follow:
-            # apply auto-follow-timeout
-            timeout = config.get('hints', 'auto-follow-timeout')
+            # apply auto_follow_timeout
+            timeout = config.val.hints.auto_follow_timeout
             keyparsers = objreg.get('keyparsers', scope='window',
                                     window=self._win_id)
             normal_parser = keyparsers[usertypes.KeyMode.normal]
@@ -773,9 +772,9 @@ class HintManager(QObject):
                     label.show()
                 else:
                     # element doesn't match anymore -> hide it, unless in rapid
-                    # mode and hide-unmatched-rapid-hints is false (see #1799)
+                    # mode and hide_unmatched_rapid_hints is false (see #1799)
                     if (not self._context.rapid or
-                            config.get('hints', 'hide-unmatched-rapid-hints')):
+                            config.val.hints.hide_unmatched_rapid_hints):
                         label.hide()
             except webelem.Error:
                 pass
@@ -795,7 +794,10 @@ class HintManager(QObject):
         else:
             self._context.filterstr = filterstr
 
+        log.hints.debug("Filtering hints on {!r}".format(filterstr))
+
         visible = []
+        # pylint: disable=not-an-iterable
         for label in self._context.all_labels:
             try:
                 if self._filter_matches(filterstr, str(label.elem)):
@@ -940,7 +942,7 @@ class WordHinter:
 
     def ensure_initialized(self):
         """Generate the used words if yet uninitialized."""
-        dictionary = config.get("hints", "dictionary")
+        dictionary = config.val.hints.dictionary
         if not self.words or self.dictionary != dictionary:
             self.words.clear()
             self.dictionary = dictionary

qutebrowser/browser/history.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -19,215 +19,110 @@
 
 """Simple history which gets written to disk."""
 
+import os
 import time
-import collections
 
-from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, QObject
+from PyQt5.QtCore import pyqtSlot, QUrl, QTimer
 
-from qutebrowser.commands import cmdutils
-from qutebrowser.utils import (utils, objreg, standarddir, log, qtutils,
-                               usertypes, message)
-from qutebrowser.config import config
-from qutebrowser.misc import lineparser, objects
+from qutebrowser.commands import cmdutils, cmdexc
+from qutebrowser.utils import (utils, objreg, log, usertypes, message,
+                               debug, standarddir, qtutils)
+from qutebrowser.misc import objects, sql
 
 
-class Entry:
-
-    """A single entry in the web history.
-
-    Attributes:
-        atime: The time the page was accessed.
-        url: The URL which was accessed as QUrl.
-        redirect: If True, don't save this entry to disk
-    """
-
-    def __init__(self, atime, url, title, redirect=False):
-        self.atime = float(atime)
-        self.url = url
-        self.title = title
-        self.redirect = redirect
-        qtutils.ensure_valid(url)
-
-    def __repr__(self):
-        return utils.get_repr(self, constructor=True, atime=self.atime,
-                              url=self.url_str(), title=self.title,
-                              redirect=self.redirect)
-
-    def __str__(self):
-        atime = str(int(self.atime))
-        if self.redirect:
-            atime += '-r'  # redirect flag
-        elems = [atime, self.url_str()]
-        if self.title:
-            elems.append(self.title)
-        return ' '.join(elems)
-
-    def __eq__(self, other):
-        return (self.atime == other.atime and
-                self.title == other.title and
-                self.url == other.url and
-                self.redirect == other.redirect)
-
-    def url_str(self):
-        """Get the URL as a lossless string."""
-        return self.url.toString(QUrl.FullyEncoded | QUrl.RemovePassword)
-
-    @classmethod
-    def from_str(cls, line):
-        """Parse a history line like '12345 http://example.com title'."""
-        data = line.split(maxsplit=2)
-        if len(data) == 2:
-            atime, url = data
-            title = ""
-        elif len(data) == 3:
-            atime, url, title = data
-        else:
-            raise ValueError("2 or 3 fields expected")
-
-        url = QUrl(url)
-        if not url.isValid():
-            raise ValueError("Invalid URL: {}".format(url.errorString()))
-
-        # https://github.com/qutebrowser/qutebrowser/issues/670
-        atime = atime.lstrip('\0')
-
-        if '-' in atime:
-            atime, flags = atime.split('-')
-        else:
-            flags = ''
-
-        if not set(flags).issubset('r'):
-            raise ValueError("Invalid flags {!r}".format(flags))
-
-        redirect = 'r' in flags
-
-        return cls(atime, url, title, redirect=redirect)
+# increment to indicate that HistoryCompletion must be regenerated
+_USER_VERSION = 1
 
 
-class WebHistory(QObject):
+class CompletionHistory(sql.SqlTable):
 
-    """The global history of visited pages.
+    """History which only has the newest entry for each URL."""
 
-    This is a little more complex as you'd expect so the history can be read
-    from disk async while new history is already arriving.
+    def __init__(self, parent=None):
+        super().__init__("CompletionHistory", ['url', 'title', 'last_atime'],
+                         constraints={'url': 'PRIMARY KEY',
+                                      'title': 'NOT NULL',
+                                      'last_atime': 'NOT NULL'},
+                         parent=parent)
+        self.create_index('CompletionHistoryAtimeIndex', 'last_atime')
 
-    self.history_dict is the main place where the history is stored, in an
-    OrderedDict (sorted by time) of URL strings mapped to Entry objects.
 
-    While reading from disk is still ongoing, the history is saved in
-    self._temp_history instead, and then appended to self.history_dict once
-    that's fully populated.
+class WebHistory(sql.SqlTable):
 
-    All history which is new in this session (rather than read from disk from a
-    previous browsing session) is also stored in self._new_history.
-    self._saved_count tracks how many of those entries were already written to
-    disk, so we can always append to the existing data.
+    """The global history of visited pages."""
 
-    Attributes:
-        history_dict: An OrderedDict of URLs read from the on-disk history.
-        _lineparser: The AppendLineParser used to save the history.
-        _new_history: A list of Entry items of the current session.
-        _saved_count: How many HistoryEntries have been written to disk.
-        _initial_read_started: Whether async_read was called.
-        _initial_read_done: Whether async_read has completed.
-        _temp_history: OrderedDict of temporary history entries before
-                       async_read was called.
+    def __init__(self, parent=None):
+        super().__init__("History", ['url', 'title', 'atime', 'redirect'],
+                         constraints={'url': 'NOT NULL',
+                                      'title': 'NOT NULL',
+                                      'atime': 'NOT NULL',
+                                      'redirect': 'NOT NULL'},
+                         parent=parent)
+        self.completion = CompletionHistory(parent=self)
+        if sql.Query('pragma user_version').run().value() < _USER_VERSION:
+            self.completion.delete_all()
+        if not self.completion:
+            # either the table is out-of-date or the user wiped it manually
+            self._rebuild_completion()
+        self.create_index('HistoryIndex', 'url')
+        self.create_index('HistoryAtimeIndex', 'atime')
+        self._contains_query = self.contains_query('url')
+        self._between_query = sql.Query('SELECT * FROM History '
+                                        'where not redirect '
+                                        'and not url like "qute://%" '
+                                        'and atime > :earliest '
+                                        'and atime <= :latest '
+                                        'ORDER BY atime desc')
 
-    Signals:
-        add_completion_item: Emitted before a new Entry is added.
-                             Used to sync with the completion.
-                             arg: The new Entry.
-        item_added: Emitted after a new Entry is added.
-                    Used to tell the savemanager that the history is dirty.
-                    arg: The new Entry.
-        cleared: Emitted after the history is cleared.
-    """
-
-    add_completion_item = pyqtSignal(Entry)
-    item_added = pyqtSignal(Entry)
-    cleared = pyqtSignal()
-    async_read_done = pyqtSignal()
-
-    def __init__(self, hist_dir, hist_name, parent=None):
-        super().__init__(parent)
-        self._initial_read_started = False
-        self._initial_read_done = False
-        self._lineparser = lineparser.AppendLineParser(hist_dir, hist_name,
-                                                       parent=self)
-        self.history_dict = collections.OrderedDict()
-        self._temp_history = collections.OrderedDict()
-        self._new_history = []
-        self._saved_count = 0
-        objreg.get('save-manager').add_saveable(
-            'history', self.save, self.item_added)
+        self._before_query = sql.Query('SELECT * FROM History '
+                                       'where not redirect '
+                                       'and not url like "qute://%" '
+                                       'and atime <= :latest '
+                                       'ORDER BY atime desc '
+                                       'limit :limit offset :offset')
 
     def __repr__(self):
         return utils.get_repr(self, length=len(self))
 
-    def __iter__(self):
-        return iter(self.history_dict.values())
+    def __contains__(self, url):
+        return self._contains_query.run(val=url).value()
 
-    def __len__(self):
-        return len(self.history_dict)
-
-    def async_read(self):
-        """Read the initial history."""
-        if self._initial_read_started:
-            log.init.debug("Ignoring async_read() because reading is started.")
-            return
-        self._initial_read_started = True
-
-        with self._lineparser.open():
-            for line in self._lineparser:
-                yield
-
-                line = line.rstrip()
-                if not line:
-                    continue
-
-                try:
-                    entry = Entry.from_str(line)
-                except ValueError as e:
-                    log.init.warning("Invalid history entry {!r}: {}!".format(
-                        line, e))
-                    continue
-
-                # This de-duplicates history entries; only the latest
-                # entry for each URL is kept. If you want to keep
-                # information about previous hits change the items in
-                # old_urls to be lists or change Entry to have a
-                # list of atimes.
-                self._add_entry(entry)
-
-        self._initial_read_done = True
-        self.async_read_done.emit()
-
-        for entry in self._temp_history.values():
-            self._add_entry(entry)
-            self._new_history.append(entry)
-            if not entry.redirect:
-                self.add_completion_item.emit(entry)
-        self._temp_history.clear()
-
-    def _add_entry(self, entry, target=None):
-        """Add an entry to self.history_dict or another given OrderedDict."""
-        if target is None:
-            target = self.history_dict
-        url_str = entry.url_str()
-        target[url_str] = entry
-        target.move_to_end(url_str)
+    def _rebuild_completion(self):
+        data = {'url': [], 'title': [], 'last_atime': []}
+        # select the latest entry for each url
+        q = sql.Query('SELECT url, title, max(atime) AS atime FROM History '
+                      'WHERE NOT redirect GROUP BY url ORDER BY atime asc')
+        for entry in q.run():
+            data['url'].append(self._format_completion_url(QUrl(entry.url)))
+            data['title'].append(entry.title)
+            data['last_atime'].append(entry.atime)
+        self.completion.insert_batch(data, replace=True)
+        sql.Query('pragma user_version = {}'.format(_USER_VERSION)).run()
 
     def get_recent(self):
         """Get the most recent history entries."""
-        old = self._lineparser.get_recent()
-        return old + [str(e) for e in self._new_history]
+        return self.select(sort_by='atime', sort_order='desc', limit=100)
 
-    def save(self):
-        """Save the history to disk."""
-        new = (str(e) for e in self._new_history[self._saved_count:])
-        self._lineparser.new_data = new
-        self._lineparser.save()
-        self._saved_count = len(self._new_history)
+    def entries_between(self, earliest, latest):
+        """Iterate non-redirect, non-qute entries between two timestamps.
+
+        Args:
+            earliest: Omit timestamps earlier than this.
+            latest: Omit timestamps later than this.
+        """
+        self._between_query.run(earliest=earliest, latest=latest)
+        return iter(self._between_query)
+
+    def entries_before(self, latest, limit, offset):
+        """Iterate non-redirect, non-qute entries occurring before a timestamp.
+
+        Args:
+            latest: Omit timestamps more recent than this.
+            limit: Max number of entries to include.
+            offset: Number of entries to skip.
+        """
+        self._before_query.run(latest=latest, limit=limit, offset=offset)
+        return iter(self._before_query)
 
     @cmdutils.register(name='history-clear', instance='web-history')
     def clear(self, force=False):
@@ -244,19 +139,32 @@ class WebHistory(QObject):
             self._do_clear()
         else:
             message.confirm_async(self._do_clear, title="Clear all browsing "
-                                "history?")
+                                  "history?")
 
     def _do_clear(self):
-        self._lineparser.clear()
-        self.history_dict.clear()
-        self._temp_history.clear()
-        self._new_history.clear()
-        self._saved_count = 0
-        self.cleared.emit()
+        self.delete_all()
+        self.completion.delete_all()
+
+    def delete_url(self, url):
+        """Remove all history entries with the given url.
+
+        Args:
+            url: URL string to delete.
+        """
+        qurl = QUrl(url)
+        qtutils.ensure_valid(qurl)
+        self.delete('url', self._format_url(qurl))
+        self.completion.delete('url', self._format_completion_url(qurl))
 
     @pyqtSlot(QUrl, QUrl, str)
     def add_from_tab(self, url, requested_url, title):
         """Add a new history entry as slot, called from a BrowserTab."""
+        if url.scheme() == 'data' or requested_url.scheme() == 'data':
+            return
+        if url.isEmpty():
+            # things set via setHtml
+            return
+
         no_formatting = QUrl.UrlFormattingOption(0)
         if (requested_url.isValid() and
                 not requested_url.matches(url, no_formatting)):
@@ -274,23 +182,158 @@ class WebHistory(QObject):
                       (hidden in completion)
             atime: Override the atime used to add the entry
         """
-        if config.get('general', 'private-browsing'):
-            return
         if not url.isValid():
             log.misc.warning("Ignoring invalid URL being added to history")
             return
 
-        if atime is None:
-            atime = time.time()
-        entry = Entry(atime, url, title, redirect=redirect)
-        if self._initial_read_done:
-            self._add_entry(entry)
-            self._new_history.append(entry)
-            self.item_added.emit(entry)
-            if not entry.redirect:
-                self.add_completion_item.emit(entry)
+        if 'no-sql-history' in objreg.get('args').debug_flags:
+            return
+
+        atime = int(atime) if (atime is not None) else int(time.time())
+
+        try:
+            self.insert({'url': self._format_url(url),
+                         'title': title,
+                         'atime': atime,
+                         'redirect': redirect})
+            if not redirect:
+                self.completion.insert({
+                    'url': self._format_completion_url(url),
+                    'title': title,
+                    'last_atime': atime
+                }, replace=True)
+        except sql.SqlError as e:
+            if e.environmental:
+                message.error("Failed to write history: {}".format(e.text()))
+            else:
+                raise
+
+    def _parse_entry(self, line):
+        """Parse a history line like '12345 http://example.com title'."""
+        if not line or line.startswith('#'):
+            return None
+        data = line.split(maxsplit=2)
+        if len(data) == 2:
+            atime, url = data
+            title = ""
+        elif len(data) == 3:
+            atime, url, title = data
         else:
-            self._add_entry(entry, target=self._temp_history)
+            raise ValueError("2 or 3 fields expected")
+
+        # http://xn--pple-43d.com/ with
+        # https://bugreports.qt.io/browse/QTBUG-60364
+        if url in ['http://.com/', 'https://.com/',
+                   'http://www..com/', 'https://www..com/']:
+            return None
+
+        url = QUrl(url)
+        if not url.isValid():
+            raise ValueError("Invalid URL: {}".format(url.errorString()))
+
+        # https://github.com/qutebrowser/qutebrowser/issues/2646
+        if url.scheme() == 'data':
+            return None
+
+        # https://github.com/qutebrowser/qutebrowser/issues/670
+        atime = atime.lstrip('\0')
+
+        if '-' in atime:
+            atime, flags = atime.split('-')
+        else:
+            flags = ''
+
+        if not set(flags).issubset('r'):
+            raise ValueError("Invalid flags {!r}".format(flags))
+
+        redirect = 'r' in flags
+        return (url, title, int(atime), redirect)
+
+    def import_txt(self):
+        """Import a history text file into sqlite if it exists.
+
+        In older versions of qutebrowser, history was stored in a text format.
+        This converts that file into the new sqlite format and moves it to a
+        backup location.
+        """
+        path = os.path.join(standarddir.data(), 'history')
+        if not os.path.isfile(path):
+            return
+
+        def action():
+            with debug.log_time(log.init, 'Import old history file to sqlite'):
+                try:
+                    self._read(path)
+                except ValueError as ex:
+                    message.error('Failed to import history: {}'.format(ex))
+                else:
+                    self._write_backup(path)
+
+        # delay to give message time to appear before locking down for import
+        message.info('Converting {} to sqlite...'.format(path))
+        QTimer.singleShot(100, action)
+
+    def _read(self, path):
+        """Import a text file into the sql database."""
+        with open(path, 'r', encoding='utf-8') as f:
+            data = {'url': [], 'title': [], 'atime': [], 'redirect': []}
+            completion_data = {'url': [], 'title': [], 'last_atime': []}
+            for (i, line) in enumerate(f):
+                try:
+                    parsed = self._parse_entry(line.strip())
+                    if parsed is None:
+                        continue
+                    url, title, atime, redirect = parsed
+                    data['url'].append(self._format_url(url))
+                    data['title'].append(title)
+                    data['atime'].append(atime)
+                    data['redirect'].append(redirect)
+                    if not redirect:
+                        completion_data['url'].append(
+                            self._format_completion_url(url))
+                        completion_data['title'].append(title)
+                        completion_data['last_atime'].append(atime)
+                except ValueError as ex:
+                    raise ValueError('Failed to parse line #{} of {}: "{}"'
+                                     .format(i, path, ex))
+        self.insert_batch(data)
+        self.completion.insert_batch(completion_data, replace=True)
+
+    def _write_backup(self, path):
+        bak = path + '.bak'
+        message.info('History import complete. Appending {} to {}'
+                     .format(path, bak))
+        with open(path, 'r', encoding='utf-8') as infile:
+            with open(bak, 'a', encoding='utf-8') as outfile:
+                for line in infile:
+                    outfile.write('\n' + line)
+        os.remove(path)
+
+    def _format_url(self, url):
+        return url.toString(QUrl.RemovePassword | QUrl.FullyEncoded)
+
+    def _format_completion_url(self, url):
+        return url.toString(QUrl.RemovePassword)
+
+    @cmdutils.register(instance='web-history', debug=True)
+    def debug_dump_history(self, dest):
+        """Dump the history to a file in the old pre-SQL format.
+
+        Args:
+            dest: Where to write the file to.
+        """
+        dest = os.path.expanduser(dest)
+
+        lines = ('{}{} {} {}'
+                 .format(int(x.atime), '-r' * x.redirect, x.url, x.title)
+                 for x in self.select(sort_by='atime', sort_order='asc'))
+
+        try:
+            with open(dest, 'w', encoding='utf-8') as f:
+                f.write('\n'.join(lines))
+            message.info("Dumped history to {}".format(dest))
+        except OSError as e:
+            raise cmdexc.CommandError('Could not write history: {}', e)
 
 
 def init(parent=None):
@@ -299,10 +342,9 @@ def init(parent=None):
     Args:
         parent: The parent to use for WebHistory.
     """
-    history = WebHistory(hist_dir=standarddir.data(), hist_name='history',
-                         parent=parent)
+    history = WebHistory(parent=parent)
     objreg.register('web-history', history)
 
-    if objects.backend == usertypes.Backend.QtWebKit:
+    if objects.backend == usertypes.Backend.QtWebKit:  # pragma: no cover
         from qutebrowser.browser.webkit import webkithistory
         webkithistory.init(history)

qutebrowser/browser/inspector.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -24,7 +24,8 @@ import binascii
 
 from PyQt5.QtWidgets import QWidget
 
-from qutebrowser.utils import log, objreg, usertypes
+from qutebrowser.config import configfiles
+from qutebrowser.utils import log, usertypes
 from qutebrowser.misc import miscwidgets, objects
 
 
@@ -67,9 +68,8 @@ class AbstractWebInspector(QWidget):
 
     def _load_state_geometry(self):
         """Load the geometry from the state file."""
-        state_config = objreg.get('state-config')
         try:
-            data = state_config['geometry']['inspector']
+            data = configfiles.state['geometry']['inspector']
             geom = base64.b64decode(data, validate=True)
         except KeyError:
             # First start
@@ -84,10 +84,9 @@ class AbstractWebInspector(QWidget):
 
     def closeEvent(self, e):
         """Save the geometry when closed."""
-        state_config = objreg.get('state-config')
         data = bytes(self.saveGeometry())
         geom = base64.b64encode(data).decode('ASCII')
-        state_config['geometry']['inspector'] = geom
+        configfiles.state['geometry']['inspector'] = geom
         super().closeEvent(e)
 
     def inspect(self, page):

qutebrowser/browser/mouse.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -85,7 +85,7 @@ class MouseEventFilter(QObject):
 
     def _handle_mouse_press(self, e):
         """Handle pressing of a mouse button."""
-        is_rocker_gesture = (config.get('input', 'rocker-gestures') and
+        is_rocker_gesture = (config.val.input.rocker_gestures and
                              e.buttons() == Qt.LeftButton | Qt.RightButton)
 
         if e.button() in [Qt.XButton1, Qt.XButton2] or is_rocker_gesture:
@@ -119,7 +119,7 @@ class MouseEventFilter(QObject):
             return True
 
         if e.modifiers() & Qt.ControlModifier:
-            divider = config.get('input', 'mouse-zoom-divider')
+            divider = config.val.zoom.mouse_divider
             if divider == 0:
                 return False
             factor = self._tab.zoom.factor() + (e.angleDelta().y() / divider)
@@ -139,7 +139,7 @@ class MouseEventFilter(QObject):
 
     def _handle_context_menu(self, _e):
         """Suppress context menus if rocker gestures are turned on."""
-        return config.get('input', 'rocker-gestures')
+        return config.val.input.rocker_gestures
 
     def _mousepress_insertmode_cb(self, elem):
         """Check if the clicked element is editable."""
@@ -157,7 +157,7 @@ class MouseEventFilter(QObject):
                           'click', only_if_normal=True)
         else:
             log.mouse.debug("Clicked non-editable element!")
-            if config.get('input', 'auto-leave-insert-mode'):
+            if config.val.input.insert_mode.auto_leave:
                 modeman.leave(self._tab.win_id, usertypes.KeyMode.insert,
                               'click', maybe=True)
 
@@ -179,7 +179,7 @@ class MouseEventFilter(QObject):
                               'click-delayed', only_if_normal=True)
             else:
                 log.mouse.debug("Clicked non-editable element (delayed)!")
-                if config.get('input', 'auto-leave-insert-mode'):
+                if config.val.input.insert_mode.auto_leave:
                     modeman.leave(self._tab.win_id, usertypes.KeyMode.insert,
                                   'click-delayed', maybe=True)
 

qutebrowser/browser/navigate.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -24,6 +24,7 @@ import posixpath
 from qutebrowser.browser import webelem
 from qutebrowser.config import config
 from qutebrowser.utils import objreg, urlutils, log, message, qtutils
+from qutebrowser.mainwindow import mainwindow
 
 
 class Error(Exception):
@@ -42,7 +43,7 @@ def incdec(url, count, inc_or_dec):
         background: Open the link in a new background tab.
         window: Open the link in a new window.
     """
-    segments = set(config.get('general', 'url-incdec-segments'))
+    segments = set(config.val.url.incdec_segments)
     try:
         new_url = urlutils.incdec_number(url, inc_or_dec, count,
                                          segments=segments)
@@ -79,12 +80,14 @@ def _find_prevnext(prev, elems):
             return e
 
     # Then check for regular links/buttons.
-    filterfunc = webelem.FILTERS[webelem.Group.prevnext]
-    elems = [e for e in elems if e.tag_name() != 'link' and filterfunc(e)]
-    option = 'prev-regexes' if prev else 'next-regexes'
+    elems = [e for e in elems if e.tag_name() != 'link']
+    option = 'prev_regexes' if prev else 'next_regexes'
     if not elems:
         return None
-    for regex in config.get('hints', option):
+
+    # pylint: disable=bad-config-option
+    for regex in getattr(config.val.hints, option):
+        # pylint: enable=bad-config-option
         log.hints.vdebug("== Checking regex '{}'.".format(regex.pattern))
         for e in elems:
             text = str(e)
@@ -128,20 +131,20 @@ def prevnext(*, browsertab, win_id, baseurl, prev=False,
             return
         qtutils.ensure_valid(url)
 
+        cur_tabbed_browser = objreg.get('tabbed-browser', scope='window',
+                                        window=win_id)
+
         if window:
-            from qutebrowser.mainwindow import mainwindow
-            new_window = mainwindow.MainWindow()
+            new_window = mainwindow.MainWindow(
+                private=cur_tabbed_browser.private)
             new_window.show()
             tabbed_browser = objreg.get('tabbed-browser', scope='window',
                                         window=new_window.win_id)
             tabbed_browser.tabopen(url, background=False)
         elif tab:
-            tabbed_browser = objreg.get('tabbed-browser', scope='window',
-                                        window=win_id)
-            tabbed_browser.tabopen(url, background=background)
+            cur_tabbed_browser.tabopen(url, background=background)
         else:
             browsertab.openurl(url)
 
-    selector = ', '.join([webelem.SELECTORS[webelem.Group.links],
-                          webelem.SELECTORS[webelem.Group.prevnext]])
-    browsertab.elements.find_css(selector, _prevnext_cb)
+    browsertab.elements.find_css(webelem.SELECTORS[webelem.Group.links],
+                                 _prevnext_cb)

qutebrowser/browser/network/pac.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -22,7 +22,7 @@
 import sys
 import functools
 
-from PyQt5.QtCore import (QObject, pyqtSignal, pyqtSlot)
+from PyQt5.QtCore import QObject, pyqtSignal, pyqtSlot, QUrl
 from PyQt5.QtNetwork import (QNetworkProxy, QNetworkRequest, QHostInfo,
                              QNetworkReply, QNetworkAccessManager,
                              QHostAddress)
@@ -155,7 +155,7 @@ class PACResolver:
                 raise ParseProxyError("Invalid number of parameters for PROXY")
             host, port = PACResolver._parse_proxy_host(config[1])
             return QNetworkProxy(QNetworkProxy.HttpProxy, host, port)
-        elif config[0] == "SOCKS":
+        elif config[0] in ["SOCKS", "SOCKS5"]:
             if len(config) != 2:
                 raise ParseProxyError("Invalid number of parameters for SOCKS")
             host, port = PACResolver._parse_proxy_host(config[1])
@@ -199,16 +199,24 @@ class PACResolver:
             err = "Cannot resolve FindProxyForURL function, got '{}' instead"
             raise EvalProxyError(err.format(self._resolver.toString()))
 
-    def resolve(self, query):
+    def resolve(self, query, from_file=False):
         """Resolve a proxy via PAC.
 
         Args:
             query: QNetworkProxyQuery.
+            from_file: Whether the proxy info is coming from a file.
 
         Return:
             A list of QNetworkProxy objects in order of preference.
         """
-        result = self._resolver.call([query.url().toString(),
+        if from_file:
+            string_flags = QUrl.PrettyDecoded
+        else:
+            string_flags = QUrl.RemoveUserInfo
+            if query.url().scheme() == 'https':
+                string_flags |= QUrl.RemovePath | QUrl.RemoveQuery
+
+        result = self._resolver.call([query.url().toString(string_flags),
                                       query.peerHostName()])
         result_str = result.toString()
         if not result.isString():
@@ -236,12 +244,24 @@ class PACFetcher(QObject):
         assert url.scheme().startswith(pac_prefix)
         url.setScheme(url.scheme()[len(pac_prefix):])
 
+        self._pac_url = url
         self._manager = QNetworkAccessManager()
         self._manager.setProxy(QNetworkProxy(QNetworkProxy.NoProxy))
-        self._reply = self._manager.get(QNetworkRequest(url))
-        self._reply.finished.connect(self._finish)
         self._pac = None
         self._error_message = None
+        self._reply = None
+
+    def __eq__(self, other):
+        # pylint: disable=protected-access
+        return self._pac_url == other._pac_url
+
+    def __repr__(self):
+        return utils.get_repr(self, url=self._pac_url, constructor=True)
+
+    def fetch(self):
+        """Fetch the proxy from the remote URL."""
+        self._reply = self._manager.get(QNetworkRequest(self._pac_url))
+        self._reply.finished.connect(self._finish)
 
     @pyqtSlot()
     def _finish(self):
@@ -292,8 +312,9 @@ class PACFetcher(QObject):
         Return a list of QNetworkProxy objects in order of preference.
         """
         self._wait()
+        from_file = self._pac_url.scheme() == 'file'
         try:
-            return self._pac.resolve(query)
+            return self._pac.resolve(query, from_file=from_file)
         except (EvalProxyError, ParseProxyError) as e:
             log.network.exception("Error in PAC resolution: {}.".format(e))
             # .invalid is guaranteed to be inaccessible in RFC 6761.
@@ -302,4 +323,4 @@ class PACFetcher(QObject):
             # Later NetworkManager.createRequest will detect this and display
             # an error message.
             error_host = "pac-resolve-error.qutebrowser.invalid"
-            return QNetworkProxy(QNetworkProxy.HttpProxy, error_host, 9)
+            return [QNetworkProxy(QNetworkProxy.HttpProxy, error_host, 9)]

qutebrowser/browser/network/proxy.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -44,7 +44,7 @@ class ProxyFactory(QNetworkProxyFactory):
         Return:
            None if proxy is correct, otherwise an error message.
         """
-        proxy = config.get('network', 'proxy')
+        proxy = config.val.content.proxy
         if isinstance(proxy, pac.PACFetcher):
             return proxy.fetch_error()
         else:
@@ -59,7 +59,7 @@ class ProxyFactory(QNetworkProxyFactory):
         Return:
             A list of QNetworkProxy objects in order of preference.
         """
-        proxy = config.get('network', 'proxy')
+        proxy = config.val.content.proxy
         if proxy is configtypes.SYSTEM_PROXY:
             proxies = QNetworkProxyFactory.systemProxyForQuery(query)
         elif isinstance(proxy, pac.PACFetcher):
@@ -69,7 +69,7 @@ class ProxyFactory(QNetworkProxyFactory):
         for p in proxies:
             if p.type() != QNetworkProxy.NoProxy:
                 capabilities = p.capabilities()
-                if config.get('network', 'proxy-dns-requests'):
+                if config.val.content.proxy_dns_requests:
                     capabilities |= QNetworkProxy.HostNameLookupCapability
                 else:
                     capabilities &= ~QNetworkProxy.HostNameLookupCapability

qutebrowser/browser/pdfjs.py

@@ -1,6 +1,7 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
 # Copyright 2015 Daniel Schadt
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/qtnetworkdownloads.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -22,18 +22,23 @@
 import io
 import shutil
 import functools
-import collections
 
+import attr
 from PyQt5.QtCore import pyqtSlot, pyqtSignal, QTimer
 from PyQt5.QtNetwork import QNetworkRequest, QNetworkReply
 
+from qutebrowser.config import config
 from qutebrowser.utils import message, usertypes, log, urlutils, utils
 from qutebrowser.browser import downloads
 from qutebrowser.browser.webkit import http
 from qutebrowser.browser.webkit.network import networkmanager
 
 
-_RetryInfo = collections.namedtuple('_RetryInfo', ['request', 'manager'])
+@attr.s
+class _RetryInfo:
+
+    request = attr.ib()
+    manager = attr.ib()
 
 
 class DownloadItem(downloads.AbstractDownloadItem):
@@ -110,6 +115,9 @@ class DownloadItem(downloads.AbstractDownloadItem):
     def _do_die(self):
         """Abort the download and emit an error."""
         self._read_timer.stop()
+        if self._reply is None:
+            log.downloads.debug("Reply gone while dying")
+            return
         self._reply.downloadProgress.disconnect()
         self._reply.finished.disconnect()
         self._reply.error.disconnect()
@@ -270,7 +278,7 @@ class DownloadItem(downloads.AbstractDownloadItem):
         if self.fileobj is None or self._reply is None:
             # No filename has been set yet (so we don't empty the buffer) or we
             # got a readyRead after the reply was finished (which happens on
-            # qute:log for example).
+            # qute://log for example).
             return
         if not self._reply.isOpen():
             raise OSError("Reply is closed!")
@@ -363,7 +371,8 @@ class DownloadManager(downloads.AbstractDownloadManager):
     def __init__(self, win_id, parent=None):
         super().__init__(parent)
         self._networkmanager = networkmanager.NetworkManager(
-            win_id, None, self)
+            win_id=win_id, tab_id=None,
+            private=config.val.content.private_browsing, parent=self)
 
     @pyqtSlot('QUrl')
     def get(self, url, *, user_agent=None, **kwargs):
@@ -407,7 +416,8 @@ class DownloadManager(downloads.AbstractDownloadManager):
                 mhtml.start_download_checked, tab=tab))
             message.global_bridge.ask(question, blocking=False)
 
-    def get_request(self, request, *, target=None, **kwargs):
+    def get_request(self, request, *, target=None,
+                    suggested_fn=None, **kwargs):
         """Start a download with a QNetworkRequest.
 
         Args:
@@ -423,7 +433,9 @@ class DownloadManager(downloads.AbstractDownloadManager):
         request.setAttribute(QNetworkRequest.CacheLoadControlAttribute,
                              QNetworkRequest.AlwaysNetwork)
 
-        if request.url().scheme().lower() != 'data':
+        if suggested_fn is not None:
+            pass
+        elif request.url().scheme().lower() != 'data':
             suggested_fn = urlutils.filename_from_url(request.url())
         else:
             # We might be downloading a binary blob embedded on a page or even
@@ -475,7 +487,7 @@ class DownloadManager(downloads.AbstractDownloadManager):
             reply: The QNetworkReply to download.
             target: Where to save the download as downloads.DownloadTarget.
             auto_remove: Whether to remove the download even if
-                         ui -> remove-finished-downloads is set to -1.
+                         downloads.remove_finished is set to -1.
 
         Return:
             The created DownloadItem.

qutebrowser/browser/qutescheme.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -17,21 +17,24 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Backend-independent qute:* code.
+"""Backend-independent qute://* code.
 
 Module attributes:
     pyeval_output: The output of the last :pyeval command.
     _HANDLERS: The handlers registered via decorators.
 """
 
-import sys
+import json
+import os
 import time
-import datetime
 import urllib.parse
+import textwrap
+import pkg_resources
 
-from PyQt5.QtCore import QUrlQuery
+from PyQt5.QtCore import QUrlQuery, QUrl
 
 import qutebrowser
+from qutebrowser.config import config, configdata, configexc, configdiff
 from qutebrowser.utils import (version, utils, jinja, log, message, docutils,
                                objreg)
 from qutebrowser.misc import objects
@@ -75,12 +78,25 @@ class QuteSchemeError(Exception):
         super().__init__(errorstring)
 
 
-class add_handler:  # pylint: disable=invalid-name
+class Redirect(Exception):
 
-    """Decorator to register a qute:* URL handler.
+    """Exception to signal a redirect should happen.
 
     Attributes:
-        _name: The 'foo' part of qute:foo
+        url: The URL to redirect to, as a QUrl.
+    """
+
+    def __init__(self, url):
+        super().__init__(url.toDisplayString())
+        self.url = url
+
+
+class add_handler:  # pylint: disable=invalid-name
+
+    """Decorator to register a qute://* URL handler.
+
+    Attributes:
+        _name: The 'foo' part of qute://foo
         backend: Limit which backends the handler can run with.
     """
 
@@ -103,11 +119,10 @@ class add_handler:  # pylint: disable=invalid-name
     def wrong_backend_handler(self, url):
         """Show an error page about using the invalid backend."""
         html = jinja.render('error.html',
-                            title="Error while opening qute:url",
+                            title="Error while opening qute://url",
                             url=url.toDisplayString(),
                             error='{} is not available with this '
-                                  'backend'.format(url.toDisplayString()),
-                            icon='')
+                                  'backend'.format(url.toDisplayString()))
         return 'text/html', html
 
 
@@ -125,13 +140,19 @@ def data_for_url(url):
     # A url like "qute:foo" is split as "scheme:path", not "scheme:host".
     log.misc.debug("url: {}, path: {}, host {}".format(
         url.toDisplayString(), path, host))
+    if path and not host:
+        new_url = QUrl()
+        new_url.setScheme('qute')
+        new_url.setHost(path)
+        new_url.setPath('/')
+        if new_url.host():  # path was a valid host
+            raise Redirect(new_url)
+
     try:
-        handler = _HANDLERS[path]
+        handler = _HANDLERS[host]
     except KeyError:
-        try:
-            handler = _HANDLERS[host]
-        except KeyError:
-            raise NoHandlerFound(url)
+        raise NoHandlerFound(url)
+
     try:
         mimetype, data = handler(url)
     except OSError as e:
@@ -150,7 +171,7 @@ def data_for_url(url):
 
 @add_handler('bookmarks')
 def qute_bookmarks(_url):
-    """Handler for qute:bookmarks. Display all quickmarks / bookmarks."""
+    """Handler for qute://bookmarks. Display all quickmarks / bookmarks."""
     bookmarks = sorted(objreg.get('bookmark-manager').marks.items(),
                        key=lambda x: x[1])  # Sort by title
     quickmarks = sorted(objreg.get('quickmark-manager').marks.items(),
@@ -163,90 +184,71 @@ def qute_bookmarks(_url):
     return 'text/html', html
 
 
-@add_handler('history')  # noqa
-def qute_history(url):
-    """Handler for qute:history. Display history."""
-    # Get current date from query parameter, if not given choose today.
-    curr_date = datetime.date.today()
-    try:
-        query_date = QUrlQuery(url).queryItemValue("date")
-        if query_date:
-            curr_date = datetime.datetime.strptime(query_date, "%Y-%m-%d")
-            curr_date = curr_date.date()
-    except ValueError:
-        log.misc.debug("Invalid date passed to qute:history: " + query_date)
+def history_data(start_time, offset=None):
+    """Return history data.
 
-    one_day = datetime.timedelta(days=1)
-    next_date = curr_date + one_day
-    prev_date = curr_date - one_day
-
-    def history_iter(reverse):
-        """Iterate through the history and get items we're interested in."""
-        curr_timestamp = time.mktime(curr_date.timetuple())
-        history = objreg.get('web-history').history_dict.values()
-        if reverse:
-            history = reversed(history)
-
-        for item in history:
-            # If we can't apply the reverse performance trick below,
-            # at least continue as early as possible with old items.
-            # This gets us down from 550ms to 123ms with 500k old items on my
-            # machine.
-            if item.atime < curr_timestamp and not reverse:
-                continue
-
-            # Convert timestamp
-            try:
-                item_atime = datetime.datetime.fromtimestamp(item.atime)
-            except (ValueError, OSError, OverflowError):
-                log.misc.debug("Invalid timestamp {}.".format(item.atime))
-                continue
-
-            if reverse and item_atime.date() < curr_date:
-                # If we could reverse the history in-place, and this entry is
-                # older than today, only older entries will follow, so we can
-                # abort here.
-                return
-
-            # Skip items not on curr_date
-            # Skip redirects
-            # Skip qute:// links
-            is_internal = item.url.scheme() == 'qute'
-            is_not_today = item_atime.date() != curr_date
-            if item.redirect or is_internal or is_not_today:
-                continue
-
-            # Use item's url as title if there's no title.
-            item_url = item.url.toDisplayString()
-            item_title = item.title if item.title else item_url
-            display_atime = item_atime.strftime("%X")
-
-            yield (item_url, item_title, display_atime)
-
-    if sys.hexversion >= 0x03050000:
-        # On Python >= 3.5 we can reverse the ordereddict in-place and thus
-        # apply an additional performance improvement in history_iter.
-        # On my machine, this gets us down from 550ms to 72us with 500k old
-        # items.
-        history = list(history_iter(reverse=True))
+    Arguments:
+        start_time: select history starting from this timestamp.
+        offset: number of items to skip
+    """
+    # history atimes are stored as ints, ensure start_time is not a float
+    start_time = int(start_time)
+    hist = objreg.get('web-history')
+    if offset is not None:
+        entries = hist.entries_before(start_time, limit=1000, offset=offset)
     else:
-        # On Python 3.4, we can't do that, so we'd need to copy the entire
-        # history to a list. There, filter first and then reverse it here.
-        history = reversed(list(history_iter(reverse=False)))
+        # end is 24hrs earlier than start
+        end_time = start_time - 24*60*60
+        entries = hist.entries_between(end_time, start_time)
 
-    html = jinja.render('history.html',
-                        title='History',
-                        history=history,
-                        curr_date=curr_date,
-                        next_date=next_date,
-                        prev_date=prev_date,
-                        today=datetime.date.today())
-    return 'text/html', html
+    return [{"url": e.url, "title": e.title or e.url, "time": e.atime}
+            for e in entries]
+
+
+@add_handler('history')
+def qute_history(url):
+    """Handler for qute://history. Display and serve history."""
+    if url.path() == '/data':
+        try:
+            offset = QUrlQuery(url).queryItemValue("offset")
+            offset = int(offset) if offset else None
+        except ValueError as e:
+            raise QuteSchemeError("Query parameter offset is invalid", e)
+        # Use start_time in query or current time.
+        try:
+            start_time = QUrlQuery(url).queryItemValue("start_time")
+            start_time = float(start_time) if start_time else time.time()
+        except ValueError as e:
+            raise QuteSchemeError("Query parameter start_time is invalid", e)
+
+        return 'text/html', json.dumps(history_data(start_time, offset))
+    else:
+        if not config.val.content.javascript.enabled:
+            return 'text/plain', b'JavaScript is required for qute://history'
+        return 'text/html', jinja.render(
+            'history.html',
+            title='History',
+            gap_interval=config.val.history_gap_interval
+        )
+
+
+@add_handler('javascript')
+def qute_javascript(url):
+    """Handler for qute://javascript.
+
+    Return content of file given as query parameter.
+    """
+    path = url.path()
+    if path:
+        path = "javascript" + os.sep.join(path.split('/'))
+        return 'text/html', utils.read_file(path, binary=False)
+    else:
+        raise QuteSchemeError("No file specified", ValueError())
 
 
 @add_handler('pyeval')
 def qute_pyeval(_url):
-    """Handler for qute:pyeval."""
+    """Handler for qute://pyeval."""
     html = jinja.render('pre.html', title='pyeval', content=pyeval_output)
     return 'text/html', html
 
@@ -254,7 +256,7 @@ def qute_pyeval(_url):
 @add_handler('version')
 @add_handler('verizon')
 def qute_version(_url):
-    """Handler for qute:version."""
+    """Handler for qute://version."""
     html = jinja.render('version.html', title='Version info',
                         version=version.version(),
                         copyright=qutebrowser.__copyright__)
@@ -263,7 +265,7 @@ def qute_version(_url):
 
 @add_handler('plainlog')
 def qute_plainlog(url):
-    """Handler for qute:plainlog.
+    """Handler for qute://plainlog.
 
     An optional query parameter specifies the minimum log level to print.
     For example, qute://log?level=warning prints warnings and errors.
@@ -283,7 +285,7 @@ def qute_plainlog(url):
 
 @add_handler('log')
 def qute_log(url):
-    """Handler for qute:log.
+    """Handler for qute://log.
 
     An optional query parameter specifies the minimum log level to print.
     For example, qute://log?level=warning prints warnings and errors.
@@ -304,27 +306,13 @@ def qute_log(url):
 
 @add_handler('gpl')
 def qute_gpl(_url):
-    """Handler for qute:gpl. Return HTML content as string."""
-    return 'text/html', utils.read_file('html/COPYING.html')
+    """Handler for qute://gpl. Return HTML content as string."""
+    return 'text/html', utils.read_file('html/LICENSE.html')
 
 
 @add_handler('help')
 def qute_help(url):
-    """Handler for qute:help."""
-    try:
-        utils.read_file('html/doc/index.html')
-    except OSError:
-        html = jinja.render(
-            'error.html',
-            title="Error while loading documentation",
-            url=url.toDisplayString(),
-            error="This most likely means the documentation was not generated "
-                  "properly. If you are running qutebrowser from the git "
-                  "repository, please run scripts/asciidoc2html.py. "
-                  "If you're running a released version this is a bug, please "
-                  "use :report to report it.",
-            icon='')
-        return 'text/html', html
+    """Handler for qute://help."""
     urlpath = url.path()
     if not urlpath or urlpath == '/':
         urlpath = 'index.html'
@@ -333,9 +321,102 @@ def qute_help(url):
     if not docutils.docs_up_to_date(urlpath):
         message.error("Your documentation is outdated! Please re-run "
                       "scripts/asciidoc2html.py.")
+
     path = 'html/doc/{}'.format(urlpath)
     if urlpath.endswith('.png'):
         return 'image/png', utils.read_file(path, binary=True)
-    else:
+
+    try:
         data = utils.read_file(path)
+    except OSError:
+        # No .html around, let's see if we find the asciidoc
+        asciidoc_path = path.replace('.html', '.asciidoc')
+        if asciidoc_path.startswith('html/doc/'):
+            asciidoc_path = asciidoc_path.replace('html/doc/', '../doc/help/')
+
+        try:
+            asciidoc = utils.read_file(asciidoc_path)
+        except OSError:
+            asciidoc = None
+
+        if asciidoc is None:
+            raise
+
+        preamble = textwrap.dedent("""
+            There was an error loading the documentation!
+
+            This most likely means the documentation was not generated
+            properly. If you are running qutebrowser from the git repository,
+            please (re)run scripts/asciidoc2html.py and reload this page.
+
+            If you're running a released version this is a bug, please use
+            :report to report it.
+
+            Falling back to the plaintext version.
+
+            ---------------------------------------------------------------
+
+
+        """)
+        return 'text/plain', (preamble + asciidoc).encode('utf-8')
+    else:
         return 'text/html', data
+
+
+@add_handler('backend-warning')
+def qute_backend_warning(_url):
+    """Handler for qute://backend-warning."""
+    html = jinja.render('backend-warning.html',
+                        distribution=version.distribution(),
+                        Distribution=version.Distribution,
+                        version=pkg_resources.parse_version,
+                        title="Legacy backend warning")
+    return 'text/html', html
+
+
+def _qute_settings_set(url):
+    """Handler for qute://settings/set."""
+    query = QUrlQuery(url)
+    option = query.queryItemValue('option', QUrl.FullyDecoded)
+    value = query.queryItemValue('value', QUrl.FullyDecoded)
+
+    # https://github.com/qutebrowser/qutebrowser/issues/727
+    if option == 'content.javascript.enabled' and value == 'false':
+        msg = ("Refusing to disable javascript via qute://settings "
+               "as it needs javascript support.")
+        message.error(msg)
+        return 'text/html', b'error: ' + msg.encode('utf-8')
+
+    try:
+        config.instance.set_str(option, value, save_yaml=True)
+        return 'text/html', b'ok'
+    except configexc.Error as e:
+        message.error(str(e))
+        return 'text/html', b'error: ' + str(e).encode('utf-8')
+
+
+@add_handler('settings')
+def qute_settings(url):
+    """Handler for qute://settings. View/change qute configuration."""
+    if url.path() == '/set':
+        return _qute_settings_set(url)
+
+    html = jinja.render('settings.html', title='settings',
+                        configdata=configdata,
+                        confget=config.instance.get_str)
+    return 'text/html', html
+
+
+@add_handler('configdiff')
+def qute_configdiff(url):
+    """Handler for qute://configdiff."""
+    if url.path() == '/old':
+        try:
+            return 'text/html', configdiff.get_diff()
+        except OSError as e:
+            error = (b'Failed to read old config: ' +
+                    str(e.strerror).encode('utf-8'))
+            return 'text/plain', error
+    else:
+        data = config.instance.dump_userconfig().encode('utf-8')
+        return 'text/plain', data

qutebrowser/browser/shared.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -21,10 +21,9 @@
 
 import html
 
-import jinja2
-
 from qutebrowser.config import config
-from qutebrowser.utils import usertypes, message, log, objreg
+from qutebrowser.utils import usertypes, message, log, objreg, jinja
+from qutebrowser.mainwindow import mainwindow
 
 
 class CallSuper(Exception):
@@ -35,16 +34,18 @@ class CallSuper(Exception):
 def custom_headers():
     """Get the combined custom headers."""
     headers = {}
-    dnt = b'1' if config.get('network', 'do-not-track') else b'0'
-    headers[b'DNT'] = dnt
-    headers[b'X-Do-Not-Track'] = dnt
 
-    config_headers = config.get('network', 'custom-headers')
-    if config_headers is not None:
-        for header, value in config_headers.items():
-            headers[header.encode('ascii')] = value.encode('ascii')
+    dnt_config = config.val.content.headers.do_not_track
+    if dnt_config is not None:
+        dnt = b'1' if dnt_config else b'0'
+        headers[b'DNT'] = dnt
+        headers[b'X-Do-Not-Track'] = dnt
 
-    accept_language = config.get('network', 'accept-language')
+    conf_headers = config.val.content.headers.custom
+    for header, value in conf_headers.items():
+        headers[header.encode('ascii')] = value.encode('ascii')
+
+    accept_language = config.val.content.headers.accept_language
     if accept_language is not None:
         headers[b'Accept-Language'] = accept_language.encode('ascii')
 
@@ -72,7 +73,7 @@ def authentication_required(url, authenticator, abort_on):
 def javascript_confirm(url, js_msg, abort_on):
     """Display a javascript confirm prompt."""
     log.js.debug("confirm: {}".format(js_msg))
-    if config.get('ui', 'modal-js-dialog'):
+    if config.val.content.javascript.modal_dialog:
         raise CallSuper
 
     msg = 'From <b>{}</b>:<br/>{}'.format(html.escape(url.toDisplayString()),
@@ -86,9 +87,9 @@ def javascript_confirm(url, js_msg, abort_on):
 def javascript_prompt(url, js_msg, default, abort_on):
     """Display a javascript prompt."""
     log.js.debug("prompt: {}".format(js_msg))
-    if config.get('ui', 'modal-js-dialog'):
+    if config.val.content.javascript.modal_dialog:
         raise CallSuper
-    if config.get('content', 'ignore-javascript-prompt'):
+    if not config.val.content.javascript.prompt:
         return (False, "")
 
     msg = '<b>{}</b> asks:<br/>{}'.format(html.escape(url.toDisplayString()),
@@ -107,10 +108,10 @@ def javascript_prompt(url, js_msg, default, abort_on):
 def javascript_alert(url, js_msg, abort_on):
     """Display a javascript alert."""
     log.js.debug("alert: {}".format(js_msg))
-    if config.get('ui', 'modal-js-dialog'):
+    if config.val.content.javascript.modal_dialog:
         raise CallSuper
 
-    if config.get('content', 'ignore-javascript-alert'):
+    if not config.val.content.javascript.alert:
         return
 
     msg = 'From <b>{}</b>:<br/>{}'.format(html.escape(url.toDisplayString()),
@@ -119,6 +120,22 @@ def javascript_alert(url, js_msg, abort_on):
                 abort_on=abort_on)
 
 
+def javascript_log_message(level, source, line, msg):
+    """Display a JavaScript log message."""
+    logstring = "[{}:{}] {}".format(source, line, msg)
+    # Needs to line up with the values allowed for the
+    # content.javascript.log setting.
+    logmap = {
+        'none': lambda arg: None,
+        'debug': log.js.debug,
+        'info': log.js.info,
+        'warning': log.js.warning,
+        'error': log.js.error,
+    }
+    logger = logmap[config.val.content.javascript.log[level.name]]
+    logger(logstring)
+
+
 def ignore_certificate_errors(url, errors, abort_on):
     """Display a certificate error question.
 
@@ -129,7 +146,7 @@ def ignore_certificate_errors(url, errors, abort_on):
     Return:
         True if the error should be ignored, False otherwise.
     """
-    ssl_strict = config.get('network', 'ssl-strict')
+    ssl_strict = config.val.content.ssl_strict
     log.webview.debug("Certificate errors {!r}, strict {}".format(
         errors, ssl_strict))
 
@@ -137,7 +154,7 @@ def ignore_certificate_errors(url, errors, abort_on):
         assert error.is_overridable(), repr(error)
 
     if ssl_strict == 'ask':
-        err_template = jinja2.Template("""
+        err_template = jinja.environment.from_string("""
             Errors while loading <b>{{url.toDisplayString()}}</b>:<br/>
             <ul>
             {% for err in errors %}
@@ -155,7 +172,7 @@ def ignore_certificate_errors(url, errors, abort_on):
             ignore = False
         return ignore
     elif ssl_strict is False:
-        log.webview.debug("ssl-strict is False, only warning about errors")
+        log.webview.debug("ssl_strict is False, only warning about errors")
         for err in errors:
             # FIXME we might want to use warn here (non-fatal error)
             # https://github.com/qutebrowser/qutebrowser/issues/114
@@ -173,7 +190,7 @@ def feature_permission(url, option, msg, yes_action, no_action, abort_on):
 
     Args:
         url: The URL the request was done for.
-        option: A (section, option) tuple for the option to check.
+        option: An option name to check.
         msg: A string like "show notifications"
         yes_action: A callable to call if the request was approved
         no_action: A callable to call if the request was denied
@@ -182,7 +199,7 @@ def feature_permission(url, option, msg, yes_action, no_action, abort_on):
     Return:
         The Question object if a question was asked, None otherwise.
     """
-    config_val = config.get(*option)
+    config_val = config.instance.get(option)
     if config_val == 'ask':
         if url.isValid():
             text = "Allow the website at <b>{}</b> to {}?".format(
@@ -216,8 +233,9 @@ def get_tab(win_id, target):
         win_id = win_id
         bg_tab = True
     elif target == usertypes.ClickTarget.window:
-        from qutebrowser.mainwindow import mainwindow
-        window = mainwindow.MainWindow()
+        tabbed_browser = objreg.get('tabbed-browser', scope='window',
+                                    window=win_id)
+        window = mainwindow.MainWindow(private=tabbed_browser.private)
         window.show()
         win_id = window.win_id
         bg_tab = False
@@ -231,15 +249,14 @@ def get_tab(win_id, target):
 
 def get_user_stylesheet():
     """Get the combined user-stylesheet."""
-    filename = config.get('ui', 'user-stylesheet')
+    css = ''
+    stylesheets = config.val.content.user_stylesheets
 
-    if filename is None:
-        css = ''
-    else:
+    for filename in stylesheets:
         with open(filename, 'r', encoding='utf-8') as f:
-            css = f.read()
+            css += f.read()
 
-    if config.get('ui', 'hide-scrollbar'):
+    if not config.val.scrolling.bar:
         css += '\nhtml > ::-webkit-scrollbar { width: 0px; height: 0px; }'
 
     return css

qutebrowser/browser/signalfilter.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/urlmarks.py

@@ -1,7 +1,7 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
-# Copyright 2015-2016 Antoni Boucher <bouanto@zoho.com>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Antoni Boucher <bouanto@zoho.com>
 #
 # This file is part of qutebrowser.
 #
@@ -77,13 +77,9 @@ class UrlMarkManager(QObject):
 
     Signals:
         changed: Emitted when anything changed.
-        added: Emitted when a new quickmark/bookmark was added.
-        removed: Emitted when an existing quickmark/bookmark was removed.
     """
 
     changed = pyqtSignal()
-    added = pyqtSignal(str, str)
-    removed = pyqtSignal(str)
 
     def __init__(self, parent=None):
         """Initialize and read quickmarks."""
@@ -121,7 +117,6 @@ class UrlMarkManager(QObject):
         """
         del self.marks[key]
         self.changed.emit()
-        self.removed.emit(key)
 
 
 class QuickmarkManager(UrlMarkManager):
@@ -133,7 +128,6 @@ class QuickmarkManager(UrlMarkManager):
         - self.marks maps names to URLs.
         - changed gets emitted with the name as first argument and the URL as
           second argument.
-        - removed gets emitted with the name as argument.
     """
 
     def _init_lineparser(self):
@@ -193,7 +187,6 @@ class QuickmarkManager(UrlMarkManager):
             """Really set the quickmark."""
             self.marks[name] = url
             self.changed.emit()
-            self.added.emit(name, url)
             log.misc.debug("Added quickmark {} for {}".format(name, url))
 
         if name in self.marks:
@@ -243,7 +236,6 @@ class BookmarkManager(UrlMarkManager):
         - self.marks maps URLs to titles.
         - changed gets emitted with the URL as first argument and the title as
           second argument.
-        - removed gets emitted with the URL as argument.
     """
 
     def _init_lineparser(self):
@@ -295,5 +287,4 @@ class BookmarkManager(UrlMarkManager):
         else:
             self.marks[urlstr] = title
             self.changed.emit()
-            self.added.emit(title, urlstr)
             return True

qutebrowser/browser/webelem.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -22,9 +22,6 @@
 Module attributes:
     Group: Enum for different kinds of groups.
     SELECTORS: CSS selectors for different groups of elements.
-    FILTERS: A dictionary of filter functions for the modes.
-             The filter for "links" filters javascript:-links and a-tags
-             without "href".
 """
 
 import collections.abc
@@ -34,21 +31,20 @@ from PyQt5.QtGui import QMouseEvent
 
 from qutebrowser.config import config
 from qutebrowser.keyinput import modeman
+from qutebrowser.mainwindow import mainwindow
 from qutebrowser.utils import log, usertypes, utils, qtutils, objreg
 
 
-Group = usertypes.enum('Group', ['all', 'links', 'images', 'url', 'prevnext',
-                                 'inputs'])
+Group = usertypes.enum('Group', ['all', 'links', 'images', 'url', 'inputs'])
 
 
 SELECTORS = {
     Group.all: ('a, area, textarea, select, input:not([type=hidden]), button, '
                 'frame, iframe, link, [onclick], [onmousedown], [role=link], '
                 '[role=option], [role=button], img'),
-    Group.links: 'a, area, link, [role=link]',
+    Group.links: 'a[href], area[href], link[href], [role=link][href]',
     Group.images: 'img',
     Group.url: '[src], [href]',
-    Group.prevnext: 'a, area, button, link, [role=button]',
     Group.inputs: ('input[type=text], input[type=email], input[type=url], '
                    'input[type=tel], input[type=number], '
                    'input[type=password], input[type=search], '
@@ -56,16 +52,6 @@ SELECTORS = {
 }
 
 
-def filter_links(elem):
-    return 'href' in elem and QUrl(elem['href']).scheme() != 'javascript'
-
-
-FILTERS = {
-    Group.links: filter_links,
-    Group.prevnext: filter_links,
-}
-
-
 class Error(Exception):
 
     """Base class for WebElement errors."""
@@ -120,10 +106,6 @@ class AbstractWebElement(collections.abc.MutableMapping):
         """Get the geometry for this element."""
         raise NotImplementedError
 
-    def style_property(self, name, *, strategy):
-        """Get the element style resolved with the given strategy."""
-        raise NotImplementedError
-
     def classes(self):
         """Get a list of classes assigned to this element."""
         raise NotImplementedError
@@ -201,7 +183,7 @@ class AbstractWebElement(collections.abc.MutableMapping):
             # at least a classid attribute. Oh, and let's hope images/...
             # DON'T have a classid attribute. HTML sucks.
             log.webelem.debug("<object type='{}'> clicked.".format(objtype))
-            return config.get('input', 'insert-mode-on-plugins')
+            return config.val.input.insert_mode.plugins
         else:
             # Image/Audio/...
             return False
@@ -266,7 +248,7 @@ class AbstractWebElement(collections.abc.MutableMapping):
             return self.is_writable()
         elif tag in ['embed', 'applet']:
             # Flash/Java/...
-            return config.get('input', 'insert-mode-on-plugins') and not strict
+            return config.val.input.insert_mode.plugins and not strict
         elif tag == 'object':
             return self._is_editable_object() and not strict
         elif tag in ['div', 'pre']:
@@ -310,6 +292,11 @@ class AbstractWebElement(collections.abc.MutableMapping):
         qtutils.ensure_valid(url)
         return url
 
+    def is_link(self):
+        """Return True if this AbstractWebElement is a link."""
+        href_tags = ['a', 'area', 'link']
+        return self.tag_name() in href_tags and 'href' in self
+
     def _mouse_pos(self):
         """Get the position to click/hover."""
         # Click the center of the largest square fitting into the top/left
@@ -326,6 +313,10 @@ class AbstractWebElement(collections.abc.MutableMapping):
             raise Error("Element position is out of view!")
         return pos
 
+    def _move_text_cursor(self):
+        """Move cursor to end after clicking."""
+        raise NotImplementedError
+
     def _click_fake_event(self, click_target):
         """Send a fake click event to the element."""
         pos = self._mouse_pos()
@@ -339,7 +330,7 @@ class AbstractWebElement(collections.abc.MutableMapping):
             usertypes.ClickTarget.tab: Qt.ControlModifier,
             usertypes.ClickTarget.tab_bg: Qt.ControlModifier,
         }
-        if config.get('tabs', 'background-tabs'):
+        if config.val.tabs.background:
             modifiers[usertypes.ClickTarget.tab] |= Qt.ShiftModifier
         else:
             modifiers[usertypes.ClickTarget.tab_bg] |= Qt.ShiftModifier
@@ -356,11 +347,7 @@ class AbstractWebElement(collections.abc.MutableMapping):
         for evt in events:
             self._tab.send_event(evt)
 
-        def after_click():
-            """Move cursor to end after clicking."""
-            if self.is_text_input() and self.is_editable():
-                self._tab.caret.move_to_end_of_document()
-        QTimer.singleShot(0, after_click)
+        QTimer.singleShot(0, self._move_text_cursor)
 
     def _click_editable(self, click_target):
         """Fake a click on an editable input field."""
@@ -378,15 +365,15 @@ class AbstractWebElement(collections.abc.MutableMapping):
             self._click_fake_event(click_target)
             return
 
+        tabbed_browser = objreg.get('tabbed-browser', scope='window',
+                                    window=self._tab.win_id)
+
         if click_target in [usertypes.ClickTarget.tab,
                             usertypes.ClickTarget.tab_bg]:
             background = click_target == usertypes.ClickTarget.tab_bg
-            tabbed_browser = objreg.get('tabbed-browser', scope='window',
-                                        window=self._tab.win_id)
             tabbed_browser.tabopen(url, background=background)
         elif click_target == usertypes.ClickTarget.window:
-            from qutebrowser.mainwindow import mainwindow
-            window = mainwindow.MainWindow()
+            window = mainwindow.MainWindow(private=tabbed_browser.private)
             window.show()
             window.tabbed_browser.tabopen(url)
         else:
@@ -407,9 +394,8 @@ class AbstractWebElement(collections.abc.MutableMapping):
             self._click_fake_event(click_target)
             return
 
-        href_tags = ['a', 'area', 'link']
         if click_target == usertypes.ClickTarget.normal:
-            if self.tag_name() in href_tags:
+            if self.is_link():
                 log.webelem.debug("Clicking via JS click()")
                 self._click_js(click_target)
             elif self.is_editable(strict=True):
@@ -422,7 +408,7 @@ class AbstractWebElement(collections.abc.MutableMapping):
         elif click_target in [usertypes.ClickTarget.tab,
                               usertypes.ClickTarget.tab_bg,
                               usertypes.ClickTarget.window]:
-            if self.tag_name() in href_tags:
+            if self.is_link():
                 self._click_href(click_target)
             else:
                 self._click_fake_event(click_target)

qutebrowser/browser/webengine/__init__.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webengine/certificateerror.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -19,9 +19,7 @@
 
 """Wrapper over a QWebEngineCertificateError."""
 
-# pylint: disable=no-name-in-module,import-error,useless-suppression
 from PyQt5.QtWebEngineWidgets import QWebEngineCertificateError
-# pylint: enable=no-name-in-module,import-error,useless-suppression
 
 from qutebrowser.utils import usertypes, utils, debug
 

qutebrowser/browser/webengine/interceptor.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -19,9 +19,7 @@
 
 """A request interceptor taking care of adblocking and custom headers."""
 
-# pylint: disable=no-name-in-module,import-error,useless-suppression
 from PyQt5.QtWebEngineCore import QWebEngineUrlRequestInterceptor
-# pylint: enable=no-name-in-module,import-error,useless-suppression
 
 from qutebrowser.config import config
 from qutebrowser.browser import shared
@@ -57,8 +55,7 @@ class RequestInterceptor(QWebEngineUrlRequestInterceptor):
             info: QWebEngineUrlRequestInfo &info
         """
         # FIXME:qtwebengine only block ads for NavigationTypeOther?
-        if (bytes(info.requestMethod()) == b'GET' and
-                self._host_blocker.is_blocked(info.requestUrl())):
+        if self._host_blocker.is_blocked(info.requestUrl()):
             log.webview.info("Request to {} blocked by host blocker.".format(
                 info.requestUrl().host()))
             info.block(True)
@@ -66,6 +63,6 @@ class RequestInterceptor(QWebEngineUrlRequestInterceptor):
         for header, value in shared.custom_headers():
             info.setHttpHeader(header, value)
 
-        user_agent = config.get('network', 'user-agent')
+        user_agent = config.val.content.headers.user_agent
         if user_agent is not None:
             info.setHttpHeader(b'User-Agent', user_agent.encode('ascii'))

qutebrowser/browser/webengine/spell.py

@@ -0,0 +1,46 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2017 Michal Siedlaczek <michal.siedlaczek@gmail.com>
+
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Installing and configuring spell-checking for QtWebEngine."""
+
+import glob
+import os
+
+from PyQt5.QtCore import QLibraryInfo
+
+
+def dictionary_dir():
+    """Return the path (str) to the QtWebEngine's dictionaries directory."""
+    datapath = QLibraryInfo.location(QLibraryInfo.DataPath)
+    return os.path.join(datapath, 'qtwebengine_dictionaries')
+
+
+def installed_file(code):
+    """Return the installed dictionary for the given code.
+
+    Return the filename of the installed dictionary or None
+    if the dictionary is not installed.
+    """
+    pathname = os.path.join(dictionary_dir(), '{}*.bdic'.format(code))
+    matching_dicts = glob.glob(pathname)
+    if matching_dicts:
+        with_extension = os.path.basename(matching_dicts[0])
+        return os.path.splitext(with_extension)[0]
+    else:
+        return None

qutebrowser/browser/webengine/tabhistory.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webengine/webenginedownloads.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -25,9 +25,7 @@ import urllib
 import functools
 
 from PyQt5.QtCore import pyqtSlot, Qt
-# pylint: disable=no-name-in-module,import-error,useless-suppression
 from PyQt5.QtWebEngineWidgets import QWebEngineDownloadItem
-# pylint: enable=no-name-in-module,import-error,useless-suppression
 
 from qutebrowser.browser import downloads
 from qutebrowser.utils import debug, usertypes, message, log, qtutils
@@ -78,32 +76,39 @@ class DownloadItem(downloads.AbstractDownloadItem):
             self.stats.finish()
         elif state == QWebEngineDownloadItem.DownloadInterrupted:
             self.successful = False
-            self.done = True
             # https://bugreports.qt.io/browse/QTBUG-56839
-            self.error.emit("Download failed")
-            self.stats.finish()
+            try:
+                reason = self._qt_item.interruptReasonString()
+            except AttributeError:
+                # Qt < 5.9
+                reason = "Download failed"
+            self._die(reason)
         else:
             raise ValueError("_on_state_changed was called with unknown state "
                              "{}".format(state_name))
 
     def _do_die(self):
         self._qt_item.downloadProgress.disconnect()
-        self._qt_item.cancel()
+        if self._qt_item.state() != QWebEngineDownloadItem.DownloadInterrupted:
+            self._qt_item.cancel()
 
     def _do_cancel(self):
         self._qt_item.cancel()
 
     def retry(self):
         # https://bugreports.qt.io/browse/QTBUG-56840
-        raise downloads.UnsupportedOperationError
+        raise downloads.UnsupportedOperationError(
+            "Retrying downloads is unsupported with QtWebEngine")
 
     def _get_open_filename(self):
         return self._filename
 
-    def _set_fileobj(self, fileobj):
+    def _set_fileobj(self, fileobj, *,
+                     autoclose=True):  # pylint: disable=unused-argument
         raise downloads.UnsupportedOperationError
 
     def _set_tempfile(self, fileobj):
+        fileobj.close()
         self._set_filename(fileobj.name, force_overwrite=True,
                            remember_directory=False)
 
@@ -144,8 +149,8 @@ def _get_suggested_filename(path):
     See https://bugreports.qt.io/browse/QTBUG-56978
     """
     filename = os.path.basename(path)
-    filename = re.sub(r'\([0-9]+\)$', '', filename)
-    if not qtutils.version_check('5.8.1'):
+    filename = re.sub(r'\([0-9]+\)(?=\.|$)', '', filename)
+    if not qtutils.version_check('5.9', compiled=False):
         # https://bugreports.qt.io/browse/QTBUG-58155
         filename = urllib.parse.unquote(filename)
         # Doing basename a *second* time because there could be a %2F in

qutebrowser/browser/webengine/webengineelem.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -18,16 +18,14 @@
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
 # FIXME:qtwebengine remove this once the stubs are gone
-# pylint: disable=unused-variable
+# pylint: disable=unused-argument
 
 """QtWebEngine specific part of the web element API."""
 
 from PyQt5.QtCore import QRect, Qt, QPoint, QEventLoop
 from PyQt5.QtGui import QMouseEvent
 from PyQt5.QtWidgets import QApplication
-# pylint: disable=no-name-in-module,import-error,useless-suppression
 from PyQt5.QtWebEngineWidgets import QWebEngineSettings
-# pylint: enable=no-name-in-module,import-error,useless-suppression
 
 from qutebrowser.utils import log, javascript
 from qutebrowser.browser import webelem
@@ -39,6 +37,38 @@ class WebEngineElement(webelem.AbstractWebElement):
 
     def __init__(self, js_dict, tab):
         super().__init__(tab)
+        # Do some sanity checks on the data we get from JS
+        js_dict_types = {
+            'id': int,
+            'text': str,
+            'value': (str, int, float),
+            'tag_name': str,
+            'outer_xml': str,
+            'class_name': str,
+            'rects': list,
+            'attributes': dict,
+        }
+        assert set(js_dict.keys()).issubset(js_dict_types.keys())
+        for name, typ in js_dict_types.items():
+            if name in js_dict and not isinstance(js_dict[name], typ):
+                raise TypeError("Got {} for {} from JS but expected {}: "
+                                "{}".format(type(js_dict[name]), name, typ,
+                                            js_dict))
+        for name, value in js_dict['attributes'].items():
+            if not isinstance(name, str):
+                raise TypeError("Got {} ({}) for attribute name from JS: "
+                                "{}".format(name, type(name), js_dict))
+            if not isinstance(value, str):
+                raise TypeError("Got {} ({}) for attribute {} from JS: "
+                                "{}".format(value, type(value), name, js_dict))
+        for rect in js_dict['rects']:
+            assert set(rect.keys()) == {'top', 'right', 'bottom', 'left',
+                                        'height', 'width'}, rect.keys()
+            for value in rect.values():
+                if not isinstance(value, (int, float)):
+                    raise TypeError("Got {} ({}) for rect from JS: "
+                                    "{}".format(value, type(value), js_dict))
+
         self._id = js_dict['id']
         self._js_dict = js_dict
 
@@ -56,9 +86,7 @@ class WebEngineElement(webelem.AbstractWebElement):
 
     def __setitem__(self, key, val):
         self._js_dict['attributes'][key] = val
-        js_code = javascript.assemble('webelem', 'set_attribute', self._id,
-            key, val)
-        self._tab.run_js_async(js_code)
+        self._js_call('set_attribute', key, val)
 
     def __delitem__(self, key):
         log.stub()
@@ -69,6 +97,11 @@ class WebEngineElement(webelem.AbstractWebElement):
     def __len__(self):
         return len(self._js_dict['attributes'])
 
+    def _js_call(self, name, *args, callback=None):
+        """Wrapper to run stuff from webelem.js."""
+        js_code = javascript.assemble('webelem', name, self._id, *args)
+        self._tab.run_js_async(js_code, callback=callback)
+
     def has_frame(self):
         return True
 
@@ -76,10 +109,6 @@ class WebEngineElement(webelem.AbstractWebElement):
         log.stub()
         return QRect()
 
-    def style_property(self, name, *, strategy):
-        log.stub()
-        return ''
-
     def classes(self):
         """Get a list of classes assigned to this element."""
         return self._js_dict['class_name'].split()
@@ -89,7 +118,9 @@ class WebEngineElement(webelem.AbstractWebElement):
 
         The returned name will always be lower-case.
         """
-        return self._js_dict['tag_name'].lower()
+        tag = self._js_dict['tag_name']
+        assert isinstance(tag, str), tag
+        return tag.lower()
 
     def outer_xml(self):
         """Get the full HTML representation of this element."""
@@ -99,15 +130,13 @@ class WebEngineElement(webelem.AbstractWebElement):
         return self._js_dict.get('value', None)
 
     def set_value(self, value):
-        js_code = javascript.assemble('webelem', 'set_value', self._id, value)
-        self._tab.run_js_async(js_code)
+        self._js_call('set_value', value)
 
     def insert_text(self, text):
         if not self.is_editable(strict=True):
             raise webelem.Error("Element is not editable!")
         log.webelem.debug("Inserting text into element {!r}".format(self))
-        js_code = javascript.assemble('webelem', 'insert_text', self._id, text)
-        self._tab.run_js_async(js_code)
+        self._js_call('insert_text', text)
 
     def rect_on_view(self, *, elem_geometry=None, no_js=False):
         """Get the geometry of the element relative to the webview.
@@ -133,7 +162,7 @@ class WebEngineElement(webelem.AbstractWebElement):
             top = rect['top']
             if width > 1 and height > 1:
                 # Fix coordinates according to zoom level
-                # We're not checking for zoom-text-only here as that doesn't
+                # We're not checking for zoom.text_only here as that doesn't
                 # exist for QtWebEngine.
                 zoom = self._tab.zoom.factor()
                 rect = QRect(left * zoom, top * zoom,
@@ -153,27 +182,27 @@ class WebEngineElement(webelem.AbstractWebElement):
     def remove_blank_target(self):
         if self._js_dict['attributes'].get('target') == '_blank':
             self._js_dict['attributes']['target'] = '_top'
-        js_code = javascript.assemble('webelem', 'remove_blank_target',
-            self._id)
-        self._tab.run_js_async(js_code)
+        self._js_call('remove_blank_target')
+
+    def _move_text_cursor(self):
+        if self.is_text_input() and self.is_editable():
+            self._js_call('move_cursor_to_end')
 
     def _click_editable(self, click_target):
         # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-58515
-        # pylint doesn't know about Qt.MouseEventSynthesizedBySystem
-        # because it was added in Qt 5.6, but we can be sure we use that with
-        # QtWebEngine.
-        # pylint: disable=no-member
         ev = QMouseEvent(QMouseEvent.MouseButtonPress, QPoint(0, 0),
                          QPoint(0, 0), QPoint(0, 0), Qt.NoButton, Qt.NoButton,
                          Qt.NoModifier, Qt.MouseEventSynthesizedBySystem)
-        # pylint: enable=no-member
         self._tab.send_event(ev)
         # This actually "clicks" the element by calling focus() on it in JS.
-        js_code = javascript.assemble('webelem', 'focus', self._id)
-        self._tab.run_js_async(js_code)
+        self._js_call('focus')
+        self._move_text_cursor()
 
     def _click_js(self, _click_target):
-        settings = QWebEngineSettings.globalSettings()
+        # FIXME:qtwebengine Have a proper API for this
+        # pylint: disable=protected-access
+        settings = self._tab._widget.settings()
+        # pylint: enable=protected-access
         attribute = QWebEngineSettings.JavascriptCanOpenWindows
         could_open_windows = settings.testAttribute(attribute)
         settings.setAttribute(attribute, True)
@@ -189,5 +218,4 @@ class WebEngineElement(webelem.AbstractWebElement):
         def reset_setting(_arg):
             settings.setAttribute(attribute, could_open_windows)
 
-        js_code = javascript.assemble('webelem', 'click', self._id)
-        self._tab.run_js_async(js_code, reset_setting)
+        self._js_call('click', callback=reset_setting)

qutebrowser/browser/webengine/webengineinspector.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -22,9 +22,7 @@
 import os
 
 from PyQt5.QtCore import QUrl
-# pylint: disable=no-name-in-module,import-error,useless-suppression
 from PyQt5.QtWebEngineWidgets import QWebEngineView
-# pylint: enable=no-name-in-module,import-error,useless-suppression
 
 from qutebrowser.browser import inspector
 

qutebrowser/browser/webengine/webenginequtescheme.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -17,24 +17,22 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-"""QtWebEngine specific qute:* handlers and glue code."""
+"""QtWebEngine specific qute://* handlers and glue code."""
 
 from PyQt5.QtCore import QBuffer, QIODevice
-# pylint: disable=no-name-in-module,import-error,useless-suppression
 from PyQt5.QtWebEngineCore import (QWebEngineUrlSchemeHandler,
                                    QWebEngineUrlRequestJob)
-# pylint: enable=no-name-in-module,import-error,useless-suppression
 
 from qutebrowser.browser import qutescheme
-from qutebrowser.utils import log
+from qutebrowser.utils import log, qtutils
 
 
 class QuteSchemeHandler(QWebEngineUrlSchemeHandler):
 
-    """Handle qute:* requests on QtWebEngine."""
+    """Handle qute://* requests on QtWebEngine."""
 
     def install(self, profile):
-        """Install the handler for qute: URLs on the given profile."""
+        """Install the handler for qute:// URLs on the given profile."""
         profile.installUrlSchemeHandler(b'qute', self)
 
     def requestStarted(self, job):
@@ -58,12 +56,15 @@ class QuteSchemeHandler(QWebEngineUrlSchemeHandler):
             job.fail(QWebEngineUrlRequestJob.UrlNotFound)
         except qutescheme.QuteSchemeOSError:
             # FIXME:qtwebengine how do we show a better error here?
-            log.misc.exception("OSError while handling qute:* URL")
+            log.misc.exception("OSError while handling qute://* URL")
             job.fail(QWebEngineUrlRequestJob.UrlNotFound)
         except qutescheme.QuteSchemeError:
             # FIXME:qtwebengine how do we show a better error here?
-            log.misc.exception("Error while handling qute:* URL")
+            log.misc.exception("Error while handling qute://* URL")
             job.fail(QWebEngineUrlRequestJob.RequestFailed)
+        except qutescheme.Redirect as e:
+            qtutils.ensure_valid(e.url)
+            job.redirect(e.url)
         else:
             log.misc.debug("Returning {} data".format(mimetype))
 

qutebrowser/browser/webengine/webenginesettings.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -17,6 +17,9 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
+# We get various "abstract but not overridden" warnings
+# pylint: disable=abstract-method
+
 """Bridge from QWebEngineSettings to our own settings.
 
 Module attributes:
@@ -26,79 +29,95 @@ Module attributes:
 
 import os
 
-# pylint: disable=no-name-in-module,import-error,useless-suppression
+from PyQt5.QtGui import QFont
 from PyQt5.QtWebEngineWidgets import (QWebEngineSettings, QWebEngineProfile,
                                       QWebEngineScript)
-# pylint: enable=no-name-in-module,import-error,useless-suppression
 
 from qutebrowser.browser import shared
+from qutebrowser.browser.webengine import spell
 from qutebrowser.config import config, websettings
-from qutebrowser.utils import objreg, utils, standarddir, javascript
+from qutebrowser.utils import utils, standarddir, javascript, qtutils, message
+
+# The default QWebEngineProfile
+default_profile = None
+# The QWebEngineProfile used for private (off-the-record) windows
+private_profile = None
 
 
-class Attribute(websettings.Attribute):
+class Base(websettings.Base):
+
+    """Base settings class with appropriate _get_global_settings."""
+
+    def _get_global_settings(self):
+        return [default_profile.settings(), private_profile.settings()]
+
+
+class Attribute(Base, websettings.Attribute):
 
     """A setting set via QWebEngineSettings::setAttribute."""
 
-    GLOBAL_SETTINGS = QWebEngineSettings.globalSettings
     ENUM_BASE = QWebEngineSettings
 
 
-class Setter(websettings.Setter):
+class Setter(Base, websettings.Setter):
 
-    """A setting set via QWebEngineSettings getter/setter methods."""
+    """A setting set via a QWebEngineSettings setter method."""
 
-    GLOBAL_SETTINGS = QWebEngineSettings.globalSettings
+    pass
 
 
-class NullStringSetter(websettings.NullStringSetter):
+class FontFamilySetter(Base, websettings.FontFamilySetter):
 
-    """A setter for settings requiring a null QString as default."""
+    """A setter for a font family.
 
-    GLOBAL_SETTINGS = QWebEngineSettings.globalSettings
+    Gets the default value from QFont.
+    """
+
+    def __init__(self, font):
+        # Mapping from WebEngineSettings::initDefaults in
+        # qtwebengine/src/core/web_engine_settings.cpp
+        font_to_qfont = {
+            QWebEngineSettings.StandardFont: QFont.Serif,
+            QWebEngineSettings.FixedFont: QFont.Monospace,
+            QWebEngineSettings.SerifFont: QFont.Serif,
+            QWebEngineSettings.SansSerifFont: QFont.SansSerif,
+            QWebEngineSettings.CursiveFont: QFont.Cursive,
+            QWebEngineSettings.FantasyFont: QFont.Fantasy,
+        }
+        super().__init__(setter=QWebEngineSettings.setFontFamily, font=font,
+                         qfont=font_to_qfont[font])
 
 
-class StaticSetter(websettings.StaticSetter):
-
-    """A setting set via static QWebEngineSettings getter/setter methods."""
-
-    GLOBAL_SETTINGS = QWebEngineSettings.globalSettings
-
-
-class ProfileSetter(websettings.Base):
+class DefaultProfileSetter(websettings.Base):
 
     """A setting set on the QWebEngineProfile."""
 
-    def __init__(self, getter, setter):
-        super().__init__()
-        self._getter = getter
+    def __init__(self, setter, default=websettings.UNSET):
+        super().__init__(default)
         self._setter = setter
 
-    def get(self, settings=None):
-        utils.unused(settings)
-        getter = getattr(QWebEngineProfile.defaultProfile(), self._getter)
-        return getter()
+    def __repr__(self):
+        return utils.get_repr(self, setter=self._setter, constructor=True)
 
     def _set(self, value, settings=None):
-        utils.unused(settings)
-        setter = getattr(QWebEngineProfile.defaultProfile(), self._setter)
+        if settings is not None:
+            raise ValueError("'settings' may not be set with "
+                             "DefaultProfileSetters!")
+        setter = getattr(default_profile, self._setter)
         setter(value)
 
 
-class PersistentCookiePolicy(ProfileSetter):
+class PersistentCookiePolicy(DefaultProfileSetter):
 
-    """The cookies -> store setting is different from other settings."""
+    """The content.cookies.store setting is different from other settings."""
 
     def __init__(self):
-        super().__init__(getter='persistentCookiesPolicy',
-                         setter='setPersistentCookiesPolicy')
-
-    def get(self, settings=None):
-        utils.unused(settings)
-        return config.get('content', 'cookies-store')
+        super().__init__('setPersistentCookiesPolicy')
 
     def _set(self, value, settings=None):
-        utils.unused(settings)
+        if settings is not None:
+            raise ValueError("'settings' may not be set with "
+                             "PersistentCookiePolicy!")
         setter = getattr(QWebEngineProfile.defaultProfile(), self._setter)
         setter(
             QWebEngineProfile.AllowPersistentCookies if value else
@@ -106,15 +125,35 @@ class PersistentCookiePolicy(ProfileSetter):
         )
 
 
+class DictionaryLanguageSetter(DefaultProfileSetter):
+
+    """Sets paths to dictionary files based on language codes."""
+
+    def __init__(self):
+        super().__init__('setSpellCheckLanguages', default=[])
+
+    def _find_installed(self, code):
+        installed_file = spell.installed_file(code)
+        if not installed_file:
+            message.warning(
+                "Language {} is not installed - see scripts/install_dict.py "
+                "in qutebrowser's sources".format(code))
+        return installed_file
+
+    def _set(self, value, settings=None):
+        if settings is not None:
+            raise ValueError("'settings' may not be set with "
+                             "DictionaryLanguageSetter!")
+        filenames = [self._find_installed(code) for code in value]
+        super()._set([f for f in filenames if f], settings)
+
+
 def _init_stylesheet(profile):
     """Initialize custom stylesheets.
 
     Mostly inspired by QupZilla:
     https://github.com/QupZilla/qupzilla/blob/v2.0/src/lib/app/mainapplication.cpp#L1063-L1101
     https://github.com/QupZilla/qupzilla/blob/v2.0/src/lib/tools/scripts.cpp#L119-L132
-
-    FIXME:qtwebengine Use QWebEngineStyleSheet once that's available
-    https://codereview.qt-project.org/#/c/148671/
     """
     old_script = profile.scripts().findScript('_qute_stylesheet')
     if not old_script.isNull():
@@ -139,19 +178,50 @@ def _init_stylesheet(profile):
     profile.scripts().insert(script)
 
 
-def _init_profile(profile):
-    """Initialize settings set on the QWebEngineProfile."""
-    profile.setCachePath(os.path.join(standarddir.cache(), 'webengine'))
-    profile.setPersistentStoragePath(
-        os.path.join(standarddir.data(), 'webengine'))
+def _set_http_headers(profile):
+    """Set the user agent and accept-language for the given profile.
+
+    We override those per request in the URL interceptor (to allow for
+    per-domain values), but this one still gets used for things like
+    window.navigator.userAgent/.languages in JS.
+    """
+    profile.setHttpUserAgent(config.val.content.headers.user_agent)
+    accept_language = config.val.content.headers.accept_language
+    if accept_language is not None:
+        profile.setHttpAcceptLanguage(accept_language)
 
 
-def update_settings(section, option):
+def _update_settings(option):
     """Update global settings when qwebsettings changed."""
-    websettings.update_mappings(MAPPINGS, section, option)
-    profile = QWebEngineProfile.defaultProfile()
-    if section == 'ui' and option in ['hide-scrollbar', 'user-stylesheet']:
-        _init_stylesheet(profile)
+    websettings.update_mappings(MAPPINGS, option)
+    if option in ['scrolling.bar', 'content.user_stylesheets']:
+        _init_stylesheet(default_profile)
+        _init_stylesheet(private_profile)
+    elif option in ['content.headers.user_agent',
+                    'content.headers.accept_language']:
+        _set_http_headers(default_profile)
+        _set_http_headers(private_profile)
+
+
+def _init_profiles():
+    """Init the two used QWebEngineProfiles."""
+    global default_profile, private_profile
+    default_profile = QWebEngineProfile.defaultProfile()
+    default_profile.setCachePath(
+        os.path.join(standarddir.cache(), 'webengine'))
+    default_profile.setPersistentStoragePath(
+        os.path.join(standarddir.data(), 'webengine'))
+    _init_stylesheet(default_profile)
+    _set_http_headers(default_profile)
+
+    private_profile = QWebEngineProfile()
+    assert private_profile.isOffTheRecord()
+    _init_stylesheet(private_profile)
+    _set_http_headers(private_profile)
+
+    if qtutils.version_check('5.8'):
+        default_profile.setSpellCheckEnabled(True)
+        private_profile.setSpellCheckEnabled(True)
 
 
 def init(args):
@@ -159,17 +229,16 @@ def init(args):
     if args.enable_webengine_inspector:
         os.environ['QTWEBENGINE_REMOTE_DEBUGGING'] = str(utils.random_port())
 
-    profile = QWebEngineProfile.defaultProfile()
-    _init_profile(profile)
-    _init_stylesheet(profile)
+    _init_profiles()
+
     # We need to do this here as a WORKAROUND for
     # https://bugreports.qt.io/browse/QTBUG-58650
-    PersistentCookiePolicy().set(config.get('content', 'cookies-store'))
-
+    if not qtutils.version_check('5.9', compiled=False):
+        PersistentCookiePolicy().set(config.val.content.cookies.store)
     Attribute(QWebEngineSettings.FullScreenSupportEnabled).set(True)
 
     websettings.init_mappings(MAPPINGS)
-    objreg.get('config').changed.connect(update_settings)
+    config.instance.changed.connect(_update_settings)
 
 
 def shutdown():
@@ -186,105 +255,84 @@ def shutdown():
 # - AllowRunningInsecureContent (5.8)
 #
 # Missing QtWebEngine fonts:
-# - FantasyFont
 # - PictographFont
 
 
 MAPPINGS = {
-    'content': {
-        'allow-images':
-            Attribute(QWebEngineSettings.AutoLoadImages),
-        'allow-javascript':
-            Attribute(QWebEngineSettings.JavascriptEnabled),
-        'javascript-can-open-windows-automatically':
-            Attribute(QWebEngineSettings.JavascriptCanOpenWindows),
-        'javascript-can-access-clipboard':
-            Attribute(QWebEngineSettings.JavascriptCanAccessClipboard),
-        'allow-plugins':
-            Attribute(QWebEngineSettings.PluginsEnabled),
-        'hyperlink-auditing':
-            Attribute(QWebEngineSettings.HyperlinkAuditingEnabled),
-        'local-content-can-access-remote-urls':
-            Attribute(QWebEngineSettings.LocalContentCanAccessRemoteUrls),
-        'local-content-can-access-file-urls':
-            Attribute(QWebEngineSettings.LocalContentCanAccessFileUrls),
-        # https://bugreports.qt.io/browse/QTBUG-58650
-        # 'cookies-store':
-        #     PersistentCookiePolicy(),
-        'webgl':
-            Attribute(QWebEngineSettings.WebGLEnabled),
-    },
-    'input': {
-        'spatial-navigation':
-            Attribute(QWebEngineSettings.SpatialNavigationEnabled),
-        'links-included-in-focus-chain':
-            Attribute(QWebEngineSettings.LinksIncludedInFocusChain),
-    },
-    'fonts': {
-        'web-family-standard':
-            Setter(getter=QWebEngineSettings.fontFamily,
-                   setter=QWebEngineSettings.setFontFamily,
-                   args=[QWebEngineSettings.StandardFont]),
-        'web-family-fixed':
-            Setter(getter=QWebEngineSettings.fontFamily,
-                   setter=QWebEngineSettings.setFontFamily,
-                   args=[QWebEngineSettings.FixedFont]),
-        'web-family-serif':
-            Setter(getter=QWebEngineSettings.fontFamily,
-                   setter=QWebEngineSettings.setFontFamily,
-                   args=[QWebEngineSettings.SerifFont]),
-        'web-family-sans-serif':
-            Setter(getter=QWebEngineSettings.fontFamily,
-                   setter=QWebEngineSettings.setFontFamily,
-                   args=[QWebEngineSettings.SansSerifFont]),
-        'web-family-cursive':
-            Setter(getter=QWebEngineSettings.fontFamily,
-                   setter=QWebEngineSettings.setFontFamily,
-                   args=[QWebEngineSettings.CursiveFont]),
-        'web-family-fantasy':
-            Setter(getter=QWebEngineSettings.fontFamily,
-                   setter=QWebEngineSettings.setFontFamily,
-                   args=[QWebEngineSettings.FantasyFont]),
-        'web-size-minimum':
-            Setter(getter=QWebEngineSettings.fontSize,
-                   setter=QWebEngineSettings.setFontSize,
-                   args=[QWebEngineSettings.MinimumFontSize]),
-        'web-size-minimum-logical':
-            Setter(getter=QWebEngineSettings.fontSize,
-                   setter=QWebEngineSettings.setFontSize,
-                   args=[QWebEngineSettings.MinimumLogicalFontSize]),
-        'web-size-default':
-            Setter(getter=QWebEngineSettings.fontSize,
-                   setter=QWebEngineSettings.setFontSize,
-                   args=[QWebEngineSettings.DefaultFontSize]),
-        'web-size-default-fixed':
-            Setter(getter=QWebEngineSettings.fontSize,
-                   setter=QWebEngineSettings.setFontSize,
-                   args=[QWebEngineSettings.DefaultFixedFontSize]),
-    },
-    'ui': {
-        'smooth-scrolling':
-            Attribute(QWebEngineSettings.ScrollAnimatorEnabled),
-    },
-    'storage': {
-        'local-storage':
-            Attribute(QWebEngineSettings.LocalStorageEnabled),
-        'cache-size':
-            ProfileSetter(getter='httpCacheMaximumSize',
-                          setter='setHttpCacheMaximumSize')
-    },
-    'general': {
-        'xss-auditing':
-            Attribute(QWebEngineSettings.XSSAuditingEnabled),
-        'default-encoding':
-            Setter(getter=QWebEngineSettings.defaultTextEncoding,
-                   setter=QWebEngineSettings.setDefaultTextEncoding),
-    }
+    'content.images':
+        Attribute(QWebEngineSettings.AutoLoadImages),
+    'content.javascript.enabled':
+        Attribute(QWebEngineSettings.JavascriptEnabled),
+    'content.javascript.can_open_tabs_automatically':
+        Attribute(QWebEngineSettings.JavascriptCanOpenWindows),
+    'content.javascript.can_access_clipboard':
+        Attribute(QWebEngineSettings.JavascriptCanAccessClipboard),
+    'content.plugins':
+        Attribute(QWebEngineSettings.PluginsEnabled),
+    'content.hyperlink_auditing':
+        Attribute(QWebEngineSettings.HyperlinkAuditingEnabled),
+    'content.local_content_can_access_remote_urls':
+        Attribute(QWebEngineSettings.LocalContentCanAccessRemoteUrls),
+    'content.local_content_can_access_file_urls':
+        Attribute(QWebEngineSettings.LocalContentCanAccessFileUrls),
+    'content.webgl':
+        Attribute(QWebEngineSettings.WebGLEnabled),
+    'content.local_storage':
+        Attribute(QWebEngineSettings.LocalStorageEnabled),
+    'content.cache.size':
+        # 0: automatically managed by QtWebEngine
+        DefaultProfileSetter('setHttpCacheMaximumSize', default=0),
+    'content.xss_auditing':
+        Attribute(QWebEngineSettings.XSSAuditingEnabled),
+    'content.default_encoding':
+        Setter(QWebEngineSettings.setDefaultTextEncoding),
+
+    'input.spatial_navigation':
+        Attribute(QWebEngineSettings.SpatialNavigationEnabled),
+    'input.links_included_in_focus_chain':
+        Attribute(QWebEngineSettings.LinksIncludedInFocusChain),
+
+    'fonts.web.family.standard':
+        FontFamilySetter(QWebEngineSettings.StandardFont),
+    'fonts.web.family.fixed':
+        FontFamilySetter(QWebEngineSettings.FixedFont),
+    'fonts.web.family.serif':
+        FontFamilySetter(QWebEngineSettings.SerifFont),
+    'fonts.web.family.sans_serif':
+        FontFamilySetter(QWebEngineSettings.SansSerifFont),
+    'fonts.web.family.cursive':
+        FontFamilySetter(QWebEngineSettings.CursiveFont),
+    'fonts.web.family.fantasy':
+        FontFamilySetter(QWebEngineSettings.FantasyFont),
+    'fonts.web.size.minimum':
+        Setter(QWebEngineSettings.setFontSize,
+               args=[QWebEngineSettings.MinimumFontSize]),
+    'fonts.web.size.minimum_logical':
+        Setter(QWebEngineSettings.setFontSize,
+               args=[QWebEngineSettings.MinimumLogicalFontSize]),
+    'fonts.web.size.default':
+        Setter(QWebEngineSettings.setFontSize,
+               args=[QWebEngineSettings.DefaultFontSize]),
+    'fonts.web.size.default_fixed':
+        Setter(QWebEngineSettings.setFontSize,
+               args=[QWebEngineSettings.DefaultFixedFontSize]),
+
+    'scrolling.smooth':
+        Attribute(QWebEngineSettings.ScrollAnimatorEnabled),
 }
 
 try:
-    MAPPINGS['general']['print-element-backgrounds'] = Attribute(
+    MAPPINGS['content.print_element_backgrounds'] = Attribute(
         QWebEngineSettings.PrintElementBackgrounds)
 except AttributeError:
     # Added in Qt 5.8
     pass
+
+
+if qtutils.version_check('5.8'):
+    MAPPINGS['spellcheck.languages'] = DictionaryLanguageSetter()
+
+
+if qtutils.version_check('5.9', compiled=False):
+    # https://bugreports.qt.io/browse/QTBUG-58650
+    MAPPINGS['content.cookies.store'] = PersistentCookiePolicy()

qutebrowser/browser/webengine/webenginetab.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -17,30 +17,27 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-# FIXME:qtwebengine remove this once the stubs are gone
-# pylint: disable=unused-variable
-
 """Wrapper over a QWebEngineView."""
 
+import math
 import functools
+import html as html_utils
 
 import sip
 from PyQt5.QtCore import pyqtSlot, Qt, QEvent, QPoint, QUrl, QTimer
 from PyQt5.QtGui import QKeyEvent
 from PyQt5.QtNetwork import QAuthenticator
-# pylint: disable=no-name-in-module,import-error,useless-suppression
 from PyQt5.QtWidgets import QApplication
-from PyQt5.QtWebEngineWidgets import (QWebEnginePage, QWebEngineScript,
-                                      QWebEngineProfile)
-# pylint: enable=no-name-in-module,import-error,useless-suppression
+from PyQt5.QtWebEngineWidgets import QWebEnginePage, QWebEngineScript
 
 from qutebrowser.browser import browsertab, mouse, shared
 from qutebrowser.browser.webengine import (webview, webengineelem, tabhistory,
                                            interceptor, webenginequtescheme,
-                                           webenginedownloads)
+                                           webenginedownloads,
+                                           webenginesettings)
 from qutebrowser.misc import miscwidgets
 from qutebrowser.utils import (usertypes, qtutils, log, javascript, utils,
-                               objreg, jinja)
+                               message, objreg, jinja, debug)
 
 
 _qute_scheme_handler = None
@@ -52,22 +49,24 @@ def init():
     # won't work...
     # https://www.riverbankcomputing.com/pipermail/pyqt/2016-September/038075.html
     global _qute_scheme_handler
-    app = QApplication.instance()
-    profile = QWebEngineProfile.defaultProfile()
 
-    log.init.debug("Initializing qute:* handler...")
+    app = QApplication.instance()
+    log.init.debug("Initializing qute://* handler...")
     _qute_scheme_handler = webenginequtescheme.QuteSchemeHandler(parent=app)
-    _qute_scheme_handler.install(profile)
+    _qute_scheme_handler.install(webenginesettings.default_profile)
+    _qute_scheme_handler.install(webenginesettings.private_profile)
 
     log.init.debug("Initializing request interceptor...")
     host_blocker = objreg.get('host-blocker')
     req_interceptor = interceptor.RequestInterceptor(
         host_blocker, parent=app)
-    req_interceptor.install(profile)
+    req_interceptor.install(webenginesettings.default_profile)
+    req_interceptor.install(webenginesettings.private_profile)
 
     log.init.debug("Initializing QtWebEngine downloads...")
     download_manager = webenginedownloads.DownloadManager(parent=app)
-    download_manager.install(profile)
+    download_manager.install(webenginesettings.default_profile)
+    download_manager.install(webenginesettings.private_profile)
     objreg.register('webengine-download-manager', download_manager)
 
 
@@ -82,17 +81,17 @@ _JS_WORLD_MAP = {
 
 class WebEngineAction(browsertab.AbstractAction):
 
-    """QtWebKit implementations related to web actions."""
+    """QtWebEngine implementations related to web actions."""
 
-    def _action(self, action):
-        self._widget.triggerPageAction(action)
+    action_class = QWebEnginePage
+    action_base = QWebEnginePage.WebAction
 
     def exit_fullscreen(self):
-        self._action(QWebEnginePage.ExitFullScreen)
+        self._widget.triggerPageAction(QWebEnginePage.ExitFullScreen)
 
     def save_page(self):
         """Save the current page."""
-        self._action(QWebEnginePage.SavePage)
+        self._widget.triggerPageAction(QWebEnginePage.SavePage)
 
 
 class WebEnginePrinting(browsertab.AbstractPrinting):
@@ -128,29 +127,37 @@ class WebEngineSearch(browsertab.AbstractSearch):
         super().__init__(parent)
         self._flags = QWebEnginePage.FindFlags(0)
 
-    def _find(self, text, flags, cb=None):
-        """Call findText on the widget with optional callback."""
-        if cb is None:
-            self._widget.findText(text, flags)
-        else:
-            self._widget.findText(text, flags, cb)
+    def _find(self, text, flags, callback, caller):
+        """Call findText on the widget."""
+        self.search_displayed = True
 
-    def search(self, text, *, ignore_case=False, reverse=False,
+        def wrapped_callback(found):
+            """Wrap the callback to do debug logging."""
+            found_text = 'found' if found else "didn't find"
+            if flags:
+                flag_text = 'with flags {}'.format(debug.qflags_key(
+                    QWebEnginePage, flags, klass=QWebEnginePage.FindFlag))
+            else:
+                flag_text = ''
+            log.webview.debug(' '.join([caller, found_text, text, flag_text])
+                              .strip())
+            if callback is not None:
+                callback(found)
+        self._widget.findText(text, flags, wrapped_callback)
+
+    def search(self, text, *, ignore_case='never', reverse=False,
                result_cb=None):
-        flags = QWebEnginePage.FindFlags(0)
-        if ignore_case == 'smart':
-            if not text.islower():
-                flags |= QWebEnginePage.FindCaseSensitively
-        elif not ignore_case:
-            flags |= QWebEnginePage.FindCaseSensitively
-        if reverse:
-            flags |= QWebEnginePage.FindBackward
-
         self.text = text
-        self._flags = flags
-        self._find(text, flags, result_cb)
+        self._flags = QWebEnginePage.FindFlags(0)
+        if self._is_case_sensitive(ignore_case):
+            self._flags |= QWebEnginePage.FindCaseSensitively
+        if reverse:
+            self._flags |= QWebEnginePage.FindBackward
+
+        self._find(text, self._flags, result_cb, 'search')
 
     def clear(self):
+        self.search_displayed = False
         self._widget.findText('')
 
     def prev_result(self, *, result_cb=None):
@@ -160,10 +167,10 @@ class WebEngineSearch(browsertab.AbstractSearch):
             flags &= ~QWebEnginePage.FindBackward
         else:
             flags |= QWebEnginePage.FindBackward
-        self._find(self.text, flags, result_cb)
+        self._find(self.text, flags, result_cb, 'prev_result')
 
     def next_result(self, *, result_cb=None):
-        self._find(self.text, self._flags, result_cb)
+        self._find(self.text, self._flags, result_cb, 'next_result')
 
 
 class WebEngineCaret(browsertab.AbstractCaret):
@@ -237,8 +244,47 @@ class WebEngineCaret(browsertab.AbstractCaret):
             raise browsertab.UnsupportedOperationError
         return self._widget.selectedText()
 
+    def _follow_selected_cb(self, js_elem, tab=False):
+        """Callback for javascript which clicks the selected element.
+
+        Args:
+            js_elem: The element serialized from javascript.
+            tab: Open in a new tab.
+        """
+        if js_elem is None:
+            return
+        assert isinstance(js_elem, dict), js_elem
+        elem = webengineelem.WebEngineElement(js_elem, tab=self._tab)
+        if tab:
+            click_type = usertypes.ClickTarget.tab
+        else:
+            click_type = usertypes.ClickTarget.normal
+
+        # Only click if we see a link
+        if elem.is_link():
+            log.webview.debug("Found link in selection, clicking. ClickTarget "
+                              "{}, elem {}".format(click_type, elem))
+            elem.click(click_type)
+
     def follow_selected(self, *, tab=False):
-        log.stub()
+        if self._tab.search.search_displayed:
+            # We are currently in search mode.
+            # let's click the link via a fake-click
+            # https://bugreports.qt.io/browse/QTBUG-60673
+            self._tab.search.clear()
+
+            log.webview.debug("Clicking a searched link via fake key press.")
+            # send a fake enter, clicking the orange selection box
+            if tab:
+                self._tab.key_press(Qt.Key_Enter, modifier=Qt.ControlModifier)
+            else:
+                self._tab.key_press(Qt.Key_Enter)
+
+        else:
+            # click an existing blue selection
+            js_code = javascript.assemble('webelem', 'find_selected_link')
+            self._tab.run_js_async(js_code, lambda jsret:
+                                   self._follow_selected_cb(jsret, tab))
 
 
 class WebEngineScroller(browsertab.AbstractScroller):
@@ -256,13 +302,10 @@ class WebEngineScroller(browsertab.AbstractScroller):
         page = widget.page()
         page.scrollPositionChanged.connect(self._update_pos)
 
-    def _key_press(self, key, count=1):
+    def _repeated_key_press(self, key, count=1, modifier=Qt.NoModifier):
+        """Send count fake key presses to this scroller's WebEngineTab."""
         for _ in range(min(count, 5000)):
-            press_evt = QKeyEvent(QEvent.KeyPress, key, Qt.NoModifier, 0, 0, 0)
-            release_evt = QKeyEvent(QEvent.KeyRelease, key, Qt.NoModifier,
-                                    0, 0, 0)
-            self._tab.send_event(press_evt)
-            self._tab.send_event(release_evt)
+            self._tab.key_press(key, modifier)
 
     @pyqtSlot()
     def _update_pos(self):
@@ -289,7 +332,7 @@ class WebEngineScroller(browsertab.AbstractScroller):
             else:
                 perc_y = min(100, round(100 / dy * jsret['px']['y']))
 
-            self._at_bottom = dy >= jsret['px']['y']
+            self._at_bottom = math.ceil(jsret['px']['y']) >= dy
             self._pos_perc = perc_x, perc_y
 
             self.perc_changed.emit(*self._pos_perc)
@@ -319,28 +362,28 @@ class WebEngineScroller(browsertab.AbstractScroller):
         self._tab.run_js_async(js_code)
 
     def up(self, count=1):
-        self._key_press(Qt.Key_Up, count)
+        self._repeated_key_press(Qt.Key_Up, count)
 
     def down(self, count=1):
-        self._key_press(Qt.Key_Down, count)
+        self._repeated_key_press(Qt.Key_Down, count)
 
     def left(self, count=1):
-        self._key_press(Qt.Key_Left, count)
+        self._repeated_key_press(Qt.Key_Left, count)
 
     def right(self, count=1):
-        self._key_press(Qt.Key_Right, count)
+        self._repeated_key_press(Qt.Key_Right, count)
 
     def top(self):
-        self._key_press(Qt.Key_Home)
+        self._tab.key_press(Qt.Key_Home)
 
     def bottom(self):
-        self._key_press(Qt.Key_End)
+        self._tab.key_press(Qt.Key_End)
 
     def page_up(self, count=1):
-        self._key_press(Qt.Key_PageUp, count)
+        self._repeated_key_press(Qt.Key_PageUp, count)
 
     def page_down(self, count=1):
-        self._key_press(Qt.Key_PageDown, count)
+        self._repeated_key_press(Qt.Key_PageDown, count)
 
     def at_top(self):
         return self.pos_px().y() == 0
@@ -356,24 +399,28 @@ class WebEngineHistory(browsertab.AbstractHistory):
     def current_idx(self):
         return self._history.currentItemIndex()
 
-    def back(self):
-        self._history.back()
-
-    def forward(self):
-        self._history.forward()
-
     def can_go_back(self):
         return self._history.canGoBack()
 
     def can_go_forward(self):
         return self._history.canGoForward()
 
+    def _item_at(self, i):
+        return self._history.itemAt(i)
+
+    def _go_to_item(self, item):
+        return self._history.goToItem(item)
+
     def serialize(self):
-        # WORKAROUND for https://github.com/qutebrowser/qutebrowser/issues/2289
-        # FIXME:qtwebengine can we get rid of this with Qt 5.8.1?
-        scheme = self._history.currentItem().url().scheme()
-        if scheme in ['view-source', 'chrome']:
-            raise browsertab.WebTabError("Can't serialize special URL!")
+        if not qtutils.version_check('5.9', compiled=False):
+            # WORKAROUND for
+            # https://github.com/qutebrowser/qutebrowser/issues/2289
+            # Don't use the history's currentItem here, because of
+            # https://bugreports.qt.io/browse/QTBUG-59599 and because it doesn't
+            # contain view-source.
+            scheme = self._tab.url().scheme()
+            if scheme in ['view-source', 'chrome']:
+                raise browsertab.WebTabError("Can't serialize special URL!")
         return qtutils.serialize(self._history)
 
     def deserialize(self, data):
@@ -398,9 +445,6 @@ class WebEngineZoom(browsertab.AbstractZoom):
     def _set_factor_internal(self, factor):
         self._widget.setZoomFactor(factor)
 
-    def factor(self):
-        return self._widget.zoomFactor()
-
 
 class WebEngineElements(browsertab.AbstractElements):
 
@@ -443,18 +487,18 @@ class WebEngineElements(browsertab.AbstractElements):
             callback(elem)
 
     def find_css(self, selector, callback, *, only_visible=False):
-        js_code = javascript.assemble('webelem', 'find_all', selector,
+        js_code = javascript.assemble('webelem', 'find_css', selector,
                                       only_visible)
         js_cb = functools.partial(self._js_cb_multiple, callback)
         self._tab.run_js_async(js_code, js_cb)
 
     def find_id(self, elem_id, callback):
-        js_code = javascript.assemble('webelem', 'element_by_id', elem_id)
+        js_code = javascript.assemble('webelem', 'find_id', elem_id)
         js_cb = functools.partial(self._js_cb_single, callback)
         self._tab.run_js_async(js_code, js_cb)
 
     def find_focused(self, callback):
-        js_code = javascript.assemble('webelem', 'focus_element')
+        js_code = javascript.assemble('webelem', 'find_focused')
         js_cb = functools.partial(self._js_cb_single, callback)
         self._tab.run_js_async(js_code, js_cb)
 
@@ -462,7 +506,7 @@ class WebEngineElements(browsertab.AbstractElements):
         assert pos.x() >= 0
         assert pos.y() >= 0
         pos /= self._tab.zoom.factor()
-        js_code = javascript.assemble('webelem', 'element_at_pos',
+        js_code = javascript.assemble('webelem', 'find_at_pos',
                                       pos.x(), pos.y())
         js_cb = functools.partial(self._js_cb_single, callback)
         self._tab.run_js_async(js_code, js_cb)
@@ -472,10 +516,11 @@ class WebEngineTab(browsertab.AbstractTab):
 
     """A QtWebEngine tab in the browser."""
 
-    def __init__(self, win_id, mode_manager, parent=None):
+    def __init__(self, *, win_id, mode_manager, private, parent=None):
         super().__init__(win_id=win_id, mode_manager=mode_manager,
-                         parent=parent)
-        widget = webview.WebEngineView(tabdata=self.data, win_id=win_id)
+                         private=private, parent=parent)
+        widget = webview.WebEngineView(tabdata=self.data, win_id=win_id,
+                                       private=private)
         self.history = WebEngineHistory(self)
         self.scroller = WebEngineScroller(self, parent=self)
         self.caret = WebEngineCaret(win_id=win_id, mode_manager=mode_manager,
@@ -556,9 +601,11 @@ class WebEngineTab(browsertab.AbstractTab):
 
     def shutdown(self):
         self.shutting_down.emit()
-        # WORKAROUND for
-        # https://bugreports.qt.io/browse/QTBUG-58563
-        self.search.clear()
+        self.action.exit_fullscreen()
+        if qtutils.version_check('5.8', exact=True, compiled=False):
+            # WORKAROUND for
+            # https://bugreports.qt.io/browse/QTBUG-58563
+            self.search.clear()
         self._widget.shutdown()
 
     def reload(self, *, force=False):
@@ -577,14 +624,12 @@ class WebEngineTab(browsertab.AbstractTab):
     def icon(self):
         return self._widget.icon()
 
-    def set_html(self, html, base_url=None):
+    def set_html(self, html, base_url=QUrl()):
         # FIXME:qtwebengine
         # check this and raise an exception if too big:
         # Warning: The content will be percent encoded before being sent to the
         # renderer via IPC. This may increase its size. The maximum size of the
         # percent encoded content is 2 megabytes minus 30 bytes.
-        if base_url is None:
-            base_url = QUrl()
         self._widget.setHtml(html, base_url)
 
     def networkaccessmanager(self):
@@ -596,8 +641,24 @@ class WebEngineTab(browsertab.AbstractTab):
     def clear_ssl_errors(self):
         raise browsertab.UnsupportedOperationError
 
+    def key_press(self, key, modifier=Qt.NoModifier):
+        press_evt = QKeyEvent(QEvent.KeyPress, key, modifier, 0, 0, 0)
+        release_evt = QKeyEvent(QEvent.KeyRelease, key, modifier,
+                                0, 0, 0)
+        self.send_event(press_evt)
+        self.send_event(release_evt)
+
     @pyqtSlot()
     def _on_history_trigger(self):
+        try:
+            self._widget.page()
+        except RuntimeError:
+            # Looks like this slot can be triggered on destroyed tabs:
+            # https://crashes.qutebrowser.org/view/3abffbed (Qt 5.9.1)
+            # wrapped C/C++ object of type WebEngineView has been deleted
+            log.misc.debug("Ignoring history trigger for destroyed tab")
+            return
+
         url = self.url()
         requested_url = self.url(requested=True)
 
@@ -615,6 +676,32 @@ class WebEngineTab(browsertab.AbstractTab):
 
         self.add_history_item.emit(url, requested_url, title)
 
+    @pyqtSlot(QUrl, 'QAuthenticator*', 'QString')
+    def _on_proxy_authentication_required(self, url, authenticator,
+                                          proxy_host):
+        """Called when a proxy needs authentication."""
+        msg = "<b>{}</b> requires a username and password.".format(
+            html_utils.escape(proxy_host))
+        answer = message.ask(
+            title="Proxy authentication required", text=msg,
+            mode=usertypes.PromptMode.user_pwd,
+            abort_on=[self.shutting_down, self.load_started])
+        if answer is not None:
+            authenticator.setUser(answer.user)
+            authenticator.setPassword(answer.password)
+        else:
+            try:
+                # pylint: disable=no-member, useless-suppression
+                sip.assign(authenticator, QAuthenticator())
+            except AttributeError:
+                url_string = url.toDisplayString()
+                error_page = jinja.render(
+                    'error.html',
+                    title="Error loading page: {}".format(url_string),
+                    url=url_string, error="Proxy authentication required",
+                    icon='')
+                self.set_html(error_page)
+
     @pyqtSlot(QUrl, 'QAuthenticator*')
     def _on_authentication_required(self, url, authenticator):
         # FIXME:qtwebengine support .netrc
@@ -632,19 +719,31 @@ class WebEngineTab(browsertab.AbstractTab):
                 error_page = jinja.render(
                     'error.html',
                     title="Error loading page: {}".format(url_string),
-                    url=url_string, error="Authentication required", icon='')
+                    url=url_string, error="Authentication required")
                 self.set_html(error_page)
 
     @pyqtSlot('QWebEngineFullScreenRequest')
     def _on_fullscreen_requested(self, request):
         request.accept()
         on = request.toggleOn()
+
+        self.data.fullscreen = on
         self.fullscreen_requested.emit(on)
         if on:
             notification = miscwidgets.FullscreenNotification(self)
             notification.show()
             notification.set_timeout(3000)
 
+    @pyqtSlot()
+    def _on_load_started(self):
+        """Clear search when a new load is started if needed."""
+        if (qtutils.version_check('5.9', compiled=False) and
+                not qtutils.version_check('5.9.2', compiled=False)):
+            # WORKAROUND for
+            # https://bugreports.qt.io/browse/QTBUG-61506
+            self.search.clear()
+        super()._on_load_started()
+
     @pyqtSlot(QWebEnginePage.RenderProcessTerminationStatus, int)
     def _on_render_process_terminated(self, status, exitcode):
         """Show an error when the renderer process terminated."""
@@ -680,6 +779,8 @@ class WebEngineTab(browsertab.AbstractTab):
         page.loadFinished.connect(self._on_load_finished)
         page.certificate_error.connect(self._on_ssl_errors)
         page.authenticationRequired.connect(self._on_authentication_required)
+        page.proxyAuthenticationRequired.connect(
+            self._on_proxy_authentication_required)
         page.fullScreenRequested.connect(self._on_fullscreen_requested)
         page.contentsSizeChanged.connect(self.contents_size_changed)
 

qutebrowser/browser/webengine/webview.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -23,28 +23,31 @@ import functools
 
 from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, PYQT_VERSION
 from PyQt5.QtGui import QPalette
-# pylint: disable=no-name-in-module,import-error,useless-suppression
 from PyQt5.QtWebEngineWidgets import QWebEngineView, QWebEnginePage
-# pylint: enable=no-name-in-module,import-error,useless-suppression
 
 from qutebrowser.browser import shared
-from qutebrowser.browser.webengine import certificateerror
+from qutebrowser.browser.webengine import certificateerror, webenginesettings
 from qutebrowser.config import config
-from qutebrowser.utils import (log, debug, usertypes, jinja, urlutils, message,
-                               objreg)
+from qutebrowser.utils import log, debug, usertypes, jinja, urlutils, message
 
 
 class WebEngineView(QWebEngineView):
 
     """Custom QWebEngineView subclass with qutebrowser-specific features."""
 
-    def __init__(self, tabdata, win_id, parent=None):
+    def __init__(self, *, tabdata, win_id, private, parent=None):
         super().__init__(parent)
         self._win_id = win_id
         self._tabdata = tabdata
 
         theme_color = self.style().standardPalette().color(QPalette.Base)
-        page = WebEnginePage(theme_color=theme_color, parent=self)
+        if private:
+            profile = webenginesettings.private_profile
+            assert profile.isOffTheRecord()
+        else:
+            profile = webenginesettings.default_profile
+        page = WebEnginePage(theme_color=theme_color, profile=profile,
+                             parent=self)
         self.setPage(page)
 
     def shutdown(self):
@@ -76,10 +79,10 @@ class WebEngineView(QWebEngineView):
             The new QWebEngineView object.
         """
         debug_type = debug.qenum_key(QWebEnginePage, wintype)
-        background_tabs = config.get('tabs', 'background-tabs')
+        background = config.val.tabs.background
 
-        log.webview.debug("createWindow with type {}, background_tabs "
-                          "{}".format(debug_type, background_tabs))
+        log.webview.debug("createWindow with type {}, background {}".format(
+            debug_type, background))
 
         if wintype == QWebEnginePage.WebBrowserWindow:
             # Shift-Alt-Click
@@ -91,13 +94,13 @@ class WebEngineView(QWebEngineView):
         elif wintype == QWebEnginePage.WebBrowserTab:
             # Middle-click / Ctrl-Click with Shift
             # FIXME:qtwebengine this also affects target=_blank links...
-            if background_tabs:
+            if background:
                 target = usertypes.ClickTarget.tab
             else:
                 target = usertypes.ClickTarget.tab_bg
         elif wintype == QWebEnginePage.WebBrowserBackgroundTab:
             # Middle-click / Ctrl-Click
-            if background_tabs:
+            if background:
                 target = usertypes.ClickTarget.tab_bg
             else:
                 target = usertypes.ClickTarget.tab
@@ -124,18 +127,18 @@ class WebEnginePage(QWebEnginePage):
     certificate_error = pyqtSignal()
     shutting_down = pyqtSignal()
 
-    def __init__(self, theme_color, parent=None):
-        super().__init__(parent)
+    def __init__(self, *, theme_color, profile, parent=None):
+        super().__init__(profile, parent)
         self._is_shutting_down = False
         self.featurePermissionRequested.connect(
             self._on_feature_permission_requested)
         self._theme_color = theme_color
         self._set_bg_color()
-        objreg.get('config').changed.connect(self._set_bg_color)
+        config.instance.changed.connect(self._set_bg_color)
 
-    @config.change_filter('colors', 'webpage.bg')
+    @config.change_filter('colors.webpage.bg')
     def _set_bg_color(self):
-        col = config.get('colors', 'webpage.bg')
+        col = config.val.colors.webpage.bg
         if col is None:
             col = self._theme_color
         self.setBackgroundColor(col)
@@ -144,11 +147,10 @@ class WebEnginePage(QWebEnginePage):
     def _on_feature_permission_requested(self, url, feature):
         """Ask the user for approval for geolocation/media/etc.."""
         options = {
-            QWebEnginePage.Geolocation: ('content', 'geolocation'),
-            QWebEnginePage.MediaAudioCapture: ('content', 'media-capture'),
-            QWebEnginePage.MediaVideoCapture: ('content', 'media-capture'),
-            QWebEnginePage.MediaAudioVideoCapture:
-                ('content', 'media-capture'),
+            QWebEnginePage.Geolocation: 'content.geolocation',
+            QWebEnginePage.MediaAudioCapture: 'content.media_capture',
+            QWebEnginePage.MediaVideoCapture: 'content.media_capture',
+            QWebEnginePage.MediaAudioVideoCapture: 'content.media_capture',
         }
         messages = {
             QWebEnginePage.Geolocation: 'access your location',
@@ -210,7 +212,7 @@ class WebEnginePage(QWebEnginePage):
         url_string = url.toDisplayString()
         error_page = jinja.render(
             'error.html', title="Error loading page: {}".format(url_string),
-            url=url_string, error=str(error), icon='')
+            url=url_string, error=str(error))
 
         if error.is_overridable():
             ignore = shared.ignore_certificate_errors(
@@ -272,19 +274,12 @@ class WebEnginePage(QWebEnginePage):
 
     def javaScriptConsoleMessage(self, level, msg, line, source):
         """Log javascript messages to qutebrowser's log."""
-        # FIXME:qtwebengine maybe unify this in the tab api somehow?
-        setting = config.get('general', 'log-javascript-console')
-        if setting == 'none':
-            return
-
-        level_to_logger = {
-            QWebEnginePage.InfoMessageLevel: log.js.info,
-            QWebEnginePage.WarningMessageLevel: log.js.warning,
-            QWebEnginePage.ErrorMessageLevel: log.js.error,
+        level_map = {
+            QWebEnginePage.InfoMessageLevel: usertypes.JsLogLevel.info,
+            QWebEnginePage.WarningMessageLevel: usertypes.JsLogLevel.warning,
+            QWebEnginePage.ErrorMessageLevel: usertypes.JsLogLevel.error,
         }
-        logstring = "[{}:{}] {}".format(source, line, msg)
-        logger = level_to_logger[level]
-        logger(logstring)
+        shared.javascript_log_message(level_map[level], source, line, msg)
 
     def acceptNavigationRequest(self,
                                 url: QUrl,

qutebrowser/browser/webkit/__init__.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #