Revert :quit to previous behavior, remove -all, alias :q to :close

Adds the --all option to quit to behave the same as before.
Also changes wq to wqa to reflect the change.

.appveyor.yml

@@ -5,15 +5,15 @@ cache:
 build: off
 environment:
     PYTHONUNBUFFERED: 1
-    PYTHON: C:\Python36\python.exe
+    PYTHON: C:\Python36-x64\python.exe
     matrix:
-    - TESTENV: py36-pyqt59
+    - TESTENV: py36-pyqt511
     - TESTENV: pylint
 
 install:
   - '%PYTHON% -m pip install -U pip'
   - '%PYTHON% -m pip install -r misc\requirements\requirements-tox.txt'
-  - 'set PATH=%PATH%;C:\Python36'
+  - 'set PATH=C:\Python36-x64;%PATH'
 
 test_script:
   - '%PYTHON% -m tox -e %TESTENV%'

.flake8

@@ -32,7 +32,7 @@ exclude = .*,__pycache__,resources.py
 # D403: First word of the first line should be properly capitalized
 #       (false-positives)
 # D413: Missing blank line after last section (not in pep257?)
-# A003: Builtin name for class attribute (needed for attrs)
+# A003: Builtin name for class attribute (needed for overridden methods)
 ignore =
     B001,B008,B305,
     E128,E226,E265,E501,E402,E266,E722,E731,
@@ -44,11 +44,11 @@ ignore =
 min-version = 3.4.0
 max-complexity = 12
 per-file-ignores =
-    tests/*/test_*.py : D100,D101,D401
-    tests/unit/browser/test_history.py : N806
-    tests/helpers/fixtures.py : N806
-    tests/unit/browser/webkit/http/test_content_disposition.py : D400
-    scripts/dev/ci/appveyor_install.py : FI53
+    /tests/**/*.py : D100,D101,D401
+    /tests/unit/browser/test_history.py : N806
+    /tests/helpers/fixtures.py : N806
+    /tests/unit/browser/webkit/http/test_content_disposition.py : D400
+    /scripts/dev/ci/appveyor_install.py : FI53
 copyright-check = True
 copyright-regexp = # Copyright [\d-]+ .*
 copyright-min-file-size = 110

.github/CODEOWNERS

@@ -8,3 +8,5 @@ tests/unit/completion/*  @rcorre
 tests/unit/misc/test_sql.py  @rcorre
 
 qutebrowser/config/configdata.yml  @mschilli87
+
+qutebrowser/javascript/caret.js @artur-shaik

.github/CONTRIBUTING.asciidoc

@@ -1,3 +1,8 @@
+IMPORTANT: I'm currently (July 2018) more busy than usual until September,
+because of exams coming up. Review of non-trivial pull requests will thus be
+delayed until then. If you're reading this note after mid-September, please
+open an issue.
+
 - Before you start to work on something, please leave a comment on the relevant
   issue (or open one). This makes sure there is no duplicate work done.
 

.gitignore

@@ -25,6 +25,7 @@ __pycache__
 /.tox
 /testresults.html
 /.cache
+/.pytest_cache
 /.testmondata
 /.hypothesis
 /.mypy_cache
@@ -40,3 +41,4 @@ TODO
 /scripts/testbrowser/cpp/webengine/testbrowser
 /scripts/testbrowser/cpp/webengine/.qmake.stash
 /scripts/dev/pylint_checkers/qute_pylint.egg-info
+/misc/file_version_info.txt

.travis.yml

@@ -14,15 +14,23 @@ matrix:
       services: docker
     - os: linux
       env: TESTENV=py36-pyqt571
-    - os: linux
-      env: TESTENV=py36-pyqt58
     - os: linux
       python: 3.5
-      env: TESTENV=py35-pyqt59
+      env: TESTENV=py35-pyqt571
     - os: linux
-      env: TESTENV=py36-pyqt59-cov
+      env: TESTENV=py36-pyqt59
+    - os: linux
+      env: TESTENV=py36-pyqt510
+    - os: linux
+      env: TESTENV=py36-pyqt511-cov
+    # https://github.com/travis-ci/travis-ci/issues/9069
+    - os: linux
+      python: 3.7
+      sudo: required
+      dist: xenial
+      env: TESTENV=py37-pyqt511
     - os: osx
-      env: TESTENV=py36 OSX=sierra
+      env: TESTENV=py37 OSX=sierra
       osx_image: xcode9.2
       language: generic
     # https://github.com/qutebrowser/qutebrowser/issues/2013
@@ -57,6 +65,10 @@ matrix:
       env: TESTENV=shellcheck
       services: docker
   fast_finish: true
+  allow_failures:
+    # https://github.com/qutebrowser/qutebrowser/issues/4055
+    - os: linux
+      env: TESTENV=py36-pyqt510
 
 cache:
   directories:

MANIFEST.in

@@ -1,6 +1,5 @@
 recursive-include qutebrowser *.py
 recursive-include qutebrowser/img *.svg *.png
-recursive-include qutebrowser/test *.py
 recursive-include qutebrowser/javascript *.js
 graft qutebrowser/html
 graft qutebrowser/3rdparty
@@ -8,7 +7,8 @@ graft icons
 graft doc/img
 graft misc/apparmor
 graft misc/userscripts
-recursive-include scripts *.py *.sh
+graft misc/requirements
+recursive-include scripts *.py *.sh *.js
 include qutebrowser/utils/testfile
 include qutebrowser/git-commit-id
 include LICENSE doc/* README.asciidoc
@@ -26,22 +26,18 @@ prune scripts/dev
 prune scripts/testbrowser/cpp
 prune .github
 exclude scripts/asciidoc2html.py
-exclude doc/notes
 recursive-exclude doc *.asciidoc
 include doc/qutebrowser.1.asciidoc
 include doc/changelog.asciidoc
 prune tests
 prune qutebrowser/3rdparty
-prune misc/requirements
-prune misc/docker
 exclude pytest.ini
-exclude qutebrowser.rcc
 exclude qutebrowser/javascript/.eslintrc.yaml
 exclude qutebrowser/javascript/.eslintignore
 exclude doc/help
 exclude .*
-exclude misc/appveyor_install.py
 exclude misc/qutebrowser.spec
 exclude misc/qutebrowser.nsi
+exclude misc/qutebrowser.rcc
 
 global-exclude __pycache__ *.pyc *.pyo

README.asciidoc

@@ -9,8 +9,6 @@ qutebrowser
 // QUTE_WEB_HIDE
 image:icons/qutebrowser-64x64.png[qutebrowser logo] *A keyboard-driven, vim-like browser based on PyQt5 and Qt.*
 
-image:https://img.shields.io/pypi/l/qutebrowser.svg?style=flat["license badge",link="https://github.com/qutebrowser/qutebrowser/blob/master/LICENSE"]
-image:https://img.shields.io/pypi/v/qutebrowser.svg?style=flat["version badge",link="https://pypi.python.org/pypi/qutebrowser/"]
 image:https://travis-ci.org/qutebrowser/qutebrowser.svg?branch=master["Build Status", link="https://travis-ci.org/qutebrowser/qutebrowser"]
 image:https://ci.appveyor.com/api/projects/status/5pyauww2k68bbow2/branch/master?svg=true["AppVeyor build status", link="https://ci.appveyor.com/project/qutebrowser/qutebrowser"]
 image:https://codecov.io/github/qutebrowser/qutebrowser/coverage.svg?branch=master["coverage badge",link="https://codecov.io/github/qutebrowser/qutebrowser?branch=master"]
@@ -44,8 +42,8 @@ Documentation
 In addition to the topics mentioned in this README, the following documents are
 available:
 
-* https://qutebrowser.org/img/cheatsheet-big.png[Key binding cheatsheet]: +
-image:https://qutebrowser.org/img/cheatsheet-small.png["qutebrowser key binding cheatsheet",link="https://qutebrowser.org/img/cheatsheet-big.png"]
+* https://raw.githubusercontent.com/qutebrowser/qutebrowser/master/doc/img/cheatsheet-big.png[Key binding cheatsheet]: +
+image:https://raw.githubusercontent.com/qutebrowser/qutebrowser/master/doc/img/cheatsheet-small.png["qutebrowser key binding cheatsheet",link="https://raw.githubusercontent.com/qutebrowser/qutebrowser/master/doc/img/cheatsheet-big.png"]
 * link:doc/quickstart.asciidoc[Quick start guide]
 * https://www.shortcutfoo.com/app/dojos/qutebrowser[Free training course] to remember those key bindings
 * link:doc/faq.asciidoc[Frequently asked questions]
@@ -91,7 +89,7 @@ https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser[mailinglist] at
 mailto:qutebrowser@lists.qutebrowser.org[].
 
 For security bugs, please contact me directly at mail@qutebrowser.org, GPG ID
-https://www.the-compiler.org/pubkey.asc[0xFD55A072].
+https://www.the-compiler.org/pubkey.asc[0x916eb0c8fd55a072].
 
 Requirements
 ------------
@@ -99,7 +97,7 @@ Requirements
 The following software and libraries are required to run qutebrowser:
 
 * http://www.python.org/[Python] 3.5 or newer (3.6 recommended)
-* http://qt.io/[Qt] 5.7.1 or newer with the following modules:
+* http://qt.io/[Qt] 5.7.1 or newer (5.11.1 recommended) with the following modules:
   - QtCore / qtbase
   - QtQuick (part of qtbase in some distributions)
   - QtSQL (part of qtbase in some distributions)
@@ -109,12 +107,12 @@ The following software and libraries are required to run qutebrowser:
     link:https://github.com/annulen/webkit/wiki[updated fork] (5.212) is
     supported
 * http://www.riverbankcomputing.com/software/pyqt/intro[PyQt] 5.7.0 or newer
-  (5.9.2 recommended) for Python 3
+  (5.11.2 recommended) for Python 3
 * https://pypi.python.org/pypi/setuptools/[pkg_resources/setuptools]
 * http://fdik.org/pyPEG/[pyPEG2]
 * http://jinja.pocoo.org/[jinja2]
 * http://pygments.org/[pygments]
-* http://pyyaml.org/wiki/PyYAML[PyYAML]
+* https://github.com/yaml/pyyaml[PyYAML]
 * http://www.attrs.org/[attrs]
 
 The following libraries are optional:

doc/backers.asciidoc

@@ -13,47 +13,75 @@ Thanks a lot to the following people who contributed to it:
 Gold sponsors
 ~~~~~~~~~~~~~
 
-TODO
+- Iggy
+- zwitschi
+- 2x Anonymous
 
 Silver sponsors
 ~~~~~~~~~~~~~~~
 
-TODO
+- https://benary.org[benaryorg]
+- https://scratchbook.ch[Claude]
+- Martin Tournoij
+- http://supported.elsensohn.ch[Thomas Elsensohn]
+- Christian Helbling
+- Gavin Troy
+- Chris King-Parra
+- Tim Das Mool Wegener
 
 Other sponsors
 ~~~~~~~~~~~~~~
 
-TODO: people with t-shirts or higher pledge levels
-
 - 7scan
+- AMD1212
+- Alex
 - Alex Suykov
 - Alexey Zhikhartsev
 - Allan Nordhøy
 - Anirudh Sanjeev
 - Anssi Puustinen
+- Anton Grensjö
+- Aristaeus
+- Armin Fisslthaler
+- Ashley Hauck
 - Benedikt Steindorf
 - Bernardo Kuri
 - Blaise Duszynski
 - Bostan
 - Bruno Oliveira
+- BunnyApocalypse
+- Christian Kellermann
 - Colin Jacobs
 - Daniel Andersson
+- Daniel Nelson
+- Daniel P. Schmidt
+- Daniel Salby
 - Danilo
 - David Beley
 - David Hollings
+- David Keijser
 - David Parrish
 - Derin Yarsuvat
 - Dmytro Kostiuchenko
+- Eero Kari
+- Epictek
+- Eric
+- Faure Hu
+- Ferus
 - Frederik Thorøe
 - G4v4g4i
+- Granitosaurus
 - Gyula Teleki
 - H
+- Heinz Bruhin
 - Hosaka
+- Ihor Radchenko
 - Iordanis Grigoriou
 - Isaac Sandaljian
 - Jakub Podeszwik
 - Jamie Anderson
 - Jasper Woudenberg
+- Jay Kamat
 - Jens Højgaard
 - Johannes
 - John Baber-Lucero
@@ -61,9 +89,11 @@ TODO: people with t-shirts or higher pledge levels
 - Kenichiro Ito
 - Kenny Low
 - Lars Ivar Igesund
+- Leulas
 - Lucas Aride Moulin
 - Ludovic Chabant
 - Lukas Gierth
+- Magnus Lindström
 - Marulkan
 - Matthew Chun-Lum
 - Matthew Cronen
@@ -80,7 +110,10 @@ TODO: people with t-shirts or higher pledge levels
 - Peter Rice
 - Philipp Middendorf
 - Pkill9
+- PluMGMK
 - Prescott
+- ProXicT
+- Ram-Z
 - Robotichead
 - Roshless
 - Ryan Ellis
@@ -90,35 +123,53 @@ TODO: people with t-shirts or higher pledge levels
 - Sean Herman
 - Sebastian Frysztak
 - Shelby Cruver
+- Simon Désaulniers
 - SirCmpwn
 - Soham Pal
+- Stephan Jauernick
 - Stewart Webb
 - Sven Reinecke
+- Timothée Floure
 - Tom Bass
+- Tom Kirchner
 - Tomas Slusny
 - Tomasz Kramkowski
 - Tommy Thomas
+- Tuscan
+- Ulrich Pötter
 - Vasilij Schneidermann
 - Vlaaaaaaad
+- XTaran
+- Z2h-A6n
+- ayekat
 - beanieuptop
+- cee
+- craftyguy
 - demure
+- dlangevi
+- epon
 - evenorbert
 - fishss
 - gsnewmark
 - guillermohs9
+- hernani
 - hubcaps
+- jnphilipp
 - lobachevsky
 - neodarz
 - nihlaeth
 - notbenh
+- nyctea
+- ongy
 - patrick suwanvithaya
 - pyratebeard
+- p≡p foundation
 - randm_dave
 - sabreman
 - toml
 - vimja
 - wiz
-- 44 Anonymous
+- 48 Anonymous
 
 2016
 ----

doc/changelog.asciidoc

@@ -15,6 +15,456 @@ breaking changes (such as renamed commands) can happen in minor releases.
 // `Fixed` for any bug fixes.
 // `Security` to invite users to upgrade in case of vulnerabilities.
 
+v1.5.0 (unreleased)
+-------------------
+
+Added
+~~~~~
+
+- The qute-pass userscript now has optional OTP support.
+
+v1.4.1 (unreleased)
+-------------------
+
+Fixed
+~~~~~
+
+- Rare crash when an error occurs in downloads.
+- Newlines are now stripped from the :version pastebin URL.
+- There's a new `mkvenv-pypi-old` environment in `tox.ini` which installs an
+  older Qt, which is needed on Ubuntu 16.04.
+- Worked around a Qt issue which redirects to a `chrome-error://` page when
+  trying to use U2F.
+- The `link_pyqt.py` script now works correctly with PyQt 5.11.
+
+v1.4.0
+------
+
+Added
+~~~~~
+
+- Support for the bundled `sip` module in PyQt 5.11 and other changes in
+  Qt/PyQt 5.11.x.
+- New `--debug-flag log-requests` to log requests to the debug log for
+  debugging.
+- New `--first` flag for `:hint` (bound to `gi` for inputs) which automatically
+  selects the first hint.
+- New `input.escape_quits_reporter` setting which can be used to avoid
+  accidentally quitting the crash reporter when pressing escape.
+- New `qute-lastpass` userscript which uses the LastPass CLI to fill passwords.
+- The Makefile now installs a `/usr/share/metainfo/qutebrowser.appdata.xml` file.
+- QtWebEngine: Support for printing from webpages via `window.print`.
+- QtWebEngine: Support for muting tabs:
+  * New `{audio}` field for `window.title_format` and `tabs.title.format` which
+    displays `[M]`/`[A]` for muted/recently audible tabs.
+  * New `:tab-mute` command (bound to `<Alt-m>`) to mute/unmute a tab.
+- QtWebEngine: Support for `content.cookies.accept` with third-party cookies
+  blocked by default (requires Qt 5.11).
+- QtWebEngine: New settings:
+  * Support for requesting persistent storage via
+    `navigator.webkitPersistentStorage.requestQuota` with a new
+    `content.persistent_storage` setting (requires Qt 5.11).
+    This setting also supports URL patterns.
+  * Support for registering custom protocol handlers via
+    `navigator.registerProtocolHandler` with a new
+    `content.register_protocol_handler` setting (requires Qt 5.11).
+    This setting also supports URL patterns.
+  * Support for WebRTC screen sharing with a new `content.desktop_capture`
+    setting (requires Qt 5.10).
+    This setting also supports URL patterns.
+  * New `content.autoplay` setting to enable/disable automatic video playback
+    (requires Qt 5.10).
+  * New `content.webrtc_public_interfaces_only` setting to only expose public
+    interfaces over WebRTC (requires Qt 5.9.2 or 5.11).
+  * New `content.canvas_reading` setting to disable reading from canvas
+    elements.
+
+Changed
+~~~~~~~
+
+- The following settings now support URL patterns:
+  * `content.headers.do_not_track`
+  * `content.headers.custom`
+  * `content.headers.accept_language`
+  * `content.headers.user_agent`
+  * `content.ssl_strict`
+  * `content.geolocation`
+  * `content.notifications`
+  * `content.media_capture`
+- The Windows/macOS releases now bundle Qt 5.11.1 which is based on
+  Chromium 65.0.3325.151 with security fixes up to Chromium 67.0.3396.87.
+- New short flags for commandline arguments: `-B` and `-T` for `--basedir` and
+  `--temp-basedir`; `-d` and `-D` for `--debug` and `--debug-flag`.
+- Deleting history items via `:history-clear` or `:completion-item-del` now
+  also removes that URL from QtWebEngine's visited links.
+- There's now completion for commands taking a variable count of arguments
+  (like `:config-cycle`).
+- QtWebEngine: On Qt 5.11.1, no reloads are needed anymore when switching
+  between pages with changed settings (e.g. `content.javascript.enabled`).
+- The `qt.force_software_rendering` setting changed from a boolean to taking
+  different values (`software-opengl`, `qt-quick` and `chromium`) for different
+  kinds of software rendering workarounds.
+- On Qt 5.11, using wayland with QtWebEngine is now possible when using
+  software rendering.
+- GreaseMonkey scripts now get their own global scope (based on the page's
+  one), which allows scripts like OneeChan to work.
+- Rapid hinting is now supported with the `yank` and `yank-primary` targets,
+  copying newline-separated links.
+- QtWebEngine: On Qt 5.11, the developer tools (inspector) can now be used
+  securely and without requiring the `--enable-webengine-inspector` option.
+- The `<Enter>` key (`:follow-selected`) now follows the currently focused
+  element if there's no selection.
+- The `--logfilter` argument now can be prepended with an exclamation mark
+  (e.g. `--logfilter '!init,destroy'`) to invert the filter.
+- `:view-source` now has a `--pygments` flag which uses the "old" way of
+  rendering sources even with QtWebEngine.
+- Improved error messages when a setting needs a newer Qt version.
+- QtWebEngine: Various improvements to make the cursor more visible in caret
+  browsing.
+- When a prompt is opened in insert/passthrough mode, the mode is restored
+  after closing the prompt.
+- On Qt 5.10 or newer, dictionaries are now read from the qutebrowser data
+  directory (e.g. `~/.local/share/qutebrowser`) instead of `/usr/share/qt`.
+  Existing dictionaries are copied over.
+- If an error while parsing `~/.netrc` occurs, the cause of the error is now
+  logged.
+- On Qt 5.9 or newer, certificate errors now show Chromium's detailed error
+  page.
+- Greasemonkey scripts now support a "@qute-js-world" tag to run them in a
+  different JavaScript context.
+
+Fixed
+~~~~~
+
+- Various subtle keyboard focus issues.
+- The security fix in v1.3.3 caused URLs with ampersands
+  (`www.example.com?one=1&two=2`) to send the wrong arguments when clicked on
+  the `qute://history` page.
+- Crash when opening a PDF page with PDF.js enabled (on QtWebKit), but no
+  PDF.js installed.
+- Crash when closing a tab shortly after opening it.
+
+Removed
+~~~~~~~
+
+- No prebuilt binaries for 32-bit Windows are supplied anymore. This is due to
+  Qt removing QtWebEngine support for those upstream. It might be possible to
+  distribute 32-bit binaries again with Qt 5.12 in December, but that will only
+  happen if it turns out enough people actually need 32-bit support.
+- `:tab-detach` which has been deprecated in v1.1.0 has been removed.
+- The `content.developer_extras` setting got removed. On QtWebKit, developer
+  extras are now automatically enabled when opening the inspector.
+
+v1.3.3
+------
+
+Security
+~~~~~~~~
+
+- An XSS vulnerability on the `qute://history` page allowed websites to inject
+  HTML into the page via a crafted title tag. This could allow them to steal
+  your browsing history. If you're currently unable to upgrade, avoid using
+  `:history`. A CVE request for this issue is pending, see
+  https://github.com/qutebrowser/qutebrowser/issues/4011[#4011] for updates.
+
+Fixed
+~~~~~
+
+- Crash in a workaround for a Qt 5.11 bug in rare circumstances.
+- Workaround for a Qt bug which preserves searches between page loads.
+- In v1.3.2 a dependency on the `PyQt5.QtQuickWidgets` module was accidentally
+  introduced. Since that module isn't packaged everywhere, it's been removed
+  again.
+
+v1.3.2
+------
+
+Fixed
+~~~~~
+
+- QtWebEngine: Improved workaround for a bug in Qt 5.11 where only the
+  top/bottom half of the window is used.
+- QtWebEngine: Work around a bug in Qt 5.11 where an endless loading-loop is
+  triggered when clicking a link with an unknown scheme.
+- QtWebEngine: When switching between pages with changed settings, less
+  unnecessary reloads are done now.
+- QtWebEngine: It's now possible to open external links such as `magnet://` or
+  `mailto:` via hints.
+
+v1.3.1
+------
+
+Fixed
+~~~~~
+
+- Work around a bug in Qt 5.11 where only the top/bottom half of the window is used.
+  This workaround is incomplete, but fixes the majority of the cases where this happens.
+- Work around keyboard focus issues with Qt 5.11.
+- Work around an issue in Qt 5.11 where e.g. activating JavaScript per-domain
+  needed a manual reload in some cases.
+- Don't crash when a ² key is pressed (e.g. on AZERTY keyboards).
+- Don't crash when a tab is opened and quickly closed again.
+
+
+v1.3.0
+------
+
+Added
+~~~~~
+
+- New `:scroll-to-anchor` command to scroll to an anchor in the document.
+- New `url.open_base_url` option to open the base URL of a searchengine when no
+  search term is given.
+- New `tabs.min_width` setting to configure the minimal width for tabs.
+- New userscripts:
+  * `getbib` to download bibtex information for DOIs on a page.
+  * `qute-keepass` to get passwords from KeePassX.
+
+Changed
+~~~~~~~
+
+- QtWebEngine: Support for JavaScript Shared Web Workers have been disabled on
+  Qt versions older than 5.11 because of security issues in in Chromium.
+  You can get the same effect in earlier versions via
+  `:set qt.args ['disable-shared-workers']`. An equivalent workaround is also
+  contained in Qt 5.9.5 and 5.10.1.
+- The file dialog for downloads now has basic tab completion based on the
+  entered text.
+- `:version` now shows OS information for POSIX OS other than Linux/macOS.
+- When there's an error inserting the text from an external editor, a backup
+  file is now saved.
+- The `window.hide_wayland_decoration` setting got renamed to
+  `window.hide_decoration` and now also works outside of wayland.
+- The `tabs.favicons.show` setting now can take three values: `'always'` (was
+  `True`), `'never'` (was `False`) and `'pinned'` (to only show favicons for
+  pinned tabs).
+- Hover tooltips on tabs now always show the webpage's title.
+- The default value for `content.host_blocking.lists` was changed to only
+  include https://github.com/StevenBlack/hosts[Steven Black's hosts-list] which
+  combines various sources.
+- Error messages when trying to wrap when `tabs.wrap` is `False` are now logged
+  to debug instead of messages.
+
+Fixed
+~~~~~
+
+- Using hints before a page is fully loaded is now possible again.
+- Selecting hints with the number keypad now works again.
+- Tab titles for tabs loaded from sessions should now really be correct instead
+  of showing the URL.
+- Loading URLs with customized settings from a session now avoids an additional
+  reload.
+- The window icon and title now get set correctly again.
+- The `tabs.switching_delay` setting now has a correct maximum value limit set.
+- The `taskadd` script now works properly when there's multi-line output.
+- QtWebEngine: Worked around issues with GreaseMonkey/stylesheets not being
+  loaded correctly in some situations.
+- The statusbar now more closely reflects the caret mode state.
+- The icon on Windows should now be displayed in a higher resolution.
+- The QtWebEngine development tools (inspector) now also work when JavaScript is
+  disabled globally.
+- Building `.exe` files now works when `upx` is installed on the system.
+- The keyhint widget now shows the correct text for chained modifiers.
+- Loading GreaseMonkey scripts now also works with Jinja2 2.8 (e.g. on Debian
+  Stable).
+- Adding styles with GreaseMonkey on fast sites now works properly.
+- Window ID 0 is now excluded properly from `:tab-take` completion.
+- A rare crash when cancelling a download has been fixed.
+- The Makefile (intended for packagers) now supports `PREFIX` properly.
+- The workaround for a black window with Nvidia graphics is now enabled on
+  non-Linux systems (like FreeBSD) as well.
+- Initial support for Qt 5.11.
+- Checking for a new version after sending a crash report now works properly
+  again.
+- `@match` in Greasemonkey scripts now more closely matches the proper pattern
+  syntax.
+- Searching via `/` or `?` now doesn't handle any characters in a special way.
+- Fixed crash when trying to retry some failed downloads on QtWebEngine.
+- An invalid spellcheck dictionary filename now doesn't crash anymore.
+- When no spellcheck dictionaries are configured, it's now disabled internally.
+  This works around an issue with entering special characters on Facebook
+  messenger.
+- The macOS release now should work again on macOS 10.11 and newer.
+
+v1.2.1
+------
+
+Fixed
+~~~~~
+
+- qutebrowser now starts properly when the PyQt5 QOpenGLFunctions package wasn't
+  found.
+- The keybinding cheatsheet on the quickstart page is now loaded from a local
+  `qute://` URL again.
+- With "tox -e mkvenv-pypi", PyQt 5.10.0 is used again instead of Qt 5.10.1,
+  because of an issue with Qt 5.10.1 which causes qutebrowser to fail to start
+  ("Could not find QtWebEngineProcess").
+- Unbinding keys which were bound in older qutebrowser versions now doesn't
+  crash anymore.
+- Fixed a crash when reloading a page which wasn't fully loaded with v1.2.0
+- Keys on the numeric keypad now fall back to the same bindings without `Num+`
+  if no `Num+` binding was found.
+- Fixed hinting on some pages with Qt < 5.10.
+- Titles are now displayed correctly again for tabs which are cloned or loaded
+  from sessions.
+- Shortcuts now correctly use `Ctrl` instead of `Command` on macOS again.
+
+v1.2.0
+------
+
+Added
+~~~~~
+
+- Initial implementation of per-domain settings:
+  * `:set` and `:config-cycle` now have a `-u`/`--pattern` argument taking a
+    https://developer.chrome.com/extensions/match_patterns[URL match pattern]
+    for supported settings.
+  * `config.set` in `config.py` now takes a third argument which is the pattern.
+  * New `with config.pattern('...') as p:` context manager for `config.py` to
+    use the shorthand syntax with a pattern.
+  * New `tsh` keybinding to toggle scripts for the current host. With a capital
+    `S`, the toggle is saved. With a capital `H`, subdomains are included. With
+    `u` instead of `h`, the exact current URL is used.
+  * New `tph` keybinding to toggle plugins, with the same additional binding
+    described above.
+- New QtWebEngine features:
+  * Caret/visual mode
+  * Authentication via ~/.netrc
+  * Retrying downloads with Qt 5.10 or newer
+  * Hinting and other features inside same-origin frames
+- New flags for existing commands:
+  * `:session-load` has a new `--delete` flag which deletes the
+    session after loading it.
+  * New `--no-last` flag for `:tab-focus` to not focus the last tab when focusing
+    the currently focused one.
+  * New `--edit` flag for `:view-source` to open the source in an external editor.
+  * New `--select` flag for `:follow-hint` which acts like the given string was entered but doesn't necessary follow the hint.
+- New special pages:
+  * `qute://bindings` (opened via `:bind`) which shows all keybindings.
+  * `qute://tabs` (opened via `:buffer`) which lists all tabs.
+- New settings:
+  * `statusbar.widgets` to configure which widgets should be shown in which
+    order in the statusbar.
+  * `tabs.mode_on_change` which replaces `tabs.persist_mode_on_change`. It can
+    now be set to `restore` which remembers input modes (input/passthrough)
+    per tab.
+  * `input.insert_mode.auto_enter` which makes it possible to disable entering
+    insert mode automatically when an editable element was clicked. Together
+    with `input.forward_unbound_keys`, this should allow for emacs-like
+    "modeless" keybindings.
+- New `:prompt-yank` command (bound to `Alt-y` by default) to yank URLs
+  referenced in prompts.
+- The `hostblock_blame` script which was removed in v1.0 was updated for the new
+  config and re-added.
+- New `cycle-inputs.js` script in `scripts/` which can be used with `:jseval -f`
+  to cycle through inputs.
+
+Changed
+~~~~~~~
+
+- Complete refactoring of key input handling, with various effects:
+  * emacs-like keychains such as `<Ctrl-X><Ctrl-C>` can now be bound.
+  * Key chains can now be bound in any mode (this allows binding unused keys in
+    hint mode).
+  * Yes/no prompts don't use keybindings from the `prompt` section anymore, they
+    have their own `yesno` section instead.
+  * Trying to bind invalid keys now shows an error.
+  * The `bindings.default` setting can now only be set in a `config.py`, and
+    existing values in `autoconfig.yml` are ignored.
+- Improvements for GreaseMonkey support:
+  * `@include` and `@exclude` now support regex matches. With QtWebEngine and Qt
+    5.8 and newer, Qt handles the matching, but similar functionality will be
+    added in Qt 5.11.
+  * Support for `@requires`
+  * Support for the GreaseMonkey 4.0 API
+- The sqlite history now uses write-ahead logging which should be
+  a performance and stability improvement.
+- When an editor is spawned with `:open-editor` and `:config-edit`, the changes
+  are now applied as soon as the file is saved in the editor.
+- The `hist_importer.py` script now only imports URL schemes qutebrowser can
+  handle.
+- Deleting a prefix (`:`, `/` or `?`) via backspace now leaves command mode.
+- Angular 1 elements and `<summary>`/`<details>` now get hints assigned.
+- `:tab-only` with pinned tabs now still closes unpinned tabs.
+- The `url.incdec_segments` option now also can take `port` as possible segment.
+- QtWebEngine: `:view-source` now uses Chromium's `view-source:` scheme.
+- Tabs now show their full title as tooltip.
+- When there are multiple unknown keys in a autoconfig.yml, they now all get
+  reported in one error.
+- More performance improvements when opening/closing many tabs.
+- The `:version` page now has a button to pastebin the information.
+- Replacements like `{url}` can now be escaped as `{{url}}`.
+
+Fixed
+~~~~~
+
+- QtWebEngine bugfixes:
+  * Improved fullscreen handling with Qt 5.10.
+  * Hinting and scrolling now works properly on special `view-source:` pages.
+  * Scroll positions are now restored correctly from sessions.
+  * `:follow-selected` should now work in more cases with Qt > 5.10.
+  * Incremental search now flickers less and doesn't move to the second result
+    when pressing Enter.
+  * Keys like `Ctrl-V` or `Shift-Insert` are now correctly handled/filtered with
+    Qt 5.10.
+  * Fixed hangs/segfaults on exit with Qt 5.10.1.
+  * Fixed favicons sometimes getting cleared with Qt 5.10.
+  * Qt download objects are now cleaned up properly when a download is removed.
+  * JavaScript messages are now not double-HTML escaped anymore on Qt < 5.11
+- QtWebKit bugfixes:
+  * Fixed GreaseMonkey-related crashes.
+  * `:view-source` now displays a valid URL.
+- URLs containing ampersands and other special chars are now shown correctly
+  when filtering them in the completion.
+- `:bookmark-add "" foo` can now be used to save the current URL with a custom
+  title.
+- `:spawn -o` now waits until the process has finished before trying to show the
+  output. Previously, it incorrectly showed the previous output immediately.
+- Suspended pages now should always load the correct page when being un-suspended.
+- Exception types are now shown properly with `:config-source` and `:config-edit`.
+- When using `:bookmark-add --toggle`, bookmarks are now saved properly.
+- Crash when opening an invalid URL from an application on macOS.
+- Crash with an empty `completion.timestamp_format`.
+- Crash when `completion.min_chars` is set in some cases.
+- HTML/JS resource files are now read into RAM on start to avoid crashes when
+  changing qutebrowser versions while it's open.
+- Setting `bindings.key_mappings` to an empty value is now allowed.
+- Bindings to an empty commands are now ignored rather than crashing.
+
+Removed
+~~~~~~~
+
+- `QUTE_SELECTED_HTML` is now not set for userscripts anymore except when called
+  via hints.
+- The `qutebrowser_viewsource` userscript has been removed as
+  `:view-source --edit` can now be used.
+- The `tabs.persist_mode_on_change` setting has been removed and replaced by
+  `tabs.mode_on_change`.
+
+v1.1.2
+------
+
+Changed
+~~~~~~~
+
+- Windows/macOS releases now bundle Qt 5.10.1 which includes security fixes from
+  Chromium up to version 64.0.3282.140.
+
+Fixed
+~~~~~
+
+- QtWebEngine: Crash with Qt 5.10.1 when using :undo on some tabs.
+- Compatibility with Python 3.7
+
+v1.1.1
+------
+
+Fixed
+~~~~~
+
+- The Makefile now actually works.
+- Fixed crashes with Qt 5.10 when closing a tab before it finished loading.
+
 v1.1.0
 ------
 
@@ -1094,7 +1544,7 @@ Added
 - New `:fake-key` command to send a fake keypress to a website or to
   qutebrowser.
 - New `--mhtml` argument for `:download` to download a page including all
-  ressources as MHTML file.
+  resources as MHTML file.
 - New option `tabs -> title-alignment` to change the alignment of tab titles.
 
 Changed
@@ -1294,7 +1744,7 @@ Added
 - New argument `--no-err-windows` to suppress all error windows.
 - New arguments `--top-navigate` and `--bottom-navigate` (`-t`/`-b`) for `:scroll-page` to specify a navigation action (e.g. automatically go to the next page when arriving at the bottom).
 - New flag `-d`/`--detach` for `:spawn` to detach the spawned process so it's not closed when qutebrowser is.
-- New flag `-v`/`--verbose` for `:spawn` to print informations when the process started/exited successfully.
+- New flag `-v`/`--verbose` for `:spawn` to print information when the process started/exited successfully.
 - Many new color settings (foreground setting for every background setting).
 - New setting `ui -> modal-js-dialog` to use the standard modal dialogs for javascript questions instead of using the statusbar.
 - New setting `colors -> webpage.bg` to set the background color to use for websites which don't set one.

doc/contributing.asciidoc

@@ -5,6 +5,11 @@ The Compiler <mail@qutebrowser.org>
 :data-uri:
 :toc:
 
+IMPORTANT: I'm currently (July 2018) more busy than usual until September,
+because of exams coming up. Review of non-trivial pull requests will thus be
+delayed until then. If you're reading this note after mid-September, please
+open an issue.
+
 I `&lt;3` footnote:[Of course, that says `<3` in HTML.] contributors!
 
 This document contains guidelines for contributing to qutebrowser, as well as
@@ -44,8 +49,8 @@ be easy to solve]
 If you prefer C++ or Javascript to Python, see the relevant issues which involve
 work in those languages:
 
-* https://github.com/qutebrowser/qutebrowser/issues?utf8=%E2%9C%93&q=is%3Aopen%20is%3Aissue%20label%3Ac%2B%2B[C++] (mostly work on Qt, the library behind qutebrowser)
-* https://github.com/qutebrowser/qutebrowser/issues?q=is%3Aopen+is%3Aissue+label%3Ajavascript[JavaScript]
+* https://github.com/qutebrowser/qutebrowser/issues?q=is%3Aopen+is%3Aissue+label%3A%22language%3A+c%2B%2B%22[C++] (mostly work on Qt, the library behind qutebrowser)
+* https://github.com/qutebrowser/qutebrowser/issues?q=is%3Aopen+is%3Aissue+label%3A%22language%3A+javascript%22[JavaScript]
 
 There are also some things to do if you don't want to write code:
 
@@ -85,6 +90,16 @@ git format-patch origin/master <1>
 <1> Replace `master` by the branch your work was based on, e.g.,
 `origin/develop`.
 
+Running qutebrowser
+-------------------
+
+After link:install.asciidoc#tox[installing qutebrowser via tox], you can run
+`.venv/bin/qutebrowser --debug --temp-basedir` to test your changes with debug
+logging enabled and without affecting existing running instances.
+
+Alternatively, you can install qutebrowser's dependencies system-wide and run
+`python3 -m qutebrowser --debug --temp-basedir`.
+
 Useful utilities
 ----------------
 
@@ -188,8 +203,8 @@ There are some useful functions for debugging in the `qutebrowser.utils.debug`
 module.
 
 When starting qutebrowser with the `--debug` flag, you also get useful debug
-logs. You can add +--logfilter _category[,category,...]_+ to restrict logging
-to the given categories.
+logs. You can add +--logfilter _[!]category[,category,...]_+ to restrict
+logging to the given categories.
 
 With `--debug` there are also some additional +debug-_*_+ commands available,
 for example `:debug-all-objects` and `:debug-all-widgets` which print a list of
@@ -375,7 +390,7 @@ The following logging levels are available for every logger:
 |error     |There was an issue and some kind of operation was abandoned.
 |warning   |There was an issue but the operation can continue running.
 |info      |General informational messages.
-|debug     |Verbose debugging informations.
+|debug     |Verbose debugging information.
 |=======================================================================
 
 [[commands]]
@@ -566,6 +581,23 @@ can be useful for debugging:
 - chrome://gpuclean/ (crashes the current renderer process!)
 - chrome://ppapiflashcrash/
 - chrome://ppapiflashhang/
+- chrome://quota-internals/ (Qt 5.11)
+- chrome://taskscheduler-internals/ (Qt 5.11)
+- chrome://sandbox/ (Qt 5.11, Linux only)
+
+QtWebEngine internals
+~~~~~~~~~~~~~~~~~~~~~
+
+This is mostly useful for qutebrowser maintainers to work around issues in Qt - if you don't understand it, don't worry, just ignore it.
+
+The hierarchy of widgets when QtWebEngine is involved looks like this:
+
+- qutebrowser has a `WebEngineTab` object, which is its abstraction over QtWebKit/QtWebEngine.
+- The `WebEngineTab` has a `_widget` attribute, which is the https://doc.qt.io/qt-5/qwebengineview.html[QWebEngineView]
+- That view has a https://doc.qt.io/qt-5/qwebenginepage.html[QWebEnginePage] for everything which doesn't require rendering.
+- The view also has a layout with exactly one element (which also is its `focusProxy()`)
+- That element is the  http://code.qt.io/cgit/qt/qtwebengine.git/tree/src/webenginewidgets/render_widget_host_view_qt_delegate_widget.cpp[RenderWidgetHostViewQtDelegateWidget] (it inherits https://doc.qt.io/qt-5/qquickwidget.html[QQuickWidget]) - also often referred to as RWHV or RWHVQDW. It can be obtained via `sip.cast(tab._widget.focusProxy(), QQuickWidget)`.
+- Calling `rootObject()` on that gives us the https://doc.qt.io/qt-5/qquickitem.html[QQuickItem] where Chromium renders into (?). With it, we can do things like `.setRotation(20)`.
 
 Style conventions
 -----------------
@@ -662,18 +694,17 @@ New PyQt release
 ~~~~~~~~~~~~~~~~
 
 * See above.
-* Install new PyQt in Windows VM (32- and 64-bit).
-* Download new installer and update PyQt installer path in `ci_install.py`.
 * Update `tox.ini`/`.travis.yml`/`.appveyor.yml` to test new versions.
 
 qutebrowser release
 ~~~~~~~~~~~~~~~~~~~
 
 * Make sure there are no unstaged changes and the tests are green.
+* Make sure all issues with the related milestone are closed.
 * Run `x=... y=...` to set the respective shell variables.
 
-* Adjust `__version_info__` in `qutebrowser/__init__.py`.
 * Update changelog (remove *(unreleased)*).
+* Adjust `__version_info__` in `qutebrowser/__init__.py`.
 * Commit.
 
 * Create annotated git tag (`git tag -s "v1.$x.$y" -m "Release v1.$x.$y"`).
@@ -683,9 +714,11 @@ qutebrowser release
 * Mark the milestone at https://github.com/qutebrowser/qutebrowser/milestones
 as closed.
 
-* Linux: Run `git checkout v1.$x.$y && python3 scripts/dev/build_release.py --upload v1.$x.$y`.
-* Windows: Run `git checkout v1.X.Y; C:\Python36-32\python scripts\dev\build_release.py --asciidoc C:\Python27\python C:\asciidoc-8.6.9\asciidoc.py --upload v1.X.Y` (replace X/Y by hand).
-* macOS: Run `git checkout v1.X.Y && python3 scripts/dev/build_release.py --upload v1.X.Y` (replace X/Y by hand).
-* On server: Run `python3 scripts/dev/download_release.py v1.X.Y` (replace X/Y by hand).
+* Linux: Run `git checkout v1.$x.$y && ./.venv/bin/python3 scripts/dev/build_release.py --upload v1.$x.$y`.
+* Windows: Run `git checkout v1.X.Y; py -3.6 scripts\dev\build_release.py --asciidoc C:\Python27\python C:\asciidoc-8.6.9\asciidoc.py --upload v1.X.Y` (replace X/Y by hand).
+* macOS: Run `pyenv shell 3.6.6 && git checkout v1.X.Y && python3 scripts/dev/build_release.py --upload v1.X.Y` (replace X/Y by hand).
+* On server:
+  - Run `python3 scripts/dev/download_release.py v1.X.Y` (replace X/Y by hand).
+  - Run `git pull github master && sudo python3 scripts/asciidoc2html.py --website /srv/http/qutebrowser`
 * Update `qutebrowser-git` PKGBUILD if dependencies/install changed.
 * Announce to qutebrowser and qutebrowser-announce mailinglist.

doc/faq.asciidoc

@@ -32,7 +32,7 @@ When qutebrowser was created, the newer
 http://webkitgtk.org/reference/webkit2gtk/stable/index.html[WebKit2 API] lacked
 basic features like proxy support, and almost no projects have started porting
 to WebKit2. In the meantime, this situation has improved a bit, but there are
-stil only a few project which have some kind of WebKit2 support (see the
+still only a few projects which have some kind of WebKit2 support (see the
 https://github.com/qutebrowser/qutebrowser#similar-projects[list of
 alternatives]).
 +
@@ -70,6 +70,31 @@ But isn't Python too slow for a browser?::
     and WebKit in C++, with the
     https://wiki.python.org/moin/GlobalInterpreterLock[GIL] released.
 
+Is qutebrowser secure?::
+    Most security issues are in the backend (which handles networking,
+    rendering, JavaScript, etc.) and not qutebrowser itself.
++
+qutebrowser uses http://wiki.qt.io/QtWebEngine[QtWebEngine] by default.
+QtWebEngine is based on Google's https://www.chromium.org/Home[Chromium]. While
+Qt only updates to a new Chromium release on every minor Qt release (all ~6
+months), every patch release backports security fixes from newer Chromium
+versions. In other words: As long as you're using an up-to-date Qt, you should
+be recieving security updates on a regular basis, without qutebrowser having to
+do anything. Chromium's process isolation and
+https://chromium.googlesource.com/chromium/src/+/master/docs/design/sandbox.md[sandboxing]
+features are also enabled as a second line of defense.
++
+http://wiki.qt.io/QtWebKit[QtWebKit] is also supported as an alternative
+backend, but hasn't seen new releases
+https://github.com/annulen/webkit/releases[in a while]. It also doesn't have any
+process isolation or sandboxing.
++
+Security issues in qutebrowser's code happen very rarely (as per March 2018,
+there has been one security issue caused by qutebrowser in over four years) and
+are fixed timely. To report security bugs, please contact me directly at
+mail@qutebrowser.org, GPG ID
+https://www.the-compiler.org/pubkey.asc[0x916eb0c8fd55a072].
+
 Is there an adblocker?::
     There is a host-based adblocker which takes /etc/hosts-like lists. A "real"
     adblocker has a
@@ -187,6 +212,37 @@ Why takes it longer to open an URL in qutebrowser than in chromium?::
     qutebrowser if it is not running already. Also check if you want
     to use webengine as backend in line 17 and change it to your
     needs.
+
+How do I make qutebrowser use greasemonkey scripts?::
+    There is currently no UI elements to handle managing greasemonkey scripts.
+    All management of what scripts are installed or disabled is done in the
+    filesystem by you. qutebrowser reads all files that have an extension of
+    `.js` from the `<data>/greasemonkey/` folder and attempts to load them.
+    Where `<data>` is the qutebrowser data directory shown in the `Paths`
+    section of the page displayed by `:version`. If you want to disable a
+    script just rename it, for example, to have `.disabled` on the end, after
+    the `.js` extension. To reload scripts from that directory run the command
+    `:greasemonkey-reload`.
++
+Troubleshooting: to check that your script is being loaded when
+`:greasemonkey-reload` runs you can start qutebrowser with the arguments
+`--debug --logfilter greasemonkey,js` and check the messages on the
+program's standard output for errors parsing or loading your script.
+You may also see javascript errors if your script is expecting an environment
+that we fail to provide.
++
+Note that there are some missing features which you may run into:
+
+. Some scripts expect `GM_xmlhttpRequest` to ignore Cross Origin Resource
+  Sharing restrictions, this is currently not supported, so scripts making
+  requests to third party sites will often fail to function correctly.
+. If your backend is a QtWebEngine version 5.8, 5.9 or 5.10 then regular
+  expressions are not supported in `@include` or `@exclude` rules. If your
+  script uses them you can re-write them to use glob expressions or convert
+  them to `@match` rules.
+  See https://wiki.greasespot.net/Metadata_Block[the wiki] for more info.
+. Any greasemonkey API function to do with adding UI elements is not currently
+  supported. That means context menu extentensions and background pages.
     
 == Troubleshooting
 
@@ -216,6 +272,29 @@ And then re-emerging qtwebengine with: +
     
     emerge -1 qtwebengine
 
+Unable to view DRM content (Netflix, Spotify, etc.).::
+    You will need to install `widevine` and set `qt.args` to point to it.
+    Qt 5.9 currently only supports widevine up to Chrome version 61.
++
+On Arch, simply install `qt5-webengine-widevine` from the AUR and run:
++
+----
+:set qt.args '["ppapi-widevine-path=/usr/lib/qt/plugins/ppapi/libwidevinecdmadapter.so"]'
+:restart
+----
++
+For other distributions, download the chromium tarball and widevine-cdm zip from
+https://aur.archlinux.org/packages/qt5-webengine-widevine/[the AUR page],
+extract `libwidevinecdmadapter.so` and `libwidevinecdm.so` files, respectively,
+and move them to the `ppapi` plugin directory in your Qt library directory (create it if it does not exist).
++
+Lastly, set your `qt.args` to point to that directory and restart qutebrowser:
++
+----
+:set qt.args '["ppapi-widevine-path=/usr/lib64/qt5/plugins/ppapi/libwidevinecdmadapter.so"]'
+:restart
+----
+
 My issue is not listed.::
     If you experience any segfaults or crashes, you can report the issue in
     https://github.com/qutebrowser/qutebrowser/issues[the issue tracker] or

doc/help/commands.asciidoc

@@ -1,6 +1,7 @@
 // DO NOT EDIT THIS FILE DIRECTLY!
 // It is autogenerated by running:
 //   $ python3 scripts/dev/src2asciidoc.py
+// vim: readonly:
 
 = Commands
 
@@ -13,6 +14,7 @@ For command arguments, there are also some variables you can use:
 
 - `{url}` expands to the URL of the current page
 - `{url:pretty}` expands to the URL in decoded format
+- `{url:host}` expands to the host part of the URL
 - `{clipboard}` expands to the clipboard contents
 - `{primary}` expands to the primary selection contents
 
@@ -91,6 +93,7 @@ It is possible to run or bind multiple commands by separating them with `;;`.
 |<<scroll,scroll>>|Scroll the current tab in the given direction.
 |<<scroll-page,scroll-page>>|Scroll the frame page-wise.
 |<<scroll-px,scroll-px>>|Scroll the current tab by 'count * dx/dy' pixels.
+|<<scroll-to-anchor,scroll-to-anchor>>|Scroll to the given anchor in the document.
 |<<scroll-to-perc,scroll-to-perc>>|Scroll to a specific percentage of the page.
 |<<search,search>>|Search for a text on the current page. With no text, clear results.
 |<<search-next,search-next>>|Continue the search to the ([count]th) next term.
@@ -108,6 +111,7 @@ It is possible to run or bind multiple commands by separating them with `;;`.
 |<<tab-focus,tab-focus>>|Select the tab given as argument/[count].
 |<<tab-give,tab-give>>|Give the current tab to a new or existing window if win_id given.
 |<<tab-move,tab-move>>|Move the current tab according to the argument and [count].
+|<<tab-mute,tab-mute>>|Mute/Unmute the current/[count]th tab.
 |<<tab-next,tab-next>>|Switch to the next tab, or switch [count] tabs forward.
 |<<tab-only,tab-only>>|Close all tabs except for the current one.
 |<<tab-pin,tab-pin>>|Pin/Unpin the current/[count]th tab.
@@ -145,14 +149,16 @@ How many pages to go back.
 
 [[bind]]
 === bind
-Syntax: +:bind [*--mode* 'mode'] [*--default*] 'key' ['command']+
+Syntax: +:bind [*--mode* 'mode'] [*--default*] ['key'] ['command']+
 
 Bind a key to a command.
 
-==== positional arguments
-* +'key'+: The keychain or special key (inside `<...>`) to bind.
-* +'command'+: The command to execute, with optional args, or not given to print the current binding.
+If no command is given, show the current binding for the given key. Using :bind without any arguments opens a page showing all keybindings.
 
+==== positional arguments
+* +'key'+: The keychain to bind. Examples of valid keychains are `gC`, `<Ctrl-X>` or `<Ctrl-C>a`.
+
+* +'command'+: The command to execute, with optional args.
 
 ==== optional arguments
 * +*-m*+, +*--mode*+: A comma-separated list of modes to bind the key in (default: `normal`). See `:help bindings.commands` for the
@@ -174,7 +180,8 @@ Save the current page as a bookmark, or a specific url.
 If no url and title are provided, then save the current page as a bookmark. If a url and title have been provided, then save the given url as a bookmark with the provided title. You can view all saved bookmarks on the link:qute://bookmarks[bookmarks page].
 
 ==== positional arguments
-* +'url'+: url to save as a bookmark. If None, use url of current page.
+* +'url'+: url to save as a bookmark. If not given, use url of current page.
+
 * +'title'+: title of the new bookmark.
 
 ==== optional arguments
@@ -218,7 +225,7 @@ Syntax: +:buffer ['index']+
 
 Select tab by index or url/title best match.
 
-Focuses window if necessary when index is given. If both index and count are given, use count.
+Focuses window if necessary when index is given. If both index and count are given, use count. With neither index nor count given, open the qute://tabs page.
 
 ==== positional arguments
 * +'index'+: The [win_id/]index of the tab to focus. Or a substring in which case the closest match will be focused.
@@ -271,7 +278,8 @@ Set all settings back to their default.
 
 [[config-cycle]]
 === config-cycle
-Syntax: +:config-cycle [*--temp*] [*--print*] 'option' ['values' ['values' ...]]+
+Syntax: +:config-cycle [*--pattern* 'pattern'] [*--temp*] [*--print*]
+             'option' ['values' ['values' ...]]+
 
 Cycle an option between multiple values.
 
@@ -280,6 +288,7 @@ Cycle an option between multiple values.
 * +'values'+: The values to cycle through.
 
 ==== optional arguments
+* +*-u*+, +*--pattern*+: The URL pattern to use.
 * +*-t*+, +*--temp*+: Set value temporarily until qutebrowser is closed.
 * +*-p*+, +*--print*+: Print the value after setting.
 
@@ -492,10 +501,16 @@ Toggle fullscreen mode.
 
 [[greasemonkey-reload]]
 === greasemonkey-reload
+Syntax: +:greasemonkey-reload [*--force*]+
+
 Re-read Greasemonkey scripts from disk.
 
 The scripts are read from a 'greasemonkey' subdirectory in qutebrowser's data directory (see `:version`).
 
+==== optional arguments
+* +*-f*+, +*--force*+: For any scripts that have required dependencies, re-download them.
+
+
 [[help]]
 === help
 Syntax: +:help [*--tab*] [*--bg*] [*--window*] ['topic']+
@@ -516,7 +531,7 @@ Show help about a command or setting.
 
 [[hint]]
 === hint
-Syntax: +:hint [*--mode* 'mode'] [*--add-history*] [*--rapid*]
+Syntax: +:hint [*--mode* 'mode'] [*--add-history*] [*--rapid*] [*--first*]
      ['group'] ['target'] ['args' ['args' ...]]+
 
 Start hinting.
@@ -586,6 +601,7 @@ Start hinting.
  `tab` (with `tabs.background_tabs=true`), `tab-bg`,
  `window`, `run`, `hover`, `userscript` and `spawn`.
 
+* +*-f*+, +*--first*+: Click the first hinted element without prompting.
 
 ==== note
 * This command does not split arguments after the last argument and handles quotes literally.
@@ -741,7 +757,13 @@ This tries to automatically click on typical _Previous Page_ or _Next Page_ link
  - `next`: Open a _next_ link.
  - `up`: Go up a level in the current URL.
  - `increment`: Increment the last number in the URL.
+ Uses the
+ link:settings.html#url.incdec_segments[url.incdec_segments]
+ config option.
  - `decrement`: Decrement the last number in the URL.
+ Uses the
+ link:settings.html#url.incdec_segments[url.incdec_segments]
+ config option.
  
 
 
@@ -1005,6 +1027,15 @@ Scroll the current tab by 'count * dx/dy' pixels.
 ==== count
 multiplier
 
+[[scroll-to-anchor]]
+=== scroll-to-anchor
+Syntax: +:scroll-to-anchor 'name'+
+
+Scroll to the given anchor in the document.
+
+==== positional arguments
+* +'name'+: The anchor to scroll to.
+
 [[scroll-to-perc]]
 === scroll-to-perc
 Syntax: +:scroll-to-perc [*--horizontal*] ['perc']+
@@ -1066,7 +1097,7 @@ Delete a session.
 
 [[session-load]]
 === session-load
-Syntax: +:session-load [*--clear*] [*--temp*] [*--force*] 'name'+
+Syntax: +:session-load [*--clear*] [*--temp*] [*--force*] [*--delete*] 'name'+
 
 Load a session.
 
@@ -1078,6 +1109,7 @@ Load a session.
 * +*-t*+, +*--temp*+: Don't set the current session for :session-save.
 * +*-f*+, +*--force*+: Force loading internal sessions (starting with an underline).
 
+* +*-d*+, +*--delete*+: Delete the saved session once it has loaded.
 
 [[session-save]]
 === session-save
@@ -1100,11 +1132,11 @@ Save a session.
 
 [[set]]
 === set
-Syntax: +:set [*--temp*] [*--print*] ['option'] ['value']+
+Syntax: +:set [*--temp*] [*--print*] [*--pattern* 'pattern'] ['option'] ['value']+
 
 Set an option.
 
-If the option name ends with '?', the value of the option is shown instead.
+If the option name ends with '?', the value of the option is shown instead. Using :set without any arguments opens a page where settings can be changed interactively.
 
 ==== positional arguments
 * +'option'+: The name of the option.
@@ -1113,6 +1145,7 @@ If the option name ends with '?', the value of the option is shown instead.
 ==== optional arguments
 * +*-t*+, +*--temp*+: Set value temporarily until qutebrowser is closed.
 * +*-p*+, +*--print*+: Print the value after setting.
+* +*-u*+, +*--pattern*+: The URL pattern to use.
 
 [[set-cmd-text]]
 === set-cmd-text
@@ -1202,7 +1235,7 @@ The tab index to close
 
 [[tab-focus]]
 === tab-focus
-Syntax: +:tab-focus ['index']+
+Syntax: +:tab-focus [*--no-last*] ['index']+
 
 Select the tab given as argument/[count].
 
@@ -1214,6 +1247,9 @@ If neither count nor index are given, it behaves like tab-next. If both are give
  last tab.
 
 
+==== optional arguments
+* +*-n*+, +*--no-last*+: Whether to avoid focusing last tab if already focused.
+
 ==== count
 The tab index to focus, starting with 1.
 
@@ -1228,6 +1264,9 @@ If no win_id is given, the tab will get detached into a new window.
 ==== positional arguments
 * +'win-id'+: The window ID of the window to give the current tab to.
 
+==== count
+Overrides win_id (index starts at 1 for win_id=0).
+
 [[tab-move]]
 === tab-move
 Syntax: +:tab-move ['index']+
@@ -1246,6 +1285,13 @@ If moving relatively: Offset. If moving absolutely: New position (default: 0). T
  overrides the index argument, if given.
 
 
+[[tab-mute]]
+=== tab-mute
+Mute/Unmute the current/[count]th tab.
+
+==== count
+The tab index to mute or unmute
+
 [[tab-next]]
 === tab-next
 Switch to the next tab, or switch [count] tabs forward.
@@ -1297,7 +1343,8 @@ Syntax: +:unbind [*--mode* 'mode'] 'key'+
 Unbind a keychain.
 
 ==== positional arguments
-* +'key'+: The keychain or special key (inside <...>) to unbind.
+* +'key'+: The keychain to unbind. See the help for `:bind` for the correct syntax for keychains.
+
 
 ==== optional arguments
 * +*-m*+, +*--mode*+: A mode to unbind the key in (default: `normal`). See `:help bindings.commands` for the available modes.
@@ -1309,12 +1356,26 @@ Re-open the last closed tab or tabs.
 
 [[version]]
 === version
+Syntax: +:version [*--paste*]+
+
 Show version information.
 
+==== optional arguments
+* +*-p*+, +*--paste*+: Paste to pastebin.
+
 [[view-source]]
 === view-source
+Syntax: +:view-source [*--edit*] [*--pygments*]+
+
 Show the source of the current page in a new tab.
 
+==== optional arguments
+* +*-e*+, +*--edit*+: Edit the source in the editor instead of opening a tab.
+* +*-p*+, +*--pygments*+: Use pygments to generate the view. This is always the case for QtWebKit. For QtWebEngine it may display
+ slightly different source.
+ Some JavaScript processing may be applied.
+
+
 [[window-only]]
 === window-only
 Close all windows except for the current one.
@@ -1402,6 +1463,7 @@ How many steps to zoom out.
 |<<prompt-accept,prompt-accept>>|Accept the current prompt.
 |<<prompt-item-focus,prompt-item-focus>>|Shift the focus of the prompt file completion menu to another item.
 |<<prompt-open-download,prompt-open-download>>|Immediately open a download.
+|<<prompt-yank,prompt-yank>>|Yank URL to clipboard or primary selection.
 |<<rl-backward-char,rl-backward-char>>|Move back a character.
 |<<rl-backward-delete-char,rl-backward-delete-char>>|Delete the character before the cursor.
 |<<rl-backward-kill-word,rl-backward-kill-word>>|Remove chars from the cursor to the beginning of the word.
@@ -1467,13 +1529,16 @@ Drop selection and keep selection mode enabled.
 
 [[follow-hint]]
 === follow-hint
-Syntax: +:follow-hint ['keystring']+
+Syntax: +:follow-hint [*--select*] ['keystring']+
 
 Follow a hint.
 
 ==== positional arguments
 * +'keystring'+: The hint to follow.
 
+==== optional arguments
+* +*-s*+, +*--select*+: Only select the given hint, don't necessarily follow it.
+
 [[leave-mode]]
 === leave-mode
 Leave the mode we're currently in.
@@ -1607,6 +1672,15 @@ If no specific command is given, this will use the system's default application
 ==== note
 * This command does not split arguments after the last argument and handles quotes literally.
 
+[[prompt-yank]]
+=== prompt-yank
+Syntax: +:prompt-yank [*--sel*]+
+
+Yank URL to clipboard or primary selection.
+
+==== optional arguments
+* +*-s*+, +*--sel*+: Use the primary selection instead of the clipboard.
+
 [[rl-backward-char]]
 === rl-backward-char
 Move back a character.

doc/help/configuring.asciidoc

@@ -3,44 +3,28 @@ Configuring qutebrowser
 
 IMPORTANT: qutebrowser's configuration system was completely rewritten in
 September 2017. This information is not applicable to older releases, and older
-information elsewhere might be outdated. **If you had an old configuration
-around and upgraded, this page will automatically open once**. To view it at a
-later time, use the `:help` command.
+information elsewhere might be outdated.
 
-Migrating older configurations
-------------------------------
+qutebrowser's config files
+--------------------------
 
-qutebrowser does no automatic migration for the new configuration. However,
-there's a special link:qute://configdiff/old[configdiff] page
-(`qute://configdiff/old`) in qutebrowser, which will show you the changes you
-did in your old configuration, compared to the old defaults.
+qutebrowser releases before v1.0.0 had a `qutebrowser.conf` and `keys.conf`
+file. Those are not used anymore since that release - see
+<<migrating,"Migrating older configurations">> for information on how to
+migrate to the new config.
 
-Other changes in default settings:
+When using `:set` and `:bind`, changes are saved to an `autoconfig.yml` file
+automatically. If you don't want to have a config file which is curated by
+hand, you can simply use those - see
+<<autoconfig,"Configuring qutebrowser via the user interface">> for details.
 
-- In v1.1.x and newer, `<Up>` and `<Down>` navigate through command history
-  if no text was entered yet.
-  With v1.0.x, they always navigate through command history instead of selecting
-  completion items. Use `<Tab>`/`<Shift-Tab>` to cycle through the completion
-  instead.
-  You can get back the old behavior by doing:
-+
-----
-:bind -m command <Up> completion-item-focus prev
-:bind -m command <Down> completion-item-focus next
-----
-+
-or always navigate through command history with
-+
-----
-:bind -m command <Up> command-history-prev
-:bind -m command <Down> command-history-next
-----
-
-- The default for `completion.web_history_max_items` is now set to `-1`, showing
-  an unlimited number of items in the completion for `:open` as the new
-  sqlite-based completion is much faster. If the `:open` completion is too slow
-  on your machine, set an appropriate limit again.
+For more advanced configuration, you can write a `config.py` file - see
+<<configpy,"Configuring qutebrowser via config.py">>. As soon as a `config.py`
+exists, the `autoconfig.yml` file **is not read anymore** by default. You need
+to <<configpy-autoconfig,load it by hand>> if you want settings done via
+`:set`/`:bind` to still persist.
 
+[[autoconfig]]
 Configuring qutebrowser via the user interface
 ----------------------------------------------
 
@@ -63,6 +47,10 @@ customizable.
 Using the link:commands.html#set[`:set`] command and command completion, you
 can quickly set settings interactively, for example `:set tabs.position left`.
 
+Some settings are also customizable for a given
+https://developer.chrome.com/apps/match_patterns[URL pattern] by doing e.g.
+`:set --pattern=*://example.com/ content.images false`.
+
 To get more help about a setting, use e.g. `:help tabs.position`.
 
 To bind and unbind keys, you can use the link:commands.html#bind[`:bind`] and
@@ -84,6 +72,7 @@ link:commands.html#config-clear[`:config-clear`] to reset the entire configurati
 and link:commands.html#config-cycle[`:config-cycle`] to cycle a setting between
 different values.
 
+[[configpy]]
 Configuring qutebrowser via config.py
 -------------------------------------
 
@@ -147,7 +136,6 @@ prefix to preserve backslashes) or a Python regex object:
 If you want to read a setting, you can use the `c` object to do so as well:
 `c.colors.tabs.even.bg = c.colors.tabs.odd.bg`.
 
-
 Using strings for setting names
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -171,6 +159,26 @@ To read a setting, use the `config.get` method:
 color = config.get('colors.completion.fg')
 ----
 
+Per-domain settings
+~~~~~~~~~~~~~~~~~~~
+
+Using `config.set`, some settings are also customizable for a given
+https://developer.chrome.com/apps/match_patterns[URL pattern]:
+
+[source,python]
+----
+config.set('content.images', False, '*://example.com/')
+----
+
+Alternatively, you can use `with config.pattern(...) as p:` to get a shortcut
+similar to `c.` which is scoped to the given domain:
+
+[source,python]
+----
+with config.pattern('*://example.com/') as p:
+    p.content.images = False
+----
+
 Binding keys
 ~~~~~~~~~~~~
 
@@ -216,13 +224,14 @@ config.bind(',v', 'spawn mpv {url}')
 To suppress loading of any default keybindings, you can set
 `c.bindings.default = {}`.
 
+[[configpy-autoconfig]]
 Loading `autoconfig.yml`
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
-By default, all customization done via `:set`, `:bind` and `:unbind` is
-temporary as soon as a `config.py` exists. The settings done that way are always
-saved in the `autoconfig.yml` file, but you'll need to explicitly load it in
-your `config.py` by doing:
+All customization done via the UI (`:set`, `:bind` and `:unbind`) is
+stored in the `autoconfig.yml` file, which is not loaded automatically as soon
+as a `config.py` exists. If you want those settings to be loaded, you'll need to
+explicitly load the `autoconfig.yml` file in your `config.py` by doing:
 
 .config.py:
 [source,python]
@@ -254,7 +263,7 @@ Getting the config directory
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 If you need to get the qutebrowser config directory, you can do so by reading
-`config.configdir`. Similarily, you can get the qutebrowser data directory via
+`config.configdir`. Similarly, you can get the qutebrowser data directory via
 `config.datadir`.
 
 This gives you a https://docs.python.org/3/library/pathlib.html[`pathlib.Path`
@@ -366,6 +375,8 @@ You can use something like this to read colors from an `~/.Xresources` file:
 
 [source,python]
 ----
+import subprocess
+
 def read_xresources(prefix):
     props = {}
     x = subprocess.run(['xrdb', '-query'], stdout=subprocess.PIPE)
@@ -376,9 +387,15 @@ def read_xresources(prefix):
     return props
 
 xresources = read_xresources('*')
-c.colors.statusbar.normal.bg = xresources['*background']
+c.colors.statusbar.normal.bg = xresources['*.background']
 ----
 
+Pre-built colorschemes
+^^^^^^^^^^^^^^^^^^^^^^
+
+- A collection of https://github.com/chriskempson/base16[base16] color-schemes can be found in https://github.com/theova/base16-qutebrowser[base16-qutebrowser] and used with https://github.com/AuditeMarlow/base16-manager[base16-manager].
+- Two implementations of the https://github.com/arcticicestudio/nord[Nord] colorscheme for qutebrowser exist: https://github.com/Linuus/nord-qutebrowser[Linuus], https://github.com/KnownAsDon/QuteBrowser-Nord-Theme[KnownAsDon]
+
 Avoiding flake8 errors
 ^^^^^^^^^^^^^^^^^^^^^^
 
@@ -404,3 +421,38 @@ from qutebrowser.config.config import ConfigContainer  # noqa: F401
 config = config  # type: ConfigAPI # noqa: F821 pylint: disable=E0602,C0103
 c = c  # type: ConfigContainer # noqa: F821 pylint: disable=E0602,C0103
 ----
+
+[[migrating]]
+Migrating older configurations
+------------------------------
+
+qutebrowser does no automatic migration for the new configuration. However,
+there's a special link:qute://configdiff/old[configdiff] page
+(`qute://configdiff/old`) in qutebrowser, which will show you the changes you
+did in your old configuration, compared to the old defaults.
+
+Other changes in default settings:
+
+- In v1.1.x and newer, `<Up>` and `<Down>` navigate through command history
+  if no text was entered yet.
+  With v1.0.x, they always navigate through command history instead of selecting
+  completion items. Use `<Tab>`/`<Shift-Tab>` to cycle through the completion
+  instead.
+  You can get back the old behavior by doing:
++
+----
+:bind -m command <Up> completion-item-focus prev
+:bind -m command <Down> completion-item-focus next
+----
++
+or always navigate through command history with
++
+----
+:bind -m command <Up> command-history-prev
+:bind -m command <Down> command-history-next
+----
+
+- The default for `completion.web_history_max_items` is now set to `-1`, showing
+  an unlimited number of items in the completion for `:open` as the new
+  sqlite-based completion is much faster. If the `:open` completion is too slow
+  on your machine, set an appropriate limit again.

doc/help/settings.asciidoc

@@ -1,6 +1,7 @@
 // DO NOT EDIT THIS FILE DIRECTLY!
 // It is autogenerated by running:
 //   $ python3 scripts/dev/src2asciidoc.py
+// vim: readonly:
 
 = Setting reference
 
@@ -108,13 +109,15 @@
 |<<completion.use_best_match,completion.use_best_match>>|Execute the best-matching command on a partial match.
 |<<completion.web_history_max_items,completion.web_history_max_items>>|Number of URLs to show in the web history.
 |<<confirm_quit,confirm_quit>>|Require a confirmation before quitting the application.
+|<<content.autoplay,content.autoplay>>|Automatically start playing `<video>` elements.
 |<<content.cache.appcache,content.cache.appcache>>|Enable support for the HTML 5 web application cache feature.
 |<<content.cache.maximum_pages,content.cache.maximum_pages>>|Maximum number of pages to hold in the global memory page cache.
 |<<content.cache.size,content.cache.size>>|Size (in bytes) of the HTTP network cache. Null to use the default value.
+|<<content.canvas_reading,content.canvas_reading>>|Allow websites to read canvas elements.
 |<<content.cookies.accept,content.cookies.accept>>|Which cookies to accept.
 |<<content.cookies.store,content.cookies.store>>|Store cookies.
 |<<content.default_encoding,content.default_encoding>>|Default encoding to use for websites.
-|<<content.developer_extras,content.developer_extras>>|Enable extra tools for Web developers.
+|<<content.desktop_capture,content.desktop_capture>>|Allow websites to share screen content.
 |<<content.dns_prefetch,content.dns_prefetch>>|Try to pre-fetch DNS entries to speed up browsing.
 |<<content.frame_flattening,content.frame_flattening>>|Expand each subframe to its contents.
 |<<content.geolocation,content.geolocation>>|Allow websites to request geolocations.
@@ -143,14 +146,17 @@
 |<<content.netrc_file,content.netrc_file>>|Netrc-file for HTTP authentication.
 |<<content.notifications,content.notifications>>|Allow websites to show notifications.
 |<<content.pdfjs,content.pdfjs>>|Allow pdf.js to view PDF files in the browser.
+|<<content.persistent_storage,content.persistent_storage>>|Allow websites to request persistent storage quota via `navigator.webkitPersistentStorage.requestQuota`.
 |<<content.plugins,content.plugins>>|Enable plugins in Web pages.
 |<<content.print_element_backgrounds,content.print_element_backgrounds>>|Draw the background color and images also when the page is printed.
 |<<content.private_browsing,content.private_browsing>>|Open new windows in private browsing mode which does not record visited pages.
 |<<content.proxy,content.proxy>>|Proxy to use.
 |<<content.proxy_dns_requests,content.proxy_dns_requests>>|Send DNS requests over the configured proxy.
+|<<content.register_protocol_handler,content.register_protocol_handler>>|Allow websites to register protocol handlers via `navigator.registerProtocolHandler`.
 |<<content.ssl_strict,content.ssl_strict>>|Validate SSL handshakes.
 |<<content.user_stylesheets,content.user_stylesheets>>|List of user stylesheet filenames to use.
 |<<content.webgl,content.webgl>>|Enable WebGL.
+|<<content.webrtc_public_interfaces_only,content.webrtc_public_interfaces_only>>|Only expose public interfaces via WebRTC.
 |<<content.windowed_fullscreen,content.windowed_fullscreen>>|Limit fullscreen to the browser window (does not expand to fill the screen).
 |<<content.xss_auditing,content.xss_auditing>>|Monitor load requests for cross-site scripting attempts.
 |<<downloads.location.directory,downloads.location.directory>>|Directory to save downloads to.
@@ -199,7 +205,9 @@
 |<<hints.scatter,hints.scatter>>|Scatter hint key chains (like Vimium) or not (like dwb).
 |<<hints.uppercase,hints.uppercase>>|Make characters in hint strings uppercase.
 |<<history_gap_interval,history_gap_interval>>|Maximum time (in minutes) between two history items for them to be considered being from the same browsing session.
+|<<input.escape_quits_reporter,input.escape_quits_reporter>>|Allow Escape to quit the crash reporter.
 |<<input.forward_unbound_keys,input.forward_unbound_keys>>|Which unbound keys to forward to the webview in normal mode.
+|<<input.insert_mode.auto_enter,input.insert_mode.auto_enter>>|Enter insert mode if an editable element is clicked.
 |<<input.insert_mode.auto_leave,input.insert_mode.auto_leave>>|Leave insert mode if a non-editable element is clicked.
 |<<input.insert_mode.auto_load,input.insert_mode.auto_load>>|Automatically enter insert mode if an editable element is focused after loading the page.
 |<<input.insert_mode.plugins,input.insert_mode.plugins>>|Switch to insert mode when clicking flash and other plugins.
@@ -229,19 +237,21 @@
 |<<statusbar.hide,statusbar.hide>>|Hide the statusbar unless a message is shown.
 |<<statusbar.padding,statusbar.padding>>|Padding (in pixels) for the statusbar.
 |<<statusbar.position,statusbar.position>>|Position of the status bar.
+|<<statusbar.widgets,statusbar.widgets>>|List of widgets displayed in the statusbar.
 |<<tabs.background,tabs.background>>|Open new tabs (middleclick/ctrl+click) in the background.
 |<<tabs.close_mouse_button,tabs.close_mouse_button>>|Mouse button with which to close tabs.
 |<<tabs.close_mouse_button_on_bar,tabs.close_mouse_button_on_bar>>|How to behave when the close mouse button is pressed on the tab bar.
 |<<tabs.favicons.scale,tabs.favicons.scale>>|Scaling factor for favicons in the tab bar.
-|<<tabs.favicons.show,tabs.favicons.show>>|Show favicons in the tab bar.
+|<<tabs.favicons.show,tabs.favicons.show>>|When to show favicons in the tab bar.
 |<<tabs.indicator.padding,tabs.indicator.padding>>|Padding (in pixels) for tab indicators.
 |<<tabs.indicator.width,tabs.indicator.width>>|Width (in pixels) of the progress indicator (0 to disable).
 |<<tabs.last_close,tabs.last_close>>|How to behave when the last tab is closed.
+|<<tabs.min_width,tabs.min_width>>|Minimum width (in pixels) of tabs (-1 for the default minimum size behavior).
+|<<tabs.mode_on_change,tabs.mode_on_change>>|When switching tabs, what input mode is applied.
 |<<tabs.mousewheel_switching,tabs.mousewheel_switching>>|Switch between tabs using the mouse wheel.
 |<<tabs.new_position.related,tabs.new_position.related>>|Position of new tabs opened from another tab.
 |<<tabs.new_position.unrelated,tabs.new_position.unrelated>>|Position of new tabs which aren't opened from another tab.
 |<<tabs.padding,tabs.padding>>|Padding (in pixels) around text for tabs.
-|<<tabs.persist_mode_on_change,tabs.persist_mode_on_change>>|Stay in insert/passthrough mode when switching tabs.
 |<<tabs.pinned.shrink,tabs.pinned.shrink>>|Shrink pinned tabs down to their contents.
 |<<tabs.position,tabs.position>>|Position of the tab bar.
 |<<tabs.select_on_remove,tabs.select_on_remove>>|Which tab to select when the focused tab is removed.
@@ -256,10 +266,11 @@
 |<<url.auto_search,url.auto_search>>|What search to start when something else than a URL is entered.
 |<<url.default_page,url.default_page>>|Page to open if :open -t/-b/-w is used without URL.
 |<<url.incdec_segments,url.incdec_segments>>|URL segments where `:navigate increment/decrement` will search for a number.
+|<<url.open_base_url,url.open_base_url>>|Open base URL of the searchengine if a searchengine shortcut is invoked without parameters.
 |<<url.searchengines,url.searchengines>>|Search engines which can be used via the address bar.
 |<<url.start_pages,url.start_pages>>|Page(s) to open at the start.
 |<<url.yank_ignored_parameters,url.yank_ignored_parameters>>|URL parameters to strip with `:yank url`.
-|<<window.hide_wayland_decoration,window.hide_wayland_decoration>>|Hide the window decoration when using wayland.
+|<<window.hide_decoration,window.hide_decoration>>|Hide the window decoration.
 |<<window.title_format,window.title_format>>|Format to use for the window title. The same placeholders like for
 |<<zoom.default,zoom.default>>|Default zoom level.
 |<<zoom.levels,zoom.levels>>|Available zoom levels.
@@ -320,7 +331,7 @@ While it's possible to add bindings with this setting, it's recommended to use `
 This setting is a dictionary containing mode names and dictionaries mapping keys to commands:
 `{mode: {key: command}}`
 If you want to map a key to another key, check the `bindings.key_mappings` setting instead.
-For special keys (can't be part of a keychain), enclose them in `<`...`>`. For modifiers, you can use either `-` or `+` as delimiters, and these names:
+For modifiers, you can use either `-` or `+` as delimiters, and these names:
 
   * Control: `Control`, `Ctrl`
 
@@ -356,11 +367,8 @@ The following modes are available:
 
 * prompt: Entered when there's a prompt to display, like for download
   locations or when invoked from JavaScript.
-  +
-  You can bind normal keys in this mode, but they will be only active when
-  a yes/no-prompt is asked. For other prompt modes, you can only bind
-  special keys.
 
+* yesno: Entered when there's a yes/no prompt displayed.
 * caret: Entered when pressing the `v` mode, used to select text using the
   keyboard.
 
@@ -377,6 +385,8 @@ Default keybindings. If you want to add bindings, modify `bindings.commands` ins
 The main purpose of this setting is that you can set it to an empty dictionary if you want to load no default keybindings at all.
 If you want to preserve default bindings (and get new bindings when there is an update), use `config.bind()` in `config.py` or the `:bind` command, and leave this setting alone.
 
+This setting can only be set in config.py.
+
 Type: <<types,Dict>>
 
 Default: 
@@ -483,6 +493,7 @@ Default:
 * +pass:[&lt;Alt-7&gt;]+: +pass:[tab-focus 7]+
 * +pass:[&lt;Alt-8&gt;]+: +pass:[tab-focus 8]+
 * +pass:[&lt;Alt-9&gt;]+: +pass:[tab-focus -1]+
+* +pass:[&lt;Alt-m&gt;]+: +pass:[tab-mute]+
 * +pass:[&lt;Ctrl-A&gt;]+: +pass:[navigate increment]+
 * +pass:[&lt;Ctrl-Alt-p&gt;]+: +pass:[print]+
 * +pass:[&lt;Ctrl-B&gt;]+: +pass:[scroll-page 0 -1]+
@@ -496,6 +507,7 @@ Default:
 * +pass:[&lt;Ctrl-Return&gt;]+: +pass:[follow-selected -t]+
 * +pass:[&lt;Ctrl-Shift-N&gt;]+: +pass:[open -p]+
 * +pass:[&lt;Ctrl-Shift-T&gt;]+: +pass:[undo]+
+* +pass:[&lt;Ctrl-Shift-Tab&gt;]+: +pass:[nop]+
 * +pass:[&lt;Ctrl-Shift-W&gt;]+: +pass:[close]+
 * +pass:[&lt;Ctrl-T&gt;]+: +pass:[open -t]+
 * +pass:[&lt;Ctrl-Tab&gt;]+: +pass:[tab-focus last]+
@@ -558,6 +570,7 @@ Default:
 * +pass:[gd]+: +pass:[download]+
 * +pass:[gf]+: +pass:[view-source]+
 * +pass:[gg]+: +pass:[scroll-to-perc 0]+
+* +pass:[gi]+: +pass:[hint inputs --first]+
 * +pass:[gl]+: +pass:[tab-move -]+
 * +pass:[gm]+: +pass:[tab-move]+
 * +pass:[go]+: +pass:[set-cmd-text :open {url:pretty}]+
@@ -580,8 +593,20 @@ Default:
 * +pass:[sk]+: +pass:[set-cmd-text -s :bind]+
 * +pass:[sl]+: +pass:[set-cmd-text -s :set -t]+
 * +pass:[ss]+: +pass:[set-cmd-text -s :set]+
+* +pass:[tPH]+: +pass:[config-cycle -p -u *://*.{url:host}/* content.plugins ;; reload]+
+* +pass:[tPh]+: +pass:[config-cycle -p -u *://{url:host}/* content.plugins ;; reload]+
+* +pass:[tPu]+: +pass:[config-cycle -p -u {url} content.plugins ;; reload]+
+* +pass:[tSH]+: +pass:[config-cycle -p -u *://*.{url:host}/* content.javascript.enabled ;; reload]+
+* +pass:[tSh]+: +pass:[config-cycle -p -u *://{url:host}/* content.javascript.enabled ;; reload]+
+* +pass:[tSu]+: +pass:[config-cycle -p -u {url} content.javascript.enabled ;; reload]+
 * +pass:[th]+: +pass:[back -t]+
 * +pass:[tl]+: +pass:[forward -t]+
+* +pass:[tpH]+: +pass:[config-cycle -p -t -u *://*.{url:host}/* content.plugins ;; reload]+
+* +pass:[tph]+: +pass:[config-cycle -p -t -u *://{url:host}/* content.plugins ;; reload]+
+* +pass:[tpu]+: +pass:[config-cycle -p -t -u {url} content.plugins ;; reload]+
+* +pass:[tsH]+: +pass:[config-cycle -p -t -u *://*.{url:host}/* content.javascript.enabled ;; reload]+
+* +pass:[tsh]+: +pass:[config-cycle -p -t -u *://{url:host}/* content.javascript.enabled ;; reload]+
+* +pass:[tsu]+: +pass:[config-cycle -p -t -u {url} content.javascript.enabled ;; reload]+
 * +pass:[u]+: +pass:[undo]+
 * +pass:[v]+: +pass:[enter-mode caret]+
 * +pass:[wB]+: +pass:[set-cmd-text -s :bookmark-load -w]+
@@ -615,6 +640,8 @@ Default:
 * +pass:[&lt;Alt-Backspace&gt;]+: +pass:[rl-backward-kill-word]+
 * +pass:[&lt;Alt-D&gt;]+: +pass:[rl-kill-word]+
 * +pass:[&lt;Alt-F&gt;]+: +pass:[rl-forward-word]+
+* +pass:[&lt;Alt-Shift-Y&gt;]+: +pass:[prompt-yank --sel]+
+* +pass:[&lt;Alt-Y&gt;]+: +pass:[prompt-yank]+
 * +pass:[&lt;Ctrl-?&gt;]+: +pass:[rl-delete-char]+
 * +pass:[&lt;Ctrl-A&gt;]+: +pass:[rl-beginning-of-line]+
 * +pass:[&lt;Ctrl-B&gt;]+: +pass:[rl-backward-char]+
@@ -632,11 +659,17 @@ Default:
 * +pass:[&lt;Shift-Tab&gt;]+: +pass:[prompt-item-focus prev]+
 * +pass:[&lt;Tab&gt;]+: +pass:[prompt-item-focus next]+
 * +pass:[&lt;Up&gt;]+: +pass:[prompt-item-focus prev]+
-* +pass:[n]+: +pass:[prompt-accept no]+
-* +pass:[y]+: +pass:[prompt-accept yes]+
 - +pass:[register]+:
 
 * +pass:[&lt;Escape&gt;]+: +pass:[leave-mode]+
+- +pass:[yesno]+:
+
+* +pass:[&lt;Alt-Shift-Y&gt;]+: +pass:[prompt-yank --sel]+
+* +pass:[&lt;Alt-Y&gt;]+: +pass:[prompt-yank]+
+* +pass:[&lt;Escape&gt;]+: +pass:[leave-mode]+
+* +pass:[&lt;Return&gt;]+: +pass:[prompt-accept]+
+* +pass:[n]+: +pass:[prompt-accept no]+
+* +pass:[y]+: +pass:[prompt-accept yes]+
 
 [[bindings.key_mappings]]
 === bindings.key_mappings
@@ -1438,11 +1471,26 @@ Default:
 
 - +pass:[never]+
 
+[[content.autoplay]]
+=== content.autoplay
+Automatically start playing `<video>` elements.
+Note this option needs a restart with QtWebEngine on Qt < 5.11.
+
+Type: <<types,Bool>>
+
+Default: +pass:[true]+
+
+On QtWebEngine, this setting requires Qt 5.10 or newer.
+
+On QtWebKit, this setting is unavailable.
+
 [[content.cache.appcache]]
 === content.cache.appcache
 Enable support for the HTML 5 web application cache feature.
 An application cache acts like an HTTP cache in some sense. For documents that use the application cache via JavaScript, the loader engine will first ask the application cache for the contents, before hitting the network.
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[true]+
@@ -1470,6 +1518,18 @@ Type: <<types,Int>>
 
 Default: empty
 
+[[content.canvas_reading]]
+=== content.canvas_reading
+Allow websites to read canvas elements.
+Note this is needed for some websites to work properly.
+This setting requires a restart.
+
+Type: <<types,Bool>>
+
+Default: +pass:[true]+
+
+This setting is only available with the QtWebEngine backend.
+
 [[content.cookies.accept]]
 === content.cookies.accept
 Which cookies to accept.
@@ -1480,12 +1540,12 @@ Valid values:
 
  * +all+: Accept all cookies.
  * +no-3rdparty+: Accept cookies from the same origin only.
- * +no-unknown-3rdparty+: Accept cookies from the same origin only, unless a cookie is already set for the domain.
+ * +no-unknown-3rdparty+: Accept cookies from the same origin only, unless a cookie is already set for the domain. On QtWebEngine, this is the same as no-3rdparty.
  * +never+: Don't accept cookies at all.
 
 Default: +pass:[no-3rdparty]+
 
-This setting is only available with the QtWebKit backend.
+On QtWebEngine, this setting requires Qt 5.11 or newer.
 
 [[content.cookies.store]]
 === content.cookies.store
@@ -1505,21 +1565,29 @@ Type: <<types,String>>
 
 Default: +pass:[iso-8859-1]+
 
-[[content.developer_extras]]
-=== content.developer_extras
-Enable extra tools for Web developers.
-This needs to be enabled for `:inspector` to work and also adds an _Inspect_ entry to the context menu. For QtWebEngine, see `--enable-webengine-inspector` in `qutebrowser --help` instead.
+[[content.desktop_capture]]
+=== content.desktop_capture
+Allow websites to share screen content.
+On Qt < 5.10, a dialog box is always displayed, even if this is set to "true".
 
-Type: <<types,Bool>>
+This setting supports URL patterns.
 
-Default: +pass:[false]+
+Type: <<types,BoolAsk>>
 
-This setting is only available with the QtWebKit backend.
+Valid values:
+
+ * +true+
+ * +false+
+ * +ask+
+
+Default: +pass:[ask]+
 
 [[content.dns_prefetch]]
 === content.dns_prefetch
 Try to pre-fetch DNS entries to speed up browsing.
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[true]+
@@ -1531,6 +1599,8 @@ This setting is only available with the QtWebKit backend.
 Expand each subframe to its contents.
 This will flatten all the frames to become one scrollable page.
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[false]+
@@ -1541,6 +1611,8 @@ This setting is only available with the QtWebKit backend.
 === content.geolocation
 Allow websites to request geolocations.
 
+This setting supports URL patterns.
+
 Type: <<types,BoolAsk>>
 
 Valid values:
@@ -1554,6 +1626,9 @@ Default: +pass:[ask]+
 [[content.headers.accept_language]]
 === content.headers.accept_language
 Value to send in the `Accept-Language` header.
+Note that the value read from JavaScript is always the global value.
+
+This setting supports URL patterns.
 
 Type: <<types,String>>
 
@@ -1563,6 +1638,8 @@ Default: +pass:[en-US,en]+
 === content.headers.custom
 Custom headers for qutebrowser HTTP requests.
 
+This setting supports URL patterns.
+
 Type: <<types,Dict>>
 
 Default: empty
@@ -1572,6 +1649,8 @@ Default: empty
 Value to send in the `DNT` header.
 When this is set to true, qutebrowser asks websites to not track your identity. If set to null, the DNT header is not sent at all.
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[true]+
@@ -1579,7 +1658,7 @@ Default: +pass:[true]+
 [[content.headers.referer]]
 === content.headers.referer
 When to send the Referer header.
-The Referer header tells websites from which website you were coming from when visting them.
+The Referer header tells websites from which website you were coming from when visiting them.
 
 Type: <<types,String>>
 
@@ -1596,6 +1675,9 @@ This setting is only available with the QtWebKit backend.
 [[content.headers.user_agent]]
 === content.headers.user_agent
 User agent to send. Unset to send the default.
+Note that the value read from JavaScript is always the global value.
+
+This setting supports URL patterns.
 
 Type: <<types,String>>
 
@@ -1625,11 +1707,7 @@ Type: <<types,List of Url>>
 
 Default: 
 
-- +pass:[https://www.malwaredomainlist.com/hostslist/hosts.txt]+
-- +pass:[http://someonewhocares.org/hosts/hosts]+
-- +pass:[http://winhelp2002.mvps.org/hosts.zip]+
-- +pass:[http://malwaredomains.lehigh.edu/files/justdomains.zip]+
-- +pass:[https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&amp;mimetype=plaintext]+
+- +pass:[https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts]+
 
 [[content.host_blocking.whitelist]]
 === content.host_blocking.whitelist
@@ -1647,6 +1725,8 @@ Default:
 === content.hyperlink_auditing
 Enable hyperlink auditing (`<a ping>`).
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[false]+
@@ -1655,6 +1735,8 @@ Default: +pass:[false]+
 === content.images
 Load images automatically in web pages.
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[true]+
@@ -1672,6 +1754,8 @@ Default: +pass:[true]+
 Allow JavaScript to read from or write to the clipboard.
 With QtWebEngine, writing the clipboard as response to a user interaction is always allowed.
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[false]+
@@ -1680,6 +1764,8 @@ Default: +pass:[false]+
 === content.javascript.can_close_tabs
 Allow JavaScript to close tabs.
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[false]+
@@ -1690,6 +1776,8 @@ This setting is only available with the QtWebKit backend.
 === content.javascript.can_open_tabs_automatically
 Allow JavaScript to open new tabs without user interaction.
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[false]+
@@ -1698,6 +1786,8 @@ Default: +pass:[false]+
 === content.javascript.enabled
 Enable JavaScript.
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[true]+
@@ -1737,6 +1827,8 @@ Default: +pass:[true]+
 === content.local_content_can_access_file_urls
 Allow locally loaded documents to access other local URLs.
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[true]+
@@ -1745,6 +1837,8 @@ Default: +pass:[true]+
 === content.local_content_can_access_remote_urls
 Allow locally loaded documents to access remote URLs.
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[false]+
@@ -1753,6 +1847,8 @@ Default: +pass:[false]+
 === content.local_storage
 Enable support for HTML 5 local storage and Web SQL.
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[true]+
@@ -1761,6 +1857,8 @@ Default: +pass:[true]+
 === content.media_capture
 Allow websites to record audio/video.
 
+This setting supports URL patterns.
+
 Type: <<types,BoolAsk>>
 
 Valid values:
@@ -1786,6 +1884,8 @@ Default: empty
 === content.notifications
 Allow websites to show notifications.
 
+This setting supports URL patterns.
+
 Type: <<types,BoolAsk>>
 
 Valid values:
@@ -1809,10 +1909,32 @@ Default: +pass:[false]+
 
 This setting is only available with the QtWebKit backend.
 
+[[content.persistent_storage]]
+=== content.persistent_storage
+Allow websites to request persistent storage quota via `navigator.webkitPersistentStorage.requestQuota`.
+
+This setting supports URL patterns.
+
+Type: <<types,BoolAsk>>
+
+Valid values:
+
+ * +true+
+ * +false+
+ * +ask+
+
+Default: +pass:[ask]+
+
+On QtWebEngine, this setting requires Qt 5.11 or newer.
+
+On QtWebKit, this setting is unavailable.
+
 [[content.plugins]]
 === content.plugins
 Enable plugins in Web pages.
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[false]+
@@ -1821,6 +1943,8 @@ Default: +pass:[false]+
 === content.print_element_backgrounds
 Draw the background color and images also when the page is printed.
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[true]+
@@ -1859,10 +1983,32 @@ Default: +pass:[true]+
 
 This setting is only available with the QtWebKit backend.
 
+[[content.register_protocol_handler]]
+=== content.register_protocol_handler
+Allow websites to register protocol handlers via `navigator.registerProtocolHandler`.
+
+This setting supports URL patterns.
+
+Type: <<types,BoolAsk>>
+
+Valid values:
+
+ * +true+
+ * +false+
+ * +ask+
+
+Default: +pass:[ask]+
+
+On QtWebEngine, this setting requires Qt 5.11 or newer.
+
+On QtWebKit, this setting is unavailable.
+
 [[content.ssl_strict]]
 === content.ssl_strict
 Validate SSL handshakes.
 
+This setting supports URL patterns.
+
 Type: <<types,BoolAsk>>
 
 Valid values:
@@ -1885,10 +2031,25 @@ Default: empty
 === content.webgl
 Enable WebGL.
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[true]+
 
+[[content.webrtc_public_interfaces_only]]
+=== content.webrtc_public_interfaces_only
+Only expose public interfaces via WebRTC.
+On Qt 5.9, this option requires a restart. On Qt 5.10, this option doesn't work at all because of a Qt bug. On Qt >= 5.11, no restart is required.
+
+Type: <<types,Bool>>
+
+Default: +pass:[false]+
+
+On QtWebEngine, this setting requires Qt 5.9.2 or newer.
+
+On QtWebKit, this setting is unavailable.
+
 [[content.windowed_fullscreen]]
 === content.windowed_fullscreen
 Limit fullscreen to the browser window (does not expand to fill the screen).
@@ -1902,6 +2063,8 @@ Default: +pass:[false]+
 Monitor load requests for cross-site scripting attempts.
 Suspicious scripts will be blocked and reported in the inspector's JavaScript console. Enabling this feature might have an impact on performance.
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[false]+
@@ -2333,6 +2496,14 @@ Type: <<types,Int>>
 
 Default: +pass:[30]+
 
+[[input.escape_quits_reporter]]
+=== input.escape_quits_reporter
+Allow Escape to quit the crash reporter.
+
+Type: <<types,Bool>>
+
+Default: +pass:[true]+
+
 [[input.forward_unbound_keys]]
 === input.forward_unbound_keys
 Which unbound keys to forward to the webview in normal mode.
@@ -2347,6 +2518,14 @@ Valid values:
 
 Default: +pass:[auto]+
 
+[[input.insert_mode.auto_enter]]
+=== input.insert_mode.auto_enter
+Enter insert mode if an editable element is clicked.
+
+Type: <<types,Bool>>
+
+Default: +pass:[true]+
+
 [[input.insert_mode.auto_leave]]
 === input.insert_mode.auto_leave
 Leave insert mode if a non-editable element is clicked.
@@ -2375,6 +2554,8 @@ Default: +pass:[false]+
 === input.links_included_in_focus_chain
 Include hyperlinks in the keyboard focus chain when tabbing.
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[true]+
@@ -2402,6 +2583,8 @@ Default: +pass:[false]+
 Enable spatial navigation.
 Spatial navigation consists in the ability to navigate between focusable elements in a Web page, such as hyperlinks and form controls, by using Left, Right, Up and Down arrow keys. For example, if the user presses the Right key, heuristics determine whether there is an element he might be trying to reach towards the right and which element he probably wants.
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[false]+
@@ -2513,12 +2696,19 @@ Default: empty
 [[qt.force_software_rendering]]
 === qt.force_software_rendering
 Force software rendering for QtWebEngine.
-This is needed for QtWebEngine to work with Nouveau drivers.
+This is needed for QtWebEngine to work with Nouveau drivers and can be useful in other scenarios related to graphic issues.
 This setting requires a restart.
 
-Type: <<types,Bool>>
+Type: <<types,String>>
 
-Default: +pass:[false]+
+Valid values:
+
+ * +software-opengl+: Tell LibGL to use a software implementation of GL (`LIBGL_ALWAYS_SOFTWARE` / `QT_XCB_FORCE_SOFTWARE_OPENGL`)
+ * +qt-quick+: Tell Qt Quick to use a software renderer instead of OpenGL. (`QT_QUICK_BACKEND=software`)
+ * +chromium+: Tell Chromium to disable GPU support and use Skia software rendering instead. (`--disable-gpu`)
+ * +none+: Don't force software rendering.
+
+Default: +pass:[none]+
 
 This setting is only available with the QtWebEngine backend.
 
@@ -2546,6 +2736,8 @@ Default: +pass:[false]+
 Enable smooth scrolling for web pages.
 Note smooth scrolling does not work with the `:scroll-px` command.
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[false]+
@@ -2682,6 +2874,31 @@ Valid values:
 
 Default: +pass:[bottom]+
 
+[[statusbar.widgets]]
+=== statusbar.widgets
+List of widgets displayed in the statusbar.
+
+Type: <<types,List of String>>
+
+Valid values:
+
+ * +url+: Current page URL.
+ * +scroll+: Percentage of the current page position like `10%`.
+ * +scroll_raw+: Raw percentage of the current page position like `10`.
+ * +history+: Display an arrow when possible to go back/forward in history.
+ * +tabs+: Current active tab, e.g. `2`.
+ * +keypress+: Display pressed keys when composing a vi command.
+ * +progress+: Progress bar for the current page loading.
+
+Default: 
+
+- +pass:[keypress]+
+- +pass:[url]+
+- +pass:[scroll]+
+- +pass:[history]+
+- +pass:[tabs]+
+- +pass:[progress]+
+
 [[tabs.background]]
 === tabs.background
 Open new tabs (middleclick/ctrl+click) in the background.
@@ -2730,11 +2947,17 @@ Default: +pass:[1.0]+
 
 [[tabs.favicons.show]]
 === tabs.favicons.show
-Show favicons in the tab bar.
+When to show favicons in the tab bar.
 
-Type: <<types,Bool>>
+Type: <<types,String>>
 
-Default: +pass:[true]+
+Valid values:
+
+ * +always+: Always show favicons.
+ * +never+: Always hide favicons.
+ * +pinned+: Show favicons only on pinned tabs.
+
+Default: +pass:[always]+
 
 [[tabs.indicator.padding]]
 === tabs.indicator.padding
@@ -2773,6 +2996,30 @@ Valid values:
 
 Default: +pass:[ignore]+
 
+[[tabs.min_width]]
+=== tabs.min_width
+Minimum width (in pixels) of tabs (-1 for the default minimum size behavior).
+This setting only applies when tabs are horizontal.
+This setting does not apply to pinned tabs, unless `tabs.pinned.shrink` is False.
+
+Type: <<types,Int>>
+
+Default: +pass:[-1]+
+
+[[tabs.mode_on_change]]
+=== tabs.mode_on_change
+When switching tabs, what input mode is applied.
+
+Type: <<types,String>>
+
+Valid values:
+
+ * +persist+: Retain the current mode.
+ * +restore+: Restore previously saved mode.
+ * +normal+: Always revert to normal mode.
+
+Default: +pass:[normal]+
+
 [[tabs.mousewheel_switching]]
 === tabs.mousewheel_switching
 Switch between tabs using the mouse wheel.
@@ -2824,14 +3071,6 @@ Default:
 - +pass:[right]+: +pass:[5]+
 - +pass:[top]+: +pass:[0]+
 
-[[tabs.persist_mode_on_change]]
-=== tabs.persist_mode_on_change
-Stay in insert/passthrough mode when switching tabs.
-
-Type: <<types,Bool>>
-
-Default: +pass:[false]+
-
 [[tabs.pinned.shrink]]
 === tabs.pinned.shrink
 Shrink pinned tabs down to their contents.
@@ -2931,11 +3170,12 @@ The following placeholders are defined:
 * `{private}`: Indicates when private mode is enabled.
 * `{current_url}`: URL of the current web page.
 * `{protocol}`: Protocol (http/https/...) of the current web page.
+* `{audio}`: Indicator for audio/mute status.
 
 
 Type: <<types,FormatString>>
 
-Default: +pass:[{index}: {title}]+
+Default: +pass:[{audio}{index}: {title}]+
 
 [[tabs.title.format_pinned]]
 === tabs.title.format_pinned
@@ -2993,6 +3233,7 @@ Type: <<types,FlagList>>
 Valid values:
 
  * +host+
+ * +port+
  * +path+
  * +query+
  * +anchor+
@@ -3002,6 +3243,14 @@ Default:
 - +pass:[path]+
 - +pass:[query]+
 
+[[url.open_base_url]]
+=== url.open_base_url
+Open base URL of the searchengine if a searchengine shortcut is invoked without parameters.
+
+Type: <<types,Bool>>
+
+Default: +pass:[false]+
+
 [[url.searchengines]]
 === url.searchengines
 Search engines which can be used via the address bar.
@@ -3037,10 +3286,12 @@ Default:
 - +pass:[utm_term]+
 - +pass:[utm_content]+
 
-[[window.hide_wayland_decoration]]
-=== window.hide_wayland_decoration
-Hide the window decoration when using wayland.
-This setting requires a restart.
+[[window.hide_decoration]]
+=== window.hide_decoration
+Hide the window decoration.
+
+This setting requires a restart on Wayland.
+
 
 Type: <<types,Bool>>
 
@@ -3101,6 +3352,8 @@ Default: +pass:[512]+
 === zoom.text_only
 Apply the zoom factor on a frame only to the text or to all content.
 
+This setting supports URL patterns.
+
 Type: <<types,Bool>>
 
 Default: +pass:[false]+
@@ -3171,7 +3424,7 @@ See the setting's valid values for more information on allowed values.
 |TextAlignment|Alignment of text.
 |TimestampTemplate|An strftime-like template for timestamps.
 
-See https://docs.python.org/3/library/datetime.html#strftime-strptime-behavior for reference.
+See https://sqlite.org/lang_datefunc.html for reference.
 |UniqueCharString|A string which may not contain duplicate chars.
 |Url|A URL as a string.
 |VerticalPosition|The position of the download bar.

doc/install.asciidoc

@@ -35,25 +35,39 @@ Ubuntu 16.04 doesn't come with an up-to-date engine (a new enough QtWebKit, or
 QtWebEngine). However, it comes with Python 3.5, so you can
 <<tox,install qutebrowser via tox>>.
 
+You'll need some basic libraries to use the tox-installed PyQt:
+
+----
+# apt install libglib2.0-0 libgl1 libfontconfig1 libx11-xcb1 libxi6 libxrender1 libdbus-1-3
+----
+
 Debian Stretch / Ubuntu 17.04 and 17.10
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Those versions come with QtWebEngine in the repositories. This makes it possible
 to install qutebrowser via the Debian package.
 
-Get the qutebrowser package from the
-https://github.com/qutebrowser/qutebrowser/releases[release page] and download
-the https://qutebrowser.org/python3-pypeg2_2.15.2-1_all.deb[PyPEG2 package].
+You'll need to download three packages:
 
-(If you are using debian testing you can just use the python3-pypeg2 package from the repos)
+- https://packages.debian.org/sid/all/python3-pypeg2/download[PyPEG2] (a library
+  used by qutebrowser which is not in the earlier repositories)
+- https://packages.debian.org/sid/all/qutebrowser/download[qutebrowser] itself
+- Either https://packages.debian.org/sid/all/qutebrowser-qtwebengine/download[qutebrowser-qtwebengine]
+  or https://packages.debian.org/sid/all/qutebrowser-qtwebkit/download[qutebrowser-qtwebkit]
+  (or both) depending on the backend you want to use. QtWebEngine is the
+  default/recommended choice.
 
-Install the packages:
+After downloading, install the packages (make sure to install all the
+downloaded qutebrowser deb files in one apt command):
 
 ----
 # apt install ./python3-pypeg2_*_all.deb
-# apt install ./qutebrowser_*_all.deb
+# apt install ./qutebrowser*.deb
 ----
 
+For an update after the initial install, you only need to download/install the
+qutebrowser package.
+
 Debian Testing / Ubuntu 18.04
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -277,6 +291,11 @@ PS C:\> Install-Package qutebrowser
 ----
 C:\> choco install qutebrowser
 ----
+* Scoop's client
+----
+C:\> scoop bucket add extras
+C:\> scoop install qutebrowser
+----
 
 Manual install
 ~~~~~~~~~~~~~~
@@ -363,8 +382,8 @@ $ git clone https://github.com/qutebrowser/qutebrowser.git
 $ cd qutebrowser
 ----
 
-Installing depdendencies (including Qt)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Installing dependencies (including Qt)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Then run tox inside the qutebrowser repository to set up a
 https://docs.python.org/3/library/venv.html[virtual environment]:
@@ -373,6 +392,10 @@ https://docs.python.org/3/library/venv.html[virtual environment]:
 $ tox -e mkvenv-pypi
 ----
 
+If your system comes with Python 3.5.3 or older (such as Ubuntu 16.04 LTS), use
+`tox -e mkvenv-pypi-old` instead. This installs an older Qt version (5.10) due
+to bugs in newer versions.
+
 This installs all needed Python dependencies in a `.venv` subfolder.
 
 This comes with an up-to-date Qt/PyQt including QtWebEngine, but has a few
@@ -408,7 +431,11 @@ Creating a wrapper script
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Running `tox` does not install a system-wide `qutebrowser` script. You can
-launch qutebrowser by doing `.venv/bin/python3 -m qutebrowser`.
+launch qutebrowser by doing:
+
+----
+.venv/bin/python3 -m qutebrowser
+----
 
 You can create a simple wrapper script to start qutebrowser somewhere in your
 `$PATH` (e.g. `/usr/local/bin/qutebrowser` or `~/bin/qutebrowser`):

doc/quickstart.asciidoc

@@ -22,9 +22,9 @@ Basic keybindings to get you started
 What to do now
 --------------
 
-* View the link:https://qutebrowser.org/img/cheatsheet-big.png[key binding cheatsheet]
+* View the link:https://raw.githubusercontent.com/qutebrowser/qutebrowser/master/doc/img/cheatsheet-big.png[key binding cheatsheet]
 to make yourself familiar with the key bindings: +
-image:https://qutebrowser.org/img/cheatsheet-small.png["qutebrowser key binding cheatsheet",link="https://qutebrowser.org/img/cheatsheet-big.png"]
+image:https://raw.githubusercontent.com/qutebrowser/qutebrowser/master/doc/img/cheatsheet-small.png["qutebrowser key binding cheatsheet",link="https://raw.githubusercontent.com/qutebrowser/qutebrowser/master/doc/img/cheatsheet-big.png"]
 * There's also a https://www.shortcutfoo.com/app/dojos/qutebrowser[free training
   course] on shortcutfoo for the keybindings - note that you need to be in
   insert mode (i) for it to work.

doc/qutebrowser.1.asciidoc

@@ -38,7 +38,7 @@ show it.
 *-h*, *--help*::
     show this help message and exit
 
-*--basedir* 'BASEDIR'::
+*-B* 'BASEDIR', *--basedir* 'BASEDIR'::
     Base directory for all storage.
 
 *-V*, *--version*::
@@ -60,7 +60,7 @@ show it.
     Which backend to use.
 
 *--enable-webengine-inspector*::
-    Enable the web inspector for QtWebEngine. Note that this is a SECURITY RISK and you should not visit untrusted websites with the inspector turned on. See https://bugreports.qt.io/browse/QTBUG-50725 for more details.
+    Enable the web inspector for QtWebEngine. Note that this is a SECURITY RISK and you should not visit untrusted websites with the inspector turned on. See https://bugreports.qt.io/browse/QTBUG-50725 for more details. This is not needed anymore since Qt 5.11 where the inspector is always enabled and secure.
 
 === debug arguments
 *-l* '{critical,error,warning,info,debug,vdebug}', *--loglevel* '{critical,error,warning,info,debug,vdebug}'::
@@ -72,7 +72,7 @@ show it.
 *--loglines* 'LOGLINES'::
     How many lines of the debug log to keep in RAM (-1: unlimited).
 
-*--debug*::
+*-d*, *--debug*::
     Turn on debugging options.
 
 *--json-logging*::
@@ -87,7 +87,7 @@ show it.
 *--nowindow*::
     Don't show the main window.
 
-*--temp-basedir*::
+*-T*, *--temp-basedir*::
     Use a temporary basedir.
 
 *--no-err-windows*::
@@ -99,7 +99,7 @@ show it.
 *--qt-flag* 'QT_FLAG'::
     Pass an argument to Qt as flag.
 
-*--debug-flag* 'DEBUG_FLAGS'::
+*-D* 'DEBUG_FLAGS', *--debug-flag* 'DEBUG_FLAGS'::
     Pass name of debugging feature to be turned on.
 // QUTE_OPTIONS_END
 

doc/stacktrace.asciidoc

@@ -37,7 +37,7 @@ is available in the repositories:
 Archlinux
 ^^^^^^^^^
 
-For Archlinux, no debug informations are provided. You can either compile Qt
+For Archlinux, no debug information is provided. You can either compile Qt
 yourself (which will take a few hours even on a modern machine) or use
 debugging symbols compiled/packaged by me (x86_64 only).
 

doc/userscripts.asciidoc

@@ -45,8 +45,6 @@ In `command` mode:
 - `QUTE_URL`: The current URL.
 - `QUTE_TITLE`: The title of the current page.
 - `QUTE_SELECTED_TEXT`: The text currently selected on the page.
-- `QUTE_SELECTED_HTML` The HTML currently selected on the page (not supported
-  with QtWebEngine).
 
 In `hints` mode:
 

misc/Makefile

@@ -1,25 +1,33 @@
 PYTHON = python3
-DESTDIR = /
+PREFIX = /usr/local
+DESTDIR =
 ICONSIZES = 16 24 32 48 64 128 256 512
 
+SETUPTOOLSOPTIONS =
+ifdef DESTDIR
+SETUPTOOLSOPTS = --root="$(DESTDIR)"
+endif
+
 .PHONY: install
 
 doc/qutebrowser.1.html:
 	a2x -f manpage doc/qutebrowser.1.asciidoc
 
 install: doc/qutebrowser.1.html
-	$(PYTHON) setup.py install --root="$(DESTDIR)" --optimize=1
+	$(PYTHON) setup.py install --prefix="$(PREFIX)" --optimize=1 $(SETUPTOOLSOPTS)
+	install -Dm644 misc/qutebrowser.appdata.xml \
+		"$(DESTDIR)$(PREFIX)/share/metainfo/qutebrowser.appdata.xml"
 	install -Dm644 doc/qutebrowser.1 \
-		"$(DESTDIR)/usr/share/man/man1/qutebrowser.1"
+		"$(DESTDIR)$(PREFIX)/share/man/man1/qutebrowser.1"
 	install -Dm644 misc/qutebrowser.desktop \
-		"$(DESTDIR)/usr/share/applications/qutebrowser.desktop"
+		"$(DESTDIR)$(PREFIX)/share/applications/qutebrowser.desktop"
 	$(foreach i,$(ICONSIZES),install -Dm644 "icons/qutebrowser-$(i)x$(i).png" \
-			"$(DESTDIR)/usr/share/icons/hicolor/$(i)x$(i)/apps/qutebrowser.png";)
+			"$(DESTDIR)$(PREFIX)/share/icons/hicolor/$(i)x$(i)/apps/qutebrowser.png";)
 	install -Dm644 icons/qutebrowser.svg \
-		"$(DESTDIR)/usr/share/icons/hicolor/scalable/apps/qutebrowser.svg"
-	install -Dm755 -t "$(DESTDIR)/usr/share/qutebrowser/userscripts/" \
+		"$(DESTDIR)$(PREFIX)/share/icons/hicolor/scalable/apps/qutebrowser.svg"
+	install -Dm755 -t "$(DESTDIR)$(PREFIX)/share/qutebrowser/userscripts/" \
 		$(wildcard misc/userscripts/*)
-	install -Dm755 -t "$(DESTDIR)/usr/share/qutebrowser/scripts/" \
+	install -Dm755 -t "$(DESTDIR)$(PREFIX)/share/qutebrowser/scripts/" \
 		$(filter-out scripts/__init__.py scripts/__pycache__ scripts/dev \
-		  scripts/testbrowser_cpp scripts/asciidoc2html.py scripts/setupcommon.py \
+		  scripts/testbrowser scripts/asciidoc2html.py scripts/setupcommon.py \
 			scripts/link_pyqt.py,$(wildcard scripts/*))

misc/cheatsheet.svg

@@ -32,22 +32,24 @@
      objecttolerance="10"
      inkscape:pageopacity="0.0"
      inkscape:pageshadow="2"
-     inkscape:zoom="1.24"
-     inkscape:cx="305.29152"
-     inkscape:cy="465.48793"
+     inkscape:zoom="1.7536248"
+     inkscape:cx="430.72917"
+     inkscape:cy="268.64059"
      inkscape:document-units="px"
      inkscape:current-layer="layer1"
      width="1024px"
      height="640px"
      showgrid="false"
-     inkscape:window-width="1024"
-     inkscape:window-height="723"
+     inkscape:window-width="2560"
+     inkscape:window-height="1440"
      inkscape:window-x="0"
      inkscape:window-y="0"
      showguides="true"
      inkscape:guide-bbox="true"
-     inkscape:window-maximized="1"
-     inkscape:snap-text-baseline="true">
+     inkscape:window-maximized="0"
+     inkscape:snap-text-baseline="true"
+     inkscape:measure-start="0,0"
+     inkscape:measure-end="0,0">
     <inkscape:grid
        id="GridFromPre046Settings"
        type="xygrid"
@@ -2688,7 +2690,8 @@
          id="flowPara5711"> </flowPara></flowRoot>    <flowRoot
        xml:space="preserve"
        id="flowRoot5691-0"
-       style="font-style:normal;font-weight:normal;font-size:12.80000019px;line-height:0.01%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1.06666672"><flowRegion
+       style="font-style:normal;font-weight:normal;font-size:12.80000019px;line-height:0.01%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1.06666672"
+       transform="translate(0,-10)"><flowRegion
          id="flowRegion5693-7"
          style="font-family:sans-serif;stroke-width:1.06666672"><rect
            id="rect5695-0"
@@ -3660,5 +3663,64 @@
          sodipodi:role="line"
          style="font-size:8.53333378px;line-height:0.89999998;stroke-width:1.06666672"
          id="tspan6220">items</tspan></text>
-  </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:12.80000019px;line-height:0%;font-family:TlwgTypewriter;text-align:start;writing-mode:lr-tb;text-anchor:start;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1.06666672"
+       x="417.29486"
+       y="205.18887"
+       id="text7245-1-3"><tspan
+         sodipodi:role="line"
+         x="417.29486"
+         y="205.18887"
+         id="tspan7366-3-6"
+         style="font-size:9.60000038px;line-height:0.89999998;stroke-width:1.06666672"> </tspan><tspan
+         sodipodi:role="line"
+         x="417.29486"
+         y="213.07179"
+         id="tspan5293-53"
+         style="font-size:8.53333378px;line-height:0.89999998;stroke-width:1.06666672">toggle</tspan><tspan
+         sodipodi:role="line"
+         x="417.29486"
+         y="220.75179"
+         style="font-size:8.53333378px;line-height:0.89999998;stroke-width:1.06666672;fill:#ff0000"
+         id="tspan6091">(12)</tspan><tspan
+         sodipodi:role="line"
+         x="417.29486"
+         y="225.70012"
+         style="font-size:8.53333378px;line-height:0.89999998;stroke-width:1.06666672"
+         id="tspan6087" /><tspan
+         sodipodi:role="line"
+         x="417.29486"
+         y="225.70012"
+         style="font-size:8.53333378px;line-height:0.89999998;stroke-width:1.06666672"
+         id="tspan6089" /></text>
+    <flowRoot
+       transform="translate(-1.2953814,90.2721)"
+       xml:space="preserve"
+       id="flowRoot5691-0-5"
+       style="font-style:normal;font-weight:normal;font-size:12.80000019px;line-height:0.01%;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:1.06666672"><flowRegion
+         id="flowRegion5693-7-6"
+         style="font-family:sans-serif;stroke-width:1.06666672"><rect
+           id="rect5695-0-2"
+           width="344"
+           height="173.33333"
+           x="19.42783"
+           y="520.07886"
+           style="font-family:sans-serif;fill:#000000;stroke-width:1.13777781" /></flowRegion><flowPara
+         style="font-weight:bold;font-size:10.66666698px;line-height:1.25;font-family:sans-serif;-inkscape-font-specification:'Sans Bold';fill:#000000;stroke-width:1.06666672"
+         id="flowPara5701-9-2"><flowSpan
+   style="font-weight:bold;font-family:sans-serif;-inkscape-font-specification:'Sans Bold';fill:#ff0000;stroke-width:1.06666672"
+   id="flowSpan5705-5-1">(12)</flowSpan>  toggling settings:</flowPara><flowPara
+         style="font-size:10.66666698px;line-height:1.25;font-family:sans-serif;fill:#000000;stroke-width:1.06666672"
+         id="flowPara6196">tsh - toggle scripts for the current host (temporarily)</flowPara><flowPara
+         style="font-size:10.66666698px;line-height:1.25;font-family:sans-serif;fill:#000000;stroke-width:1.06666672"
+         id="flowPara6200">tSh - like <flowSpan
+   style="font-style:italic"
+   id="flowSpan6202">tsh</flowSpan>, but permanently</flowPara><flowPara
+         style="font-size:10.66666698px;line-height:1.25;font-family:sans-serif;fill:#000000;stroke-width:1.06666672"
+         id="flowPara6206">tsH/tsu - like <flowSpan
+   style="font-style:italic"
+   id="flowSpan6210">tsh</flowSpan>, but including subdomains / with exact URL</flowPara><flowPara
+         style="font-size:10.66666698px;line-height:1.25;font-family:sans-serif;fill:#000000;stroke-width:1.06666672"
+         id="flowPara6208">tph - toggle plugins</flowPara></flowRoot>  </g>
 </svg>

misc/qutebrowser.appdata.xml

@@ -9,9 +9,9 @@
 	<description>
 		<p>
 			qutebrowser is a keyboard-focused browser with a minimal GUI.
-			It was inspired by other browsers/addons like dwb and Vimperator/Pentadactyl, 
+			It was inspired by other browsers/addons like dwb and Vimperator/Pentadactyl,
 			and is based on Python and PyQt5.
-		</p>	
+		</p>
 	</description>
 	<categories>
 		<category>Network</category>
@@ -40,4 +40,9 @@
 	<url type="help">https://qutebrowser.org/doc/help/</url>
 	<url type="bugtracker">https://github.com/qutebrowser/qutebrowser/issues/</url>
 	<url type="donation">https://github.com/qutebrowser/qutebrowser#donating</url>
+	<releases>
+		<release version="1.3.0" date="2018-05-04"/>
+		<release version="1.2.1" date="2018-03-14"/>
+		<release version="1.2.0" date="2018-03-09"/>
+	</releases>
 </component>

misc/qutebrowser.desktop

@@ -1,6 +1,7 @@
 [Desktop Entry]
 Name=qutebrowser
 GenericName=Web Browser
+Comment=A keyboard-driven, vim-like browser based on PyQt5
 Icon=qutebrowser
 Type=Application
 Categories=Network;WebBrowser;

misc/qutebrowser.spec

@@ -15,7 +15,7 @@ def get_data_files():
         ('../qutebrowser/img', 'img'),
         ('../qutebrowser/javascript', 'javascript'),
         ('../qutebrowser/html/doc', 'html/doc'),
-        ('../qutebrowser/git-commit-id', ''),
+        ('../qutebrowser/git-commit-id', '.'),
         ('../qutebrowser/config/configdata.yml', 'config'),
     ]
 
@@ -58,14 +58,15 @@ exe = EXE(pyz,
           icon=icon,
           debug=False,
           strip=False,
-          upx=True,
-          console=False )
+          upx=False,
+          console=False,
+          version='misc/file_version_info.txt')
 coll = COLLECT(exe,
                a.binaries,
                a.zipfiles,
                a.datas,
                strip=False,
-               upx=True,
+               upx=False,
                name='qutebrowser')
 
 app = BUNDLE(coll,

misc/requirements/requirements-check-manifest.txt

@@ -1,3 +1,3 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-check-manifest==0.36
+check-manifest==0.37

misc/requirements/requirements-codecov.txt

@@ -1,9 +1,9 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-certifi==2017.11.5
+certifi==2018.4.16
 chardet==3.0.4
-codecov==2.0.13
-coverage==4.4.2
-idna==2.6
-requests==2.18.4
+codecov==2.0.15
+coverage==4.5.1
+idna==2.7
+requests==2.19.1
 urllib3==1.22

misc/requirements/requirements-flake8.txt

@@ -1,25 +1,27 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-attrs==17.4.0
+attrs==18.1.0
 flake8==3.5.0
-flake8-bugbear==17.12.0
-flake8-builtins==1.0.post0
+flake8-bugbear==18.2.0
+flake8-builtins==1.4.1  # rq.filter: != 1.4.0
 flake8-comprehensions==1.4.1
 flake8-copyright==0.2.0
-flake8-debugger==3.0.0
+flake8-debugger==3.1.0
 flake8-deprecated==1.3
 flake8-docstrings==1.3.0
 flake8-future-import==0.4.4
 flake8-mock==0.3
-flake8-per-file-ignores==0.4
+flake8-per-file-ignores==0.6
 flake8-polyfill==1.0.2
 flake8-string-format==0.2.3
 flake8-tidy-imports==1.1.0
 flake8-tuple==0.2.13
 mccabe==0.6.1
-pep8-naming==0.5.0
-pycodestyle==2.3.1
+pathmatch==0.2.1
+pep8-naming==0.7.0
+pycodestyle==2.3.1  # rq.filter: < 2.4.0
 pydocstyle==2.1.1
-pyflakes==1.6.0
+pyflakes==2.0.0
 six==1.11.0
 snowballstemmer==1.2.1
+typing==3.6.4

misc/requirements/requirements-flake8.txt-raw

@@ -1,6 +1,6 @@
 flake8
 flake8-bugbear
-flake8-builtins
+flake8-builtins!=1.4.0
 flake8-comprehensions
 flake8-copyright
 flake8-debugger
@@ -15,3 +15,9 @@ flake8-tuple
 pep8-naming
 pydocstyle
 pyflakes
+
+# https://github.com/PyCQA/pycodestyle/issues/741
+#@ filter: pycodestyle < 2.4.0
+
+# https://github.com/gforcada/flake8-builtins/issues/36
+#@ filter: flake8-builtins != 1.4.0

misc/requirements/requirements-pip.txt

@@ -1,8 +1,8 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
 appdirs==1.4.3
-packaging==16.8
+packaging==17.1
 pyparsing==2.2.0
-setuptools==38.4.0
+setuptools==40.0.0
 six==1.11.0
-wheel==0.30.0
+wheel==0.31.1

misc/requirements/requirements-pyinstaller.txt

@@ -4,4 +4,4 @@ altgraph==0.15
 future==0.16.0
 macholib==1.9
 pefile==2017.11.5
--e git+https://github.com/pyinstaller/pyinstaller.git@develop#egg=PyInstaller
+PyInstaller==3.3.1

misc/requirements/requirements-pyinstaller.txt-raw

@@ -1,4 +1 @@
--e git+https://github.com/pyinstaller/pyinstaller.git@develop#egg=PyInstaller
-
-# remove @commit-id for scm installs
-#@ replace: @.*# @develop#
+PyInstaller

misc/requirements/requirements-pylint-master.txt

@@ -1,18 +1,18 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
 -e git+https://github.com/PyCQA/astroid.git#egg=astroid
-certifi==2017.11.5
+certifi==2018.4.16
 chardet==3.0.4
-github3.py==0.9.6
-idna==2.6
-isort==4.2.15
+github3.py==1.1.0
+idna==2.7
+isort==4.3.4
 lazy-object-proxy==1.3.1
 mccabe==0.6.1
 -e git+https://github.com/PyCQA/pylint.git#egg=pylint
+python-dateutil==2.7.3
 ./scripts/dev/pylint_checkers
-requests==2.18.4
+requests==2.19.1
 six==1.11.0
 uritemplate==3.0.0
-uritemplate.py==3.0.2
 urllib3==1.22
 wrapt==1.10.11

misc/requirements/requirements-pylint.txt

@@ -1,18 +1,18 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-astroid==1.6.0
-certifi==2017.11.5
+astroid==1.6.5
+certifi==2018.4.16
 chardet==3.0.4
-github3.py==0.9.6
-idna==2.6
-isort==4.2.15
+github3.py==1.1.0
+idna==2.7
+isort==4.3.4
 lazy-object-proxy==1.3.1
 mccabe==0.6.1
-pylint==1.8.1
+pylint==1.9.2
+python-dateutil==2.7.3
 ./scripts/dev/pylint_checkers
-requests==2.18.4
+requests==2.19.1
 six==1.11.0
 uritemplate==3.0.0
-uritemplate.py==3.0.2
 urllib3==1.22
 wrapt==1.10.11

misc/requirements/requirements-pyqt.txt

@@ -1,4 +1,4 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-PyQt5==5.9.2
-sip==4.19.6
+PyQt5==5.11.2
+PyQt5-sip==4.19.11

misc/requirements/requirements-pyroma.txt

@@ -1,4 +1,4 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
 docutils==0.14
-pyroma==2.3
+pyroma==2.3.1

misc/requirements/requirements-tests-git.txt

@@ -35,8 +35,4 @@ git+https://github.com/pallets/markupsafe.git
 hg+http://bitbucket.org/birkenfeld/pygments-main
 hg+https://bitbucket.org/fdik/pypeg
 git+https://github.com/python-attrs/attrs.git
-
-# Fails to build:
-# gcc: error: ext/_yaml.c: No such file or directory
-# hg+https://bitbucket.org/xi/pyyaml
-PyYAML==3.12
+git+https://github.com/yaml/pyyaml.git

misc/requirements/requirements-tests.txt

@@ -1,39 +1,40 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-attrs==17.4.0
+attrs==18.1.0
 beautifulsoup4==4.6.0
-cheroot==6.0.0
+cheroot==6.3.2.post0
 click==6.7
 # colorama==0.3.9
-coverage==4.4.2
+coverage==4.5.1
 EasyProcess==0.2.3
 fields==5.0.0
-Flask==0.12.2
+Flask==1.0.2
 glob2==0.6
 hunter==2.0.2
-hypothesis==3.44.16
+hypothesis==3.66.1
 itsdangerous==0.24
 # Jinja2==2.10
 Mako==1.0.7
 # MarkupSafe==1.0
-parse==1.8.2
+more-itertools==4.2.0
+parse==1.8.4
 parse-type==0.4.2
 pluggy==0.6.0
-py==1.5.2
-py-cpuinfo==3.3.0
-pytest==3.3.1  # rq.filter: != 3.3.2
-pytest-bdd==2.19.0
+py==1.5.4
+py-cpuinfo==4.0.0
+pytest==3.6.3
+pytest-bdd==2.21.0
 pytest-benchmark==3.1.1
 pytest-cov==2.5.1
-pytest-faulthandler==1.3.1
-pytest-instafail==0.3.0
-pytest-mock==1.6.3
-pytest-qt==2.3.1
+pytest-faulthandler==1.5.0
+pytest-instafail==0.4.0
+pytest-mock==1.10.0
+pytest-qt==2.4.1
 pytest-repeat==0.4.1
-pytest-rerunfailures==4.0
+pytest-rerunfailures==4.1
 pytest-travis-fold==1.3.0
-pytest-xvfb==1.0.0
+pytest-xvfb==1.1.0
 PyVirtualDisplay==0.2.1
 six==1.11.0
-vulture==0.26
+vulture==0.28
 Werkzeug==0.14.1

misc/requirements/requirements-tests.txt-raw

@@ -4,7 +4,7 @@ coverage
 Flask
 hunter
 hypothesis
-pytest==3.3.1
+pytest
 pytest-bdd
 pytest-benchmark
 pytest-cov
@@ -19,4 +19,3 @@ pytest-xvfb
 vulture
 
 #@ ignore: Jinja2, MarkupSafe, colorama
-#@ filter: pytest != 3.3.2

misc/requirements/requirements-tox.txt

@@ -1,7 +1,7 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
 pluggy==0.6.0
-py==1.5.2
+py==1.5.4
 six==1.11.0
-tox==2.9.1
-virtualenv==15.1.0
+tox==3.1.1
+virtualenv==16.0.0

misc/requirements/requirements-tox.txt-raw

@@ -1,4 +1 @@
 tox
-
-# The latest tox release still depends on pluggy < 0.4...
-pluggy==0.4.0

misc/requirements/requirements-vulture.txt

@@ -1,3 +1,3 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-vulture==0.26
+vulture==0.28

misc/userscripts/getbib

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""Qutebrowser userscript scraping the current web page for DOIs and downloading
+corresponding bibtex information.
+
+Set the environment variable 'QUTE_BIB_FILEPATH' to indicate the path to
+download to. Otherwise, bibtex information is downloaded to '/tmp' and hence
+deleted at reboot.
+
+Installation: see qute://help/userscripts.html
+
+Inspired by
+https://ocefpaf.github.io/python4oceanographers/blog/2014/05/19/doi2bibtex/
+"""
+
+import os
+import sys
+import shutil
+import re
+from collections import Counter
+from urllib import parse as url_parse
+from urllib import request as url_request
+
+
+FIFO_PATH = os.getenv("QUTE_FIFO")
+
+def message_fifo(message, level="warning"):
+    """Send message to qutebrowser FIFO. The level must be one of 'info',
+    'warning' (default) or 'error'."""
+    with open(FIFO_PATH, "w") as fifo:
+        fifo.write("message-{} '{}'".format(level, message))
+
+
+source = os.getenv("QUTE_TEXT")
+with open(source) as f:
+    text = f.read()
+
+# find DOIs on page using regex
+dval = re.compile(r'(10\.(\d)+/([^(\s\>\"\<)])+)')
+# https://stackoverflow.com/a/10324802/3865876, too strict
+# dval = re.compile(r'\b(10[.][0-9]{4,}(?:[.][0-9]+)*/(?:(?!["&\'<>])\S)+)\b')
+dois = dval.findall(text)
+dois = Counter(e[0] for e in dois)
+try:
+    doi = dois.most_common(1)[0][0]
+except IndexError:
+    message_fifo("No DOIs found on page")
+    sys.exit()
+message_fifo("Found {} DOIs on page, selecting {}".format(len(dois), doi),
+             level="info")
+
+# get bibtex data corresponding to DOI
+url = "http://dx.doi.org/" + url_parse.quote(doi)
+headers = dict(Accept='text/bibliography; style=bibtex')
+request = url_request.Request(url, headers=headers)
+response = url_request.urlopen(request)
+status_code = response.getcode()
+if status_code >= 400:
+    message_fifo("Request returned {}".format(status_code))
+    sys.exit()
+
+# obtain content and format it
+bibtex = response.read().decode("utf-8").strip()
+bibtex = bibtex.replace(" ", "\n    ", 1).\
+    replace("}, ", "},\n    ").replace("}}", "}\n}")
+
+# append to file
+bib_filepath = os.getenv("QUTE_BIB_FILEPATH", "/tmp/qute.bib")
+with open(bib_filepath, "a") as f:
+    f.write(bibtex + "\n\n")

misc/userscripts/open_download

@@ -52,7 +52,7 @@ die() {
 if ! [ -d "$DOWNLOAD_DIR" ] ; then
     die "Download directory »$DOWNLOAD_DIR« not found!"
 fi
-if ! which "${ROFI_CMD}" > /dev/null ; then
+if ! command -v "${ROFI_CMD}" > /dev/null ; then
     die "Rofi command »${ROFI_CMD}« not found in PATH!"
 fi
 

misc/userscripts/password_fill

@@ -220,7 +220,7 @@ user_pattern='^(user|username|login): '
 GPG_OPTS=( "--quiet" "--yes" "--compress-algo=none" "--no-encrypt-to" )
 GPG="gpg"
 export GPG_TTY="${GPG_TTY:-$(tty 2>/dev/null)}"
-which gpg2 &>/dev/null && GPG="gpg2"
+command -v gpg2 &>/dev/null && GPG="gpg2"
 [[ -n $GPG_AGENT_INFO || $GPG == "gpg2" ]] && GPG_OPTS+=( "--batch" "--use-agent" )
 
 pass_backend() {

misc/userscripts/qute-keepass

@@ -0,0 +1,261 @@
+#!/usr/bin/env python3
+
+# Copyright 2018 Jay Kamat <jaygkamat@gmail.com>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""This userscript allows for insertion of usernames and passwords from keepass
+databases using pykeepass. Since it is a userscript, it must be run from
+qutebrowser.
+
+A sample invocation of this script is:
+
+:spawn --userscript qute-keepass -p ~/KeePassFiles/MainDatabase.kdbx
+
+And a sample binding
+
+:bind --mode=insert <ctrl-i> spawn --userscript qute-keepass -p ~/KeePassFiles/MainDatabase.kdbx
+
+-p or --path is a required argument.
+
+--keyfile-path allows you to specify a keepass keyfile. If you only use a
+keyfile, also add --no-password as well. Specifying --no-password without
+--keyfile-path will lead to an error.
+
+login information is inserted using :insert-text and :fake-key <Tab>, which
+means you must have a cursor in position before initiating this userscript. If
+you do not do this, you will get 'element not editable' errors.
+
+If keepass takes a while to open the DB, you might want to consider reducing
+the number of transform rounds in your database settings.
+
+Dependencies: pykeepass (in python3), PyQt5. Without pykeepass, you will get an
+exit code of 100.
+
+********************!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!******************
+
+WARNING: The login details are viewable as plaintext in qutebrowser's debug log
+(qute://log) and could be compromised if you decide to submit a crash report!
+
+********************!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!******************
+
+"""
+
+# pylint: disable=bad-builtin
+
+import argparse
+import enum
+import functools
+import os
+import shlex
+import subprocess
+import sys
+
+from PyQt5.QtCore import QUrl
+from PyQt5.QtWidgets import QApplication, QInputDialog, QLineEdit
+
+try:
+    import pykeepass
+except ImportError as e:
+    print("pykeepass not found: {}".format(str(e)), file=sys.stderr)
+
+    # Since this is a common error, try to print it to the FIFO if we can.
+    if 'QUTE_FIFO' in os.environ:
+        with open(os.environ['QUTE_FIFO'], 'w') as fifo:
+            fifo.write('message-error "pykeepass failed to be imported."\n')
+            fifo.flush()
+    sys.exit(100)
+
+argument_parser = argparse.ArgumentParser(
+    description="Fill passwords using keepass.",
+    formatter_class=argparse.RawDescriptionHelpFormatter,
+    epilog=__doc__)
+argument_parser.add_argument('url', nargs='?', default=os.getenv('QUTE_URL'))
+argument_parser.add_argument('--path', '-p', required=True,
+                             help='Path to the keepass db.')
+argument_parser.add_argument('--keyfile-path', '-k', default=None,
+                             help='Path to a keepass keyfile')
+argument_parser.add_argument(
+    '--no-password', action='store_true',
+    help='Supply if no password is required to unlock this database. '
+    'Only allowed with --keyfile-path')
+argument_parser.add_argument(
+    '--dmenu-invocation', '-d', default='dmenu',
+    help='Invocation used to execute a dmenu-provider')
+argument_parser.add_argument(
+    '--dmenu-format', '-f', default='{title}: {username}',
+    help='Format string for keys to display in dmenu.'
+    ' Must generate a unique string.')
+argument_parser.add_argument(
+    '--no-insert-mode', '-n', dest='insert_mode', action='store_false',
+    help="Don't automatically enter insert mode")
+argument_parser.add_argument(
+    '--io-encoding', '-i', default='UTF-8',
+    help='Encoding used to communicate with subprocesses')
+group = argument_parser.add_mutually_exclusive_group()
+group.add_argument('--username-fill-only', '-e',
+                   action='store_true', help='Only insert username')
+group.add_argument('--password-fill-only', '-w',
+                   action='store_true', help='Only insert password')
+
+CMD_DELAY = 50
+
+
+class ExitCodes(enum.IntEnum):
+    """Stores various exit codes groups to use."""
+    SUCCESS = 0
+    FAILURE = 1
+    # 1 is automatically used if Python throws an exception
+    NO_CANDIDATES = 2
+    USER_QUIT = 3
+    DB_OPEN_FAIL = 4
+
+    INTERNAL_ERROR = 10
+
+
+def qute_command(command):
+    with open(os.environ['QUTE_FIFO'], 'w') as fifo:
+        fifo.write(command + '\n')
+        fifo.flush()
+
+
+def stderr(to_print):
+    """Extra functionality to echo out errors to qb ui."""
+    print(to_print, file=sys.stderr)
+    qute_command('message-error "{}"'.format(to_print))
+
+
+def dmenu(items, invocation, encoding):
+    """Runs dmenu with given arguments."""
+    command = shlex.split(invocation)
+    process = subprocess.run(command, input='\n'.join(items).encode(encoding),
+                             stdout=subprocess.PIPE)
+    return process.stdout.decode(encoding).strip()
+
+
+def get_password():
+    """Get a keepass db password from user."""
+    _app = QApplication(sys.argv)
+    text, ok = QInputDialog.getText(
+        None, "KeePass DB Password",
+        "Please enter your KeePass Master Password",
+        QLineEdit.Password)
+    if not ok:
+        stderr('Password Prompt Rejected.')
+        sys.exit(ExitCodes.USER_QUIT)
+    return text
+
+
+def find_candidates(args, host):
+    """Finds candidates that match host"""
+    file_path = os.path.expanduser(args.path)
+
+    # TODO find a way to keep the db open, so we don't open (and query
+    # password) it every time
+
+    pw = None
+    if not args.no_password:
+        pw = get_password()
+
+    kf = args.keyfile_path
+    if kf:
+        kf = os.path.expanduser(kf)
+
+    try:
+        kp = pykeepass.PyKeePass(file_path, password=pw, keyfile=kf)
+    except Exception as e:
+        stderr("There was an error opening the DB: {}".format(str(e)))
+
+    return kp.find_entries(url="{}{}{}".format(".*", host, ".*"), regex=True)
+
+
+def candidate_to_str(args, candidate):
+    """Turns candidate into a human readable string for dmenu"""
+    return args.dmenu_format.format(title=candidate.title,
+                                    url=candidate.url,
+                                    username=candidate.username,
+                                    path=candidate.path,
+                                    uuid=candidate.uuid)
+
+
+def candidate_to_secret(candidate):
+    """Turns candidate into a generic (user, password) tuple"""
+    return (candidate.username, candidate.password)
+
+
+def run(args):
+    """Runs qute-keepass"""
+    if not args.url:
+        argument_parser.print_help()
+        return ExitCodes.FAILURE
+
+    url_host = QUrl(args.url).host()
+
+    if not url_host:
+        stderr('{} was not parsed as a valid URL!'.format(args.url))
+        return ExitCodes.INTERNAL_ERROR
+
+    # Find candidates matching the host of the given URL
+    candidates = find_candidates(args, url_host)
+    if not candidates:
+        stderr('No candidates for URL {!r} found!'.format(args.url))
+        return ExitCodes.NO_CANDIDATES
+
+    # Create a map so we can get turn the resulting string from dmenu back into
+    # a candidate
+    candidates_strs = list(map(functools.partial(candidate_to_str, args),
+                               candidates))
+    candidates_map = dict(zip(candidates_strs, candidates))
+
+    if len(candidates) == 1:
+        selection = candidates.pop()
+    else:
+        selection = dmenu(candidates_strs,
+                          args.dmenu_invocation,
+                          args.io_encoding)
+
+        if selection not in candidates_map:
+            stderr("'{}' was not a valid entry!").format(selection)
+            return ExitCodes.USER_QUIT
+
+        selection = candidates_map[selection]
+
+    username, password = candidate_to_secret(selection)
+
+    insert_mode = ';; enter-mode insert' if args.insert_mode else ''
+    if args.username_fill_only:
+        qute_command('insert-text {}{}'.format(username, insert_mode))
+    elif args.password_fill_only:
+        qute_command('insert-text {}{}'.format(password, insert_mode))
+    else:
+        # Enter username and password using insert-key and fake-key <Tab>
+        # (which supports more passwords than fake-key only), then switch back
+        # into insert-mode, so the form can be directly submitted by hitting
+        # enter afterwards. It dosen't matter when we go into insert mode, but
+        # the other commands need to be be executed sequentially, so we add
+        # delays with later.
+        qute_command('insert-text {} ;;'
+                     'later {} fake-key <Tab> ;;'
+                     'later {} insert-text {}{}'
+                     .format(username, CMD_DELAY,
+                             CMD_DELAY * 2, password, insert_mode))
+
+    return ExitCodes.SUCCESS
+
+
+if __name__ == '__main__':
+    arguments = argument_parser.parse_args()
+    sys.exit(run(arguments))

misc/userscripts/qute-lastpass

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+
+# Copyright 2017 Chris Braun (cryzed) <cryzed@googlemail.com>
+# Adapted for LastPass by Wayne Cheng (welps) <waynethecheng@gmail.com>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published bjy
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Insert login information using lastpass CLI and a dmenu-compatible application (e.g. dmenu, rofi -dmenu, ...). 
+A short demonstration can be seen here: https://i.imgur.com/zA61NrF.gifv.
+"""
+
+USAGE = """The domain of the site has to be in the name of the LastPass entry, for example: "github.com/cryzed" or
+"websites/github.com".  The login information is inserted by emulating key events using qutebrowser's fake-key command in this manner:
+[USERNAME]<Tab>[PASSWORD], which is compatible with almost all login forms.
+
+You must log into LastPass CLI using `lpass login <email>` prior to use of this script. The LastPass CLI agent only holds your master password for an hour by default. If you wish to change this, please see `man lpass`.
+
+To use in qutebrowser, run: `spawn --userscript qute-lastpass`
+"""
+
+EPILOG = """Dependencies: tldextract (Python 3 module), LastPass CLI (1.3 or newer)
+
+WARNING: The login details are viewable as plaintext in qutebrowser's debug log (qute://log) and might be shared if
+you decide to submit a crash report!"""
+
+import argparse
+import enum
+import fnmatch
+import functools
+import os
+import re
+import shlex
+import subprocess
+import sys
+import json
+import tldextract
+
+argument_parser = argparse.ArgumentParser(
+    description=__doc__, usage=USAGE, epilog=EPILOG)
+argument_parser.add_argument('url', nargs='?', default=os.getenv('QUTE_URL'))
+argument_parser.add_argument('--dmenu-invocation', '-d', default='rofi -dmenu',
+                             help='Invocation used to execute a dmenu-provider')
+argument_parser.add_argument('--no-insert-mode', '-n', dest='insert_mode', action='store_false',
+                             help="Don't automatically enter insert mode")
+argument_parser.add_argument('--io-encoding', '-i', default='UTF-8',
+                             help='Encoding used to communicate with subprocesses')
+argument_parser.add_argument('--merge-candidates', '-m', action='store_true',
+                             help='Merge pass candidates for fully-qualified and registered domain name')
+group = argument_parser.add_mutually_exclusive_group()
+group.add_argument('--username-only', '-e',
+                   action='store_true', help='Only insert username')
+group.add_argument('--password-only', '-w',
+                   action='store_true', help='Only insert password')
+
+stderr = functools.partial(print, file=sys.stderr)
+
+class ExitCodes(enum.IntEnum):
+    SUCCESS = 0
+    FAILURE = 1
+    # 1 is automatically used if Python throws an exception
+    NO_PASS_CANDIDATES = 2
+    COULD_NOT_MATCH_USERNAME = 3
+    COULD_NOT_MATCH_PASSWORD = 4
+
+def qute_command(command):
+    with open(os.environ['QUTE_FIFO'], 'w') as fifo:
+        fifo.write(command + '\n')
+        fifo.flush()
+
+def pass_(domain, encoding):
+    args = ['lpass', 'show', '-x', '-j', '-G', '.*{:s}.*'.format(domain)]
+    process = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    err = process.stderr.decode(encoding).strip()
+    if err:
+        msg = "LastPass CLI returned for {:s} - {:s}".format(domain, err)
+        stderr(msg)
+        return '[]'
+
+    out = process.stdout.decode(encoding).strip()
+
+    return out
+
+def dmenu(items, invocation, encoding):
+    command = shlex.split(invocation)
+    process = subprocess.run(command, input='\n'.join(
+        items).encode(encoding), stdout=subprocess.PIPE)
+    return process.stdout.decode(encoding).strip()
+
+
+def fake_key_raw(text):
+    for character in text:
+        # Escape all characters by default, space requires special handling
+        sequence = '" "' if character == ' ' else '\{}'.format(character)
+        qute_command('fake-key {}'.format(sequence))
+
+
+def main(arguments):
+    if not arguments.url:
+        argument_parser.print_help()
+        return ExitCodes.FAILURE
+
+    extract_result = tldextract.extract(arguments.url)
+
+    # Try to find candidates using targets in the following order: fully-qualified domain name (includes subdomains),
+    # the registered domain name and finally: the IPv4 address if that's what
+    # the URL represents
+    candidates = []
+    for target in filter(None, [extract_result.fqdn, extract_result.registered_domain, extract_result.subdomain + extract_result.domain, extract_result.domain, extract_result.ipv4]):
+        target_candidates = json.loads(pass_(target, arguments.io_encoding))
+        if not target_candidates:
+            continue
+
+        candidates = candidates + target_candidates
+        if not arguments.merge_candidates:
+            break
+    else:
+        if not candidates:
+            stderr('No pass candidates for URL {!r} found!'.format(
+                arguments.url))
+            return ExitCodes.NO_PASS_CANDIDATES
+
+    if len(candidates) == 1:
+        selection = candidates.pop()
+    else:
+        choices = ["{:s} | {:s} | {:s} | {:s}".format(c["id"], c["name"], c["url"], c["username"]) for c in candidates]
+        choice = dmenu(choices, arguments.dmenu_invocation, arguments.io_encoding)
+        choiceId = choice.split("|")[0].strip()
+        selection = next((c for (i, c) in enumerate(candidates) if c["id"] == choiceId), None)
+
+    # Nothing was selected, simply return
+    if not selection:
+        return ExitCodes.SUCCESS
+
+    username = selection["username"]
+    password = selection["password"]
+
+    if arguments.username_only:
+        fake_key_raw(username)
+    elif arguments.password_only:
+        fake_key_raw(password)
+    else:
+        # Enter username and password using fake-key and <Tab> (which seems to work almost universally), then switch
+        # back into insert-mode, so the form can be directly submitted by
+        # hitting enter afterwards
+        fake_key_raw(username)
+        qute_command('fake-key <Tab>')
+        fake_key_raw(password)
+
+    if arguments.insert_mode:
+        qute_command('enter-mode insert')
+
+    return ExitCodes.SUCCESS
+
+
+if __name__ == '__main__':
+    arguments = argument_parser.parse_args()
+    sys.exit(main(arguments))

misc/userscripts/qute-pass

@@ -25,9 +25,17 @@ demonstration can be seen here: https://i.imgur.com/KN3XuZP.gif.
 USAGE = """The domain of the site has to appear as a segment in the pass path, for example: "github.com/cryzed" or
 "websites/github.com". How the username and password are determined is freely configurable using the CLI arguments. The
 login information is inserted by emulating key events using qutebrowser's fake-key command in this manner:
-[USERNAME]<Tab>[PASSWORD], which is compatible with almost all login forms."""
+[USERNAME]<Tab>[PASSWORD], which is compatible with almost all login forms.
 
-EPILOG = """Dependencies: tldextract (Python 3 module), pass.
+Suggested bindings similar to Uzbl's `formfiller` script:
+
+    config.bind('<z><l>', 'spawn --userscript qute-pass')
+    config.bind('<z><u><l>', 'spawn --userscript qute-pass --username-only')
+    config.bind('<z><p><l>', 'spawn --userscript qute-pass --password-only')
+    config.bind('<z><o><l>', 'spawn --userscript qute-pass --otp-only')
+"""
+
+EPILOG = """Dependencies: tldextract (Python 3 module), pass, pass-otp (optional).
 For issues and feedback please use: https://github.com/cryzed/qutebrowser-userscripts.
 
 WARNING: The login details are viewable as plaintext in qutebrowser's debug log (qute://log) and might be shared if
@@ -66,6 +74,7 @@ argument_parser.add_argument('--merge-candidates', '-m', action='store_true',
 group = argument_parser.add_mutually_exclusive_group()
 group.add_argument('--username-only', '-e', action='store_true', help='Only insert username')
 group.add_argument('--password-only', '-w', action='store_true', help='Only insert password')
+group.add_argument('--otp-only', '-o', action='store_true', help='Only insert OTP code')
 
 stderr = functools.partial(print, file=sys.stderr)
 
@@ -98,17 +107,32 @@ def find_pass_candidates(domain, password_store_path):
     return candidates
 
 
-def pass_(path, encoding):
-    process = subprocess.run(['pass', path], stdout=subprocess.PIPE)
+def _run_pass(command, encoding):
+    process = subprocess.run(command, stdout=subprocess.PIPE)
     return process.stdout.decode(encoding).strip()
 
 
+def pass_(path, encoding):
+    return _run_pass(['pass', path], encoding)
+
+
+def pass_otp(path, encoding):
+    return _run_pass(['pass', 'otp', path], encoding)
+
+
 def dmenu(items, invocation, encoding):
     command = shlex.split(invocation)
     process = subprocess.run(command, input='\n'.join(items).encode(encoding), stdout=subprocess.PIPE)
     return process.stdout.decode(encoding).strip()
 
 
+def fake_key_raw(text):
+    for character in text:
+        # Escape all characters by default, space requires special handling
+        sequence = '" "' if character == ' ' else '\{}'.format(character)
+        qute_command('fake-key {}'.format(sequence))
+
+
 def main(arguments):
     if not arguments.url:
         argument_parser.print_help()
@@ -158,15 +182,22 @@ def main(arguments):
         return ExitCodes.COULD_NOT_MATCH_PASSWORD
     password = match.group(1)
 
-    insert_mode = ';; enter-mode insert' if arguments.insert_mode else ''
     if arguments.username_only:
-        qute_command('fake-key {}{}'.format(username, insert_mode))
+        fake_key_raw(username)
     elif arguments.password_only:
-        qute_command('fake-key {}{}'.format(password, insert_mode))
+        fake_key_raw(password)
+    elif arguments.otp_only:
+        otp = pass_otp(selection, arguments.io_encoding)
+        fake_key_raw(otp)
     else:
         # Enter username and password using fake-key and <Tab> (which seems to work almost universally), then switch
         # back into insert-mode, so the form can be directly submitted by hitting enter afterwards
-        qute_command('fake-key {} ;; fake-key <Tab> ;; fake-key {}{}'.format(username, password, insert_mode))
+        fake_key_raw(username)
+        qute_command('fake-key <Tab>')
+        fake_key_raw(password)
+
+    if arguments.insert_mode:
+        qute_command('enter-mode insert')
 
     return ExitCodes.SUCCESS
 

misc/userscripts/qutebrowser_viewsource

@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright 2015 Zach-Button <zachrey.button@gmail.com>
-# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
-#
-# This file is part of qutebrowser.
-#
-# qutebrowser is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# qutebrowser is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
-
-#
-# This script fetches the unprocessed HTML source for a page and opens it in vim.
-#	:bind gf spawn --userscript qutebrowser_viewsource
-#
-# Caveat: Does not use authentication of any kind. Add it in if you want it to.
-#
-
-path=$(mktemp --tmpdir qutebrowser_XXXXXXXX.html)
-
-curl "$QUTE_URL" > "$path"
-urxvt -e vim "$path"
-
-rm "$path"

misc/userscripts/readability

@@ -13,7 +13,11 @@
 from __future__ import absolute_import
 import codecs, os
 
-tmpfile=os.path.expanduser('~/.local/share/qutebrowser/userscripts/readability.html')
+tmpfile = os.path.join(
+    os.environ.get('QUTE_DATA_DIR',
+                   os.path.expanduser('~/.local/share/qutebrowser')),
+    'userscripts/readability.html')
+
 if not os.path.exists(os.path.dirname(tmpfile)):
     os.makedirs(os.path.dirname(tmpfile))
 

misc/userscripts/taskadd

@@ -28,7 +28,7 @@
 if msg="$(task add "$title" "$*" 2>&1)"; then
     # annotate the new task with the url, send the output back to the browser
     task +LATEST annotate "$QUTE_URL"
-    echo "message-info '$msg'" >> "$QUTE_FIFO"
+    echo "message-info '$(echo "$msg" | head -n 1)'" >> "$QUTE_FIFO"
 else
-    echo "message-error '$msg'" >> "$QUTE_FIFO"
+    echo "message-error '$(echo "$msg" | head -n 1)'" >> "$QUTE_FIFO"
 fi

misc/userscripts/tor_identity

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Copyright 2018 jnphilipp <mail@jnphilipp.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+# Change your tor identity.
+#
+# Set a hotkey to launch this script, then:
+#   :bind ti spawn --userscript tor_identity PASSWORD
+#
+# Use the hotkey to change your tor identity, press 'ti' to change it.
+# https://stem.torproject.org/faq.html#how-do-i-request-a-new-identity-from-tor
+#
+
+import os
+import sys
+
+try:
+    from stem import Signal
+    from stem.control import Controller
+except ImportError:
+    if os.getenv('QUTE_FIFO'):
+        with open(os.environ['QUTE_FIFO'], 'w') as f:
+            f.write('message-error "Failed to import stem."')
+    else:
+        print('Failed to import stem.')
+
+
+password = sys.argv[1]
+with Controller.from_port(port=9051) as controller:
+    controller.authenticate(password)
+    controller.signal(Signal.NEWNYM)
+    if os.getenv('QUTE_FIFO'):
+        with open(os.environ['QUTE_FIFO'], 'w') as f:
+            f.write('message-info "Tor identity changed."')
+    else:
+        print('Tor identity changed.')

misc/userscripts/view_in_mpv

@@ -49,7 +49,7 @@ msg() {
 
 MPV_COMMAND=${MPV_COMMAND:-mpv}
 # Warning: spaces in single flags are not supported
-MPV_FLAGS=${MPV_FLAGS:- --force-window --no-terminal --keep-open=yes --ytdl --ytdl-raw-options=yes-playlist=}
+MPV_FLAGS=${MPV_FLAGS:- --force-window --no-terminal --keep-open=yes --ytdl}
 IFS=" " read -r -a video_command <<< "$MPV_COMMAND $MPV_FLAGS"
 
 js() {

pytest.ini

@@ -1,4 +1,5 @@
 [pytest]
+log_level = NOTSET
 addopts = --strict -rfEw --faulthandler-timeout=90 --instafail --pythonwarnings error --benchmark-columns=Min,Max,Median
 testpaths = tests
 markers =
@@ -25,8 +26,11 @@ markers =
     this: Used to mark tests during development
     no_invalid_lines: Don't fail on unparseable lines in end2end tests
     issue2478: Tests which are broken on Windows with QtWebEngine, https://github.com/qutebrowser/qutebrowser/issues/2478
+    issue3572: Tests which are broken with QtWebEngine and Qt 5.10, https://github.com/qutebrowser/qutebrowser/issues/3572
+    qtbug60673: Tests which are broken if the conversion from orange selection to real selection  is flaky
     fake_os: Fake utils.is_* to a fake operating system
     unicode_locale: Tests which need an unicode locale to work
+    qtwebkit6021_skip: Tests which would fail on WebKit version 602.1
 qt_log_level_fail = WARNING
 qt_log_ignore =
     ^SpellCheck: .*
@@ -59,4 +63,8 @@ qt_log_ignore =
     ^inotify_add_watch\(".*"\) failed: "No space left on device"
     ^QSettings::value: Empty key passed
     ^Icon theme ".*" not found
+    ^Error receiving trust for a CA certificate
 xfail_strict = true
+filterwarnings =
+    # This happens in many qutebrowser dependencies...
+    ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working:DeprecationWarning

qutebrowser.py

@@ -2,7 +2,7 @@
 
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2015 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/__init__.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -22,11 +22,11 @@
 import os.path
 
 __author__ = "Florian Bruhin"
-__copyright__ = "Copyright 2014-2017 Florian Bruhin (The Compiler)"
+__copyright__ = "Copyright 2014-2018 Florian Bruhin (The Compiler)"
 __license__ = "GPL"
 __maintainer__ = __author__
 __email__ = "mail@qutebrowser.org"
-__version_info__ = (1, 1, 0)
+__version_info__ = (1, 4, 0)
 __version__ = '.'.join(str(e) for e in __version_info__)
 __description__ = "A keyboard-driven, vim-like browser based on PyQt5."
 

qutebrowser/__main__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/app.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -64,7 +64,7 @@ from qutebrowser.completion.models import miscmodels
 from qutebrowser.commands import cmdutils, runners, cmdexc
 from qutebrowser.config import config, websettings, configfiles, configinit
 from qutebrowser.browser import (urlmarks, adblock, history, browsertab,
-                                 downloads, greasemonkey)
+                                 qtnetworkdownloads, downloads, greasemonkey)
 from qutebrowser.browser.network import proxy
 from qutebrowser.browser.webkit import cookies, cache
 from qutebrowser.browser.webkit.network import networkmanager
@@ -95,6 +95,7 @@ def run(args):
 
     log.init.debug("Initializing directories...")
     standarddir.init(args)
+    utils.preload_resources()
 
     log.init.debug("Initializing config...")
     configinit.early_init(args)
@@ -339,7 +340,7 @@ def _open_startpage(win_id=None):
     for cur_win_id in list(window_ids):  # Copying as the dict could change
         tabbed_browser = objreg.get('tabbed-browser', scope='window',
                                     window=cur_win_id)
-        if tabbed_browser.count() == 0:
+        if tabbed_browser.widget.count() == 0:
             log.init.debug("Opening start pages")
             for url in config.val.url.start_pages:
                 tabbed_browser.tabopen(url)
@@ -491,6 +492,10 @@ def _init_modules(args, crash_handler):
     diskcache = cache.DiskCache(standarddir.cache(), parent=qApp)
     objreg.register('cache', diskcache)
 
+    log.init.debug("Initializing downloads...")
+    download_manager = qtnetworkdownloads.DownloadManager(parent=qApp)
+    objreg.register('qtnetwork-download-manager', download_manager)
+
     log.init.debug("Initializing Greasemonkey...")
     greasemonkey.init()
 
@@ -676,7 +681,6 @@ class Quitter:
     @cmdutils.argument('session', completion=miscmodels.session)
     def quit(self, save=False, session=None):
         """Quit qutebrowser.
-
         Args:
             save: When given, save the open windows even if auto_save.session
                   is turned off.
@@ -768,6 +772,8 @@ class Quitter:
                         pre_text="Error while saving {}".format(key))
         # Disable storage so removing tempdir will work
         websettings.shutdown()
+        # Disable application proxy factory to fix segfaults with Qt 5.10.1
+        proxy.shutdown()
         # Re-enable faulthandler to stdout, then remove crash log
         log.destroy.debug("Deactivating crash log...")
         objreg.get('crash-handler').destroy_crashlogfile()
@@ -836,7 +842,11 @@ class Application(QApplication):
     def event(self, e):
         """Handle macOS FileOpen events."""
         if e.type() == QEvent.FileOpen:
-            open_url(e.url(), no_raise=True)
+            url = e.url()
+            if url.isValid():
+                open_url(url, no_raise=True)
+            else:
+                message.error("Invalid URL: {}".format(url.errorString()))
         else:
             return super().event(e)
 
@@ -874,6 +884,7 @@ class EventFilter(QObject):
         self._handlers = {
             QEvent.KeyPress: self._handle_key_event,
             QEvent.KeyRelease: self._handle_key_event,
+            QEvent.ShortcutOverride: self._handle_key_event,
         }
 
     def _handle_key_event(self, event):
@@ -891,7 +902,7 @@ class EventFilter(QObject):
             return False
         try:
             man = objreg.get('mode-manager', scope='window', window='current')
-            return man.eventFilter(event)
+            return man.handle_event(event)
         except objreg.RegistryUnavailableError:
             # No window available yet, or not a MainWindow
             return False

qutebrowser/browser/__init__.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/adblock.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -94,14 +94,8 @@ class HostBlocker:
         _done_count: How many files have been read successfully.
         _local_hosts_file: The path to the blocked-hosts file.
         _config_hosts_file: The path to a blocked-hosts in ~/.config
-
-    Class attributes:
-        WHITELISTED: Hosts which never should be blocked.
     """
 
-    WHITELISTED = ('localhost', 'localhost.localdomain', 'broadcasthost',
-                   'local')
-
     def __init__(self):
         self._blocked_hosts = set()
         self._config_blocked_hosts = set()
@@ -176,8 +170,7 @@ class HostBlocker:
                               self._config_blocked_hosts)
         self._blocked_hosts = set()
         self._done_count = 0
-        download_manager = objreg.get('qtnetwork-download-manager',
-                                      scope='window', window='last-focused')
+        download_manager = objreg.get('qtnetwork-download-manager')
         for url in config.val.content.host_blocking.lists:
             if url.scheme() == 'file':
                 filename = url.toLocalFile()
@@ -235,16 +228,14 @@ class HostBlocker:
         parts = line.split()
         if len(parts) == 1:
             # "one host per line" format
-            host = parts[0]
-        elif len(parts) == 2:
-            # /etc/hosts format
-            host = parts[1]
+            hosts = [parts[0]]
         else:
-            log.misc.error("Failed to parse: {!r}".format(line))
-            return False
+            # /etc/hosts format
+            hosts = parts[1:]
 
-        if host not in self.WHITELISTED:
-            self._blocked_hosts.add(host)
+        for host in hosts:
+            if '.' in host and not host.endswith('.localdomain'):
+                self._blocked_hosts.add(host)
 
         return True
 

qutebrowser/browser/browsertab.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -22,17 +22,22 @@
 import enum
 import itertools
 
-import sip
 import attr
 from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, QObject, QSizeF, Qt
 from PyQt5.QtGui import QIcon
 from PyQt5.QtWidgets import QWidget, QApplication
 
+import pygments
+import pygments.lexers
+import pygments.formatters
+
 from qutebrowser.keyinput import modeman
 from qutebrowser.config import config
-from qutebrowser.utils import utils, objreg, usertypes, log, qtutils
+from qutebrowser.utils import (utils, objreg, usertypes, log, qtutils,
+                               urlutils, message)
 from qutebrowser.misc import miscwidgets, objects
 from qutebrowser.browser import mouse, hints
+from qutebrowser.qt import sip
 
 
 tab_id_gen = itertools.count(0)
@@ -95,18 +100,30 @@ class TabData:
                    load.
         inspector: The QWebInspector used for this webview.
         viewing_source: Set if we're currently showing a source view.
+                        Only used when sources are shown via pygments.
+        open_target: Where to open the next link.
+                     Only used for QtWebKit.
         override_target: Override for open_target for fake clicks (like hints).
                          Only used for QtWebKit.
         pinned: Flag to pin the tab.
         fullscreen: Whether the tab has a video shown fullscreen currently.
+        netrc_used: Whether netrc authentication was performed.
+        input_mode: current input mode for the tab.
     """
 
     keep_icon = attr.ib(False)
     viewing_source = attr.ib(False)
     inspector = attr.ib(None)
+    open_target = attr.ib(usertypes.ClickTarget.normal)
     override_target = attr.ib(None)
     pinned = attr.ib(False)
     fullscreen = attr.ib(False)
+    netrc_used = attr.ib(False)
+    input_mode = attr.ib(usertypes.KeyMode.normal)
+
+    def should_show_icon(self):
+        return (config.val.tabs.favicons.show == 'always' or
+                config.val.tabs.favicons.show == 'pinned' and self.pinned)
 
 
 class AbstractAction:
@@ -121,8 +138,9 @@ class AbstractAction:
     action_class = None
     action_base = None
 
-    def __init__(self):
+    def __init__(self, tab):
         self._widget = None
+        self._tab = tab
 
     def exit_fullscreen(self):
         """Exit the fullscreen mode."""
@@ -139,6 +157,31 @@ class AbstractAction:
             raise WebTabError("{} is not a valid web action!".format(name))
         self._widget.triggerPageAction(member)
 
+    def show_source(self,
+                    pygments=False):  # pylint: disable=redefined-outer-name
+        """Show the source of the current page in a new tab."""
+        raise NotImplementedError
+
+    def _show_source_pygments(self):
+
+        def show_source_cb(source):
+            """Show source as soon as it's ready."""
+            # WORKAROUND for https://github.com/PyCQA/pylint/issues/491
+            # pylint: disable=no-member
+            lexer = pygments.lexers.HtmlLexer()
+            formatter = pygments.formatters.HtmlFormatter(
+                full=True, linenos='table')
+            # pylint: enable=no-member
+            highlighted = pygments.highlight(source, lexer, formatter)
+
+            tb = objreg.get('tabbed-browser', scope='window',
+                            window=self._tab.win_id)
+            new_tab = tb.tabopen(background=False, related=True)
+            new_tab.set_html(highlighted, self._tab.url())
+            new_tab.data.viewing_source = True
+
+        self._tab.dump_async(show_source_cb)
+
 
 class AbstractPrinting:
 
@@ -245,10 +288,10 @@ class AbstractZoom(QObject):
         _default_zoom_changed: Whether the zoom was changed from the default.
     """
 
-    def __init__(self, win_id, parent=None):
+    def __init__(self, tab, parent=None):
         super().__init__(parent)
+        self._tab = tab
         self._widget = None
-        self._win_id = win_id
         self._default_zoom_changed = False
         self._init_neighborlist()
         config.instance.changed.connect(self._on_config_changed)
@@ -322,12 +365,18 @@ class AbstractZoom(QObject):
 
 class AbstractCaret(QObject):
 
-    """Attribute of AbstractTab for caret browsing."""
+    """Attribute of AbstractTab for caret browsing.
 
-    def __init__(self, win_id, tab, mode_manager, parent=None):
+    Signals:
+        selection_toggled: Emitted when the selection was toggled.
+                           arg: Whether the selection is now active.
+    """
+
+    selection_toggled = pyqtSignal(bool)
+
+    def __init__(self, tab, mode_manager, parent=None):
         super().__init__(parent)
         self._tab = tab
-        self._win_id = win_id
         self._widget = None
         self.selection_enabled = False
         mode_manager.entered.connect(self._on_mode_entered)
@@ -336,7 +385,7 @@ class AbstractCaret(QObject):
     def _on_mode_entered(self, mode):
         raise NotImplementedError
 
-    def _on_mode_left(self):
+    def _on_mode_left(self, mode):
         raise NotImplementedError
 
     def move_to_next_line(self, count=1):
@@ -390,11 +439,15 @@ class AbstractCaret(QObject):
     def drop_selection(self):
         raise NotImplementedError
 
-    def has_selection(self):
+    def selection(self, callback):
         raise NotImplementedError
 
-    def selection(self, html=False):
-        raise NotImplementedError
+    def _follow_enter(self, tab):
+        """Follow a link by faking an enter press."""
+        if tab:
+            self._tab.key_press(Qt.Key_Enter, modifier=Qt.ControlModifier)
+        else:
+            self._tab.key_press(Qt.Key_Enter)
 
     def follow_selected(self, *, tab=False):
         raise NotImplementedError
@@ -432,6 +485,9 @@ class AbstractScroller(QObject):
     def to_point(self, point):
         raise NotImplementedError
 
+    def to_anchor(self, name):
+        raise NotImplementedError
+
     def delta(self, x=0, y=0):
         raise NotImplementedError
 
@@ -578,6 +634,33 @@ class AbstractElements:
         raise NotImplementedError
 
 
+class AbstractAudio(QObject):
+
+    """Handling of audio/muting for this tab."""
+
+    muted_changed = pyqtSignal(bool)
+    recently_audible_changed = pyqtSignal(bool)
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._widget = None
+
+    def set_muted(self, muted: bool):
+        """Set this tab as muted or not."""
+        raise NotImplementedError
+
+    def is_muted(self):
+        """Whether this tab is muted."""
+        raise NotImplementedError
+
+    def toggle_muted(self):
+        self.set_muted(not self.is_muted())
+
+    def is_recently_audible(self):
+        """Whether this tab has had audio playing recently."""
+        raise NotImplementedError
+
+
 class AbstractTab(QWidget):
 
     """A wrapper over the given widget to hide its API and expose another one.
@@ -609,6 +692,7 @@ class AbstractTab(QWidget):
                                      process terminated.
                                      arg 0: A TerminationStatus member.
                                      arg 1: The exit code.
+        predicted_navigation: Emitted before we tell Qt to open a URL.
     """
 
     window_close_requested = pyqtSignal()
@@ -626,6 +710,7 @@ class AbstractTab(QWidget):
     add_history_item = pyqtSignal(QUrl, QUrl, str)  # url, requested url, title
     fullscreen_requested = pyqtSignal(bool)
     renderer_process_terminated = pyqtSignal(TerminationStatus, int)
+    predicted_navigation = pyqtSignal(QUrl)
 
     def __init__(self, *, win_id, mode_manager, private, parent=None):
         self.private = private
@@ -639,16 +724,6 @@ class AbstractTab(QWidget):
         tab_registry[self.tab_id] = self
         objreg.register('tab', self, registry=self.registry)
 
-        # self.history = AbstractHistory(self)
-        # self.scroller = AbstractScroller(self, parent=self)
-        # self.caret = AbstractCaret(win_id=win_id, tab=self,
-        #                            mode_manager=mode_manager, parent=self)
-        # self.zoom = AbstractZoom(win_id=win_id)
-        # self.search = AbstractSearch(parent=self)
-        # self.printing = AbstractPrinting()
-        # self.elements = AbstractElements(self)
-        # self.action = AbstractAction()
-
         self.data = TabData()
         self._layout = miscwidgets.WrapperLayout(self)
         self._widget = None
@@ -666,6 +741,8 @@ class AbstractTab(QWidget):
         objreg.register('hintmanager', hintmanager, scope='tab',
                         window=self.win_id, tab=self.tab_id)
 
+        self.predicted_navigation.connect(self._on_predicted_navigation)
+
     def _set_widget(self, widget):
         # pylint: disable=protected-access
         self._widget = widget
@@ -678,6 +755,8 @@ class AbstractTab(QWidget):
         self.printing._widget = widget
         self.action._widget = widget
         self.elements._widget = widget
+        self.audio._widget = widget
+        self.settings._settings = widget.settings()
 
         self._install_event_filter()
         self.zoom.set_default()
@@ -708,10 +787,24 @@ class AbstractTab(QWidget):
         if getattr(evt, 'posted', False):
             raise utils.Unreachable("Can't re-use an event which was already "
                                     "posted!")
+
         recipient = self.event_target()
+        if recipient is None:
+            # https://github.com/qutebrowser/qutebrowser/issues/3888
+            log.webview.warning("Unable to find event target!")
+            return
+
         evt.posted = True
         QApplication.postEvent(recipient, evt)
 
+    @pyqtSlot(QUrl)
+    def _on_predicted_navigation(self, url):
+        """Adjust the title if we are going to visit an URL soon."""
+        qtutils.ensure_valid(url)
+        url_string = url.toDisplayString()
+        log.webview.debug("Predicted navigation: {}".format(url_string))
+        self.title_changed.emit(url_string)
+
     @pyqtSlot(QUrl)
     def _on_url_changed(self, url):
         """Update title when URL has changed and no title is available."""
@@ -727,6 +820,23 @@ class AbstractTab(QWidget):
         self._set_load_status(usertypes.LoadStatus.loading)
         self.load_started.emit()
 
+    @pyqtSlot(usertypes.NavigationRequest)
+    def _on_navigation_request(self, navigation):
+        """Handle common acceptNavigationRequest code."""
+        url = utils.elide(navigation.url.toDisplayString(), 100)
+        log.webview.debug("navigation request: url {}, type {}, is_main_frame "
+                          "{}".format(url,
+                                      navigation.navigation_type,
+                                      navigation.is_main_frame))
+
+        if (navigation.navigation_type == navigation.Type.link_clicked and
+                not navigation.url.isValid()):
+            msg = urlutils.get_errstring(navigation.url,
+                                         "Invalid link clicked")
+            message.error(msg)
+            self.data.open_target = usertypes.ClickTarget.normal
+            navigation.accepted = False
+
     def handle_auto_insert_mode(self, ok):
         """Handle `input.insert_mode.auto_load` after loading finished."""
         if not config.val.input.insert_mode.auto_load or not ok:
@@ -749,6 +859,10 @@ class AbstractTab(QWidget):
 
     @pyqtSlot(bool)
     def _on_load_finished(self, ok):
+        if sip.isdeleted(self._widget):
+            # https://github.com/qutebrowser/qutebrowser/issues/3498
+            return
+
         sess_manager = objreg.get('session-manager')
         sess_manager.save_autosave()
 
@@ -761,7 +875,9 @@ class AbstractTab(QWidget):
             self._set_load_status(usertypes.LoadStatus.warn)
         else:
             self._set_load_status(usertypes.LoadStatus.error)
+
         self.load_finished.emit(ok)
+
         if not self.title():
             self.title_changed.emit(self.url().toDisplayString())
 
@@ -777,10 +893,6 @@ class AbstractTab(QWidget):
         self._progress = perc
         self.load_progress.emit(perc)
 
-    @pyqtSlot()
-    def _on_ssl_errors(self):
-        self._has_ssl_errors = True
-
     def url(self, requested=False):
         raise NotImplementedError
 
@@ -790,11 +902,12 @@ class AbstractTab(QWidget):
     def load_status(self):
         return self._load_status
 
-    def _openurl_prepare(self, url):
+    def _openurl_prepare(self, url, *, predict=True):
         qtutils.ensure_valid(url)
-        self.title_changed.emit(url.toDisplayString())
+        if predict:
+            self.predicted_navigation.emit(url)
 
-    def openurl(self, url):
+    def openurl(self, url, *, predict=True):
         raise NotImplementedError
 
     def reload(self, *, force=False):
@@ -811,7 +924,7 @@ class AbstractTab(QWidget):
         raise NotImplementedError
 
     def dump_async(self, callback, *, plain=False):
-        """Dump the current page to a file ascync.
+        """Dump the current page's html asynchronously.
 
         The given callback will be called with the result when dumping is
         complete.

qutebrowser/browser/commands.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -26,20 +26,16 @@ import functools
 import typing
 
 from PyQt5.QtWidgets import QApplication, QTabBar, QDialog
-from PyQt5.QtCore import Qt, QUrl, QEvent, QUrlQuery
-from PyQt5.QtGui import QKeyEvent
+from PyQt5.QtCore import pyqtSlot, Qt, QUrl, QEvent, QUrlQuery
 from PyQt5.QtPrintSupport import QPrintDialog, QPrintPreviewDialog
-import pygments
-import pygments.lexers
-import pygments.formatters
 
 from qutebrowser.commands import userscripts, cmdexc, cmdutils, runners
 from qutebrowser.config import config, configdata
 from qutebrowser.browser import (urlmarks, browsertab, inspector, navigate,
                                  webelem, downloads)
-from qutebrowser.keyinput import modeman
+from qutebrowser.keyinput import modeman, keyutils
 from qutebrowser.utils import (message, usertypes, log, qtutils, urlutils,
-                               objreg, utils, debug, standarddir)
+                               objreg, utils, standarddir)
 from qutebrowser.utils.usertypes import KeyMode
 from qutebrowser.misc import editor, guiprocess
 from qutebrowser.completion.models import urlmodel, miscmodels
@@ -57,7 +53,6 @@ class CommandDispatcher:
     cmdutils.register() decorators are run, currentWidget() will return None.
 
     Attributes:
-        _editor: The ExternalEditor object.
         _win_id: The window ID the CommandDispatcher is associated with.
         _tabbed_browser: The TabbedBrowser used.
     """
@@ -77,16 +72,16 @@ class CommandDispatcher:
 
     def _count(self):
         """Convenience method to get the widget count."""
-        return self._tabbed_browser.count()
+        return self._tabbed_browser.widget.count()
 
     def _set_current_index(self, idx):
         """Convenience method to set the current widget index."""
         cmdutils.check_overflow(idx, 'int')
-        self._tabbed_browser.setCurrentIndex(idx)
+        self._tabbed_browser.widget.setCurrentIndex(idx)
 
     def _current_index(self):
         """Convenience method to get the current widget index."""
-        return self._tabbed_browser.currentIndex()
+        return self._tabbed_browser.widget.currentIndex()
 
     def _current_url(self):
         """Convenience method to get the current url."""
@@ -105,7 +100,7 @@ class CommandDispatcher:
 
     def _current_widget(self):
         """Get the currently active widget from a command."""
-        widget = self._tabbed_browser.currentWidget()
+        widget = self._tabbed_browser.widget.currentWidget()
         if widget is None:
             raise cmdexc.CommandError("No WebView available yet!")
         return widget
@@ -151,10 +146,10 @@ class CommandDispatcher:
             None if no widget was found.
         """
         if count is None:
-            return self._tabbed_browser.currentWidget()
+            return self._tabbed_browser.widget.currentWidget()
         elif 1 <= count <= self._count():
             cmdutils.check_overflow(count + 1, 'int')
-            return self._tabbed_browser.widget(count - 1)
+            return self._tabbed_browser.widget.widget(count - 1)
         else:
             return None
 
@@ -167,7 +162,7 @@ class CommandDispatcher:
             if not show_error:
                 return
             raise cmdexc.CommandError("No last focused tab!")
-        idx = self._tabbed_browser.indexOf(tab)
+        idx = self._tabbed_browser.widget.indexOf(tab)
         if idx == -1:
             raise cmdexc.CommandError("Last focused tab vanished!")
         self._set_current_index(idx)
@@ -216,7 +211,7 @@ class CommandDispatcher:
                       what's configured in 'tabs.select_on_remove'.
             count: The tab index to close, or None
         """
-        tabbar = self._tabbed_browser.tabBar()
+        tabbar = self._tabbed_browser.widget.tabBar()
         selection_override = self._get_selection_override(prev, next_,
                                                           opposite)
 
@@ -268,7 +263,7 @@ class CommandDispatcher:
             return
 
         to_pin = not tab.data.pinned
-        self._tabbed_browser.set_tab_pinned(tab, to_pin)
+        self._tabbed_browser.widget.set_tab_pinned(tab, to_pin)
 
     @cmdutils.register(instance='command-dispatcher', name='open',
                        maxsplit=0, scope='window')
@@ -487,7 +482,8 @@ class CommandDispatcher:
         """
         cmdutils.check_exclusive((bg, window), 'bw')
         curtab = self._current_widget()
-        cur_title = self._tabbed_browser.page_title(self._current_index())
+        cur_title = self._tabbed_browser.widget.page_title(
+            self._current_index())
         try:
             history = curtab.history.serialize()
         except browsertab.WebTabError as e:
@@ -503,18 +499,18 @@ class CommandDispatcher:
         newtab = new_tabbed_browser.tabopen(background=bg)
         new_tabbed_browser = objreg.get('tabbed-browser', scope='window',
                                         window=newtab.win_id)
-        idx = new_tabbed_browser.indexOf(newtab)
+        idx = new_tabbed_browser.widget.indexOf(newtab)
 
-        new_tabbed_browser.set_page_title(idx, cur_title)
-        if config.val.tabs.favicons.show:
-            new_tabbed_browser.setTabIcon(idx, curtab.icon())
+        new_tabbed_browser.widget.set_page_title(idx, cur_title)
+        if curtab.data.should_show_icon():
+            new_tabbed_browser.widget.setTabIcon(idx, curtab.icon())
             if config.val.tabs.tabs_are_windows:
-                new_tabbed_browser.window().setWindowIcon(curtab.icon())
+                new_tabbed_browser.widget.window().setWindowIcon(curtab.icon())
 
         newtab.data.keep_icon = True
         newtab.history.deserialize(history)
         newtab.zoom.set_factor(curtab.zoom.factor())
-        new_tabbed_browser.set_tab_pinned(newtab, curtab.data.pinned)
+        new_tabbed_browser.widget.set_tab_pinned(newtab, curtab.data.pinned)
         return newtab
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
@@ -536,14 +532,19 @@ class CommandDispatcher:
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
     @cmdutils.argument('win_id', completion=miscmodels.window)
-    def tab_give(self, win_id: int = None):
+    @cmdutils.argument('count', count=True)
+    def tab_give(self, win_id: int = None, count=None):
         """Give the current tab to a new or existing window if win_id given.
 
         If no win_id is given, the tab will get detached into a new window.
 
         Args:
             win_id: The window ID of the window to give the current tab to.
+            count: Overrides win_id (index starts at 1 for win_id=0).
         """
+        if count is not None:
+            win_id = count - 1
+
         if win_id == self._win_id:
             raise cmdexc.CommandError("Can't give a tab to the same window")
 
@@ -565,12 +566,6 @@ class CommandDispatcher:
         tabbed_browser.tabopen(self._current_url())
         self._tabbed_browser.close_tab(self._current_widget(), add_undo=False)
 
-    @cmdutils.register(instance='command-dispatcher', scope='window',
-                       deprecated='Use :tab-give instead!')
-    def tab_detach(self):
-        """Deprecated way to detach a tab."""
-        self.tab_give()
-
     def _back_forward(self, tab, bg, window, count, forward):
         """Helper function for :back/:forward."""
         history = self._current_widget().history
@@ -638,7 +633,13 @@ class CommandDispatcher:
                 - `next`: Open a _next_ link.
                 - `up`: Go up a level in the current URL.
                 - `increment`: Increment the last number in the URL.
+                  Uses the
+                  link:settings.html#url.incdec_segments[url.incdec_segments]
+                  config option.
                 - `decrement`: Decrement the last number in the URL.
+                  Uses the
+                  link:settings.html#url.incdec_segments[url.incdec_segments]
+                  config option.
 
             tab: Open in a new tab.
             bg: Open in a background tab.
@@ -761,6 +762,15 @@ class CommandDispatcher:
 
         self._current_widget().scroller.to_perc(x, y)
 
+    @cmdutils.register(instance='command-dispatcher', scope='window')
+    def scroll_to_anchor(self, name):
+        """Scroll to the given anchor in the document.
+
+        Args:
+            name: The anchor to scroll to.
+        """
+        self._current_widget().scroller.to_anchor(name)
+
     @cmdutils.register(instance='command-dispatcher', scope='window')
     @cmdutils.argument('count', count=True)
     @cmdutils.argument('top_navigate', metavar='ACTION',
@@ -839,7 +849,7 @@ class CommandDispatcher:
             keep: Stay in visual mode after yanking the selection.
         """
         if what == 'title':
-            s = self._tabbed_browser.page_title(self._current_index())
+            s = self._tabbed_browser.widget.page_title(self._current_index())
         elif what == 'domain':
             port = self._current_url().port()
             s = '{}://{}{}'.format(self._current_url().scheme(),
@@ -849,14 +859,21 @@ class CommandDispatcher:
             s = self._yank_url(what)
             what = 'URL'  # For printing
         elif what == 'selection':
+            def _selection_callback(s):
+                if not s:
+                    message.info("Nothing to yank")
+                    return
+                self._yank_to_target(s, sel, what, keep)
+
             caret = self._current_widget().caret
-            s = caret.selection()
-            if not caret.has_selection() or not s:
-                message.info("Nothing to yank")
-                return
+            caret.selection(callback=_selection_callback)
+            return
         else:  # pragma: no cover
             raise ValueError("Invalid value {!r} for `what'.".format(what))
 
+        self._yank_to_target(s, sel, what, keep)
+
+    def _yank_to_target(self, s, sel, what, keep):
         if sel and utils.supports_selection():
             target = "primary selection"
         else:
@@ -944,7 +961,7 @@ class CommandDispatcher:
             force: Avoid confirmation for pinned tabs.
         """
         cmdutils.check_exclusive((prev, next_), 'pn')
-        cur_idx = self._tabbed_browser.currentIndex()
+        cur_idx = self._tabbed_browser.widget.currentIndex()
         assert cur_idx != -1
 
         def _to_close(i):
@@ -953,22 +970,25 @@ class CommandDispatcher:
                         (prev and i < cur_idx) or
                         (next_ and i > cur_idx))
 
-        # Check to see if we are closing any pinned tabs
-        if not force:
-            for i, tab in enumerate(self._tabbed_browser.widgets()):
-                if _to_close(i) and tab.data.pinned:
-                    self._tabbed_browser.tab_close_prompt_if_pinned(
-                        tab,
-                        force,
-                        lambda: self.tab_only(
-                            prev=prev, next_=next_, force=True))
-                    return
-
+        # close as many tabs as we can
         first_tab = True
+        pinned_tabs_cleanup = False
         for i, tab in enumerate(self._tabbed_browser.widgets()):
             if _to_close(i):
-                self._tabbed_browser.close_tab(tab, new_undo=first_tab)
-                first_tab = False
+                if force or not tab.data.pinned:
+                    self._tabbed_browser.close_tab(tab, new_undo=first_tab)
+                    first_tab = False
+                else:
+                    pinned_tabs_cleanup = tab
+
+        # Check to see if we would like to close any pinned tabs
+        if pinned_tabs_cleanup:
+            self._tabbed_browser.tab_close_prompt_if_pinned(
+                pinned_tabs_cleanup,
+                force,
+                lambda: self.tab_only(
+                    prev=prev, next_=next_, force=True),
+                text="Are you sure you want to close pinned tabs?")
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
     def undo(self):
@@ -996,7 +1016,7 @@ class CommandDispatcher:
         elif config.val.tabs.wrap:
             self._set_current_index(newidx % self._count())
         else:
-            raise cmdexc.CommandError("First tab")
+            log.webview.debug("First tab")
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
     @cmdutils.argument('count', count=True)
@@ -1016,7 +1036,7 @@ class CommandDispatcher:
         elif config.val.tabs.wrap:
             self._set_current_index(newidx % self._count())
         else:
-            raise cmdexc.CommandError("Last tab")
+            log.webview.debug("Last tab")
 
     def _resolve_buffer_index(self, index):
         """Resolve a buffer index to the tabbedbrowser and tab.
@@ -1058,11 +1078,11 @@ class CommandDispatcher:
 
         tabbed_browser = objreg.get('tabbed-browser', scope='window',
                                     window=win_id)
-        if not 0 < idx <= tabbed_browser.count():
+        if not 0 < idx <= tabbed_browser.widget.count():
             raise cmdexc.CommandError(
                 "There's no tab with index {}!".format(idx))
 
-        return (tabbed_browser, tabbed_browser.widget(idx-1))
+        return (tabbed_browser, tabbed_browser.widget.widget(idx-1))
 
     @cmdutils.register(instance='command-dispatcher', scope='window',
                        maxsplit=0)
@@ -1074,29 +1094,32 @@ class CommandDispatcher:
         Focuses window if necessary when index is given. If both index and
         count are given, use count.
 
+        With neither index nor count given, open the qute://tabs page.
+
         Args:
             index: The [win_id/]index of the tab to focus. Or a substring
                    in which case the closest match will be focused.
             count: The tab index to focus, starting with 1.
         """
         if count is None and index is None:
-            raise cmdexc.CommandError("buffer: Either a count or the argument "
-                                      "index must be specified.")
+            self.openurl('qute://tabs/', tab=True)
+            return
 
         if count is not None:
             index = str(count)
 
         tabbed_browser, tab = self._resolve_buffer_index(index)
 
-        window = tabbed_browser.window()
+        window = tabbed_browser.widget.window()
         window.activateWindow()
         window.raise_()
-        tabbed_browser.setCurrentWidget(tab)
+        tabbed_browser.widget.setCurrentWidget(tab)
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
     @cmdutils.argument('index', choices=['last'])
     @cmdutils.argument('count', count=True)
-    def tab_focus(self, index: typing.Union[str, int] = None, count=None):
+    def tab_focus(self, index: typing.Union[str, int] = None,
+                  count=None, no_last=False):
         """Select the tab given as argument/[count].
 
         If neither count nor index are given, it behaves like tab-next.
@@ -1108,13 +1131,14 @@ class CommandDispatcher:
                    Negative indices count from the end, such that -1 is the
                    last tab.
             count: The tab index to focus, starting with 1.
+            no_last: Whether to avoid focusing last tab if already focused.
         """
         index = count if count is not None else index
 
         if index == 'last':
             self._tab_focus_last()
             return
-        elif index == self._current_index() + 1:
+        elif not no_last and index == self._current_index() + 1:
             self._tab_focus_last(show_error=False)
             return
         elif index is None:
@@ -1173,7 +1197,7 @@ class CommandDispatcher:
         cur_idx = self._current_index()
         cmdutils.check_overflow(cur_idx, 'int')
         cmdutils.check_overflow(new_idx, 'int')
-        self._tabbed_browser.tabBar().moveTab(cur_idx, new_idx)
+        self._tabbed_browser.widget.tabBar().moveTab(cur_idx, new_idx)
 
     @cmdutils.register(instance='command-dispatcher', scope='window',
                        maxsplit=0, no_replace_variables=True)
@@ -1204,9 +1228,29 @@ class CommandDispatcher:
 
         log.procs.debug("Executing {} with args {}, userscript={}".format(
             cmd, args, userscript))
+
+        @pyqtSlot()
+        def _on_proc_finished():
+            if output:
+                tb = objreg.get('tabbed-browser', scope='window',
+                                window='last-focused')
+                tb.openurl(QUrl('qute://spawn-output'), newtab=True)
+
         if userscript:
+            def _selection_callback(s):
+                try:
+                    runner = self._run_userscript(s, cmd, args, verbose)
+                    runner.finished.connect(_on_proc_finished)
+                except cmdexc.CommandError as e:
+                    message.error(str(e))
+
             # ~ expansion is handled by the userscript module.
-            self._run_userscript(cmd, *args, verbose=verbose)
+            # dirty hack for async call because of:
+            # https://bugreports.qt.io/browse/QTBUG-53134
+            # until it fixed or blocked async call implemented:
+            # https://github.com/qutebrowser/qutebrowser/issues/3327
+            caret = self._current_widget().caret
+            caret.selection(callback=_selection_callback)
         else:
             cmd = os.path.expanduser(cmd)
             proc = guiprocess.GUIProcess(what='command', verbose=verbose,
@@ -1215,18 +1259,14 @@ class CommandDispatcher:
                 proc.start_detached(cmd, args)
             else:
                 proc.start(cmd, args)
-
-        if output:
-            tabbed_browser = objreg.get('tabbed-browser', scope='window',
-                                        window='last-focused')
-            tabbed_browser.openurl(QUrl('qute://spawn-output'), newtab=True)
+            proc.finished.connect(_on_proc_finished)
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
     def home(self):
         """Open main startpage in current tab."""
         self.openurl(config.val.url.start_pages[0])
 
-    def _run_userscript(self, cmd, *args, verbose=False):
+    def _run_userscript(self, selection, cmd, args, verbose):
         """Run a userscript given as argument.
 
         Args:
@@ -1236,21 +1276,15 @@ class CommandDispatcher:
         """
         env = {
             'QUTE_MODE': 'command',
+            'QUTE_SELECTED_TEXT': selection,
         }
 
         idx = self._current_index()
         if idx != -1:
-            env['QUTE_TITLE'] = self._tabbed_browser.page_title(idx)
-
-        tab = self._tabbed_browser.currentWidget()
-        if tab is not None and tab.caret.has_selection():
-            env['QUTE_SELECTED_TEXT'] = tab.caret.selection()
-            try:
-                env['QUTE_SELECTED_HTML'] = tab.caret.selection(html=True)
-            except browsertab.UnsupportedOperationError:
-                pass
+            env['QUTE_TITLE'] = self._tabbed_browser.widget.page_title(idx)
 
         # FIXME:qtwebengine: If tab is None, run_async will fail!
+        tab = self._tabbed_browser.widget.currentWidget()
 
         try:
             url = self._tabbed_browser.current_url()
@@ -1260,10 +1294,11 @@ class CommandDispatcher:
             env['QUTE_URL'] = url.toString(QUrl.FullyEncoded)
 
         try:
-            userscripts.run_async(tab, cmd, *args, win_id=self._win_id,
-                                  env=env, verbose=verbose)
+            runner = userscripts.run_async(
+                tab, cmd, *args, win_id=self._win_id, env=env, verbose=verbose)
         except userscripts.Error as e:
             raise cmdexc.CommandError(e)
+        return runner
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
     def quickmark_save(self):
@@ -1325,7 +1360,8 @@ class CommandDispatcher:
         link:qute://bookmarks[bookmarks page].
 
         Args:
-            url: url to save as a bookmark. If None, use url of current page.
+            url: url to save as a bookmark. If not given, use url of current
+                 page.
             title: title of the new bookmark.
             toggle: remove the bookmark instead of raising an error if it
                     already exists.
@@ -1334,7 +1370,7 @@ class CommandDispatcher:
             raise cmdexc.CommandError('Title must be provided if url has '
                                       'been provided')
         bookmark_manager = objreg.get('bookmark-manager')
-        if url is None:
+        if not url:
             url = self._current_url()
         else:
             try:
@@ -1419,6 +1455,7 @@ class CommandDispatcher:
             if tab.data.inspector is None:
                 tab.data.inspector = inspector.create()
                 tab.data.inspector.inspect(page)
+                tab.data.inspector.show()
             else:
                 tab.data.inspector.toggle(page)
         except inspector.WebInspectorError as e:
@@ -1434,8 +1471,7 @@ class CommandDispatcher:
             mhtml_: Download the current page and all assets as mhtml file.
         """
         # FIXME:qtwebengine do this with the QtWebEngine download manager?
-        download_manager = objreg.get('qtnetwork-download-manager',
-                                      scope='window', window=self._win_id)
+        download_manager = objreg.get('qtnetwork-download-manager')
         target = None
         if dest is not None:
             dest = downloads.transform_path(dest)
@@ -1480,34 +1516,31 @@ class CommandDispatcher:
             )
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
-    def view_source(self):
-        """Show the source of the current page in a new tab."""
-        tab = self._current_widget()
-        if tab.data.viewing_source:
-            raise cmdexc.CommandError("Already viewing source!")
+    def view_source(self, edit=False, pygments=False):
+        """Show the source of the current page in a new tab.
 
+        Args:
+            edit: Edit the source in the editor instead of opening a tab.
+            pygments: Use pygments to generate the view. This is always
+                      the case for QtWebKit. For QtWebEngine it may display
+                      slightly different source.
+                      Some JavaScript processing may be applied.
+        """
+        tab = self._current_widget()
         try:
             current_url = self._current_url()
         except cmdexc.CommandError as e:
             message.error(str(e))
             return
 
-        def show_source_cb(source):
-            """Show source as soon as it's ready."""
-            # WORKAROUND for https://github.com/PyCQA/pylint/issues/491
-            # pylint: disable=no-member
-            lexer = pygments.lexers.HtmlLexer()
-            formatter = pygments.formatters.HtmlFormatter(
-                full=True, linenos='table',
-                title='Source for {}'.format(current_url.toDisplayString()))
-            # pylint: enable=no-member
-            highlighted = pygments.highlight(source, lexer, formatter)
+        if current_url.scheme() == 'view-source' or tab.data.viewing_source:
+            raise cmdexc.CommandError("Already viewing source!")
 
-            new_tab = self._tabbed_browser.tabopen()
-            new_tab.set_html(highlighted)
-            new_tab.data.viewing_source = True
-
-        tab.dump_async(show_source_cb)
+        if edit:
+            ed = editor.ExternalEditor(self._tabbed_browser)
+            tab.dump_async(ed.edit)
+        else:
+            tab.action.show_source(pygments)
 
     @cmdutils.register(instance='command-dispatcher', scope='window',
                        debug=True)
@@ -1613,9 +1646,11 @@ class CommandDispatcher:
 
         caret_position = elem.caret_position()
 
-        ed = editor.ExternalEditor(self._tabbed_browser)
-        ed.editing_finished.connect(functools.partial(
-            self.on_editing_finished, elem))
+        ed = editor.ExternalEditor(watch=True, parent=self._tabbed_browser)
+        ed.file_updated.connect(functools.partial(
+            self.on_file_updated, ed, elem))
+        ed.editing_finished.connect(lambda: mainwindow.raise_window(
+            objreg.last_focused_window(), alert=False))
         ed.edit(text, caret_position)
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
@@ -1628,10 +1663,10 @@ class CommandDispatcher:
         tab = self._current_widget()
         tab.elements.find_focused(self._open_editor_cb)
 
-    def on_editing_finished(self, elem, text):
+    def on_file_updated(self, ed, elem, text):
         """Write the editor text into the form field and clean up tempfile.
 
-        Callback for GUIProcess when the editor was closed.
+        Callback for GUIProcess when the edited text was updated.
 
         Args:
             elem: The WebElementWrapper which was modified.
@@ -1639,12 +1674,12 @@ class CommandDispatcher:
         """
         try:
             elem.set_value(text)
-        except webelem.OrphanedError as e:
+        except webelem.OrphanedError:
             message.error('Edited element vanished')
+            ed.backup()
         except webelem.Error as e:
-            raise cmdexc.CommandError(str(e))
-
-        mainwindow.raise_window(objreg.last_focused_window(), alert=False)
+            message.error(str(e))
+            ed.backup()
 
     @cmdutils.register(instance='command-dispatcher', maxsplit=0,
                        scope='window')
@@ -1753,10 +1788,10 @@ class CommandDispatcher:
         """
         self.set_mark("'")
         tab = self._current_widget()
-        if tab.search.search_displayed:
-            tab.search.clear()
 
         if not text:
+            if tab.search.search_displayed:
+                tab.search.clear()
             return
 
         options = {
@@ -2087,15 +2122,13 @@ class CommandDispatcher:
             global_: If given, the keys are sent to the qutebrowser UI.
         """
         try:
-            keyinfos = utils.parse_keystring(keystring)
-        except utils.KeyParseError as e:
+            sequence = keyutils.KeySequence.parse(keystring)
+        except keyutils.KeyParseError as e:
             raise cmdexc.CommandError(str(e))
 
-        for keyinfo in keyinfos:
-            press_event = QKeyEvent(QEvent.KeyPress, keyinfo.key,
-                                    keyinfo.modifiers, keyinfo.text)
-            release_event = QKeyEvent(QEvent.KeyRelease, keyinfo.key,
-                                      keyinfo.modifiers, keyinfo.text)
+        for keyinfo in sequence:
+            press_event = keyinfo.to_event(QEvent.KeyPress)
+            release_event = keyinfo.to_event(QEvent.KeyRelease)
 
             if global_:
                 window = QApplication.focusWindow()
@@ -2138,7 +2171,7 @@ class CommandDispatcher:
         ed = editor.ExternalEditor(self._tabbed_browser)
 
         # Passthrough for openurl args (e.g. -t, -b, -w)
-        ed.editing_finished.connect(functools.partial(
+        ed.file_updated.connect(functools.partial(
             self._open_if_changed, old_url=old_url, bg=bg, tab=tab,
             window=window, private=private, related=related))
 
@@ -2195,12 +2228,22 @@ class CommandDispatcher:
                 pass
             return
 
-        window = self._tabbed_browser.window()
-        if window.isFullScreen():
-            window.setWindowState(
-                window.state_before_fullscreen & ~Qt.WindowFullScreen)
-        else:
-            window.state_before_fullscreen = window.windowState()
-            window.showFullScreen()
-        log.misc.debug('state before fullscreen: {}'.format(
-            debug.qflags_key(Qt, window.state_before_fullscreen)))
+        window = self._tabbed_browser.widget.window()
+        window.setWindowState(window.windowState() ^ Qt.WindowFullScreen)
+
+    @cmdutils.register(instance='command-dispatcher', scope='window',
+                       name='tab-mute')
+    @cmdutils.argument('count', count=True)
+    def tab_mute(self, count=None):
+        """Mute/Unmute the current/[count]th tab.
+
+        Args:
+            count: The tab index to mute or unmute, or None
+        """
+        tab = self._cntwidget(count)
+        if tab is None:
+            return
+        try:
+            tab.audio.toggle_muted()
+        except browsertab.WebTabError as e:
+            raise cmdexc.CommandError(e)

qutebrowser/browser/downloads.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -29,14 +29,14 @@ import pathlib
 import tempfile
 import enum
 
-import sip
 from PyQt5.QtCore import (pyqtSlot, pyqtSignal, Qt, QObject, QModelIndex,
-                          QTimer, QAbstractListModel)
+                          QTimer, QAbstractListModel, QUrl)
 
 from qutebrowser.commands import cmdexc, cmdutils
 from qutebrowser.config import config
 from qutebrowser.utils import (usertypes, standarddir, utils, message, log,
                                qtutils)
+from qutebrowser.qt import sip
 
 
 ModelRole = enum.IntEnum('ModelRole', ['item'], start=Qt.UserRole)
@@ -166,6 +166,7 @@ def get_filename_question(*, suggested_filename, url, parent=None):
     q.title = "Save file to:"
     q.text = "Please enter a location for <b>{}</b>".format(
         html.escape(url.toDisplayString()))
+    q.url = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded)
     q.mode = usertypes.PromptMode.download
     q.completed.connect(q.deleteLater)
     q.default = _path_suggestion(suggested_filename)
@@ -237,11 +238,14 @@ class FileDownloadTarget(_DownloadTarget):
 
     Attributes:
         filename: Filename where the download should be saved.
+        force_overwrite: Whether to overwrite the target without
+                         prompting the user.
     """
 
-    def __init__(self, filename):
+    def __init__(self, filename, force_overwrite=False):
         # pylint: disable=super-init-not-called
         self.filename = filename
+        self.force_overwrite = force_overwrite
 
     def suggested_filename(self):
         return os.path.basename(self.filename)
@@ -737,7 +741,8 @@ class AbstractDownloadItem(QObject):
         if isinstance(target, FileObjDownloadTarget):
             self._set_fileobj(target.fileobj, autoclose=False)
         elif isinstance(target, FileDownloadTarget):
-            self._set_filename(target.filename)
+            self._set_filename(
+                target.filename, force_overwrite=target.force_overwrite)
         elif isinstance(target, OpenFileDownloadTarget):
             try:
                 fobj = temp_download_manager.get_tmpfile(self.basename)

qutebrowser/browser/downloadview.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -21,13 +21,13 @@
 
 import functools
 
-import sip
 from PyQt5.QtCore import pyqtSlot, QSize, Qt, QTimer
 from PyQt5.QtWidgets import QListView, QSizePolicy, QMenu, QStyleFactory
 
 from qutebrowser.browser import downloads
 from qutebrowser.config import config
 from qutebrowser.utils import qtutils, utils, objreg
+from qutebrowser.qt import sip
 
 
 def update_geometry(obj):

qutebrowser/browser/greasemonkey.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2017-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -25,12 +25,16 @@ import json
 import fnmatch
 import functools
 import glob
+import textwrap
 
 import attr
 from PyQt5.QtCore import pyqtSignal, QObject, QUrl
 
-from qutebrowser.utils import log, standarddir, jinja, objreg
+from qutebrowser.utils import (log, standarddir, jinja, objreg, utils,
+                               javascript, urlmatch, version, usertypes)
 from qutebrowser.commands import cmdutils
+from qutebrowser.browser import downloads
+from qutebrowser.misc import objects
 
 
 def _scripts_dir():
@@ -45,13 +49,16 @@ class GreasemonkeyScript:
     def __init__(self, properties, code):
         self._code = code
         self.includes = []
+        self.matches = []
         self.excludes = []
+        self.requires = []
         self.description = None
         self.name = None
         self.namespace = None
         self.run_at = None
         self.script_meta = None
         self.runs_on_sub_frames = True
+        self.jsworld = "main"
         for name, value in properties:
             if name == 'name':
                 self.name = value
@@ -59,14 +66,20 @@ class GreasemonkeyScript:
                 self.namespace = value
             elif name == 'description':
                 self.description = value
-            elif name in ['include', 'match']:
+            elif name == 'include':
                 self.includes.append(value)
+            elif name == 'match':
+                self.matches.append(value)
             elif name in ['exclude', 'exclude_match']:
                 self.excludes.append(value)
             elif name == 'run-at':
                 self.run_at = value
             elif name == 'noframes':
                 self.runs_on_sub_frames = False
+            elif name == 'require':
+                self.requires.append(value)
+            elif name == 'qute-js-world':
+                self.jsworld = value
 
     HEADER_REGEX = r'// ==UserScript==|\n+// ==/UserScript==\n'
     PROPS_REGEX = r'// @(?P<prop>[^\s]+)\s*(?P<val>.*)'
@@ -86,7 +99,7 @@ class GreasemonkeyScript:
             props = ""
         script = cls(re.findall(cls.PROPS_REGEX, props), source)
         script.script_meta = props
-        if not props:
+        if not script.includes and not script.matches:
             script.includes = ['*']
         return script
 
@@ -94,28 +107,42 @@ class GreasemonkeyScript:
         """Return the processed JavaScript code of this script.
 
         Adorns the source code with GM_* methods for Greasemonkey
-        compatibility and wraps it in an IFFE to hide it within a
+        compatibility and wraps it in an IIFE to hide it within a
         lexical scope. Note that this means line numbers in your
         browser's debugger/inspector will not match up to the line
         numbers in the source script directly.
         """
-        return jinja.js_environment.get_template(
-            'greasemonkey_wrapper.js').render(
-                scriptName="/".join([self.namespace or '', self.name]),
-                scriptInfo=self._meta_json(),
-                scriptMeta=self.script_meta,
-                scriptSource=self._code)
+        # Don't use Proxy on this webkit version, the support isn't there.
+        use_proxy = not (
+            objects.backend == usertypes.Backend.QtWebKit and
+            version.qWebKitVersion() == '602.1')
+        template = jinja.js_environment.get_template('greasemonkey_wrapper.js')
+        return template.render(
+            scriptName=javascript.string_escape(
+                "/".join([self.namespace or '', self.name])),
+            scriptInfo=self._meta_json(),
+            scriptMeta=javascript.string_escape(self.script_meta),
+            scriptSource=self._code,
+            use_proxy=use_proxy)
 
     def _meta_json(self):
         return json.dumps({
             'name': self.name,
             'description': self.description,
-            'matches': self.includes,
+            'matches': self.matches,
             'includes': self.includes,
             'excludes': self.excludes,
             'run-at': self.run_at,
         })
 
+    def add_required_script(self, source):
+        """Add the source of a required script to this script."""
+        # The additional source is indented in case it also contains a
+        # metadata block. Because we pass everything at once to
+        # QWebEngineScript and that would parse the first metadata block
+        # found as the valid one.
+        self._code = "\n".join([textwrap.indent(source, "    "), self._code])
+
 
 @attr.s
 class MatchingScripts(object):
@@ -128,6 +155,42 @@ class MatchingScripts(object):
     idle = attr.ib(default=attr.Factory(list))
 
 
+class GreasemonkeyMatcher:
+
+    """Check whether scripts should be loaded for a given URL."""
+
+    # https://wiki.greasespot.net/Include_and_exclude_rules#Greaseable_schemes
+    # Limit the schemes scripts can run on due to unreasonable levels of
+    # exploitability
+    GREASEABLE_SCHEMES = ['http', 'https', 'ftp', 'file']
+
+    def __init__(self, url):
+        self._url = url
+        self._url_string = url.toString(QUrl.FullyEncoded)
+        self.is_greaseable = url.scheme() in self.GREASEABLE_SCHEMES
+
+    def _match_pattern(self, pattern):
+        # For include and exclude rules if they start and end with '/' they
+        # should be treated as a (ecma syntax) regular expression.
+        if pattern.startswith('/') and pattern.endswith('/'):
+            matches = re.search(pattern[1:-1], self._url_string, flags=re.I)
+            return matches is not None
+
+        # Otherwise they are glob expressions.
+        return fnmatch.fnmatch(self._url_string, pattern)
+
+    def matches(self, script):
+        """Check whether the URL matches filtering rules of the script."""
+        assert self.is_greaseable
+        matching_includes = any(self._match_pattern(pat)
+                                for pat in script.includes)
+        matching_match = any(urlmatch.UrlPattern(pat).matches(self._url)
+                             for pat in script.matches)
+        matching_excludes = any(self._match_pattern(pat)
+                                for pat in script.excludes)
+        return (matching_includes or matching_match) and not matching_excludes
+
+
 class GreasemonkeyManager(QObject):
 
     """Manager of userscripts and a Greasemonkey compatible environment.
@@ -135,26 +198,31 @@ class GreasemonkeyManager(QObject):
     Signals:
         scripts_reloaded: Emitted when scripts are reloaded from disk.
             Any cached or already-injected scripts should be
-            considered obselete.
+            considered obsolete.
     """
 
     scripts_reloaded = pyqtSignal()
-    # https://wiki.greasespot.net/Include_and_exclude_rules#Greaseable_schemes
-    # Limit the schemes scripts can run on due to unreasonable levels of
-    # exploitability
-    greaseable_schemes = ['http', 'https', 'ftp', 'file']
 
     def __init__(self, parent=None):
         super().__init__(parent)
+        self._run_start = []
+        self._run_end = []
+        self._run_idle = []
+        self._in_progress_dls = []
+
         self.load_scripts()
 
     @cmdutils.register(name='greasemonkey-reload',
                        instance='greasemonkey')
-    def load_scripts(self):
+    def load_scripts(self, force=False):
         """Re-read Greasemonkey scripts from disk.
 
         The scripts are read from a 'greasemonkey' subdirectory in
         qutebrowser's data directory (see `:version`).
+
+        Args:
+            force: For any scripts that have required dependencies,
+                   re-download them.
         """
         self._run_start = []
         self._run_end = []
@@ -170,42 +238,132 @@ class GreasemonkeyManager(QObject):
                 script = GreasemonkeyScript.parse(script_file.read())
                 if not script.name:
                     script.name = script_filename
-
-                if script.run_at == 'document-start':
-                    self._run_start.append(script)
-                elif script.run_at == 'document-end':
-                    self._run_end.append(script)
-                elif script.run_at == 'document-idle':
-                    self._run_idle.append(script)
-                else:
-                    log.greasemonkey.warning("Script {} has invalid run-at "
-                                             "defined, defaulting to "
-                                             "document-end"
-                                             .format(script_path))
-                    # Default as per
-                    # https://wiki.greasespot.net/Metadata_Block#.40run-at
-                    self._run_end.append(script)
-                log.greasemonkey.debug("Loaded script: {}".format(script.name))
+                self.add_script(script, force)
         self.scripts_reloaded.emit()
 
+    def add_script(self, script, force=False):
+        """Add a GreasemonkeyScript to this manager.
+
+        Args:
+            force: Fetch and overwrite any dependancies which are
+                   already locally cached.
+        """
+        if script.requires:
+            log.greasemonkey.debug(
+                "Deferring script until requirements are "
+                "fulfilled: {}".format(script.name))
+            self._get_required_scripts(script, force)
+        else:
+            self._add_script(script)
+
+    def _add_script(self, script):
+        if script.run_at == 'document-start':
+            self._run_start.append(script)
+        elif script.run_at == 'document-end':
+            self._run_end.append(script)
+        elif script.run_at == 'document-idle':
+            self._run_idle.append(script)
+        else:
+            if script.run_at:
+                log.greasemonkey.warning("Script {} has invalid run-at "
+                                         "defined, defaulting to "
+                                         "document-end"
+                                         .format(script.name))
+                # Default as per
+                # https://wiki.greasespot.net/Metadata_Block#.40run-at
+            self._run_end.append(script)
+        log.greasemonkey.debug("Loaded script: {}".format(script.name))
+
+    def _required_url_to_file_path(self, url):
+        requires_dir = os.path.join(_scripts_dir(), 'requires')
+        if not os.path.exists(requires_dir):
+            os.mkdir(requires_dir)
+        return os.path.join(requires_dir, utils.sanitize_filename(url))
+
+    def _on_required_download_finished(self, script, download):
+        self._in_progress_dls.remove(download)
+        if not self._add_script_with_requires(script):
+            log.greasemonkey.debug(
+                "Finished download {} for script {} "
+                "but some requirements are still pending"
+                .format(download.basename, script.name))
+
+    def _add_script_with_requires(self, script, quiet=False):
+        """Add a script with pending downloads to this GreasemonkeyManager.
+
+        Specifically a script that has dependancies specified via an
+        `@require` rule.
+
+        Args:
+            script: The GreasemonkeyScript to add.
+            quiet: True to suppress the scripts_reloaded signal after
+                   adding `script`.
+        Returns: True if the script was added, False if there are still
+                 dependancies being downloaded.
+        """
+        # See if we are still waiting on any required scripts for this one
+        for dl in self._in_progress_dls:
+            if dl.requested_url in script.requires:
+                return False
+
+        # Need to add the required scripts to the IIFE now
+        for url in reversed(script.requires):
+            target_path = self._required_url_to_file_path(url)
+            log.greasemonkey.debug(
+                "Adding required script for {} to IIFE: {}"
+                .format(script.name, url))
+            with open(target_path, encoding='utf8') as f:
+                script.add_required_script(f.read())
+
+        self._add_script(script)
+        if not quiet:
+            self.scripts_reloaded.emit()
+        return True
+
+    def _get_required_scripts(self, script, force=False):
+        required_dls = [(url, self._required_url_to_file_path(url))
+                        for url in script.requires]
+        if not force:
+            required_dls = [(url, path) for (url, path) in required_dls
+                            if not os.path.exists(path)]
+        if not required_dls:
+            # All the required files exist already
+            self._add_script_with_requires(script, quiet=True)
+            return
+
+        download_manager = objreg.get('qtnetwork-download-manager')
+
+        for url, target_path in required_dls:
+            target = downloads.FileDownloadTarget(target_path,
+                                                  force_overwrite=True)
+            download = download_manager.get(QUrl(url), target=target,
+                                            auto_remove=True)
+            download.requested_url = url
+            self._in_progress_dls.append(download)
+            if download.successful:
+                self._on_required_download_finished(script, download)
+            else:
+                download.finished.connect(
+                    functools.partial(self._on_required_download_finished,
+                                      script, download))
+
     def scripts_for(self, url):
         """Fetch scripts that are registered to run for url.
 
         returns a tuple of lists of scripts meant to run at (document-start,
         document-end, document-idle)
         """
-        if url.scheme() not in self.greaseable_schemes:
+        matcher = GreasemonkeyMatcher(url)
+        if not matcher.is_greaseable:
             return MatchingScripts(url, [], [], [])
-        match = functools.partial(fnmatch.fnmatch,
-                                  url.toString(QUrl.FullyEncoded))
-        tester = (lambda script:
-                  any(match(pat) for pat in script.includes) and
-                  not any(match(pat) for pat in script.excludes))
         return MatchingScripts(
-            url,
-            [script for script in self._run_start if tester(script)],
-            [script for script in self._run_end if tester(script)],
-            [script for script in self._run_idle if tester(script)]
+            url=url,
+            start=[script for script in self._run_start
+                   if matcher.matches(script)],
+            end=[script for script in self._run_end
+                 if matcher.matches(script)],
+            idle=[script for script in self._run_idle
+                  if matcher.matches(script)]
         )
 
     def all_scripts(self):

qutebrowser/browser/hints.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -22,6 +22,7 @@
 import collections
 import functools
 import math
+import os
 import re
 import html
 import enum
@@ -154,6 +155,7 @@ class HintContext:
         to_follow: The link to follow when enter is pressed.
         args: Custom arguments for userscript/spawn
         rapid: Whether to do rapid hinting.
+        first_run: Whether the action is run for the 1st time in rapid hinting.
         add_history: Whether to add yanked or spawned link to the history.
         filterstr: Used to save the filter string for restoring in rapid mode.
         tab: The WebTab object we started hinting in.
@@ -166,12 +168,14 @@ class HintContext:
     baseurl = attr.ib(None)
     to_follow = attr.ib(None)
     rapid = attr.ib(False)
+    first_run = attr.ib(True)
     add_history = attr.ib(False)
     filterstr = attr.ib(None)
     args = attr.ib(attr.Factory(list))
     tab = attr.ib(None)
     group = attr.ib(None)
     hint_mode = attr.ib(None)
+    first = attr.ib(False)
 
     def get_args(self, urlstr):
         """Get the arguments, with {hint-url} replaced by the given URL."""
@@ -240,7 +244,18 @@ class HintActions:
         if url.scheme() == 'mailto':
             flags |= QUrl.RemoveScheme
         urlstr = url.toString(flags)
-        utils.set_clipboard(urlstr, selection=sel)
+
+        new_content = urlstr
+
+        # only second and consecutive yanks are to append to the clipboard
+        if context.rapid and not context.first_run:
+            try:
+                old_content = utils.get_clipboard(selection=sel)
+            except utils.ClipboardEmptyError:
+                pass
+            else:
+                new_content = os.linesep.join([old_content, new_content])
+        utils.set_clipboard(new_content, selection=sel)
 
         msg = "Yanked URL to {}: {}".format(
             "primary selection" if sel else "clipboard",
@@ -291,8 +306,7 @@ class HintActions:
         user_agent = context.tab.user_agent()
 
         # FIXME:qtwebengine do this with QtWebEngine downloads?
-        download_manager = objreg.get('qtnetwork-download-manager',
-                                      scope='window', window=self._win_id)
+        download_manager = objreg.get('qtnetwork-download-manager')
         download_manager.get(url, qnam=qnam, user_agent=user_agent,
                              prompt_download_directory=prompt)
 
@@ -613,6 +627,9 @@ class HintManager(QObject):
         modeman.enter(self._win_id, usertypes.KeyMode.hint,
                       'HintManager.start')
 
+        if self._context.first:
+            self._fire(strings[0])
+            return
         # to make auto_follow == 'always' work
         self._handle_auto_follow()
 
@@ -621,7 +638,8 @@ class HintManager(QObject):
     @cmdutils.argument('win_id', win_id=True)
     def start(self,  # pylint: disable=keyword-arg-before-vararg
               group=webelem.Group.all, target=Target.normal,
-              *args, win_id, mode=None, add_history=False, rapid=False):
+              *args, win_id, mode=None, add_history=False, rapid=False,
+              first=False):
         """Start hinting.
 
         Args:
@@ -632,6 +650,7 @@ class HintManager(QObject):
                    `window`, `run`, `hover`, `userscript` and `spawn`.
             add_history: Whether to add the spawned or yanked link to the
                          browsing history.
+            first: Click the first hinted element without prompting.
             group: The element types to hint.
 
                 - `all`: All clickable elements.
@@ -683,7 +702,7 @@ class HintManager(QObject):
         """
         tabbed_browser = objreg.get('tabbed-browser', scope='window',
                                     window=self._win_id)
-        tab = tabbed_browser.currentWidget()
+        tab = tabbed_browser.widget.currentWidget()
         if tab is None:
             raise cmdexc.CommandError("No WebView available yet!")
 
@@ -695,7 +714,8 @@ class HintManager(QObject):
         if rapid:
             if target in [Target.tab_bg, Target.window, Target.run,
                           Target.hover, Target.userscript, Target.spawn,
-                          Target.download, Target.normal, Target.current]:
+                          Target.download, Target.normal, Target.current,
+                          Target.yank, Target.yank_primary]:
                 pass
             elif target == Target.tab and config.val.tabs.background:
                 pass
@@ -714,6 +734,7 @@ class HintManager(QObject):
         self._context.rapid = rapid
         self._context.hint_mode = mode
         self._context.add_history = add_history
+        self._context.first = first
         try:
             self._context.baseurl = tabbed_browser.current_url()
         except qtutils.QtValueError:
@@ -908,22 +929,32 @@ class HintManager(QObject):
         except HintingError as e:
             message.error(str(e))
 
+        if self._context is not None:
+            self._context.first_run = False
+
     @cmdutils.register(instance='hintmanager', scope='tab',
                        modes=[usertypes.KeyMode.hint])
-    def follow_hint(self, keystring=None):
+    def follow_hint(self, select=False, keystring=None):
         """Follow a hint.
 
         Args:
+            select: Only select the given hint, don't necessarily follow it.
             keystring: The hint to follow, or None.
         """
         if keystring is None:
             if self._context.to_follow is None:
                 raise cmdexc.CommandError("No hint to follow")
+            elif select:
+                raise cmdexc.CommandError("Can't use --select without hint.")
             else:
                 keystring = self._context.to_follow
         elif keystring not in self._context.labels:
             raise cmdexc.CommandError("No hint {}!".format(keystring))
-        self._fire(keystring)
+
+        if select:
+            self.handle_partial_key(keystring)
+        else:
+            self._fire(keystring)
 
     @pyqtSlot(usertypes.KeyMode)
     def on_mode_left(self, mode):

qutebrowser/browser/history.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -23,7 +23,7 @@ import os
 import time
 import contextlib
 
-from PyQt5.QtCore import pyqtSlot, QUrl, QTimer
+from PyQt5.QtCore import pyqtSlot, QUrl, QTimer, pyqtSignal
 
 from qutebrowser.commands import cmdutils, cmdexc
 from qutebrowser.utils import (utils, objreg, log, usertypes, message,
@@ -52,6 +52,11 @@ class WebHistory(sql.SqlTable):
 
     """The global history of visited pages."""
 
+    # All web history cleared
+    history_cleared = pyqtSignal()
+    # one url cleared
+    url_cleared = pyqtSignal(QUrl)
+
     def __init__(self, parent=None):
         super().__init__("History", ['url', 'title', 'atime', 'redirect'],
                          constraints={'url': 'NOT NULL',
@@ -157,6 +162,7 @@ class WebHistory(sql.SqlTable):
         with self._handle_sql_errors():
             self.delete_all()
             self.completion.delete_all()
+        self.history_cleared.emit()
 
     def delete_url(self, url):
         """Remove all history entries with the given url.
@@ -168,11 +174,12 @@ class WebHistory(sql.SqlTable):
         qtutils.ensure_valid(qurl)
         self.delete('url', self._format_url(qurl))
         self.completion.delete('url', self._format_completion_url(qurl))
+        self.url_cleared.emit(qurl)
 
     @pyqtSlot(QUrl, QUrl, str)
     def add_from_tab(self, url, requested_url, title):
         """Add a new history entry as slot, called from a BrowserTab."""
-        if any(url.scheme() == 'data' or
+        if any(url.scheme() in ('data', 'view-source') or
                (url.scheme(), url.host()) == ('qute', 'back')
                for url in (url, requested_url)):
             return

qutebrowser/browser/inspector.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -87,6 +87,8 @@ class AbstractWebInspector(QWidget):
         data = bytes(self.saveGeometry())
         geom = base64.b64encode(data).decode('ASCII')
         configfiles.state['geometry']['inspector'] = geom
+
+        self.inspect(None)
         super().closeEvent(e)
 
     def inspect(self, page):
@@ -99,3 +101,4 @@ class AbstractWebInspector(QWidget):
             self.hide()
         else:
             self.inspect(page)
+            self.show()

qutebrowser/browser/mouse.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -22,7 +22,7 @@
 from PyQt5.QtCore import QObject, QEvent, Qt, QTimer
 
 from qutebrowser.config import config
-from qutebrowser.utils import message, log, usertypes
+from qutebrowser.utils import message, log, usertypes, qtutils, objreg
 from qutebrowser.keyinput import modeman
 
 
@@ -40,11 +40,12 @@ class ChildEventFilter(QObject):
         _widget: The widget expected to send out childEvents.
     """
 
-    def __init__(self, eventfilter, widget, parent=None):
+    def __init__(self, eventfilter, widget, win_id, parent=None):
         super().__init__(parent)
         self._filter = eventfilter
         assert widget is not None
         self._widget = widget
+        self._win_id = win_id
 
     def eventFilter(self, obj, event):
         """Act on ChildAdded events."""
@@ -54,6 +55,29 @@ class ChildEventFilter(QObject):
                 obj, child))
             assert obj is self._widget
             child.installEventFilter(self._filter)
+
+            if qtutils.version_check('5.11', compiled=False, exact=True):
+                # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-68076
+                pass_modes = [usertypes.KeyMode.command,
+                              usertypes.KeyMode.prompt,
+                              usertypes.KeyMode.yesno]
+                if modeman.instance(self._win_id).mode not in pass_modes:
+                    tabbed_browser = objreg.get('tabbed-browser',
+                                                scope='window',
+                                                window=self._win_id)
+                    current_index = tabbed_browser.widget.currentIndex()
+                    try:
+                        widget_index = tabbed_browser.widget.indexOf(
+                            self._widget.parent())
+                    except RuntimeError:
+                        widget_index = -1
+                    if current_index == widget_index:
+                        QTimer.singleShot(0, self._widget.setFocus)
+
+        elif event.type() == QEvent.ChildRemoved:
+            child = event.child()
+            log.mouse.debug("{}: removed child {}".format(obj, child))
+
         return False
 
 
@@ -151,8 +175,9 @@ class MouseEventFilter(QObject):
 
         if elem.is_editable():
             log.mouse.debug("Clicked editable element!")
-            modeman.enter(self._tab.win_id, usertypes.KeyMode.insert,
-                          'click', only_if_normal=True)
+            if config.val.input.insert_mode.auto_enter:
+                modeman.enter(self._tab.win_id, usertypes.KeyMode.insert,
+                              'click', only_if_normal=True)
         else:
             log.mouse.debug("Clicked non-editable element!")
             if config.val.input.insert_mode.auto_leave:

qutebrowser/browser/navigate.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/network/pac.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/network/proxy.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -34,6 +34,10 @@ def init():
     QNetworkProxyFactory.setApplicationProxyFactory(proxy_factory)
 
 
+def shutdown():
+    QNetworkProxyFactory.setApplicationProxyFactory(None)
+
+
 class ProxyFactory(QNetworkProxyFactory):
 
     """Factory for proxies to be used by qutebrowser."""
@@ -61,6 +65,9 @@ class ProxyFactory(QNetworkProxyFactory):
         """
         proxy = config.val.content.proxy
         if proxy is configtypes.SYSTEM_PROXY:
+            # On Linux, use "export http_proxy=socks5://host:port" to manually
+            # set system proxy.
+            # ref. http://doc.qt.io/qt-5/qnetworkproxyfactory.html#systemProxyForQuery
             proxies = QNetworkProxyFactory.systemProxyForQuery(query)
         elif isinstance(proxy, pac.PACFetcher):
             proxies = proxy.resolve(query)

qutebrowser/browser/pdfjs.py

@@ -1,7 +1,7 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
 # Copyright 2015 Daniel Schadt
-# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/qtnetworkdownloads.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -20,6 +20,7 @@
 """Download manager."""
 
 import io
+import os.path
 import shutil
 import functools
 
@@ -28,7 +29,7 @@ from PyQt5.QtCore import pyqtSlot, pyqtSignal, QTimer
 from PyQt5.QtNetwork import QNetworkRequest, QNetworkReply
 
 from qutebrowser.config import config
-from qutebrowser.utils import message, usertypes, log, urlutils, utils
+from qutebrowser.utils import message, usertypes, log, urlutils, utils, debug
 from qutebrowser.browser import downloads
 from qutebrowser.browser.webkit import http
 from qutebrowser.browser.webkit.network import networkmanager
@@ -161,6 +162,7 @@ class DownloadItem(downloads.AbstractDownloadItem):
             QTimer.singleShot(0, lambda: self._die(reply.errorString()))
 
     def _do_cancel(self):
+        self._read_timer.stop()
         if self._reply is not None:
             self._reply.finished.disconnect(self._on_reply_finished)
             self._reply.abort()
@@ -198,21 +200,23 @@ class DownloadItem(downloads.AbstractDownloadItem):
 
     def _ask_confirm_question(self, title, msg):
         no_action = functools.partial(self.cancel, remove_data=False)
+        url = 'file://{}'.format(self._filename)
         message.confirm_async(title=title, text=msg,
                               yes_action=self._after_set_filename,
                               no_action=no_action, cancel_action=no_action,
-                              abort_on=[self.cancelled, self.error])
+                              abort_on=[self.cancelled, self.error], url=url)
 
     def _ask_create_parent_question(self, title, msg,
                                     force_overwrite, remember_directory):
         no_action = functools.partial(self.cancel, remove_data=False)
+        url = 'file://{}'.format(os.path.dirname(self._filename))
         message.confirm_async(title=title, text=msg,
                               yes_action=(lambda:
                                           self._after_create_parent_question(
                                               force_overwrite,
                                               remember_directory)),
                               no_action=no_action, cancel_action=no_action,
-                              abort_on=[self.cancelled, self.error])
+                              abort_on=[self.cancelled, self.error], url=url)
 
     def _set_fileobj(self, fileobj, *, autoclose=True):
         """Set the file object to write the download to.
@@ -303,7 +307,14 @@ class DownloadItem(downloads.AbstractDownloadItem):
         """Handle QNetworkReply errors."""
         if code == QNetworkReply.OperationCanceledError:
             return
-        self._die(self._reply.errorString())
+
+        if self._reply is None:
+            error = "Unknown error: {}".format(
+                debug.qenum_key(QNetworkReply, code))
+        else:
+            error = self._reply.errorString()
+
+        self._die(error)
 
     @pyqtSlot()
     def _on_read_timer_timeout(self):
@@ -378,10 +389,10 @@ class DownloadManager(downloads.AbstractDownloadManager):
         _networkmanager: A NetworkManager for generic downloads.
     """
 
-    def __init__(self, win_id, parent=None):
+    def __init__(self, parent=None):
         super().__init__(parent)
         self._networkmanager = networkmanager.NetworkManager(
-            win_id=win_id, tab_id=None,
+            win_id=None, tab_id=None,
             private=config.val.content.private_browsing, parent=self)
 
     @pyqtSlot('QUrl')

qutebrowser/browser/qutescheme.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -24,12 +24,14 @@ Module attributes:
     _HANDLERS: The handlers registered via decorators.
 """
 
+import html
 import json
 import os
 import time
 import textwrap
 import mimetypes
 import urllib
+import collections
 
 import pkg_resources
 from PyQt5.QtCore import QUrlQuery, QUrl
@@ -39,6 +41,7 @@ from qutebrowser.config import config, configdata, configexc, configdiff
 from qutebrowser.utils import (version, utils, jinja, log, message, docutils,
                                objreg, urlutils)
 from qutebrowser.misc import objects
+from qutebrowser.qt import sip
 
 
 pyeval_output = ":pyeval was never called"
@@ -121,12 +124,12 @@ class add_handler:  # noqa: N801,N806 pylint: disable=invalid-name
 
     def wrong_backend_handler(self, url):
         """Show an error page about using the invalid backend."""
-        html = jinja.render('error.html',
-                            title="Error while opening qute://url",
-                            url=url.toDisplayString(),
-                            error='{} is not available with this '
-                                  'backend'.format(url.toDisplayString()))
-        return 'text/html', html
+        src = jinja.render('error.html',
+                           title="Error while opening qute://url",
+                           url=url.toDisplayString(),
+                           error='{} is not available with this '
+                                 'backend'.format(url.toDisplayString()))
+        return 'text/html', src
 
 
 def data_for_url(url):
@@ -175,7 +178,7 @@ def data_for_url(url):
     except OSError as e:
         # FIXME:qtwebengine how to handle this?
         raise QuteSchemeOSError(e)
-    except QuteSchemeError as e:
+    except QuteSchemeError:
         raise
 
     assert mimetype is not None, url
@@ -194,11 +197,32 @@ def qute_bookmarks(_url):
     quickmarks = sorted(objreg.get('quickmark-manager').marks.items(),
                         key=lambda x: x[0])  # Sort by name
 
-    html = jinja.render('bookmarks.html',
-                        title='Bookmarks',
-                        bookmarks=bookmarks,
-                        quickmarks=quickmarks)
-    return 'text/html', html
+    src = jinja.render('bookmarks.html',
+                       title='Bookmarks',
+                       bookmarks=bookmarks,
+                       quickmarks=quickmarks)
+    return 'text/html', src
+
+
+@add_handler('tabs')
+def qute_tabs(_url):
+    """Handler for qute://tabs. Display information about all open tabs."""
+    tabs = collections.defaultdict(list)
+    for win_id, window in objreg.window_registry.items():
+        if sip.isdeleted(window):
+            continue
+        tabbed_browser = objreg.get('tabbed-browser',
+                                    scope='window',
+                                    window=win_id)
+        for tab in tabbed_browser.widgets():
+            if tab.url() not in [QUrl("qute://tabs/"), QUrl("qute://tabs")]:
+                urlstr = tab.url().toDisplayString()
+                tabs[str(win_id)].append((tab.title(), urlstr))
+
+    src = jinja.render('tabs.html',
+                       title='Tabs',
+                       tab_list_by_window=tabs)
+    return 'text/html', src
 
 
 def history_data(start_time, offset=None):
@@ -218,8 +242,9 @@ def history_data(start_time, offset=None):
         end_time = start_time - 24*60*60
         entries = hist.entries_between(end_time, start_time)
 
-    return [{"url": e.url, "title": e.title or e.url, "time": e.atime}
-            for e in entries]
+    return [{"url": e.url,
+             "title": html.escape(e.title) or html.escape(e.url),
+             "time": e.atime} for e in entries]
 
 
 @add_handler('history')
@@ -240,8 +265,6 @@ def qute_history(url):
 
         return 'text/html', json.dumps(history_data(start_time, offset))
     else:
-        if not config.val.content.javascript.enabled:
-            return 'text/plain', b'JavaScript is required for qute://history'
         return 'text/html', jinja.render(
             'history.html',
             title='History',
@@ -266,25 +289,25 @@ def qute_javascript(url):
 @add_handler('pyeval')
 def qute_pyeval(_url):
     """Handler for qute://pyeval."""
-    html = jinja.render('pre.html', title='pyeval', content=pyeval_output)
-    return 'text/html', html
+    src = jinja.render('pre.html', title='pyeval', content=pyeval_output)
+    return 'text/html', src
 
 
 @add_handler('spawn-output')
 def qute_spawn_output(_url):
     """Handler for qute://spawn-output."""
-    html = jinja.render('pre.html', title='spawn output', content=spawn_output)
-    return 'text/html', html
+    src = jinja.render('pre.html', title='spawn output', content=spawn_output)
+    return 'text/html', src
 
 
 @add_handler('version')
 @add_handler('verizon')
 def qute_version(_url):
     """Handler for qute://version."""
-    html = jinja.render('version.html', title='Version info',
-                        version=version.version(),
-                        copyright=qutebrowser.__copyright__)
-    return 'text/html', html
+    src = jinja.render('version.html', title='Version info',
+                       version=version.version(),
+                       copyright=qutebrowser.__copyright__)
+    return 'text/html', src
 
 
 @add_handler('plainlog')
@@ -302,8 +325,8 @@ def qute_plainlog(url):
         if not level:
             level = 'vdebug'
         text = log.ram_handler.dump_log(html=False, level=level)
-    html = jinja.render('pre.html', title='log', content=text)
-    return 'text/html', html
+    src = jinja.render('pre.html', title='log', content=text)
+    return 'text/html', src
 
 
 @add_handler('log')
@@ -322,8 +345,8 @@ def qute_log(url):
             level = 'vdebug'
         html_log = log.ram_handler.dump_log(html=True, level=level)
 
-    html = jinja.render('log.html', title='log', content=html_log)
-    return 'text/html', html
+    src = jinja.render('log.html', title='log', content=html_log)
+    return 'text/html', src
 
 
 @add_handler('gpl')
@@ -394,12 +417,12 @@ def qute_help(url):
 @add_handler('backend-warning')
 def qute_backend_warning(_url):
     """Handler for qute://backend-warning."""
-    html = jinja.render('backend-warning.html',
-                        distribution=version.distribution(),
-                        Distribution=version.Distribution,
-                        version=pkg_resources.parse_version,
-                        title="Legacy backend warning")
-    return 'text/html', html
+    src = jinja.render('backend-warning.html',
+                       distribution=version.distribution(),
+                       Distribution=version.Distribution,
+                       version=pkg_resources.parse_version,
+                       title="Legacy backend warning")
+    return 'text/html', src
 
 
 def _qute_settings_set(url):
@@ -429,10 +452,26 @@ def qute_settings(url):
     if url.path() == '/set':
         return _qute_settings_set(url)
 
-    html = jinja.render('settings.html', title='settings',
-                        configdata=configdata,
-                        confget=config.instance.get_str)
-    return 'text/html', html
+    src = jinja.render('settings.html', title='settings',
+                       configdata=configdata,
+                       confget=config.instance.get_str)
+    return 'text/html', src
+
+
+@add_handler('bindings')
+def qute_bindings(_url):
+    """Handler for qute://bindings. View keybindings."""
+    bindings = {}
+    defaults = config.val.bindings.default
+    modes = set(defaults.keys()).union(config.val.bindings.commands)
+    modes.remove('normal')
+    modes = ['normal'] + sorted(list(modes))
+    for mode in modes:
+        bindings[mode] = config.key_instance.get_bindings_for(mode)
+
+    src = jinja.render('bindings.html', title='Bindings',
+                       bindings=bindings)
+    return 'text/html', src
 
 
 @add_handler('back')
@@ -441,10 +480,10 @@ def qute_back(url):
 
     Simple page to free ram / lazy load a site, goes back on focusing the tab.
     """
-    html = jinja.render(
+    src = jinja.render(
         'back.html',
         title='Suspended: ' + urllib.parse.unquote(url.fragment()))
-    return 'text/html', html
+    return 'text/html', src
 
 
 @add_handler('configdiff')
@@ -460,3 +499,10 @@ def qute_configdiff(url):
     else:
         data = config.instance.dump_userconfig().encode('utf-8')
         return 'text/plain', data
+
+
+@add_handler('pastebin-version')
+def qute_pastebin_version(_url):
+    """Handler that pastebins the version string."""
+    version.pastebin_version()
+    return 'text/plain', b'Paste called.'

qutebrowser/browser/shared.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -19,7 +19,11 @@
 
 """Various utilities shared between webpage/webview subclasses."""
 
+import os
 import html
+import netrc
+
+from PyQt5.QtCore import QUrl
 
 from qutebrowser.config import config
 from qutebrowser.utils import usertypes, message, log, objreg, jinja, utils
@@ -27,25 +31,25 @@ from qutebrowser.mainwindow import mainwindow
 
 
 class CallSuper(Exception):
-
     """Raised when the caller should call the superclass instead."""
 
 
-def custom_headers():
+def custom_headers(url):
     """Get the combined custom headers."""
     headers = {}
 
-    dnt_config = config.val.content.headers.do_not_track
+    dnt_config = config.instance.get('content.headers.do_not_track', url=url)
     if dnt_config is not None:
         dnt = b'1' if dnt_config else b'0'
         headers[b'DNT'] = dnt
         headers[b'X-Do-Not-Track'] = dnt
 
-    conf_headers = config.val.content.headers.custom
+    conf_headers = config.instance.get('content.headers.custom', url=url)
     for header, value in conf_headers.items():
         headers[header.encode('ascii')] = value.encode('ascii')
 
-    accept_language = config.val.content.headers.accept_language
+    accept_language = config.instance.get('content.headers.accept_language',
+                                          url=url)
     if accept_language is not None:
         headers[b'Accept-Language'] = accept_language.encode('ascii')
 
@@ -61,30 +65,33 @@ def authentication_required(url, authenticator, abort_on):
     else:
         msg = '<b>{}</b> needs authentication'.format(
             html.escape(url.toDisplayString()))
+    urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded)
     answer = message.ask(title="Authentication required", text=msg,
                          mode=usertypes.PromptMode.user_pwd,
-                         abort_on=abort_on)
+                         abort_on=abort_on, url=urlstr)
     if answer is not None:
         authenticator.setUser(answer.user)
         authenticator.setPassword(answer.password)
     return answer
 
 
-def javascript_confirm(url, js_msg, abort_on):
+def javascript_confirm(url, js_msg, abort_on, *, escape_msg=True):
     """Display a javascript confirm prompt."""
     log.js.debug("confirm: {}".format(js_msg))
     if config.val.content.javascript.modal_dialog:
         raise CallSuper
 
+    js_msg = html.escape(js_msg) if escape_msg else js_msg
     msg = 'From <b>{}</b>:<br/>{}'.format(html.escape(url.toDisplayString()),
-                                          html.escape(js_msg))
+                                          js_msg)
+    urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded)
     ans = message.ask('Javascript confirm', msg,
                       mode=usertypes.PromptMode.yesno,
-                      abort_on=abort_on)
+                      abort_on=abort_on, url=urlstr)
     return bool(ans)
 
 
-def javascript_prompt(url, js_msg, default, abort_on):
+def javascript_prompt(url, js_msg, default, abort_on, *, escape_msg=True):
     """Display a javascript prompt."""
     log.js.debug("prompt: {}".format(js_msg))
     if config.val.content.javascript.modal_dialog:
@@ -92,12 +99,14 @@ def javascript_prompt(url, js_msg, default, abort_on):
     if not config.val.content.javascript.prompt:
         return (False, "")
 
+    js_msg = html.escape(js_msg) if escape_msg else js_msg
     msg = '<b>{}</b> asks:<br/>{}'.format(html.escape(url.toDisplayString()),
-                                          html.escape(js_msg))
+                                          js_msg)
+    urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded)
     answer = message.ask('Javascript prompt', msg,
                          mode=usertypes.PromptMode.text,
                          default=default,
-                         abort_on=abort_on)
+                         abort_on=abort_on, url=urlstr)
 
     if answer is None:
         return (False, "")
@@ -105,7 +114,7 @@ def javascript_prompt(url, js_msg, default, abort_on):
         return (True, answer)
 
 
-def javascript_alert(url, js_msg, abort_on):
+def javascript_alert(url, js_msg, abort_on, *, escape_msg=True):
     """Display a javascript alert."""
     log.js.debug("alert: {}".format(js_msg))
     if config.val.content.javascript.modal_dialog:
@@ -114,10 +123,12 @@ def javascript_alert(url, js_msg, abort_on):
     if not config.val.content.javascript.alert:
         return
 
+    js_msg = html.escape(js_msg) if escape_msg else js_msg
     msg = 'From <b>{}</b>:<br/>{}'.format(html.escape(url.toDisplayString()),
-                                          html.escape(js_msg))
+                                          js_msg)
+    urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded)
     message.ask('Javascript alert', msg, mode=usertypes.PromptMode.alert,
-                abort_on=abort_on)
+                abort_on=abort_on, url=urlstr)
 
 
 def javascript_log_message(level, source, line, msg):
@@ -146,7 +157,7 @@ def ignore_certificate_errors(url, errors, abort_on):
     Return:
         True if the error should be ignored, False otherwise.
     """
-    ssl_strict = config.val.content.ssl_strict
+    ssl_strict = config.instance.get('content.ssl_strict', url=url)
     log.webview.debug("Certificate errors {!r}, strict {}".format(
         errors, ssl_strict))
 
@@ -164,9 +175,10 @@ def ignore_certificate_errors(url, errors, abort_on):
         """.strip())
         msg = err_template.render(url=url, errors=errors)
 
+        urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded)
         ignore = message.ask(title="Certificate errors - continue?", text=msg,
                              mode=usertypes.PromptMode.yesno, default=False,
-                             abort_on=abort_on)
+                             abort_on=abort_on, url=urlstr)
         if ignore is None:
             # prompt aborted
             ignore = False
@@ -185,7 +197,8 @@ def ignore_certificate_errors(url, errors, abort_on):
     raise utils.Unreachable
 
 
-def feature_permission(url, option, msg, yes_action, no_action, abort_on):
+def feature_permission(url, option, msg, yes_action, no_action, abort_on,
+                       blocking=False):
     """Handle a feature permission request.
 
     Args:
@@ -195,22 +208,36 @@ def feature_permission(url, option, msg, yes_action, no_action, abort_on):
         yes_action: A callable to call if the request was approved
         no_action: A callable to call if the request was denied
         abort_on: A list of signals which interrupt the question.
+        blocking: If True, ask a blocking question.
 
     Return:
-        The Question object if a question was asked, None otherwise.
+        The Question object if a question was asked (and blocking=False),
+        None otherwise.
     """
-    config_val = config.instance.get(option)
+    config_val = config.instance.get(option, url=url)
     if config_val == 'ask':
         if url.isValid():
+            urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded)
             text = "Allow the website at <b>{}</b> to {}?".format(
                 html.escape(url.toDisplayString()), msg)
         else:
+            urlstr = None
             text = "Allow the website to {}?".format(msg)
 
-        return message.confirm_async(
-            yes_action=yes_action, no_action=no_action,
-            cancel_action=no_action, abort_on=abort_on,
-            title='Permission request', text=text)
+        if blocking:
+            answer = message.ask(abort_on=abort_on, title='Permission request',
+                                 text=text, url=urlstr,
+                                 mode=usertypes.PromptMode.yesno)
+            if answer:
+                yes_action()
+            else:
+                no_action()
+            return None
+        else:
+            return message.confirm_async(
+                yes_action=yes_action, no_action=no_action,
+                cancel_action=no_action, abort_on=abort_on,
+                title='Permission request', text=text, url=urlstr)
     elif config_val:
         yes_action()
         return None
@@ -260,3 +287,41 @@ def get_user_stylesheet():
         css += '\nhtml > ::-webkit-scrollbar { width: 0px; height: 0px; }'
 
     return css
+
+
+def netrc_authentication(url, authenticator):
+    """Perform authorization using netrc.
+
+    Args:
+        url: The URL the request was done for.
+        authenticator: QAuthenticator object used to set credentials provided.
+
+    Return:
+        True if netrc found credentials for the URL.
+        False otherwise.
+    """
+    if 'HOME' not in os.environ:
+        # We'll get an OSError by netrc if 'HOME' isn't available in
+        # os.environ. We don't want to log that, so we prevent it
+        # altogether.
+        return False
+    user, password = None, None
+    try:
+        net = netrc.netrc(config.val.content.netrc_file)
+        authenticators = net.authenticators(url.host())
+        if authenticators is not None:
+            (user, _account, password) = authenticators
+    except FileNotFoundError:
+        log.misc.debug("No .netrc file found")
+    except OSError as e:
+        log.misc.exception("Unable to read the netrc file: {}".format(e))
+    except netrc.NetrcParseError as e:
+        log.misc.exception("Error when parsing the netrc file: {}".format(e))
+
+    if user is None:
+        return False
+
+    authenticator.setUser(user)
+    authenticator.setPassword(password)
+
+    return True

qutebrowser/browser/signalfilter.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -76,11 +76,11 @@ class SignalFilter(QObject):
         tabbed_browser = objreg.get('tabbed-browser', scope='window',
                                     window=self._win_id)
         try:
-            tabidx = tabbed_browser.indexOf(tab)
+            tabidx = tabbed_browser.widget.indexOf(tab)
         except RuntimeError:
             # The tab has been deleted already
             return
-        if tabidx == tabbed_browser.currentIndex():
+        if tabidx == tabbed_browser.widget.currentIndex():
             if log_signal:
                 log.signals.debug("emitting: {} (tab {})".format(
                     debug.dbg_signal(signal, args), tabidx))

qutebrowser/browser/urlmarks.py

@@ -1,7 +1,7 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
-# Copyright 2015-2017 Antoni Boucher <bouanto@zoho.com>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2018 Antoni Boucher <bouanto@zoho.com>
 #
 # This file is part of qutebrowser.
 #
@@ -161,7 +161,7 @@ class QuickmarkManager(UrlMarkManager):
             "Add quickmark:", usertypes.PromptMode.text,
             functools.partial(self.quickmark_add, urlstr),
             text="Please enter a quickmark name for<br/><b>{}</b>".format(
-                html.escape(url.toDisplayString())))
+                html.escape(url.toDisplayString())), url=urlstr)
 
     @cmdutils.register(instance='quickmark-manager')
     def quickmark_add(self, url, name):
@@ -192,7 +192,7 @@ class QuickmarkManager(UrlMarkManager):
         if name in self.marks:
             message.confirm_async(
                 title="Override existing quickmark?",
-                yes_action=set_mark, default=True)
+                yes_action=set_mark, default=True, url=url)
         else:
             set_mark()
 
@@ -280,7 +280,7 @@ class BookmarkManager(UrlMarkManager):
 
         if urlstr in self.marks:
             if toggle:
-                del self.marks[urlstr]
+                self.delete(urlstr)
                 return False
             else:
                 raise AlreadyExistsError("Bookmark already exists!")

qutebrowser/browser/webelem.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -41,8 +41,10 @@ Group = enum.Enum('Group', ['all', 'links', 'images', 'url', 'inputs'])
 
 SELECTORS = {
     Group.all: ('a, area, textarea, select, input:not([type=hidden]), button, '
-                'frame, iframe, link, [onclick], [onmousedown], [role=link], '
-                '[role=option], [role=button], img'),
+                'frame, iframe, link, summary, [onclick], [onmousedown], '
+                '[role=link], [role=option], [role=button], img, '
+                # Angular 1 selectors
+                '[ng-click], [ngClick], [data-ng-click], [x-ng-click]'),
     Group.links: 'a[href], area[href], link[href], [role=link][href]',
     Group.images: 'img',
     Group.url: '[src], [href]',
@@ -305,6 +307,10 @@ class AbstractWebElement(collections.abc.MutableMapping):
         href_tags = ['a', 'area', 'link']
         return self.tag_name() in href_tags and 'href' in self
 
+    def _requires_user_interaction(self):
+        """Return True if clicking this element needs user interaction."""
+        raise NotImplementedError
+
     def _mouse_pos(self):
         """Get the position to click/hover."""
         # Click the center of the largest square fitting into the top/left
@@ -403,14 +409,15 @@ class AbstractWebElement(collections.abc.MutableMapping):
             return
 
         if click_target == usertypes.ClickTarget.normal:
-            if self.is_link():
+            if self.is_link() and not self._requires_user_interaction():
                 log.webelem.debug("Clicking via JS click()")
                 self._click_js(click_target)
             elif self.is_editable(strict=True):
                 log.webelem.debug("Clicking via JS focus()")
                 self._click_editable(click_target)
-                modeman.enter(self._tab.win_id, usertypes.KeyMode.insert,
-                              'clicking input')
+                if config.val.input.insert_mode.auto_enter:
+                    modeman.enter(self._tab.win_id, usertypes.KeyMode.insert,
+                                  'clicking input')
             else:
                 self._click_fake_event(click_target)
         elif click_target in [usertypes.ClickTarget.tab,

qutebrowser/browser/webengine/__init__.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webengine/certificateerror.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -28,6 +28,10 @@ class CertificateErrorWrapper(usertypes.AbstractCertificateErrorWrapper):
 
     """A wrapper over a QWebEngineCertificateError."""
 
+    def __init__(self, error):
+        super().__init__(error)
+        self.ignore = False
+
     def __str__(self):
         return self._error.errorDescription()
 
@@ -37,5 +41,8 @@ class CertificateErrorWrapper(usertypes.AbstractCertificateErrorWrapper):
                                         self._error.error()),
             string=str(self))
 
+    def url(self):
+        return self._error.url()
+
     def is_overridable(self):
         return self._error.isOverridable()

qutebrowser/browser/webengine/cookies.py

@@ -0,0 +1,48 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Filter for QtWebEngine cookies."""
+
+from qutebrowser.config import config
+from qutebrowser.utils import utils
+
+
+def _accept_cookie(request):
+    """Check whether the given cookie should be accepted."""
+    accept = config.val.content.cookies.accept
+    if accept == 'all':
+        return True
+    elif accept in ['no-3rdparty', 'no-unknown-3rdparty']:
+        return not request.thirdParty
+    elif accept == 'never':
+        return False
+    else:
+        raise utils.Unreachable
+
+
+def install_filter(profile):
+    """Install the cookie filter on the given profile.
+
+    On Qt < 5.11, the filter isn't installed.
+    """
+    store = profile.cookieStore()
+    try:
+        store.setCookieFilter(_accept_cookie)
+    except AttributeError:
+        pass

qutebrowser/browser/webengine/interceptor.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -19,20 +19,22 @@
 
 """A request interceptor taking care of adblocking and custom headers."""
 
-from PyQt5.QtWebEngineCore import QWebEngineUrlRequestInterceptor
+from PyQt5.QtWebEngineCore import (QWebEngineUrlRequestInterceptor,
+                                   QWebEngineUrlRequestInfo)
 
 from qutebrowser.config import config
 from qutebrowser.browser import shared
-from qutebrowser.utils import utils, log
+from qutebrowser.utils import utils, log, debug
 
 
 class RequestInterceptor(QWebEngineUrlRequestInterceptor):
 
     """Handle ad blocking and custom headers."""
 
-    def __init__(self, host_blocker, parent=None):
+    def __init__(self, host_blocker, args, parent=None):
         super().__init__(parent)
         self._host_blocker = host_blocker
+        self._args = args
 
     def install(self, profile):
         """Install the interceptor on the given QWebEngineProfile."""
@@ -54,15 +56,29 @@ class RequestInterceptor(QWebEngineUrlRequestInterceptor):
         Args:
             info: QWebEngineUrlRequestInfo &info
         """
+        if 'log-requests' in self._args.debug_flags:
+            resource_type = debug.qenum_key(QWebEngineUrlRequestInfo,
+                                            info.resourceType())
+            navigation_type = debug.qenum_key(QWebEngineUrlRequestInfo,
+                                              info.navigationType())
+            log.webview.debug("{} {}, first-party {}, resource {}, "
+                              "navigation {}".format(
+                                  bytes(info.requestMethod()).decode('ascii'),
+                                  info.requestUrl().toDisplayString(),
+                                  info.firstPartyUrl().toDisplayString(),
+                                  resource_type, navigation_type))
+
+        url = info.requestUrl()
+
         # FIXME:qtwebengine only block ads for NavigationTypeOther?
-        if self._host_blocker.is_blocked(info.requestUrl()):
+        if self._host_blocker.is_blocked(url):
             log.webview.info("Request to {} blocked by host blocker.".format(
-                info.requestUrl().host()))
+                url.host()))
             info.block(True)
 
-        for header, value in shared.custom_headers():
+        for header, value in shared.custom_headers(url=url):
             info.setHttpHeader(header, value)
 
-        user_agent = config.val.content.headers.user_agent
+        user_agent = config.instance.get('content.headers.user_agent', url=url)
         if user_agent is not None:
             info.setHttpHeader(b'User-Agent', user_agent.encode('ascii'))

qutebrowser/browser/webengine/spell.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2017 Michal Siedlaczek <michal.siedlaczek@gmail.com>
+# Copyright 2017-2018 Michal Siedlaczek <michal.siedlaczek@gmail.com>
 
 # This file is part of qutebrowser.
 #
@@ -21,38 +21,53 @@
 
 import glob
 import os
+import os.path
 import re
+import shutil
 
 from PyQt5.QtCore import QLibraryInfo
-from qutebrowser.utils import log
+from qutebrowser.utils import log, message, standarddir, qtutils
+
+dict_version_re = re.compile(r".+-(?P<version>[0-9]+-[0-9]+?)\.bdic")
 
 
 def version(filename):
     """Extract the version number from the dictionary file name."""
-    version_re = re.compile(r".+-(?P<version>[0-9]+-[0-9]+?)\.bdic")
-    match = version_re.fullmatch(filename)
+    match = dict_version_re.match(filename)
     if match is None:
-        raise ValueError('the given dictionary file name is malformed: {}'
-                         .format(filename))
+        message.warning(
+            "Found a dictionary with a malformed name: {}".format(filename))
+        return None
     return tuple(int(n) for n in match.group('version').split('-'))
 
 
-def dictionary_dir():
+def dictionary_dir(old=False):
     """Return the path (str) to the QtWebEngine's dictionaries directory."""
-    datapath = QLibraryInfo.location(QLibraryInfo.DataPath)
+    if qtutils.version_check('5.10', compiled=False) and not old:
+        datapath = standarddir.data()
+    else:
+        datapath = QLibraryInfo.location(QLibraryInfo.DataPath)
     return os.path.join(datapath, 'qtwebengine_dictionaries')
 
 
 def local_files(code):
-    """Return all installed dictionaries for the given code."""
+    """Return all installed dictionaries for the given code.
+
+    The returned dictionaries are sorted by version, therefore the latest will
+    be the first element. The list will be empty if no dictionaries are found.
+    """
     pathname = os.path.join(dictionary_dir(), '{}*.bdic'.format(code))
     matching_dicts = glob.glob(pathname)
-    files = []
-    for matching_dict in sorted(matching_dicts, key=version, reverse=True):
-        filename = os.path.basename(matching_dict)
-        log.config.debug('Found file for dict {}: {}'.format(code, filename))
-        files.append(filename)
-    return files
+    versioned_dicts = []
+    for matching_dict in matching_dicts:
+        parsed_version = version(matching_dict)
+        if parsed_version is not None:
+            filename = os.path.basename(matching_dict)
+            log.config.debug('Found file for dict {}: {}'
+                             .format(code, filename))
+            versioned_dicts.append((parsed_version, filename))
+    return [filename for version, filename
+            in sorted(versioned_dicts, reverse=True)]
 
 
 def local_filename(code):
@@ -63,3 +78,16 @@ def local_filename(code):
     """
     all_installed = local_files(code)
     return os.path.splitext(all_installed[0])[0] if all_installed else None
+
+
+def init():
+    """Initialize the dictionary path if supported."""
+    if qtutils.version_check('5.10', compiled=False):
+        new_dir = dictionary_dir()
+        old_dir = dictionary_dir(old=True)
+        os.environ['QTWEBENGINE_DICTIONARIES_PATH'] = new_dir
+        try:
+            if os.path.exists(old_dir) and not os.path.exists(new_dir):
+                shutil.copytree(old_dir, new_dir)
+        except OSError:
+            log.misc.exception("Failed to copy old dictionaries")

qutebrowser/browser/webengine/tabhistory.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webengine/webenginedownloads.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -45,6 +45,10 @@ class DownloadItem(downloads.AbstractDownloadItem):
         qt_item.downloadProgress.connect(self.stats.on_download_progress)
         qt_item.stateChanged.connect(self._on_state_changed)
 
+        # Ensure wrapped qt_item is deleted manually when the wrapper object
+        # is deleted. See https://github.com/qutebrowser/qutebrowser/issues/3373
+        self.destroyed.connect(self._qt_item.deleteLater)
+
     def _is_page_download(self):
         """Check if this item is a page (i.e. mhtml) download."""
         return (self._qt_item.savePageFormat() !=
@@ -96,9 +100,19 @@ class DownloadItem(downloads.AbstractDownloadItem):
         self._qt_item.cancel()
 
     def retry(self):
-        # https://bugreports.qt.io/browse/QTBUG-56840
-        raise downloads.UnsupportedOperationError(
-            "Retrying downloads is unsupported with QtWebEngine")
+        state = self._qt_item.state()
+        if state != QWebEngineDownloadItem.DownloadInterrupted:
+            log.downloads.warning(
+                "Trying to retry download in state {}".format(
+                    debug.qenum_key(QWebEngineDownloadItem, state)))
+            return
+
+        try:
+            self._qt_item.resume()
+        except AttributeError:
+            raise downloads.UnsupportedOperationError(
+                "Retrying downloads is unsupported with QtWebEngine on "
+                "Qt/PyQt < 5.10")
 
     def _get_open_filename(self):
         return self._filename
@@ -125,6 +139,7 @@ class DownloadItem(downloads.AbstractDownloadItem):
         question = usertypes.Question()
         question.title = title
         question.text = msg
+        question.url = 'file://{}'.format(self._filename)
         question.mode = usertypes.PromptMode.yesno
         question.answered_yes.connect(self._after_set_filename)
         question.answered_no.connect(no_action)
@@ -139,6 +154,7 @@ class DownloadItem(downloads.AbstractDownloadItem):
         question = usertypes.Question()
         question.title = title
         question.text = msg
+        question.url = 'file://{}'.format(os.path.dirname(self._filename))
         question.mode = usertypes.PromptMode.yesno
         question.answered_yes.connect(lambda:
                                       self._after_create_parent_question(

qutebrowser/browser/webengine/webengineelem.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -27,7 +27,7 @@ from PyQt5.QtGui import QMouseEvent
 from PyQt5.QtWidgets import QApplication
 from PyQt5.QtWebEngineWidgets import QWebEngineSettings
 
-from qutebrowser.utils import log, javascript
+from qutebrowser.utils import log, javascript, urlutils
 from qutebrowser.browser import webelem
 
 
@@ -198,6 +198,13 @@ class WebEngineElement(webelem.AbstractWebElement):
         if self.is_text_input() and self.is_editable():
             self._js_call('move_cursor_to_end')
 
+    def _requires_user_interaction(self):
+        baseurl = self._tab.url()
+        url = self.resolve_url(baseurl)
+        if url is None:
+            return True
+        return url.scheme() not in urlutils.WEBENGINE_SCHEMES
+
     def _click_editable(self, click_target):
         # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-58515
         ev = QMouseEvent(QMouseEvent.MouseButtonPress, QPoint(0, 0),

qutebrowser/browser/webengine/webengineinspector.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -22,7 +22,7 @@
 import os
 
 from PyQt5.QtCore import QUrl
-from PyQt5.QtWebEngineWidgets import QWebEngineView
+from PyQt5.QtWebEngineWidgets import QWebEngineView, QWebEngineSettings
 
 from qutebrowser.browser import inspector
 
@@ -35,16 +35,31 @@ class WebEngineInspector(inspector.AbstractWebInspector):
         super().__init__(parent)
         self.port = None
         view = QWebEngineView()
+        settings = view.settings()
+        settings.setAttribute(QWebEngineSettings.JavascriptEnabled, True)
         self._set_widget(view)
 
-    def inspect(self, _page):
-        """Set up the inspector."""
+    def _inspect_old(self, page):
+        """Set up the inspector for Qt < 5.11."""
         try:
             port = int(os.environ['QTWEBENGINE_REMOTE_DEBUGGING'])
         except KeyError:
             raise inspector.WebInspectorError(
-                "Debugging is not enabled. See 'qutebrowser --help' for "
-                "details.")
+                "QtWebEngine inspector is not enabled. See "
+                "'qutebrowser --help' for details.")
         url = QUrl('http://localhost:{}/'.format(port))
-        self._widget.load(url)
-        self.show()
+
+        if page is None:
+            self._widget.load(QUrl('about:blank'))
+        else:
+            self._widget.load(url)
+
+    def _inspect_new(self, page):
+        """Set up the inspector for Qt >= 5.11."""
+        self._widget.page().setInspectedPage(page)
+
+    def inspect(self, page):
+        try:
+            self._inspect_new(page)
+        except AttributeError:
+            self._inspect_old(page)

qutebrowser/browser/webengine/webenginequtescheme.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -34,6 +34,10 @@ class QuteSchemeHandler(QWebEngineUrlSchemeHandler):
     def install(self, profile):
         """Install the handler for qute:// URLs on the given profile."""
         profile.installUrlSchemeHandler(b'qute', self)
+        if qtutils.version_check('5.11', compiled=False):
+            # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-63378
+            profile.installUrlSchemeHandler(b'chrome-error', self)
+            profile.installUrlSchemeHandler(b'chrome-extension', self)
 
     def requestStarted(self, job):
         """Handle a request for a qute: scheme.
@@ -45,6 +49,12 @@ class QuteSchemeHandler(QWebEngineUrlSchemeHandler):
             job: QWebEngineUrlRequestJob
         """
         url = job.requestUrl()
+
+        if url.scheme() in ['chrome-error', 'chrome-extension']:
+            # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-63378
+            job.fail(QWebEngineUrlRequestJob.UrlInvalid)
+            return
+
         assert job.requestMethod() == b'GET'
         assert url.scheme() == 'qute'
         log.misc.debug("Got request for {}".format(url.toDisplayString()))

qutebrowser/browser/webengine/webenginesettings.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -17,9 +17,6 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-# We get various "abstract but not overridden" warnings
-# pylint: disable=abstract-method
-
 """Bridge from QWebEngineSettings to our own settings.
 
 Module attributes:
@@ -29,369 +26,287 @@ Module attributes:
 
 import os
 
-import sip
 from PyQt5.QtGui import QFont
 from PyQt5.QtWebEngineWidgets import (QWebEngineSettings, QWebEngineProfile,
-                                      QWebEngineScript)
+                                      QWebEnginePage)
 
-from qutebrowser.browser import shared
 from qutebrowser.browser.webengine import spell
 from qutebrowser.config import config, websettings
-from qutebrowser.utils import (utils, standarddir, javascript, qtutils,
-                               message, log, objreg)
+from qutebrowser.config.websettings import AttributeInfo as Attr
+from qutebrowser.utils import utils, standarddir, qtutils, message, log
 
 # The default QWebEngineProfile
 default_profile = None
 # The QWebEngineProfile used for private (off-the-record) windows
 private_profile = None
+# The global WebEngineSettings object
+global_settings = None
 
 
-class Base(websettings.Base):
+class _SettingsWrapper:
 
-    """Base settings class with appropriate _get_global_settings."""
+    """Expose a QWebEngineSettings interface which acts on all profiles.
 
-    def _get_global_settings(self):
-        return [default_profile.settings(), private_profile.settings()]
-
-
-class Attribute(Base, websettings.Attribute):
-
-    """A setting set via QWebEngineSettings::setAttribute."""
-
-    ENUM_BASE = QWebEngineSettings
-
-
-class Setter(Base, websettings.Setter):
-
-    """A setting set via a QWebEngineSettings setter method."""
-
-    pass
-
-
-class FontFamilySetter(Base, websettings.FontFamilySetter):
-
-    """A setter for a font family.
-
-    Gets the default value from QFont.
+    For read operations, the default profile value is always used.
     """
 
-    def __init__(self, font):
-        # Mapping from WebEngineSettings::initDefaults in
-        # qtwebengine/src/core/web_engine_settings.cpp
-        font_to_qfont = {
-            QWebEngineSettings.StandardFont: QFont.Serif,
-            QWebEngineSettings.FixedFont: QFont.Monospace,
-            QWebEngineSettings.SerifFont: QFont.Serif,
-            QWebEngineSettings.SansSerifFont: QFont.SansSerif,
-            QWebEngineSettings.CursiveFont: QFont.Cursive,
-            QWebEngineSettings.FantasyFont: QFont.Fantasy,
+    def __init__(self):
+        self._settings = [default_profile.settings(),
+                          private_profile.settings()]
+
+    def setAttribute(self, *args, **kwargs):
+        for settings in self._settings:
+            settings.setAttribute(*args, **kwargs)
+
+    def setFontFamily(self, *args, **kwargs):
+        for settings in self._settings:
+            settings.setFontFamily(*args, **kwargs)
+
+    def setFontSize(self, *args, **kwargs):
+        for settings in self._settings:
+            settings.setFontSize(*args, **kwargs)
+
+    def setDefaultTextEncoding(self, *args, **kwargs):
+        for settings in self._settings:
+            settings.setDefaultTextEncoding(*args, **kwargs)
+
+    def testAttribute(self, *args, **kwargs):
+        return self._settings[0].testAttribute(*args, **kwargs)
+
+    def fontSize(self, *args, **kwargs):
+        return self._settings[0].fontSize(*args, **kwargs)
+
+    def fontFamily(self, *args, **kwargs):
+        return self._settings[0].fontFamily(*args, **kwargs)
+
+    def defaultTextEncoding(self, *args, **kwargs):
+        return self._settings[0].defaultTextEncoding(*args, **kwargs)
+
+
+class WebEngineSettings(websettings.AbstractSettings):
+
+    """A wrapper for the config for QWebEngineSettings."""
+
+    _ATTRIBUTES = {
+        'content.xss_auditing':
+            Attr(QWebEngineSettings.XSSAuditingEnabled),
+        'content.images':
+            Attr(QWebEngineSettings.AutoLoadImages),
+        'content.javascript.enabled':
+            Attr(QWebEngineSettings.JavascriptEnabled),
+        'content.javascript.can_open_tabs_automatically':
+            Attr(QWebEngineSettings.JavascriptCanOpenWindows),
+        'content.javascript.can_access_clipboard':
+            Attr(QWebEngineSettings.JavascriptCanAccessClipboard),
+        'content.plugins':
+            Attr(QWebEngineSettings.PluginsEnabled),
+        'content.hyperlink_auditing':
+            Attr(QWebEngineSettings.HyperlinkAuditingEnabled),
+        'content.local_content_can_access_remote_urls':
+            Attr(QWebEngineSettings.LocalContentCanAccessRemoteUrls),
+        'content.local_content_can_access_file_urls':
+            Attr(QWebEngineSettings.LocalContentCanAccessFileUrls),
+        'content.webgl':
+            Attr(QWebEngineSettings.WebGLEnabled),
+        'content.local_storage':
+            Attr(QWebEngineSettings.LocalStorageEnabled),
+        'content.desktop_capture':
+            Attr(QWebEngineSettings.ScreenCaptureEnabled,
+                 converter=lambda val: True if val == 'ask' else val),
+        # 'ask' is handled via the permission system,
+        # or a hardcoded dialog on Qt < 5.10
+
+        'input.spatial_navigation':
+            Attr(QWebEngineSettings.SpatialNavigationEnabled),
+        'input.links_included_in_focus_chain':
+            Attr(QWebEngineSettings.LinksIncludedInFocusChain),
+
+        'scrolling.smooth':
+            Attr(QWebEngineSettings.ScrollAnimatorEnabled),
+    }
+
+    _FONT_SIZES = {
+        'fonts.web.size.minimum':
+            QWebEngineSettings.MinimumFontSize,
+        'fonts.web.size.minimum_logical':
+            QWebEngineSettings.MinimumLogicalFontSize,
+        'fonts.web.size.default':
+            QWebEngineSettings.DefaultFontSize,
+        'fonts.web.size.default_fixed':
+            QWebEngineSettings.DefaultFixedFontSize,
+    }
+
+    _FONT_FAMILIES = {
+        'fonts.web.family.standard': QWebEngineSettings.StandardFont,
+        'fonts.web.family.fixed': QWebEngineSettings.FixedFont,
+        'fonts.web.family.serif': QWebEngineSettings.SerifFont,
+        'fonts.web.family.sans_serif': QWebEngineSettings.SansSerifFont,
+        'fonts.web.family.cursive': QWebEngineSettings.CursiveFont,
+        'fonts.web.family.fantasy': QWebEngineSettings.FantasyFont,
+    }
+
+    # Mapping from WebEngineSettings::initDefaults in
+    # qtwebengine/src/core/web_engine_settings.cpp
+    _FONT_TO_QFONT = {
+        QWebEngineSettings.StandardFont: QFont.Serif,
+        QWebEngineSettings.FixedFont: QFont.Monospace,
+        QWebEngineSettings.SerifFont: QFont.Serif,
+        QWebEngineSettings.SansSerifFont: QFont.SansSerif,
+        QWebEngineSettings.CursiveFont: QFont.Cursive,
+        QWebEngineSettings.FantasyFont: QFont.Fantasy,
+    }
+
+    def __init__(self, settings):
+        super().__init__(settings)
+        # Attributes which don't exist in all Qt versions.
+        new_attributes = {
+            # Qt 5.8
+            'content.print_element_backgrounds':
+                ('PrintElementBackgrounds', None),
+            # Qt 5.11
+            'content.autoplay':
+                ('PlaybackRequiresUserGesture', lambda val: not val),
         }
-        super().__init__(setter=QWebEngineSettings.setFontFamily, font=font,
-                         qfont=font_to_qfont[font])
+        for name, (attribute, converter) in new_attributes.items():
+            try:
+                value = getattr(QWebEngineSettings, attribute)
+            except AttributeError:
+                continue
+
+            self._ATTRIBUTES[name] = Attr(value, converter=converter)
 
 
-class DefaultProfileSetter(websettings.Base):
+class ProfileSetter:
 
-    """A setting set on the QWebEngineProfile."""
+    """Helper to set various settings on a profile."""
 
-    def __init__(self, setter, converter=None, default=websettings.UNSET):
-        super().__init__(default)
-        self._setter = setter
-        self._converter = converter
+    def __init__(self, profile):
+        self._profile = profile
 
-    def __repr__(self):
-        return utils.get_repr(self, setter=self._setter, constructor=True)
+    def init_profile(self):
+        """Initialize settings on the given profile."""
+        self.set_http_headers()
+        self.set_http_cache_size()
 
-    def _set(self, value, settings=None):
-        if settings is not None:
-            raise ValueError("'settings' may not be set with "
-                             "DefaultProfileSetters!")
+        settings = self._profile.settings()
+        settings.setAttribute(
+            QWebEngineSettings.FullScreenSupportEnabled, True)
+        try:
+            settings.setAttribute(
+                QWebEngineSettings.FocusOnNavigationEnabled, False)
+        except AttributeError:
+            # Added in Qt 5.8
+            pass
 
-        setter = getattr(default_profile, self._setter)
-        if self._converter is not None:
-            value = self._converter(value)
+        if qtutils.version_check('5.8'):
+            self.set_dictionary_language()
 
-        setter(value)
+    def set_http_headers(self):
+        """Set the user agent and accept-language for the given profile.
 
+        We override those per request in the URL interceptor (to allow for
+        per-domain values), but this one still gets used for things like
+        window.navigator.userAgent/.languages in JS.
+        """
+        self._profile.setHttpUserAgent(config.val.content.headers.user_agent)
+        accept_language = config.val.content.headers.accept_language
+        if accept_language is not None:
+            self._profile.setHttpAcceptLanguage(accept_language)
 
-class PersistentCookiePolicy(DefaultProfileSetter):
+    def set_http_cache_size(self):
+        """Initialize the HTTP cache size for the given profile."""
+        size = config.val.content.cache.size
+        if size is None:
+            size = 0
+        else:
+            size = qtutils.check_overflow(size, 'int', fatal=False)
 
-    """The content.cookies.store setting is different from other settings."""
+        # 0: automatically managed by QtWebEngine
+        self._profile.setHttpCacheMaximumSize(size)
 
-    def __init__(self):
-        super().__init__('setPersistentCookiesPolicy')
+    def set_persistent_cookie_policy(self):
+        """Set the HTTP Cookie size for the given profile."""
+        assert not self._profile.isOffTheRecord()
+        if config.val.content.cookies.store:
+            value = QWebEngineProfile.AllowPersistentCookies
+        else:
+            value = QWebEngineProfile.NoPersistentCookies
+        self._profile.setPersistentCookiesPolicy(value)
 
-    def _set(self, value, settings=None):
-        if settings is not None:
-            raise ValueError("'settings' may not be set with "
-                             "PersistentCookiePolicy!")
-        setter = getattr(QWebEngineProfile.defaultProfile(), self._setter)
-        setter(
-            QWebEngineProfile.AllowPersistentCookies if value else
-            QWebEngineProfile.NoPersistentCookies
-        )
+    def set_dictionary_language(self, warn=True):
+        """Load the given dictionaries."""
+        filenames = []
+        for code in config.val.spellcheck.languages or []:
+            local_filename = spell.local_filename(code)
+            if not local_filename:
+                if warn:
+                    message.warning("Language {} is not installed - see "
+                                    "scripts/dictcli.py in qutebrowser's "
+                                    "sources".format(code))
+                continue
 
+            filenames.append(local_filename)
 
-class DictionaryLanguageSetter(DefaultProfileSetter):
-
-    """Sets paths to dictionary files based on language codes."""
-
-    def __init__(self):
-        super().__init__('setSpellCheckLanguages', default=[])
-
-    def _find_installed(self, code):
-        local_filename = spell.local_filename(code)
-        if not local_filename:
-            message.warning(
-                "Language {} is not installed - see scripts/dictcli.py "
-                "in qutebrowser's sources".format(code))
-        return local_filename
-
-    def _set(self, value, settings=None):
-        if settings is not None:
-            raise ValueError("'settings' may not be set with "
-                             "DictionaryLanguageSetter!")
-        filenames = [self._find_installed(code) for code in value]
         log.config.debug("Found dicts: {}".format(filenames))
-        super()._set([f for f in filenames if f], settings)
-
-
-def _init_stylesheet(profile):
-    """Initialize custom stylesheets.
-
-    Partially inspired by QupZilla:
-    https://github.com/QupZilla/qupzilla/blob/v2.0/src/lib/app/mainapplication.cpp#L1063-L1101
-    """
-    old_script = profile.scripts().findScript('_qute_stylesheet')
-    if not old_script.isNull():
-        profile.scripts().remove(old_script)
-
-    css = shared.get_user_stylesheet()
-    source = '\n'.join([
-        '"use strict";',
-        'window._qutebrowser = window._qutebrowser || {};',
-        utils.read_file('javascript/stylesheet.js'),
-        javascript.assemble('stylesheet', 'set_css', css),
-    ])
-
-    script = QWebEngineScript()
-    script.setName('_qute_stylesheet')
-    script.setInjectionPoint(QWebEngineScript.DocumentCreation)
-    script.setWorldId(QWebEngineScript.ApplicationWorld)
-    script.setRunsOnSubFrames(True)
-    script.setSourceCode(source)
-    profile.scripts().insert(script)
-
-
-def _update_stylesheet():
-    """Update the custom stylesheet in existing tabs."""
-    css = shared.get_user_stylesheet()
-    code = javascript.assemble('stylesheet', 'set_css', css)
-    for win_id, window in objreg.window_registry.items():
-        # We could be in the middle of destroying a window here
-        if sip.isdeleted(window):
-            continue
-        tab_registry = objreg.get('tab-registry', scope='window',
-                                  window=win_id)
-        for tab in tab_registry.values():
-            tab.run_js_async(code)
-
-
-def _set_http_headers(profile):
-    """Set the user agent and accept-language for the given profile.
-
-    We override those per request in the URL interceptor (to allow for
-    per-domain values), but this one still gets used for things like
-    window.navigator.userAgent/.languages in JS.
-    """
-    profile.setHttpUserAgent(config.val.content.headers.user_agent)
-    accept_language = config.val.content.headers.accept_language
-    if accept_language is not None:
-        profile.setHttpAcceptLanguage(accept_language)
+        self._profile.setSpellCheckLanguages(filenames)
+        self._profile.setSpellCheckEnabled(bool(filenames))
 
 
 def _update_settings(option):
     """Update global settings when qwebsettings changed."""
-    websettings.update_mappings(MAPPINGS, option)
-    if option in ['scrolling.bar', 'content.user_stylesheets']:
-        _init_stylesheet(default_profile)
-        _init_stylesheet(private_profile)
-        _update_stylesheet()
-    elif option in ['content.headers.user_agent',
-                    'content.headers.accept_language']:
-        _set_http_headers(default_profile)
-        _set_http_headers(private_profile)
+    global_settings.update_setting(option)
+
+    if option in ['content.headers.user_agent',
+                  'content.headers.accept_language']:
+        default_profile.setter.set_http_headers()
+        private_profile.setter.set_http_headers()
+    elif option == 'content.cache.size':
+        default_profile.setter.set_http_cache_size()
+        private_profile.setter.set_http_cache_size()
+    elif (option == 'content.cookies.store' and
+          # https://bugreports.qt.io/browse/QTBUG-58650
+          qtutils.version_check('5.9', compiled=False)):
+        default_profile.setter.set_persistent_cookie_policy()
+        # We're not touching the private profile's cookie policy.
+    elif option == 'spellcheck.languages':
+        default_profile.setter.set_dictionary_language()
+        private_profile.setter.set_dictionary_language(warn=False)
 
 
 def _init_profiles():
     """Init the two used QWebEngineProfiles."""
     global default_profile, private_profile
+
     default_profile = QWebEngineProfile.defaultProfile()
+    default_profile.setter = ProfileSetter(default_profile)
     default_profile.setCachePath(
         os.path.join(standarddir.cache(), 'webengine'))
     default_profile.setPersistentStoragePath(
         os.path.join(standarddir.data(), 'webengine'))
-    _init_stylesheet(default_profile)
-    _set_http_headers(default_profile)
+    default_profile.setter.init_profile()
+    default_profile.setter.set_persistent_cookie_policy()
 
     private_profile = QWebEngineProfile()
+    private_profile.setter = ProfileSetter(private_profile)
     assert private_profile.isOffTheRecord()
-    _init_stylesheet(private_profile)
-    _set_http_headers(private_profile)
-
-    if qtutils.version_check('5.8'):
-        default_profile.setSpellCheckEnabled(True)
-        private_profile.setSpellCheckEnabled(True)
-
-
-def inject_userscripts():
-    """Register user JavaScript files with the global profiles."""
-    # The Greasemonkey metadata block support in QtWebEngine only starts at
-    # Qt 5.8. With 5.7.1, we need to inject the scripts ourselves in response
-    # to urlChanged.
-    if not qtutils.version_check('5.8'):
-        return
-
-    # Since we are inserting scripts into profile.scripts they won't
-    # just get replaced by new gm scripts like if we were injecting them
-    # ourselves so we need to remove all gm scripts, while not removing
-    # any other stuff that might have been added. Like the one for
-    # stylesheets.
-    greasemonkey = objreg.get('greasemonkey')
-    for profile in [default_profile, private_profile]:
-        scripts = profile.scripts()
-        for script in scripts.toList():
-            if script.name().startswith("GM-"):
-                log.greasemonkey.debug('Removing script: {}'
-                                       .format(script.name()))
-                removed = scripts.remove(script)
-                assert removed, script.name()
-
-        # Then add the new scripts.
-        for script in greasemonkey.all_scripts():
-            # @run-at (and @include/@exclude/@match) is parsed by
-            # QWebEngineScript.
-            new_script = QWebEngineScript()
-            new_script.setWorldId(QWebEngineScript.MainWorld)
-            new_script.setSourceCode(script.code())
-            new_script.setName("GM-{}".format(script.name))
-            new_script.setRunsOnSubFrames(script.runs_on_sub_frames)
-            log.greasemonkey.debug('adding script: {}'
-                                   .format(new_script.name()))
-            scripts.insert(new_script)
+    private_profile.setter.init_profile()
 
 
 def init(args):
     """Initialize the global QWebSettings."""
-    if args.enable_webengine_inspector:
+    if (args.enable_webengine_inspector and
+            not hasattr(QWebEnginePage, 'setInspectedPage')):  # only Qt < 5.11
         os.environ['QTWEBENGINE_REMOTE_DEBUGGING'] = str(utils.random_port())
 
+    spell.init()
+
     _init_profiles()
-
-    # We need to do this here as a WORKAROUND for
-    # https://bugreports.qt.io/browse/QTBUG-58650
-    if not qtutils.version_check('5.9', compiled=False):
-        PersistentCookiePolicy().set(config.val.content.cookies.store)
-    Attribute(QWebEngineSettings.FullScreenSupportEnabled).set(True)
-
-    websettings.init_mappings(MAPPINGS)
     config.instance.changed.connect(_update_settings)
 
+    global global_settings
+    global_settings = WebEngineSettings(_SettingsWrapper())
+    global_settings.init_settings()
+
 
 def shutdown():
-    # FIXME:qtwebengine do we need to do something for a clean shutdown here?
     pass
-
-
-# Missing QtWebEngine attributes:
-# - ScreenCaptureEnabled
-# - Accelerated2dCanvasEnabled
-# - AutoLoadIconsForPage
-# - TouchIconsEnabled
-# - FocusOnNavigationEnabled (5.8)
-# - AllowRunningInsecureContent (5.8)
-#
-# Missing QtWebEngine fonts:
-# - PictographFont
-
-
-MAPPINGS = {
-    'content.images':
-        Attribute(QWebEngineSettings.AutoLoadImages),
-    'content.javascript.enabled':
-        Attribute(QWebEngineSettings.JavascriptEnabled),
-    'content.javascript.can_open_tabs_automatically':
-        Attribute(QWebEngineSettings.JavascriptCanOpenWindows),
-    'content.javascript.can_access_clipboard':
-        Attribute(QWebEngineSettings.JavascriptCanAccessClipboard),
-    'content.plugins':
-        Attribute(QWebEngineSettings.PluginsEnabled),
-    'content.hyperlink_auditing':
-        Attribute(QWebEngineSettings.HyperlinkAuditingEnabled),
-    'content.local_content_can_access_remote_urls':
-        Attribute(QWebEngineSettings.LocalContentCanAccessRemoteUrls),
-    'content.local_content_can_access_file_urls':
-        Attribute(QWebEngineSettings.LocalContentCanAccessFileUrls),
-    'content.webgl':
-        Attribute(QWebEngineSettings.WebGLEnabled),
-    'content.local_storage':
-        Attribute(QWebEngineSettings.LocalStorageEnabled),
-    'content.cache.size':
-        # 0: automatically managed by QtWebEngine
-        DefaultProfileSetter('setHttpCacheMaximumSize', default=0,
-                             converter=lambda val:
-                             qtutils.check_overflow(val, 'int', fatal=False)),
-    'content.xss_auditing':
-        Attribute(QWebEngineSettings.XSSAuditingEnabled),
-    'content.default_encoding':
-        Setter(QWebEngineSettings.setDefaultTextEncoding),
-
-    'input.spatial_navigation':
-        Attribute(QWebEngineSettings.SpatialNavigationEnabled),
-    'input.links_included_in_focus_chain':
-        Attribute(QWebEngineSettings.LinksIncludedInFocusChain),
-
-    'fonts.web.family.standard':
-        FontFamilySetter(QWebEngineSettings.StandardFont),
-    'fonts.web.family.fixed':
-        FontFamilySetter(QWebEngineSettings.FixedFont),
-    'fonts.web.family.serif':
-        FontFamilySetter(QWebEngineSettings.SerifFont),
-    'fonts.web.family.sans_serif':
-        FontFamilySetter(QWebEngineSettings.SansSerifFont),
-    'fonts.web.family.cursive':
-        FontFamilySetter(QWebEngineSettings.CursiveFont),
-    'fonts.web.family.fantasy':
-        FontFamilySetter(QWebEngineSettings.FantasyFont),
-    'fonts.web.size.minimum':
-        Setter(QWebEngineSettings.setFontSize,
-               args=[QWebEngineSettings.MinimumFontSize]),
-    'fonts.web.size.minimum_logical':
-        Setter(QWebEngineSettings.setFontSize,
-               args=[QWebEngineSettings.MinimumLogicalFontSize]),
-    'fonts.web.size.default':
-        Setter(QWebEngineSettings.setFontSize,
-               args=[QWebEngineSettings.DefaultFontSize]),
-    'fonts.web.size.default_fixed':
-        Setter(QWebEngineSettings.setFontSize,
-               args=[QWebEngineSettings.DefaultFixedFontSize]),
-
-    'scrolling.smooth':
-        Attribute(QWebEngineSettings.ScrollAnimatorEnabled),
-}
-
-try:
-    MAPPINGS['content.print_element_backgrounds'] = Attribute(
-        QWebEngineSettings.PrintElementBackgrounds)
-except AttributeError:
-    # Added in Qt 5.8
-    pass
-
-
-if qtutils.version_check('5.8'):
-    MAPPINGS['spellcheck.languages'] = DictionaryLanguageSetter()
-
-
-if qtutils.version_check('5.9', compiled=False):
-    # https://bugreports.qt.io/browse/QTBUG-58650
-    MAPPINGS['content.cookies.store'] = PersistentCookiePolicy()

qutebrowser/browser/webengine/webenginetab.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -21,24 +21,27 @@
 
 import math
 import functools
+import sys
+import re
 import html as html_utils
 
-import sip
 from PyQt5.QtCore import (pyqtSignal, pyqtSlot, Qt, QEvent, QPoint, QPointF,
-                          QUrl, QTimer)
-from PyQt5.QtGui import QKeyEvent
+                          QUrl, QTimer, QObject, qVersion)
+from PyQt5.QtGui import QKeyEvent, QIcon
 from PyQt5.QtNetwork import QAuthenticator
 from PyQt5.QtWidgets import QApplication
 from PyQt5.QtWebEngineWidgets import QWebEnginePage, QWebEngineScript
 
+from qutebrowser.config import configdata, config
 from qutebrowser.browser import browsertab, mouse, shared
 from qutebrowser.browser.webengine import (webview, webengineelem, tabhistory,
                                            interceptor, webenginequtescheme,
-                                           webenginedownloads,
-                                           webenginesettings)
+                                           cookies, webenginedownloads,
+                                           webenginesettings, certificateerror)
 from qutebrowser.misc import miscwidgets
 from qutebrowser.utils import (usertypes, qtutils, log, javascript, utils,
                                message, objreg, jinja, debug)
+from qutebrowser.qt import sip
 
 
 _qute_scheme_handler = None
@@ -59,8 +62,9 @@ def init():
 
     log.init.debug("Initializing request interceptor...")
     host_blocker = objreg.get('host-blocker')
+    args = objreg.get('args')
     req_interceptor = interceptor.RequestInterceptor(
-        host_blocker, parent=app)
+        host_blocker, args=args, parent=app)
     req_interceptor.install(webenginesettings.default_profile)
     req_interceptor.install(webenginesettings.private_profile)
 
@@ -70,9 +74,17 @@ def init():
     download_manager.install(webenginesettings.private_profile)
     objreg.register('webengine-download-manager', download_manager)
 
-    greasemonkey = objreg.get('greasemonkey')
-    greasemonkey.scripts_reloaded.connect(webenginesettings.inject_userscripts)
-    webenginesettings.inject_userscripts()
+    log.init.debug("Initializing cookie filter...")
+    cookies.install_filter(webenginesettings.default_profile)
+    cookies.install_filter(webenginesettings.private_profile)
+
+    # Clear visited links on web history clear
+    hist = objreg.get('web-history')
+    for p in [webenginesettings.default_profile,
+              webenginesettings.private_profile]:
+        hist.history_cleared.connect(p.clearAllVisitedLinks)
+        hist.url_cleared.connect(lambda url, profile=p:
+                                 profile.clearVisitedLinks([url]))
 
 
 # Mapping worlds from usertypes.JsWorld to QWebEngineScript world IDs.
@@ -98,6 +110,23 @@ class WebEngineAction(browsertab.AbstractAction):
         """Save the current page."""
         self._widget.triggerPageAction(QWebEnginePage.SavePage)
 
+    def show_source(self, pygments=False):
+        if pygments:
+            self._show_source_pygments()
+            return
+
+        try:
+            self._widget.triggerPageAction(QWebEnginePage.ViewSource)
+        except AttributeError:
+            # Qt < 5.8
+            tb = objreg.get('tabbed-browser', scope='window',
+                            window=self._tab.win_id)
+            urlstr = self._tab.url().toString(QUrl.RemoveUserInfo)
+            # The original URL becomes the path of a view-source: URL
+            # (without a host), but query/fragment should stay.
+            url = QUrl('view-source:' + urlstr)
+            tb.tabopen(url, background=False, related=True)
+
 
 class WebEnginePrinting(browsertab.AbstractPrinting):
 
@@ -169,6 +198,12 @@ class WebEngineSearch(browsertab.AbstractSearch):
 
     def search(self, text, *, ignore_case='never', reverse=False,
                result_cb=None):
+        # Don't go to next entry on duplicate search
+        if self.text == text and self.search_displayed:
+            log.webview.debug("Ignoring duplicate search request"
+                              " for {}".format(text))
+            return
+
         self.text = text
         self._flags = QWebEnginePage.FindFlags(0)
         if self._is_case_sensitive(ignore_case):
@@ -201,70 +236,104 @@ class WebEngineCaret(browsertab.AbstractCaret):
 
     @pyqtSlot(usertypes.KeyMode)
     def _on_mode_entered(self, mode):
-        pass
+        if mode != usertypes.KeyMode.caret:
+            return
+
+        if self._tab.search.search_displayed:
+            # We are currently in search mode.
+            # convert the search to a blue selection so we can operate on it
+            # https://bugreports.qt.io/browse/QTBUG-60673
+            self._tab.search.clear()
+
+        self._tab.run_js_async(
+            javascript.assemble('caret',
+                                'setPlatform', sys.platform, qVersion()))
+        self._js_call('setInitialCursor', self._selection_cb)
+
+    def _selection_cb(self, enabled):
+        """Emit selection_toggled based on setInitialCursor."""
+        if enabled is None:
+            log.webview.debug("Ignoring selection status None")
+            return
+        self.selection_toggled.emit(enabled)
 
     @pyqtSlot(usertypes.KeyMode)
-    def _on_mode_left(self):
-        pass
+    def _on_mode_left(self, mode):
+        if mode != usertypes.KeyMode.caret:
+            return
+
+        self.drop_selection()
+        self._js_call('disableCaret')
 
     def move_to_next_line(self, count=1):
-        log.stub()
+        for _ in range(count):
+            self._js_call('moveDown')
 
     def move_to_prev_line(self, count=1):
-        log.stub()
+        for _ in range(count):
+            self._js_call('moveUp')
 
     def move_to_next_char(self, count=1):
-        log.stub()
+        for _ in range(count):
+            self._js_call('moveRight')
 
     def move_to_prev_char(self, count=1):
-        log.stub()
+        for _ in range(count):
+            self._js_call('moveLeft')
 
     def move_to_end_of_word(self, count=1):
-        log.stub()
+        for _ in range(count):
+            self._js_call('moveToEndOfWord')
 
     def move_to_next_word(self, count=1):
-        log.stub()
+        for _ in range(count):
+            self._js_call('moveToNextWord')
 
     def move_to_prev_word(self, count=1):
-        log.stub()
+        for _ in range(count):
+            self._js_call('moveToPreviousWord')
 
     def move_to_start_of_line(self):
-        log.stub()
+        self._js_call('moveToStartOfLine')
 
     def move_to_end_of_line(self):
-        log.stub()
+        self._js_call('moveToEndOfLine')
 
     def move_to_start_of_next_block(self, count=1):
-        log.stub()
+        for _ in range(count):
+            self._js_call('moveToStartOfNextBlock')
 
     def move_to_start_of_prev_block(self, count=1):
-        log.stub()
+        for _ in range(count):
+            self._js_call('moveToStartOfPrevBlock')
 
     def move_to_end_of_next_block(self, count=1):
-        log.stub()
+        for _ in range(count):
+            self._js_call('moveToEndOfNextBlock')
 
     def move_to_end_of_prev_block(self, count=1):
-        log.stub()
+        for _ in range(count):
+            self._js_call('moveToEndOfPrevBlock')
 
     def move_to_start_of_document(self):
-        log.stub()
+        self._js_call('moveToStartOfDocument')
 
     def move_to_end_of_document(self):
-        log.stub()
+        self._js_call('moveToEndOfDocument')
 
     def toggle_selection(self):
-        log.stub()
+        self._js_call('toggleSelection', self.selection_toggled.emit)
 
     def drop_selection(self):
-        log.stub()
+        self._js_call('dropSelection')
 
-    def has_selection(self):
-        return self._widget.hasSelection()
-
-    def selection(self, html=False):
-        if html:
-            raise browsertab.UnsupportedOperationError
-        return self._widget.selectedText()
+    def selection(self, callback):
+        # Not using selectedText() as WORKAROUND for
+        # https://bugreports.qt.io/browse/QTBUG-53134
+        # Even on Qt 5.10 selectedText() seems to work poorly, see
+        # https://github.com/qutebrowser/qutebrowser/issues/3523
+        self._tab.run_js_async(javascript.assemble('caret', 'getSelection'),
+                               callback)
 
     def _follow_selected_cb(self, js_elem, tab=False):
         """Callback for javascript which clicks the selected element.
@@ -275,6 +344,11 @@ class WebEngineCaret(browsertab.AbstractCaret):
         """
         if js_elem is None:
             return
+        if js_elem == "focused":
+            # we had a focused element, not a selected one. Just send <enter>
+            self._follow_enter(tab)
+            return
+
         assert isinstance(js_elem, dict), js_elem
         elem = webengineelem.WebEngineElement(js_elem, tab=self._tab)
         if tab:
@@ -297,17 +371,17 @@ class WebEngineCaret(browsertab.AbstractCaret):
 
             log.webview.debug("Clicking a searched link via fake key press.")
             # send a fake enter, clicking the orange selection box
-            if tab:
-                self._tab.key_press(Qt.Key_Enter, modifier=Qt.ControlModifier)
-            else:
-                self._tab.key_press(Qt.Key_Enter)
-
+            self._follow_enter(tab)
         else:
             # click an existing blue selection
-            js_code = javascript.assemble('webelem', 'find_selected_link')
+            js_code = javascript.assemble('webelem',
+                                          'find_selected_focused_link')
             self._tab.run_js_async(js_code, lambda jsret:
                                    self._follow_selected_cb(jsret, tab))
 
+    def _js_call(self, command, callback=None):
+        self._tab.run_js_async(javascript.assemble('caret', command), callback)
+
 
 class WebEngineScroller(browsertab.AbstractScroller):
 
@@ -327,7 +401,7 @@ class WebEngineScroller(browsertab.AbstractScroller):
 
     def _repeated_key_press(self, key, count=1, modifier=Qt.NoModifier):
         """Send count fake key presses to this scroller's WebEngineTab."""
-        for _ in range(min(count, 5000)):
+        for _ in range(min(count, 1000)):
             self._tab.key_press(key, modifier)
 
     @pyqtSlot(QPointF)
@@ -380,6 +454,11 @@ class WebEngineScroller(browsertab.AbstractScroller):
         js_code = javascript.assemble('window', 'scroll', point.x(), point.y())
         self._tab.run_js_async(js_code)
 
+    def to_anchor(self, name):
+        url = self._tab.url()
+        url.setFragment(name)
+        self._tab.openurl(url)
+
     def delta(self, x=0, y=0):
         self._tab.run_js_async(javascript.assemble('window', 'scrollBy', x, y))
 
@@ -435,7 +514,8 @@ class WebEngineHistory(browsertab.AbstractHistory):
         return self._history.itemAt(i)
 
     def _go_to_item(self, item):
-        return self._history.goToItem(item)
+        self._tab.predicted_navigation.emit(item.url())
+        self._history.goToItem(item)
 
     def serialize(self):
         if not qtutils.version_check('5.9', compiled=False):
@@ -453,15 +533,23 @@ class WebEngineHistory(browsertab.AbstractHistory):
         return qtutils.deserialize(data, self._history)
 
     def load_items(self, items):
+        if items:
+            self._tab.predicted_navigation.emit(items[-1].url)
+
         stream, _data, cur_data = tabhistory.serialize(items)
         qtutils.deserialize_stream(stream, self._history)
+
+        @pyqtSlot()
+        def _on_load_finished():
+            self._tab.scroller.to_point(cur_data['scroll-pos'])
+            self._tab.load_finished.disconnect(_on_load_finished)
+
         if cur_data is not None:
             if 'zoom' in cur_data:
                 self._tab.zoom.set_factor(cur_data['zoom'])
             if ('scroll-pos' in cur_data and
                     self._tab.scroller.pos_px() == QPoint(0, 0)):
-                QTimer.singleShot(0, functools.partial(
-                    self._tab.scroller.to_point, cur_data['scroll-pos']))
+                self._tab.load_finished.connect(_on_load_finished)
 
 
 class WebEngineZoom(browsertab.AbstractZoom):
@@ -538,6 +626,333 @@ class WebEngineElements(browsertab.AbstractElements):
         self._tab.run_js_async(js_code, js_cb)
 
 
+class WebEngineAudio(browsertab.AbstractAudio):
+
+    """QtWebEngine implemementations related to audio/muting."""
+
+    def _connect_signals(self):
+        page = self._widget.page()
+        page.audioMutedChanged.connect(self.muted_changed)
+        page.recentlyAudibleChanged.connect(self.recently_audible_changed)
+
+    def set_muted(self, muted: bool):
+        page = self._widget.page()
+        page.setAudioMuted(muted)
+
+    def is_muted(self):
+        page = self._widget.page()
+        return page.isAudioMuted()
+
+    def is_recently_audible(self):
+        page = self._widget.page()
+        return page.recentlyAudible()
+
+
+class _WebEnginePermissions(QObject):
+
+    """Handling of various permission-related signals."""
+
+    _abort_questions = pyqtSignal()
+
+    def __init__(self, tab, parent=None):
+        super().__init__(parent)
+        self._tab = tab
+        self._widget = None
+
+    def connect_signals(self):
+        """Connect related signals from the QWebEnginePage."""
+        page = self._widget.page()
+        page.fullScreenRequested.connect(
+            self._on_fullscreen_requested)
+        page.featurePermissionRequested.connect(
+            self._on_feature_permission_requested)
+
+        if qtutils.version_check('5.11'):
+            page.quotaRequested.connect(
+                self._on_quota_requested)
+            page.registerProtocolHandlerRequested.connect(
+                self._on_register_protocol_handler_requested)
+
+        self._tab.shutting_down.connect(self._abort_questions)
+        self._tab.load_started.connect(self._abort_questions)
+
+    @pyqtSlot('QWebEngineFullScreenRequest')
+    def _on_fullscreen_requested(self, request):
+        request.accept()
+        on = request.toggleOn()
+
+        self._tab.data.fullscreen = on
+        self._tab.fullscreen_requested.emit(on)
+        if on:
+            notification = miscwidgets.FullscreenNotification(self._widget)
+            notification.show()
+            notification.set_timeout(3000)
+
+    @pyqtSlot(QUrl, 'QWebEnginePage::Feature')
+    def _on_feature_permission_requested(self, url, feature):
+        """Ask the user for approval for geolocation/media/etc.."""
+        options = {
+            QWebEnginePage.Geolocation: 'content.geolocation',
+            QWebEnginePage.MediaAudioCapture: 'content.media_capture',
+            QWebEnginePage.MediaVideoCapture: 'content.media_capture',
+            QWebEnginePage.MediaAudioVideoCapture: 'content.media_capture',
+        }
+        messages = {
+            QWebEnginePage.Geolocation: 'access your location',
+            QWebEnginePage.MediaAudioCapture: 'record audio',
+            QWebEnginePage.MediaVideoCapture: 'record video',
+            QWebEnginePage.MediaAudioVideoCapture: 'record audio/video',
+        }
+        try:
+            options.update({
+                QWebEnginePage.DesktopVideoCapture:
+                    'content.desktop_capture',
+                QWebEnginePage.DesktopAudioVideoCapture:
+                    'content.desktop_capture',
+            })
+            messages.update({
+                QWebEnginePage.DesktopVideoCapture:
+                    'capture your desktop',
+                QWebEnginePage.DesktopAudioVideoCapture:
+                    'capture your desktop and audio',
+            })
+        except AttributeError:
+            # Added in Qt 5.10
+            pass
+
+        assert options.keys() == messages.keys()
+
+        page = self._widget.page()
+
+        if feature not in options:
+            log.webview.error("Unhandled feature permission {}".format(
+                debug.qenum_key(QWebEnginePage, feature)))
+            page.setFeaturePermission(url, feature,
+                                      QWebEnginePage.PermissionDeniedByUser)
+            return
+
+        yes_action = functools.partial(
+            page.setFeaturePermission, url, feature,
+            QWebEnginePage.PermissionGrantedByUser)
+        no_action = functools.partial(
+            page.setFeaturePermission, url, feature,
+            QWebEnginePage.PermissionDeniedByUser)
+
+        question = shared.feature_permission(
+            url=url, option=options[feature], msg=messages[feature],
+            yes_action=yes_action, no_action=no_action,
+            abort_on=[self._abort_questions])
+
+        if question is not None:
+            page.featurePermissionRequestCanceled.connect(
+                functools.partial(self._on_feature_permission_cancelled,
+                                  question, url, feature))
+
+    def _on_feature_permission_cancelled(self, question, url, feature,
+                                         cancelled_url, cancelled_feature):
+        """Slot invoked when a feature permission request was cancelled.
+
+        To be used with functools.partial.
+        """
+        if url == cancelled_url and feature == cancelled_feature:
+            try:
+                question.abort()
+            except RuntimeError:
+                # The question could already be deleted, e.g. because it was
+                # aborted after a loadStarted signal.
+                pass
+
+    def _on_quota_requested(self, request):
+        size = utils.format_size(request.requestedSize())
+        shared.feature_permission(
+            url=request.origin(),
+            option='content.persistent_storage',
+            msg='use {} of persistent storage'.format(size),
+            yes_action=request.accept, no_action=request.reject,
+            abort_on=[self._abort_questions],
+            blocking=True)
+
+    def _on_register_protocol_handler_requested(self, request):
+        shared.feature_permission(
+            url=request.origin(),
+            option='content.register_protocol_handler',
+            msg='open all {} links'.format(request.scheme()),
+            yes_action=request.accept, no_action=request.reject,
+            abort_on=[self._abort_questions],
+            blocking=True)
+
+
+class _WebEngineScripts(QObject):
+
+    def __init__(self, tab, parent=None):
+        super().__init__(parent)
+        self._tab = tab
+        self._widget = None
+        self._greasemonkey = objreg.get('greasemonkey')
+
+    def connect_signals(self):
+        config.instance.changed.connect(self._on_config_changed)
+
+    @pyqtSlot(str)
+    def _on_config_changed(self, option):
+        if option in ['scrolling.bar', 'content.user_stylesheets']:
+            self._init_stylesheet()
+            self._update_stylesheet()
+
+    def _update_stylesheet(self):
+        """Update the custom stylesheet in existing tabs."""
+        css = shared.get_user_stylesheet()
+        code = javascript.assemble('stylesheet', 'set_css', css)
+        self._tab.run_js_async(code)
+
+    def _inject_early_js(self, name, js_code, *,
+                         world=QWebEngineScript.ApplicationWorld,
+                         subframes=False):
+        """Inject the given script to run early on a page load.
+
+        This runs the script both on DocumentCreation and DocumentReady as on
+        some internal pages, DocumentCreation will not work.
+
+        That is a WORKAROUND for https://bugreports.qt.io/browse/QTBUG-66011
+        """
+        scripts = self._widget.page().scripts()
+        for injection in ['creation', 'ready']:
+            injection_points = {
+                'creation': QWebEngineScript.DocumentCreation,
+                'ready': QWebEngineScript.DocumentReady,
+            }
+            script = QWebEngineScript()
+            script.setInjectionPoint(injection_points[injection])
+            script.setSourceCode(js_code)
+            script.setWorldId(world)
+            script.setRunsOnSubFrames(subframes)
+            script.setName('_qute_{}_{}'.format(name, injection))
+            scripts.insert(script)
+
+    def _remove_early_js(self, name):
+        """Remove an early QWebEngineScript."""
+        scripts = self._widget.page().scripts()
+        for injection in ['creation', 'ready']:
+            full_name = '_qute_{}_{}'.format(name, injection)
+            script = scripts.findScript(full_name)
+            if not script.isNull():
+                scripts.remove(script)
+
+    def init(self):
+        """Initialize global qutebrowser JavaScript."""
+        js_code = javascript.wrap_global(
+            'scripts',
+            utils.read_file('javascript/scroll.js'),
+            utils.read_file('javascript/webelem.js'),
+            utils.read_file('javascript/caret.js'),
+        )
+        self._inject_early_js('js',
+                              utils.read_file('javascript/print.js'),
+                              subframes=True,
+                              world=QWebEngineScript.MainWorld)
+        # FIXME:qtwebengine what about subframes=True?
+        self._inject_early_js('js', js_code, subframes=True)
+        self._init_stylesheet()
+
+        # The Greasemonkey metadata block support in QtWebEngine only starts at
+        # Qt 5.8. With 5.7.1, we need to inject the scripts ourselves in
+        # response to urlChanged.
+        if not qtutils.version_check('5.8'):
+            self._tab.url_changed.connect(
+                self._inject_greasemonkey_scripts_for_url)
+        else:
+            self._greasemonkey.scripts_reloaded.connect(
+                self._inject_all_greasemonkey_scripts)
+            self._inject_all_greasemonkey_scripts()
+
+    def _init_stylesheet(self):
+        """Initialize custom stylesheets.
+
+        Partially inspired by QupZilla:
+        https://github.com/QupZilla/qupzilla/blob/v2.0/src/lib/app/mainapplication.cpp#L1063-L1101
+        """
+        self._remove_early_js('stylesheet')
+        css = shared.get_user_stylesheet()
+        js_code = javascript.wrap_global(
+            'stylesheet',
+            utils.read_file('javascript/stylesheet.js'),
+            javascript.assemble('stylesheet', 'set_css', css),
+        )
+        self._inject_early_js('stylesheet', js_code, subframes=True)
+
+    @pyqtSlot(QUrl)
+    def _inject_greasemonkey_scripts_for_url(self, url):
+        matching_scripts = self._greasemonkey.scripts_for(url)
+        self._inject_greasemonkey_scripts(
+            matching_scripts.start, QWebEngineScript.DocumentCreation, True)
+        self._inject_greasemonkey_scripts(
+            matching_scripts.end, QWebEngineScript.DocumentReady, False)
+        self._inject_greasemonkey_scripts(
+            matching_scripts.idle, QWebEngineScript.Deferred, False)
+
+    @pyqtSlot()
+    def _inject_all_greasemonkey_scripts(self):
+        scripts = self._greasemonkey.all_scripts()
+        self._inject_greasemonkey_scripts(scripts)
+
+    def _inject_greasemonkey_scripts(self, scripts=None, injection_point=None,
+                                     remove_first=True):
+        """Register user JavaScript files with the current tab.
+
+        Args:
+            scripts: A list of GreasemonkeyScripts, or None to add all
+                     known by the Greasemonkey subsystem.
+            injection_point: The QWebEngineScript::InjectionPoint stage
+                             to inject the script into, None to use
+                             auto-detection.
+            remove_first: Whether to remove all previously injected
+                          scripts before adding these ones.
+        """
+        if sip.isdeleted(self._widget):
+            return
+
+        # Since we are inserting scripts into a per-tab collection,
+        # rather than just injecting scripts on page load, we need to
+        # make sure we replace existing scripts, not just add new ones.
+        # While, taking care not to remove any other scripts that might
+        # have been added elsewhere, like the one for stylesheets.
+        page_scripts = self._widget.page().scripts()
+        if remove_first:
+            for script in page_scripts.toList():
+                if script.name().startswith("GM-"):
+                    log.greasemonkey.debug('Removing script: {}'
+                                           .format(script.name()))
+                    removed = page_scripts.remove(script)
+                    assert removed, script.name()
+
+        if not scripts:
+            return
+
+        for script in scripts:
+            new_script = QWebEngineScript()
+            try:
+                world = int(script.jsworld)
+            except ValueError:
+                try:
+                    world = _JS_WORLD_MAP[usertypes.JsWorld[
+                        script.jsworld.lower()]]
+                except KeyError:
+                    log.greasemonkey.error(
+                        "script {} has invalid value for '@qute-js-world'"
+                        ": {}".format(script.name, script.jsworld))
+                    continue
+            new_script.setWorldId(world)
+            new_script.setSourceCode(script.code())
+            new_script.setName("GM-{}".format(script.name))
+            new_script.setRunsOnSubFrames(script.runs_on_sub_frames)
+            # Override the @run-at value parsed by QWebEngineScript if desired.
+            if injection_point:
+                new_script.setInjectionPoint(injection_point)
+            log.greasemonkey.debug('adding script: {}'
+                                   .format(new_script.name()))
+            page_scripts.insert(new_script)
+
+
 class WebEngineTab(browsertab.AbstractTab):
 
     """A QtWebEngine tab in the browser.
@@ -557,54 +972,63 @@ class WebEngineTab(browsertab.AbstractTab):
                                        private=private)
         self.history = WebEngineHistory(self)
         self.scroller = WebEngineScroller(self, parent=self)
-        self.caret = WebEngineCaret(win_id=win_id, mode_manager=mode_manager,
+        self.caret = WebEngineCaret(mode_manager=mode_manager,
                                     tab=self, parent=self)
-        self.zoom = WebEngineZoom(win_id=win_id, parent=self)
+        self.zoom = WebEngineZoom(tab=self, parent=self)
         self.search = WebEngineSearch(parent=self)
         self.printing = WebEnginePrinting()
-        self.elements = WebEngineElements(self)
-        self.action = WebEngineAction()
+        self.elements = WebEngineElements(tab=self)
+        self.action = WebEngineAction(tab=self)
+        self.audio = WebEngineAudio(parent=self)
+        self._permissions = _WebEnginePermissions(tab=self, parent=self)
+        self._scripts = _WebEngineScripts(tab=self, parent=self)
+        # We're assigning settings in _set_widget
+        self.settings = webenginesettings.WebEngineSettings(settings=None)
         self._set_widget(widget)
         self._connect_signals()
         self.backend = usertypes.Backend.QtWebEngine
-        self._init_js()
         self._child_event_filter = None
         self._saved_zoom = None
+        self._reload_url = None
+        self._scripts.init()
 
-    def _init_js(self):
-        js_code = '\n'.join([
-            '"use strict";',
-            'window._qutebrowser = window._qutebrowser || {};',
-            utils.read_file('javascript/scroll.js'),
-            utils.read_file('javascript/webelem.js'),
-        ])
-        script = QWebEngineScript()
-        script.setInjectionPoint(QWebEngineScript.DocumentCreation)
-        script.setSourceCode(js_code)
-
-        page = self._widget.page()
-        script.setWorldId(QWebEngineScript.ApplicationWorld)
-
-        # FIXME:qtwebengine  what about runsOnSubFrames?
-        page.scripts().insert(script)
+    def _set_widget(self, widget):
+        # pylint: disable=protected-access
+        super()._set_widget(widget)
+        self._permissions._widget = widget
+        self._scripts._widget = widget
 
     def _install_event_filter(self):
-        self._widget.focusProxy().installEventFilter(self._mouse_event_filter)
+        fp = self._widget.focusProxy()
+        if fp is not None:
+            fp.installEventFilter(self._mouse_event_filter)
         self._child_event_filter = mouse.ChildEventFilter(
             eventfilter=self._mouse_event_filter, widget=self._widget,
-            parent=self)
+            win_id=self.win_id, parent=self)
         self._widget.installEventFilter(self._child_event_filter)
 
     @pyqtSlot()
     def _restore_zoom(self):
+        if sip.isdeleted(self._widget):
+            # https://github.com/qutebrowser/qutebrowser/issues/3498
+            return
         if self._saved_zoom is None:
             return
         self.zoom.set_factor(self._saved_zoom)
         self._saved_zoom = None
 
-    def openurl(self, url):
+    def openurl(self, url, *, predict=True):
+        """Open the given URL in this tab.
+
+        Arguments:
+            url: The QUrl to open.
+            predict: If set to False, predicted_navigation is not emitted.
+        """
+        if sip.isdeleted(self._widget):
+            # https://github.com/qutebrowser/qutebrowser/issues/3896
+            return
         self._saved_zoom = self.zoom.factor()
-        self._openurl_prepare(url)
+        self._openurl_prepare(url, predict=predict)
         self._widget.load(url)
 
     def url(self, requested=False):
@@ -636,10 +1060,6 @@ class WebEngineTab(browsertab.AbstractTab):
     def shutdown(self):
         self.shutting_down.emit()
         self.action.exit_fullscreen()
-        if qtutils.version_check('5.8', exact=True, compiled=False):
-            # WORKAROUND for
-            # https://bugreports.qt.io/browse/QTBUG-58563
-            self.search.clear()
         self._widget.shutdown()
 
     def reload(self, *, force=False):
@@ -682,6 +1102,16 @@ class WebEngineTab(browsertab.AbstractTab):
         self.send_event(press_evt)
         self.send_event(release_evt)
 
+    def _show_error_page(self, url, error):
+        """Show an error page in the tab."""
+        log.misc.debug("Showing error page for {}".format(error))
+        url_string = url.toDisplayString()
+        error_page = jinja.render(
+            'error.html',
+            title="Error loading page: {}".format(url_string),
+            url=url_string, error=error)
+        self.set_html(error_page)
+
     @pyqtSlot()
     def _on_history_trigger(self):
         try:
@@ -716,10 +1146,11 @@ class WebEngineTab(browsertab.AbstractTab):
         """Called when a proxy needs authentication."""
         msg = "<b>{}</b> requires a username and password.".format(
             html_utils.escape(proxy_host))
+        urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded)
         answer = message.ask(
             title="Proxy authentication required", text=msg,
             mode=usertypes.PromptMode.user_pwd,
-            abort_on=[self.shutting_down, self.load_started])
+            abort_on=[self.shutting_down, self.load_started], url=urlstr)
         if answer is not None:
             authenticator.setUser(answer.user)
             authenticator.setPassword(answer.password)
@@ -729,21 +1160,19 @@ class WebEngineTab(browsertab.AbstractTab):
                 sip.assign(authenticator, QAuthenticator())
                 # pylint: enable=no-member, useless-suppression
             except AttributeError:
-                url_string = url.toDisplayString()
-                error_page = jinja.render(
-                    'error.html',
-                    title="Error loading page: {}".format(url_string),
-                    url=url_string, error="Proxy authentication required",
-                    icon='')
-                self.set_html(error_page)
+                self._show_error_page(url, "Proxy authentication required")
 
     @pyqtSlot(QUrl, 'QAuthenticator*')
     def _on_authentication_required(self, url, authenticator):
-        # FIXME:qtwebengine support .netrc
-        answer = shared.authentication_required(
-            url, authenticator, abort_on=[self.shutting_down,
-                                          self.load_started])
-        if answer is None:
+        netrc_success = False
+        if not self.data.netrc_used:
+            self.data.netrc_used = True
+            netrc_success = shared.netrc_authentication(url, authenticator)
+        if not netrc_success:
+            abort_on = [self.shutting_down, self.load_started]
+            answer = shared.authentication_required(url, authenticator,
+                                                    abort_on)
+        if not netrc_success and answer is None:
             try:
                 # pylint: disable=no-member, useless-suppression
                 sip.assign(authenticator, QAuthenticator())
@@ -751,34 +1180,17 @@ class WebEngineTab(browsertab.AbstractTab):
             except AttributeError:
                 # WORKAROUND for
                 # https://www.riverbankcomputing.com/pipermail/pyqt/2016-December/038400.html
-                url_string = url.toDisplayString()
-                error_page = jinja.render(
-                    'error.html',
-                    title="Error loading page: {}".format(url_string),
-                    url=url_string, error="Authentication required")
-                self.set_html(error_page)
-
-    @pyqtSlot('QWebEngineFullScreenRequest')
-    def _on_fullscreen_requested(self, request):
-        request.accept()
-        on = request.toggleOn()
-
-        self.data.fullscreen = on
-        self.fullscreen_requested.emit(on)
-        if on:
-            notification = miscwidgets.FullscreenNotification(self)
-            notification.show()
-            notification.set_timeout(3000)
+                self._show_error_page(url, "Authentication required")
 
     @pyqtSlot()
     def _on_load_started(self):
         """Clear search when a new load is started if needed."""
-        if (qtutils.version_check('5.9', compiled=False) and
-                not qtutils.version_check('5.9.2', compiled=False)):
-            # WORKAROUND for
-            # https://bugreports.qt.io/browse/QTBUG-61506
-            self.search.clear()
+        # WORKAROUND for
+        # https://bugreports.qt.io/browse/QTBUG-61506
+        # (seems to be back in later Qt versions as well)
+        self.search.clear()
         super()._on_load_started()
+        self.data.netrc_used = False
 
     @pyqtSlot(QWebEnginePage.RenderProcessTerminationStatus, int)
     def _on_render_process_terminated(self, status, exitcode):
@@ -820,6 +1232,124 @@ class WebEngineTab(browsertab.AbstractTab):
         if not ok:
             self._load_finished_fake.emit(False)
 
+    def _error_page_workaround(self, html):
+        """Check if we're displaying a Chromium error page.
+
+        This gets only called if we got loadFinished(False) without JavaScript,
+        so we can display at least some error page.
+
+        WORKAROUND for https://bugreports.qt.io/browse/QTBUG-66643
+        Needs to check the page content as a WORKAROUND for
+        https://bugreports.qt.io/browse/QTBUG-66661
+        """
+        match = re.search(r'"errorCode":"([^"]*)"', html)
+        if match is None:
+            return
+        self._show_error_page(self.url(), error=match.group(1))
+
+    @pyqtSlot(bool)
+    def _on_load_finished(self, ok):
+        """Display a static error page if JavaScript is disabled."""
+        super()._on_load_finished(ok)
+        js_enabled = self.settings.test_attribute('content.javascript.enabled')
+        if not ok and not js_enabled:
+            self.dump_async(self._error_page_workaround)
+
+        if ok and self._reload_url is not None:
+            # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-66656
+            log.config.debug(
+                "Loading {} again because of config change".format(
+                    self._reload_url.toDisplayString()))
+            QTimer.singleShot(100, functools.partial(self.openurl,
+                                                     self._reload_url,
+                                                     predict=False))
+            self._reload_url = None
+
+        if not qtutils.version_check('5.10', compiled=False):
+            # We can't do this when we have the loadFinished workaround as that
+            # sometimes clears icons without loading a new page.
+            # In general, this is handled by Qt, but when loading takes long,
+            # the old icon is still displayed.
+            self.icon_changed.emit(QIcon())
+
+    @pyqtSlot(certificateerror.CertificateErrorWrapper)
+    def _on_ssl_errors(self, error):
+        self._has_ssl_errors = True
+
+        url = error.url()
+        log.webview.debug("Certificate error: {}".format(error))
+
+        if error.is_overridable():
+            error.ignore = shared.ignore_certificate_errors(
+                url, [error], abort_on=[self.shutting_down, self.load_started])
+        else:
+            log.webview.error("Non-overridable certificate error: "
+                              "{}".format(error))
+
+        log.webview.debug("ignore {}, URL {}, requested {}".format(
+            error.ignore, url, self.url(requested=True)))
+
+        # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-56207
+        # We can't really know when to show an error page, as the error might
+        # have happened when loading some resource.
+        # However, self.url() is not available yet and the requested URL
+        # might not match the URL we get from the error - so we just apply a
+        # heuristic here.
+        if (not qtutils.version_check('5.9') and
+                not error.ignore and
+                url.matches(self.url(requested=True), QUrl.RemoveScheme)):
+            self._show_error_page(url, str(error))
+
+    @pyqtSlot(QUrl)
+    def _on_predicted_navigation(self, url):
+        """If we know we're going to visit an URL soon, change the settings.
+
+        This is a WORKAROUND for https://bugreports.qt.io/browse/QTBUG-66656
+        """
+        super()._on_predicted_navigation(url)
+        if not qtutils.version_check('5.11.1', compiled=False):
+            self.settings.update_for_url(url)
+
+    @pyqtSlot(usertypes.NavigationRequest)
+    def _on_navigation_request(self, navigation):
+        super()._on_navigation_request(navigation)
+
+        if navigation.url == QUrl('qute://print'):
+            command_dispatcher = objreg.get('command-dispatcher',
+                                            scope='window',
+                                            window=self.win_id)
+            command_dispatcher.printpage()
+            navigation.accepted = False
+
+        if not navigation.accepted or not navigation.is_main_frame:
+            return
+
+        settings_needing_reload = {
+            'content.plugins',
+            'content.javascript.enabled',
+            'content.javascript.can_access_clipboard',
+            'content.print_element_backgrounds',
+            'input.spatial_navigation',
+        }
+        assert settings_needing_reload.issubset(configdata.DATA)
+
+        changed = self.settings.update_for_url(navigation.url)
+        reload_needed = changed & settings_needing_reload
+
+        # On Qt < 5.11, we don't don't need a reload when type == link_clicked.
+        # On Qt 5.11.0, we always need a reload.
+        # On Qt > 5.11.0, we never need a reload:
+        # https://codereview.qt-project.org/#/c/229525/1
+        # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-66656
+        if qtutils.version_check('5.11.1', compiled=False):
+            reload_needed = False
+        elif not qtutils.version_check('5.11.0', exact=True, compiled=False):
+            if navigation.navigation_type == navigation.Type.link_clicked:
+                reload_needed = False
+
+        if reload_needed:
+            self._reload_url = navigation.url
+
     def _connect_signals(self):
         view = self._widget
         page = view.page()
@@ -832,8 +1362,8 @@ class WebEngineTab(browsertab.AbstractTab):
         page.authenticationRequired.connect(self._on_authentication_required)
         page.proxyAuthenticationRequired.connect(
             self._on_proxy_authentication_required)
-        page.fullScreenRequested.connect(self._on_fullscreen_requested)
         page.contentsSizeChanged.connect(self.contents_size_changed)
+        page.navigation_request.connect(self._on_navigation_request)
 
         view.titleChanged.connect(self.title_changed)
         view.urlChanged.connect(self._on_url_changed)
@@ -854,5 +1384,12 @@ class WebEngineTab(browsertab.AbstractTab):
             page.loadFinished.connect(self._restore_zoom)
             page.loadFinished.connect(self._on_load_finished)
 
+        self.predicted_navigation.connect(self._on_predicted_navigation)
+
+        # pylint: disable=protected-access
+        self.audio._connect_signals()
+        self._permissions.connect_signals()
+        self._scripts.connect_signals()
+
     def event_target(self):
-        return self._widget.focusProxy()
+        return self._widget.render_widget()

qutebrowser/browser/webengine/webview.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -19,18 +19,17 @@
 
 """The main browser widget for QtWebEngine."""
 
-import functools
-
-from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, PYQT_VERSION
+from PyQt5.QtCore import pyqtSignal, QUrl, PYQT_VERSION
 from PyQt5.QtGui import QPalette
-from PyQt5.QtWebEngineWidgets import (QWebEngineView, QWebEnginePage,
-                                      QWebEngineScript)
+from PyQt5.QtWidgets import QWidget
+from PyQt5.QtWebEngineWidgets import QWebEngineView, QWebEnginePage
 
 from qutebrowser.browser import shared
-from qutebrowser.browser.webengine import certificateerror, webenginesettings
+from qutebrowser.browser.webengine import webenginesettings, certificateerror
 from qutebrowser.config import config
-from qutebrowser.utils import (log, debug, usertypes, jinja, urlutils, message,
-                               objreg, qtutils)
+from qutebrowser.utils import log, debug, usertypes, objreg, qtutils
+from qutebrowser.misc import miscwidgets
+from qutebrowser.qt import sip
 
 
 class WebEngineView(QWebEngineView):
@@ -52,6 +51,35 @@ class WebEngineView(QWebEngineView):
                              parent=self)
         self.setPage(page)
 
+        if qtutils.version_check('5.11', compiled=False):
+            # Set a PseudoLayout as a WORKAROUND for
+            # https://bugreports.qt.io/browse/QTBUG-68224
+            # and other related issues.
+            sip.delete(self.layout())
+            self._layout = miscwidgets.PseudoLayout(self)
+
+    def render_widget(self):
+        """Get the RenderWidgetHostViewQt for this view.
+
+        Normally, this would always be the focusProxy().
+        However, it sometimes isn't, so we use this as a WORKAROUND for
+        https://bugreports.qt.io/browse/QTBUG-68727
+        """
+        if 'lost-focusproxy' not in objreg.get('args').debug_flags:
+            proxy = self.focusProxy()
+            if proxy is not None:
+                return proxy
+
+        # We don't want e.g. a QMenu.
+        rwhv_class = 'QtWebEngineCore::RenderWidgetHostViewQtDelegateWidget'
+        children = [c for c in self.findChildren(QWidget)
+                    if c.isVisible() and c.inherits(rwhv_class)]
+
+        log.webview.debug("Found possibly lost focusProxy: {}"
+                          .format(children))
+
+        return children[-1] if children else None
+
     def shutdown(self):
         self.page().shutdown()
 
@@ -123,21 +151,22 @@ class WebEnginePage(QWebEnginePage):
 
     Signals:
         certificate_error: Emitted on certificate errors.
+                           Needs to be directly connected to a slot setting the
+                           'ignore' attribute.
         shutting_down: Emitted when the page is shutting down.
+        navigation_request: Emitted on acceptNavigationRequest.
     """
 
-    certificate_error = pyqtSignal()
+    certificate_error = pyqtSignal(certificateerror.CertificateErrorWrapper)
     shutting_down = pyqtSignal()
+    navigation_request = pyqtSignal(usertypes.NavigationRequest)
 
     def __init__(self, *, theme_color, profile, parent=None):
         super().__init__(profile, parent)
         self._is_shutting_down = False
-        self.featurePermissionRequested.connect(
-            self._on_feature_permission_requested)
         self._theme_color = theme_color
         self._set_bg_color()
         config.instance.changed.connect(self._set_bg_color)
-        self.urlChanged.connect(self._inject_userjs)
 
     @config.change_filter('colors.webpage.bg')
     def _set_bg_color(self):
@@ -146,106 +175,26 @@ class WebEnginePage(QWebEnginePage):
             col = self._theme_color
         self.setBackgroundColor(col)
 
-    @pyqtSlot(QUrl, 'QWebEnginePage::Feature')
-    def _on_feature_permission_requested(self, url, feature):
-        """Ask the user for approval for geolocation/media/etc.."""
-        options = {
-            QWebEnginePage.Geolocation: 'content.geolocation',
-            QWebEnginePage.MediaAudioCapture: 'content.media_capture',
-            QWebEnginePage.MediaVideoCapture: 'content.media_capture',
-            QWebEnginePage.MediaAudioVideoCapture: 'content.media_capture',
-        }
-        messages = {
-            QWebEnginePage.Geolocation: 'access your location',
-            QWebEnginePage.MediaAudioCapture: 'record audio',
-            QWebEnginePage.MediaVideoCapture: 'record video',
-            QWebEnginePage.MediaAudioVideoCapture: 'record audio/video',
-        }
-        assert options.keys() == messages.keys()
-
-        if feature not in options:
-            log.webview.error("Unhandled feature permission {}".format(
-                debug.qenum_key(QWebEnginePage, feature)))
-            self.setFeaturePermission(url, feature,
-                                      QWebEnginePage.PermissionDeniedByUser)
-            return
-
-        yes_action = functools.partial(
-            self.setFeaturePermission, url, feature,
-            QWebEnginePage.PermissionGrantedByUser)
-        no_action = functools.partial(
-            self.setFeaturePermission, url, feature,
-            QWebEnginePage.PermissionDeniedByUser)
-
-        question = shared.feature_permission(
-            url=url, option=options[feature], msg=messages[feature],
-            yes_action=yes_action, no_action=no_action,
-            abort_on=[self.shutting_down, self.loadStarted])
-
-        if question is not None:
-            self.featurePermissionRequestCanceled.connect(
-                functools.partial(self._on_feature_permission_cancelled,
-                                  question, url, feature))
-
-    def _on_feature_permission_cancelled(self, question, url, feature,
-                                         cancelled_url, cancelled_feature):
-        """Slot invoked when a feature permission request was cancelled.
-
-        To be used with functools.partial.
-        """
-        if url == cancelled_url and feature == cancelled_feature:
-            try:
-                question.abort()
-            except RuntimeError:
-                # The question could already be deleted, e.g. because it was
-                # aborted after a loadStarted signal.
-                pass
-
     def shutdown(self):
         self._is_shutting_down = True
         self.shutting_down.emit()
 
     def certificateError(self, error):
         """Handle certificate errors coming from Qt."""
-        self.certificate_error.emit()
-        url = error.url()
         error = certificateerror.CertificateErrorWrapper(error)
-        log.webview.debug("Certificate error: {}".format(error))
-
-        url_string = url.toDisplayString()
-        error_page = jinja.render(
-            'error.html', title="Error loading page: {}".format(url_string),
-            url=url_string, error=str(error))
-
-        if error.is_overridable():
-            ignore = shared.ignore_certificate_errors(
-                url, [error], abort_on=[self.loadStarted, self.shutting_down])
-        else:
-            log.webview.error("Non-overridable certificate error: "
-                              "{}".format(error))
-            ignore = False
-
-        # We can't really know when to show an error page, as the error might
-        # have happened when loading some resource.
-        # However, self.url() is not available yet and self.requestedUrl()
-        # might not match the URL we get from the error - so we just apply a
-        # heuristic here.
-        # See https://bugreports.qt.io/browse/QTBUG-56207
-        log.webview.debug("ignore {}, URL {}, requested {}".format(
-            ignore, url, self.requestedUrl()))
-        if not ignore and url.matches(self.requestedUrl(), QUrl.RemoveScheme):
-            self.setHtml(error_page)
-
-        return ignore
+        self.certificate_error.emit(error)
+        return error.ignore
 
     def javaScriptConfirm(self, url, js_msg):
         """Override javaScriptConfirm to use qutebrowser prompts."""
         if self._is_shutting_down:
             return False
+        escape_msg = qtutils.version_check('5.11', compiled=False)
         try:
             return shared.javascript_confirm(url, js_msg,
                                              abort_on=[self.loadStarted,
-                                                       self.shutting_down])
+                                                       self.shutting_down],
+                                             escape_msg=escape_msg)
         except shared.CallSuper:
             return super().javaScriptConfirm(url, js_msg)
 
@@ -255,12 +204,14 @@ class WebEnginePage(QWebEnginePage):
         # https://www.riverbankcomputing.com/pipermail/pyqt/2016-November/038293.html
         def javaScriptPrompt(self, url, js_msg, default):
             """Override javaScriptPrompt to use qutebrowser prompts."""
+            escape_msg = qtutils.version_check('5.11', compiled=False)
             if self._is_shutting_down:
                 return (False, "")
             try:
                 return shared.javascript_prompt(url, js_msg, default,
                                                 abort_on=[self.loadStarted,
-                                                          self.shutting_down])
+                                                          self.shutting_down],
+                                                escape_msg=escape_msg)
             except shared.CallSuper:
                 return super().javaScriptPrompt(url, js_msg, default)
 
@@ -268,10 +219,12 @@ class WebEnginePage(QWebEnginePage):
         """Override javaScriptAlert to use qutebrowser prompts."""
         if self._is_shutting_down:
             return
+        escape_msg = qtutils.version_check('5.11', compiled=False)
         try:
             shared.javascript_alert(url, js_msg,
                                     abort_on=[self.loadStarted,
-                                              self.shutting_down])
+                                              self.shutting_down],
+                                    escape_msg=escape_msg)
         except shared.CallSuper:
             super().javaScriptAlert(url, js_msg)
 
@@ -288,58 +241,23 @@ class WebEnginePage(QWebEnginePage):
                                 url: QUrl,
                                 typ: QWebEnginePage.NavigationType,
                                 is_main_frame: bool):
-        """Override acceptNavigationRequest to handle clicked links.
-
-        This only show an error on invalid links - everything else is handled
-        in createWindow.
-        """
-        log.webview.debug("navigation request: url {}, type {}, is_main_frame "
-                          "{}".format(url.toDisplayString(),
-                                      debug.qenum_key(QWebEnginePage, typ),
-                                      is_main_frame))
-        if (typ == QWebEnginePage.NavigationTypeLinkClicked and
-                not url.isValid()):
-            msg = urlutils.get_errstring(url, "Invalid link clicked")
-            message.error(msg)
-            return False
-        return True
-
-    @pyqtSlot('QUrl')
-    def _inject_userjs(self, url):
-        """Inject userscripts registered for `url` into the current page."""
-        if qtutils.version_check('5.8'):
-            # Handled in webenginetab with the builtin Greasemonkey
-            # support.
-            return
-
-        # Using QWebEnginePage.scripts() to hold the user scripts means
-        # we don't have to worry ourselves about where to inject the
-        # page but also means scripts hang around for the tab lifecycle.
-        # So clear them here.
-        scripts = self.scripts()
-        for script in scripts.toList():
-            if script.name().startswith("GM-"):
-                log.greasemonkey.debug("Removing script: {}"
-                                       .format(script.name()))
-                removed = scripts.remove(script)
-                assert removed, script.name()
-
-        def _add_script(script, injection_point):
-            new_script = QWebEngineScript()
-            new_script.setInjectionPoint(injection_point)
-            new_script.setWorldId(QWebEngineScript.MainWorld)
-            new_script.setSourceCode(script.code())
-            new_script.setName("GM-{}".format(script.name))
-            new_script.setRunsOnSubFrames(script.runs_on_sub_frames)
-            log.greasemonkey.debug("Adding script: {}"
-                                   .format(new_script.name()))
-            scripts.insert(new_script)
-
-        greasemonkey = objreg.get('greasemonkey')
-        matching_scripts = greasemonkey.scripts_for(url)
-        for script in matching_scripts.start:
-            _add_script(script, QWebEngineScript.DocumentCreation)
-        for script in matching_scripts.end:
-            _add_script(script, QWebEngineScript.DocumentReady)
-        for script in matching_scripts.idle:
-            _add_script(script, QWebEngineScript.Deferred)
+        """Override acceptNavigationRequest to forward it to the tab API."""
+        type_map = {
+            QWebEnginePage.NavigationTypeLinkClicked:
+                usertypes.NavigationRequest.Type.link_clicked,
+            QWebEnginePage.NavigationTypeTyped:
+                usertypes.NavigationRequest.Type.typed,
+            QWebEnginePage.NavigationTypeFormSubmitted:
+                usertypes.NavigationRequest.Type.form_submitted,
+            QWebEnginePage.NavigationTypeBackForward:
+                usertypes.NavigationRequest.Type.back_forward,
+            QWebEnginePage.NavigationTypeReload:
+                usertypes.NavigationRequest.Type.reloaded,
+            QWebEnginePage.NavigationTypeOther:
+                usertypes.NavigationRequest.Type.other,
+        }
+        navigation = usertypes.NavigationRequest(url=url,
+                                                 navigation_type=type_map[typ],
+                                                 is_main_frame=is_main_frame)
+        self.navigation_request.emit(navigation)
+        return navigation.accepted

qutebrowser/browser/webkit/__init__.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webkit/cache.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webkit/certificateerror.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webkit/cookies.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webkit/http.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2018 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #