Release v0.10.0

Fixes #2347

.eslintignore

@@ -1 +0,0 @@
-qutebrowser/3rdparty/pdfjs/*

.eslintrc

@@ -1,49 +0,0 @@
-# vim: ft=yaml
-
-env:
-   browser: true
-
-rules:
-    block-scoped-var: 2
-    dot-location: 2
-    default-case: 2
-    guard-for-in: 2
-    no-div-regex: 2
-    no-param-reassign: 2
-    no-eq-null: 2
-    no-floating-decimal: 2
-    no-self-compare: 2
-    no-throw-literal: 2
-    no-void: 2
-    radix: 2
-    wrap-iife: [2, "inside"]
-    brace-style: [2, "1tbs", {"allowSingleLine": true}]
-    comma-style: [2, "last"]
-    consistent-this: [2, "self"]
-    func-style: [2, "declaration"]
-    indent: [2, 4, {"SwitchCase": 1}]
-    linebreak-style: [2, "unix"]
-    max-nested-callbacks: [2, 3]
-    no-lonely-if: 2
-    no-multiple-empty-lines: [2, {"max": 2}]
-    no-nested-ternary: 2
-    no-unneeded-ternary: 2
-    operator-assignment: [2, "always"]
-    operator-linebreak: [2, "after"]
-    keyword-spacing: 2
-    space-before-blocks: [2, "always"]
-    space-before-function-paren: [2, {"anonymous": "never", "named": "never"}]
-    object-curly-spacing: [2, "never"]
-    array-bracket-spacing: [2, "never"]
-    computed-property-spacing: [2, "never"]
-    space-in-parens: [2, "never"]
-    space-unary-ops: [2, {"words": true, "nonwords": false}]
-    spaced-comment: [2, "always"]
-    max-depth: [2, 5]
-    max-len: [2, 79, 4]
-    max-params: [2, 5]
-    max-statements: [2, 30]
-    no-bitwise: 2
-    quote-props: [2, "always"]
-    global-strict: 0
-    quotes: 0

.flake8

@@ -6,6 +6,9 @@ exclude = .*,__pycache__,resources.py
 # E501: Line too long
 # E402: module level import not at top of file
 # E266: too many leading '#' for block comment
+# E722: do not use bare except
+# E731: do not assign a lambda expression, use a def
+#       (for pytest's __tracebackhide__)
 # F401: Unused import
 # N802: function name should be lowercase
 # P101: format string does contain unindexed parameters
@@ -21,18 +24,12 @@ exclude = .*,__pycache__,resources.py
 # D402: First line should not be function's signature (false-positives)
 # D403: First word of the first line should be properly capitalized
 #       (false-positives)
-# H101: Use TODO(NAME)
-# H201: bare except
-# H238: Use new-stule classes
-# H301: one import per line
-# H306: imports not in alphabetical order
 ignore =
-    E128,E226,E265,E501,E402,E266,
+    E128,E226,E265,E501,E402,E266,E722,E731,
     F401,
     N802,
     P101,P102,P103,
-    D102,D103,D104,D105,D209,D211,D402,D403,
-    H101,H201,H238,H301,H306
+    D102,D103,D104,D105,D209,D211,D402,D403
 min-version = 3.4.0
 max-complexity = 12
 putty-auto-ignore = True
@@ -41,6 +38,8 @@ putty-ignore =
     /# pylint: disable=wildcard-import/ : +F403
     /# pragma: no mccabe/ : +C901
     tests/*/test_*.py : +D100,D101,D401
+    tests/unit/browser/webkit/test_history.py : +N806
+    tests/helpers/fixtures.py : +N806
     tests/unit/browser/webkit/http/test_content_disposition.py : +D400
     scripts/dev/ci/appveyor_install.py : +FI53
 copyright-check = True

.github/ISSUE_TEMPLATE.md

@@ -1,4 +1,2 @@
-Please remember to mention your version info (qutebrowser, Qt, PyQt,
-OS/distribution) from the `qute:version` page or `qutebrowser --version`
-
----
+<!-- If this is a bug report, please remember to mention your version info from
+the `qute:version` page or `qutebrowser --version` -->

.gitignore

@@ -20,7 +20,7 @@ __pycache__
 /FAQ.html
 /INSTALL.html
 /qutebrowser/html/doc/
-/.venv
+/.venv*
 /.coverage
 /htmlcov
 /coverage.xml
@@ -33,7 +33,12 @@ __pycache__
 /prof
 /venv
 TODO
-/scripts/testbrowser_cpp/Makefile
-/scripts/testbrowser_cpp/main.o
-/scripts/testbrowser_cpp/testbrowser
+/scripts/testbrowser_cpp/webkit/Makefile
+/scripts/testbrowser_cpp/webkit/main.o
+/scripts/testbrowser_cpp/webkit/testbrowser
+/scripts/testbrowser_cpp/webkit/.qmake.stash
+/scripts/testbrowser_cpp/webengine/Makefile
+/scripts/testbrowser_cpp/webengine/main.o
+/scripts/testbrowser_cpp/webengine/testbrowser
+/scripts/testbrowser_cpp/webengine/.qmake.stash
 /scripts/dev/pylint_checkers/qute_pylint.egg-info

.pylintrc

@@ -68,9 +68,12 @@ max-args=10
 valid-metaclass-classmethod-first-arg=cls
 
 [TYPECHECK]
+# WORKAROUND for https://github.com/PyCQA/astroid/pull/357
+ignored-modules=pytest
 # MsgType added as WORKAROUND for
 # https://bitbucket.org/logilab/pylint/issues/690/
 # UnsetObject because pylint infers any objreg.get(...) as UnsetObject.
 ignored-classes=qutebrowser.utils.objreg.UnsetObject,
                 qutebrowser.browser.webkit.webelem.WebElementWrapper,
-                scripts.dev.check_coverage.MsgType
+                scripts.dev.check_coverage.MsgType,
+                qutebrowser.browser.downloads.UnsupportedAttribute

.travis.yml

@@ -13,13 +13,33 @@ matrix:
       env: DOCKER=archlinux
       services: docker
     - os: linux
-      env: DOCKER=ubuntu-wily
+      env: DOCKER=archlinux-webengine QUTE_BDD_WEBENGINE=true
+      services: docker
+    - os: linux
+      env: DOCKER=archlinux-ng
       services: docker
     - os: linux
       env: DOCKER=ubuntu-xenial
       services: docker
+    - os: linux
+      language: python
+      python: 3.6
+      env: TESTENV=py36-pyqt571
+    - os: linux
+      language: python
+      python: 3.5
+      env: TESTENV=py35-pyqt58
+    - os: linux
+      language: python
+      python: 3.6
+      env: TESTENV=py36-pyqt58
     - os: osx
-      env: TESTENV=py35
+      env: TESTENV=py36 OSX=elcapitan
+      osx_image: xcode7.3
+    # https://github.com/qutebrowser/qutebrowser/issues/2013
+    # - os: osx
+    #   env: TESTENV=py35 OSX=yosemite
+    #   osx_image: xcode6.4
     - os: linux
       env: TESTENV=pylint
     - os: linux
@@ -38,12 +58,14 @@ matrix:
       env: TESTENV=eslint
   allow_failures:
     - os: osx
-      env: TESTENV=py35
+      env: TESTENV=py36 OSX=elcapitan
+      osx_image: xcode7.3
+  fast_finish: true
 
 cache:
   directories:
     - $HOME/.cache/pip
-    - $HOME/build/The-Compiler/qutebrowser/.cache
+    - $HOME/build/qutebrowser/qutebrowser/.cache
 
 before_install:
   # We need to do this so we pick up the system-wide python properly
@@ -63,8 +85,8 @@ notifications:
     - https://buildtimetrend.herokuapp.com/travis
   irc:
     channels:
-      - "chat.freenode.net#qutebrowser"
-    on_success: change
+      - "chat.freenode.net#qutebrowser-dev"
+    on_success: always
     on_failure: always
     skip_join: true
     template:

CHANGELOG.asciidoc

@@ -14,6 +14,320 @@ This project adheres to http://semver.org/[Semantic Versioning].
 // `Fixed` for any bug fixes.
 // `Security` to invite users to upgrade in case of vulnerabilities.
 
+v0.10.0
+-------
+
+Added
+~~~~~
+
+- Userscripts now have a new `$QUTE_COMMANDLINE_TEXT` environment variable, containing the current commandline contents
+- New `ripbang` userscript to create a searchengine from a duckduckgo bang
+- link:https://github.com/annulen/webkit/wiki[QtWebKit Reloaded] (also called QtWebKit-NG) is now fully supported
+- Various new functionality with the QtWebEngine backend:
+    * Printing support with Qt >= 5.8
+    * Proxy support with Qt >= 5.8
+    * Tor the `general -> print-element-backgrounds` option with Qt >= 5.8
+    * The `content -> cookies-store` option
+    * The `storage -> cache-size` option
+    * The `colors -> webpage.bg` option
+    * The HTML5 fullscreen API (e.g. youtube videos) with QtWebEngine
+    * `:download --mhtml`
+- New `qute:history` URL and `:history` command to show the browsing history
+- Open tabs are now auto-saved on each successful load and restored in case of a crash
+- `:jseval` now has a `--file` flag so you can pass a javascript file
+- `:session-save` now has a `--only-active-window` flag to only save the active window
+- OS X builds are back, and built with QtWebEngine
+
+Changed
+~~~~~~~
+
+- PyQt 5.7/Qt 5.7.1 is now required for the QtWebEngine backend
+- Scrolling with the scrollwheel while holding shift now scrolls sideways
+- New way of clicking hints with which solves various small issues
+- When yanking a mailto: link via hints, the mailto: prefix is now stripped
+- Zoom level messages are now not stacked on top of each other anymore
+- qutebrowser now automatically uses QtWebEngine if QtWebKit is unavailable
+- :history-clear now asks for a confirmation, unless it's run with --force.
+- `input -> mouse-zoom-divider` can now be 0 to disable zooming by mouse wheel
+- `network -> proxy` can also be set to `pac+file://...` now to
+  use a local proxy autoconfig file (on QtWebKit)
+
+Fixed
+~~~~~
+
+- Various bugs with Qt 5.8 and QtWebEngine:
+    * Segfault when closing a window
+    * Segfault with when closing a tab with a search active
+    * Fixed various mouse actions (like automatically entering insert mode) not working
+    * Fixed hints sometimes not working
+    * Segfault when opening a URL after a QtWebEngine renderer process crash
+- Other QtWebEngine fixes:
+    * Insert mode now gets entered correctly with a non-100% zoom
+    * Crash reports are now re-enabled when using QtWebEngine
+    * Fixed crashes when closing tabs while hinting
+    * Using :undo or :tab-clone with a view-source:// or chrome:// tab is now prevented, as it segfaults
+- `:enter-mode` now refuses to enter modes which can't be entered manually (which caused crashes)
+- `:record-macro` (`q`) now doesn't try to record macros for special keys without a text
+- Fixed PAC (proxy autoconfig) not working with QtWebKit
+- `:download --mhtml` now uses the new file dialog
+- Word hints are now upper-cased correctly when hints -> uppercase is true
+- Font validation is now more permissive in the config, allowing e.g. "Terminus
+  (TTF)" as font name
+- Fixed starting on newer PyQt/sip versions with LibreSSL
+- When downloading files with QtWebKit, a User-Agent header is set when possible
+- Fixed showing of keybindings in the :help completion
+- `:navigate prev/next` now detects `rel` attributes on `<a>` elements, and
+  handles multiple `rel` attributes correctly
+- Fixed a crash when hinting with target `userscript` and spawning a non-existing script
+- Lines in Jupyter notebook now trigger insert mode
+
+v0.9.1
+------
+
+Fixed
+~~~~~
+
+- Prevent websites from downloading files to a location outside of the download
+  folder with QtWebEngine.
+
+v0.9.0
+------
+
+Added
+~~~~~
+
+- *New dependency:* qutebrowser now depends on the Qt QML module, which is
+   packaged separately in some distributions (as Qt Declarative/QML/Quick).
+- New `:rl-backward-kill-word` command which does what `:rl-unix-word-rubout`
+  did before v0.8.0.
+- New `:rl-unix-filename-rubout` command which is similar to readline's
+  `unix-filename-rubout`.
+- New `fonts -> completion.category` setting to customize the font used for
+  completion category headers.
+- New `:debug-log-capacity` command to adjust how many lines are logged into RAM
+  (to report bugs which are difficult to reproduce).
+- New `hide-unmatched-rapid-hints` option to not hide hint unmatched hint labels
+  in rapid mode.
+- New `{clipboard}` and `{primary}` replacements for the commandline which
+  replace the `:paste` command.
+- New `:insert-text` command to insert a given text into a field on the page,
+  which replaces `:paste-primary` together with the `{primary}` replacement.
+- New `:window-only` command to close all other windows.
+- New `prev-category` and `next-category` arguments to `:completion-item-focus`
+  to focus the previous/next category in the completion (bound to `<Ctrl-Tab>`
+  and `<Ctrl-Shift-Tab>` by default).
+- New `:click-element` command to fake a click on a element.
+- New `:debug-log-filter` command to change console log filtering on-the-fly.
+- New `:debug-log-level` command to change the console loglevel on-the-fly.
+- New `general -> yank-ignored-url-parameters` option to configure which URL
+  parameters (like `utm_source` etc.) to strip off when yanking an URL.
+- Support for the
+  https://developer.mozilla.org/en-US/docs/Web/API/Page_Visibility_API[HTML5 page visibility API]
+- New `readability` userscript which shows a readable version of a page (using
+  the `readability-lxml` python package)
+- New `cast` userscript to show a video on a Google Chromecast
+- New `:run-with-count` command which replaces the (undocumented) `:count:command` syntax.
+- New `:record-macro` (`q`) and `:run-macro` (`@`) commands for keyboard macros.
+- New `ui -> hide-scrollbar` setting to hide the scrollbar independently of the
+  `user-stylesheet` setting.
+- New `general -> default-open-dispatcher` setting to configure what to open
+  downloads with (instead of e.g. `xdg-open` on Linux).
+- Support for PAC (proxy autoconfig) with QtWebKit
+
+Changed
+~~~~~~~
+
+- Hints are now drawn natively in Qt instead of using web elements. This has a
+  few implications for users:
+    * The `hints -> opacity` setting does not exist anymore, but you can use
+      `rgba(r, g, b, alpha)` colors instead for `colors -> hints.bg`.
+    * The `hints -> font` setting is not affected by
+      `fonts -> web-family-fixed` anymore. Thus, a transformer got added to
+      change `Monospace` to `${_monospace}`.
+    * Gradients in hint colors can now be configured by using `qlineargradient`
+      and friends instead of `-webkit-gradient`. The most common cases get
+      migrated automatically, but if you drastically changed the defaults,
+      you'll need to manually adjust your config.
+    * Styling hints by styling `qutehint` elements in `user-stylesheet` was
+      never officially supported and does not work anymore.
+    * Hints are now not affected by the page's stylesheet or zoom anymore.
+- `:bookmark-add` now has a `--toggle` flag which deletes the bookmark if it
+  already exists.
+- `:bookmark-load` now has a `--delete` flag which deletes the bookmark after
+  loading it.
+- `:open` now also accepts quickmark names instead of URLs
+- `:tab-move` now optionally takes an index for absolute moving.
+- Commands taking either an argument or a count (like `:zoom` or `:tab-focus`)
+  now prefer the count instead of showing an error message.
+- `:open` now has an `--implicit` argument to treat the opened tab as implicit
+  (i.e. to open it at the position it would be opened if it was a clicked link)
+- `:download-open` and `:prompt-open-download` now have an optional `cmdline`
+  argument to pass a commandline to open the download with.
+- `:yank` now has a position argument to select what to yank instead of using
+  flags.
+- Replacements like `{url}` can now also be used in the middle of an argument.
+  Consequently, commands taking another command (`:later`, `:repeat` and
+  `:bind`) now don't immediately evaluate variables.
+- Tab titles in the `:buffer` completion now update correctly when a page's
+  title is changed via javascript.
+- `:hint` now has a `--mode <mode>` flag to override the hint mode configured
+  using the `hints -> mode` setting.
+- With `new-instance-open-target` set to a tab option, the tab is now opened in
+  the most recently focused (instead of the last opened) window. This can be
+  configured with the new `new-instance-open-target.window` setting.
+  It can also be set to `last-visible` to show the pages in the most recently
+  visible window, or `first-opened` to use the first (oldest) available window.
+- Word hints now are more clever about getting the element text from some elements.
+- Completions for `:help` and `:bind` now also show hidden commands
+- The `:buffer` completion now also filters using the first column (id).
+- `:undo` has been improved to reopen tabs at the position they were closed.
+- `:navigate` now takes a count for `up`/`increment`/`decrement`.
+- The `hints -> auto-follow` setting now can be set to
+  `always`/`full-match`/`unique-match`/`never` to more precisely control when
+  hints should be followed automatically.
+- Counts can now be used with special keybindings (e.g. with modifiers).
+  This was already implemented for v0.7.0 originally, but got reverted because
+  it caused some issues and then never re-applied.
+- Sending a command to an existing instance (via "qutebrowser :reload") now
+  doesn't mark it as urgent anymore.
+- `tabs -> title-format` now treats an empty string as valid.
+- Bindings for `:`, `/` and `?` are now configured explicitly and not hardcoded
+  anymore.
+- The `completion -> show` setting can now be set to `always`, `auto` or
+  `never`.
+- `:open-editor` can now be used in any mode.
+- Lots of improvements to and bugfixes for the QtWebEngine backend, such as
+  working hints. However, using qutebrowser directly from git is still advised
+  when using `--backend webengine`.
+- `content -> javascript-can-open-windows` got renamed to
+  `javascript-can-open-windows-automatically`.
+- `:prompt-accept` now optionally accepts a value which overrides the one
+  entered in the input box. `yes` and `no` can be used as values for yes/no
+  questions.
+- The new `--qt-arg` and `--qt-flag` arguments can be used to pass
+  arguments/flags to Qt's commandline.
+- Error/warning/info messages are now shown stacked above the statusbar.
+  This also added various new settings:
+    * `colors -> messages.fg.error` (renamed from `statusbar.fg.error`)
+    * `colors -> messages.bg.error` (renamed from `statusbar.bg.error`)
+    * `colors -> messages.border.error`
+    * `colors -> messages.fg.warning` (renamed from `statusbar.fg.warning`)
+    * `colors -> messages.bg.warning` (renamed from `statusbar.bg.warning`)
+    * `colors -> messages.border.warning`
+    * `colors -> messages.fg.info`
+    * `colors -> messages.bg.info`
+    * `colors -> messages.border.info`
+    * `fonts -> messages.error`
+    * `fonts -> messages.warning`
+    * `fonts -> messages.info`
+- The `qute:settings` page now also shows option descriptions.
+- `qute:version` and `qutebrowser --version` now show various important paths
+- `:spawn`/userscripts now show a nicer error when a script wasn't found
+- Various functionality now works when javascript is disabled with QtWebKit
+- Various commands/settings taking `left`/`right`/`previous` arguments now take
+  `prev`/`next`/`last-used` to remove ambiguity.
+- The `ui -> user-stylesheet` setting now only takes filenames, not CSS snippets
+- `ui -> window-title-format` now has a new `{backend} ` replacement
+- `:hint` has a new `--add-history` argument to add the URL to the history for
+  yank/spawn targets.
+- `:set` now cycles through values if more than one argument is given.
+- `:open` now opens `default-page` without an URL even without `-t`/`-b`/`-w` given.
+
+Deprecated
+~~~~~~~~~~
+
+- The `:paste` command got deprecated as `:open` with `{clipboard}` and
+  `{primary}` can be used instead.
+- The `:paste-primary` command got deprecated as `:insert-text {primary}` can
+  be used instead.
+- The `:prompt-yes` and `:prompt-no` commands got deprecated as
+  `:prompt-accept yes` and `:prompt-accept no` can be used instead.
+
+Removed
+~~~~~~~
+
+- The `:yank-selected` command got merged into `:yank` as `:yank selection`
+  and thus removed.
+- The `:completion-item-prev` and `:completion-item-next` commands got merged
+  into a new `:completion-focus {prev,next}` command and thus removed.
+- The `ui -> hide-mouse-cursor` setting since it was completely broken and
+  nobody seemed to care.
+- The `hints -> opacity` setting - see the "Changed" section for details.
+- The `completion -> auto-open` setting got merged into `completion -> show` and
+  thus removed.
+- All `--qt-*` arguments got replaced by `--qt-arg` and `--qt-flag` and thus
+  removed.
+- The `-c`/`--confdir`, `--datadir` and `--cachedir` arguments got removed, as
+  `--basedir` should be sufficient.
+
+Fixed
+~~~~~
+
+- `:undo` now doesn't undo tabs "closed" by `:tab-detach` anymore.
+- Fixed an issue with hint chars not being cleared correctly when leaving hint
+  mode.
+- `:tab-detach` now fails correctly when there's only one tab open.
+- Various small issues with the command completion
+- Fixed hang when using multiple spaces in a row with the URL completion
+- qutebrowser now still starts with an incorrectly configured
+  `$XDG_RUNTIME_DIR`.
+- Fixed crash when a userscript writes invalid unicode data to the FIFO
+- Fixed crash when a included HTML was not found
+
+v0.8.3
+------
+
+Fixed
+~~~~~
+
+- Fixed crash when doing `:<space><enter>`, another corner-case introduced in v0.8.0
+- Fixed `:open-editor` (`<Ctrl-e>`) on Windows
+- Fixed crash when setting `general -> auto-save-interval` to a too big value.
+- Fixed crash when using hints on Void Linux.
+- Fixed compatibility with Python 3.5.2+ on Debian unstable
+- Compatibility with pdfjs v1.6.210
+- `:bind` can now be used to bind to an alias (binding by editing `keys.conf`
+  already worked before)
+- The command completion now updates correctly when changing aliases
+- The tabbar now displays correctly with the Adwaita Qt theme
+- The default `sk` keybinding now sets the commandline to `:bind` correctly
+- Fixed crash when closing a window without focusing it
+- Userscripts now can access QUTE_FIFO correctly on Windows
+
+v0.8.2
+------
+
+Fixed
+~~~~~
+
+- Fixed `general -> private-browsing` not being set correctly until a restart
+  (which caused e.g. local storage to be enabled).
+- When hinting input fields (`:t`), also consider input elements without a type.
+- Fixed crash when opening an invalid URL with a percent-encoded and a real @ in it
+- Fixed default `;o` and `;O` bindings
+- Fixed local storage not working (and possible other bugs) when using a
+  relative path with `--basedir`.
+- Fixed crash when deleting a quickmark with Ctrl-D
+- Fixed HTML5 video playback on Windows
+- Fixed crash when using `:prompt-open-download` with a file with chars not
+  encodable with the OS' filesystem encoding (e.g. with `LC_ALL=C`)
+- Fixed `:prompt-open-download` with a too long filename (> 255 bytes)
+- Fixed crash when cancelling a download after doing `:prompt-open-download`
+- Fixed crash when writing a download to disk fails with
+  `:prompt-open-download`.
+- Fixed `:restart` deleting the basedir when it was given with `--basedir`.
+
+v0.8.1
+------
+
+Fixed
+~~~~~
+
+- Fix crash when pressing enter without a command
+- Adjust error message to point out QtWebEngine is unsupported with the OS
+  X .app currently.
+- Hide Harfbuzz warning with the OS X .app
+
 v0.8.0
 ------
 
@@ -33,7 +347,7 @@ Added
   `$QUTE_DOWNLOAD_DIR` available for userscripts.
 - New option `ui` -> `status-position` to configure the position of the
   status bar (top/bottom).
-- New `--pdf <filename>` argument for `:print` which can be used to generate a
+- New `--pdf <filename>` argument for `:print` WHICH can be used to generate a
   PDF without a dialog.
 
 Changed
@@ -108,7 +422,6 @@ Changed
 - `:navigate` now clears the URL fragment
 - `:completion-item-del` (`Ctrl-D`) can now be used in `:buffer` completion to
   close a tab
-- Counts can now be used with special keybindings (e.g. with modifiers)
 - Various SSL ciphers are now disabled by default. With recent Qt/OpenSSL
   versions those already all are disabled, but with older versions they might
   not be.
@@ -568,7 +881,7 @@ Fixed
 - Fixed horrible completion performance when the `shrink` option was set.
 - Sessions now store zoom/scroll-position correctly.
 
-https://github.com/The-Compiler/qutebrowser/releases/tag/v0.2.1[v0.2.1]
+https://github.com/qutebrowser/qutebrowser/releases/tag/v0.2.1[v0.2.1]
 -----------------------------------------------------------------------
 
 Fixed
@@ -576,7 +889,7 @@ Fixed
 
 - Added missing manpage (doc/qutebrowser.1.asciidoc) to archive.
 
-https://github.com/The-Compiler/qutebrowser/releases/tag/v0.2.0[v0.2.0]
+https://github.com/qutebrowser/qutebrowser/releases/tag/v0.2.0[v0.2.0]
 -----------------------------------------------------------------------
 
 Added
@@ -719,7 +1032,7 @@ Fixed
 - Add a timeout to pastebin HTTP replies.
 - Various other fixes for small/rare bugs.
 
-https://github.com/The-Compiler/qutebrowser/releases/tag/v0.1.4[v0.1.4]
+https://github.com/qutebrowser/qutebrowser/releases/tag/v0.1.4[v0.1.4]
 -----------------------------------------------------------------------
 
 Changed
@@ -763,7 +1076,7 @@ Security
 * Stop the icon database from being created when private-browsing is set to true.
 * Disable insecure SSL ciphers.
 
-https://github.com/The-Compiler/qutebrowser/releases/tag/v0.1.3[v0.1.3]
+https://github.com/qutebrowser/qutebrowser/releases/tag/v0.1.3[v0.1.3]
 -----------------------------------------------------------------------
 
 Changed
@@ -797,7 +1110,7 @@ Security
 
 * Fix for HTTP passwords accidentally being written to debug log.
 
-https://github.com/The-Compiler/qutebrowser/releases/tag/v0.1.2[v0.1.2]
+https://github.com/qutebrowser/qutebrowser/releases/tag/v0.1.2[v0.1.2]
 -----------------------------------------------------------------------
 
 Changed
@@ -829,7 +1142,7 @@ Fixed
 * Fix user-stylesheet setting with an empty value.
 
 
-https://github.com/The-Compiler/qutebrowser/releases/tag/v0.1.1[v0.1.1]
+https://github.com/qutebrowser/qutebrowser/releases/tag/v0.1.1[v0.1.1]
 -----------------------------------------------------------------------
 
 Added
@@ -887,7 +1200,7 @@ Fixed
 * Ensure the docs get included in `freeze.py`.
 * Fix crash with `:zoom`.
 
-https://github.com/The-Compiler/qutebrowser/releases/tag/v0.1[v0.1]
+https://github.com/qutebrowser/qutebrowser/releases/tag/v0.1[v0.1]
 -------------------------------------------------------------------
 
 Initial release.

CONTRIBUTING.asciidoc

@@ -11,9 +11,9 @@ This document contains guidelines for contributing to qutebrowser, as well as
 useful hints when doing so.
 
 If anything mentioned here would prevent you from contributing, please let me
-know, and contribute anyways! The guidelines are only meant to make life easier
-for me, but if you don't follow anything in here, I won't be mad at you. I will
-probably change it for you then, though.
+know, and contribute anyways! The guidelines are meant to make life easier for
+me, but if you don't follow everything in here, I won't be mad at you. In
+fact, I will probably change it for you.
 
 If you have any problems, I'm more than happy to help! You can get help in
 several ways:
@@ -34,17 +34,17 @@ this. It might be a good idea to ask on the mailing list or IRC channel to make
 sure nobody else started working on the same thing already.
 
 If you want to find something useful to do, check the
-https://github.com/The-Compiler/qutebrowser/issues[issue tracker]. Some
+https://github.com/qutebrowser/qutebrowser/issues[issue tracker]. Some
 pointers:
 
-* https://github.com/The-Compiler/qutebrowser/labels/easy[Issues which should
+* https://github.com/qutebrowser/qutebrowser/labels/easy[Issues which should
 be easy to solve]
-* https://github.com/The-Compiler/qutebrowser/labels/not%20code[Issues which
+* https://github.com/qutebrowser/qutebrowser/labels/not%20code[Issues which
 require little/no coding]
 
 There are also some things to do if you don't want to write code:
 
-* Help the community, e.g. on the mailinglist and the IRC channel.
+* Help the community, e.g., on the mailinglist and the IRC channel.
 * Improve the documentation.
 * Help on the website and graphics (logo, etc.).
 
@@ -55,12 +55,12 @@ qutebrowser uses http://git-scm.com/[git] for its development. You can clone
 the repo like this:
 
 ----
-git clone https://github.com/The-Compiler/qutebrowser.git
+git clone https://github.com/qutebrowser/qutebrowser.git
 ----
 
 If you don't know git, a http://git-scm.com/[git cheatsheet] might come in
 handy. Of course, if using git is the issue which prevents you from
-contributing, feel free to send normal patches instead, e.g. generated via
+contributing, feel free to send normal patches instead, e.g., generated via
 `diff -Nur`.
 
 Getting patches
@@ -77,7 +77,7 @@ based on your changes like this:
 ----
 git format-patch origin/master <1>
 ----
-<1> Replace `master` by the branch your work was based on, e.g.
+<1> Replace `master` by the branch your work was based on, e.g.,
 `origin/develop`.
 
 Useful utilities
@@ -89,7 +89,7 @@ Checkers
 qutebrowser uses http://tox.readthedocs.org/en/latest/[tox] to run its
 unittests and several linters/checkers.
 
-Currently, following tox environments are available:
+Currently, the following tox environments are available:
 
 * Tests using https://www.pytest.org[pytest]:
   - `py34`: Run pytest for python-3.4.
@@ -117,18 +117,18 @@ Currently, following tox environments are available:
     - VCS conflict markers
     - common spelling mistakes
 
-The default test suite is run with `tox`, the list of default
-environments is obtained with `tox -l` .
+The default test suite is run with `tox`; the list of default
+environments is obtained with `tox -l`.
 
 Please make sure the checks run without any warnings on your new contributions.
 
-There's of course the possibility of false-positives, and the following
+There's always the possibility of false positives; the following
 techniques are useful to handle these:
 
 * Use `_foo` for unused parameters, with `foo` being a descriptive name. Using
 `_` is discouraged.
-* If you think you have a good reason to suppress a message, add the following
-comment:
+* If you think you have a good reason to suppress a message, then add the 
+following comment:
 +
 ----
 # pylint: disable=message-name
@@ -186,10 +186,10 @@ the output in four different ways:
 Debugging
 ~~~~~~~~~
 
-In the `qutebrowser.utils.debug` module there are some useful functions for
-debugging.
+There are some useful functions for debugging in the `qutebrowser.utils.debug`
+module.
 
-When starting qutebrowser with the `--debug` flag you also get useful debug
+When starting qutebrowser with the `--debug` flag, you also get useful debug
 logs. You can add +--logfilter _category[,category,...]_+ to restrict logging
 to the given categories.
 
@@ -276,10 +276,10 @@ Hints
 Python and Qt objects
 ~~~~~~~~~~~~~~~~~~~~~
 
-For many tasks, there are solutions in both Qt and the Python standard library
-available.
+For many tasks, there are solutions available in both Qt and the Python
+standard library.
 
-In qutebrowser, the policy is usually using the Python libraries, as they
+In qutebrowser, the policy is usually to use the Python libraries, as they
 provide exceptions and other benefits.
 
 There are some exceptions to that:
@@ -292,7 +292,7 @@ slots.
 
 When using Qt objects, two issues must be taken care of:
 
-* Methods of Qt objects report their status by using their return values,
+* Methods of Qt objects report their status with their return values,
 instead of using exceptions.
 +
 If a function gets or returns a Qt object which has an `.isValid()`
@@ -304,13 +304,12 @@ on all such objects. It will raise
 If a function returns something else on error, the return value should
 carefully be checked.
 
-* Methods of Qt objects have certain maximum values, based on their
+* Methods of Qt objects have certain maximum values based on their
 underlying C++ types.
 +
-When passing a numeric parameter to a Qt function, all numbers should
-be range-checked using `qutebrowser.qtutils.check_overflow`, or
-passing a value which is too large should be avoided by other means
-(e.g. by setting a maximum value for a config object).
+To avoid passing too large of a numeric parameter to a Qt function, all 
+numbers should be range-checked using `qutebrowser.qtutils.check_overflow`,
+or by other means (e.g. by setting a maximum value for a config object).
 
 [[object-registry]]
 The object registry
@@ -339,8 +338,8 @@ window=_win-id_, tab=_tab-id_])+. The default scope is `global`.
 All objects can be printed by starting with the `--debug` flag and using the
 `:debug-all-objects` command.
 
-The registry is mainly used for <<commands,command handlers>> but also can be
-useful in places where using Qt's
+The registry is mainly used for <<commands,command handlers>>, but it can
+also be useful in places where using Qt's
 http://doc.qt.io/qt-5/signalsandslots.html[signals and slots] mechanism would
 be difficult.
 
@@ -350,7 +349,7 @@ Logging
 Logging is used at various places throughout the qutebrowser code. If you add a
 new feature, you should also add some strategic debug logging.
 
-Unless other Python projects, qutebrowser doesn't use a logger per file,
+Unlike other Python projects, qutebrowser doesn't use a logger per file,
 instead it uses custom-named loggers.
 
 The existing loggers are defined in `qutebrowser.utils.log`. If your feature
@@ -413,11 +412,11 @@ selects which object registry (global, per-tab, etc.) to use.  See the
 <<object-registry,object registry>> section for details.
 
 There are also other arguments to customize the way the command is
-registered, see the class documentation for `register` in
+registered; see the class documentation for `register` in
 `qutebrowser.commands.cmdutils` for details.
 
 The types of the function arguments are inferred based on their default values,
-e.g. an argument `foo=True` will be converted to a flag `-f`/`--foo` in
+e.g., an argument `foo=True` will be converted to a flag `-f`/`--foo` in
 qutebrowser's commandline.
 
 The type can be overridden using Python's
@@ -435,11 +434,11 @@ Possible values:
       value.
     - A python enum type: All members of the enum are possible values.
     - A `typing.Union` of multiple types above: Any of these types are valid
-      values, e.g. `typing.Union[str, int]`
+      values, e.g., `typing.Union[str, int]`
 
 You can customize how an argument is handled using the `@cmdutils.argument`
-decorator *after* `@cmdutils.register`. This can e.g. be used to customize the
-flag an argument should get:
+decorator *after* `@cmdutils.register`. This can, for example, be used to 
+customize the flag an argument should get:
 
 [source,python]
 ----
@@ -482,13 +481,13 @@ qutebrowser handles two different types of URLs: URLs as a string, and URLs as
 the Qt `QUrl` type. As this can get confusing quickly, please follow the
 following guidelines:
 
-* Convert a string to a QUrl object as early as possible, i.e. directly after
+* Convert a string to a QUrl object as early as possible, i.e., directly after
 the user did enter it.
     - Use `utils.urlutils.fuzzy_url` if the URL is entered by the user
       somewhere.
     - Be sure you handle `utils.urlutils.FuzzyError` and display an error
       message to the user.
-* Convert a `QUrl` object to a string as late as possible, e.g. before
+* Convert a `QUrl` object to a string as late as possible, i.e., before
 displaying it to the user.
     - If you want to display the URL to the user, use `url.toDisplayString()`
       so password information is removed.
@@ -514,8 +513,8 @@ This is needed so valgrind handles self-modifying code correctly:
 [quote]
 ____
 This option controls Valgrind's detection of self-modifying code. If no
-checking is done, if a program executes some code, then overwrites it with new
-code, and executes the new code, Valgrind will continue to execute the
+checking is done and a program executes some code, overwrites it with new
+code, and then executes the new code, Valgrind will continue to execute the
 translations it made for the old code. This will likely lead to incorrect
 behavior and/or crashes.
 
@@ -527,6 +526,27 @@ generate code and subsequently overwrite part or all of it. Running with all
 will slow Valgrind down noticeably.
 ____
 
+Setting up a Windows Development Environment
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Install https://www.python.org/downloads/release/python-344/[Python 3.4]
+* Install https://sourceforge.net/projects/pyqt/files/PyQt5/PyQt-5.5.1/[PyQt 5.5]
+* Create a file at `C:\Windows\system32\python3.bat` with the following content:
+  `@C:\Python34\python %*`
+  This will make the Python 3.4 interpreter available as `python3`, which is used by various development scripts.
+* Install git from the https://git-scm.com/download/win[git-scm downloads page]
+  Try not to enable `core.autocrlf`, since that will cause `flake8` to complain a lot. Use an editor that can deal with plain line feeds instead.
+* Clone your favourite qutebrowser repository.
+* To install tox, open an elevated cmd, enter your working directory and run `pip install -rmisc/requirements/requirements-tox.txt`.
+
+Note that the `flake8` tox env might not run due to encoding errors despite having LANG/LC_* set correctly.
+
+Rebuilding the website
+~~~~~~~~~~~~~~~~~~~~~~
+
+If you want to rebuild the website, run `./scripts/asciidoc2html.py --website <outputdir>`.
+
+
 Style conventions
 -----------------
 
@@ -615,16 +635,8 @@ and make sure all bugs marked as resolved are actually fixed.
 * Grep for `WORKAROUND` in the code and test if fixed stuff works without the
 workaround.
 * Check relevant
-https://github.com/The-Compiler/qutebrowser/issues?q=is%3Aopen+is%3Aissue+label%3Aqt[qutebrowser
+https://github.com/qutebrowser/qutebrowser/issues?q=is%3Aopen+is%3Aissue+label%3Aqt[qutebrowser
 bugs] and check if they're fixed.
-* As soon as Homebrew updated, update the custom OS X bottle:
-  - Update https://github.com/The-Compiler/homebrew-qt5-webkit/blob/master/Formula/qt5.rb[qt5.rb]
-  - `brew install --build-from-source --build-bottle --verbose qt5.rb`
-  - `brew bottle qt5.rb`
-  - `brew install --build-from-source --build-bottle --verbose pyqt5`
-  - `brew bottle pyqt5`
-  - Upload bottles to github
-  - Adjust `scripts/dev/ci/travis_install.sh`
 
 New PyQt release
 ~~~~~~~~~~~~~~~~
@@ -632,11 +644,13 @@ New PyQt release
 * See above
 * Install new PyQt in Windows VM (32- and 64-bit)
 * Download new installer and update PyQt installer path in `ci_install.py`.
+* Update `tox.ini`/`.travis.yml`/`.appveyor.yml` to test new versions
 
 qutebrowser release
 ~~~~~~~~~~~~~~~~~~~
 
 * Make sure there are no unstaged changes and the tests are green.
+* Run `x=... y=...` to set the respective shell variables
 
 * Add newest config to `tests/unit/config/old_configs` and update `test_upgrade_version`
     - `python -m qutebrowser --basedir conf :quit`
@@ -644,30 +658,20 @@ qutebrowser release
     - `rm -r conf`
     - commit
 * Adjust `__version_info__` in `qutebrowser/__init__.py`.
-* Remove *(unreleased)* from changelog.
+* Update changelog (remove *(unreleased)*)
 * Run tests again
-* Run `asciidoc2html.py`.
 * Commit
 
-* Create annotated git tag (`git tag -s "v0.X.Y" -m "Release v0.X.Y"`)
+* Create annotated git tag (`git tag -s "v0.$x.$y" -m "Release v0.$x.$y"`)
+* `git push origin`; `git push origin v0.$x.$y`
 * If committing on minor branch, cherry-pick release commit to master.
-* `git push origin`; `git push origin v0.X.Y`
 * Create release on github
-* Mark the milestone at https://github.com/The-Compiler/qutebrowser/milestones
+* Mark the milestone at https://github.com/qutebrowser/qutebrowser/milestones
 as closed.
 
-* Run `scripts/dev/build_release.py` on Linux to build an sdist
-* Upload to PyPI: `twine upload dist/foo{,.asc}`
-* Create Windows packages via `C:\Python34_x32\python scripts\dev\build_release.py --asciidoc C:\Python27\python C:\asciidoc-8.6.9\asciidoc.py`
-* Upload to github
-
-* Upload to qutebrowser.org with checksum/GPG
-    - On server: `sudo mkdir -p /srv/http/qutebrowser/releases/v0.X.Y/windows`
-    - `rsync -avPh dist/ tonks:`
-    - On server: `sudo mv qutebrowser-0.X.Y.tar.gz* /srv/http/qutebrowser/releases/v0.X.Y`
-    - Upload windows release:
-           - `scp bb-win8:proj/qutebrowser/qutebrowser-0.X.Y-windows.zip .`
-           - `aunpack qutebrowser-0.X.Y-windows.zip`
-           - `sudo mv qutebrowser-0.X.Y-windows/* /srv/http/qutebrowser/releases/v0.X.Y/windows`
+* Linux: Run `python3 scripts/dev/build_release.py --upload v0.$x.$y`
+* Windows: Run `C:\Python34_x32\python scripts\dev\build_release.py --asciidoc C:\Python27\python C:\asciidoc-8.6.9\asciidoc.py --upload v0.X.Y` (replace X/Y by hand)
+* OS X: Run `python3 scripts/dev/build_release.py --upload v0.X.Y` (replace X/Y by hand)
+* On server: Run `python3 scripts/dev/download_release.py v0.X.Y` (replace X/Y by hand)
 * Update `qutebrowser-git` PKGBUILD if dependencies/install changed
-* Announce to qutebrowser mailinglist
+* Announce to qutebrowser and qutebrowser-announce mailinglist

FAQ.asciidoc

@@ -105,13 +105,25 @@ It also works nicely with rapid hints:
 How do I use qutebrowser with mutt?::
     Due to a Qt limitation, local files without `.html` extensions are
     "downloaded" instead of displayed, see
-    https://github.com/The-Compiler/qutebrowser/issues/566[#566]. You can work
+    https://github.com/qutebrowser/qutebrowser/issues/566[#566]. You can work
     around this by using this in your `mailcap`:
 +
 ----
     text/html; mv %s %s.html && qutebrowser %s.html >/dev/null 2>/dev/null; needsterminal;
 ----
 
+What is the difference between bookmarks and quickmarks?::
+    Bookmarks will always use the title of the website as their name, but with quickmarks
+    you can set your own title.
++
+For example, if you bookmark multiple food recipe websites and use `:open`, 
+you have to type the title or address of the website.
++
+When using quickmark, you can give them all names, like
+`foodrecipes1`, `foodrecipes2` and so on. When you type 
+`:open foodrecipes`, you will see a list of all the food recipe sites,
+without having to remember the exact website title or address.
+
 == Troubleshooting
 
 Configuration not saved after modifying config.::
@@ -129,7 +141,7 @@ Experiencing freezing on sites like duckduckgo and youtube.::
     This issue could be caused by stale plugin files installed by `mozplugger`
     if mozplugger was subsequently removed.
     Try exiting qutebrowser and removing `~/.mozilla/plugins/mozplugger*.so`.
-    See https://github.com/The-Compiler/qutebrowser/issues/357[Issue #357]
+    See https://github.com/qutebrowser/qutebrowser/issues/357[Issue #357]
     for more details.
 
 Experiencing segfaults (crashes) on Debian systems.::
@@ -143,7 +155,7 @@ Segfaults on Facebook, Medium, Amazon, ...::
     visiting these sites. This is caused by various bugs in Qt which have been
     fixed in Qt 5.4. However Debian and Ubuntu are slow to adopt or upgrade
     some packages. On Debian Jessie, it's recommended to use the experimental
-    repos as described in https://github.com/The-Compiler/qutebrowser/blob/master/INSTALL.asciidoc#on-debian--ubuntu[INSTALL].
+    repos as described in https://github.com/qutebrowser/qutebrowser/blob/master/INSTALL.asciidoc#on-debian--ubuntu[INSTALL].
 +
 Since Ubuntu Trusty (using Qt 5.2.1),
 https://bugreports.qt.io/browse/QTBUG-42417?jql=component%20%3D%20WebKit%20and%20resolution%20%3D%20Done%20and%20fixVersion%20in%20(5.3.0%2C%20%225.3.0%20Alpha%22%2C%20%225.3.0%20Beta1%22%2C%20%225.3.0%20RC1%22%2C%205.3.1%2C%205.3.2%2C%205.4.0%2C%20%225.4.0%20Alpha%22%2C%20%225.4.0%20Beta%22%2C%20%225.4.0%20RC%22)%20and%20priority%20in%20(%22P2%3A%20Important%22%2C%20%22P1%3A%20Critical%22%2C%20%22P0%3A%20Blocker%22)[over
@@ -154,7 +166,7 @@ https://bugreports.qt.io/browse/QTBUG-42417?jql=component%20%3D%20WebKit%20and%2
 
 My issue is not listed.::
     If you experience any segfaults or crashes, you can report the issue in
-    https://github.com/The-Compiler/qutebrowser/issues[the issue tracker] or
+    https://github.com/qutebrowser/qutebrowser/issues[the issue tracker] or
     using the `:report` command.
     If you are reporting a segfault, make sure you read the
     link:doc/stacktrace.asciidoc[guide] on how to report them with all needed

INSTALL.asciidoc

@@ -21,10 +21,12 @@ Using the packages
 Install the dependencies via apt-get:
 
 ----
-# apt-get install python3-lxml python-tox python3-pyqt5 python3-pyqt5.qtwebkit python3-sip python3-jinja2 python3-pygments python3-yaml
+# apt-get install python3-lxml python-tox python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python3-sip python3-jinja2 python3-pygments python3-yaml
 ----
 
-Get the packages from the https://github.com/The-Compiler/qutebrowser/releases[release page]
+Get the qutebrowser package from the
+https://github.com/qutebrowser/qutebrowser/releases[release page] and download
+the https://qutebrowser.org/python3-pypeg2_2.15.2-1_all.deb[PyPEG2 package].
 
 Install the packages:
 
@@ -54,7 +56,7 @@ Then install the packages like this:
 
 ----
 # apt-get update
-# apt-get install -t experimental python3-pyqt5 python3-pyqt5.qtwebkit python3-sip python3-dev
+# apt-get install -t experimental python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python3-sip python3-dev
 # apt-get install python-tox
 ----
 
@@ -72,7 +74,7 @@ For distributions other than Debian or if you prefer to not use the
 experimental repo:
 
 ----
-# apt-get install python3-pyqt5 python3-pyqt5.qtwebkit python-tox python3-sip python3-dev
+# apt-get install python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python-tox python3-sip python3-dev
 ----
 
 To generate the documentation for the `:help` command, when using the git
@@ -185,6 +187,24 @@ Or add the repo manually:
 # zypper install qutebrowser
 ----
 
+On OpenBSD
+----------
+
+qutebrowser is in http://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/www/qutebrowser/[OpenBSD ports].
+
+Manual install:
+
+----
+# cd /usr/ports/www/qutebrowser
+# make install
+----
+
+Or alternatively if you're using `-current` (or OpenBSD 6.1 once it's been released):
+
+----
+# pkg_add qutebrowser
+----
+
 On Windows
 ----------
 
@@ -194,7 +214,7 @@ Prebuilt binaries
 ~~~~~~~~~~~~~~~~~
 
 Prebuilt standalone packages and MSI installers
-https://github.com/The-Compiler/qutebrowser/releases[are built] for every
+https://github.com/qutebrowser/qutebrowser/releases[are built] for every
 release.
 
 https://chocolatey.org/packages/qutebrowser[Chocolatey package]
@@ -229,41 +249,44 @@ Then <<tox,install qutebrowser via tox>>.
 On OS X
 -------
 
+Prebuilt binary
+~~~~~~~~~~~~~~~
+
 The easiest way to install qutebrowser on OS X is to use the prebuilt `.app`
 files from the
-https://github.com/The-Compiler/qutebrowser/releases[release page].
+https://github.com/qutebrowser/qutebrowser/releases[release page].
+
+This binary is also available through the
+https://caskroom.github.io/[Homebrew Cask] package manager:
+
+----
+$ brew cask install qutebrowser
+----
+
+Manual Install
+~~~~~~~~~~~~~~
 
 Alternatively, you can install the dependencies via a package manager (like
 http://brew.sh/[Homebrew] or https://www.macports.org/[MacPorts]) and run
 qutebrowser from source.
 
-For Homebrew, a few extra steps are necessary since Homebrew dropped QtWebKit
-from Qt 5.6 - however, some users reported this didn't work for them, so using
-the `.app` is strongly encouraged.
-
-This installs a Qt 5.5 and symlinks it so PyQt5 will work with it instead of Qt
-5.6. This requires that `qt5` is not installed via Homebrew:
+==== Homebrew
 
 ----
-$ brew install python3 d-bus mysql sip xz
-$ brew install homebrew/versions/qt55
-$ brew install --ignore-dependencies pyqt5
-$ ln -s /usr/local/opt/qt55 /usr/local/opt/qt5
-
-$ pip3.5 install qutebrowser
+$ brew install qt5
+$ pip3 install qutebrowser
 ----
 
-For MacPorts, run:
+Homebrew's builds of Qt and PyQt no longer include QtWebKit - if you need
+QtWebKit support, it is necessary to build from source. The build takes several
+hours on an average laptop.
 
 ----
-$ sudo port install python34 py34-jinja2 asciidoc py34-pygments py34-pyqt5
-$ sudo pip3.4 install qutebrowser
+$ brew install qt5 --with-qtwebkit
+$ brew install -s pyqt5
+$ pip3 install qutebrowser
 ----
 
-The preferences for qutebrowser are stored in
-`~/Library/Preferences/qutebrowser`, the application data is stored in
-`~/Library/Application Support/qutebrowser`.
-
 Packagers
 ---------
 
@@ -282,7 +305,7 @@ First of all, clone the repository using http://git-scm.org/[git] and switch
 into the repository folder:
 
 ----
-$ git clone https://github.com/The-Compiler/qutebrowser.git
+$ git clone https://github.com/qutebrowser/qutebrowser.git
 $ cd qutebrowser
 ----
 
@@ -294,6 +317,12 @@ https://docs.python.org/3/library/venv.html[virtual environment]:
 $ tox -e mkvenv
 ----
 
+On Windows, run tox with the 'mkvenv-win' option, however make sure that ONLY Python3 is in your PATH before running tox.
+
+----
+$ tox -e mkvenv-win
+----
+
 This installs all needed Python dependencies in a `.venv` subfolder. The
 system-wide Qt5/PyQt5 installations are symlinked into the virtual environment.
 

MANIFEST.in

@@ -35,8 +35,8 @@ exclude pytest.ini
 exclude qutebrowser.rcc
 exclude .coveragerc
 exclude .pylintrc
-exclude .eslintrc
-exclude .eslintignore
+exclude qutebrowser/javascript/.eslintrc.yaml
+exclude qutebrowser/javascript/.eslintignore
 exclude doc/help
 exclude .appveyor.yml
 exclude .travis.yml

README.asciidoc

@@ -1,26 +1,26 @@
 // If you are reading this in plaintext or on PyPi:
 //
 // A rendered version is available at:
-// https://github.com/The-Compiler/qutebrowser/blob/master/README.asciidoc
+// https://github.com/qutebrowser/qutebrowser/blob/master/README.asciidoc
 
 qutebrowser
 ===========
 
 // QUTE_WEB_HIDE
-image:icons/qutebrowser-64x64.png[qutebrowser logo] *A keyboard-driven, vim-like browser based on PyQt5 and QtWebKit.*
+image:icons/qutebrowser-64x64.png[qutebrowser logo] *A keyboard-driven, vim-like browser based on PyQt5 and Qt.*
 
-image:https://img.shields.io/pypi/l/qutebrowser.svg?style=flat["license badge",link="https://github.com/The-Compiler/qutebrowser/blob/master/COPYING"]
+image:https://img.shields.io/pypi/l/qutebrowser.svg?style=flat["license badge",link="https://github.com/qutebrowser/qutebrowser/blob/master/COPYING"]
 image:https://img.shields.io/pypi/v/qutebrowser.svg?style=flat["version badge",link="https://pypi.python.org/pypi/qutebrowser/"]
-image:https://requires.io/github/The-Compiler/qutebrowser/requirements.svg?branch=master["requirements badge",link="https://requires.io/github/The-Compiler/qutebrowser/requirements/?branch=master"]
-image:https://travis-ci.org/The-Compiler/qutebrowser.svg?branch=master["Build Status", link="https://travis-ci.org/The-Compiler/qutebrowser"]
-image:https://ci.appveyor.com/api/projects/status/9gmnuip6i1oq7046?svg=true["AppVeyor build status", link="https://ci.appveyor.com/project/The-Compiler/qutebrowser"]
-image:https://codecov.io/github/The-Compiler/qutebrowser/coverage.svg?branch=master["coverage badge",link="https://codecov.io/github/The-Compiler/qutebrowser?branch=master"]
+image:https://requires.io/github/qutebrowser/qutebrowser/requirements.svg?branch=master["requirements badge",link="https://requires.io/github/qutebrowser/qutebrowser/requirements/?branch=master"]
+image:https://travis-ci.org/qutebrowser/qutebrowser.svg?branch=master["Build Status", link="https://travis-ci.org/qutebrowser/qutebrowser"]
+image:https://ci.appveyor.com/api/projects/status/5pyauww2k68bbow2/branch/master?svg=true["AppVeyor build status", link="https://ci.appveyor.com/project/qutebrowser/qutebrowser"]
+image:https://codecov.io/github/qutebrowser/qutebrowser/coverage.svg?branch=master["coverage badge",link="https://codecov.io/github/qutebrowser/qutebrowser?branch=master"]
 
-link:http://www.qutebrowser.org[website] | link:http://blog.qutebrowser.org[blog] | link:https://github.com/The-Compiler/qutebrowser/releases[releases]
+link:https://www.qutebrowser.org[website] | link:https://blog.qutebrowser.org[blog] | link:https://github.com/qutebrowser/qutebrowser/releases[releases]
 // QUTE_WEB_HIDE_END
 
 qutebrowser is a keyboard-focused browser with a minimal GUI. It's based
-on Python, PyQt5 and QtWebKit and free software, licensed under the GPL.
+on Python and PyQt5 and free software, licensed under the GPL.
 
 It was inspired by other browsers/addons like dwb and Vimperator/Pentadactyl.
 
@@ -35,7 +35,7 @@ image:doc/img/hints.png["screenshot 4",width=300,link="doc/img/hints.png"]
 Downloads
 ---------
 
-See the https://github.com/The-Compiler/qutebrowser/releases[github releases
+See the https://github.com/qutebrowser/qutebrowser/releases[github releases
 page] for available downloads (currently a source archive, and standalone
 packages as well as MSI installers for Windows).
 
@@ -48,9 +48,10 @@ Documentation
 In addition to the topics mentioned in this README, the following documents are
 available:
 
-* A http://qutebrowser.org/img/cheatsheet-big.png[key binding cheatsheet]: +
-image:http://qutebrowser.org/img/cheatsheet-small.png["qutebrowser key binding cheatsheet",link="http://qutebrowser.org/img/cheatsheet-big.png"]
+* A https://qutebrowser.org/img/cheatsheet-big.png[key binding cheatsheet]: +
+image:https://qutebrowser.org/img/cheatsheet-small.png["qutebrowser key binding cheatsheet",link="https://qutebrowser.org/img/cheatsheet-big.png"]
 * link:doc/quickstart.asciidoc[Quick start guide]
+* A https://www.shortcutfoo.com/app/dojos/qutebrowser[free training course] to remember those key bindings.
 * link:FAQ.asciidoc[Frequently asked questions]
 * link:CONTRIBUTING.asciidoc[Contributing to qutebrowser]
 * link:INSTALL.asciidoc[INSTALL]
@@ -69,6 +70,9 @@ message to the
 https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser[mailinglist] at
 mailto:qutebrowser@lists.qutebrowser.org[].
 
+There's also a https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser-announce[announce-only mailinglist]
+at mailto:qutebrowser-announce@lists.qutebrowser.org[].
+
 Contributions / Bugs
 --------------------
 
@@ -86,7 +90,7 @@ https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser[mailinglist] at
 mailto:qutebrowser@lists.qutebrowser.org[].
 
 For security bugs, please contact me directly at mail@qutebrowser.org, GPG ID
-http://www.the-compiler.org/pubkey.asc[0xFD55A072].
+https://www.the-compiler.org/pubkey.asc[0xFD55A072].
 
 Requirements
 ------------
@@ -95,7 +99,7 @@ The following software and libraries are required to run qutebrowser:
 
 * http://www.python.org/[Python] 3.4 or newer
 * http://qt.io/[Qt] 5.2.0 or newer (5.5.1 recommended)
-* QtWebKit
+* QtWebKit (old or link:https://github.com/annulen/webkit/wiki[reloaded]/NG) or QtWebEngine
 * http://www.riverbankcomputing.com/software/pyqt/intro[PyQt] 5.2.0 or newer
 (5.5.1 recommended) for Python 3
 * https://pypi.python.org/pypi/setuptools/[pkg_resources/setuptools]
@@ -142,47 +146,64 @@ Contributors, sorted by the number of commits in descending order:
 * Florian Bruhin
 * Daniel Schadt
 * Ryan Roden-Corrent
+* Jan Verbeek
+* Jakub Klinkovský
 * Antoni Boucher
 * Lamar Pavel
+* Marshall Lochbaum
 * Bruno Oliveira
 * Alexander Cogneau
 * Felix Van der Jeugt
+* Daniel Karbach
 * Martin Tournoij
-* Jakub Klinkovský
+* Kevin Velghe
 * Raphael Pierzina
 * Joel Torstensson
-* Jan Verbeek
-* Tarcisio Fedrizzi
 * Patric Schmitz
+* Tarcisio Fedrizzi
 * Claude
 * Corentin Julé
 * meles5
 * Philipp Hansch
+* Imran Sobir
 * Panagiotis Ktistakis
 * Artur Shaik
 * Nathan Isom
 * Thorsten Wißmann
-* Kevin Velghe
 * Austin Anderson
+* Fritz Reichwald
 * Jimmy
-* Marshall Lochbaum
+* Niklas Haas
+* Maciej Wołczyk
+* Spreadyy
 * Alexey "Averrin" Nabrodov
+* nanjekyejoannah
 * avk
 * ZDarian
 * Milan Svoboda
 * John ShaggyTwoDope Jenkins
-* Peter Vilim
 * Clayton Craft
+* Peter Vilim
+* knaggita
 * Oliver Caldwell
+* Julian Weigt
+* Tomasz Kramkowski
+* Sebastian Frysztak
+* Nikolay Amiantov
+* Julie Engel
 * Jonas Schürmann
 * error800
+* Michael Hoang
 * Liam BEGUIN
+* Daniel Fiser
 * skinnay
 * Zach-Button
-* Tomasz Kramkowski
+* Samuel Walladge
+* Peter Rice
 * Ismail S
 * Halfwit
 * David Vogt
+* Claire Cavanaugh
 * rikn00
 * kanikaa1234
 * haitaka
@@ -190,9 +211,12 @@ Contributors, sorted by the number of commits in descending order:
 * Michał Góral
 * Michael Ilsaas
 * Martin Zimmermann
-* Fritz Reichwald
+* Jussi Timperi
+* Cosmin Popescu
 * Brian Jackson
+* thuck
 * sbinix
+* rsteube
 * neeasade
 * jnphilipp
 * Tobias Patzl
@@ -200,37 +224,51 @@ Contributors, sorted by the number of commits in descending order:
 * Samuel Loury
 * Peter Michely
 * Panashe M. Fundira
+* Lucas Hoffmann
 * Link
 * Larry Hynes
+* Kirill A. Shutemov
 * Johannes Altmanninger
 * Jeremy Kaplan
 * Ismail
 * Edgar Hipp
 * Daryl Finlay
+* pkill9
+* arza
 * adam
 * Samir Benmendil
 * Regina Hug
 * Mathias Fussenegger
 * Marcelo Santos
+* Joel Bradshaw
 * Jean-Louis Fuchs
-* Fritz V155 Reichwald
 * Franz Fellner
+* Eric Drechsel
 * zwarag
 * xd1le
+* rmortens
 * oniondreams
 * issue
 * haxwithaxe
 * evan
 * dylan araps
+* addictedtoflames
 * Xitian9
 * Tomas Orsava
+* Tom Janson
 * Tobias Werth
 * Tim Harder
 * Thiago Barroso Perrotta
 * Sorokin Alexei
+* Simon Désaulniers
+* Rok Mandeljc
 * Noah Huesser
+* Moez Bouhlel
 * Matthias Lisin
 * Marcel Schilling
+* Lazlow Carmichael
+* Kevin Wang
+* Ján Kobezda
 * Johannes Martinsson
 * Jean-Christophe Petkovich
 * Jay Kamat
@@ -239,10 +277,12 @@ Contributors, sorted by the number of commits in descending order:
 * Gregor Pohl
 * Eivind Uggedal
 * Dietrich Daroch
+* Derek Sivers
 * Daniel Lu
 * Arseniy Seroka
 * Andy Balaam
 * Andreas Fischer
+* Akselmo
 // QUTE_AUTHORS_END
 
 The following people have contributed graphics:

doc/help/commands.asciidoc

@@ -1,5 +1,23 @@
+// DO NOT EDIT THIS FILE DIRECTLY!
+// It is autogenerated from docstrings by running:
+//   $ python3 scripts/dev/src2asciidoc.py
+
 = Commands
 
+In qutebrowser, all keybindings are mapped to commands.
+
+Some commands are hidden, which means they don't show up in the command
+completion when pressing `:`, as they're typically not useful to run by hand.
+
+For command arguments, there are also some variables you can use:
+
+- `{url}` expands to the URL of the current page
+- `{url:pretty}` expands to the URL in decoded format
+- `{clipboard}` expands to the clipboard contents
+- `{primary}` expands to the primary selection contents
+
+It is possible to run or bind multiple commands by separating them with `;;`.
+
 == Normal commands
 .Quick reference
 [options="header",width="75%",cols="25%,75%"]
@@ -26,26 +44,28 @@
 |<<fullscreen,fullscreen>>|Toggle fullscreen mode.
 |<<help,help>>|Show help about a command or setting.
 |<<hint,hint>>|Start hinting.
+|<<history,history>>|Show browsing history.
 |<<history-clear,history-clear>>|Clear all browsing history.
 |<<home,home>>|Open main startpage in current tab.
+|<<insert-text,insert-text>>|Insert text at cursor position.
 |<<inspector,inspector>>|Toggle the web inspector.
 |<<jseval,jseval>>|Evaluate a JavaScript string.
-|<<jump-mark,jump-mark>>|Jump to the mark named by `key`.
 |<<later,later>>|Execute a command after some time.
 |<<messages,messages>>|Show a log of past messages.
 |<<navigate,navigate>>|Open typical prev/next links or navigate using the URL path.
 |<<open,open>>|Open a URL in the current/[count]th tab.
-|<<paste,paste>>|Open a page from the clipboard.
 |<<print,print>>|Print the current/[count]th tab.
 |<<quickmark-add,quickmark-add>>|Add a new quickmark.
 |<<quickmark-del,quickmark-del>>|Delete a quickmark.
 |<<quickmark-load,quickmark-load>>|Load a quickmark.
 |<<quickmark-save,quickmark-save>>|Save the current page as a quickmark.
 |<<quit,quit>>|Quit qutebrowser.
+|<<record-macro,record-macro>>|Start or stop recording a macro.
 |<<reload,reload>>|Reload the current/[count]th tab.
 |<<repeat,repeat>>|Repeat a given command.
 |<<report,report>>|Report a bug in qutebrowser.
 |<<restart,restart>>|Restart qutebrowser while keeping existing tabs open.
+|<<run-macro,run-macro>>|Run a recorded macro.
 |<<save,save>>|Save configs and state.
 |<<search,search>>|Search for a text on the current page. With no text, clear results.
 |<<session-delete,session-delete>>|Delete a session.
@@ -53,23 +73,22 @@
 |<<session-save,session-save>>|Save a session.
 |<<set,set>>|Set an option.
 |<<set-cmd-text,set-cmd-text>>|Preset the statusbar to some text.
-|<<set-mark,set-mark>>|Set a mark at the current scroll position in the current tab.
 |<<spawn,spawn>>|Spawn a command in a shell.
 |<<stop,stop>>|Stop loading in the current/[count]th tab.
 |<<tab-clone,tab-clone>>|Duplicate the current tab.
 |<<tab-close,tab-close>>|Close the current/[count]th tab.
 |<<tab-detach,tab-detach>>|Detach the current tab to its own window.
 |<<tab-focus,tab-focus>>|Select the tab given as argument/[count].
-|<<tab-move,tab-move>>|Move the current tab.
+|<<tab-move,tab-move>>|Move the current tab according to the argument and [count].
 |<<tab-next,tab-next>>|Switch to the next tab, or switch [count] tabs forward.
 |<<tab-only,tab-only>>|Close all tabs except for the current one.
 |<<tab-prev,tab-prev>>|Switch to the previous tab, or switch [count] tabs back.
 |<<unbind,unbind>>|Unbind a keychain.
 |<<undo,undo>>|Re-open a closed tab (optionally skipping [count] closed tabs).
-|<<view-source,view-source>>|Show the source of the current page.
+|<<view-source,view-source>>|Show the source of the current page in a new tab.
+|<<window-only,window-only>>|Close all windows except for the current one.
 |<<wq,wq>>|Save open pages and quit.
-|<<yank,yank>>|Yank the current URL/title to the clipboard or primary selection.
-|<<yank-selected,yank-selected>>|Yank the selected text to the clipboard or primary selection.
+|<<yank,yank>>|Yank something to the clipboard or primary selection.
 |<<zoom,zoom>>|Set the zoom level for the current tab.
 |<<zoom-in,zoom-in>>|Increase the zoom level for the current tab.
 |<<zoom-out,zoom-out>>|Decrease the zoom level for the current tab.
@@ -78,7 +97,7 @@
 === adblock-update
 Update the adblock block lists.
 
-This updates ~/.local/share/qutebrowser/blocked-hosts with downloaded host lists and re-reads ~/.config/qutebrowser/blocked-hosts.
+This updates `~/.local/share/qutebrowser/blocked-hosts` with downloaded host lists and re-reads `~/.config/qutebrowser/blocked-hosts`.
 
 [[back]]
 === back
@@ -113,10 +132,11 @@ Bind a key to a command.
 ==== note
 * This command does not split arguments after the last argument and handles quotes literally.
 * With this command, +;;+ is interpreted literally instead of splitting off a second command.
+* This command does not replace variables like +\{url\}+.
 
 [[bookmark-add]]
 === bookmark-add
-Syntax: +:bookmark-add ['url'] ['title']+
+Syntax: +:bookmark-add [*--toggle*] ['url'] ['title']+
 
 Save the current page as a bookmark, or a specific url.
 
@@ -126,6 +146,10 @@ If no url and title are provided, then save the current page as a bookmark. If a
 * +'url'+: url to save as a bookmark. If None, use url of current page.
 * +'title'+: title of the new bookmark.
 
+==== optional arguments
+* +*-t*+, +*--toggle*+: remove the bookmark instead of raising an error if it already exists.
+
+
 [[bookmark-del]]
 === bookmark-del
 Syntax: +:bookmark-del ['url']+
@@ -141,7 +165,7 @@ Delete a bookmark.
 
 [[bookmark-load]]
 === bookmark-load
-Syntax: +:bookmark-load [*--tab*] [*--bg*] [*--window*] 'url'+
+Syntax: +:bookmark-load [*--tab*] [*--bg*] [*--window*] [*--delete*] 'url'+
 
 Load a bookmark.
 
@@ -152,6 +176,7 @@ Load a bookmark.
 * +*-t*+, +*--tab*+: Load the bookmark in a new tab.
 * +*-b*+, +*--bg*+: Load the bookmark in a new background tab.
 * +*-w*+, +*--window*+: Load the bookmark in a new window.
+* +*-d*+, +*--delete*+: Whether to delete the bookmark afterwards.
 
 ==== note
 * This command does not split arguments after the last argument and handles quotes literally.
@@ -212,11 +237,24 @@ The index of the download to delete.
 
 [[download-open]]
 === download-open
+Syntax: +:download-open ['cmdline']+
+
 Open the last/[count]th download.
 
+If no specific command is given, this will use the system's default application to open the file.
+
+==== positional arguments
+* +'cmdline'+: The command which should be used to open the file. A `{}` is expanded to the temporary file name. If no `{}` is
+ present, the filename is automatically appended to the
+ cmdline.
+
+
 ==== count
 The index of the download to open.
 
+==== note
+* This command does not split arguments after the last argument and handles quotes literally.
+
 [[download-remove]]
 === download-remove
 Syntax: +:download-remove [*--all*]+
@@ -282,8 +320,13 @@ How many pages to go forward.
 
 [[fullscreen]]
 === fullscreen
+Syntax: +:fullscreen [*--leave*]+
+
 Toggle fullscreen mode.
 
+==== optional arguments
+* +*-l*+, +*--leave*+: Only leave fullscreen if it was entered by the page.
+
 [[help]]
 === help
 Syntax: +:help [*--tab*] [*--bg*] [*--window*] ['topic']+
@@ -304,12 +347,13 @@ Show help about a command or setting.
 
 [[hint]]
 === hint
-Syntax: +:hint [*--rapid*] ['group'] ['target'] ['args' ['args' ...]]+
+Syntax: +:hint [*--rapid*] [*--mode* 'mode'] [*--add-history*]
+     ['group'] ['target'] ['args' ['args' ...]]+
 
 Start hinting.
 
 ==== positional arguments
-* +'group'+: The hinting mode to use. 
+* +'group'+: The element types to hint. 
 
  - `all`: All clickable elements.
  - `links`: Only links.
@@ -360,20 +404,59 @@ Start hinting.
 * +*-r*+, +*--rapid*+: Whether to do rapid hinting. This is only possible with targets `tab` (with background-tabs=true), `tab-bg`,
  `window`, `run`, `hover`, `userscript` and `spawn`.
 
+* +*-m*+, +*--mode*+: The hinting mode to use. 
+
+ - `number`: Use numeric hints.
+ - `letter`: Use the chars in the hints->chars settings.
+ - `word`: Use hint words based on the html elements and the
+ extra words.
+ 
+
+
+* +*-a*+, +*--add-history*+: Whether to add the spawned or yanked link to the browsing history.
+
 
 ==== note
 * This command does not split arguments after the last argument and handles quotes literally.
 
+[[history]]
+=== history
+Syntax: +:history [*--tab*] [*--bg*] [*--window*]+
+
+Show browsing history.
+
+==== optional arguments
+* +*-t*+, +*--tab*+: Open in a new tab.
+* +*-b*+, +*--bg*+: Open in a background tab.
+* +*-w*+, +*--window*+: Open in a new window.
+
 [[history-clear]]
 === history-clear
+Syntax: +:history-clear [*--force*]+
+
 Clear all browsing history.
 
 Note this only clears the global history (e.g. `~/.local/share/qutebrowser/history` on Linux) but not cookies, the back/forward history of a tab, cache or other persistent data.
 
+==== optional arguments
+* +*-f*+, +*--force*+: Don't ask for confirmation.
+
 [[home]]
 === home
 Open main startpage in current tab.
 
+[[insert-text]]
+=== insert-text
+Syntax: +:insert-text 'text'+
+
+Insert text at cursor position.
+
+==== positional arguments
+* +'text'+: The text to insert.
+
+==== note
+* This command does not split arguments after the last argument and handles quotes literally.
+
 [[inspector]]
 === inspector
 Toggle the web inspector.
@@ -382,29 +465,23 @@ Note: Due a bug in Qt, the inspector will show incorrect request headers in the
 
 [[jseval]]
 === jseval
-Syntax: +:jseval [*--quiet*] 'js-code'+
+Syntax: +:jseval [*--file*] [*--quiet*] [*--world* 'world'] 'js-code'+
 
 Evaluate a JavaScript string.
 
 ==== positional arguments
-* +'js-code'+: The string to evaluate.
+* +'js-code'+: The string/file to evaluate.
 
 ==== optional arguments
+* +*-f*+, +*--file*+: Interpret js-code as a path to a file.
 * +*-q*+, +*--quiet*+: Don't show resulting JS object.
+* +*-w*+, +*--world*+: Ignored on QtWebKit. On QtWebEngine, a world ID or name to run the snippet in.
+
 
 ==== note
 * This command does not split arguments after the last argument and handles quotes literally.
 * With this command, +;;+ is interpreted literally instead of splitting off a second command.
 
-[[jump-mark]]
-=== jump-mark
-Syntax: +:jump-mark 'key'+
-
-Jump to the mark named by `key`.
-
-==== positional arguments
-* +'key'+: mark identifier; capital indicates a global mark
-
 [[later]]
 === later
 Syntax: +:later 'ms' 'command'+
@@ -418,6 +495,7 @@ Execute a command after some time.
 ==== note
 * This command does not split arguments after the last argument and handles quotes literally.
 * With this command, +;;+ is interpreted literally instead of splitting off a second command.
+* This command does not replace variables like +\{url\}+.
 
 [[messages]]
 === messages
@@ -460,16 +538,24 @@ This tries to automatically click on typical _Previous Page_ or _Next Page_ link
 * +*-b*+, +*--bg*+: Open in a background tab.
 * +*-w*+, +*--window*+: Open in a new window.
 
+==== count
+For `increment` and `decrement`, the number to change the URL by. For `up`, the number of levels to go up in the URL.
+
+
 [[open]]
 === open
-Syntax: +:open [*--bg*] [*--tab*] [*--window*] ['url']+
+Syntax: +:open [*--implicit*] [*--bg*] [*--tab*] [*--window*] ['url']+
 
 Open a URL in the current/[count]th tab.
 
+If the URL contains newlines, each line gets opened in its own tab.
+
 ==== positional arguments
 * +'url'+: The URL to open.
 
 ==== optional arguments
+* +*-i*+, +*--implicit*+: If opening a new tab, treat the tab as implicit (like clicking on a link).
+
 * +*-b*+, +*--bg*+: Open in a new background tab.
 * +*-t*+, +*--tab*+: Open in a new tab.
 * +*-w*+, +*--window*+: Open in a new window.
@@ -480,20 +566,6 @@ The tab index to open the URL in.
 ==== note
 * This command does not split arguments after the last argument and handles quotes literally.
 
-[[paste]]
-=== paste
-Syntax: +:paste [*--sel*] [*--tab*] [*--bg*] [*--window*]+
-
-Open a page from the clipboard.
-
-If the pasted text contains newlines, each line gets opened in its own tab.
-
-==== optional arguments
-* +*-s*+, +*--sel*+: Use the primary selection instead of the clipboard.
-* +*-t*+, +*--tab*+: Open in a new tab.
-* +*-b*+, +*--bg*+: Open in a background tab.
-* +*-w*+, +*--window*+: Open in new window.
-
 [[print]]
 === print
 Syntax: +:print [*--preview*] [*--pdf* 'file']+
@@ -558,6 +630,15 @@ Save the current page as a quickmark.
 === quit
 Quit qutebrowser.
 
+[[record-macro]]
+=== record-macro
+Syntax: +:record-macro ['register']+
+
+Start or stop recording a macro.
+
+==== positional arguments
+* +'register'+: Which register to store the macro in.
+
 [[reload]]
 === reload
 Syntax: +:reload [*--force*]+
@@ -583,6 +664,7 @@ Repeat a given command.
 ==== note
 * This command does not split arguments after the last argument and handles quotes literally.
 * With this command, +;;+ is interpreted literally instead of splitting off a second command.
+* This command does not replace variables like +\{url\}+.
 
 [[report]]
 === report
@@ -592,6 +674,18 @@ Report a bug in qutebrowser.
 === restart
 Restart qutebrowser while keeping existing tabs open.
 
+[[run-macro]]
+=== run-macro
+Syntax: +:run-macro ['register']+
+
+Run a recorded macro.
+
+==== positional arguments
+* +'register'+: Which macro to run.
+
+==== count
+How many times to run the macro.
+
 [[save]]
 === save
 Syntax: +:save ['what' ['what' ...]]+
@@ -647,7 +741,8 @@ Load a session.
 
 [[session-save]]
 === session-save
-Syntax: +:session-save [*--current*] [*--quiet*] [*--force*] ['name']+
+Syntax: +:session-save [*--current*] [*--quiet*] [*--force*] [*--only-active-window*]
+             ['name']+
 
 Save a session.
 
@@ -659,10 +754,11 @@ Save a session.
 * +*-c*+, +*--current*+: Save the current session instead of the default.
 * +*-q*+, +*--quiet*+: Don't show confirmation message.
 * +*-f*+, +*--force*+: Force saving internal sessions (starting with an underline).
+* +*-o*+, +*--only-active-window*+: Saves only tabs of the currently active window.
 
 [[set]]
 === set
-Syntax: +:set [*--temp*] [*--print*] ['section'] ['option'] ['value']+
+Syntax: +:set [*--temp*] [*--print*] ['section'] ['option'] ['values' ['values' ...]]+
 
 Set an option.
 
@@ -671,7 +767,7 @@ If the option name ends with '?', the value of the option is shown instead. If t
 ==== positional arguments
 * +'section'+: The section where the option is in.
 * +'option'+: The name of the option.
-* +'value'+: The value to set.
+* +'values'+: The value to set, or the values to cycle through.
 
 ==== optional arguments
 * +*-t*+, +*--temp*+: Set value temporarily.
@@ -683,8 +779,6 @@ Syntax: +:set-cmd-text [*--space*] [*--append*] 'text'+
 
 Preset the statusbar to some text.
 
-You can use the `{url}` and `{url:pretty}` variables here which will get replaced by the encoded/decoded URL.
-
 ==== positional arguments
 * +'text'+: The commandline to set.
 
@@ -695,23 +789,12 @@ You can use the `{url}` and `{url:pretty}` variables here which will get replace
 ==== note
 * This command does not split arguments after the last argument and handles quotes literally.
 
-[[set-mark]]
-=== set-mark
-Syntax: +:set-mark 'key'+
-
-Set a mark at the current scroll position in the current tab.
-
-==== positional arguments
-* +'key'+: mark identifier; capital indicates a global mark
-
 [[spawn]]
 === spawn
 Syntax: +:spawn [*--userscript*] [*--verbose*] [*--detach*] 'cmdline'+
 
 Spawn a command in a shell.
 
-Note the `{url}` and `{url:pretty}` variables might be useful here. `{url}` gets replaced by the URL in fully encoded format and `{url:pretty}` uses a "pretty form" with most percent-encoded characters decoded.
-
 ==== positional arguments
 * +'cmdline'+: The commandline to execute.
 
@@ -747,13 +830,13 @@ Duplicate the current tab.
 
 [[tab-close]]
 === tab-close
-Syntax: +:tab-close [*--left*] [*--right*] [*--opposite*]+
+Syntax: +:tab-close [*--prev*] [*--next*] [*--opposite*]+
 
 Close the current/[count]th tab.
 
 ==== optional arguments
-* +*-l*+, +*--left*+: Force selecting the tab to the left of the current tab.
-* +*-r*+, +*--right*+: Force selecting the tab to the right of the current tab.
+* +*-p*+, +*--prev*+: Force selecting the tab before the current tab.
+* +*-n*+, +*--next*+: Force selecting the tab after the current tab.
 * +*-o*+, +*--opposite*+: Force selecting the tab in the opposite direction of what's configured in 'tabs->select-on-remove'.
 
 
@@ -770,11 +853,12 @@ Syntax: +:tab-focus ['index']+
 
 Select the tab given as argument/[count].
 
-If neither count nor index are given, it behaves like tab-next.
+If neither count nor index are given, it behaves like tab-next. If both are given, use count.
 
 ==== positional arguments
-* +'index'+: The tab index to focus, starting with 1. The special value `last` focuses the last focused tab. Negative indexes
- counts from the end, such that -1 is the last tab.
+* +'index'+: The tab index to focus, starting with 1. The special value `last` focuses the last focused tab (regardless of count).
+ Negative indices count from the end, such that -1 is the
+ last tab.
 
 
 ==== count
@@ -782,16 +866,20 @@ The tab index to focus, starting with 1.
 
 [[tab-move]]
 === tab-move
-Syntax: +:tab-move ['direction']+
+Syntax: +:tab-move ['index']+
 
-Move the current tab.
+Move the current tab according to the argument and [count].
+
+If neither is given, move it to the first position.
 
 ==== positional arguments
-* +'direction'+: `+` or `-` for relative moving, not given for absolute moving.
+* +'index'+: `+` or `-` to move relative to the current tab by count, or a default of 1 space.
+ A tab index to move to that index.
 
 
 ==== count
-If moving absolutely: New position (default: 0) If moving relatively: Offset.
+If moving relatively: Offset. If moving absolutely: New position (default: 0). This
+ overrides the index argument, if given.
 
 
 [[tab-next]]
@@ -803,13 +891,13 @@ How many tabs to switch forward.
 
 [[tab-only]]
 === tab-only
-Syntax: +:tab-only [*--left*] [*--right*]+
+Syntax: +:tab-only [*--prev*] [*--next*]+
 
 Close all tabs except for the current one.
 
 ==== optional arguments
-* +*-l*+, +*--left*+: Keep tabs to the left of the current.
-* +*-r*+, +*--right*+: Keep tabs to the right of the current.
+* +*-p*+, +*--prev*+: Keep tabs before the current.
+* +*-n*+, +*--next*+: Keep tabs after the current.
 
 [[tab-prev]]
 === tab-prev
@@ -835,7 +923,11 @@ Re-open a closed tab (optionally skipping [count] closed tabs).
 
 [[view-source]]
 === view-source
-Show the source of the current page.
+Show the source of the current page in a new tab.
+
+[[window-only]]
+=== window-only
+Close all windows except for the current one.
 
 [[wq]]
 === wq
@@ -848,25 +940,25 @@ Save open pages and quit.
 
 [[yank]]
 === yank
-Syntax: +:yank [*--title*] [*--sel*] [*--domain*] [*--pretty*]+
+Syntax: +:yank [*--sel*] [*--keep*] ['what']+
 
-Yank the current URL/title to the clipboard or primary selection.
+Yank something to the clipboard or primary selection.
 
-==== optional arguments
-* +*-t*+, +*--title*+: Yank the title instead of the URL.
-* +*-s*+, +*--sel*+: Use the primary selection instead of the clipboard.
-* +*-d*+, +*--domain*+: Yank only the scheme, domain, and port number.
-* +*-p*+, +*--pretty*+: Yank the URL in pretty decoded form.
+==== positional arguments
+* +'what'+: What to yank. 
+
+ - `url`: The current URL.
+ - `pretty-url`: The URL in pretty decoded form.
+ - `title`: The current page's title.
+ - `domain`: The current scheme, domain, and port number.
+ - `selection`: The selection under the cursor.
+ 
 
-[[yank-selected]]
-=== yank-selected
-Syntax: +:yank-selected [*--sel*] [*--keep*]+
 
-Yank the selected text to the clipboard or primary selection.
 
 ==== optional arguments
 * +*-s*+, +*--sel*+: Use the primary selection instead of the clipboard.
-* +*-k*+, +*--keep*+: If given, stay in visual mode after yanking.
+* +*-k*+, +*--keep*+: Stay in visual mode after yanking the selection.
 
 [[zoom]]
 === zoom
@@ -874,7 +966,7 @@ Syntax: +:zoom ['zoom']+
 
 Set the zoom level for the current tab.
 
-The zoom can be given as argument or as [count]. If neither of both is given, the zoom is set to the default zoom.
+The zoom can be given as argument or as [count]. If neither is given, the zoom is set to the default zoom. If both are given, use [count].
 
 ==== positional arguments
 * +'zoom'+: The zoom percentage to set.
@@ -903,16 +995,17 @@ How many steps to zoom out.
 |==============
 |Command|Description
 |<<clear-keychain,clear-keychain>>|Clear the currently entered key chain.
+|<<click-element,click-element>>|Click the element matching the given filter.
 |<<command-accept,command-accept>>|Execute the command currently in the commandline.
 |<<command-history-next,command-history-next>>|Go forward in the commandline history.
 |<<command-history-prev,command-history-prev>>|Go back in the commandline history.
 |<<completion-item-del,completion-item-del>>|Delete the current completion item.
-|<<completion-item-next,completion-item-next>>|Select the next completion item.
-|<<completion-item-prev,completion-item-prev>>|Select the previous completion item.
+|<<completion-item-focus,completion-item-focus>>|Shift the focus of the completion menu to another item.
 |<<drop-selection,drop-selection>>|Drop selection and keep selection mode enabled.
 |<<enter-mode,enter-mode>>|Enter a key mode.
 |<<follow-hint,follow-hint>>|Follow a hint.
 |<<follow-selected,follow-selected>>|Follow the selected text.
+|<<jump-mark,jump-mark>>|Jump to the mark named by `key`.
 |<<leave-mode,leave-mode>>|Leave the mode we're currently in.
 |<<message-error,message-error>>|Show an error message in the statusbar.
 |<<message-info,message-info>>|Show an info message in the statusbar.
@@ -933,14 +1026,13 @@ How many steps to zoom out.
 |<<move-to-start-of-next-block,move-to-start-of-next-block>>|Move the cursor or selection to the start of next block.
 |<<move-to-start-of-prev-block,move-to-start-of-prev-block>>|Move the cursor or selection to the start of previous block.
 |<<open-editor,open-editor>>|Open an external editor with the currently selected form field.
-|<<paste-primary,paste-primary>>|Paste the primary selection at cursor position.
 |<<prompt-accept,prompt-accept>>|Accept the current prompt.
-|<<prompt-no,prompt-no>>|Answer no to a yes/no prompt.
+|<<prompt-item-focus,prompt-item-focus>>|Shift the focus of the prompt file completion menu to another item.
 |<<prompt-open-download,prompt-open-download>>|Immediately open a download.
-|<<prompt-yes,prompt-yes>>|Answer yes to a yes/no prompt.
 |<<repeat-command,repeat-command>>|Repeat the last executed command.
 |<<rl-backward-char,rl-backward-char>>|Move back a character.
 |<<rl-backward-delete-char,rl-backward-delete-char>>|Delete the character before the cursor.
+|<<rl-backward-kill-word,rl-backward-kill-word>>|Remove chars from the cursor to the beginning of the word.
 |<<rl-backward-word,rl-backward-word>>|Move back to the start of the current or previous word.
 |<<rl-beginning-of-line,rl-beginning-of-line>>|Move to the start of the line.
 |<<rl-delete-char,rl-delete-char>>|Delete the character after the cursor.
@@ -949,21 +1041,41 @@ How many steps to zoom out.
 |<<rl-forward-word,rl-forward-word>>|Move forward to the end of the next word.
 |<<rl-kill-line,rl-kill-line>>|Remove chars from the cursor to the end of the line.
 |<<rl-kill-word,rl-kill-word>>|Remove chars from the cursor to the end of the current word.
+|<<rl-unix-filename-rubout,rl-unix-filename-rubout>>|Remove chars from the cursor to the previous path separator.
 |<<rl-unix-line-discard,rl-unix-line-discard>>|Remove chars backward from the cursor to the beginning of the line.
 |<<rl-unix-word-rubout,rl-unix-word-rubout>>|Remove chars from the cursor to the beginning of the word.
 |<<rl-yank,rl-yank>>|Paste the most recently deleted text.
+|<<run-with-count,run-with-count>>|Run a command with the given count.
 |<<scroll,scroll>>|Scroll the current tab in the given direction.
 |<<scroll-page,scroll-page>>|Scroll the frame page-wise.
 |<<scroll-perc,scroll-perc>>|Scroll to a specific percentage of the page.
 |<<scroll-px,scroll-px>>|Scroll the current tab by 'count * dx/dy' pixels.
 |<<search-next,search-next>>|Continue the search to the ([count]th) next term.
 |<<search-prev,search-prev>>|Continue the search to the ([count]th) previous term.
+|<<set-mark,set-mark>>|Set a mark at the current scroll position in the current tab.
 |<<toggle-selection,toggle-selection>>|Toggle caret selection mode.
 |==============
 [[clear-keychain]]
 === clear-keychain
 Clear the currently entered key chain.
 
+[[click-element]]
+=== click-element
+Syntax: +:click-element [*--target* 'target'] [*--force-event*] 'filter' 'value'+
+
+Click the element matching the given filter.
+
+The given filter needs to result in exactly one element, otherwise, an error is shown.
+
+==== positional arguments
+* +'filter'+: How to filter the elements. id: Get an element based on its ID.
+
+* +'value'+: The value to filter for.
+
+==== optional arguments
+* +*-t*+, +*--target*+: How to open the clicked element (normal/tab/tab-bg/window).
+* +*-f*+, +*--force-event*+: Force generating a fake click event.
+
 [[command-accept]]
 === command-accept
 Execute the command currently in the commandline.
@@ -980,13 +1092,14 @@ Go back in the commandline history.
 === completion-item-del
 Delete the current completion item.
 
-[[completion-item-next]]
-=== completion-item-next
-Select the next completion item.
+[[completion-item-focus]]
+=== completion-item-focus
+Syntax: +:completion-item-focus 'which'+
 
-[[completion-item-prev]]
-=== completion-item-prev
-Select the previous completion item.
+Shift the focus of the completion menu to another item.
+
+==== positional arguments
+* +'which'+: 'next', 'prev', 'next-category', or 'prev-category'.
 
 [[drop-selection]]
 === drop-selection
@@ -1019,6 +1132,15 @@ Follow the selected text.
 ==== optional arguments
 * +*-t*+, +*--tab*+: Load the selected link in a new tab.
 
+[[jump-mark]]
+=== jump-mark
+Syntax: +:jump-mark 'key'+
+
+Jump to the mark named by `key`.
+
+==== positional arguments
+* +'key'+: mark identifier; capital indicates a global mark
+
 [[leave-mode]]
 === leave-mode
 Leave the mode we're currently in.
@@ -1041,6 +1163,9 @@ Show an info message in the statusbar.
 ==== positional arguments
 * +'text'+: The text to show.
 
+==== count
+How many times to show the message
+
 [[message-warning]]
 === message-warning
 Syntax: +:message-warning 'text'+
@@ -1149,25 +1274,41 @@ Open an external editor with the currently selected form field.
 
 The editor which should be launched can be configured via the `general -> editor` config option.
 
-[[paste-primary]]
-=== paste-primary
-Paste the primary selection at cursor position.
-
 [[prompt-accept]]
 === prompt-accept
+Syntax: +:prompt-accept ['value']+
+
 Accept the current prompt.
 
-[[prompt-no]]
-=== prompt-no
-Answer no to a yes/no prompt.
+==== positional arguments
+* +'value'+: If given, uses this value instead of the entered one. For boolean prompts, "yes"/"no" are accepted as value.
+
+
+[[prompt-item-focus]]
+=== prompt-item-focus
+Syntax: +:prompt-item-focus 'which'+
+
+Shift the focus of the prompt file completion menu to another item.
+
+==== positional arguments
+* +'which'+: 'next', 'prev'
 
 [[prompt-open-download]]
 === prompt-open-download
+Syntax: +:prompt-open-download ['cmdline']+
+
 Immediately open a download.
 
-[[prompt-yes]]
-=== prompt-yes
-Answer yes to a yes/no prompt.
+If no specific command is given, this will use the system's default application to open the file.
+
+==== positional arguments
+* +'cmdline'+: The command which should be used to open the file. A `{}` is expanded to the temporary file name. If no `{}` is
+ present, the filename is automatically appended to the
+ cmdline.
+
+
+==== note
+* This command does not split arguments after the last argument and handles quotes literally.
 
 [[repeat-command]]
 === repeat-command
@@ -1188,6 +1329,12 @@ Delete the character before the cursor.
 
 This acts like readline's backward-delete-char.
 
+[[rl-backward-kill-word]]
+=== rl-backward-kill-word
+Remove chars from the cursor to the beginning of the word.
+
+This acts like readline's backward-kill-word. Any non-alphanumeric character is considered a word delimiter.
+
 [[rl-backward-word]]
 === rl-backward-word
 Move back to the start of the current or previous word.
@@ -1236,6 +1383,12 @@ Remove chars from the cursor to the end of the current word.
 
 This acts like readline's kill-word.
 
+[[rl-unix-filename-rubout]]
+=== rl-unix-filename-rubout
+Remove chars from the cursor to the previous path separator.
+
+This acts like readline's unix-filename-rubout.
+
 [[rl-unix-line-discard]]
 === rl-unix-line-discard
 Remove chars backward from the cursor to the beginning of the line.
@@ -1246,7 +1399,7 @@ This acts like readline's unix-line-discard.
 === rl-unix-word-rubout
 Remove chars from the cursor to the beginning of the word.
 
-This acts like readline's unix-word-rubout.
+This acts like readline's unix-word-rubout. Whitespace is used as a word delimiter.
 
 [[rl-yank]]
 === rl-yank
@@ -1254,6 +1407,26 @@ Paste the most recently deleted text.
 
 This acts like readline's yank.
 
+[[run-with-count]]
+=== run-with-count
+Syntax: +:run-with-count 'count-arg' 'command'+
+
+Run a command with the given count.
+
+If run_with_count itself is run with a count, it multiplies count_arg.
+
+==== positional arguments
+* +'count-arg'+: The count to pass to the command.
+* +'command'+: The command to run, with optional args.
+
+==== count
+The count that run_with_count itself received.
+
+==== note
+* This command does not split arguments after the last argument and handles quotes literally.
+* With this command, +;;+ is interpreted literally instead of splitting off a second command.
+* This command does not replace variables like +\{url\}+.
+
 [[scroll]]
 === scroll
 Syntax: +:scroll 'direction'+
@@ -1330,6 +1503,15 @@ Continue the search to the ([count]th) previous term.
 ==== count
 How many elements to ignore.
 
+[[set-mark]]
+=== set-mark
+Syntax: +:set-mark 'key'+
+
+Set a mark at the current scroll position in the current tab.
+
+==== positional arguments
+* +'key'+: mark identifier; capital indicates a global mark
+
 [[toggle-selection]]
 === toggle-selection
 Toggle caret selection mode.
@@ -1348,6 +1530,9 @@ These commands are mainly intended for debugging. They are hidden if qutebrowser
 |<<debug-console,debug-console>>|Show the debugging console.
 |<<debug-crash,debug-crash>>|Crash for debugging purposes.
 |<<debug-dump-page,debug-dump-page>>|Dump the current page's content to a file.
+|<<debug-log-capacity,debug-log-capacity>>|Change the number of log lines to be stored in RAM.
+|<<debug-log-filter,debug-log-filter>>|Change the log filter for console logging.
+|<<debug-log-level,debug-log-level>>|Change the log level for console logging.
 |<<debug-pyeval,debug-pyeval>>|Evaluate a python string and display the results as a web page.
 |<<debug-set-fake-clipboard,debug-set-fake-clipboard>>|Put data into the fake clipboard and enable logging, used for tests.
 |<<debug-trace,debug-trace>>|Trace executed code via hunter.
@@ -1390,6 +1575,33 @@ Dump the current page's content to a file.
 ==== optional arguments
 * +*-p*+, +*--plain*+: Write plain text instead of HTML.
 
+[[debug-log-capacity]]
+=== debug-log-capacity
+Syntax: +:debug-log-capacity 'capacity'+
+
+Change the number of log lines to be stored in RAM.
+
+==== positional arguments
+* +'capacity'+: Number of lines for the log.
+
+[[debug-log-filter]]
+=== debug-log-filter
+Syntax: +:debug-log-filter 'filters'+
+
+Change the log filter for console logging.
+
+==== positional arguments
+* +'filters'+: A comma separated list of logger names.
+
+[[debug-log-level]]
+=== debug-log-level
+Syntax: +:debug-log-level 'level'+
+
+Change the log level for console logging.
+
+==== positional arguments
+* +'level'+: The log level to set.
+
 [[debug-pyeval]]
 === debug-pyeval
 Syntax: +:debug-pyeval [*--quiet*] 's'+

doc/help/index.asciidoc

@@ -6,13 +6,13 @@ Documentation
 
 The following help pages are currently available:
 
-* link:quickstart.html[Quick start guide]
-* link:FAQ.html[Frequently asked questions]
-* link:CHANGELOG.html[Change Log]
+* link:../quickstart.html[Quick start guide]
+* link:../../FAQ.html[Frequently asked questions]
+* link:../../CHANGELOG.html[Change Log]
 * link:commands.html[Documentation of commands]
 * link:settings.html[Documentation of settings]
-* link:userscripts.html[How to write userscripts]
-* link:CONTRIBUTING.html[Contributing to qutebrowser]
+* link:../userscripts.html[How to write userscripts]
+* link:../../CONTRIBUTING.html[Contributing to qutebrowser]
 
 Getting help
 ------------

doc/help/settings.asciidoc

@@ -1,3 +1,7 @@
+// DO NOT EDIT THIS FILE DIRECTLY!
+// It is autogenerated from docstrings by running:
+//   $ python3 scripts/dev/src2asciidoc.py
+
 = Settings
 
 .Quick reference for section ``general''
@@ -6,6 +10,8 @@
 |Setting|Description
 |<<general-ignore-case,ignore-case>>|Whether to find text on a page case-insensitively.
 |<<general-startpage,startpage>>|The default page(s) to open at the start, separated by commas.
+|<<general-yank-ignored-url-parameters,yank-ignored-url-parameters>>|The URL parameters to strip with :yank url, separated by commas.
+|<<general-default-open-dispatcher,default-open-dispatcher>>|The default program used to open downloads. Set to an empty string to use the default internal handler.
 |<<general-default-page,default-page>>|The page to open if :open -t/-b/-w is used without URL. Use `about:blank` for a blank page.
 |<<general-auto-search,auto-search>>|Whether to start a search when something else than a URL is entered.
 |<<general-auto-save-config,auto-save-config>>|Whether to save the config automatically on quit.
@@ -16,9 +22,10 @@
 |<<general-developer-extras,developer-extras>>|Enable extra tools for Web developers.
 |<<general-print-element-backgrounds,print-element-backgrounds>>|Whether the background color and images are also drawn when the page is printed.
 |<<general-xss-auditing,xss-auditing>>|Whether load requests should be monitored for cross-site scripting attempts.
-|<<general-site-specific-quirks,site-specific-quirks>>|Enable workarounds for broken sites.
+|<<general-site-specific-quirks,site-specific-quirks>>|Enable QtWebKit workarounds for broken sites.
 |<<general-default-encoding,default-encoding>>|Default encoding to use for websites.
 |<<general-new-instance-open-target,new-instance-open-target>>|How to open links in an existing instance if a new one is launched.
+|<<general-new-instance-open-target.window,new-instance-open-target.window>>|Which window to choose when opening links as new tabs.
 |<<general-log-javascript-console,log-javascript-console>>|How to log javascript console messages.
 |<<general-save-session,save-session>>|Whether to always save the open pages.
 |<<general-session-default-name,session-default-name>>|The name of the session to save by default, or empty for the last loaded session.
@@ -38,17 +45,19 @@
 |<<ui-confirm-quit,confirm-quit>>|Whether to confirm quitting the application.
 |<<ui-zoom-text-only,zoom-text-only>>|Whether the zoom factor on a frame applies only to the text or to all content.
 |<<ui-frame-flattening,frame-flattening>>|Whether to  expand each subframe to its contents.
-|<<ui-user-stylesheet,user-stylesheet>>|User stylesheet to use (absolute filename, filename relative to the config directory or CSS string). Will expand environment variables.
+|<<ui-user-stylesheet,user-stylesheet>>|User stylesheet to use (absolute filename or filename relative to the config directory). Will expand environment variables.
+|<<ui-hide-scrollbar,hide-scrollbar>>|Hide the main scrollbar.
 |<<ui-css-media-type,css-media-type>>|Set the CSS media type.
 |<<ui-smooth-scrolling,smooth-scrolling>>|Whether to enable smooth scrolling for webpages.
 |<<ui-remove-finished-downloads,remove-finished-downloads>>|Number of milliseconds to wait before removing finished downloads. Will not be removed if value is -1.
 |<<ui-hide-statusbar,hide-statusbar>>|Whether to hide the statusbar unless a message is shown.
 |<<ui-statusbar-padding,statusbar-padding>>|Padding for statusbar (top, bottom, left, right).
 |<<ui-window-title-format,window-title-format>>|The format to use for the window title. The following placeholders are defined:
-|<<ui-hide-mouse-cursor,hide-mouse-cursor>>|Whether to hide the mouse cursor.
 |<<ui-modal-js-dialog,modal-js-dialog>>|Use standard JavaScript modal dialog for alert() and confirm()
 |<<ui-hide-wayland-decoration,hide-wayland-decoration>>|Hide the window decoration when using wayland (requires restart)
 |<<ui-keyhint-blacklist,keyhint-blacklist>>|Keychains that shouldn't be shown in the keyhint dialog
+|<<ui-prompt-radius,prompt-radius>>|The rounding radius for the edges of prompts.
+|<<ui-prompt-filebrowser,prompt-filebrowser>>|Show a filebrowser in upload/download prompts.
 |==============
 
 .Quick reference for section ``network''
@@ -64,16 +73,16 @@
 |<<network-ssl-strict,ssl-strict>>|Whether to validate SSL handshakes.
 |<<network-dns-prefetch,dns-prefetch>>|Whether to try to pre-fetch DNS entries to speed up browsing.
 |<<network-custom-headers,custom-headers>>|Set custom headers for qutebrowser HTTP requests.
+|<<network-netrc-file,netrc-file>>|Set location of a netrc-file for HTTP authentication. If empty, ~/.netrc is used.
 |==============
 
 .Quick reference for section ``completion''
 [options="header",width="75%",cols="25%,75%"]
 |==============
 |Setting|Description
-|<<completion-auto-open,auto-open>>|Automatically open completion when typing.
+|<<completion-show,show>>|When to show the autocompletion window.
 |<<completion-download-path-suggestion,download-path-suggestion>>|What to display in the download filename input.
 |<<completion-timestamp-format,timestamp-format>>|How to format timestamps (e.g. for history)
-|<<completion-show,show>>|Whether to show the autocompletion window.
 |<<completion-height,height>>|The height of the completion, in px or as percentage of the window.
 |<<completion-cmd-history-max-items,cmd-history-max-items>>|How many commands to save in the command history.
 |<<completion-web-history-max-items,web-history-max-items>>|How many URLs to show in the web history.
@@ -87,8 +96,8 @@
 [options="header",width="75%",cols="25%,75%"]
 |==============
 |Setting|Description
-|<<input-timeout,timeout>>|Timeout for ambiguous key bindings.
-|<<input-partial-timeout,partial-timeout>>|Timeout for partially typed key bindings.
+|<<input-timeout,timeout>>|Timeout (in milliseconds) for ambiguous key bindings.
+|<<input-partial-timeout,partial-timeout>>|Timeout (in milliseconds) for partially typed key bindings.
 |<<input-insert-mode-on-plugins,insert-mode-on-plugins>>|Whether to switch to insert mode when clicking flash and other plugins.
 |<<input-auto-leave-insert-mode,auto-leave-insert-mode>>|Whether to leave insert mode if a non-editable element is clicked.
 |<<input-auto-insert-mode,auto-insert-mode>>|Whether to automatically enter insert mode if an editable element is focused after page load.
@@ -139,7 +148,7 @@
 |<<storage-offline-storage-database,offline-storage-database>>|Whether support for the HTML 5 offline storage feature is enabled.
 |<<storage-offline-web-application-storage,offline-web-application-storage>>|Whether support for the HTML 5 web application cache feature is enabled.
 |<<storage-local-storage,local-storage>>|Whether support for the HTML 5 local storage feature is enabled.
-|<<storage-cache-size,cache-size>>|Size of the HTTP network cache.
+|<<storage-cache-size,cache-size>>|Size of the HTTP network cache. Empty to use the default value.
 |==============
 
 .Quick reference for section ``content''
@@ -154,7 +163,8 @@
 |<<content-hyperlink-auditing,hyperlink-auditing>>|Enable or disable hyperlink auditing (<a ping>).
 |<<content-geolocation,geolocation>>|Allow websites to request geolocations.
 |<<content-notifications,notifications>>|Allow websites to show notifications.
-|<<content-javascript-can-open-windows,javascript-can-open-windows>>|Whether JavaScript programs can open new windows.
+|<<content-media-capture,media-capture>>|Allow websites to record audio/video.
+|<<content-javascript-can-open-windows-automatically,javascript-can-open-windows-automatically>>|Whether JavaScript programs can open new windows without user interaction.
 |<<content-javascript-can-close-windows,javascript-can-close-windows>>|Whether JavaScript programs can close windows.
 |<<content-javascript-can-access-clipboard,javascript-can-access-clipboard>>|Whether JavaScript programs can read or write to the clipboard.
 |<<content-ignore-javascript-prompt,ignore-javascript-prompt>>|Whether all javascript prompts should be ignored.
@@ -162,7 +172,7 @@
 |<<content-local-content-can-access-remote-urls,local-content-can-access-remote-urls>>|Whether locally loaded documents are allowed to access remote urls.
 |<<content-local-content-can-access-file-urls,local-content-can-access-file-urls>>|Whether locally loaded documents are allowed to access other local urls.
 |<<content-cookies-accept,cookies-accept>>|Control which cookies to accept.
-|<<content-cookies-store,cookies-store>>|Whether to store cookies.
+|<<content-cookies-store,cookies-store>>|Whether to store cookies. Note this option needs a restart with QtWebEngine.
 |<<content-host-block-lists,host-block-lists>>|List of URLs of lists which contain hosts to block.
 |<<content-host-blocking-enabled,host-blocking-enabled>>|Whether host blocking is enabled.
 |<<content-host-blocking-whitelist,host-blocking-whitelist>>|List of domains that should always be loaded, despite being ad-blocked.
@@ -174,18 +184,18 @@
 |==============
 |Setting|Description
 |<<hints-border,border>>|CSS border value for hints.
-|<<hints-opacity,opacity>>|Opacity for hints.
 |<<hints-mode,mode>>|Mode to use for hints.
 |<<hints-chars,chars>>|Chars used for hint strings.
 |<<hints-min-chars,min-chars>>|Minimum number of chars used for hint strings.
 |<<hints-scatter,scatter>>|Whether to scatter hint key chains (like Vimium) or not (like dwb). Ignored for number hints.
 |<<hints-uppercase,uppercase>>|Make chars in hint strings uppercase.
 |<<hints-dictionary,dictionary>>|The dictionary file to be used by the word hints.
-|<<hints-auto-follow,auto-follow>>|Follow a hint immediately when the hint text is completely matched.
-|<<hints-auto-follow-timeout,auto-follow-timeout>>|A timeout to inhibit normal-mode key bindings after a successfulauto-follow.
+|<<hints-auto-follow,auto-follow>>|Controls when a hint can be automatically followed without the user pressing Enter.
+|<<hints-auto-follow-timeout,auto-follow-timeout>>|A timeout (in milliseconds) to inhibit normal-mode key bindings after a successful auto-follow.
 |<<hints-next-regexes,next-regexes>>|A comma-separated list of regexes to use for 'next' links.
 |<<hints-prev-regexes,prev-regexes>>|A comma-separated list of regexes to use for 'prev' links.
 |<<hints-find-implementation,find-implementation>>|Which implementation to use to find elements to hint.
+|<<hints-hide-unmatched-rapid-hints,hide-unmatched-rapid-hints>>|Controls hiding unmatched hints in rapid mode.
 |==============
 
 .Quick reference for section ``colors''
@@ -208,12 +218,6 @@
 |<<colors-completion.scrollbar.bg,completion.scrollbar.bg>>|Color of the scrollbar in completion view
 |<<colors-statusbar.fg,statusbar.fg>>|Foreground color of the statusbar.
 |<<colors-statusbar.bg,statusbar.bg>>|Background color of the statusbar.
-|<<colors-statusbar.fg.error,statusbar.fg.error>>|Foreground color of the statusbar if there was an error.
-|<<colors-statusbar.bg.error,statusbar.bg.error>>|Background color of the statusbar if there was an error.
-|<<colors-statusbar.fg.warning,statusbar.fg.warning>>|Foreground color of the statusbar if there is a warning.
-|<<colors-statusbar.bg.warning,statusbar.bg.warning>>|Background color of the statusbar if there is a warning.
-|<<colors-statusbar.fg.prompt,statusbar.fg.prompt>>|Foreground color of the statusbar if there is a prompt.
-|<<colors-statusbar.bg.prompt,statusbar.bg.prompt>>|Background color of the statusbar if there is a prompt.
 |<<colors-statusbar.fg.insert,statusbar.fg.insert>>|Foreground color of the statusbar in insert mode.
 |<<colors-statusbar.bg.insert,statusbar.bg.insert>>|Background color of the statusbar in insert mode.
 |<<colors-statusbar.fg.command,statusbar.fg.command>>|Foreground color of the statusbar in command mode.
@@ -243,7 +247,7 @@
 |<<colors-tabs.indicator.error,tabs.indicator.error>>|Color for the tab indicator on errors..
 |<<colors-tabs.indicator.system,tabs.indicator.system>>|Color gradient interpolation system for the tab indicator.
 |<<colors-hints.fg,hints.fg>>|Font color for hints.
-|<<colors-hints.bg,hints.bg>>|Background color for hints.
+|<<colors-hints.bg,hints.bg>>|Background color for hints. Note that you can use a `rgba(...)` value for transparency.
 |<<colors-hints.fg.match,hints.fg.match>>|Font color for the matched part of hints.
 |<<colors-downloads.bg.bar,downloads.bg.bar>>|Background color for the download bar.
 |<<colors-downloads.fg.start,downloads.fg.start>>|Color gradient start for download text.
@@ -258,6 +262,18 @@
 |<<colors-keyhint.fg,keyhint.fg>>|Text color for the keyhint widget.
 |<<colors-keyhint.fg.suffix,keyhint.fg.suffix>>|Highlight color for keys to complete the current keychain
 |<<colors-keyhint.bg,keyhint.bg>>|Background color of the keyhint widget.
+|<<colors-messages.fg.error,messages.fg.error>>|Foreground color of an error message.
+|<<colors-messages.bg.error,messages.bg.error>>|Background color of an error message.
+|<<colors-messages.border.error,messages.border.error>>|Border color of an error message.
+|<<colors-messages.fg.warning,messages.fg.warning>>|Foreground color a warning message.
+|<<colors-messages.bg.warning,messages.bg.warning>>|Background color of a warning message.
+|<<colors-messages.border.warning,messages.border.warning>>|Border color of an error message.
+|<<colors-messages.fg.info,messages.fg.info>>|Foreground color an info message.
+|<<colors-messages.bg.info,messages.bg.info>>|Background color of an info message.
+|<<colors-messages.border.info,messages.border.info>>|Border color of an info message.
+|<<colors-prompts.fg,prompts.fg>>|Foreground color for prompts.
+|<<colors-prompts.bg,prompts.bg>>|Background color for prompts.
+|<<colors-prompts.selected.bg,prompts.selected.bg>>|Background color for the selected item in filename prompts.
 |==============
 
 .Quick reference for section ``fonts''
@@ -266,6 +282,7 @@
 |Setting|Description
 |<<fonts-_monospace,_monospace>>|Default monospace fonts.
 |<<fonts-completion,completion>>|Font used in the completion widget.
+|<<fonts-completion.category,completion.category>>|Font used in the completion categories.
 |<<fonts-tabbar,tabbar>>|Font used in the tab bar.
 |<<fonts-statusbar,statusbar>>|Font used in the statusbar.
 |<<fonts-downloads,downloads>>|Font used for the downloadbar.
@@ -282,6 +299,10 @@
 |<<fonts-web-size-default,web-size-default>>|The default font size for regular text.
 |<<fonts-web-size-default-fixed,web-size-default-fixed>>|The default font size for fixed-pitch text.
 |<<fonts-keyhint,keyhint>>|Font used in the keyhint widget.
+|<<fonts-messages.error,messages.error>>|Font used for error messages.
+|<<fonts-messages.warning,messages.warning>>|Font used for warning messages.
+|<<fonts-messages.info,messages.info>>|Font used for info messages.
+|<<fonts-prompts,prompts>>|Font used for prompts.
 |==============
 
 == general
@@ -303,7 +324,21 @@ Default: +pass:[smart]+
 === startpage
 The default page(s) to open at the start, separated by commas.
 
-Default: +pass:[https://duckduckgo.com]+
+Default: +pass:[https://start.duckduckgo.com]+
+
+[[general-yank-ignored-url-parameters]]
+=== yank-ignored-url-parameters
+The URL parameters to strip with :yank url, separated by commas.
+
+Default: +pass:[ref,utm_source,utm_medium,utm_campaign,utm_term,utm_content]+
+
+[[general-default-open-dispatcher]]
+=== default-open-dispatcher
+The default program used to open downloads. Set to an empty string to use the default internal handler.
+
+Any {} in the string will be expanded to the filename, else the filename will be appended.
+
+Default: empty
 
 [[general-default-page]]
 === default-page
@@ -366,11 +401,13 @@ Valid values:
 
 Default: +pass:[false]+
 
+This setting is only available with the QtWebKit backend.
+
 [[general-developer-extras]]
 === developer-extras
 Enable extra tools for Web developers.
 
-This needs to be enabled for `:inspector` to work and also adds an _Inspect_ entry to the context menu.
+This needs to be enabled for `:inspector` to work and also adds an _Inspect_ entry to the context menu. For QtWebEngine, see 'qutebrowser --help' instead.
 
 Valid values:
 
@@ -379,9 +416,12 @@ Valid values:
 
 Default: +pass:[false]+
 
+This setting is only available with the QtWebKit backend.
+
 [[general-print-element-backgrounds]]
 === print-element-backgrounds
 Whether the background color and images are also drawn when the page is printed.
+This setting only works with Qt 5.8 or newer when using the QtWebEngine backend.
 
 Valid values:
 
@@ -405,7 +445,7 @@ Default: +pass:[false]+
 
 [[general-site-specific-quirks]]
 === site-specific-quirks
-Enable workarounds for broken sites.
+Enable QtWebKit workarounds for broken sites.
 
 Valid values:
 
@@ -414,6 +454,8 @@ Valid values:
 
 Default: +pass:[true]+
 
+This setting is only available with the QtWebKit backend.
+
 [[general-default-encoding]]
 === default-encoding
 Default encoding to use for websites.
@@ -436,6 +478,19 @@ Valid values:
 
 Default: +pass:[tab]+
 
+[[general-new-instance-open-target.window]]
+=== new-instance-open-target.window
+Which window to choose when opening links as new tabs.
+
+Valid values:
+
+ * +first-opened+: Open new tabs in the first (oldest) opened window.
+ * +last-opened+: Open new tabs in the last (newest) opened window.
+ * +last-focused+: Open new tabs in the most recently focused window.
+ * +last-visible+: Open new tabs in the most recently visible window.
+
+Default: +pass:[last-focused]+
+
 [[general-log-javascript-console]]
 === log-javascript-console
 How to log javascript console messages.
@@ -556,6 +611,8 @@ Valid values:
 
 Default: +pass:[false]+
 
+This setting is only available with the QtWebKit backend.
+
 [[ui-frame-flattening]]
 === frame-flattening
 Whether to  expand each subframe to its contents.
@@ -569,11 +626,24 @@ Valid values:
 
 Default: +pass:[false]+
 
+This setting is only available with the QtWebKit backend.
+
 [[ui-user-stylesheet]]
 === user-stylesheet
-User stylesheet to use (absolute filename, filename relative to the config directory or CSS string). Will expand environment variables.
+User stylesheet to use (absolute filename or filename relative to the config directory). Will expand environment variables.
 
-Default: +pass:[::-webkit-scrollbar { width: 0px; height: 0px; }]+
+Default: empty
+
+[[ui-hide-scrollbar]]
+=== hide-scrollbar
+Hide the main scrollbar.
+
+Valid values:
+
+ * +true+
+ * +false+
+
+Default: +pass:[true]+
 
 [[ui-css-media-type]]
 === css-media-type
@@ -581,6 +651,8 @@ Set the CSS media type.
 
 Default: empty
 
+This setting is only available with the QtWebKit backend.
+
 [[ui-smooth-scrolling]]
 === smooth-scrolling
 Whether to enable smooth scrolling for webpages.
@@ -626,20 +698,10 @@ The format to use for the window title. The following placeholders are defined:
 * `{id}`: The internal window ID of this window.
 * `{scroll_pos}`: The page scroll position.
 * `{host}`: The host of the current web page.
+* `{backend}`: Either 'webkit' or 'webengine'
 
 Default: +pass:[{perc}{title}{title_sep}qutebrowser]+
 
-[[ui-hide-mouse-cursor]]
-=== hide-mouse-cursor
-Whether to hide the mouse cursor.
-
-Valid values:
-
- * +true+
- * +false+
-
-Default: +pass:[false]+
-
 [[ui-modal-js-dialog]]
 === modal-js-dialog
 Use standard JavaScript modal dialog for alert() and confirm()
@@ -670,6 +732,23 @@ Globs are supported, so ';*' will blacklist all keychainsstarting with ';'. Use
 
 Default: empty
 
+[[ui-prompt-radius]]
+=== prompt-radius
+The rounding radius for the edges of prompts.
+
+Default: +pass:[8]+
+
+[[ui-prompt-filebrowser]]
+=== prompt-filebrowser
+Show a filebrowser in upload/download prompts.
+
+Valid values:
+
+ * +true+
+ * +false+
+
+Default: +pass:[true]+
+
 == network
 Settings related to the network.
 
@@ -702,6 +781,8 @@ Valid values:
 
 Default: +pass:[same-domain]+
 
+This setting is only available with the QtWebKit backend.
+
 [[network-user-agent]]
 === user-agent
 User agent to send. Empty to send the default.
@@ -714,6 +795,8 @@ The proxy to use.
 
 In addition to the listed values, you can use a `socks://...` or `http://...` URL.
 
+This setting only works with Qt 5.8 or newer when using the QtWebEngine backend.
+
 Valid values:
 
  * +system+: Use the system wide proxy.
@@ -732,6 +815,8 @@ Valid values:
 
 Default: +pass:[true]+
 
+This setting is only available with the QtWebKit backend.
+
 [[network-ssl-strict]]
 === ssl-strict
 Whether to validate SSL handshakes.
@@ -755,25 +840,34 @@ Valid values:
 
 Default: +pass:[true]+
 
+This setting is only available with the QtWebKit backend.
+
 [[network-custom-headers]]
 === custom-headers
 Set custom headers for qutebrowser HTTP requests.
 
 Default: empty
 
+[[network-netrc-file]]
+=== netrc-file
+Set location of a netrc-file for HTTP authentication. If empty, ~/.netrc is used.
+
+Default: empty
+
 == completion
 Options related to completion and command history.
 
-[[completion-auto-open]]
-=== auto-open
-Automatically open completion when typing.
+[[completion-show]]
+=== show
+When to show the autocompletion window.
 
 Valid values:
 
- * +true+
- * +false+
+ * +always+: Whenever a completion is available.
+ * +auto+: Whenever a completion is requested.
+ * +never+: Never.
 
-Default: +pass:[true]+
+Default: +pass:[always]+
 
 [[completion-download-path-suggestion]]
 === download-path-suggestion
@@ -793,17 +887,6 @@ How to format timestamps (e.g. for history)
 
 Default: +pass:[%Y-%m-%d]+
 
-[[completion-show]]
-=== show
-Whether to show the autocompletion window.
-
-Valid values:
-
- * +true+
- * +false+
-
-Default: +pass:[true]+
-
 [[completion-height]]
 === height
 The height of the completion, in px or as percentage of the window.
@@ -865,7 +948,7 @@ Options related to input modes.
 
 [[input-timeout]]
 === timeout
-Timeout for ambiguous key bindings.
+Timeout (in milliseconds) for ambiguous key bindings.
 
 If the current input forms both a complete match and a partial match, the complete match will be executed after this time.
 
@@ -873,7 +956,7 @@ Default: +pass:[500]+
 
 [[input-partial-timeout]]
 === partial-timeout
-Timeout for partially typed key bindings.
+Timeout (in milliseconds) for partially typed key bindings.
 
 If the current input forms only partial matches, the keystring will be cleared after this time.
 
@@ -985,11 +1068,11 @@ Which tab to select when the focused tab is removed.
 
 Valid values:
 
- * +left+: Select the tab on the left.
- * +right+: Select the tab on the right.
- * +previous+: Select the previously selected tab.
+ * +prev+: Select the tab which came before the closed one (left in horizontal, above in vertical).
+ * +next+: Select the tab which came after the closed one (right in horizontal, below in vertical).
+ * +last-used+: Select the previously selected tab.
 
-Default: +pass:[right]+
+Default: +pass:[next]+
 
 [[tabs-new-tab-position]]
 === new-tab-position
@@ -997,12 +1080,12 @@ How new tabs are positioned.
 
 Valid values:
 
- * +left+: On the left of the current tab.
- * +right+: On the right of the current tab.
- * +first+: At the left end.
- * +last+: At the right end.
+ * +prev+: Before the current tab.
+ * +next+: After the current tab.
+ * +first+: At the beginning.
+ * +last+: At the end.
 
-Default: +pass:[right]+
+Default: +pass:[next]+
 
 [[tabs-new-tab-position-explicit]]
 === new-tab-position-explicit
@@ -1010,10 +1093,10 @@ How new tabs opened explicitly are positioned.
 
 Valid values:
 
- * +left+: On the left of the current tab.
- * +right+: On the right of the current tab.
- * +first+: At the left end.
- * +last+: At the right end.
+ * +prev+: Before the current tab.
+ * +next+: After the current tab.
+ * +first+: At the beginning.
+ * +last+: At the end.
 
 Default: +pass:[last]+
 
@@ -1143,6 +1226,7 @@ The format to use for the tab title. The following placeholders are defined:
 * `{id}`: The internal tab ID of this tab.
 * `{scroll_pos}`: The page scroll position.
 * `{host}`: The host of the current web page.
+* `{backend}`: Either 'webkit' or 'webengine'
 
 Default: +pass:[{index}: {title}]+
 
@@ -1223,6 +1307,8 @@ For more information about the feature, please refer to: http://webkit.org/blog/
 
 Default: empty
 
+This setting is only available with the QtWebKit backend.
+
 [[storage-object-cache-capacities]]
 === object-cache-capacities
 The capacities for the global memory cache for dead objects such as stylesheets or scripts. Syntax: cacheMinDeadCapacity, cacheMaxDead, totalCapacity.
@@ -1235,18 +1321,24 @@ _totalCapacity_ specifies the maximum number of bytes that the cache should cons
 
 Default: empty
 
+This setting is only available with the QtWebKit backend.
+
 [[storage-offline-storage-default-quota]]
 === offline-storage-default-quota
 Default quota for new offline storage databases.
 
 Default: empty
 
+This setting is only available with the QtWebKit backend.
+
 [[storage-offline-web-application-cache-quota]]
 === offline-web-application-cache-quota
 Quota for the offline web application cache.
 
 Default: empty
 
+This setting is only available with the QtWebKit backend.
+
 [[storage-offline-storage-database]]
 === offline-storage-database
 Whether support for the HTML 5 offline storage feature is enabled.
@@ -1258,6 +1350,8 @@ Valid values:
 
 Default: +pass:[true]+
 
+This setting is only available with the QtWebKit backend.
+
 [[storage-offline-web-application-storage]]
 === offline-web-application-storage
 Whether support for the HTML 5 web application cache feature is enabled.
@@ -1273,6 +1367,8 @@ Valid values:
 
 Default: +pass:[true]+
 
+This setting is only available with the QtWebKit backend.
+
 [[storage-local-storage]]
 === local-storage
 Whether support for the HTML 5 local storage feature is enabled.
@@ -1286,9 +1382,9 @@ Default: +pass:[true]+
 
 [[storage-cache-size]]
 === cache-size
-Size of the HTTP network cache.
+Size of the HTTP network cache. Empty to use the default value.
 
-Default: +pass:[52428800]+
+Default: empty
 
 == content
 Loaded plugins/scripts and allowed actions.
@@ -1337,7 +1433,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: +pass:[false]+
+Default: +pass:[true]+
 
 [[content-css-regions]]
 === css-regions
@@ -1350,6 +1446,8 @@ Valid values:
 
 Default: +pass:[true]+
 
+This setting is only available with the QtWebKit backend.
+
 [[content-hyperlink-auditing]]
 === hyperlink-auditing
 Enable or disable hyperlink auditing (<a ping>).
@@ -1385,9 +1483,23 @@ Valid values:
 
 Default: +pass:[ask]+
 
-[[content-javascript-can-open-windows]]
-=== javascript-can-open-windows
-Whether JavaScript programs can open new windows.
+[[content-media-capture]]
+=== media-capture
+Allow websites to record audio/video.
+
+Valid values:
+
+ * +true+
+ * +false+
+ * +ask+
+
+Default: +pass:[ask]+
+
+This setting is only available with the QtWebEngine backend.
+
+[[content-javascript-can-open-windows-automatically]]
+=== javascript-can-open-windows-automatically
+Whether JavaScript programs can open new windows without user interaction.
 
 Valid values:
 
@@ -1407,6 +1519,8 @@ Valid values:
 
 Default: +pass:[false]+
 
+This setting is only available with the QtWebKit backend.
+
 [[content-javascript-can-access-clipboard]]
 === javascript-can-access-clipboard
 Whether JavaScript programs can read or write to the clipboard.
@@ -1475,9 +1589,11 @@ Valid values:
 
 Default: +pass:[no-3rdparty]+
 
+This setting is only available with the QtWebKit backend.
+
 [[content-cookies-store]]
 === cookies-store
-Whether to store cookies.
+Whether to store cookies. Note this option needs a restart with QtWebEngine.
 
 Valid values:
 
@@ -1496,7 +1612,7 @@ The file can be in one of the following formats:
 - One host per line
 - A zip-file of any of the above, with either only one file, or a file named 'hosts' (with any extension).
 
-Default: +pass:[http://www.malwaredomainlist.com/hostslist/hosts.txt,http://someonewhocares.org/hosts/hosts,http://winhelp2002.mvps.org/hosts.zip,http://malwaredomains.lehigh.edu/files/justdomains.zip,http://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&amp;mimetype=plaintext]+
+Default: +pass:[https://www.malwaredomainlist.com/hostslist/hosts.txt,http://someonewhocares.org/hosts/hosts,http://winhelp2002.mvps.org/hosts.zip,http://malwaredomains.lehigh.edu/files/justdomains.zip,https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&amp;mimetype=plaintext]+
 
 [[content-host-blocking-enabled]]
 === host-blocking-enabled
@@ -1541,19 +1657,13 @@ CSS border value for hints.
 
 Default: +pass:[1px solid #E3BE23]+
 
-[[hints-opacity]]
-=== opacity
-Opacity for hints.
-
-Default: +pass:[0.7]+
-
 [[hints-mode]]
 === mode
 Mode to use for hints.
 
 Valid values:
 
- * +number+: Use numeric hints.
+ * +number+: Use numeric hints. (In this mode you can also type letters form the hinted element to filter and reduce the number of elements that are hinted.)
  * +letter+: Use the chars in the hints -> chars setting.
  * +word+: Use hints words based on the html elements and the extra words.
 
@@ -1601,18 +1711,20 @@ Default: +pass:[/usr/share/dict/words]+
 
 [[hints-auto-follow]]
 === auto-follow
-Follow a hint immediately when the hint text is completely matched.
+Controls when a hint can be automatically followed without the user pressing Enter.
 
 Valid values:
 
- * +true+
- * +false+
+ * +always+: Auto-follow whenever there is only a single hint on a page.
+ * +unique-match+: Auto-follow whenever there is a unique non-empty match in either the hint string (word mode) or filter (number mode).
+ * +full-match+: Follow the hint when the user typed the whole hint (letter, word or number mode) or the element's text (only in number mode).
+ * +never+: The user will always need to press Enter to follow a hint.
 
-Default: +pass:[true]+
+Default: +pass:[unique-match]+
 
 [[hints-auto-follow-timeout]]
 === auto-follow-timeout
-A timeout to inhibit normal-mode key bindings after a successfulauto-follow.
+A timeout (in milliseconds) to inhibit normal-mode key bindings after a successful auto-follow.
 
 Default: +pass:[0]+
 
@@ -1637,7 +1749,18 @@ Valid values:
  * +javascript+: Better but slower
  * +python+: Slightly worse but faster
 
-Default: +pass:[javascript]+
+Default: +pass:[python]+
+
+[[hints-hide-unmatched-rapid-hints]]
+=== hide-unmatched-rapid-hints
+Controls hiding unmatched hints in rapid mode.
+
+Valid values:
+
+ * +true+
+ * +false+
+
+Default: +pass:[true]+
 
 == searchengines
 Definitions of search engines which can be used via the address bar.
@@ -1647,7 +1770,7 @@ The searchengine named `DEFAULT` is used when `general -> auto-search` is true a
 Aliases for commands.
 By default, no aliases are defined. Example which adds a new command `:qtb` to open qutebrowsers website:
 
-`qtb = open http://www.qutebrowser.org/`
+`qtb = open https://www.qutebrowser.org/`
 
 == colors
 Colors used in the UI.
@@ -1760,42 +1883,6 @@ Background color of the statusbar.
 
 Default: +pass:[black]+
 
-[[colors-statusbar.fg.error]]
-=== statusbar.fg.error
-Foreground color of the statusbar if there was an error.
-
-Default: +pass:[${statusbar.fg}]+
-
-[[colors-statusbar.bg.error]]
-=== statusbar.bg.error
-Background color of the statusbar if there was an error.
-
-Default: +pass:[red]+
-
-[[colors-statusbar.fg.warning]]
-=== statusbar.fg.warning
-Foreground color of the statusbar if there is a warning.
-
-Default: +pass:[${statusbar.fg}]+
-
-[[colors-statusbar.bg.warning]]
-=== statusbar.bg.warning
-Background color of the statusbar if there is a warning.
-
-Default: +pass:[darkorange]+
-
-[[colors-statusbar.fg.prompt]]
-=== statusbar.fg.prompt
-Foreground color of the statusbar if there is a prompt.
-
-Default: +pass:[${statusbar.fg}]+
-
-[[colors-statusbar.bg.prompt]]
-=== statusbar.bg.prompt
-Background color of the statusbar if there is a prompt.
-
-Default: +pass:[darkblue]+
-
 [[colors-statusbar.fg.insert]]
 === statusbar.fg.insert
 Foreground color of the statusbar in insert mode.
@@ -1979,9 +2066,9 @@ Default: +pass:[black]+
 
 [[colors-hints.bg]]
 === hints.bg
-Background color for hints.
+Background color for hints. Note that you can use a `rgba(...)` value for transparency.
 
-Default: +pass:[-webkit-gradient(linear, left top, left bottom, color-stop(0%,#FFF785), color-stop(100%,#FFC542))]+
+Default: +pass:[qlineargradient(x1:0, y1:0, x2:0, y2:1, stop:0 rgba(255, 247, 133, 0.8), stop:1 rgba(255, 197, 66, 0.8))]+
 
 [[colors-hints.fg.match]]
 === hints.fg.match
@@ -2081,6 +2168,78 @@ Background color of the keyhint widget.
 
 Default: +pass:[rgba(0, 0, 0, 80%)]+
 
+[[colors-messages.fg.error]]
+=== messages.fg.error
+Foreground color of an error message.
+
+Default: +pass:[white]+
+
+[[colors-messages.bg.error]]
+=== messages.bg.error
+Background color of an error message.
+
+Default: +pass:[red]+
+
+[[colors-messages.border.error]]
+=== messages.border.error
+Border color of an error message.
+
+Default: +pass:[#bb0000]+
+
+[[colors-messages.fg.warning]]
+=== messages.fg.warning
+Foreground color a warning message.
+
+Default: +pass:[white]+
+
+[[colors-messages.bg.warning]]
+=== messages.bg.warning
+Background color of a warning message.
+
+Default: +pass:[darkorange]+
+
+[[colors-messages.border.warning]]
+=== messages.border.warning
+Border color of an error message.
+
+Default: +pass:[#d47300]+
+
+[[colors-messages.fg.info]]
+=== messages.fg.info
+Foreground color an info message.
+
+Default: +pass:[white]+
+
+[[colors-messages.bg.info]]
+=== messages.bg.info
+Background color of an info message.
+
+Default: +pass:[black]+
+
+[[colors-messages.border.info]]
+=== messages.border.info
+Border color of an info message.
+
+Default: +pass:[#333333]+
+
+[[colors-prompts.fg]]
+=== prompts.fg
+Foreground color for prompts.
+
+Default: +pass:[white]+
+
+[[colors-prompts.bg]]
+=== prompts.bg
+Background color for prompts.
+
+Default: +pass:[darkblue]+
+
+[[colors-prompts.selected.bg]]
+=== prompts.selected.bg
+Background color for the selected item in filename prompts.
+
+Default: +pass:[#308cc6]+
+
 == fonts
 Fonts used for the UI, with optional style/weight/size.
 
@@ -2100,6 +2259,12 @@ Font used in the completion widget.
 
 Default: +pass:[8pt ${_monospace}]+
 
+[[fonts-completion.category]]
+=== completion.category
+Font used in the completion categories.
+
+Default: +pass:[bold ${completion}]+
+
 [[fonts-tabbar]]
 === tabbar
 Font used in the tab bar.
@@ -2122,7 +2287,7 @@ Default: +pass:[8pt ${_monospace}]+
 === hints
 Font used for the hints.
 
-Default: +pass:[bold 13px Monospace]+
+Default: +pass:[bold 13px ${_monospace}]+
 
 [[fonts-debug-console]]
 === debug-console
@@ -2195,3 +2360,27 @@ Default: empty
 Font used in the keyhint widget.
 
 Default: +pass:[8pt ${_monospace}]+
+
+[[fonts-messages.error]]
+=== messages.error
+Font used for error messages.
+
+Default: +pass:[8pt ${_monospace}]+
+
+[[fonts-messages.warning]]
+=== messages.warning
+Font used for warning messages.
+
+Default: +pass:[8pt ${_monospace}]+
+
+[[fonts-messages.info]]
+=== messages.info
+Font used for info messages.
+
+Default: +pass:[8pt ${_monospace}]+
+
+[[fonts-prompts]]
+=== prompts
+Font used for prompts.
+
+Default: +pass:[8pt sans-serif]+

doc/quickstart.asciidoc

@@ -9,7 +9,8 @@ Basic keybindings to get you started
 ------------------------------------
 
 * Use the arrow keys or `hjkl` to move around a webpage (vim-like syntax is used in quite a few places)
-* To go to a new webpage, press `o`, then type a url, then press Enter (Use `O` to open the url in a new tab). If what you've typed isn't a url, then a search engine will be used instead (DuckDuckGo, by default)
+* To go to a new webpage, press `o`, then type a url, then press Enter (Use `O` to open the url in a new tab, `go` to edit the current URL)
+* If what you've typed isn't a url, then a search engine will be used instead (DuckDuckGo, by default)
 * To switch between tabs, use `J` (next tab) and `K` (previous tab), or press `<Alt-num>`, where `num` is the position of the tab to switch to
 * To close the current tab, press `d` (and press `u` to undo closing a tab)
 * Use `H` and `L` to go back and forth in the history
@@ -24,13 +25,16 @@ What to do now
 * View the link:http://qutebrowser.org/img/cheatsheet-big.png[key binding cheatsheet]
 to make yourself familiar with the key bindings: +
 image:http://qutebrowser.org/img/cheatsheet-small.png["qutebrowser key binding cheatsheet",link="http://qutebrowser.org/img/cheatsheet-big.png"]
+* There's also a https://www.shortcutfoo.com/app/dojos/qutebrowser[free training
+  course] on shortcutfoo for the keybindings - note that you need to be in
+  insert mode (i) for it to work.
 * Run `:adblock-update` to download adblock lists and activate adblocking.
 * If you just cloned the repository, you'll need to run
 `scripts/asciidoc2html.py` to generate the documentation.
 * Go to the link:qute://settings[settings page] to set up qutebrowser the way you want it.
 * Subscribe to
-https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser[the mailinglist]
-where there are weekly "what's new in qutebrowser" posts.
+https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser[the mailinglist] or
+https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser-announce[the announce-only mailinglist].
 * Let me know what features you are missing or things that need (even small!)
 improvements.
 

doc/qutebrowser.1.asciidoc

@@ -7,10 +7,10 @@
 :man source: qutebrowser
 :man manual: qutebrowser manpage
 :toc:
-:homepage: http://www.qutebrowser.org/
+:homepage: https://www.qutebrowser.org/
 
 == NAME
-qutebrowser - a keyboard-driven, vim-like browser based on PyQt5 and QtWebKit.
+qutebrowser - a keyboard-driven, vim-like browser based on PyQt5.
 
 == SYNOPSIS
 *qutebrowser* ['-OPTION' ['...']] [':COMMAND' ['...']] ['URL' ['...']]
@@ -38,17 +38,8 @@ show it.
 *-h*, *--help*::
     show this help message and exit
 
-*-c* 'CONFDIR', *--confdir* 'CONFDIR'::
-    Set config directory (empty for no config storage).
-
-*--datadir* 'DATADIR'::
-    Set data directory (empty for no data storage).
-
-*--cachedir* 'CACHEDIR'::
-    Set cache directory (empty for no cache storage).
-
 *--basedir* 'BASEDIR'::
-    Base directory for all storage. Other --*dir arguments are ignored if this is given.
+    Base directory for all storage.
 
 *-V*, *--version*::
     Show version and quit.
@@ -68,6 +59,9 @@ show it.
 *--backend* '{webkit,webengine}'::
     Which backend to use (webengine backend is EXPERIMENTAL!).
 
+*--enable-webengine-inspector*::
+    Enable the web inspector for QtWebEngine. Note that this is a SECURITY RISK and you should not visit untrusted websites with the inspector turned on. See https://bugreports.qt.io/browse/QTBUG-50725 for more details.
+
 === debug arguments
 *-l* '{critical,error,warning,info,debug,vdebug}', *--loglevel* '{critical,error,warning,info,debug,vdebug}'::
     Set loglevel
@@ -111,23 +105,11 @@ show it.
 *--no-err-windows*::
     Don't show any error windows (used for tests/smoke.py).
 
-*--qt-name* 'NAME'::
-    Set the window name.
+*--qt-arg* 'NAME' 'VALUE'::
+    Pass an argument with a value to Qt. For example, you can do `--qt-arg geometry 650x555+200+300` to set the window geometry.
 
-*--qt-style* 'STYLE'::
-    Set the Qt GUI style to use.
-
-*--qt-stylesheet* 'STYLESHEET'::
-    Override the Qt application stylesheet.
-
-*--qt-widgetcount*::
-    Print debug message at the end about number of widgets left undestroyed and maximum number of widgets existed at the same time.
-
-*--qt-reverse*::
-    Set the application's layout direction to right-to-left.
-
-*--qt-qmljsdebugger* 'port:PORT[,block]'::
-    Activate the QML/JS debugger with a specified port. 'block' is optional and will make the application wait until a debugger connects to it.
+*--qt-flag* 'QT_FLAG'::
+    Pass an argument to Qt as flag.
 // QUTE_OPTIONS_END
 
 == FILES
@@ -145,7 +127,7 @@ defaults.
 
 == BUGS
 Bugs are tracked in the Github issue tracker at 
-https://github.com/The-Compiler/qutebrowser/issues.
+https://github.com/qutebrowser/qutebrowser/issues.
 
 If you found a bug, use the built-in ':report' command to create a bug report
 with all information needed.
@@ -155,7 +137,7 @@ https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser[mailinglist] at
 mailto:qutebrowser@lists.qutebrowser.org[] instead.
 
 For security bugs, please contact me directly at me@the-compiler.org, GPG ID
-http://www.the-compiler.org/pubkey.asc[0xFD55A072].
+https://www.the-compiler.org/pubkey.asc[0xFD55A072].
 
 == COPYRIGHT
 This program is free software: you can redistribute it and/or modify it under
@@ -171,12 +153,14 @@ You should have received a copy of the GNU General Public License along with
 this program.  If not, see <http://www.gnu.org/licenses/>.
 
 == RESOURCES
-* Website: http://www.qutebrowser.org/
+* Website: https://www.qutebrowser.org/
 * Mailinglist: mailto:qutebrowser@lists.qutebrowser.org[] /
 https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser
+* Announce-only mailinglist: mailto:qutebrowser-announce@lists.qutebrowser.org[] /
+https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser-announce
 * IRC: irc://irc.freenode.org/#qutebrowser[`#qutebrowser`] on
 http://freenode.net/[Freenode]
-* Github: https://github.com/The-Compiler/qutebrowser
+* Github: https://github.com/qutebrowser/qutebrowser
 
 == AUTHOR
 *qutebrowser* was written by Florian Bruhin. All contributors can be found in

doc/stacktrace.asciidoc

@@ -41,21 +41,8 @@ For Archlinux, no debug informations are provided. You can either compile Qt
 yourself (which will take a few hours even on a modern machine) or use
 debugging symbols compiled/packaged by me (x86_64 only).
 
-.To compile by yourself
-
-----
-$ git clone https://github.com/The-Compiler/qt-debug-pkgbuild.git
-$ cd qt-debug-pkgbuild
-$ git checkout symbols
-$ export DEBUG_CFLAGS='-ggdb3 -fvar-tracking-assignments -Og'
-$ export DEBUG_CXXFLAGS='-ggdb3 -fvar-tracking-assignments -Og'
-$ cd qt5
-$ makepkg -si --pkg qt5-base-debug,qt5-webkit-debug,qt5-webengine-debug
-$ cd ../pyqt5
-$ makepkg -si --pkg pyqt5-common-debug,python-pyqt5-debug
-----
-
-.To install my pre-built packages
+To install my pre-built packages
+++++++++++++++++++++++++++++++++
 
 First download and sign the key:
 
@@ -76,12 +63,30 @@ Server = http://qutebrowser.org/qt-debug/$arch
 Then install the packages:
 
 ----
-# pacman -Suy pyqt5-common-debug python-pyqt5-debug qt5-base-debug qt5-webkit-debug,qt5-webengine-debug
+# pacman -Suy pyqt5-common-debug python-pyqt5-debug qt5-base-debug qt5-webkit-debug qt5-webengine-debug
 ----
 
 The `-debug` packages conflict with the non-debug variants - it's safe to
 remove them.
 
+To compile by yourself
+++++++++++++++++++++++
+
+Note that building Qt will likely take multiple hours, even on a recent system.
+I'd also expect it to take around 6 GB of RAM and 30 GB of disk space for a
+successful compile run.
+
+----
+$ git clone https://github.com/qutebrowser/qt-debug-pkgbuild.git
+$ cd qt-debug-pkgbuild
+$ export DEBUG_CFLAGS='-ggdb3 -fvar-tracking-assignments -Og'
+$ export DEBUG_CXXFLAGS='-ggdb3 -fvar-tracking-assignments -Og'
+$ cd qt5
+$ makepkg -si --pkg qt5-base-debug,qt5-webkit-debug,qt5-webengine-debug
+$ cd ../pyqt5
+$ makepkg -si --pkg pyqt5-common-debug,python-pyqt5-debug
+----
+
 Getting the stack trace
 ~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -169,7 +174,7 @@ When you see the _qutebrowser.exe has stopped working_ window, do not click
 file displayed there.
 
 Now install
-http://www.microsoft.com/en-us/download/details.aspx?id=42933[DebugDiag] from
+https://www.microsoft.com/en-us/download/details.aspx?id=49924[DebugDiag] from
 Microsoft, then run the *DebugDiag 2 Analysis* tool. There, check
 *CrashHangAnalysis* and add your crash dump via *Add Data files*. Then click
 *Start analysis*.

doc/userscripts.asciidoc

@@ -22,6 +22,8 @@ To call a userscript, it needs to be stored in your data directory under
 `userscripts` (for example: `~/.local/share/qutebrowser/userscripts/myscript`), 
 or just use an absolute path.
 
+NOTE: On Windows, only userscripts with `com`, `bat`, or `exe` extensions will be launched.
+
 Getting information
 -------------------
 
@@ -36,13 +38,15 @@ The following environment variables will be set when a userscript is launched:
 - `QUTE_CONFIG_DIR`: Path of the directory containing qutebrowser's configuration.
 - `QUTE_DATA_DIR`: Path of the directory containing qutebrowser's data.
 - `QUTE_DOWNLOAD_DIR`: Path of the downloads directory.
+- `QUTE_COMMANDLINE_TEXT`: Text currently in qutebrowser's command line.
 
 In `command` mode:
 
 - `QUTE_URL`: The current URL.
 - `QUTE_TITLE`: The title of the current page.
 - `QUTE_SELECTED_TEXT`: The text currently selected on the page.
-- `QUTE_SELECTED_HTML` The HTML currently selected on the page.
+- `QUTE_SELECTED_HTML` The HTML currently selected on the page (not supported
+  with QtWebEngine).
 
 In `hints` mode:
 

misc/docker/archlinux/Dockerfile

@@ -1,35 +0,0 @@
-FROM base/archlinux
-MAINTAINER Florian Bruhin <me@the-compiler.org>
-
-RUN echo 'Server = http://mirror.de.leaseweb.net/archlinux/$repo/os/$arch' > /etc/pacman.d/mirrorlist
-RUN pacman-key --init && pacman-key --populate archlinux && pacman -Sy --noconfirm archlinux-keyring
-
-RUN pacman -Suyy --noconfirm
-RUN pacman-db-upgrade
-
-RUN pacman -S --noconfirm \
-    git \
-    python-tox \
-    qt5-base \
-    qt5-webkit \
-    python-pyqt5 \
-    xorg-xinit \
-    herbstluftwm \
-    xorg-server-xvfb
-
-RUN echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen && locale-gen
-
-RUN useradd user && mkdir /home/user && chown -R user:users /home/user
-USER user
-WORKDIR /home/user
-
-ENV DISPLAY=:0
-ENV LC_ALL=en_US.UTF-8
-ENV LANG=en_US.UTF-8
-
-CMD Xvfb -screen 0 800x600x24 :0 & \
-    sleep 2 && \
-    herbstluftwm & \
-    git clone /outside qutebrowser.git && \
-    cd qutebrowser.git && \
-    tox -e py35

misc/docker/debian-jessie/Dockerfile

@@ -1,35 +0,0 @@
-FROM debian:jessie
-MAINTAINER Florian Bruhin <me@the-compiler.org>
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get -y update && \
-    apt-get -y dist-upgrade && \
-    apt-get -y install --no-install-recommends \
-            python3-pyqt5 \
-            python3-pyqt5.qtwebkit \
-            python-tox \
-            python3-sip \
-            xvfb \
-            git \
-            python3-setuptools \
-            wget \
-            herbstluftwm \
-            locales \
-            libjs-pdf
-RUN echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen && locale-gen
-
-RUN useradd user && mkdir /home/user && chown -R user:users /home/user
-USER user
-WORKDIR /home/user
-
-ENV DISPLAY=:0
-ENV LC_ALL=en_US.UTF-8
-ENV LANG=en_US.UTF-8
-
-CMD Xvfb -screen 0 800x600x24 :0 & \
-    sleep 2 && \
-    herbstluftwm & \
-    git clone /outside qutebrowser.git && \
-    cd qutebrowser.git && \
-    tox -e py34

misc/docker/ubuntu-wily/Dockerfile

@@ -1,37 +0,0 @@
-FROM ubuntu:wily
-MAINTAINER Florian Bruhin <me@the-compiler.org>
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get -y update && \
-    apt-get -y dist-upgrade && \
-    apt-get -y install --no-install-recommends \
-            python3-pyqt5 \
-            python3-pyqt5.qtwebkit \
-            python-tox \
-            python3-sip \
-            xvfb \
-            git \
-            python3-setuptools \
-            wget \
-            herbstluftwm \
-            language-pack-en \
-            libjs-pdf \
-            dbus
-
-RUN dbus-uuidgen --ensure
-
-RUN useradd user && mkdir /home/user && chown -R user:users /home/user
-USER user
-WORKDIR /home/user
-
-ENV DISPLAY=:0
-ENV LC_ALL=en_US.UTF-8
-ENV LANG=en_US.UTF-8
-
-CMD Xvfb -screen 0 800x600x24 :0 & \
-    sleep 2 && \
-    herbstluftwm & \
-    git clone /outside qutebrowser.git && \
-    cd qutebrowser.git && \
-    tox -e py34

misc/docker/ubuntu-xenial/Dockerfile

@@ -1,37 +0,0 @@
-FROM ubuntu:xenial
-MAINTAINER Florian Bruhin <me@the-compiler.org>
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get -y update && \
-    apt-get -y dist-upgrade && \
-    apt-get -y install --no-install-recommends \
-            python3-pyqt5 \
-            python3-pyqt5.qtwebkit \
-            python-tox \
-            python3-sip \
-            xvfb \
-            git \
-            python3-setuptools \
-            wget \
-            herbstluftwm \
-            language-pack-en \
-            libjs-pdf \
-            dbus
-
-RUN dbus-uuidgen --ensure
-
-RUN useradd user && mkdir /home/user && chown -R user:users /home/user
-USER user
-WORKDIR /home/user
-
-ENV DISPLAY=:0
-ENV LC_ALL=en_US.UTF-8
-ENV LANG=en_US.UTF-8
-
-CMD Xvfb -screen 0 800x600x24 :0 & \
-    sleep 2 && \
-    herbstluftwm & \
-    git clone /outside qutebrowser.git && \
-    cd qutebrowser.git && \
-    tox -e py35

misc/qutebrowser.spec

@@ -70,5 +70,9 @@ coll = COLLECT(exe,
 app = BUNDLE(coll,
              name='qutebrowser.app',
              icon=icon,
-             info_plist={'NSHighResolutionCapable': 'True'},
-             bundle_identifier=None)
+             info_plist={
+                 'NSHighResolutionCapable': 'True',
+                 'NSSupportsAutomaticGraphicsSwitching': 'True',
+             },
+             # https://github.com/pyinstaller/pyinstaller/blob/b78bfe530cdc2904f65ce098bdf2de08c9037abb/PyInstaller/hooks/hook-PyQt5.QtWebEngineWidgets.py#L24
+             bundle_identifier='org.qt-project.Qt.QtWebEngineCore')

misc/requirements/requirements-check-manifest.txt

@@ -1,3 +1,3 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-check-manifest==0.31
+check-manifest==0.35

misc/requirements/requirements-codecov.txt

@@ -1,5 +1,5 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
 codecov==2.0.5
-coverage==4.1
-requests==2.10.0
+coverage==4.3.4
+requests==2.13.0

misc/requirements/requirements-cxfreeze.txt

@@ -1,3 +1,3 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-cx-Freeze==4.3.4
+cx-Freeze==4.3.4  # rq.filter: < 5.0.0

misc/requirements/requirements-cxfreeze.txt-raw

@@ -1 +1,5 @@
-cx_Freeze
+cx-Freeze < 5.0.0
+
+# We'll probably switch to PyInstaller soon, and 5.x doesn't install without a
+# compiler?
+#@ filter: cx-Freeze < 5.0.0

misc/requirements/requirements-flake8.txt

@@ -2,25 +2,19 @@
 
 flake8==2.6.2  # rq.filter: < 3.0.0
 flake8-copyright==0.2.0
-flake8-debugger==1.4.0
-flake8-deprecated==1.0
-flake8-docstrings==0.2.8
+flake8-debugger==1.4.0  # rq.filter: != 2.0.0
+flake8-deprecated==1.1
+flake8-docstrings==1.0.3
 flake8-future-import==0.4.3
-flake8-mock==0.2
-flake8-pep3101==0.4
+flake8-mock==0.3
+flake8-pep3101==1.0
+flake8-polyfill==1.0.1
 flake8-putty==0.4.0
-flake8-string-format==0.2.2
-flake8-tidy-imports==1.0.2
+flake8-string-format==0.2.3
+flake8-tidy-imports==1.0.5
 flake8-tuple==0.2.12
-hacking==0.11.0
-mccabe==0.5.0
-packaging==16.7
-pbr==1.10.0
-pep257==0.7.0  # still needed by flake8-docstrings but ignored
-pep8==1.7.0
+mccabe==0.6.1
 pep8-naming==0.4.1
-pycodestyle==2.0.0
-pydocstyle==1.0.0
-pyflakes==1.2.3
-pyparsing==2.1.5
-six==1.10.0
+pycodestyle==2.3.1
+pydocstyle==1.1.1
+pyflakes==1.5.0

misc/requirements/requirements-flake8.txt-raw

@@ -1,6 +1,6 @@
 flake8<3.0.0
 flake8-copyright
-flake8-debugger
+flake8-debugger!=2.0.0
 flake8-deprecated
 flake8-docstrings
 flake8-future-import
@@ -10,14 +10,17 @@ flake8-putty
 flake8-string-format
 flake8-tidy-imports
 flake8-tuple
-hacking
 pep8-naming
 pydocstyle
 pyflakes
 
-pep8==1.7.0
+# Pinned to 2.0.0 otherwise
+pycodestyle==2.3.1
+# Pinned to 0.5.3 otherwise
+mccabe==0.6.1
 
-#@ comment: pep257 still needed by flake8-docstrings but ignored
-
-# Waiting until hacking/flake8-tuple are updated
+# Waiting until flake8-putty updated
 #@ filter: flake8 < 3.0.0
+
+# https://github.com/JBKahn/flake8-debugger/issues/5
+#@ filter: flake8-debugger != 2.0.0

misc/requirements/requirements-pip.txt

@@ -1 +1,8 @@
-pip==8.1.2
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
+appdirs==1.4.2
+packaging==16.8
+pyparsing==2.1.10
+setuptools==34.3.0
+six==1.10.0
+wheel==0.29.0

misc/requirements/requirements-pyinstaller.txt

@@ -1,3 +1,5 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-PyInstaller==3.2
+-e git+https://github.com/pyinstaller/pyinstaller.git@develop#egg=PyInstaller
+PyQt5==5.8
+sip==4.19.1

misc/requirements/requirements-pyinstaller.txt-raw

@@ -1 +1,5 @@
-PyInstaller
+-e git+https://github.com/pyinstaller/pyinstaller.git@develop#egg=PyInstaller
+PyQt5
+
+# remove @commit-id for scm installs
+#@ replace: @.*# @develop#

misc/requirements/requirements-pylint-master.txt

@@ -1,11 +1,11 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
 -e git+https://github.com/PyCQA/astroid.git#egg=astroid
+editdistance==0.3.1
 isort==4.2.5
 lazy-object-proxy==1.2.2
-mccabe==0.5.0
+mccabe==0.6.1
 -e git+https://github.com/PyCQA/pylint.git#egg=pylint
 ./scripts/dev/pylint_checkers
-requests==2.10.0
-six==1.10.0
+requests==2.13.0
 wrapt==1.10.8

misc/requirements/requirements-pylint-master.txt-raw

@@ -3,9 +3,6 @@
 ./scripts/dev/pylint_checkers
 requests
 
-# https://github.com/PyCQA/pylint/issues/932
-mccabe==0.5.0
-
 # remove @commit-id for scm installs
 #@ replace: @.*# #
 

misc/requirements/requirements-pylint.txt

@@ -1,11 +1,13 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-astroid==1.4.7
+astroid==1.4.9
+github3.py==0.9.6
 isort==4.2.5
 lazy-object-proxy==1.2.2
-mccabe==0.5.0
-pylint==1.6.4
+mccabe==0.6.1
+pylint==1.6.5
 ./scripts/dev/pylint_checkers
-requests==2.10.0
-six==1.10.0
+requests==2.13.0
+uritemplate==3.0.0
+uritemplate.py==3.0.2
 wrapt==1.10.8

misc/requirements/requirements-pylint.txt-raw

@@ -1,6 +1,7 @@
 pylint
 ./scripts/dev/pylint_checkers
 requests
+github3.py
 
 # fix qute-pylint location
 #@ replace: qute-pylint==.* ./scripts/dev/pylint_checkers

misc/requirements/requirements-pyroma.txt

@@ -1,4 +1,4 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-docutils==0.12
-pyroma==2.0.2
+docutils==0.13.1
+pyroma==2.2

misc/requirements/requirements-tests-git.txt

@@ -0,0 +1,49 @@
+bzr+lp:beautifulsoup
+git+https://github.com/cherrypy/cheroot.git
+hg+https://bitbucket.org/ned/coveragepy
+git+https://github.com/micheles/decorator.git
+git+https://github.com/pallets/flask.git
+git+https://github.com/miracle2k/python-glob2.git
+git+https://github.com/Runscope/httpbin.git
+git+https://github.com/HypothesisWorks/hypothesis-python.git
+git+https://github.com/pallets/itsdangerous.git
+git+https://bitbucket.org/zzzeek/mako.git
+git+https://github.com/r1chardj0n3s/parse.git
+git+https://github.com/jenisys/parse_type.git
+hg+https://bitbucket.org/pytest-dev/py
+git+https://github.com/pytest-dev/pytest.git@features
+git+https://github.com/pytest-dev/pytest-bdd.git
+
+# This is broken at the moment because logfail tries to access
+# LogCaptureHandler
+# git+https://github.com/eisensheng/pytest-catchlog.git
+pytest-catchlog==1.2.2
+
+git+https://github.com/pytest-dev/pytest-cov.git
+git+https://github.com/pytest-dev/pytest-faulthandler.git
+git+https://github.com/pytest-dev/pytest-instafail.git
+git+https://github.com/pytest-dev/pytest-mock.git
+git+https://github.com/pytest-dev/pytest-qt.git
+git+https://github.com/pytest-dev/pytest-repeat.git
+git+https://github.com/pytest-dev/pytest-rerunfailures.git
+git+https://github.com/abusalimov/pytest-travis-fold.git
+git+https://github.com/fschulze/pytest-warnings.git
+git+https://github.com/The-Compiler/pytest-xvfb.git
+hg+https://bitbucket.org/gutworth/six
+hg+https://bitbucket.org/jendrikseipp/vulture
+git+https://github.com/pallets/werkzeug.git
+
+
+## qutebrowser dependencies
+
+git+https://github.com/tartley/colorama.git
+hg+https://bitbucket.org/cthedot/cssutils
+git+https://github.com/pallets/jinja.git
+git+https://github.com/pallets/markupsafe.git
+hg+http://bitbucket.org/birkenfeld/pygments-main
+hg+https://bitbucket.org/fdik/pypeg
+
+# Fails to build:
+# gcc: error: ext/_yaml.c: No such file or directory
+# hg+https://bitbucket.org/xi/pyyaml
+PyYAML==3.12

misc/requirements/requirements-tests.txt

@@ -1,32 +1,36 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-beautifulsoup4==4.5.0
-CherryPy==7.1.0
-coverage==4.1
-decorator==4.0.10
-Flask==0.10.1  # rq.filter: < 0.11.0
-glob2==0.4.1
-httpbin==0.4.1
-hypothesis==3.4.2
+beautifulsoup4==4.5.3
+cheroot==5.1.0
+click==6.7
+coverage==4.3.4
+decorator==4.0.11
+EasyProcess==0.2.3
+Flask==0.12
+glob2==0.5
+httpbin==0.5.0
+hypothesis==3.6.1
 itsdangerous==0.24
-# Jinja2==2.8
-Mako==1.0.4
+# Jinja2==2.9.5
+Mako==1.0.6
 # MarkupSafe==0.23
 parse==1.6.6
 parse-type==0.3.4
-py==1.4.31
-pytest==2.9.2
-pytest-bdd==2.17.0
+py==1.4.32
+pytest==3.0.6
+pytest-bdd==2.18.1
+pytest-benchmark==3.0.0
 pytest-catchlog==1.2.2
-pytest-cov==2.3.0
-pytest-faulthandler==1.3.0
+pytest-cov==2.4.0
+pytest-faulthandler==1.3.1
 pytest-instafail==0.3.0
-pytest-mock==1.1
-pytest-qt==1.11.0
-pytest-repeat==0.3.0
-pytest-rerunfailures==2.0.0
+pytest-mock==1.5.0
+pytest-qt==2.1.0
+pytest-repeat==0.4.1
+pytest-rerunfailures==2.1.0
 pytest-travis-fold==1.2.0
-pytest-xvfb==0.2.0
-six==1.10.0
-vulture==0.10
-Werkzeug==0.11.10
+pytest-warnings==0.2.0
+pytest-xvfb==1.0.0
+PyVirtualDisplay==0.2.1
+vulture==0.12
+Werkzeug==0.11.15

misc/requirements/requirements-tests.txt-raw

@@ -1,11 +1,12 @@
 beautifulsoup4
-CherryPy
+cheroot
 coverage
-Flask==0.10.1
+Flask
 httpbin
 hypothesis
 pytest
 pytest-bdd
+pytest-benchmark
 pytest-catchlog
 pytest-cov
 pytest-faulthandler
@@ -15,8 +16,8 @@ pytest-qt
 pytest-repeat
 pytest-rerunfailures
 pytest-travis-fold
+pytest-warnings
 pytest-xvfb
 vulture
 
-#@ filter: Flask < 0.11.0
 #@ ignore: Jinja2, MarkupSafe

misc/requirements/requirements-tox.txt

@@ -1,6 +1,6 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-pluggy==0.3.1
-py==1.4.31
-tox==2.3.1
-virtualenv==15.0.2
+pluggy==0.4.0
+py==1.4.32
+tox==2.6.0
+virtualenv==15.1.0

misc/requirements/requirements-tox.txt-raw

@@ -1 +1,4 @@
 tox
+
+# The latest tox release still depends on pluggy < 0.4...
+pluggy==0.4.0

misc/requirements/requirements-vulture.txt

@@ -1,3 +1,3 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-vulture==0.10
+vulture==0.12

misc/userscripts/cast

@@ -0,0 +1,150 @@
+#!/usr/bin/env bash
+#
+# Behaviour
+#   Userscript for qutebrowser which casts the url passed in $1 to the default
+#   ChromeCast device in the network using the program `castnow`
+#
+# Usage
+#   You can launch the script from qutebrowser as follows:
+#       spawn --userscript ${PATH_TO_FILE} {url}
+#
+#   Then, you can control the chromecast by launching the simple command
+#   `castnow` in a shell which will connect to the running castnow instance.
+#
+#   For stopping the script, issue the command `pkill -f castnow` which would
+#   then let the rest of the userscript execute for cleaning temporary file.
+#
+# Thanks
+#   This userscript borrows Thorsten Wißmann's javascript code from his `mpv`
+#   userscript.
+#
+# Dependencies
+#   - castnow, https://github.com/xat/castnow
+#
+# Author
+#   Simon Désaulniers <sim.desaulniers@gmail.com>
+
+if [ -z "$QUTE_FIFO" ] ; then
+    cat 1>&2 <<EOF
+Error: $0 can not be run as a standalone script.
+
+It is a qutebrowser userscript. In order to use it, call it using
+'spawn --userscript' as described in qute://help/userscripts.html
+EOF
+    exit 1
+fi
+
+msg() {
+    local cmd="$1"
+    shift
+    local msg="$*"
+    if [ -z "$QUTE_FIFO" ] ; then
+        echo "$cmd: $msg" >&2
+    else
+        echo "message-$cmd '${msg//\'/\\\'}'" >> "$QUTE_FIFO"
+    fi
+}
+
+js() {
+cat <<EOF
+
+    function descendantOfTagName(child, ancestorTagName) {
+        // tells whether child has some (proper) ancestor
+        // with the tag name ancestorTagName
+        while (child.parentNode != null) {
+            child = child.parentNode;
+            if (typeof child.tagName === 'undefined') break;
+            if (child.tagName.toUpperCase() == ancestorTagName.toUpperCase()) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    var App = {};
+
+    var all_videos = [];
+    all_videos.push.apply(all_videos, document.getElementsByTagName("video"));
+    all_videos.push.apply(all_videos, document.getElementsByTagName("object"));
+    all_videos.push.apply(all_videos, document.getElementsByTagName("embed"));
+    App.backup_videos = Array();
+    App.all_replacements = Array();
+    for (i = 0; i < all_videos.length; i++) {
+        var video = all_videos[i];
+        if (descendantOfTagName(video, "object")) {
+            // skip tags that are contained in an object, because we hide
+            // the object anyway.
+            continue;
+        }
+        var replacement = document.createElement("div");
+        replacement.innerHTML = "
+            <p style=\\"margin-bottom: 0.5em\\">
+            The video is being cast on your ChromeCast device.
+            </p>
+            <p>
+            In order to restore this particular video
+            <a style=\\"font-weight: bold;
+                        color: white;
+                        background: transparent;
+                     \\"
+               onClick=\\"restore_video(this, " + i + ");\\"
+               href=\\"javascript: restore_video(this, " + i + ")\\"
+              >click here</a>.
+            </p>
+        ";
+        replacement.style.position = "relative";
+        replacement.style.zIndex = "100003000000";
+        replacement.style.fontSize = "1rem";
+        replacement.style.textAlign = "center";
+        replacement.style.verticalAlign = "middle";
+        replacement.style.height = "100%";
+        replacement.style.background = "#101010";
+        replacement.style.color = "white";
+        replacement.style.border = "4px dashed #545454";
+        replacement.style.padding = "2em";
+        replacement.style.margin = "auto";
+        App.all_replacements[i] = replacement;
+        App.backup_videos[i] = video;
+        video.parentNode.replaceChild(replacement, video);
+    }
+
+    function restore_video(obj, index) {
+        obj = App.all_replacements[index];
+        video = App.backup_videos[index];
+        console.log(video);
+        obj.parentNode.replaceChild(video, obj);
+    }
+
+    /** force repainting the video, thanks to:
+     * http://martinwolf.org/2014/06/10/force-repaint-of-an-element-with-javascript/
+     */
+    var siteHeader = document.getElementById('header');
+    siteHeader.style.display='none';
+    siteHeader.offsetHeight; // no need to store this anywhere, the reference is enough
+    siteHeader.style.display='block';
+
+EOF
+}
+
+printjs() {
+    js | sed 's,//.*$,,' | tr '\n' ' '
+}
+echo "jseval -q $(printjs)" >> "$QUTE_FIFO"
+
+tmpdir=$(mktemp -d)
+file_to_cast=${tmpdir}/qutecast
+
+# kill any running instance of castnow
+pkill -f /usr/bin/castnow
+
+# start youtube download in stream mode (-o -) into temporary file
+youtube-dl -qo - "$1" > ${file_to_cast} &
+ytdl_pid=$!
+
+msg info "Casting $1" >> "$QUTE_FIFO"
+# start castnow in stream mode to cast on ChromeCast
+tail -F "${file_to_cast}" | castnow -
+
+# cleanup remaining background process and file on disk
+kill ${ytdl_pid}
+rm -rf ${tmpdir}

misc/userscripts/password_fill

@@ -342,10 +342,14 @@ cat <<EOF
         for (var j = 0; j < inputs.length; j++) {
             var input = inputs[j];
             if (input.type == "text" || input.type == "email") {
+                input.focus();
                 input.value = "$(javascript_escape "${username}")";
+                input.blur();
             }
             if (input.type == "password") {
+                input.focus();
                 input.value = "$(javascript_escape "${password}")";
+                input.blur();
             }
         }
     };

misc/userscripts/qutebrowser_viewsource

@@ -24,9 +24,9 @@
 # Caveat: Does not use authentication of any kind. Add it in if you want it to.
 #
 
-path=/tmp/qutebrowser_$(mktemp XXXXXXXX).html
+path=$(mktemp --tmpdir qutebrowser_XXXXXXXX.html)
 
-curl "$QUTE_URL" > $path
+curl "$QUTE_URL" > "$path"
 urxvt -e vim "$path"
 
 rm "$path"

misc/userscripts/readability

@@ -0,0 +1,25 @@
+#!/usr/bin/env python2
+#
+# Executes python-readability on current page and opens the summary as new tab.
+#
+# Usage:
+#   :spawn --userscript readability
+#
+from __future__ import absolute_import
+import codecs, os
+from readability.readability import Document
+
+tmpfile=os.path.expanduser('~/.local/share/qutebrowser/userscripts/readability.html')
+if not os.path.exists(os.path.dirname(tmpfile)):
+    os.makedirs(os.path.dirname(tmpfile))
+
+with codecs.open(os.environ['QUTE_HTML'], 'r', 'utf-8') as source:
+    doc = Document(source.read())
+    content = doc.summary().replace('<html>', '<html><head><title>%s</title></head>' % doc.title())
+
+    with codecs.open(tmpfile, 'w', 'utf-8') as target:
+        target.write('<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />')
+        target.write(content)
+
+    with open(os.environ['QUTE_FIFO'], 'w') as fifo:
+        fifo.write('open -t %s' % tmpfile)

misc/userscripts/ripbang

@@ -0,0 +1,27 @@
+#!/usr/bin/env python2
+#
+# Adds DuckDuckGo bang as searchengine.
+#
+# Usage:
+#   :spawn --userscript ripbang [bang]...
+#
+# Example:
+#   :spawn --userscript ripbang amazon maps
+#
+import os, re, requests, sys, urllib
+
+for argument in sys.argv[1:]:
+    bang = '!' + argument
+    r = requests.get('https://duckduckgo.com/',
+                     params={'q': bang + ' SEARCHTEXT'})
+
+    searchengine = urllib.unquote(re.search("url=[^']+", r.text).group(0))
+    searchengine = searchengine.replace('url=', '')
+    searchengine = searchengine.replace('/l/?kh=-1&uddg=', '')
+    searchengine = searchengine.replace('SEARCHTEXT', '{}')
+
+    if os.getenv('QUTE_FIFO'):
+        with open(os.environ['QUTE_FIFO'], 'w') as fifo:
+            fifo.write('set searchengines %s %s' % (bang, searchengine))
+    else:
+        print '%s %s' % (bang, searchengine)

misc/userscripts/rss

@@ -0,0 +1,122 @@
+#!/bin/sh
+
+# Copyright 2016 Jan Verbeek (blyxxyz) <ring@openmailbox.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+# This script keeps track of URLs in RSS feeds and opens new ones.
+# New feeds can be added with ':spawn -u /path/to/userscripts/rss add' or
+# ':spawn -u /path/to/userscripts/rss <url>'.
+# New items can be opened with ':spawn -u /path/to/userscripts/rss'.
+# The script doesn't really parse XML, and searches for things that look like
+# item links. It might open things that aren't real links, and it might miss
+# real links.
+
+config_dir="$HOME/.qute-rss"
+
+add_feed () {
+    touch "feeds"
+    if grep -Fq "$1" "feeds"; then
+        notice "$1 is saved already."
+    else
+        printf "%s\n" "$1" >> "feeds"
+    fi
+}
+
+# Show an error message and exit
+fail () {
+    echo "message-error '$*'" > "$QUTE_FIFO"
+    exit 1
+}
+
+# Get a sorted list of item URLs from a RSS feed
+get_items () {
+    $curl "$@" | grep "$text_only" -zo -e '<guid[^<>]*>[^<>]*</guid>' \
+                      -e '<link[^<>]*>[^<>]*</link>' \
+                      -e '<link[^<>]*href="[^"]*"' |
+        grep "$text_only" -o 'http[^<>"]*' | sort | uniq
+}
+
+# Show an info message
+notice () {
+    echo "message-info '$*'" > "$QUTE_FIFO"
+}
+
+# Update a database of a feed and open new URLs
+read_items () {
+    cd read_urls
+    feed_file="$(echo "$1" | tr -d /)"
+    feed_temp_file="$(mktemp "$feed_file.tmp.XXXXXXXXXX")"
+    feed_new_items="$(mktemp "$feed_file.new.XXXXXXXXXX")"
+    get_items "$1" > "$feed_temp_file"
+    if [ ! -s "$feed_temp_file" ]; then
+        notice "No items found for $1."
+        rm "$feed_temp_file" "$feed_new_items"
+    elif [ ! -f "$feed_file" ]; then
+        notice "$1 is a new feed. All items will be marked as read."
+        mv "$feed_temp_file" "$feed_file"
+        rm "$feed_new_items"
+    else
+        sort -o "$feed_file" "$feed_file"
+        comm -2 -3 "$feed_temp_file" "$feed_file" | tee "$feed_new_items"
+        cat "$feed_new_items" >> "$feed_file"
+        sort -o "$feed_file" "$feed_file"
+        rm "$feed_temp_file" "$feed_new_items"
+    fi | while read item; do
+        echo "open -t $item" > "$QUTE_FIFO"
+    done
+}
+
+if [ ! -d "$config_dir/read_urls" ]; then
+    notice "Creating configuration directory."
+    mkdir -p "$config_dir/read_urls"
+fi
+
+cd "$config_dir"
+
+if [ $# != 0 ]; then
+    for arg in "$@"; do
+        if [ "$arg" = "add" ]; then
+            add_feed "$QUTE_URL"
+        else
+            add_feed "$arg"
+        fi
+    done
+    exit
+fi
+
+if [ ! -f "feeds" ]; then
+    fail "Add feeds by running ':spawn -u rss add' or ':spawn -u rss <url>'."
+fi
+
+if curl --version >&-; then
+    curl="curl -sL"
+elif wget --version >&-; then
+    curl="wget -qO -"
+else
+    fail "Either curl or wget is needed to run this script."
+fi
+
+# Detect GNU grep so we can force it to treat everything as text
+if < /dev/null grep --help 2>&1 | grep -q -- -a; then
+    text_only="-a"
+fi
+
+while read feed_url; do
+    read_items "$feed_url" &
+done < "$config_dir/feeds"
+
+wait

pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-addopts = --strict -rfEw --faulthandler-timeout=70 --instafail
+addopts = --strict -rfEw --faulthandler-timeout=70 --instafail --pythonwarnings error
 markers =
     gui: Tests using the GUI (e.g. spawning widgets)
     posix: Tests which only can run on a POSIX OS.
@@ -14,7 +14,16 @@ markers =
     end2end: End to end tests which run qutebrowser as subprocess
     xfail_norun: xfail the test with out running it
     ci: Tests which should only run on CI.
-    flaky_once: Try to rerun this test once if it fails
+    qtwebengine_todo: Features still missing with QtWebEngine
+    qtwebengine_skip: Tests not applicable with QtWebEngine
+    qtwebkit_skip: Tests not applicable with QtWebKit
+    qtwebkit_ng_xfail: Tests failing with QtWebKit-NG
+    qtwebkit_ng_skip: Tests skipped with QtWebKit-NG
+    qtwebengine_flaky: Tests which are flaky (and currently skipped) with QtWebEngine
+    qtwebengine_osx_xfail: Tests which fail on OS X with QtWebEngine
+    js_prompt: Tests needing to display a javascript prompt
+    this: Used to mark tests during development
+    no_invalid_lines: Don't fail on unparseable lines in end2end tests
 qt_log_level_fail = WARNING
 qt_log_ignore =
     ^SpellCheck: .*
@@ -23,6 +32,9 @@ qt_log_ignore =
     ^QProcess: Destroyed while process .* is still running\.
     ^"Method "GetAll" with signature "s" on interface "org\.freedesktop\.DBus\.Properties" doesn't exist
     ^"Method \\"GetAll\\" with signature \\"s\\" on interface \\"org\.freedesktop\.DBus\.Properties\\" doesn't exist\\n"
+    ^propsReply "Method \\"GetAll\\" with signature \\"s\\" on interface \\"org\.freedesktop\.DBus\.Properties\\" doesn't exist\\n"
+    ^nmReply "Method \\"GetDevices\\" with signature \\"\\" on interface \\"org\.freedesktop\.NetworkManager\\" doesn't exist\\n"
+    ^"Object path cannot be empty"
     ^virtual void QSslSocketBackendPrivate::transmit\(\) SSL write failed with error: -9805
     ^virtual void QSslSocketBackendPrivate::transmit\(\) SSLRead failed with: -9805
     ^Type conversion already registered from type .*
@@ -30,11 +42,13 @@ qt_log_ignore =
     ^QWaitCondition: Destroyed while threads are still waiting
     ^QXcbXSettings::QXcbXSettings\(QXcbScreen\*\) Failed to get selection owner for XSETTINGS_S atom
     ^QStandardPaths: XDG_RUNTIME_DIR not set, defaulting to .*
-    ^QXcbClipboard: SelectionRequest too old
     ^QGeoclueMaster error creating GeoclueMasterClient\.
     ^Geoclue error: Process org\.freedesktop\.Geoclue\.Master exited with status 127
     ^QObject::connect: Cannot connect \(null\)::stateChanged\(QNetworkSession::State\) to QNetworkReplyHttpImpl::_q_networkSessionStateChanged\(QNetworkSession::State\)
     ^QXcbClipboard: Cannot transfer data, no data available
     ^load glyph failed
-qt_wait_signal_raising = true
+    ^Error when parsing the netrc file
+    ^Image of format '' blocked because it is not considered safe. If you are sure it is safe to do so, you can white-list the format by setting the environment variable QTWEBKIT_IMAGEFORMAT_WHITELIST=
+    ^QPainter::end: Painter ended with \d+ saved states
+    ^QSslSocket: cannot resolve SSLv[23]_(client|server)_method
 xfail_strict = true

qutebrowser.desktop

@@ -8,3 +8,4 @@ Exec=qutebrowser %u
 Terminal=false
 StartupNotify=false
 MimeType=text/html;text/xml;application/xhtml+xml;application/xml;application/rdf+xml;image/gif;image/jpeg;image/png;x-scheme-handler/http;x-scheme-handler/https;
+Keywords=Browser

qutebrowser/__init__.py

@@ -17,9 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-# pylint: disable=line-too-long
-
-"""A keyboard-driven, vim-like browser based on PyQt5 and QtWebKit."""
+"""A keyboard-driven, vim-like browser based on PyQt5."""
 
 import os.path
 
@@ -28,8 +26,8 @@ __copyright__ = "Copyright 2014-2016 Florian Bruhin (The Compiler)"
 __license__ = "GPL"
 __maintainer__ = __author__
 __email__ = "mail@qutebrowser.org"
-__version_info__ = (0, 8, 0)
+__version_info__ = (0, 10, 0)
 __version__ = '.'.join(str(e) for e in __version_info__)
-__description__ = "A keyboard-driven, vim-like browser based on PyQt5 and QtWebKit."
+__description__ = "A keyboard-driven, vim-like browser based on PyQt5."
 
 basedir = os.path.dirname(os.path.realpath(__file__))

qutebrowser/app.py

@@ -32,10 +32,9 @@ import datetime
 import tokenize
 
 from PyQt5.QtWidgets import QApplication, QWidget
-from PyQt5.QtWebKit import QWebSettings
-from PyQt5.QtGui import QDesktopServices, QPixmap, QIcon, QCursor, QWindow
+from PyQt5.QtGui import QDesktopServices, QPixmap, QIcon, QWindow
 from PyQt5.QtCore import (pyqtSlot, qInstallMessageHandler, QTimer, QUrl,
-                          QObject, Qt, QEvent, pyqtSignal)
+                          QObject, QEvent, pyqtSignal)
 try:
     import hunter
 except ImportError:
@@ -46,12 +45,15 @@ import qutebrowser.resources
 from qutebrowser.completion.models import instances as completionmodels
 from qutebrowser.commands import cmdutils, runners, cmdexc
 from qutebrowser.config import style, config, websettings, configexc
-from qutebrowser.browser import urlmarks, adblock
-from qutebrowser.browser.webkit import cookies, cache, history, downloads
-from qutebrowser.browser.webkit.network import (qutescheme, proxy,
-                                                networkmanager)
-from qutebrowser.mainwindow import mainwindow
-from qutebrowser.misc import readline, ipc, savemanager, sessions, crashsignal
+from qutebrowser.browser import (urlmarks, adblock, history, browsertab,
+                                 downloads)
+from qutebrowser.browser.network import proxy
+from qutebrowser.browser.webkit import cookies, cache
+from qutebrowser.browser.webkit.network import networkmanager
+from qutebrowser.keyinput import macros
+from qutebrowser.mainwindow import mainwindow, prompt
+from qutebrowser.misc import (readline, ipc, savemanager, sessions,
+                              crashsignal, earlyinit)
 from qutebrowser.misc import utilcmds  # pylint: disable=unused-import
 from qutebrowser.utils import (log, version, message, utils, qtutils, urlutils,
                                objreg, usertypes, standarddir, error, debug)
@@ -63,10 +65,6 @@ qApp = None
 
 def run(args):
     """Initialize everything and run the application."""
-    if args.version:
-        print(version.version())
-        sys.exit(usertypes.Exit.ok)
-
     if args.temp_basedir:
         args.basedir = tempfile.mkdtemp(prefix='qutebrowser-basedir-')
 
@@ -80,6 +78,13 @@ def run(args):
     qApp.setApplicationVersion(qutebrowser.__version__)
     qApp.lastWindowClosed.connect(quitter.on_last_window_closed)
 
+    log.init.debug("Initializing directories...")
+    standarddir.init(args)
+
+    if args.version:
+        print(version.version())
+        sys.exit(usertypes.Exit.ok)
+
     crash_handler = crashsignal.CrashHandler(
         app=qApp, quitter=quitter, args=args, parent=qApp)
     crash_handler.activate()
@@ -149,12 +154,13 @@ def init(args, crash_handler):
     config_obj = objreg.get('config')
     config_obj.style_changed.connect(style.get_stylesheet.cache_clear)
     qApp.focusChanged.connect(on_focus_changed)
-    qApp.focusChanged.connect(message.on_focus_changed)
 
     QDesktopServices.setUrlHandler('http', open_desktopservices_url)
     QDesktopServices.setUrlHandler('https', open_desktopservices_url)
     QDesktopServices.setUrlHandler('qute', open_desktopservices_url)
 
+    macros.init()
+
     log.init.debug("Init done!")
     crash_handler.raise_crashdlg()
 
@@ -181,8 +187,7 @@ def _process_args(args):
         try:
             config_obj.set('temp', sect, opt, val)
         except (configexc.Error, configparser.Error) as e:
-            message.error('current', "set: {} - {}".format(
-                e.__class__.__name__, e))
+            message.error("set: {} - {}".format(e.__class__.__name__, e))
 
     if not args.override_restore:
         _load_session(args.session)
@@ -198,6 +203,9 @@ def _process_args(args):
     _open_startpage()
     _open_quickstart(args)
 
+    delta = datetime.datetime.now() - earlyinit.START_TIME
+    log.init.debug("Init finished after {}s".format(delta.total_seconds()))
+
 
 def _load_session(name):
     """Load the default session.
@@ -206,21 +214,23 @@ def _load_session(name):
         name: The name of the session to load, or None to read state file.
     """
     state_config = objreg.get('state-config')
-    if name is None:
+    session_manager = objreg.get('session-manager')
+    if name is None and session_manager.exists('_autosave'):
+        name = '_autosave'
+    elif name is None:
         try:
             name = state_config['general']['session']
         except KeyError:
             # No session given as argument and none in the session file ->
             # start without loading a session
             return
-    session_manager = objreg.get('session-manager')
+
     try:
         session_manager.load(name)
     except sessions.SessionNotFoundError:
-        message.error('current', "Session {} not found!".format(name))
+        message.error("Session {} not found!".format(name))
     except sessions.SessionError as e:
-        message.error('current', "Failed to load session {}: {}".format(
-            name, e))
+        message.error("Failed to load session {}: {}".format(name, e))
     try:
         del state_config['general']['session']
     except KeyError:
@@ -272,8 +282,8 @@ def process_pos_args(args, via_ipc=False, cwd=None, target_arg=None):
             try:
                 url = urlutils.fuzzy_url(cmd, cwd, relative=True)
             except urlutils.InvalidUrlError as e:
-                message.error('current', "Error in startup argument '{}': "
-                              "{}".format(cmd, e))
+                message.error("Error in startup argument '{}': {}".format(
+                    cmd, e))
             else:
                 background = open_target in ['tab-bg', 'tab-bg-silent']
                 tabbed_browser.tabopen(url, background=background,
@@ -302,8 +312,7 @@ def _open_startpage(win_id=None):
                 try:
                     url = urlutils.fuzzy_url(urlstr, do_search=False)
                 except urlutils.InvalidUrlError as e:
-                    message.error('current', "Error when opening startpage: "
-                                  "{}".format(e))
+                    message.error("Error when opening startpage: {}".format(e))
                     tabbed_browser.tabopen(QUrl('about:blank'))
                 else:
                     tabbed_browser.tabopen(url)
@@ -315,8 +324,8 @@ def _open_quickstart(args):
     Args:
         args: The argparse namespace.
     """
-    if args.datadir is not None or args.basedir is not None:
-        # With --datadir or --basedir given, don't open quickstart.
+    if args.basedir is not None:
+        # With --basedir given, don't open quickstart.
         return
     state_config = objreg.get('state-config')
     try:
@@ -327,7 +336,7 @@ def _open_quickstart(args):
         tabbed_browser = objreg.get('tabbed-browser', scope='window',
                                     window='last-focused')
         tabbed_browser.tabopen(
-            QUrl('http://www.qutebrowser.org/quickstart.html'))
+            QUrl('https://www.qutebrowser.org/quickstart.html'))
         state_config['general']['quickstart-done'] = '1'
 
 
@@ -339,19 +348,20 @@ def _save_version():
 
 def on_focus_changed(_old, new):
     """Register currently focused main window in the object registry."""
+    if new is None:
+        return
+
     if not isinstance(new, QWidget):
         log.misc.debug("on_focus_changed called with non-QWidget {!r}".format(
             new))
+        return
 
-    if new is None or not isinstance(new, mainwindow.MainWindow):
-        try:
-            objreg.delete('last-focused-main-window')
-        except KeyError:
-            pass
-        qApp.restoreOverrideCursor()
-    else:
-        objreg.register('last-focused-main-window', new.window(), update=True)
-        _maybe_hide_mouse_cursor()
+    window = new.window()
+    if isinstance(window, mainwindow.MainWindow):
+        objreg.register('last-focused-main-window', window, update=True)
+        # A focused window must also be visible, and in this case we should
+        # consider it as the most recently looked-at window
+        objreg.register('last-visible-main-window', window, update=True)
 
 
 def open_desktopservices_url(url):
@@ -362,17 +372,6 @@ def open_desktopservices_url(url):
     tabbed_browser.tabopen(url)
 
 
-@config.change_filter('ui', 'hide-mouse-cursor', function=True)
-def _maybe_hide_mouse_cursor():
-    """Hide the mouse cursor if it isn't yet and it's configured."""
-    if config.get('ui', 'hide-mouse-cursor'):
-        if qApp.overrideCursor() is not None:
-            return
-        qApp.setOverrideCursor(QCursor(Qt.BlankCursor))
-    else:
-        qApp.restoreOverrideCursor()
-
-
 def _init_modules(args, crash_handler):
     """Initialize all 'modules' which need to be initialized.
 
@@ -381,63 +380,76 @@ def _init_modules(args, crash_handler):
         crash_handler: The CrashHandler instance.
     """
     # pylint: disable=too-many-statements
+    log.init.debug("Initializing prompts...")
+    prompt.init()
+
     log.init.debug("Initializing save manager...")
     save_manager = savemanager.SaveManager(qApp)
     objreg.register('save-manager', save_manager)
     save_manager.add_saveable('version', _save_version)
+
     log.init.debug("Initializing network...")
     networkmanager.init()
+
+    if qtutils.version_check('5.8'):
+        # Otherwise we can only initialize it for QtWebKit because of crashes
+        log.init.debug("Initializing proxy...")
+        proxy.init()
+
     log.init.debug("Initializing readline-bridge...")
     readline_bridge = readline.ReadlineBridge()
     objreg.register('readline-bridge', readline_bridge)
-    log.init.debug("Initializing directories...")
-    standarddir.init(args)
+
     log.init.debug("Initializing config...")
     config.init(qApp)
     save_manager.init_autosave()
+
     log.init.debug("Initializing web history...")
     history.init(qApp)
+
     log.init.debug("Initializing crashlog...")
     if not args.no_err_windows:
         crash_handler.handle_segfault()
+
     log.init.debug("Initializing sessions...")
     sessions.init(qApp)
-    log.init.debug("Initializing js-bridge...")
-    js_bridge = qutescheme.JSBridge(qApp)
-    objreg.register('js-bridge', js_bridge)
+
     log.init.debug("Initializing websettings...")
-    websettings.init()
+    websettings.init(args)
+
     log.init.debug("Initializing adblock...")
     host_blocker = adblock.HostBlocker()
     host_blocker.read_hosts()
     objreg.register('host-blocker', host_blocker)
+
     log.init.debug("Initializing quickmarks...")
     quickmark_manager = urlmarks.QuickmarkManager(qApp)
     objreg.register('quickmark-manager', quickmark_manager)
+
     log.init.debug("Initializing bookmarks...")
     bookmark_manager = urlmarks.BookmarkManager(qApp)
     objreg.register('bookmark-manager', bookmark_manager)
-    log.init.debug("Initializing proxy...")
-    proxy.init()
+
     log.init.debug("Initializing cookies...")
     cookie_jar = cookies.CookieJar(qApp)
     ram_cookie_jar = cookies.RAMCookieJar(qApp)
     objreg.register('cookie-jar', cookie_jar)
     objreg.register('ram-cookie-jar', ram_cookie_jar)
+
     log.init.debug("Initializing cache...")
     diskcache = cache.DiskCache(standarddir.cache(), parent=qApp)
     objreg.register('cache', diskcache)
+
     log.init.debug("Initializing completions...")
     completionmodels.init()
+
     log.init.debug("Misc initialization...")
     if config.get('ui', 'hide-wayland-decoration'):
         os.environ['QT_WAYLAND_DISABLE_WINDOWDECORATION'] = '1'
     else:
         os.environ.pop('QT_WAYLAND_DISABLE_WINDOWDECORATION', None)
-    _maybe_hide_mouse_cursor()
-    objreg.get('config').changed.connect(_maybe_hide_mouse_cursor)
-    temp_downloads = downloads.TempDownloadManager(qApp)
-    objreg.register('temporary-downloads', temp_downloads)
+    # Init backend-specific stuff
+    browsertab.init()
 
 
 def _init_late_modules(args):
@@ -525,7 +537,7 @@ class Quitter:
             if not os.path.isdir(cwd):
                 # Probably running from a python egg. Let's fallback to
                 # cwd=None and see if that works out.
-                # See https://github.com/The-Compiler/qutebrowser/issues/323
+                # See https://github.com/qutebrowser/qutebrowser/issues/323
                 cwd = None
 
         # Add all open pages so they get reopened.
@@ -551,8 +563,9 @@ class Quitter:
             argdict['session'] = session
             argdict['override_restore'] = False
         # Ensure :restart works with --temp-basedir
-        argdict['temp_basedir'] = False
-        argdict['temp_basedir_restarted'] = True
+        if self._args.temp_basedir:
+            argdict['temp_basedir'] = False
+            argdict['temp_basedir_restarted'] = True
 
         # Dump the data
         data = json.dumps(argdict)
@@ -644,13 +657,7 @@ class Quitter:
             session_manager.save(sessions.default, last_window=last_window,
                                  load_next_time=True)
 
-        deferrer = False
-        for win_id in objreg.window_registry:
-            prompter = objreg.get('prompter', None, scope='window',
-                                  window=win_id)
-            if prompter is not None and prompter.shutdown():
-                deferrer = True
-        if deferrer:
+        if prompt.prompt_queue.shutdown():
             # If shutdown was called while we were asking a question, we're in
             # a still sub-eventloop (which gets quit now) and not in the main
             # one.
@@ -699,9 +706,7 @@ class Quitter:
                         e, self._args, "Error while saving!",
                         pre_text="Error while saving {}".format(key))
         # Disable storage so removing tempdir will work
-        QWebSettings.setIconDatabasePath('')
-        QWebSettings.setOfflineWebApplicationCachePath('')
-        QWebSettings.globalSettings().setLocalStoragePath('')
+        websettings.shutdown()
         # Re-enable faulthandler to stdout, then remove crash log
         log.destroy.debug("Deactivating crash log...")
         objreg.get('crash-handler').destroy_crashlogfile()
@@ -711,13 +716,14 @@ class Quitter:
             atexit.register(shutil.rmtree, self._args.basedir,
                             ignore_errors=True)
         # Delete temp download dir
-        objreg.get('temporary-downloads').cleanup()
+        downloads.temp_download_manager.cleanup()
         # If we don't kill our custom handler here we might get segfaults
         log.destroy.debug("Deactivating message handler...")
         qInstallMessageHandler(None)
         # Now we can hopefully quit without segfaults
         log.destroy.debug("Deferring QApplication::exit...")
         objreg.get('signal-handler').deactivate()
+        objreg.get('session-manager').delete_autosave()
         # We use a singleshot timer to exit here to minimize the likelihood of
         # segfaults.
         QTimer.singleShot(0, functools.partial(qApp.exit, status))
@@ -739,6 +745,7 @@ class Application(QApplication):
 
     Attributes:
         _args: ArgumentParser instance.
+        _last_focus_object: The last focused object's repr.
     """
 
     new_window = pyqtSignal(mainwindow.MainWindow)
@@ -749,6 +756,8 @@ class Application(QApplication):
         Args:
             Argument namespace from argparse.
         """
+        self._last_focus_object = None
+
         qt_args = qtutils.get_args(args)
         log.init.debug("Qt arguments: {}, based on {}".format(qt_args, args))
         super().__init__(qt_args)
@@ -765,7 +774,10 @@ class Application(QApplication):
     @pyqtSlot(QObject)
     def on_focus_object_changed(self, obj):
         """Log when the focus object changed."""
-        log.misc.debug("Focus object changed: {!r}".format(obj))
+        output = repr(obj)
+        if self._last_focus_object != output:
+            log.misc.debug("Focus object changed: {}".format(output))
+        self._last_focus_object = output
 
     def __repr__(self):
         return utils.get_repr(self)

qutebrowser/browser/adblock.py

@@ -26,9 +26,10 @@ import posixpath
 import zipfile
 import fnmatch
 
+from qutebrowser.browser import downloads
 from qutebrowser.config import config
-from qutebrowser.utils import objreg, standarddir, log, message, usertypes
-from qutebrowser.commands import cmdutils, cmdexc
+from qutebrowser.utils import objreg, standarddir, log, message
+from qutebrowser.commands import cmdutils
 
 
 def guess_zip_filename(zf):
@@ -112,17 +113,11 @@ class HostBlocker:
         self._done_count = 0
 
         data_dir = standarddir.data()
-        if data_dir is None:
-            self._local_hosts_file = None
-        else:
-            self._local_hosts_file = os.path.join(data_dir, 'blocked-hosts')
+        self._local_hosts_file = os.path.join(data_dir, 'blocked-hosts')
         self.on_config_changed()
 
         config_dir = standarddir.config()
-        if config_dir is None:
-            self._config_hosts_file = None
-        else:
-            self._config_hosts_file = os.path.join(config_dir, 'blocked-hosts')
+        self._config_hosts_file = os.path.join(config_dir, 'blocked-hosts')
 
         objreg.get('config').changed.connect(self.on_config_changed)
 
@@ -145,7 +140,7 @@ class HostBlocker:
         Return:
             True if a read was attempted, False otherwise
         """
-        if filename is None or not os.path.exists(filename):
+        if not os.path.exists(filename):
             return False
 
         try:
@@ -161,9 +156,6 @@ class HostBlocker:
         """Read hosts from the existing blocked-hosts file."""
         self._blocked_hosts = set()
 
-        if self._local_hosts_file is None:
-            return
-
         self._read_hosts_file(self._config_hosts_file,
                               self._config_blocked_hosts)
 
@@ -174,26 +166,22 @@ class HostBlocker:
             args = objreg.get('args')
             if (config.get('content', 'host-block-lists') is not None and
                     args.basedir is None):
-                message.info('current',
-                             "Run :adblock-update to get adblock lists.")
+                message.info("Run :adblock-update to get adblock lists.")
 
     @cmdutils.register(instance='host-blocker')
-    @cmdutils.argument('win_id', win_id=True)
-    def adblock_update(self, win_id):
+    def adblock_update(self):
         """Update the adblock block lists.
 
-        This updates ~/.local/share/qutebrowser/blocked-hosts with downloaded
-        host lists and re-reads ~/.config/qutebrowser/blocked-hosts.
+        This updates `~/.local/share/qutebrowser/blocked-hosts` with downloaded
+        host lists and re-reads `~/.config/qutebrowser/blocked-hosts`.
         """
         self._read_hosts_file(self._config_hosts_file,
                               self._config_blocked_hosts)
-        if self._local_hosts_file is None:
-            raise cmdexc.CommandError("No data storage is configured!")
         self._blocked_hosts = set()
         self._done_count = 0
         urls = config.get('content', 'host-block-lists')
-        download_manager = objreg.get('download-manager', scope='window',
-                                      window='last-focused')
+        download_manager = objreg.get('qtnetwork-download-manager',
+                                      scope='window', window='last-focused')
         if urls is None:
             return
         for url in urls:
@@ -201,8 +189,8 @@ class HostBlocker:
                 try:
                     fileobj = open(url.path(), 'rb')
                 except OSError as e:
-                    message.error(win_id, "adblock: Error while reading {}: "
-                                  "{}".format(url.path(), e.strerror))
+                    message.error("adblock: Error while reading {}: {}".format(
+                        url.path(), e.strerror))
                     continue
                 download = FakeDownload(fileobj)
                 self._in_progress.append(download)
@@ -210,7 +198,7 @@ class HostBlocker:
             else:
                 fobj = io.BytesIO()
                 fobj.name = 'adblock: ' + url.host()
-                target = usertypes.FileObjDownloadTarget(fobj)
+                target = downloads.FileObjDownloadTarget(fobj)
                 download = download_manager.get(url, target=target,
                                                 auto_remove=True)
                 self._in_progress.append(download)
@@ -231,9 +219,9 @@ class HostBlocker:
         try:
             f = get_fileobj(byte_io)
         except (OSError, UnicodeDecodeError, zipfile.BadZipFile,
-                zipfile.LargeZipFile) as e:
-            message.error('current', "adblock: Error while reading {}: {} - "
-                          "{}".format(byte_io.name, e.__class__.__name__, e))
+                zipfile.LargeZipFile, LookupError) as e:
+            message.error("adblock: Error while reading {}: {} - {}".format(
+                byte_io.name, e.__class__.__name__, e))
             return
         for line in f:
             line_count += 1
@@ -261,7 +249,7 @@ class HostBlocker:
                 self._blocked_hosts.add(host)
         log.misc.debug("{}: read {} lines".format(byte_io.name, line_count))
         if error_count > 0:
-            message.error('current', "adblock: {} read errors for {}".format(
+            message.error("adblock: {} read errors for {}".format(
                 error_count, byte_io.name))
 
     def on_lists_downloaded(self):
@@ -269,14 +257,14 @@ class HostBlocker:
         with open(self._local_hosts_file, 'w', encoding='utf-8') as f:
             for host in sorted(self._blocked_hosts):
                 f.write(host + '\n')
-            message.info('current', "adblock: Read {} hosts from {} sources."
-                         .format(len(self._blocked_hosts), self._done_count))
+            message.info("adblock: Read {} hosts from {} sources.".format(
+                len(self._blocked_hosts), self._done_count))
 
     @config.change_filter('content', 'host-block-lists')
     def on_config_changed(self):
         """Update files when the config changed."""
         urls = config.get('content', 'host-block-lists')
-        if urls is None and self._local_hosts_file is not None:
+        if urls is None:
             try:
                 os.remove(self._local_hosts_file)
             except FileNotFoundError:

qutebrowser/browser/browsertab.py

@@ -21,13 +21,15 @@
 
 import itertools
 
-from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, QObject, QPoint
+from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, QObject, QSizeF
 from PyQt5.QtGui import QIcon
-from PyQt5.QtWidgets import QWidget, QLayout
+from PyQt5.QtWidgets import QWidget, QApplication
 
 from qutebrowser.keyinput import modeman
 from qutebrowser.config import config
-from qutebrowser.utils import utils, objreg, usertypes, message, log, qtutils
+from qutebrowser.utils import utils, objreg, usertypes, log, qtutils
+from qutebrowser.misc import miscwidgets, objects
+from qutebrowser.browser import mouse, hints
 
 
 tab_id_gen = itertools.count(0)
@@ -43,7 +45,7 @@ def create(win_id, parent=None):
     # Importing modules here so we don't depend on QtWebEngine without the
     # argument and to avoid circular imports.
     mode_manager = modeman.instance(win_id)
-    if objreg.get('args').backend == 'webengine':
+    if objects.backend == usertypes.Backend.QtWebEngine:
         from qutebrowser.browser.webengine import webenginetab
         tab_class = webenginetab.WebEngineTab
     else:
@@ -52,38 +54,33 @@ def create(win_id, parent=None):
     return tab_class(win_id=win_id, mode_manager=mode_manager, parent=parent)
 
 
+def init():
+    """Initialize backend-specific modules."""
+    if objects.backend == usertypes.Backend.QtWebEngine:
+        from qutebrowser.browser.webengine import webenginetab
+        webenginetab.init()
+    else:
+        from qutebrowser.browser.webkit import webkittab
+        webkittab.init()
+
+
 class WebTabError(Exception):
 
     """Base class for various errors."""
 
 
-class WrapperLayout(QLayout):
+class UnsupportedOperationError(WebTabError):
 
-    """A Qt layout which simply wraps a single widget.
+    """Raised when an operation is not supported with the given backend."""
 
-    This is used so the widget is hidden behind a AbstractTab API and can't
-    easily be accidentally accessed.
-    """
 
-    def __init__(self, widget, parent=None):
-        super().__init__(parent)
-        self._widget = widget
-
-    def addItem(self, _widget):
-        raise AssertionError("Should never be called!")
-
-    def sizeHint(self):
-        return self._widget.sizeHint()
-
-    def itemAt(self, _index):  # pragma: no cover
-        # For some reason this sometimes gets called by Qt.
-        return None
-
-    def takeAt(self, _index):
-        raise AssertionError("Should never be called!")
-
-    def setGeometry(self, rect):
-        self._widget.setGeometry(rect)
+TerminationStatus = usertypes.enum('TerminationStatus', [
+    'normal',
+    'abnormal',  # non-zero exit status
+    'crashed',   # e.g. segfault
+    'killed',
+    'unknown',
+])
 
 
 class TabData:
@@ -95,14 +92,31 @@ class TabData:
                    load.
         inspector: The QWebInspector used for this webview.
         viewing_source: Set if we're currently showing a source view.
+        override_target: Override for open_target for fake clicks (like hints).
+                         Only used for QtWebKit.
     """
 
-    __slots__ = ['keep_icon', 'viewing_source', 'inspector']
-
     def __init__(self):
         self.keep_icon = False
         self.viewing_source = False
         self.inspector = None
+        self.override_target = None
+
+
+class AbstractAction:
+
+    """Attribute of AbstractTab for Qt WebActions."""
+
+    def __init__(self):
+        self._widget = None
+
+    def exit_fullscreen(self):
+        """Exit the fullscreen mode."""
+        raise NotImplementedError
+
+    def save_page(self):
+        """Save the current page."""
+        raise NotImplementedError
 
 
 class AbstractPrinting:
@@ -118,10 +132,20 @@ class AbstractPrinting:
     def check_printer_support(self):
         raise NotImplementedError
 
+    def check_preview_support(self):
+        raise NotImplementedError
+
     def to_pdf(self, filename):
         raise NotImplementedError
 
-    def to_printer(self, printer):
+    def to_printer(self, printer, callback=None):
+        """Print the tab.
+
+        Args:
+            printer: The QPrinter to print to.
+            callback: Called with a boolean
+                      (True if printing succeeded, False otherwise)
+        """
         raise NotImplementedError
 
 
@@ -193,7 +217,7 @@ class AbstractZoom(QObject):
         # # FIXME:qtwebengine is this needed?
         # # For some reason, this signal doesn't get disconnected automatically
         # # when the WebView is destroyed on older PyQt versions.
-        # # See https://github.com/The-Compiler/qutebrowser/issues/390
+        # # See https://github.com/qutebrowser/qutebrowser/issues/390
         # self.destroyed.connect(functools.partial(
         #     cfg.changed.disconnect, self.init_neighborlist))
 
@@ -226,6 +250,9 @@ class AbstractZoom(QObject):
         self.set_factor(float(level) / 100, fuzzyval=False)
         return level
 
+    def _set_factor_internal(self, factor):
+        raise NotImplementedError
+
     def set_factor(self, factor, *, fuzzyval=True):
         """Zoom to a given zoom factor.
 
@@ -247,19 +274,6 @@ class AbstractZoom(QObject):
         default_zoom = config.get('ui', 'default-zoom')
         self._set_factor_internal(float(default_zoom) / 100)
 
-    @pyqtSlot(QPoint)
-    def _on_mouse_wheel_zoom(self, delta):
-        """Handle zooming via mousewheel requested by the web view."""
-        divider = config.get('input', 'mouse-zoom-divider')
-        factor = self.factor() + delta.y() / divider
-        if factor < 0:
-            return
-        perc = int(100 * factor)
-        message.info(self._win_id, "Zoom level: {}%".format(perc))
-        self._neighborlist.fuzzyval = perc
-        self._set_factor_internal(factor)
-        self._default_zoom_changed = True
-
 
 class AbstractCaret(QObject):
 
@@ -351,6 +365,12 @@ class AbstractScroller(QObject):
         super().__init__(parent)
         self._tab = tab
         self._widget = None
+        self.perc_changed.connect(self._log_scroll_pos_change)
+
+    @pyqtSlot()
+    def _log_scroll_pos_change(self):
+        log.webview.vdebug("Scroll position changed to {}".format(
+            self.pos_px()))
 
     def _init_widget(self, widget):
         self._widget = widget
@@ -446,6 +466,55 @@ class AbstractHistory:
         raise NotImplementedError
 
 
+class AbstractElements:
+
+    """Finding and handling of elements on the page."""
+
+    def __init__(self, tab):
+        self._widget = None
+        self._tab = tab
+
+    def find_css(self, selector, callback, *, only_visible=False):
+        """Find all HTML elements matching a given selector async.
+
+        Args:
+            callback: The callback to be called when the search finished.
+            selector: The CSS selector to search for.
+            only_visible: Only show elements which are visible on screen.
+        """
+        raise NotImplementedError
+
+    def find_id(self, elem_id, callback):
+        """Find the HTML element with the given ID async.
+
+        Args:
+            callback: The callback to be called when the search finished.
+            elem_id: The ID to search for.
+        """
+        raise NotImplementedError
+
+    def find_focused(self, callback):
+        """Find the focused element on the page async.
+
+        Args:
+            callback: The callback to be called when the search finished.
+                      Called with a WebEngineElement or None.
+        """
+        raise NotImplementedError
+
+    def find_at_pos(self, pos, callback):
+        """Find the element at the given position async.
+
+        This is also called "hit test" elsewhere.
+
+        Args:
+            pos: The QPoint to get the element for.
+            callback: The callback to be called when the search finished.
+                      Called with a WebEngineElement or None.
+        """
+        raise NotImplementedError
+
+
 class AbstractTab(QWidget):
 
     """A wrapper over the given widget to hide its API and expose another one.
@@ -469,6 +538,13 @@ class AbstractTab(QWidget):
         new_tab_requested: Emitted when a new tab should be opened with the
                            given URL.
         load_status_changed: The loading status changed
+        fullscreen_requested: Fullscreen display was requested by the page.
+                              arg: True if fullscreen should be turned on,
+                                   False if it should be turned off.
+        renderer_process_terminated: Emitted when the underlying renderer
+                                     process terminated.
+                                     arg 0: A TerminationStatus member.
+                                     arg 1: The exit code.
     """
 
     window_close_requested = pyqtSignal()
@@ -482,8 +558,12 @@ class AbstractTab(QWidget):
     new_tab_requested = pyqtSignal(QUrl)
     url_changed = pyqtSignal(QUrl)
     shutting_down = pyqtSignal()
+    contents_size_changed = pyqtSignal(QSizeF)
+    add_history_item = pyqtSignal(QUrl, QUrl, str)  # url, requested url, title
+    fullscreen_requested = pyqtSignal(bool)
+    renderer_process_terminated = pyqtSignal(TerminationStatus, int)
 
-    def __init__(self, win_id, parent=None):
+    def __init__(self, win_id, mode_manager, parent=None):
         self.win_id = win_id
         self.tab_id = next(tab_id_gen)
         super().__init__(parent)
@@ -496,32 +576,49 @@ class AbstractTab(QWidget):
 
         # self.history = AbstractHistory(self)
         # self.scroller = AbstractScroller(self, parent=self)
-        # self.caret = AbstractCaret(win_id=win_id, tab=self, mode_manager=...,
-        #                            parent=self)
+        # self.caret = AbstractCaret(win_id=win_id, tab=self,
+        #                            mode_manager=mode_manager, parent=self)
         # self.zoom = AbstractZoom(win_id=win_id)
         # self.search = AbstractSearch(parent=self)
         # self.printing = AbstractPrinting()
+        # self.elements = AbstractElements(self)
+        # self.action = AbstractAction()
+
         self.data = TabData()
-        self._layout = None
+        self._layout = miscwidgets.WrapperLayout(self)
         self._widget = None
         self._progress = 0
         self._has_ssl_errors = False
+        self._mode_manager = mode_manager
         self._load_status = usertypes.LoadStatus.none
+        self._mouse_event_filter = mouse.MouseEventFilter(
+            self, parent=self)
         self.backend = None
 
+        # FIXME:qtwebengine  Should this be public api via self.hints?
+        #                    Also, should we get it out of objreg?
+        hintmanager = hints.HintManager(win_id, self.tab_id, parent=self)
+        objreg.register('hintmanager', hintmanager, scope='tab',
+                        window=self.win_id, tab=self.tab_id)
+
     def _set_widget(self, widget):
         # pylint: disable=protected-access
-        self._layout = WrapperLayout(widget, self)
         self._widget = widget
+        self._layout.wrap(self, widget)
         self.history._history = widget.history()
         self.scroller._init_widget(widget)
         self.caret._widget = widget
         self.zoom._widget = widget
         self.search._widget = widget
         self.printing._widget = widget
-        widget.mouse_wheel_zoom.connect(self.zoom._on_mouse_wheel_zoom)
-        widget.setParent(self)
-        self.setFocusProxy(widget)
+        self.action._widget = widget
+        self.elements._widget = widget
+
+        self._install_event_filter()
+        self.zoom.set_default()
+
+    def _install_event_filter(self):
+        raise NotImplementedError
 
     def _set_load_status(self, val):
         """Setter for load_status."""
@@ -531,6 +628,25 @@ class AbstractTab(QWidget):
         self._load_status = val
         self.load_status_changed.emit(val.name)
 
+    def event_target(self):
+        """Return the widget events should be sent to."""
+        raise NotImplementedError
+
+    def send_event(self, evt):
+        """Send the given event to the underlying widget.
+
+        The event will be sent via QApplication.postEvent.
+        Note that a posted event may not be re-used in any way!
+        """
+        # This only gives us some mild protection against re-using events, but
+        # it's certainly better than a segfault.
+        if getattr(evt, 'posted', False):
+            raise AssertionError("Can't re-use an event which was already "
+                                 "posted!")
+        recipient = self.event_target()
+        evt.posted = True
+        QApplication.postEvent(recipient, evt)
+
     @pyqtSlot(QUrl)
     def _on_url_changed(self, url):
         """Update title when URL has changed and no title is available."""
@@ -546,14 +662,36 @@ class AbstractTab(QWidget):
         self._set_load_status(usertypes.LoadStatus.loading)
         self.load_started.emit()
 
+    def _handle_auto_insert_mode(self, ok):
+        """Handle auto-insert-mode after loading finished."""
+        if not config.get('input', 'auto-insert-mode') or not ok:
+            return
+
+        cur_mode = self._mode_manager.mode
+        if cur_mode == usertypes.KeyMode.insert:
+            return
+
+        def _auto_insert_mode_cb(elem):
+            """Called from JS after finding the focused element."""
+            if elem is None:
+                log.webview.debug("No focused element!")
+                return
+            if elem.is_editable():
+                modeman.enter(self.win_id, usertypes.KeyMode.insert,
+                              'load finished', only_if_normal=True)
+
+        self.elements.find_focused(_auto_insert_mode_cb)
+
     @pyqtSlot(bool)
     def _on_load_finished(self, ok):
+        sess_manager = objreg.get('session-manager')
+        sess_manager.save_autosave()
+
         if ok and not self._has_ssl_errors:
             if self.url().scheme() == 'https':
                 self._set_load_status(usertypes.LoadStatus.success_https)
             else:
                 self._set_load_status(usertypes.LoadStatus.success)
-
         elif ok:
             self._set_load_status(usertypes.LoadStatus.warn)
         else:
@@ -561,6 +699,12 @@ class AbstractTab(QWidget):
         self.load_finished.emit(ok)
         if not self.title():
             self.title_changed.emit(self.url().toDisplayString())
+        self._handle_auto_insert_mode(ok)
+
+    @pyqtSlot()
+    def _on_history_trigger(self):
+        """Emit add_history_item when triggered by backend-specific signal."""
+        raise NotImplementedError
 
     @pyqtSlot(int)
     def _on_load_progress(self, perc):
@@ -571,7 +715,7 @@ class AbstractTab(QWidget):
     def _on_ssl_errors(self):
         self._has_ssl_errors = True
 
-    def url(self):
+    def url(self, requested=False):
         raise NotImplementedError
 
     def progress(self):
@@ -604,19 +748,17 @@ class AbstractTab(QWidget):
         """
         raise NotImplementedError
 
-    def run_js_async(self, code, callback=None):
+    def run_js_async(self, code, callback=None, *, world=None):
         """Run javascript async.
 
         The given callback will be called with the result when running JS is
         complete.
-        """
-        raise NotImplementedError
 
-    def run_js_blocking(self, code):
-        """Run javascript and block.
-
-        This returns the result to the caller. Its use should be avoided when
-        possible as it runs a local event loop for QtWebEngine.
+        Args:
+            code: The javascript code to run.
+            callback: The callback to call with the result, or None.
+            world: A world ID (int or usertypes.JsWorld member) to run the JS
+                   in the main world or in another isolated world.
         """
         raise NotImplementedError
 
@@ -632,6 +774,22 @@ class AbstractTab(QWidget):
     def set_html(self, html, base_url):
         raise NotImplementedError
 
+    def networkaccessmanager(self):
+        """Get the QNetworkAccessManager for this tab.
+
+        This is only implemented for QtWebKit.
+        For QtWebEngine, always returns None.
+        """
+        raise NotImplementedError
+
+    def user_agent(self):
+        """Get the user agent for this tab.
+
+        This is only implemented for QtWebKit.
+        For QtWebEngine, always returns None.
+        """
+        raise NotImplementedError
+
     def __repr__(self):
         try:
             url = utils.elide(self.url().toDisplayString(QUrl.EncodeUnicode),

qutebrowser/browser/downloadview.py

@@ -25,7 +25,7 @@ import sip
 from PyQt5.QtCore import pyqtSlot, QSize, Qt, QTimer
 from PyQt5.QtWidgets import QListView, QSizePolicy, QMenu
 
-from qutebrowser.browser.webkit import downloads
+from qutebrowser.browser import downloads
 from qutebrowser.config import style
 from qutebrowser.utils import qtutils, utils, objreg
 
@@ -39,8 +39,8 @@ def update_geometry(obj):
 
     Here we check if obj ("self") was deleted and just ignore the event if so.
 
-    Original bug:   https://github.com/The-Compiler/qutebrowser/issues/167
-    Workaround bug: https://github.com/The-Compiler/qutebrowser/issues/171
+    Original bug:   https://github.com/qutebrowser/qutebrowser/issues/167
+    Workaround bug: https://github.com/qutebrowser/qutebrowser/issues/171
     """
     def _update_geometry():
         """Actually update the geometry if the object still exists."""
@@ -83,7 +83,7 @@ class DownloadView(QListView):
         self.setFlow(QListView.LeftToRight)
         self.setSpacing(1)
         self._menu = None
-        model = objreg.get('download-manager', scope='window', window=win_id)
+        model = objreg.get('download-model', scope='window', window=win_id)
         model.rowsInserted.connect(functools.partial(update_geometry, self))
         model.rowsRemoved.connect(functools.partial(update_geometry, self))
         model.dataChanged.connect(functools.partial(update_geometry, self))
@@ -113,7 +113,7 @@ class DownloadView(QListView):
         item = self.model().data(index, downloads.ModelRole.item)
         if item.done and item.successful:
             item.open_file()
-            self.model().remove_item(item)
+            item.remove()
 
     def _get_menu_actions(self, item):
         """Get the available context menu actions for a given DownloadItem.
@@ -135,8 +135,7 @@ class DownloadView(QListView):
                 actions.append(("Open", item.open_file))
             else:
                 actions.append(("Retry", item.retry))
-            actions.append(("Remove",
-                            functools.partial(model.remove_item, item)))
+            actions.append(("Remove", item.remove))
         else:
             actions.append(("Cancel", item.cancel))
         if model.can_clear():

qutebrowser/browser/history.py

@@ -22,13 +22,13 @@
 import time
 import collections
 
-from PyQt5.QtCore import pyqtSignal, QUrl, QObject
-from PyQt5.QtWebKit import QWebHistoryInterface
+from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, QObject
 
 from qutebrowser.commands import cmdutils
-from qutebrowser.utils import utils, objreg, standarddir, log, qtutils
+from qutebrowser.utils import (utils, objreg, standarddir, log, qtutils,
+                               usertypes, message)
 from qutebrowser.config import config
-from qutebrowser.misc import lineparser
+from qutebrowser.misc import lineparser, objects
 
 
 class Entry:
@@ -88,12 +88,8 @@ class Entry:
         if not url.isValid():
             raise ValueError("Invalid URL: {}".format(url.errorString()))
 
-        if atime.startswith('\0'):
-            log.init.debug(
-                "Removing NUL bytes from entry {!r} - see "
-                "https://github.com/The-Compiler/qutebrowser/issues/"
-                "670".format(data))
-            atime = atime.lstrip('\0')
+        # https://github.com/qutebrowser/qutebrowser/issues/670
+        atime = atime.lstrip('\0')
 
         if '-' in atime:
             atime, flags = atime.split('-')
@@ -108,34 +104,6 @@ class Entry:
         return cls(atime, url, title, redirect=redirect)
 
 
-class WebHistoryInterface(QWebHistoryInterface):
-
-    """Glue code between WebHistory and Qt's QWebHistoryInterface.
-
-    Attributes:
-        _history: The WebHistory object.
-    """
-
-    def __init__(self, webhistory, parent=None):
-        super().__init__(parent)
-        self._history = webhistory
-
-    def addHistoryEntry(self, url_string):
-        """Required for a QWebHistoryInterface impl, obsoleted by add_url."""
-        pass
-
-    def historyContains(self, url_string):
-        """Called by WebKit to determine if a URL is contained in the history.
-
-        Args:
-            url_string: The URL (as string) to check for.
-
-        Return:
-            True if the url is in the history, False otherwise.
-        """
-        return url_string in self._history.history_dict
-
-
 class WebHistory(QObject):
 
     """The global history of visited pages.
@@ -157,7 +125,6 @@ class WebHistory(QObject):
 
     Attributes:
         history_dict: An OrderedDict of URLs read from the on-disk history.
-        _hist_dir: The directory to store the history in
         _lineparser: The AppendLineParser used to save the history.
         _new_history: A list of Entry items of the current session.
         _saved_count: How many HistoryEntries have been written to disk.
@@ -185,7 +152,6 @@ class WebHistory(QObject):
         super().__init__(parent)
         self._initial_read_started = False
         self._initial_read_done = False
-        self._hist_dir = hist_dir
         self._lineparser = lineparser.AppendLineParser(hist_dir, hist_name,
                                                        parent=self)
         self.history_dict = collections.OrderedDict()
@@ -211,12 +177,6 @@ class WebHistory(QObject):
             return
         self._initial_read_started = True
 
-        if self._hist_dir is None:
-            self._initial_read_done = True
-            self.async_read_done.emit()
-            assert not self._temp_history
-            return
-
         with self._lineparser.open():
             for line in self._lineparser:
                 yield
@@ -270,13 +230,23 @@ class WebHistory(QObject):
         self._saved_count = len(self._new_history)
 
     @cmdutils.register(name='history-clear', instance='web-history')
-    def clear(self):
+    def clear(self, force=False):
         """Clear all browsing history.
 
         Note this only clears the global history
         (e.g. `~/.local/share/qutebrowser/history` on Linux) but not cookies,
         the back/forward history of a tab, cache or other persistent data.
+
+        Args:
+            force: Don't ask for confirmation.
         """
+        if force:
+            self._do_clear()
+        else:
+            message.confirm_async(self._do_clear, title="Clear all browsing "
+                                "history?")
+
+    def _do_clear(self):
         self._lineparser.clear()
         self.history_dict.clear()
         self._temp_history.clear()
@@ -284,8 +254,19 @@ class WebHistory(QObject):
         self._saved_count = 0
         self.cleared.emit()
 
+    @pyqtSlot(QUrl, QUrl, str)
+    def add_from_tab(self, url, requested_url, title):
+        """Add a new history entry as slot, called from a BrowserTab."""
+        no_formatting = QUrl.UrlFormattingOption(0)
+        if (requested_url.isValid() and
+                not requested_url.matches(url, no_formatting)):
+            # If the url of the page is different than the url of the link
+            # originally clicked, save them both.
+            self.add_url(requested_url, title, redirect=True)
+        self.add_url(url, title)
+
     def add_url(self, url, title="", *, redirect=False, atime=None):
-        """Called by WebKit when a URL should be added to the history.
+        """Called via add_from_tab when a URL should be added to the history.
 
         Args:
             url: A url (as QUrl) to add to the history.
@@ -295,6 +276,10 @@ class WebHistory(QObject):
         """
         if config.get('general', 'private-browsing'):
             return
+        if not url.isValid():
+            log.misc.warning("Ignoring invalid URL being added to history")
+            return
+
         if atime is None:
             atime = time.time()
         entry = Entry(atime, url, title, redirect=redirect)
@@ -318,5 +303,6 @@ def init(parent=None):
                          parent=parent)
     objreg.register('web-history', history)
 
-    interface = WebHistoryInterface(history, parent=history)
-    QWebHistoryInterface.setDefaultInterface(interface)
+    if objects.backend == usertypes.Backend.QtWebKit:
+        from qutebrowser.browser.webkit import webkithistory
+        webkithistory.init(history)

qutebrowser/browser/inspector.py

@@ -17,31 +17,53 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Customized QWebInspector."""
+"""Base class for a QtWebKit/QtWebEngine web inspector."""
 
 import base64
 import binascii
 
-from PyQt5.QtWebKitWidgets import QWebInspector
+from PyQt5.QtWidgets import QWidget
 
-from qutebrowser.utils import log, objreg
+from qutebrowser.utils import log, objreg, usertypes
+from qutebrowser.misc import miscwidgets, objects
 
 
-class WebInspector(QWebInspector):
+def create(parent=None):
+    """Get a WebKitInspector/WebEngineInspector.
+
+    Args:
+        parent: The Qt parent to set.
+    """
+    # Importing modules here so we don't depend on QtWebEngine without the
+    # argument and to avoid circular imports.
+    if objects.backend == usertypes.Backend.QtWebEngine:
+        from qutebrowser.browser.webengine import webengineinspector
+        return webengineinspector.WebEngineInspector(parent)
+    else:
+        from qutebrowser.browser.webkit import webkitinspector
+        return webkitinspector.WebKitInspector(parent)
+
+
+class WebInspectorError(Exception):
+
+    """Raised when the inspector could not be initialized."""
+
+    pass
+
+
+class AbstractWebInspector(QWidget):
 
     """A customized WebInspector which stores its geometry."""
 
     def __init__(self, parent=None):
         super().__init__(parent)
+        self._widget = None
+        self._layout = miscwidgets.WrapperLayout(self)
         self._load_state_geometry()
 
-    def closeEvent(self, e):
-        """Save the geometry when closed."""
-        state_config = objreg.get('state-config')
-        data = bytes(self.saveGeometry())
-        geom = base64.b64encode(data).decode('ASCII')
-        state_config['geometry']['inspector'] = geom
-        super().closeEvent(e)
+    def _set_widget(self, widget):
+        self._widget = widget
+        self._layout.wrap(self, widget)
 
     def _load_state_geometry(self):
         """Load the geometry from the state file."""
@@ -59,3 +81,21 @@ class WebInspector(QWebInspector):
             ok = self.restoreGeometry(geom)
             if not ok:
                 log.init.warning("Error while loading geometry.")
+
+    def closeEvent(self, e):
+        """Save the geometry when closed."""
+        state_config = objreg.get('state-config')
+        data = bytes(self.saveGeometry())
+        geom = base64.b64encode(data).decode('ASCII')
+        state_config['geometry']['inspector'] = geom
+        super().closeEvent(e)
+
+    def inspect(self, page):
+        """Inspect the given QWeb(Engine)Page."""
+        raise NotImplementedError
+
+    def toggle(self, page):
+        if self._widget.isVisible():
+            self.hide()
+        else:
+            self.inspect(page)

qutebrowser/browser/mouse.py

@@ -0,0 +1,216 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Mouse handling for a browser tab."""
+
+
+from qutebrowser.config import config
+from qutebrowser.utils import message, log, usertypes
+from qutebrowser.keyinput import modeman
+
+
+from PyQt5.QtCore import QObject, QEvent, Qt, QTimer
+
+
+class ChildEventFilter(QObject):
+
+    """An event filter re-adding MouseEventFilter on ChildEvent.
+
+    This is needed because QtWebEngine likes to randomly change its
+    focusProxy...
+
+    FIXME:qtwebengine Add a test for this happening
+
+    Attributes:
+        _filter: The event filter to install.
+        _widget: The widget expected to send out childEvents.
+    """
+
+    def __init__(self, eventfilter, widget, parent=None):
+        super().__init__(parent)
+        self._filter = eventfilter
+        assert widget is not None
+        self._widget = widget
+
+    def eventFilter(self, obj, event):
+        """Act on ChildAdded events."""
+        if event.type() == QEvent.ChildAdded:
+            child = event.child()
+            log.mouse.debug("{} got new child {}, installing filter".format(
+                obj, child))
+            assert obj is self._widget
+            child.installEventFilter(self._filter)
+        return False
+
+
+class MouseEventFilter(QObject):
+
+    """Handle mouse events on a tab.
+
+    Attributes:
+        _tab: The browsertab object this filter is installed on.
+        _handlers: A dict of handler functions for the handled events.
+        _ignore_wheel_event: Whether to ignore the next wheelEvent.
+        _check_insertmode_on_release: Whether an insertmode check should be
+                                      done when the mouse is released.
+    """
+
+    def __init__(self, tab, *, parent=None):
+        super().__init__(parent)
+        self._tab = tab
+        self._handlers = {
+            QEvent.MouseButtonPress: self._handle_mouse_press,
+            QEvent.MouseButtonRelease: self._handle_mouse_release,
+            QEvent.Wheel: self._handle_wheel,
+            QEvent.ContextMenu: self._handle_context_menu,
+        }
+        self._ignore_wheel_event = False
+        self._check_insertmode_on_release = False
+
+    def _handle_mouse_press(self, e):
+        """Handle pressing of a mouse button."""
+        is_rocker_gesture = (config.get('input', 'rocker-gestures') and
+                             e.buttons() == Qt.LeftButton | Qt.RightButton)
+
+        if e.button() in [Qt.XButton1, Qt.XButton2] or is_rocker_gesture:
+            self._mousepress_backforward(e)
+            return True
+
+        self._ignore_wheel_event = True
+
+        if e.button() != Qt.NoButton:
+            self._tab.elements.find_at_pos(e.pos(),
+                                           self._mousepress_insertmode_cb)
+
+        return False
+
+    def _handle_mouse_release(self, _e):
+        """Handle releasing of a mouse button."""
+        # We want to make sure we check the focus element after the WebView is
+        # updated completely.
+        QTimer.singleShot(0, self._mouserelease_insertmode)
+        return False
+
+    def _handle_wheel(self, e):
+        """Zoom on Ctrl-Mousewheel.
+
+        Args:
+            e: The QWheelEvent.
+        """
+        if self._ignore_wheel_event:
+            # See https://github.com/qutebrowser/qutebrowser/issues/395
+            self._ignore_wheel_event = False
+            return True
+
+        if e.modifiers() & Qt.ControlModifier:
+            divider = config.get('input', 'mouse-zoom-divider')
+            if divider == 0:
+                return False
+            factor = self._tab.zoom.factor() + (e.angleDelta().y() / divider)
+            if factor < 0:
+                return False
+            perc = int(100 * factor)
+            message.info("Zoom level: {}%".format(perc), replace=True)
+            self._tab.zoom.set_factor(factor)
+        elif e.modifiers() & Qt.ShiftModifier:
+            if e.angleDelta().y() > 0:
+                self._tab.scroller.left()
+            else:
+                self._tab.scroller.right()
+            return True
+
+        return False
+
+    def _handle_context_menu(self, _e):
+        """Suppress context menus if rocker gestures are turned on."""
+        return config.get('input', 'rocker-gestures')
+
+    def _mousepress_insertmode_cb(self, elem):
+        """Check if the clicked element is editable."""
+        if elem is None:
+            # Something didn't work out, let's find the focus element after
+            # a mouse release.
+            log.mouse.debug("Got None element, scheduling check on "
+                            "mouse release")
+            self._check_insertmode_on_release = True
+            return
+
+        if elem.is_editable():
+            log.mouse.debug("Clicked editable element!")
+            modeman.enter(self._tab.win_id, usertypes.KeyMode.insert,
+                          'click', only_if_normal=True)
+        else:
+            log.mouse.debug("Clicked non-editable element!")
+            if config.get('input', 'auto-leave-insert-mode'):
+                modeman.leave(self._tab.win_id, usertypes.KeyMode.insert,
+                              'click', maybe=True)
+
+    def _mouserelease_insertmode(self):
+        """If we have an insertmode check scheduled, handle it."""
+        if not self._check_insertmode_on_release:
+            return
+        self._check_insertmode_on_release = False
+
+        def mouserelease_insertmode_cb(elem):
+            """Callback which gets called from JS."""
+            if elem is None:
+                log.mouse.debug("Element vanished!")
+                return
+
+            if elem.is_editable():
+                log.mouse.debug("Clicked editable element (delayed)!")
+                modeman.enter(self._tab.win_id, usertypes.KeyMode.insert,
+                              'click-delayed', only_if_normal=True)
+            else:
+                log.mouse.debug("Clicked non-editable element (delayed)!")
+                if config.get('input', 'auto-leave-insert-mode'):
+                    modeman.leave(self._tab.win_id, usertypes.KeyMode.insert,
+                                  'click-delayed', maybe=True)
+
+        self._tab.elements.find_focused(mouserelease_insertmode_cb)
+
+    def _mousepress_backforward(self, e):
+        """Handle back/forward mouse button presses.
+
+        Args:
+            e: The QMouseEvent.
+        """
+        if e.button() in [Qt.XButton1, Qt.LeftButton]:
+            # Back button on mice which have it, or rocker gesture
+            if self._tab.history.can_go_back():
+                self._tab.history.back()
+            else:
+                message.error("At beginning of history.")
+        elif e.button() in [Qt.XButton2, Qt.RightButton]:
+            # Forward button on mice which have it, or rocker gesture
+            if self._tab.history.can_go_forward():
+                self._tab.history.forward()
+            else:
+                message.error("At end of history.")
+
+    def eventFilter(self, obj, event):
+        """Filter events going to a QWeb(Engine)View."""
+        evtype = event.type()
+        if evtype not in self._handlers:
+            return False
+        if obj is not self._tab.event_target():
+            log.mouse.debug("Ignoring {} to {}".format(
+                event.__class__.__name__, obj))
+            return False
+        return self._handlers[evtype](event)

qutebrowser/browser/navigate.py

@@ -0,0 +1,147 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Implementation of :navigate."""
+
+import posixpath
+
+from qutebrowser.browser import webelem
+from qutebrowser.config import config
+from qutebrowser.utils import objreg, urlutils, log, message, qtutils
+
+
+class Error(Exception):
+
+    """Raised when the navigation can't be done."""
+
+
+def incdec(url, count, inc_or_dec):
+    """Helper method for :navigate when `where' is increment/decrement.
+
+    Args:
+        url: The current url.
+        count: How much to increment or decrement by.
+        inc_or_dec: Either 'increment' or 'decrement'.
+        tab: Whether to open the link in a new tab.
+        background: Open the link in a new background tab.
+        window: Open the link in a new window.
+    """
+    segments = set(config.get('general', 'url-incdec-segments'))
+    try:
+        new_url = urlutils.incdec_number(url, inc_or_dec, count,
+                                         segments=segments)
+    except urlutils.IncDecError as error:
+        raise Error(error.msg)
+    return new_url
+
+
+def path_up(url, count):
+    """Helper method for :navigate when `where' is up.
+
+    Args:
+        url: The current url.
+        count: The number of levels to go up in the url.
+    """
+    path = url.path()
+    if not path or path == '/':
+        raise Error("Can't go up!")
+    for _i in range(0, min(count, path.count('/'))):
+        path = posixpath.join(path, posixpath.pardir)
+    url.setPath(path)
+    return url
+
+
+def _find_prevnext(prev, elems):
+    """Find a prev/next element in the given list of elements."""
+    # First check for <link rel="prev(ious)|next">
+    rel_values = {'prev', 'previous'} if prev else {'next'}
+    for e in elems:
+        if e.tag_name() not in ['link', 'a'] or 'rel' not in e:
+            continue
+        if set(e['rel'].split(' ')) & rel_values:
+            log.hints.debug("Found {!r} with rel={}".format(e, e['rel']))
+            return e
+
+    # Then check for regular links/buttons.
+    filterfunc = webelem.FILTERS[webelem.Group.prevnext]
+    elems = [e for e in elems if e.tag_name() != 'link' and filterfunc(e)]
+    option = 'prev-regexes' if prev else 'next-regexes'
+    if not elems:
+        return None
+    for regex in config.get('hints', option):
+        log.hints.vdebug("== Checking regex '{}'.".format(regex.pattern))
+        for e in elems:
+            text = str(e)
+            if not text:
+                continue
+            if regex.search(text):
+                log.hints.debug("Regex '{}' matched on '{}'.".format(
+                    regex.pattern, text))
+                return e
+            else:
+                log.hints.vdebug("No match on '{}'!".format(text))
+    return None
+
+
+def prevnext(*, browsertab, win_id, baseurl, prev=False,
+             tab=False, background=False, window=False):
+    """Click a "previous"/"next" element on the page.
+
+    Args:
+        browsertab: The WebKitTab/WebEngineTab of the page.
+        baseurl: The base URL of the current tab.
+        prev: True to open a "previous" link, False to open a "next" link.
+        tab: True to open in a new tab, False for the current tab.
+        background: True to open in a background tab.
+        window: True to open in a new window, False for the current one.
+    """
+    def _prevnext_cb(elems):
+        if elems is None:
+            message.error("There was an error while getting hint elements")
+            return
+
+        elem = _find_prevnext(prev, elems)
+        word = 'prev' if prev else 'forward'
+
+        if elem is None:
+            message.error("No {} links found!".format(word))
+            return
+        url = elem.resolve_url(baseurl)
+        if url is None:
+            message.error("No {} links found!".format(word))
+            return
+        qtutils.ensure_valid(url)
+
+        if window:
+            from qutebrowser.mainwindow import mainwindow
+            new_window = mainwindow.MainWindow()
+            new_window.show()
+            tabbed_browser = objreg.get('tabbed-browser', scope='window',
+                                        window=new_window.win_id)
+            tabbed_browser.tabopen(url, background=False)
+        elif tab:
+            tabbed_browser = objreg.get('tabbed-browser', scope='window',
+                                        window=win_id)
+            tabbed_browser.tabopen(url, background=background)
+        else:
+            browsertab.openurl(url)
+
+    selector = ', '.join([webelem.SELECTORS[webelem.Group.links],
+                          webelem.SELECTORS[webelem.Group.prevnext]])
+    browsertab.elements.find_css(selector, _prevnext_cb)

qutebrowser/browser/network/__init__.py

@@ -0,0 +1,3 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+"""Modules related to network operations."""

qutebrowser/browser/network/pac.py

@@ -0,0 +1,305 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Evaluation of PAC scripts."""
+
+import sys
+import functools
+
+from PyQt5.QtCore import (QObject, pyqtSignal, pyqtSlot)
+from PyQt5.QtNetwork import (QNetworkProxy, QNetworkRequest, QHostInfo,
+                             QNetworkReply, QNetworkAccessManager,
+                             QHostAddress)
+from PyQt5.QtQml import QJSEngine, QJSValue
+
+from qutebrowser.utils import log, utils, qtutils
+
+
+class ParseProxyError(Exception):
+
+    """Error while parsing PAC result string."""
+
+    pass
+
+
+class EvalProxyError(Exception):
+
+    """Error while evaluating PAC script."""
+
+    pass
+
+
+def _js_slot(*args):
+    """Wrap a methods as a JavaScript function.
+
+    Register a PACContext method as a JavaScript function, and catch
+    exceptions returning them as JavaScript Error objects.
+
+    Args:
+        args: Types of method arguments.
+
+    Return: Wrapped method.
+    """
+    def _decorator(method):
+        @functools.wraps(method)
+        def new_method(self, *args, **kwargs):
+            try:
+                return method(self, *args, **kwargs)
+            except:
+                e = str(sys.exc_info()[0])
+                log.network.exception("PAC evaluation error")
+                # pylint: disable=protected-access
+                return self._error_con.callAsConstructor([e])
+                # pylint: enable=protected-access
+        return pyqtSlot(*args, result=QJSValue)(new_method)
+    return _decorator
+
+
+class _PACContext(QObject):
+
+    """Implementation of PAC API functions that require native calls.
+
+    See https://developer.mozilla.org/en-US/docs/Mozilla/Projects/Necko/Proxy_Auto-Configuration_(PAC)_file
+    """
+
+    JS_DEFINITIONS = """
+        function dnsResolve(host) {
+            return PAC.dnsResolve(host);
+        }
+
+        function myIpAddress() {
+            return PAC.myIpAddress();
+        }
+    """
+
+    def __init__(self, engine):
+        """Create a new PAC API implementation instance.
+
+        Args:
+            engine: QJSEngine which is used for running PAC.
+        """
+        super().__init__(parent=engine)
+        self._engine = engine
+        self._error_con = engine.globalObject().property("Error")
+
+    @_js_slot(str)
+    def dnsResolve(self, host):
+        """Resolve a DNS hostname.
+
+        Resolves the given DNS hostname into an IP address, and returns it
+        in the dot-separated format as a string.
+
+        Args:
+            host: hostname to resolve.
+        """
+        ips = QHostInfo.fromName(host)
+        if ips.error() != QHostInfo.NoError or not ips.addresses():
+            err_f = "Failed to resolve host during PAC evaluation: {}"
+            log.network.info(err_f.format(host))
+            return QJSValue(QJSValue.NullValue)
+        else:
+            return ips.addresses()[0].toString()
+
+    @_js_slot()
+    def myIpAddress(self):
+        """Get host IP address.
+
+        Return the server IP address of the current machine, as a string in
+        the dot-separated integer format.
+        """
+        return QHostAddress(QHostAddress.LocalHost).toString()
+
+
+class PACResolver:
+
+    """Evaluate PAC script files and resolve proxies."""
+
+    @staticmethod
+    def _parse_proxy_host(host_str):
+        host, _colon, port_str = host_str.partition(':')
+        try:
+            port = int(port_str)
+        except ValueError:
+            raise ParseProxyError("Invalid port number")
+        return (host, port)
+
+    @staticmethod
+    def _parse_proxy_entry(proxy_str):
+        """Parse one proxy string entry, as described in PAC specification."""
+        config = [c.strip() for c in proxy_str.split(' ') if c]
+        if not config:
+            raise ParseProxyError("Empty proxy entry")
+        elif config[0] == "DIRECT":
+            if len(config) != 1:
+                raise ParseProxyError("Invalid number of parameters for " +
+                                      "DIRECT")
+            return QNetworkProxy(QNetworkProxy.NoProxy)
+        elif config[0] == "PROXY":
+            if len(config) != 2:
+                raise ParseProxyError("Invalid number of parameters for PROXY")
+            host, port = PACResolver._parse_proxy_host(config[1])
+            return QNetworkProxy(QNetworkProxy.HttpProxy, host, port)
+        elif config[0] == "SOCKS":
+            if len(config) != 2:
+                raise ParseProxyError("Invalid number of parameters for SOCKS")
+            host, port = PACResolver._parse_proxy_host(config[1])
+            return QNetworkProxy(QNetworkProxy.Socks5Proxy, host, port)
+        else:
+            err = "Unknown proxy type: {}"
+            raise ParseProxyError(err.format(config[0]))
+
+    @staticmethod
+    def _parse_proxy_string(proxy_str):
+        proxies = proxy_str.split(';')
+        return [PACResolver._parse_proxy_entry(x) for x in proxies]
+
+    def _evaluate(self, js_code, js_file):
+        ret = self._engine.evaluate(js_code, js_file)
+        if ret.isError():
+            err = "JavaScript error while evaluating PAC file: {}"
+            raise EvalProxyError(err.format(ret.toString()))
+
+    def __init__(self, pac_str):
+        """Create a PAC resolver.
+
+        Args:
+            pac_str: JavaScript code containing PAC resolver.
+        """
+        self._engine = QJSEngine()
+
+        self._ctx = _PACContext(self._engine)
+        self._engine.globalObject().setProperty(
+            "PAC", self._engine.newQObject(self._ctx))
+        self._evaluate(_PACContext.JS_DEFINITIONS, "pac_js_definitions")
+        self._evaluate(utils.read_file("javascript/pac_utils.js"), "pac_utils")
+        proxy_config = self._engine.newObject()
+        proxy_config.setProperty("bindings", self._engine.newObject())
+        self._engine.globalObject().setProperty("ProxyConfig", proxy_config)
+
+        self._evaluate(pac_str, "pac")
+        global_js_object = self._engine.globalObject()
+        self._resolver = global_js_object.property("FindProxyForURL")
+        if not self._resolver.isCallable():
+            err = "Cannot resolve FindProxyForURL function, got '{}' instead"
+            raise EvalProxyError(err.format(self._resolver.toString()))
+
+    def resolve(self, query):
+        """Resolve a proxy via PAC.
+
+        Args:
+            query: QNetworkProxyQuery.
+
+        Return:
+            A list of QNetworkProxy objects in order of preference.
+        """
+        result = self._resolver.call([query.url().toString(),
+                                      query.peerHostName()])
+        result_str = result.toString()
+        if not result.isString():
+            err = "Got strange value from FindProxyForURL: '{}'"
+            raise EvalProxyError(err.format(result_str))
+        return self._parse_proxy_string(result_str)
+
+
+class PACFetcher(QObject):
+
+    """Asynchronous fetcher of PAC files."""
+
+    finished = pyqtSignal()
+
+    def __init__(self, url, parent=None):
+        """Resolve a PAC proxy from URL.
+
+        Args:
+            url: QUrl of a PAC proxy.
+        """
+        super().__init__(parent)
+
+        pac_prefix = "pac+"
+
+        assert url.scheme().startswith(pac_prefix)
+        url.setScheme(url.scheme()[len(pac_prefix):])
+
+        self._manager = QNetworkAccessManager()
+        self._manager.setProxy(QNetworkProxy(QNetworkProxy.NoProxy))
+        self._reply = self._manager.get(QNetworkRequest(url))
+        self._reply.finished.connect(self._finish)
+        self._pac = None
+        self._error_message = None
+
+    @pyqtSlot()
+    def _finish(self):
+        if self._reply.error() != QNetworkReply.NoError:
+            error = "Can't fetch PAC file from URL, error code {}: {}"
+            self._error_message = error.format(
+                self._reply.error(), self._reply.errorString())
+            log.network.error(self._error_message)
+        else:
+            try:
+                pacscript = bytes(self._reply.readAll()).decode("utf-8")
+            except UnicodeError as e:
+                error = "Invalid encoding of a PAC file: {}"
+                self._error_message = error.format(e)
+                log.network.exception(self._error_message)
+            try:
+                self._pac = PACResolver(pacscript)
+                log.network.debug("Successfully evaluated PAC file.")
+            except EvalProxyError as e:
+                error = "Error in PAC evaluation: {}"
+                self._error_message = error.format(e)
+                log.network.exception(self._error_message)
+        self._manager = None
+        self._reply = None
+        self.finished.emit()
+
+    def _wait(self):
+        """Wait until a reply from the remote server is received."""
+        if self._manager is not None:
+            loop = qtutils.EventLoop()
+            self.finished.connect(loop.quit)
+            loop.exec_()
+
+    def fetch_error(self):
+        """Check if PAC script is successfully fetched.
+
+        Return None iff PAC script is downloaded and evaluated successfully,
+        error string otherwise.
+        """
+        self._wait()
+        return self._error_message
+
+    def resolve(self, query):
+        """Resolve a query via PAC.
+
+        Args: QNetworkProxyQuery.
+
+        Return a list of QNetworkProxy objects in order of preference.
+        """
+        self._wait()
+        try:
+            return self._pac.resolve(query)
+        except (EvalProxyError, ParseProxyError) as e:
+            log.network.exception("Error in PAC resolution: {}.".format(e))
+            # .invalid is guaranteed to be inaccessible in RFC 6761.
+            # Port 9 is for DISCARD protocol -- DISCARD servers act like
+            # /dev/null.
+            # Later NetworkManager.createRequest will detect this and display
+            # an error message.
+            error_host = "pac-resolve-error.qutebrowser.invalid"
+            return QNetworkProxy(QNetworkProxy.HttpProxy, error_host, 9)

qutebrowser/browser/network/proxy.py

@@ -23,17 +23,33 @@
 from PyQt5.QtNetwork import QNetworkProxy, QNetworkProxyFactory
 
 from qutebrowser.config import config, configtypes
+from qutebrowser.utils import objreg
+from qutebrowser.browser.network import pac
 
 
 def init():
     """Set the application wide proxy factory."""
-    QNetworkProxyFactory.setApplicationProxyFactory(ProxyFactory())
+    proxy_factory = ProxyFactory()
+    objreg.register('proxy-factory', proxy_factory)
+    QNetworkProxyFactory.setApplicationProxyFactory(proxy_factory)
 
 
 class ProxyFactory(QNetworkProxyFactory):
 
     """Factory for proxies to be used by qutebrowser."""
 
+    def get_error(self):
+        """Check if proxy can't be resolved.
+
+        Return:
+           None if proxy is correct, otherwise an error message.
+        """
+        proxy = config.get('network', 'proxy')
+        if isinstance(proxy, pac.PACFetcher):
+            return proxy.fetch_error()
+        else:
+            return None
+
     def queryProxy(self, query):
         """Get the QNetworkProxies for a query.
 
@@ -46,6 +62,8 @@ class ProxyFactory(QNetworkProxyFactory):
         proxy = config.get('network', 'proxy')
         if proxy is configtypes.SYSTEM_PROXY:
             proxies = QNetworkProxyFactory.systemProxyForQuery(query)
+        elif isinstance(proxy, pac.PACFetcher):
+            proxies = proxy.resolve(query)
         else:
             proxies = [proxy]
         for p in proxies:

qutebrowser/browser/pdfjs.py

@@ -23,8 +23,7 @@ import os
 
 from PyQt5.QtCore import QUrl
 
-from qutebrowser.browser.webkit import webelem
-from qutebrowser.utils import utils
+from qutebrowser.utils import utils, javascript
 
 
 class PDFJSNotFound(Exception):
@@ -63,9 +62,11 @@ def _generate_pdfjs_script(url):
         url: The url of the pdf page as QUrl.
     """
     return (
-        'PDFJS.verbosity = PDFJS.VERBOSITY_LEVELS.info;\n'
-        'PDFView.open("{url}");\n'
-    ).format(url=webelem.javascript_escape(url.toString(QUrl.FullyEncoded)))
+        'document.addEventListener("DOMContentLoaded", function() {{\n'
+        '  PDFJS.verbosity = PDFJS.VERBOSITY_LEVELS.info;\n'
+        '  (window.PDFView || window.PDFViewerApplication).open("{url}");\n'
+        '}});\n'
+    ).format(url=javascript.string_escape(url.toString(QUrl.FullyEncoded)))
 
 
 def fix_urls(asset):
@@ -99,6 +100,8 @@ SYSTEM_PDFJS_PATHS = [
     # Debian pdf.js-common
     # Arch Linux pdfjs (AUR)
     '/usr/share/pdf.js/',
+    # Arch Linux pdf.js (AUR)
+    '/usr/share/javascript/pdf.js/',
     # Debian libjs-pdf
     '/usr/share/javascript/pdf/',
     # fallback

qutebrowser/browser/qtnetworkdownloads.py

@@ -0,0 +1,528 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Download manager."""
+
+import io
+import shutil
+import functools
+import collections
+
+from PyQt5.QtCore import pyqtSlot, pyqtSignal, QTimer
+from PyQt5.QtNetwork import QNetworkRequest, QNetworkReply
+
+from qutebrowser.utils import message, usertypes, log, urlutils, utils
+from qutebrowser.browser import downloads
+from qutebrowser.browser.webkit import http
+from qutebrowser.browser.webkit.network import networkmanager
+
+
+_RetryInfo = collections.namedtuple('_RetryInfo', ['request', 'manager'])
+
+
+class DownloadItem(downloads.AbstractDownloadItem):
+
+    """A single download currently running.
+
+    There are multiple ways the data can flow from the QNetworkReply to the
+    disk.
+
+    If the filename/file object is known immediately when starting the
+    download, QNetworkReply's readyRead writes to the target file directly.
+
+    If not, readyRead is ignored and with self._read_timer we periodically read
+    into the self._buffer BytesIO slowly, so some broken servers don't close
+    our connection.
+
+    As soon as we know the file object, we copy self._buffer over and the next
+    readyRead will write to the real file object.
+
+    Class attributes:
+        _MAX_REDIRECTS: The maximum redirection count.
+
+    Attributes:
+        _retry_info: A _RetryInfo instance.
+        _redirects: How many time we were redirected already.
+        _buffer: A BytesIO object to buffer incoming data until we know the
+                 target file.
+        _read_timer: A Timer which reads the QNetworkReply into self._buffer
+                     periodically.
+        _manager: The DownloadManager which started this download
+        _reply: The QNetworkReply associated with this download.
+        _autoclose: Whether to close the associated file when the download is
+                    done.
+
+    Signals:
+        adopt_download: Emitted when a download is retried and should be
+                        adopted by the QNAM if needed.
+                        arg 0: The new DownloadItem
+    """
+
+    _MAX_REDIRECTS = 10
+    adopt_download = pyqtSignal(object)  # DownloadItem
+
+    def __init__(self, reply, manager):
+        """Constructor.
+
+        Args:
+            reply: The QNetworkReply to download.
+        """
+        super().__init__(parent=manager)
+        self.fileobj = None
+        self.raw_headers = {}
+
+        self._autoclose = True
+        self._manager = manager
+        self._retry_info = None
+        self._reply = None
+        self._buffer = io.BytesIO()
+        self._read_timer = usertypes.Timer(self, name='download-read-timer')
+        self._read_timer.setInterval(500)
+        self._read_timer.timeout.connect(self._on_read_timer_timeout)
+        self._redirects = 0
+        self._init_reply(reply)
+
+    def _create_fileobj(self):
+        """Create a file object using the internal filename."""
+        try:
+            fileobj = open(self._filename, 'wb')
+        except OSError as e:
+            self._die(e.strerror)
+        else:
+            self._set_fileobj(fileobj)
+
+    def _do_die(self):
+        """Abort the download and emit an error."""
+        self._read_timer.stop()
+        self._reply.downloadProgress.disconnect()
+        self._reply.finished.disconnect()
+        self._reply.error.disconnect()
+        self._reply.readyRead.disconnect()
+        with log.hide_qt_warning('QNetworkReplyImplPrivate::error: Internal '
+                                 'problem, this method must only be called '
+                                 'once.'):
+            # See https://codereview.qt-project.org/#/c/107863/
+            self._reply.abort()
+        self._reply.deleteLater()
+        self._reply = None
+        if self.fileobj is not None:
+            try:
+                self.fileobj.close()
+            except OSError:
+                log.downloads.exception("Error while closing file object")
+
+    def _init_reply(self, reply):
+        """Set a new reply and connect its signals.
+
+        Args:
+            reply: The QNetworkReply to handle.
+        """
+        self.done = False
+        self.successful = False
+        self._reply = reply
+        reply.setReadBufferSize(16 * 1024 * 1024)  # 16 MB
+        reply.downloadProgress.connect(self.stats.on_download_progress)
+        reply.finished.connect(self._on_reply_finished)
+        reply.error.connect(self._on_reply_error)
+        reply.readyRead.connect(self._on_ready_read)
+        reply.metaDataChanged.connect(self._on_meta_data_changed)
+        self._retry_info = _RetryInfo(request=reply.request(),
+                                      manager=reply.manager())
+        if not self.fileobj:
+            self._read_timer.start()
+        # We could have got signals before we connected slots to them.
+        # Here no signals are connected to the DownloadItem yet, so we use a
+        # singleShot QTimer to emit them after they are connected.
+        if reply.error() != QNetworkReply.NoError:
+            QTimer.singleShot(0, lambda: self._die(reply.errorString()))
+
+    def _do_cancel(self):
+        if self._reply is not None:
+            self._reply.finished.disconnect(self._on_reply_finished)
+            self._reply.abort()
+            self._reply.deleteLater()
+            self._reply = None
+        if self.fileobj is not None:
+            self.fileobj.close()
+        self.cancelled.emit()
+
+    @pyqtSlot()
+    def retry(self):
+        """Retry a failed download."""
+        assert self.done
+        assert not self.successful
+        new_reply = self._retry_info.manager.get(self._retry_info.request)
+        new_download = self._manager.fetch(new_reply,
+                                           suggested_filename=self.basename)
+        self.adopt_download.emit(new_download)
+        self.cancel()
+
+    def _get_open_filename(self):
+        filename = self._filename
+        if filename is None:
+            filename = getattr(self.fileobj, 'name', None)
+        return filename
+
+    def _ensure_can_set_filename(self, filename):
+        if self.fileobj is not None:  # pragma: no cover
+            raise ValueError("fileobj was already set! filename: {}, "
+                             "existing: {}, fileobj {}".format(
+                                 filename, self._filename, self.fileobj))
+
+    def _after_set_filename(self):
+        self._create_fileobj()
+
+    def _ask_confirm_question(self, title, msg):
+        no_action = functools.partial(self.cancel, remove_data=False)
+        message.confirm_async(title=title, text=msg,
+                              yes_action=self._after_set_filename,
+                              no_action=no_action, cancel_action=no_action,
+                              abort_on=[self.cancelled, self.error])
+
+    def _set_fileobj(self, fileobj, *, autoclose=True):
+        """"Set the file object to write the download to.
+
+        Args:
+            fileobj: A file-like object.
+        """
+        if self.fileobj is not None:  # pragma: no cover
+            raise ValueError("fileobj was already set! Old: {}, new: "
+                             "{}".format(self.fileobj, fileobj))
+        self.fileobj = fileobj
+        self._autoclose = autoclose
+        try:
+            self._read_timer.stop()
+            log.downloads.debug("buffer: {} bytes".format(self._buffer.tell()))
+            self._buffer.seek(0)
+            shutil.copyfileobj(self._buffer, fileobj)
+            self._buffer.close()
+            if self._reply.isFinished():
+                # Downloading to the buffer in RAM has already finished so we
+                # write out the data and clean up now.
+                self._on_reply_finished()
+            else:
+                # Since the buffer already might be full, on_ready_read might
+                # not be called at all anymore, so we force it here to flush
+                # the buffer and continue receiving new data.
+                self._on_ready_read()
+        except OSError as e:
+            self._die(e.strerror)
+
+    def _set_tempfile(self, fileobj):
+        self._set_fileobj(fileobj)
+
+    def _finish_download(self):
+        """Write buffered data to disk and finish the QNetworkReply."""
+        log.downloads.debug("Finishing download...")
+        if self._reply.isOpen():
+            self.fileobj.write(self._reply.readAll())
+        if self._autoclose:
+            self.fileobj.close()
+        self.successful = self._reply.error() == QNetworkReply.NoError
+        self._reply.close()
+        self._reply.deleteLater()
+        self._reply = None
+        self.finished.emit()
+        self.done = True
+        log.downloads.debug("Download {} finished".format(self.basename))
+        self.data_changed.emit()
+
+    @pyqtSlot()
+    def _on_reply_finished(self):
+        """Clean up when the download was finished.
+
+        Note when this gets called, only the QNetworkReply has finished. This
+        doesn't mean the download (i.e. writing data to the disk) is finished
+        as well. Therefore, we can't close() the QNetworkReply in here yet.
+        """
+        if self._reply is None:
+            return
+        self._read_timer.stop()
+        self.stats.finish()
+        is_redirected = self._handle_redirect()
+        if is_redirected:
+            return
+        log.downloads.debug("Reply finished, fileobj {}".format(self.fileobj))
+        if self.fileobj is not None:
+            # We can do a "delayed" write immediately to empty the buffer and
+            # clean up.
+            self._finish_download()
+
+    @pyqtSlot()
+    def _on_ready_read(self):
+        """Read available data and save file when ready to read."""
+        if self.fileobj is None or self._reply is None:
+            # No filename has been set yet (so we don't empty the buffer) or we
+            # got a readyRead after the reply was finished (which happens on
+            # qute:log for example).
+            return
+        if not self._reply.isOpen():
+            raise OSError("Reply is closed!")
+        try:
+            self.fileobj.write(self._reply.readAll())
+        except OSError as e:
+            self._die(e.strerror)
+
+    @pyqtSlot('QNetworkReply::NetworkError')
+    def _on_reply_error(self, code):
+        """Handle QNetworkReply errors."""
+        if code == QNetworkReply.OperationCanceledError:
+            return
+        else:
+            self._die(self._reply.errorString())
+
+    @pyqtSlot()
+    def _on_read_timer_timeout(self):
+        """Read some bytes from the QNetworkReply periodically."""
+        if not self._reply.isOpen():
+            raise OSError("Reply is closed!")
+        data = self._reply.read(1024)
+        if data is not None:
+            self._buffer.write(data)
+
+    @pyqtSlot()
+    def _on_meta_data_changed(self):
+        """Update the download's metadata."""
+        if self._reply is None:
+            return
+        self.raw_headers = {}
+        for key, value in self._reply.rawHeaderPairs():
+            self.raw_headers[bytes(key)] = bytes(value)
+
+    def _handle_redirect(self):
+        """Handle an HTTP redirect.
+
+        Return:
+            True if the download was redirected, False otherwise.
+        """
+        redirect = self._reply.attribute(
+            QNetworkRequest.RedirectionTargetAttribute)
+        if redirect is None or redirect.isEmpty():
+            return False
+        new_url = self._reply.url().resolved(redirect)
+        new_request = self._reply.request()
+        if new_url == new_request.url():
+            return False
+
+        if self._redirects > self._MAX_REDIRECTS:
+            self._die("Maximum redirection count reached!")
+            self.delete()
+            return True  # so on_reply_finished aborts
+
+        log.downloads.debug("{}: Handling redirect".format(self))
+        self._redirects += 1
+        new_request.setUrl(new_url)
+        old_reply = self._reply
+        old_reply.finished.disconnect(self._on_reply_finished)
+        self._read_timer.stop()
+        self._reply = None
+        if self.fileobj is not None:
+            self.fileobj.seek(0)
+
+        log.downloads.debug("redirected: {} -> {}".format(
+            old_reply.url(), new_request.url()))
+        new_reply = old_reply.manager().get(new_request)
+        self._init_reply(new_reply)
+
+        old_reply.deleteLater()
+        return True
+
+    def _uses_nam(self, nam):
+        """Check if this download uses the given QNetworkAccessManager."""
+        running_nam = self._reply is not None and self._reply.manager() is nam
+        # user could request retry after tab is closed.
+        retry_nam = (self.done and (not self.successful) and
+                     self._retry_info.manager is nam)
+        return running_nam or retry_nam
+
+
+class DownloadManager(downloads.AbstractDownloadManager):
+
+    """Manager for currently running downloads.
+
+    Attributes:
+        _networkmanager: A NetworkManager for generic downloads.
+    """
+
+    def __init__(self, win_id, parent=None):
+        super().__init__(parent)
+        self._networkmanager = networkmanager.NetworkManager(
+            win_id, None, self)
+
+    @pyqtSlot('QUrl')
+    def get(self, url, *, user_agent=None, **kwargs):
+        """Start a download with a link URL.
+
+        Args:
+            url: The URL to get, as QUrl
+            user_agent: The UA to set for the request, or None.
+            **kwargs: passed to get_request().
+
+        Return:
+            The created DownloadItem.
+        """
+        if not url.isValid():
+            urlutils.invalid_url_error(url, "start download")
+            return
+        req = QNetworkRequest(url)
+        if user_agent is not None:
+            req.setHeader(QNetworkRequest.UserAgentHeader, user_agent)
+        return self.get_request(req, **kwargs)
+
+    def get_mhtml(self, tab, target):
+        """Download the given tab as mhtml to the given DownloadTarget."""
+        assert tab.backend == usertypes.Backend.QtWebKit
+        from qutebrowser.browser.webkit import mhtml
+
+        if target is not None:
+            mhtml.start_download_checked(target, tab=tab)
+            return
+
+        suggested_fn = utils.sanitize_filename(tab.title() + ".mhtml")
+
+        filename = downloads.immediate_download_path()
+        if filename is not None:
+            target = downloads.FileDownloadTarget(filename)
+            mhtml.start_download_checked(target, tab=tab)
+        else:
+            question = downloads.get_filename_question(
+                suggested_filename=suggested_fn, url=tab.url(), parent=tab)
+            question.answered.connect(functools.partial(
+                mhtml.start_download_checked, tab=tab))
+            message.global_bridge.ask(question, blocking=False)
+
+    def get_request(self, request, *, target=None, **kwargs):
+        """Start a download with a QNetworkRequest.
+
+        Args:
+            request: The QNetworkRequest to download.
+            target: Where to save the download as downloads.DownloadTarget.
+            **kwargs: Passed to _fetch_request.
+
+        Return:
+            The created DownloadItem.
+        """
+        # WORKAROUND for Qt corrupting data loaded from cache:
+        # https://bugreports.qt.io/browse/QTBUG-42757
+        request.setAttribute(QNetworkRequest.CacheLoadControlAttribute,
+                             QNetworkRequest.AlwaysNetwork)
+
+        if request.url().scheme().lower() != 'data':
+            suggested_fn = urlutils.filename_from_url(request.url())
+        else:
+            # We might be downloading a binary blob embedded on a page or even
+            # generated dynamically via javascript. We try to figure out a more
+            # sensible name than the base64 content of the data.
+            origin = request.originatingObject()
+            try:
+                origin_url = origin.url()
+            except AttributeError:
+                # Raised either if origin is None or some object that doesn't
+                # have its own url. We're probably fine with a default fallback
+                # then.
+                suggested_fn = 'binary blob'
+            else:
+                # Use the originating URL as a base for the filename (works
+                # e.g. for pdf.js).
+                suggested_fn = urlutils.filename_from_url(origin_url)
+
+        if suggested_fn is None:
+            suggested_fn = 'qutebrowser-download'
+
+        return self._fetch_request(request,
+                                   target=target,
+                                   suggested_filename=suggested_fn,
+                                   **kwargs)
+
+    def _fetch_request(self, request, *, qnam=None, **kwargs):
+        """Download a QNetworkRequest to disk.
+
+        Args:
+            request: The QNetworkRequest to download.
+            qnam: The QNetworkAccessManager to use.
+            **kwargs: passed to fetch().
+
+        Return:
+            The created DownloadItem.
+        """
+        if qnam is None:
+            qnam = self._networkmanager
+        reply = qnam.get(request)
+        return self.fetch(reply, **kwargs)
+
+    @pyqtSlot('QNetworkReply')
+    def fetch(self, reply, *, target=None, auto_remove=False,
+              suggested_filename=None, prompt_download_directory=None):
+        """Download a QNetworkReply to disk.
+
+        Args:
+            reply: The QNetworkReply to download.
+            target: Where to save the download as downloads.DownloadTarget.
+            auto_remove: Whether to remove the download even if
+                         ui -> remove-finished-downloads is set to -1.
+
+        Return:
+            The created DownloadItem.
+        """
+        if not suggested_filename:
+            try:
+                suggested_filename = target.suggested_filename()
+            except downloads.NoFilenameError:
+                _, suggested_filename = http.parse_content_disposition(reply)
+        log.downloads.debug("fetch: {} -> {}".format(reply.url(),
+                                                     suggested_filename))
+        download = DownloadItem(reply, manager=self)
+        self._init_item(download, auto_remove, suggested_filename)
+
+        if target is not None:
+            download.set_target(target)
+            return download
+
+        # Neither filename nor fileobj were given
+
+        filename = downloads.immediate_download_path(prompt_download_directory)
+        if filename is not None:
+            # User doesn't want to be asked, so just use the download_dir
+            target = downloads.FileDownloadTarget(filename)
+            download.set_target(target)
+            return download
+
+        # Ask the user for a filename
+        question = downloads.get_filename_question(
+            suggested_filename=suggested_filename, url=reply.url(),
+            parent=self)
+        self._init_filename_question(question, download)
+        message.global_bridge.ask(question, blocking=False)
+
+        return download
+
+    def has_downloads_with_nam(self, nam):
+        """Check if the DownloadManager has any downloads with the given QNAM.
+
+        Args:
+            nam: The QNetworkAccessManager to check.
+
+        Return:
+            A boolean.
+        """
+        assert nam.adopted_downloads == 0
+        for download in self.downloads:
+            if download._uses_nam(nam):  # pylint: disable=protected-access
+                nam.adopt_download(download)
+        return nam.adopted_downloads

qutebrowser/browser/qutescheme.py

@@ -0,0 +1,341 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Backend-independent qute:* code.
+
+Module attributes:
+    pyeval_output: The output of the last :pyeval command.
+    _HANDLERS: The handlers registered via decorators.
+"""
+
+import sys
+import time
+import datetime
+import urllib.parse
+
+from PyQt5.QtCore import QUrlQuery
+
+import qutebrowser
+from qutebrowser.utils import (version, utils, jinja, log, message, docutils,
+                               objreg)
+from qutebrowser.misc import objects
+
+
+pyeval_output = ":pyeval was never called"
+
+
+_HANDLERS = {}
+
+
+class NoHandlerFound(Exception):
+
+    """Raised when no handler was found for the given URL."""
+
+    pass
+
+
+class QuteSchemeOSError(Exception):
+
+    """Called when there was an OSError inside a handler."""
+
+    pass
+
+
+class QuteSchemeError(Exception):
+
+    """Exception to signal that a handler should return an ErrorReply.
+
+    Attributes correspond to the arguments in
+    networkreply.ErrorNetworkReply.
+
+    Attributes:
+        errorstring: Error string to print.
+        error: Numerical error value.
+    """
+
+    def __init__(self, errorstring, error):
+        self.errorstring = errorstring
+        self.error = error
+        super().__init__(errorstring)
+
+
+class add_handler:  # pylint: disable=invalid-name
+
+    """Decorator to register a qute:* URL handler.
+
+    Attributes:
+        _name: The 'foo' part of qute:foo
+        backend: Limit which backends the handler can run with.
+    """
+
+    def __init__(self, name, backend=None):
+        self._name = name
+        self._backend = backend
+        self._function = None
+
+    def __call__(self, function):
+        self._function = function
+        _HANDLERS[self._name] = self.wrapper
+        return function
+
+    def wrapper(self, *args, **kwargs):
+        if self._backend is not None and objects.backend != self._backend:
+            return self.wrong_backend_handler(*args, **kwargs)
+        else:
+            return self._function(*args, **kwargs)
+
+    def wrong_backend_handler(self, url):
+        """Show an error page about using the invalid backend."""
+        html = jinja.render('error.html',
+                            title="Error while opening qute:url",
+                            url=url.toDisplayString(),
+                            error='{} is not available with this '
+                                  'backend'.format(url.toDisplayString()),
+                            icon='')
+        return 'text/html', html
+
+
+def data_for_url(url):
+    """Get the data to show for the given URL.
+
+    Args:
+        url: The QUrl to show.
+
+    Return:
+        A (mimetype, data) tuple.
+    """
+    path = url.path()
+    host = url.host()
+    # A url like "qute:foo" is split as "scheme:path", not "scheme:host".
+    log.misc.debug("url: {}, path: {}, host {}".format(
+        url.toDisplayString(), path, host))
+    try:
+        handler = _HANDLERS[path]
+    except KeyError:
+        try:
+            handler = _HANDLERS[host]
+        except KeyError:
+            raise NoHandlerFound(url)
+    try:
+        mimetype, data = handler(url)
+    except OSError as e:
+        # FIXME:qtwebengine how to handle this?
+        raise QuteSchemeOSError(e)
+    except QuteSchemeError as e:
+        raise
+
+    assert mimetype is not None, url
+    if mimetype == 'text/html' and isinstance(data, str):
+        # We let handlers return HTML as text
+        data = data.encode('utf-8', errors='xmlcharrefreplace')
+
+    return mimetype, data
+
+
+@add_handler('bookmarks')
+def qute_bookmarks(_url):
+    """Handler for qute:bookmarks. Display all quickmarks / bookmarks."""
+    bookmarks = sorted(objreg.get('bookmark-manager').marks.items(),
+                       key=lambda x: x[1])  # Sort by title
+    quickmarks = sorted(objreg.get('quickmark-manager').marks.items(),
+                        key=lambda x: x[0])  # Sort by name
+
+    html = jinja.render('bookmarks.html',
+                        title='Bookmarks',
+                        bookmarks=bookmarks,
+                        quickmarks=quickmarks)
+    return 'text/html', html
+
+
+@add_handler('history')  # noqa
+def qute_history(url):
+    """Handler for qute:history. Display history."""
+    # Get current date from query parameter, if not given choose today.
+    curr_date = datetime.date.today()
+    try:
+        query_date = QUrlQuery(url).queryItemValue("date")
+        if query_date:
+            curr_date = datetime.datetime.strptime(query_date, "%Y-%m-%d")
+            curr_date = curr_date.date()
+    except ValueError:
+        log.misc.debug("Invalid date passed to qute:history: " + query_date)
+
+    one_day = datetime.timedelta(days=1)
+    next_date = curr_date + one_day
+    prev_date = curr_date - one_day
+
+    def history_iter(reverse):
+        """Iterate through the history and get items we're interested in."""
+        curr_timestamp = time.mktime(curr_date.timetuple())
+        history = objreg.get('web-history').history_dict.values()
+        if reverse:
+            history = reversed(history)
+
+        for item in history:
+            # If we can't apply the reverse performance trick below,
+            # at least continue as early as possible with old items.
+            # This gets us down from 550ms to 123ms with 500k old items on my
+            # machine.
+            if item.atime < curr_timestamp and not reverse:
+                continue
+
+            # Convert timestamp
+            try:
+                item_atime = datetime.datetime.fromtimestamp(item.atime)
+            except (ValueError, OSError, OverflowError):
+                log.misc.debug("Invalid timestamp {}.".format(item.atime))
+                continue
+
+            if reverse and item_atime.date() < curr_date:
+                # If we could reverse the history in-place, and this entry is
+                # older than today, only older entries will follow, so we can
+                # abort here.
+                return
+
+            # Skip items not on curr_date
+            # Skip redirects
+            # Skip qute:// links
+            is_internal = item.url.scheme() == 'qute'
+            is_not_today = item_atime.date() != curr_date
+            if item.redirect or is_internal or is_not_today:
+                continue
+
+            # Use item's url as title if there's no title.
+            item_url = item.url.toDisplayString()
+            item_title = item.title if item.title else item_url
+            display_atime = item_atime.strftime("%X")
+
+            yield (item_url, item_title, display_atime)
+
+    if sys.hexversion >= 0x03050000:
+        # On Python >= 3.5 we can reverse the ordereddict in-place and thus
+        # apply an additional performance improvement in history_iter.
+        # On my machine, this gets us down from 550ms to 72us with 500k old
+        # items.
+        history = list(history_iter(reverse=True))
+    else:
+        # On Python 3.4, we can't do that, so we'd need to copy the entire
+        # history to a list. There, filter first and then reverse it here.
+        history = reversed(list(history_iter(reverse=False)))
+
+    html = jinja.render('history.html',
+                        title='History',
+                        history=history,
+                        curr_date=curr_date,
+                        next_date=next_date,
+                        prev_date=prev_date,
+                        today=datetime.date.today())
+    return 'text/html', html
+
+
+@add_handler('pyeval')
+def qute_pyeval(_url):
+    """Handler for qute:pyeval."""
+    html = jinja.render('pre.html', title='pyeval', content=pyeval_output)
+    return 'text/html', html
+
+
+@add_handler('version')
+@add_handler('verizon')
+def qute_version(_url):
+    """Handler for qute:version."""
+    html = jinja.render('version.html', title='Version info',
+                        version=version.version(),
+                        copyright=qutebrowser.__copyright__)
+    return 'text/html', html
+
+
+@add_handler('plainlog')
+def qute_plainlog(url):
+    """Handler for qute:plainlog.
+
+    An optional query parameter specifies the minimum log level to print.
+    For example, qute://log?level=warning prints warnings and errors.
+    Level can be one of: vdebug, debug, info, warning, error, critical.
+    """
+    if log.ram_handler is None:
+        text = "Log output was disabled."
+    else:
+        try:
+            level = urllib.parse.parse_qs(url.query())['level'][0]
+        except KeyError:
+            level = 'vdebug'
+        text = log.ram_handler.dump_log(html=False, level=level)
+    html = jinja.render('pre.html', title='log', content=text)
+    return 'text/html', html
+
+
+@add_handler('log')
+def qute_log(url):
+    """Handler for qute:log.
+
+    An optional query parameter specifies the minimum log level to print.
+    For example, qute://log?level=warning prints warnings and errors.
+    Level can be one of: vdebug, debug, info, warning, error, critical.
+    """
+    if log.ram_handler is None:
+        html_log = None
+    else:
+        try:
+            level = urllib.parse.parse_qs(url.query())['level'][0]
+        except KeyError:
+            level = 'vdebug'
+        html_log = log.ram_handler.dump_log(html=True, level=level)
+
+    html = jinja.render('log.html', title='log', content=html_log)
+    return 'text/html', html
+
+
+@add_handler('gpl')
+def qute_gpl(_url):
+    """Handler for qute:gpl. Return HTML content as string."""
+    return 'text/html', utils.read_file('html/COPYING.html')
+
+
+@add_handler('help')
+def qute_help(url):
+    """Handler for qute:help."""
+    try:
+        utils.read_file('html/doc/index.html')
+    except OSError:
+        html = jinja.render(
+            'error.html',
+            title="Error while loading documentation",
+            url=url.toDisplayString(),
+            error="This most likely means the documentation was not generated "
+                  "properly. If you are running qutebrowser from the git "
+                  "repository, please run scripts/asciidoc2html.py. "
+                  "If you're running a released version this is a bug, please "
+                  "use :report to report it.",
+            icon='')
+        return 'text/html', html
+    urlpath = url.path()
+    if not urlpath or urlpath == '/':
+        urlpath = 'index.html'
+    else:
+        urlpath = urlpath.lstrip('/')
+    if not docutils.docs_up_to_date(urlpath):
+        message.error("Your documentation is outdated! Please re-run "
+                      "scripts/asciidoc2html.py.")
+    path = 'html/doc/{}'.format(urlpath)
+    if urlpath.endswith('.png'):
+        return 'image/png', utils.read_file(path, binary=True)
+    else:
+        data = utils.read_file(path)
+        return 'text/html', data

qutebrowser/browser/shared.py

@@ -0,0 +1,245 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Various utilities shared between webpage/webview subclasses."""
+
+import html
+
+import jinja2
+
+from qutebrowser.config import config
+from qutebrowser.utils import usertypes, message, log, objreg
+
+
+class CallSuper(Exception):
+
+    """Raised when the caller should call the superclass instead."""
+
+
+def custom_headers():
+    """Get the combined custom headers."""
+    headers = {}
+    dnt = b'1' if config.get('network', 'do-not-track') else b'0'
+    headers[b'DNT'] = dnt
+    headers[b'X-Do-Not-Track'] = dnt
+
+    config_headers = config.get('network', 'custom-headers')
+    if config_headers is not None:
+        for header, value in config_headers.items():
+            headers[header.encode('ascii')] = value.encode('ascii')
+
+    accept_language = config.get('network', 'accept-language')
+    if accept_language is not None:
+        headers[b'Accept-Language'] = accept_language.encode('ascii')
+
+    return sorted(headers.items())
+
+
+def authentication_required(url, authenticator, abort_on):
+    """Ask a prompt for an authentication question."""
+    realm = authenticator.realm()
+    if realm:
+        msg = '<b>{}</b> says:<br/>{}'.format(
+            html.escape(url.toDisplayString()), html.escape(realm))
+    else:
+        msg = '<b>{}</b> needs authentication'.format(
+            html.escape(url.toDisplayString()))
+    answer = message.ask(title="Authentication required", text=msg,
+                         mode=usertypes.PromptMode.user_pwd,
+                         abort_on=abort_on)
+    if answer is not None:
+        authenticator.setUser(answer.user)
+        authenticator.setPassword(answer.password)
+    return answer
+
+
+def javascript_confirm(url, js_msg, abort_on):
+    """Display a javascript confirm prompt."""
+    log.js.debug("confirm: {}".format(js_msg))
+    if config.get('ui', 'modal-js-dialog'):
+        raise CallSuper
+
+    msg = 'From <b>{}</b>:<br/>{}'.format(html.escape(url.toDisplayString()),
+                                          html.escape(js_msg))
+    ans = message.ask('Javascript confirm', msg,
+                      mode=usertypes.PromptMode.yesno,
+                      abort_on=abort_on)
+    return bool(ans)
+
+
+def javascript_prompt(url, js_msg, default, abort_on):
+    """Display a javascript prompt."""
+    log.js.debug("prompt: {}".format(js_msg))
+    if config.get('ui', 'modal-js-dialog'):
+        raise CallSuper
+    if config.get('content', 'ignore-javascript-prompt'):
+        return (False, "")
+
+    msg = '<b>{}</b> asks:<br/>{}'.format(html.escape(url.toDisplayString()),
+                                          html.escape(js_msg))
+    answer = message.ask('Javascript prompt', msg,
+                         mode=usertypes.PromptMode.text,
+                         default=default,
+                         abort_on=abort_on)
+
+    if answer is None:
+        return (False, "")
+    else:
+        return (True, answer)
+
+
+def javascript_alert(url, js_msg, abort_on):
+    """Display a javascript alert."""
+    log.js.debug("alert: {}".format(js_msg))
+    if config.get('ui', 'modal-js-dialog'):
+        raise CallSuper
+
+    if config.get('content', 'ignore-javascript-alert'):
+        return
+
+    msg = 'From <b>{}</b>:<br/>{}'.format(html.escape(url.toDisplayString()),
+                                          html.escape(js_msg))
+    message.ask('Javascript alert', msg, mode=usertypes.PromptMode.alert,
+                abort_on=abort_on)
+
+
+def ignore_certificate_errors(url, errors, abort_on):
+    """Display a certificate error question.
+
+    Args:
+        url: The URL the errors happened in
+        errors: A list of QSslErrors or QWebEngineCertificateErrors
+
+    Return:
+        True if the error should be ignored, False otherwise.
+    """
+    ssl_strict = config.get('network', 'ssl-strict')
+    log.webview.debug("Certificate errors {!r}, strict {}".format(
+        errors, ssl_strict))
+
+    for error in errors:
+        assert error.is_overridable(), repr(error)
+
+    if ssl_strict == 'ask':
+        err_template = jinja2.Template("""
+            Errors while loading <b>{{url.toDisplayString()}}</b>:<br/>
+            <ul>
+            {% for err in errors %}
+                <li>{{err}}</li>
+            {% endfor %}
+            </ul>
+        """.strip())
+        msg = err_template.render(url=url, errors=errors)
+
+        ignore = message.ask(title="Certificate errors - continue?", text=msg,
+                             mode=usertypes.PromptMode.yesno, default=False,
+                             abort_on=abort_on)
+        if ignore is None:
+            # prompt aborted
+            ignore = False
+        return ignore
+    elif ssl_strict is False:
+        log.webview.debug("ssl-strict is False, only warning about errors")
+        for err in errors:
+            # FIXME we might want to use warn here (non-fatal error)
+            # https://github.com/qutebrowser/qutebrowser/issues/114
+            message.error('Certificate error: {}'.format(err))
+        return True
+    elif ssl_strict is True:
+        return False
+    else:
+        raise ValueError("Invalid ssl_strict value {!r}".format(ssl_strict))
+    raise AssertionError("Not reached")
+
+
+def feature_permission(url, option, msg, yes_action, no_action, abort_on):
+    """Handle a feature permission request.
+
+    Args:
+        url: The URL the request was done for.
+        option: A (section, option) tuple for the option to check.
+        msg: A string like "show notifications"
+        yes_action: A callable to call if the request was approved
+        no_action: A callable to call if the request was denied
+        abort_on: A list of signals which interrupt the question.
+
+    Return:
+        The Question object if a question was asked, None otherwise.
+    """
+    config_val = config.get(*option)
+    if config_val == 'ask':
+        if url.isValid():
+            text = "Allow the website at <b>{}</b> to {}?".format(
+                html.escape(url.toDisplayString()), msg)
+        else:
+            text = "Allow the website to {}?".format(msg)
+
+        return message.confirm_async(
+            yes_action=yes_action, no_action=no_action,
+            cancel_action=no_action, abort_on=abort_on,
+            title='Permission request', text=text)
+    elif config_val:
+        yes_action()
+        return None
+    else:
+        no_action()
+        return None
+
+
+def get_tab(win_id, target):
+    """Get a tab widget for the given usertypes.ClickTarget.
+
+    Args:
+        win_id: The window ID to open new tabs in
+        target: A usertypes.ClickTarget
+    """
+    if target == usertypes.ClickTarget.tab:
+        win_id = win_id
+        bg_tab = False
+    elif target == usertypes.ClickTarget.tab_bg:
+        win_id = win_id
+        bg_tab = True
+    elif target == usertypes.ClickTarget.window:
+        from qutebrowser.mainwindow import mainwindow
+        window = mainwindow.MainWindow()
+        window.show()
+        win_id = window.win_id
+        bg_tab = False
+    else:
+        raise ValueError("Invalid ClickTarget {}".format(target))
+
+    tabbed_browser = objreg.get('tabbed-browser', scope='window',
+                                window=win_id)
+    return tabbed_browser.tabopen(url=None, background=bg_tab)
+
+
+def get_user_stylesheet():
+    """Get the combined user-stylesheet."""
+    filename = config.get('ui', 'user-stylesheet')
+
+    if filename is None:
+        css = ''
+    else:
+        with open(filename, 'r', encoding='utf-8') as f:
+            css = f.read()
+
+    if config.get('ui', 'hide-scrollbar'):
+        css += '\nhtml > ::-webkit-scrollbar { width: 0px; height: 0px; }'
+
+    return css

qutebrowser/browser/urlmarks.py

@@ -26,6 +26,7 @@ to a file on shutdown, so it makes sense to keep them as strings here.
 """
 
 import os
+import html
 import os.path
 import functools
 import collections
@@ -33,7 +34,7 @@ import collections
 from PyQt5.QtCore import pyqtSignal, QUrl, QObject
 
 from qutebrowser.utils import (message, usertypes, qtutils, urlutils,
-                               standarddir, objreg)
+                               standarddir, objreg, log)
 from qutebrowser.commands import cmdutils
 from qutebrowser.misc import lineparser
 
@@ -72,8 +73,7 @@ class UrlMarkManager(QObject):
 
     Attributes:
         marks: An OrderedDict of all quickmarks/bookmarks.
-        _lineparser: The LineParser used for the marks, or None
-                     (when qutebrowser is started with -c '').
+        _lineparser: The LineParser used for the marks
 
     Signals:
         changed: Emitted when anything changed.
@@ -90,10 +90,6 @@ class UrlMarkManager(QObject):
         super().__init__(parent)
 
         self.marks = collections.OrderedDict()
-        self._lineparser = None
-
-        if standarddir.config() is None:
-            return
 
         self._init_lineparser()
         for line in self._lineparser:
@@ -114,10 +110,8 @@ class UrlMarkManager(QObject):
 
     def save(self):
         """Save the marks to disk."""
-        if self._lineparser is not None:
-            self._lineparser.data = [' '.join(tpl)
-                                     for tpl in self.marks.items()]
-            self._lineparser.save()
+        self._lineparser.data = [' '.join(tpl) for tpl in self.marks.items()]
+        self._lineparser.save()
 
     def delete(self, key):
         """Delete a quickmark/bookmark.
@@ -155,45 +149,44 @@ class QuickmarkManager(UrlMarkManager):
         try:
             key, url = line.rsplit(maxsplit=1)
         except ValueError:
-            message.error('current', "Invalid quickmark '{}'".format(line))
+            message.error("Invalid quickmark '{}'".format(line))
         else:
             self.marks[key] = url
 
-    def prompt_save(self, win_id, url):
+    def prompt_save(self, url):
         """Prompt for a new quickmark name to be added and add it.
 
         Args:
-            win_id: The current window ID.
             url: The quickmark url as a QUrl.
         """
         if not url.isValid():
-            urlutils.invalid_url_error(win_id, url, "save quickmark")
+            urlutils.invalid_url_error(url, "save quickmark")
             return
         urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded)
         message.ask_async(
-            win_id, "Add quickmark:", usertypes.PromptMode.text,
-            functools.partial(self.quickmark_add, win_id, urlstr))
+            "Add quickmark:", usertypes.PromptMode.text,
+            functools.partial(self.quickmark_add, urlstr),
+            text="Please enter a quickmark name for<br/><b>{}</b>".format(
+                html.escape(url.toDisplayString())))
 
     @cmdutils.register(instance='quickmark-manager')
-    @cmdutils.argument('win_id', win_id=True)
-    def quickmark_add(self, win_id, url, name):
+    def quickmark_add(self, url, name):
         """Add a new quickmark.
 
         You can view all saved quickmarks on the
         link:qute://bookmarks[bookmarks page].
 
         Args:
-            win_id: The window ID to display the errors in.
             url: The url to add as quickmark.
             name: The name for the new quickmark.
         """
         # We don't raise cmdexc.CommandError here as this can be called async
         # via prompt_save.
         if not name:
-            message.error(win_id, "Can't set mark with empty name!")
+            message.error("Can't set mark with empty name!")
             return
         if not url:
-            message.error(win_id, "Can't set mark with empty URL!")
+            message.error("Can't set mark with empty URL!")
             return
 
         def set_mark():
@@ -201,10 +194,12 @@ class QuickmarkManager(UrlMarkManager):
             self.marks[name] = url
             self.changed.emit()
             self.added.emit(name, url)
+            log.misc.debug("Added quickmark {} for {}".format(name, url))
 
         if name in self.marks:
             message.confirm_async(
-                win_id, "Override existing quickmark?", set_mark, default=True)
+                title="Override existing quickmark?",
+                yes_action=set_mark, default=True)
         else:
             set_mark()
 
@@ -272,12 +267,18 @@ class BookmarkManager(UrlMarkManager):
         elif len(parts) == 1:
             self.marks[parts[0]] = ''
 
-    def add(self, url, title):
+    def add(self, url, title, *, toggle=False):
         """Add a new bookmark.
 
         Args:
             url: The url to add as bookmark.
             title: The title for the new bookmark.
+            toggle: remove the bookmark instead of raising an error if it
+                    already exists.
+
+        Return:
+            True if the bookmark was added, and False if it was
+            removed (only possible if toggle is True).
         """
         if not url.isValid():
             errstr = urlutils.get_errstring(url)
@@ -286,8 +287,13 @@ class BookmarkManager(UrlMarkManager):
         urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded)
 
         if urlstr in self.marks:
-            raise AlreadyExistsError("Bookmark already exists!")
+            if toggle:
+                del self.marks[urlstr]
+                return False
+            else:
+                raise AlreadyExistsError("Bookmark already exists!")
         else:
             self.marks[urlstr] = title
             self.changed.emit()
             self.added.emit(title, urlstr)
+            return True

qutebrowser/browser/webelem.py

@@ -0,0 +1,437 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Generic web element related code.
+
+Module attributes:
+    Group: Enum for different kinds of groups.
+    SELECTORS: CSS selectors for different groups of elements.
+    FILTERS: A dictionary of filter functions for the modes.
+             The filter for "links" filters javascript:-links and a-tags
+             without "href".
+"""
+
+import collections.abc
+
+from PyQt5.QtCore import QUrl, Qt, QEvent, QTimer
+from PyQt5.QtGui import QMouseEvent
+
+from qutebrowser.config import config
+from qutebrowser.keyinput import modeman
+from qutebrowser.utils import log, usertypes, utils, qtutils, objreg
+
+
+Group = usertypes.enum('Group', ['all', 'links', 'images', 'url', 'prevnext',
+                                 'inputs'])
+
+
+SELECTORS = {
+    Group.all: ('a, area, textarea, select, input:not([type=hidden]), button, '
+                'frame, iframe, link, [onclick], [onmousedown], [role=link], '
+                '[role=option], [role=button], img'),
+    Group.links: 'a, area, link, [role=link]',
+    Group.images: 'img',
+    Group.url: '[src], [href]',
+    Group.prevnext: 'a, area, button, link, [role=button]',
+    Group.inputs: ('input[type=text], input[type=email], input[type=url], '
+                   'input[type=tel], input[type=number], '
+                   'input[type=password], input[type=search], '
+                   'input:not([type]), textarea'),
+}
+
+
+def filter_links(elem):
+    return 'href' in elem and QUrl(elem['href']).scheme() != 'javascript'
+
+
+FILTERS = {
+    Group.links: filter_links,
+    Group.prevnext: filter_links,
+}
+
+
+class Error(Exception):
+
+    """Base class for WebElement errors."""
+
+    pass
+
+
+class AbstractWebElement(collections.abc.MutableMapping):
+
+    """A wrapper around QtWebKit/QtWebEngine web element.
+
+    Attributes:
+        tab: The tab associated with this element.
+    """
+
+    def __init__(self, tab):
+        self._tab = tab
+
+    def __eq__(self, other):
+        raise NotImplementedError
+
+    def __str__(self):
+        raise NotImplementedError
+
+    def __getitem__(self, key):
+        raise NotImplementedError
+
+    def __setitem__(self, key, val):
+        raise NotImplementedError
+
+    def __delitem__(self, key):
+        raise NotImplementedError
+
+    def __iter__(self):
+        raise NotImplementedError
+
+    def __len__(self):
+        raise NotImplementedError
+
+    def __repr__(self):
+        try:
+            html = utils.compact_text(self.outer_xml(), 500)
+        except Error:
+            html = None
+        return utils.get_repr(self, html=html)
+
+    def has_frame(self):
+        """Check if this element has a valid frame attached."""
+        raise NotImplementedError
+
+    def geometry(self):
+        """Get the geometry for this element."""
+        raise NotImplementedError
+
+    def style_property(self, name, *, strategy):
+        """Get the element style resolved with the given strategy."""
+        raise NotImplementedError
+
+    def classes(self):
+        """Get a list of classes assigned to this element."""
+        raise NotImplementedError
+
+    def tag_name(self):
+        """Get the tag name of this element.
+
+        The returned name will always be lower-case.
+        """
+        raise NotImplementedError
+
+    def outer_xml(self):
+        """Get the full HTML representation of this element."""
+        raise NotImplementedError
+
+    def value(self):
+        """Get the value attribute for this element, or None."""
+        raise NotImplementedError
+
+    def set_value(self, value):
+        """Set the element value."""
+        raise NotImplementedError
+
+    def insert_text(self, text):
+        """Insert the given text into the element."""
+        raise NotImplementedError
+
+    def rect_on_view(self, *, elem_geometry=None, no_js=False):
+        """Get the geometry of the element relative to the webview.
+
+        Uses the getClientRects() JavaScript method to obtain the collection of
+        rectangles containing the element and returns the first rectangle which
+        is large enough (larger than 1px times 1px). If all rectangles returned
+        by getClientRects() are too small, falls back to elem.rect_on_view().
+
+        Skipping of small rectangles is due to <a> elements containing other
+        elements with "display:block" style, see
+        https://github.com/qutebrowser/qutebrowser/issues/1298
+
+        Args:
+            elem_geometry: The geometry of the element, or None.
+                           Calling QWebElement::geometry is rather expensive so
+                           we want to avoid doing it twice.
+            no_js: Fall back to the Python implementation
+        """
+        raise NotImplementedError
+
+    def is_writable(self):
+        """Check whether an element is writable."""
+        return not ('disabled' in self or 'readonly' in self)
+
+    def is_content_editable(self):
+        """Check if an element has a contenteditable attribute.
+
+        Args:
+            elem: The QWebElement to check.
+
+        Return:
+            True if the element has a contenteditable attribute,
+            False otherwise.
+        """
+        try:
+            return self['contenteditable'].lower() not in ['false', 'inherit']
+        except KeyError:
+            return False
+
+    def _is_editable_object(self):
+        """Check if an object-element is editable."""
+        if 'type' not in self:
+            log.webelem.debug("<object> without type clicked...")
+            return False
+        objtype = self['type'].lower()
+        if objtype.startswith('application/') or 'classid' in self:
+            # Let's hope flash/java stuff has an application/* mimetype OR
+            # at least a classid attribute. Oh, and let's hope images/...
+            # DON'T have a classid attribute. HTML sucks.
+            log.webelem.debug("<object type='{}'> clicked.".format(objtype))
+            return config.get('input', 'insert-mode-on-plugins')
+        else:
+            # Image/Audio/...
+            return False
+
+    def _is_editable_input(self):
+        """Check if an input-element is editable.
+
+        Return:
+            True if the element is editable, False otherwise.
+        """
+        try:
+            objtype = self['type'].lower()
+        except KeyError:
+            return self.is_writable()
+        else:
+            if objtype in ['text', 'email', 'url', 'tel', 'number', 'password',
+                           'search']:
+                return self.is_writable()
+            else:
+                return False
+
+    def _is_editable_classes(self):
+        """Check if an element is editable based on its classes.
+
+        Return:
+            True if the element is editable, False otherwise.
+        """
+        # Beginnings of div-classes which are actually some kind of editor.
+        classes = {
+            'div': ['CodeMirror',  # Javascript editor over a textarea
+                    'kix-',  # Google Docs editor
+                    'ace_'],  # http://ace.c9.io/
+            'pre': ['CodeMirror'],
+        }
+        relevant_classes = classes[self.tag_name()]
+        for klass in self.classes():
+            if any([klass.strip().startswith(e) for e in relevant_classes]):
+                return True
+        return False
+
+    def is_editable(self, strict=False):
+        """Check whether we should switch to insert mode for this element.
+
+        Args:
+            strict: Whether to do stricter checking so only fields where we can
+                    get the value match, for use with the :editor command.
+
+        Return:
+            True if we should switch to insert mode, False otherwise.
+        """
+        roles = ('combobox', 'textbox')
+        log.webelem.debug("Checking if element is editable: {}".format(
+            repr(self)))
+        tag = self.tag_name()
+        if self.is_content_editable() and self.is_writable():
+            return True
+        elif self.get('role', None) in roles and self.is_writable():
+            return True
+        elif tag == 'input':
+            return self._is_editable_input()
+        elif tag == 'textarea':
+            return self.is_writable()
+        elif tag in ['embed', 'applet']:
+            # Flash/Java/...
+            return config.get('input', 'insert-mode-on-plugins') and not strict
+        elif tag == 'object':
+            return self._is_editable_object() and not strict
+        elif tag in ['div', 'pre']:
+            return self._is_editable_classes() and not strict
+        return False
+
+    def is_text_input(self):
+        """Check if this element is some kind of text box."""
+        roles = ('combobox', 'textbox')
+        tag = self.tag_name()
+        return self.get('role', None) in roles or tag in ['input', 'textarea']
+
+    def remove_blank_target(self):
+        """Remove target from link."""
+        raise NotImplementedError
+
+    def resolve_url(self, baseurl):
+        """Resolve the URL in the element's src/href attribute.
+
+        Args:
+            baseurl: The URL to base relative URLs on as QUrl.
+
+        Return:
+            A QUrl with the absolute URL, or None.
+        """
+        if baseurl.isRelative():
+            raise ValueError("Need an absolute base URL!")
+
+        for attr in ['href', 'src']:
+            if attr in self:
+                text = self[attr].strip()
+                break
+        else:
+            return None
+
+        url = QUrl(text)
+        if not url.isValid():
+            return None
+        if url.isRelative():
+            url = baseurl.resolved(url)
+        qtutils.ensure_valid(url)
+        return url
+
+    def _mouse_pos(self):
+        """Get the position to click/hover."""
+        # Click the center of the largest square fitting into the top/left
+        # corner of the rectangle, this will help if part of the <a> element
+        # is hidden behind other elements
+        # https://github.com/qutebrowser/qutebrowser/issues/1005
+        rect = self.rect_on_view()
+        if rect.width() > rect.height():
+            rect.setWidth(rect.height())
+        else:
+            rect.setHeight(rect.width())
+        pos = rect.center()
+        if pos.x() < 0 or pos.y() < 0:
+            raise Error("Element position is out of view!")
+        return pos
+
+    def _click_fake_event(self, click_target):
+        """Send a fake click event to the element."""
+        pos = self._mouse_pos()
+
+        log.webelem.debug("Sending fake click to {!r} at position {} with "
+                          "target {}".format(self, pos, click_target))
+
+        modifiers = {
+            usertypes.ClickTarget.normal: Qt.NoModifier,
+            usertypes.ClickTarget.window: Qt.AltModifier | Qt.ShiftModifier,
+            usertypes.ClickTarget.tab: Qt.ControlModifier,
+            usertypes.ClickTarget.tab_bg: Qt.ControlModifier,
+        }
+        if config.get('tabs', 'background-tabs'):
+            modifiers[usertypes.ClickTarget.tab] |= Qt.ShiftModifier
+        else:
+            modifiers[usertypes.ClickTarget.tab_bg] |= Qt.ShiftModifier
+
+        events = [
+            QMouseEvent(QEvent.MouseMove, pos, Qt.NoButton, Qt.NoButton,
+                        Qt.NoModifier),
+            QMouseEvent(QEvent.MouseButtonPress, pos, Qt.LeftButton,
+                        Qt.LeftButton, modifiers[click_target]),
+            QMouseEvent(QEvent.MouseButtonRelease, pos, Qt.LeftButton,
+                        Qt.NoButton, modifiers[click_target]),
+        ]
+
+        for evt in events:
+            self._tab.send_event(evt)
+
+        def after_click():
+            """Move cursor to end after clicking."""
+            if self.is_text_input() and self.is_editable():
+                self._tab.caret.move_to_end_of_document()
+        QTimer.singleShot(0, after_click)
+
+    def _click_editable(self, click_target):
+        """Fake a click on an editable input field."""
+        raise NotImplementedError
+
+    def _click_js(self, click_target):
+        """Fake a click by using the JS .click() method."""
+        raise NotImplementedError
+
+    def _click_href(self, click_target):
+        """Fake a click on an element with a href by opening the link."""
+        baseurl = self._tab.url()
+        url = self.resolve_url(baseurl)
+        if url is None:
+            self._click_fake_event(click_target)
+            return
+
+        if click_target in [usertypes.ClickTarget.tab,
+                            usertypes.ClickTarget.tab_bg]:
+            background = click_target == usertypes.ClickTarget.tab_bg
+            tabbed_browser = objreg.get('tabbed-browser', scope='window',
+                                        window=self._tab.win_id)
+            tabbed_browser.tabopen(url, background=background)
+        elif click_target == usertypes.ClickTarget.window:
+            from qutebrowser.mainwindow import mainwindow
+            window = mainwindow.MainWindow()
+            window.show()
+            window.tabbed_browser.tabopen(url)
+        else:
+            raise ValueError("Unknown ClickTarget {}".format(click_target))
+
+    def click(self, click_target, *, force_event=False):
+        """Simulate a click on the element.
+
+        Args:
+            click_target: A usertypes.ClickTarget member, what kind of click
+                          to simulate.
+            force_event: Force generating a fake mouse event.
+        """
+        log.webelem.debug("Clicking {!r} with click_target {}, force_event {}"
+                          .format(self, click_target, force_event))
+
+        if force_event:
+            self._click_fake_event(click_target)
+            return
+
+        href_tags = ['a', 'area', 'link']
+        if click_target == usertypes.ClickTarget.normal:
+            if self.tag_name() in href_tags:
+                log.webelem.debug("Clicking via JS click()")
+                self._click_js(click_target)
+            elif self.is_editable(strict=True):
+                log.webelem.debug("Clicking via JS focus()")
+                self._click_editable(click_target)
+                modeman.enter(self._tab.win_id, usertypes.KeyMode.insert,
+                              'clicking input')
+            else:
+                self._click_fake_event(click_target)
+        elif click_target in [usertypes.ClickTarget.tab,
+                              usertypes.ClickTarget.tab_bg,
+                              usertypes.ClickTarget.window]:
+            if self.tag_name() in href_tags:
+                self._click_href(click_target)
+            else:
+                self._click_fake_event(click_target)
+        else:
+            raise ValueError("Unknown ClickTarget {}".format(click_target))
+
+    def hover(self):
+        """Simulate a mouse hover over the element."""
+        pos = self._mouse_pos()
+        event = QMouseEvent(QEvent.MouseMove, pos, Qt.NoButton, Qt.NoButton,
+                            Qt.NoModifier)
+        self._tab.send_event(event)

qutebrowser/browser/webengine/certificateerror.py

@@ -0,0 +1,43 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Wrapper over a QWebEngineCertificateError."""
+
+# pylint: disable=no-name-in-module,import-error,useless-suppression
+from PyQt5.QtWebEngineWidgets import QWebEngineCertificateError
+# pylint: enable=no-name-in-module,import-error,useless-suppression
+
+from qutebrowser.utils import usertypes, utils, debug
+
+
+class CertificateErrorWrapper(usertypes.AbstractCertificateErrorWrapper):
+
+    """A wrapper over a QWebEngineCertificateError."""
+
+    def __str__(self):
+        return self._error.errorDescription()
+
+    def __repr__(self):
+        return utils.get_repr(
+            self, error=debug.qenum_key(QWebEngineCertificateError,
+                                        self._error.error()),
+            string=str(self))
+
+    def is_overridable(self):
+        return self._error.isOverridable()

qutebrowser/browser/webengine/interceptor.py

@@ -0,0 +1,71 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""A request interceptor taking care of adblocking and custom headers."""
+
+# pylint: disable=no-name-in-module,import-error,useless-suppression
+from PyQt5.QtWebEngineCore import QWebEngineUrlRequestInterceptor
+# pylint: enable=no-name-in-module,import-error,useless-suppression
+
+from qutebrowser.config import config
+from qutebrowser.browser import shared
+from qutebrowser.utils import utils, log
+
+
+class RequestInterceptor(QWebEngineUrlRequestInterceptor):
+
+    """Handle ad blocking and custom headers."""
+
+    def __init__(self, host_blocker, parent=None):
+        super().__init__(parent)
+        self._host_blocker = host_blocker
+
+    def install(self, profile):
+        """Install the interceptor on the given QWebEngineProfile."""
+        profile.setRequestInterceptor(self)
+
+    # Gets called in the IO thread -> showing crash window will fail
+    @utils.prevent_exceptions(None)
+    def interceptRequest(self, info):
+        """Handle the given request.
+
+        Reimplementing this virtual function and setting the interceptor on a
+        profile makes it possible to intercept URL requests. This function is
+        executed on the IO thread, and therefore running long tasks here will
+        block networking.
+
+        info contains the information about the URL request and will track
+        internally whether its members have been altered.
+
+        Args:
+            info: QWebEngineUrlRequestInfo &info
+        """
+        # FIXME:qtwebengine only block ads for NavigationTypeOther?
+        if (bytes(info.requestMethod()) == b'GET' and
+                self._host_blocker.is_blocked(info.requestUrl())):
+            log.webview.info("Request to {} blocked by host blocker.".format(
+                info.requestUrl().host()))
+            info.block(True)
+
+        for header, value in shared.custom_headers():
+            info.setHttpHeader(header, value)
+
+        user_agent = config.get('network', 'user-agent')
+        if user_agent is not None:
+            info.setHttpHeader(b'User-Agent', user_agent.encode('ascii'))

qutebrowser/browser/webengine/tabhistory.py

@@ -0,0 +1,117 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""QWebHistory serializer for QtWebEngine."""
+
+import time
+
+from PyQt5.QtCore import QByteArray, QDataStream, QIODevice, QUrl
+
+from qutebrowser.utils import qtutils
+
+
+HISTORY_STREAM_VERSION = 3
+
+
+def _serialize_item(item, stream):
+    """Serialize a single WebHistoryItem into a QDataStream.
+
+    Args:
+        item: The WebHistoryItem to write.
+        stream: The QDataStream to write to.
+    """
+    ### Thanks to Otter Browser:
+    ### https://github.com/OtterBrowser/otter-browser/blob/v0.9.10/src/modules/backends/web/qtwebengine/QtWebEngineWebWidget.cpp#L1210
+    ### src/core/web_contents_adapter.cpp serializeNavigationHistory
+    ## toQt(entry->GetVirtualURL());
+    qtutils.serialize_stream(stream, item.url)
+    ## toQt(entry->GetTitle());
+    stream.writeQString(item.title)
+    ## QByteArray(encodedPageState.data(), encodedPageState.size());
+    qtutils.serialize_stream(stream, QByteArray())
+    ## static_cast<qint32>(entry->GetTransitionType());
+    # chromium/ui/base/page_transition_types.h
+    stream.writeInt32(0)  # PAGE_TRANSITION_LINK
+    ## entry->GetHasPostData();
+    stream.writeBool(False)
+    ## toQt(entry->GetReferrer().url);
+    qtutils.serialize_stream(stream, QUrl())
+    ## static_cast<qint32>(entry->GetReferrer().policy);
+    # chromium/third_party/WebKit/public/platform/WebReferrerPolicy.h
+    stream.writeInt32(0)  # WebReferrerPolicyAlways
+    ## toQt(entry->GetOriginalRequestURL());
+    qtutils.serialize_stream(stream, item.original_url)
+    ## entry->GetIsOverridingUserAgent();
+    stream.writeBool(False)
+    ## static_cast<qint64>(entry->GetTimestamp().ToInternalValue());
+    stream.writeInt64(int(time.time()))
+    ## entry->GetHttpStatusCode();
+    stream.writeInt(200)
+
+
+def serialize(items):
+    """Serialize a list of QWebHistoryItems to a data stream.
+
+    Args:
+        items: An iterable of WebHistoryItems.
+
+    Return:
+        A (stream, data, user_data) tuple.
+            stream: The reset QDataStream.
+            data: The QByteArray with the raw data.
+            cur_user_data: The user data for the current item or None.
+
+    Warning:
+        If 'data' goes out of scope, reading from 'stream' will result in a
+        segfault!
+    """
+    data = QByteArray()
+    stream = QDataStream(data, QIODevice.ReadWrite)
+    cur_user_data = None
+
+    current_idx = None
+
+    for i, item in enumerate(items):
+        if item.active:
+            if current_idx is not None:
+                raise ValueError("Multiple active items ({} and {}) "
+                                 "found!".format(current_idx, i))
+            current_idx = i
+            cur_user_data = item.user_data
+
+    if items:
+        if current_idx is None:
+            raise ValueError("No active item found!")
+    else:
+        current_idx = -1
+
+    ### src/core/web_contents_adapter.cpp serializeNavigationHistory
+    # kHistoryStreamVersion
+    stream.writeInt(HISTORY_STREAM_VERSION)
+    # count
+    stream.writeInt(len(items))
+    # currentIndex
+    stream.writeInt(current_idx)
+
+    for item in items:
+        _serialize_item(item, stream)
+
+    stream.device().reset()
+    qtutils.check_qdatastream(stream)
+    return stream, data, cur_user_data

qutebrowser/browser/webengine/webenginedownloads.py

@@ -0,0 +1,210 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""QtWebEngine specific code for downloads."""
+
+import re
+import os.path
+import urllib
+import functools
+
+from PyQt5.QtCore import pyqtSlot, Qt
+# pylint: disable=no-name-in-module,import-error,useless-suppression
+from PyQt5.QtWebEngineWidgets import QWebEngineDownloadItem
+# pylint: enable=no-name-in-module,import-error,useless-suppression
+
+from qutebrowser.browser import downloads
+from qutebrowser.utils import debug, usertypes, message, log, qtutils
+
+
+class DownloadItem(downloads.AbstractDownloadItem):
+
+    """A wrapper over a QWebEngineDownloadItem.
+
+    Attributes:
+        _qt_item: The wrapped item.
+    """
+
+    def __init__(self, qt_item, parent=None):
+        super().__init__(parent)
+        self._qt_item = qt_item
+        qt_item.downloadProgress.connect(self.stats.on_download_progress)
+        qt_item.stateChanged.connect(self._on_state_changed)
+
+    def _is_page_download(self):
+        """Check if this item is a page (i.e. mhtml) download."""
+        return (self._qt_item.savePageFormat() !=
+                QWebEngineDownloadItem.UnknownSaveFormat)
+
+    @pyqtSlot(QWebEngineDownloadItem.DownloadState)
+    def _on_state_changed(self, state):
+        state_name = debug.qenum_key(QWebEngineDownloadItem, state)
+        log.downloads.debug("State for {!r} changed to {}".format(
+            self, state_name))
+
+        if state == QWebEngineDownloadItem.DownloadRequested:
+            pass
+        elif state == QWebEngineDownloadItem.DownloadInProgress:
+            pass
+        elif state == QWebEngineDownloadItem.DownloadCompleted:
+            log.downloads.debug("Download {} finished".format(self.basename))
+            if self._is_page_download():
+                # Same logging as QtWebKit mhtml downloads.
+                log.downloads.debug("File successfully written.")
+            self.successful = True
+            self.done = True
+            self.finished.emit()
+            self.stats.finish()
+        elif state == QWebEngineDownloadItem.DownloadCancelled:
+            self.successful = False
+            self.done = True
+            self.cancelled.emit()
+            self.stats.finish()
+        elif state == QWebEngineDownloadItem.DownloadInterrupted:
+            self.successful = False
+            self.done = True
+            # https://bugreports.qt.io/browse/QTBUG-56839
+            self.error.emit("Download failed")
+            self.stats.finish()
+        else:
+            raise ValueError("_on_state_changed was called with unknown state "
+                             "{}".format(state_name))
+
+    def _do_die(self):
+        self._qt_item.downloadProgress.disconnect()
+        self._qt_item.cancel()
+
+    def _do_cancel(self):
+        self._qt_item.cancel()
+
+    def retry(self):
+        # https://bugreports.qt.io/browse/QTBUG-56840
+        raise downloads.UnsupportedOperationError
+
+    def _get_open_filename(self):
+        return self._filename
+
+    def _set_fileobj(self, fileobj):
+        raise downloads.UnsupportedOperationError
+
+    def _set_tempfile(self, fileobj):
+        self._set_filename(fileobj.name, force_overwrite=True,
+                           remember_directory=False)
+
+    def _ensure_can_set_filename(self, filename):
+        state = self._qt_item.state()
+        if state != QWebEngineDownloadItem.DownloadRequested:
+            state_name = debug.qenum_key(QWebEngineDownloadItem, state)
+            raise ValueError("Trying to set filename {} on {!r} which is "
+                             "state {} (not in requested state)!".format(
+                                 filename, self, state_name))
+
+    def _ask_confirm_question(self, title, msg):
+        no_action = functools.partial(self.cancel, remove_data=False)
+        question = usertypes.Question()
+        question.title = title
+        question.text = msg
+        question.mode = usertypes.PromptMode.yesno
+        question.answered_yes.connect(self._after_set_filename)
+        question.answered_no.connect(no_action)
+        question.cancelled.connect(no_action)
+        self.cancelled.connect(question.abort)
+        self.error.connect(question.abort)
+        message.global_bridge.ask(question, blocking=True)
+
+    def _after_set_filename(self):
+        self._qt_item.setPath(self._filename)
+        self._qt_item.accept()
+
+
+def _get_suggested_filename(path):
+    """Convert a path we got from chromium to a suggested filename.
+
+    Chromium thinks we want to download stuff to ~/Download, so even if we
+    don't, we get downloads with a suffix like (1) for files existing there.
+
+    We simply strip the suffix off via regex.
+
+    See https://bugreports.qt.io/browse/QTBUG-56978
+    """
+    filename = os.path.basename(path)
+    filename = re.sub(r'\([0-9]+\)$', '', filename)
+    if not qtutils.version_check('5.8.1'):
+        # https://bugreports.qt.io/browse/QTBUG-58155
+        filename = urllib.parse.unquote(filename)
+        # Doing basename a *second* time because there could be a %2F in
+        # there...
+        filename = os.path.basename(filename)
+    return filename
+
+
+class DownloadManager(downloads.AbstractDownloadManager):
+
+    """Manager for currently running downloads.
+
+    Attributes:
+        _mhtml_target: DownloadTarget for the next MHTML download.
+    """
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._mhtml_target = None
+
+    def install(self, profile):
+        """Set up the download manager on a QWebEngineProfile."""
+        profile.downloadRequested.connect(self.handle_download,
+                                          Qt.DirectConnection)
+
+    @pyqtSlot(QWebEngineDownloadItem)
+    def handle_download(self, qt_item):
+        """Start a download coming from a QWebEngineProfile."""
+        suggested_filename = _get_suggested_filename(qt_item.path())
+
+        download = DownloadItem(qt_item)
+        self._init_item(download, auto_remove=False,
+                        suggested_filename=suggested_filename)
+
+        if self._mhtml_target is not None:
+            download.set_target(self._mhtml_target)
+            self._mhtml_target = None
+            return
+
+        filename = downloads.immediate_download_path()
+        if filename is not None:
+            # User doesn't want to be asked, so just use the download_dir
+            target = downloads.FileDownloadTarget(filename)
+            download.set_target(target)
+            return
+
+        # Ask the user for a filename - needs to be blocking!
+        question = downloads.get_filename_question(
+            suggested_filename=suggested_filename, url=qt_item.url(),
+            parent=self)
+        self._init_filename_question(question, download)
+
+        message.global_bridge.ask(question, blocking=True)
+        # The filename is set via the question.answered signal, connected in
+        # _init_filename_question.
+
+    def get_mhtml(self, tab, target):
+        """Download the given tab as mhtml to the given target."""
+        assert tab.backend == usertypes.Backend.QtWebEngine
+        assert self._mhtml_target is None, self._mhtml_target
+        self._mhtml_target = target
+        tab.action.save_page()

qutebrowser/browser/webengine/webengineelem.py

@@ -0,0 +1,193 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+# FIXME:qtwebengine remove this once the stubs are gone
+# pylint: disable=unused-variable
+
+"""QtWebEngine specific part of the web element API."""
+
+from PyQt5.QtCore import QRect, Qt, QPoint, QEventLoop
+from PyQt5.QtGui import QMouseEvent
+from PyQt5.QtWidgets import QApplication
+# pylint: disable=no-name-in-module,import-error,useless-suppression
+from PyQt5.QtWebEngineWidgets import QWebEngineSettings
+# pylint: enable=no-name-in-module,import-error,useless-suppression
+
+from qutebrowser.utils import log, javascript
+from qutebrowser.browser import webelem
+
+
+class WebEngineElement(webelem.AbstractWebElement):
+
+    """A web element for QtWebEngine, using JS under the hood."""
+
+    def __init__(self, js_dict, tab):
+        super().__init__(tab)
+        self._id = js_dict['id']
+        self._js_dict = js_dict
+
+    def __str__(self):
+        return self._js_dict.get('text', '')
+
+    def __eq__(self, other):
+        if not isinstance(other, WebEngineElement):
+            return NotImplemented
+        return self._id == other._id  # pylint: disable=protected-access
+
+    def __getitem__(self, key):
+        attrs = self._js_dict['attributes']
+        return attrs[key]
+
+    def __setitem__(self, key, val):
+        self._js_dict['attributes'][key] = val
+        js_code = javascript.assemble('webelem', 'set_attribute', self._id,
+            key, val)
+        self._tab.run_js_async(js_code)
+
+    def __delitem__(self, key):
+        log.stub()
+
+    def __iter__(self):
+        return iter(self._js_dict['attributes'])
+
+    def __len__(self):
+        return len(self._js_dict['attributes'])
+
+    def has_frame(self):
+        return True
+
+    def geometry(self):
+        log.stub()
+        return QRect()
+
+    def style_property(self, name, *, strategy):
+        log.stub()
+        return ''
+
+    def classes(self):
+        """Get a list of classes assigned to this element."""
+        return self._js_dict['class_name'].split()
+
+    def tag_name(self):
+        """Get the tag name of this element.
+
+        The returned name will always be lower-case.
+        """
+        return self._js_dict['tag_name'].lower()
+
+    def outer_xml(self):
+        """Get the full HTML representation of this element."""
+        return self._js_dict['outer_xml']
+
+    def value(self):
+        return self._js_dict.get('value', None)
+
+    def set_value(self, value):
+        js_code = javascript.assemble('webelem', 'set_value', self._id, value)
+        self._tab.run_js_async(js_code)
+
+    def insert_text(self, text):
+        if not self.is_editable(strict=True):
+            raise webelem.Error("Element is not editable!")
+        log.webelem.debug("Inserting text into element {!r}".format(self))
+        js_code = javascript.assemble('webelem', 'insert_text', self._id, text)
+        self._tab.run_js_async(js_code)
+
+    def rect_on_view(self, *, elem_geometry=None, no_js=False):
+        """Get the geometry of the element relative to the webview.
+
+        Skipping of small rectangles is due to <a> elements containing other
+        elements with "display:block" style, see
+        https://github.com/qutebrowser/qutebrowser/issues/1298
+
+        Args:
+            elem_geometry: The geometry of the element, or None.
+                           Calling QWebElement::geometry is rather expensive so
+                           we want to avoid doing it twice.
+            no_js: Fall back to the Python implementation
+        """
+        rects = self._js_dict['rects']
+        for rect in rects:
+            # FIXME:qtwebengine
+            # width = rect.get("width", 0)
+            # height = rect.get("height", 0)
+            width = rect['width']
+            height = rect['height']
+            left = rect['left']
+            top = rect['top']
+            if width > 1 and height > 1:
+                # Fix coordinates according to zoom level
+                # We're not checking for zoom-text-only here as that doesn't
+                # exist for QtWebEngine.
+                zoom = self._tab.zoom.factor()
+                rect = QRect(left * zoom, top * zoom,
+                             width * zoom, height * zoom)
+                # FIXME:qtwebengine
+                # frame = self._elem.webFrame()
+                # while frame is not None:
+                #     # Translate to parent frames' position (scroll position
+                #     # is taken care of inside getClientRects)
+                #     rect.translate(frame.geometry().topLeft())
+                #     frame = frame.parentFrame()
+                return rect
+        log.webelem.debug("Couldn't find rectangle for {!r} ({})".format(
+            self, rects))
+        return QRect()
+
+    def remove_blank_target(self):
+        if self._js_dict['attributes'].get('target') == '_blank':
+            self._js_dict['attributes']['target'] = '_top'
+        js_code = javascript.assemble('webelem', 'remove_blank_target',
+            self._id)
+        self._tab.run_js_async(js_code)
+
+    def _click_editable(self, click_target):
+        # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-58515
+        # pylint doesn't know about Qt.MouseEventSynthesizedBySystem
+        # because it was added in Qt 5.6, but we can be sure we use that with
+        # QtWebEngine.
+        # pylint: disable=no-member
+        ev = QMouseEvent(QMouseEvent.MouseButtonPress, QPoint(0, 0),
+                         QPoint(0, 0), QPoint(0, 0), Qt.NoButton, Qt.NoButton,
+                         Qt.NoModifier, Qt.MouseEventSynthesizedBySystem)
+        # pylint: enable=no-member
+        self._tab.send_event(ev)
+        # This actually "clicks" the element by calling focus() on it in JS.
+        js_code = javascript.assemble('webelem', 'focus', self._id)
+        self._tab.run_js_async(js_code)
+
+    def _click_js(self, _click_target):
+        settings = QWebEngineSettings.globalSettings()
+        attribute = QWebEngineSettings.JavascriptCanOpenWindows
+        could_open_windows = settings.testAttribute(attribute)
+        settings.setAttribute(attribute, True)
+
+        # Get QtWebEngine do apply the settings
+        # (it does so with a 0ms QTimer...)
+        # This is also used in Qt's tests:
+        # https://github.com/qt/qtwebengine/commit/5e572e88efa7ba7c2b9138ec19e606d3e345ac90
+        qapp = QApplication.instance()
+        qapp.processEvents(QEventLoop.ExcludeSocketNotifiers |
+                           QEventLoop.ExcludeUserInputEvents)
+
+        def reset_setting(_arg):
+            settings.setAttribute(attribute, could_open_windows)
+
+        js_code = javascript.assemble('webelem', 'click', self._id)
+        self._tab.run_js_async(js_code, reset_setting)

qutebrowser/browser/webengine/webengineinspector.py

@@ -0,0 +1,52 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Customized QWebInspector for QtWebEngine."""
+
+import os
+
+from PyQt5.QtCore import QUrl
+# pylint: disable=no-name-in-module,import-error,useless-suppression
+from PyQt5.QtWebEngineWidgets import QWebEngineView
+# pylint: enable=no-name-in-module,import-error,useless-suppression
+
+from qutebrowser.browser import inspector
+
+
+class WebEngineInspector(inspector.AbstractWebInspector):
+
+    """A web inspector for QtWebEngine."""
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self.port = None
+        view = QWebEngineView()
+        self._set_widget(view)
+
+    def inspect(self, _page):
+        """Set up the inspector."""
+        try:
+            port = int(os.environ['QTWEBENGINE_REMOTE_DEBUGGING'])
+        except KeyError:
+            raise inspector.WebInspectorError(
+                "Debugging is not enabled. See 'qutebrowser --help' for "
+                "details.")
+        url = QUrl('http://localhost:{}/'.format(port))
+        self._widget.load(url)
+        self.show()

qutebrowser/browser/webengine/webenginequtescheme.py

@@ -0,0 +1,78 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""QtWebEngine specific qute:* handlers and glue code."""
+
+from PyQt5.QtCore import QBuffer, QIODevice
+# pylint: disable=no-name-in-module,import-error,useless-suppression
+from PyQt5.QtWebEngineCore import (QWebEngineUrlSchemeHandler,
+                                   QWebEngineUrlRequestJob)
+# pylint: enable=no-name-in-module,import-error,useless-suppression
+
+from qutebrowser.browser import qutescheme
+from qutebrowser.utils import log
+
+
+class QuteSchemeHandler(QWebEngineUrlSchemeHandler):
+
+    """Handle qute:* requests on QtWebEngine."""
+
+    def install(self, profile):
+        """Install the handler for qute: URLs on the given profile."""
+        profile.installUrlSchemeHandler(b'qute', self)
+
+    def requestStarted(self, job):
+        """Handle a request for a qute: scheme.
+
+        This method must be reimplemented by all custom URL scheme handlers.
+        The request is asynchronous and does not need to be handled right away.
+
+        Args:
+            job: QWebEngineUrlRequestJob
+        """
+        url = job.requestUrl()
+        assert job.requestMethod() == b'GET'
+        assert url.scheme() == 'qute'
+        log.misc.debug("Got request for {}".format(url.toDisplayString()))
+        try:
+            mimetype, data = qutescheme.data_for_url(url)
+        except qutescheme.NoHandlerFound:
+            log.misc.debug("No handler found for {}".format(
+                url.toDisplayString()))
+            job.fail(QWebEngineUrlRequestJob.UrlNotFound)
+        except qutescheme.QuteSchemeOSError:
+            # FIXME:qtwebengine how do we show a better error here?
+            log.misc.exception("OSError while handling qute:* URL")
+            job.fail(QWebEngineUrlRequestJob.UrlNotFound)
+        except qutescheme.QuteSchemeError:
+            # FIXME:qtwebengine how do we show a better error here?
+            log.misc.exception("Error while handling qute:* URL")
+            job.fail(QWebEngineUrlRequestJob.RequestFailed)
+        else:
+            log.misc.debug("Returning {} data".format(mimetype))
+
+            # We can't just use the QBuffer constructor taking a QByteArray,
+            # because that somehow segfaults...
+            # https://www.riverbankcomputing.com/pipermail/pyqt/2016-September/038075.html
+            buf = QBuffer(parent=self)
+            buf.open(QIODevice.WriteOnly)
+            buf.write(data)
+            buf.seek(0)
+            buf.close()
+            job.reply(mimetype.encode('ascii'), buf)

qutebrowser/browser/webengine/webenginesettings.py

@@ -0,0 +1,288 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Bridge from QWebEngineSettings to our own settings.
+
+Module attributes:
+    ATTRIBUTES: A mapping from internal setting names to QWebEngineSetting enum
+                constants.
+"""
+
+import os
+
+# pylint: disable=no-name-in-module,import-error,useless-suppression
+from PyQt5.QtWebEngineWidgets import (QWebEngineSettings, QWebEngineProfile,
+                                      QWebEngineScript)
+# pylint: enable=no-name-in-module,import-error,useless-suppression
+
+from qutebrowser.browser import shared
+from qutebrowser.config import config, websettings
+from qutebrowser.utils import objreg, utils, standarddir, javascript
+
+
+class Attribute(websettings.Attribute):
+
+    """A setting set via QWebEngineSettings::setAttribute."""
+
+    GLOBAL_SETTINGS = QWebEngineSettings.globalSettings
+    ENUM_BASE = QWebEngineSettings
+
+
+class Setter(websettings.Setter):
+
+    """A setting set via QWebEngineSettings getter/setter methods."""
+
+    GLOBAL_SETTINGS = QWebEngineSettings.globalSettings
+
+
+class NullStringSetter(websettings.NullStringSetter):
+
+    """A setter for settings requiring a null QString as default."""
+
+    GLOBAL_SETTINGS = QWebEngineSettings.globalSettings
+
+
+class StaticSetter(websettings.StaticSetter):
+
+    """A setting set via static QWebEngineSettings getter/setter methods."""
+
+    GLOBAL_SETTINGS = QWebEngineSettings.globalSettings
+
+
+class ProfileSetter(websettings.Base):
+
+    """A setting set on the QWebEngineProfile."""
+
+    def __init__(self, getter, setter):
+        super().__init__()
+        profile = QWebEngineProfile.defaultProfile()
+        self._getter = getattr(profile, getter)
+        self._setter = getattr(profile, setter)
+
+    def get(self, settings=None):
+        utils.unused(settings)
+        return self._getter()
+
+    def _set(self, value, settings=None):
+        utils.unused(settings)
+        self._setter(value)
+
+
+class PersistentCookiePolicy(ProfileSetter):
+
+    """The cookies -> store setting is different from other settings."""
+
+    def __init__(self):
+        super().__init__(getter='persistentCookiesPolicy',
+                         setter='setPersistentCookiesPolicy')
+
+    def get(self, settings=None):
+        utils.unused(settings)
+        return config.get('content', 'cookies-store')
+
+    def _set(self, value, settings=None):
+        utils.unused(settings)
+        self._setter(
+            QWebEngineProfile.AllowPersistentCookies if value else
+            QWebEngineProfile.NoPersistentCookies
+        )
+
+
+def _init_stylesheet(profile):
+    """Initialize custom stylesheets.
+
+    Mostly inspired by QupZilla:
+    https://github.com/QupZilla/qupzilla/blob/v2.0/src/lib/app/mainapplication.cpp#L1063-L1101
+    https://github.com/QupZilla/qupzilla/blob/v2.0/src/lib/tools/scripts.cpp#L119-L132
+
+    FIXME:qtwebengine Use QWebEngineStyleSheet once that's available
+    https://codereview.qt-project.org/#/c/148671/
+    """
+    old_script = profile.scripts().findScript('_qute_stylesheet')
+    if not old_script.isNull():
+        profile.scripts().remove(old_script)
+
+    css = shared.get_user_stylesheet()
+    source = """
+        (function() {{
+            var css = document.createElement('style');
+            css.setAttribute('type', 'text/css');
+            css.appendChild(document.createTextNode('{}'));
+            document.getElementsByTagName('head')[0].appendChild(css);
+        }})()
+    """.format(javascript.string_escape(css))
+
+    script = QWebEngineScript()
+    script.setName('_qute_stylesheet')
+    script.setInjectionPoint(QWebEngineScript.DocumentReady)
+    script.setWorldId(QWebEngineScript.ApplicationWorld)
+    script.setRunsOnSubFrames(True)
+    script.setSourceCode(source)
+    profile.scripts().insert(script)
+
+
+def _init_profile(profile):
+    """Initialize settings set on the QWebEngineProfile."""
+    profile.setCachePath(os.path.join(standarddir.cache(), 'webengine'))
+    profile.setPersistentStoragePath(
+        os.path.join(standarddir.data(), 'webengine'))
+
+
+def update_settings(section, option):
+    """Update global settings when qwebsettings changed."""
+    websettings.update_mappings(MAPPINGS, section, option)
+    profile = QWebEngineProfile.defaultProfile()
+    if section == 'ui' and option in ['hide-scrollbar', 'user-stylesheet']:
+        _init_stylesheet(profile)
+
+
+def init(args):
+    """Initialize the global QWebSettings."""
+    if args.enable_webengine_inspector:
+        os.environ['QTWEBENGINE_REMOTE_DEBUGGING'] = str(utils.random_port())
+
+    profile = QWebEngineProfile.defaultProfile()
+    _init_profile(profile)
+    _init_stylesheet(profile)
+    # We need to do this here as a WORKAROUND for
+    # https://bugreports.qt.io/browse/QTBUG-58650
+    PersistentCookiePolicy().set(config.get('content', 'cookies-store'))
+
+    Attribute(QWebEngineSettings.FullScreenSupportEnabled).set(True)
+
+    websettings.init_mappings(MAPPINGS)
+    objreg.get('config').changed.connect(update_settings)
+
+
+def shutdown():
+    # FIXME:qtwebengine do we need to do something for a clean shutdown here?
+    pass
+
+
+# Missing QtWebEngine attributes:
+# - ScreenCaptureEnabled
+# - Accelerated2dCanvasEnabled
+# - AutoLoadIconsForPage
+# - TouchIconsEnabled
+# - FocusOnNavigationEnabled (5.8)
+# - AllowRunningInsecureContent (5.8)
+#
+# Missing QtWebEngine fonts:
+# - FantasyFont
+# - PictographFont
+
+
+MAPPINGS = {
+    'content': {
+        'allow-images':
+            Attribute(QWebEngineSettings.AutoLoadImages),
+        'allow-javascript':
+            Attribute(QWebEngineSettings.JavascriptEnabled),
+        'javascript-can-open-windows-automatically':
+            Attribute(QWebEngineSettings.JavascriptCanOpenWindows),
+        'javascript-can-access-clipboard':
+            Attribute(QWebEngineSettings.JavascriptCanAccessClipboard),
+        'allow-plugins':
+            Attribute(QWebEngineSettings.PluginsEnabled),
+        'hyperlink-auditing':
+            Attribute(QWebEngineSettings.HyperlinkAuditingEnabled),
+        'local-content-can-access-remote-urls':
+            Attribute(QWebEngineSettings.LocalContentCanAccessRemoteUrls),
+        'local-content-can-access-file-urls':
+            Attribute(QWebEngineSettings.LocalContentCanAccessFileUrls),
+        # https://bugreports.qt.io/browse/QTBUG-58650
+        # 'cookies-store':
+        #     PersistentCookiePolicy(),
+        'webgl':
+            Attribute(QWebEngineSettings.WebGLEnabled),
+    },
+    'input': {
+        'spatial-navigation':
+            Attribute(QWebEngineSettings.SpatialNavigationEnabled),
+        'links-included-in-focus-chain':
+            Attribute(QWebEngineSettings.LinksIncludedInFocusChain),
+    },
+    'fonts': {
+        'web-family-standard':
+            Setter(getter=QWebEngineSettings.fontFamily,
+                   setter=QWebEngineSettings.setFontFamily,
+                   args=[QWebEngineSettings.StandardFont]),
+        'web-family-fixed':
+            Setter(getter=QWebEngineSettings.fontFamily,
+                   setter=QWebEngineSettings.setFontFamily,
+                   args=[QWebEngineSettings.FixedFont]),
+        'web-family-serif':
+            Setter(getter=QWebEngineSettings.fontFamily,
+                   setter=QWebEngineSettings.setFontFamily,
+                   args=[QWebEngineSettings.SerifFont]),
+        'web-family-sans-serif':
+            Setter(getter=QWebEngineSettings.fontFamily,
+                   setter=QWebEngineSettings.setFontFamily,
+                   args=[QWebEngineSettings.SansSerifFont]),
+        'web-family-cursive':
+            Setter(getter=QWebEngineSettings.fontFamily,
+                   setter=QWebEngineSettings.setFontFamily,
+                   args=[QWebEngineSettings.CursiveFont]),
+        'web-family-fantasy':
+            Setter(getter=QWebEngineSettings.fontFamily,
+                   setter=QWebEngineSettings.setFontFamily,
+                   args=[QWebEngineSettings.FantasyFont]),
+        'web-size-minimum':
+            Setter(getter=QWebEngineSettings.fontSize,
+                   setter=QWebEngineSettings.setFontSize,
+                   args=[QWebEngineSettings.MinimumFontSize]),
+        'web-size-minimum-logical':
+            Setter(getter=QWebEngineSettings.fontSize,
+                   setter=QWebEngineSettings.setFontSize,
+                   args=[QWebEngineSettings.MinimumLogicalFontSize]),
+        'web-size-default':
+            Setter(getter=QWebEngineSettings.fontSize,
+                   setter=QWebEngineSettings.setFontSize,
+                   args=[QWebEngineSettings.DefaultFontSize]),
+        'web-size-default-fixed':
+            Setter(getter=QWebEngineSettings.fontSize,
+                   setter=QWebEngineSettings.setFontSize,
+                   args=[QWebEngineSettings.DefaultFixedFontSize]),
+    },
+    'ui': {
+        'smooth-scrolling':
+            Attribute(QWebEngineSettings.ScrollAnimatorEnabled),
+    },
+    'storage': {
+        'local-storage':
+            Attribute(QWebEngineSettings.LocalStorageEnabled),
+        'cache-size':
+            ProfileSetter(getter='httpCacheMaximumSize',
+                          setter='setHttpCacheMaximumSize')
+    },
+    'general': {
+        'xss-auditing':
+            Attribute(QWebEngineSettings.XSSAuditingEnabled),
+        'default-encoding':
+            Setter(getter=QWebEngineSettings.defaultTextEncoding,
+                   setter=QWebEngineSettings.setDefaultTextEncoding),
+    }
+}
+
+try:
+    MAPPINGS['general']['print-element-backgrounds'] = Attribute(
+        QWebEngineSettings.PrintElementBackgrounds)
+except AttributeError:
+    # Added in Qt 5.8
+    pass

qutebrowser/browser/webengine/webenginetab.py

@@ -22,16 +22,77 @@
 
 """Wrapper over a QWebEngineView."""
 
-from PyQt5.QtCore import pyqtSlot, Qt, QEvent, QPoint
-from PyQt5.QtGui import QKeyEvent, QIcon
-from PyQt5.QtWidgets import QApplication
+import functools
+
+import sip
+from PyQt5.QtCore import pyqtSlot, Qt, QEvent, QPoint, QUrl, QTimer
+from PyQt5.QtGui import QKeyEvent
+from PyQt5.QtNetwork import QAuthenticator
 # pylint: disable=no-name-in-module,import-error,useless-suppression
-from PyQt5.QtWebEngineWidgets import QWebEnginePage
+from PyQt5.QtWidgets import QApplication
+from PyQt5.QtWebEngineWidgets import (QWebEnginePage, QWebEngineScript,
+                                      QWebEngineProfile)
 # pylint: enable=no-name-in-module,import-error,useless-suppression
 
-from qutebrowser.browser import browsertab
-from qutebrowser.browser.webengine import webview
-from qutebrowser.utils import usertypes, qtutils, log, utils
+from qutebrowser.browser import browsertab, mouse, shared
+from qutebrowser.browser.webengine import (webview, webengineelem, tabhistory,
+                                           interceptor, webenginequtescheme,
+                                           webenginedownloads)
+from qutebrowser.misc import miscwidgets
+from qutebrowser.utils import (usertypes, qtutils, log, javascript, utils,
+                               objreg, jinja)
+
+
+_qute_scheme_handler = None
+
+
+def init():
+    """Initialize QtWebEngine-specific modules."""
+    # For some reason we need to keep a reference, otherwise the scheme handler
+    # won't work...
+    # https://www.riverbankcomputing.com/pipermail/pyqt/2016-September/038075.html
+    global _qute_scheme_handler
+    app = QApplication.instance()
+    profile = QWebEngineProfile.defaultProfile()
+
+    log.init.debug("Initializing qute:* handler...")
+    _qute_scheme_handler = webenginequtescheme.QuteSchemeHandler(parent=app)
+    _qute_scheme_handler.install(profile)
+
+    log.init.debug("Initializing request interceptor...")
+    host_blocker = objreg.get('host-blocker')
+    req_interceptor = interceptor.RequestInterceptor(
+        host_blocker, parent=app)
+    req_interceptor.install(profile)
+
+    log.init.debug("Initializing QtWebEngine downloads...")
+    download_manager = webenginedownloads.DownloadManager(parent=app)
+    download_manager.install(profile)
+    objreg.register('webengine-download-manager', download_manager)
+
+
+# Mapping worlds from usertypes.JsWorld to QWebEngineScript world IDs.
+_JS_WORLD_MAP = {
+    usertypes.JsWorld.main: QWebEngineScript.MainWorld,
+    usertypes.JsWorld.application: QWebEngineScript.ApplicationWorld,
+    usertypes.JsWorld.user: QWebEngineScript.UserWorld,
+    usertypes.JsWorld.jseval: QWebEngineScript.UserWorld + 1,
+}
+
+
+class WebEngineAction(browsertab.AbstractAction):
+
+    """QtWebKit implementations related to web actions."""
+
+    def _action(self, action):
+        self._widget.triggerPageAction(action)
+
+    def exit_fullscreen(self):
+        self._action(QWebEnginePage.ExitFullScreen)
+
+    def save_page(self):
+        """Save the current page."""
+        self._action(QWebEnginePage.SavePage)
 
 
 class WebEnginePrinting(browsertab.AbstractPrinting):
@@ -39,20 +100,24 @@ class WebEnginePrinting(browsertab.AbstractPrinting):
     """QtWebEngine implementations related to printing."""
 
     def check_pdf_support(self):
-        if not hasattr(self._widget.page(), 'printToPdf'):
-            raise browsertab.WebTabError(
-                "Printing to PDF is unsupported with QtWebEngine on Qt > 5.7")
+        return True
 
     def check_printer_support(self):
+        if not hasattr(self._widget.page(), 'print'):
+            raise browsertab.WebTabError(
+                "Printing is unsupported with QtWebEngine on Qt < 5.8")
+
+    def check_preview_support(self):
         raise browsertab.WebTabError(
-            "Printing is unsupported with QtWebEngine")
+            "Print previews are unsupported with QtWebEngine")
 
     def to_pdf(self, filename):
         self._widget.page().printToPdf(filename)
 
-    def to_printer(self, printer):
-        # Should never be called
-        assert False
+    def to_printer(self, printer, callback=None):
+        if callback is None:
+            callback = lambda _ok: None
+        self._widget.page().print(printer, callback)
 
 
 class WebEngineSearch(browsertab.AbstractSearch):
@@ -95,7 +160,7 @@ class WebEngineSearch(browsertab.AbstractSearch):
             flags &= ~QWebEnginePage.FindBackward
         else:
             flags |= QWebEnginePage.FindBackward
-        self._find(self.text, self._flags, result_cb)
+        self._find(self.text, flags, result_cb)
 
     def next_result(self, *, result_cb=None):
         self._find(self.text, self._flags, result_cb)
@@ -107,11 +172,11 @@ class WebEngineCaret(browsertab.AbstractCaret):
 
     @pyqtSlot(usertypes.KeyMode)
     def _on_mode_entered(self, mode):
-        log.stub()
+        pass
 
     @pyqtSlot(usertypes.KeyMode)
     def _on_mode_left(self):
-        log.stub()
+        pass
 
     def move_to_next_line(self, count=1):
         log.stub()
@@ -169,7 +234,7 @@ class WebEngineCaret(browsertab.AbstractCaret):
 
     def selection(self, html=False):
         if html:
-            raise NotImplementedError
+            raise browsertab.UnsupportedOperationError
         return self._widget.selectedText()
 
     def follow_selected(self, *, tab=False):
@@ -182,45 +247,55 @@ class WebEngineScroller(browsertab.AbstractScroller):
 
     def __init__(self, tab, parent=None):
         super().__init__(tab, parent)
-        self._pos_perc = (None, None)
+        self._pos_perc = (0, 0)
         self._pos_px = QPoint()
+        self._at_bottom = False
 
     def _init_widget(self, widget):
         super()._init_widget(widget)
         page = widget.page()
-        try:
-            page.scrollPositionChanged.connect(
-                self._on_scroll_pos_changed)
-        except AttributeError:
-            log.stub('scrollPositionChanged, on Qt < 5.7')
-        self._on_scroll_pos_changed()
+        page.scrollPositionChanged.connect(self._update_pos)
 
     def _key_press(self, key, count=1):
-        # FIXME:qtwebengine Abort scrolling if the minimum/maximum was reached.
-        press_evt = QKeyEvent(QEvent.KeyPress, key, Qt.NoModifier, 0, 0, 0)
-        release_evt = QKeyEvent(QEvent.KeyRelease, key, Qt.NoModifier, 0, 0, 0)
-        recipient = self._widget.focusProxy()
-        for _ in range(count):
-            # If we get a segfault here, we might want to try sendEvent
-            # instead.
-            QApplication.postEvent(recipient, press_evt)
-            QApplication.postEvent(recipient, release_evt)
+        for _ in range(min(count, 5000)):
+            press_evt = QKeyEvent(QEvent.KeyPress, key, Qt.NoModifier, 0, 0, 0)
+            release_evt = QKeyEvent(QEvent.KeyRelease, key, Qt.NoModifier,
+                                    0, 0, 0)
+            self._tab.send_event(press_evt)
+            self._tab.send_event(release_evt)
 
     @pyqtSlot()
-    def _on_scroll_pos_changed(self):
+    def _update_pos(self):
         """Update the scroll position attributes when it changed."""
-        def update_scroll_pos(jsret):
+        def update_pos_cb(jsret):
             """Callback after getting scroll position via JS."""
-            assert isinstance(jsret, dict)
-            self._pos_perc = (jsret['perc']['x'], jsret['perc']['y'])
+            if jsret is None:
+                # This can happen when the callback would get called after
+                # shutting down a tab
+                return
+            log.webview.vdebug(jsret)
+            assert isinstance(jsret, dict), jsret
             self._pos_px = QPoint(jsret['px']['x'], jsret['px']['y'])
+
+            dx = jsret['scroll']['width'] - jsret['inner']['width']
+            if dx == 0:
+                perc_x = 0
+            else:
+                perc_x = min(100, round(100 / dx * jsret['px']['x']))
+
+            dy = jsret['scroll']['height'] - jsret['inner']['height']
+            if dy == 0:
+                perc_y = 0
+            else:
+                perc_y = min(100, round(100 / dy * jsret['px']['y']))
+
+            self._at_bottom = dy >= jsret['px']['y']
+            self._pos_perc = perc_x, perc_y
+
             self.perc_changed.emit(*self._pos_perc)
 
-        js_code = """
-            {scroll_js}
-            scroll_pos();
-        """.format(scroll_js=utils.read_file('javascript/scroll.js'))
-        self._tab.run_js_async(js_code, update_scroll_pos)
+        js_code = javascript.assemble('scroll', 'pos')
+        self._tab.run_js_async(js_code, update_pos_cb)
 
     def pos_px(self):
         return self._pos_px
@@ -229,26 +304,18 @@ class WebEngineScroller(browsertab.AbstractScroller):
         return self._pos_perc
 
     def to_perc(self, x=None, y=None):
-        js_code = """
-            {scroll_js}
-            scroll_to_perc({x}, {y});
-        """.format(scroll_js=utils.read_file('javascript/scroll.js'),
-                   x='undefined' if x is None else x,
-                   y='undefined' if y is None else y)
+        js_code = javascript.assemble('scroll', 'to_perc', x, y)
         self._tab.run_js_async(js_code)
 
     def to_point(self, point):
-        self._tab.run_js_async("window.scroll({x}, {y});".format(
-            x=point.x(), y=point.y()))
+        js_code = javascript.assemble('window', 'scroll', point.x(), point.y())
+        self._tab.run_js_async(js_code)
 
     def delta(self, x=0, y=0):
-        self._tab.run_js_async("window.scrollBy({x}, {y});".format(x=x, y=y))
+        self._tab.run_js_async(javascript.assemble('window', 'scrollBy', x, y))
 
     def delta_page(self, x=0, y=0):
-        js_code = """
-            {scroll_js}
-            scroll_delta_page({x}, {y});
-        """.format(scroll_js=utils.read_file('javascript/scroll.js'), x=x, y=y)
+        js_code = javascript.assemble('scroll', 'delta_page', x, y)
         self._tab.run_js_async(js_code)
 
     def up(self, count=1):
@@ -279,7 +346,7 @@ class WebEngineScroller(browsertab.AbstractScroller):
         return self.pos_px().y() == 0
 
     def at_bottom(self):
-        log.stub()
+        return self._at_bottom
 
 
 class WebEngineHistory(browsertab.AbstractHistory):
@@ -302,13 +369,26 @@ class WebEngineHistory(browsertab.AbstractHistory):
         return self._history.canGoForward()
 
     def serialize(self):
+        # WORKAROUND for https://github.com/qutebrowser/qutebrowser/issues/2289
+        # FIXME:qtwebengine can we get rid of this with Qt 5.8.1?
+        scheme = self._history.currentItem().url().scheme()
+        if scheme in ['view-source', 'chrome']:
+            raise browsertab.WebTabError("Can't serialize special URL!")
         return qtutils.serialize(self._history)
 
     def deserialize(self, data):
         return qtutils.deserialize(data, self._history)
 
     def load_items(self, items):
-        log.stub()
+        stream, _data, cur_data = tabhistory.serialize(items)
+        qtutils.deserialize_stream(stream, self._history)
+        if cur_data is not None:
+            if 'zoom' in cur_data:
+                self._tab.zoom.set_factor(cur_data['zoom'])
+            if ('scroll-pos' in cur_data and
+                    self._tab.scroller.pos_px() == QPoint(0, 0)):
+                QTimer.singleShot(0, functools.partial(
+                    self._tab.scroller.to_point, cur_data['scroll-pos']))
 
 
 class WebEngineZoom(browsertab.AbstractZoom):
@@ -322,13 +402,80 @@ class WebEngineZoom(browsertab.AbstractZoom):
         return self._widget.zoomFactor()
 
 
+class WebEngineElements(browsertab.AbstractElements):
+
+    """QtWebEngine implemementations related to elements on the page."""
+
+    def _js_cb_multiple(self, callback, js_elems):
+        """Handle found elements coming from JS and call the real callback.
+
+        Args:
+            callback: The callback to call with the found elements.
+                      Called with None if there was an error.
+            js_elems: The elements serialized from javascript.
+        """
+        if js_elems is None:
+            callback(None)
+            return
+
+        elems = []
+        for js_elem in js_elems:
+            elem = webengineelem.WebEngineElement(js_elem, tab=self._tab)
+            elems.append(elem)
+        callback(elems)
+
+    def _js_cb_single(self, callback, js_elem):
+        """Handle a found focus elem coming from JS and call the real callback.
+
+        Args:
+            callback: The callback to call with the found element.
+                      Called with a WebEngineElement or None.
+            js_elem: The element serialized from javascript.
+        """
+        debug_str = ('None' if js_elem is None
+                     else utils.elide(repr(js_elem), 1000))
+        log.webview.debug("Got element from JS: {}".format(debug_str))
+
+        if js_elem is None:
+            callback(None)
+        else:
+            elem = webengineelem.WebEngineElement(js_elem, tab=self._tab)
+            callback(elem)
+
+    def find_css(self, selector, callback, *, only_visible=False):
+        js_code = javascript.assemble('webelem', 'find_all', selector,
+                                      only_visible)
+        js_cb = functools.partial(self._js_cb_multiple, callback)
+        self._tab.run_js_async(js_code, js_cb)
+
+    def find_id(self, elem_id, callback):
+        js_code = javascript.assemble('webelem', 'element_by_id', elem_id)
+        js_cb = functools.partial(self._js_cb_single, callback)
+        self._tab.run_js_async(js_code, js_cb)
+
+    def find_focused(self, callback):
+        js_code = javascript.assemble('webelem', 'focus_element')
+        js_cb = functools.partial(self._js_cb_single, callback)
+        self._tab.run_js_async(js_code, js_cb)
+
+    def find_at_pos(self, pos, callback):
+        assert pos.x() >= 0
+        assert pos.y() >= 0
+        pos /= self._tab.zoom.factor()
+        js_code = javascript.assemble('webelem', 'element_at_pos',
+                                      pos.x(), pos.y())
+        js_cb = functools.partial(self._js_cb_single, callback)
+        self._tab.run_js_async(js_code, js_cb)
+
+
 class WebEngineTab(browsertab.AbstractTab):
 
     """A QtWebEngine tab in the browser."""
 
     def __init__(self, win_id, mode_manager, parent=None):
-        super().__init__(win_id)
-        widget = webview.WebEngineView()
+        super().__init__(win_id=win_id, mode_manager=mode_manager,
+                         parent=parent)
+        widget = webview.WebEngineView(tabdata=self.data, win_id=win_id)
         self.history = WebEngineHistory(self)
         self.scroller = WebEngineScroller(self, parent=self)
         self.caret = WebEngineCaret(win_id=win_id, mode_manager=mode_manager,
@@ -336,16 +483,57 @@ class WebEngineTab(browsertab.AbstractTab):
         self.zoom = WebEngineZoom(win_id=win_id, parent=self)
         self.search = WebEngineSearch(parent=self)
         self.printing = WebEnginePrinting()
+        self.elements = WebEngineElements(self)
+        self.action = WebEngineAction()
         self._set_widget(widget)
         self._connect_signals()
         self.backend = usertypes.Backend.QtWebEngine
+        self._init_js()
+        self._child_event_filter = None
+        self._saved_zoom = None
+
+    def _init_js(self):
+        js_code = '\n'.join([
+            '"use strict";',
+            'window._qutebrowser = {};',
+            utils.read_file('javascript/scroll.js'),
+            utils.read_file('javascript/webelem.js'),
+        ])
+        script = QWebEngineScript()
+        script.setInjectionPoint(QWebEngineScript.DocumentCreation)
+        script.setSourceCode(js_code)
+
+        page = self._widget.page()
+        script.setWorldId(QWebEngineScript.ApplicationWorld)
+
+        # FIXME:qtwebengine  what about runsOnSubFrames?
+        page.scripts().insert(script)
+
+    def _install_event_filter(self):
+        self._widget.focusProxy().installEventFilter(self._mouse_event_filter)
+        self._child_event_filter = mouse.ChildEventFilter(
+            eventfilter=self._mouse_event_filter, widget=self._widget,
+            parent=self)
+        self._widget.installEventFilter(self._child_event_filter)
+
+    @pyqtSlot()
+    def _restore_zoom(self):
+        if self._saved_zoom is None:
+            return
+        self.zoom.set_factor(self._saved_zoom)
+        self._saved_zoom = None
 
     def openurl(self, url):
+        self._saved_zoom = self.zoom.factor()
         self._openurl_prepare(url)
         self._widget.load(url)
 
-    def url(self):
-        return self._widget.url()
+    def url(self, requested=False):
+        page = self._widget.page()
+        if requested:
+            return page.requestedUrl()
+        else:
+            return page.url()
 
     def dump_async(self, callback, *, plain=False):
         if plain:
@@ -353,31 +541,25 @@ class WebEngineTab(browsertab.AbstractTab):
         else:
             self._widget.page().toHtml(callback)
 
-    def run_js_async(self, code, callback=None):
-        if callback is None:
-            self._widget.page().runJavaScript(code)
+    def run_js_async(self, code, callback=None, *, world=None):
+        if world is None:
+            world_id = QWebEngineScript.ApplicationWorld
+        elif isinstance(world, int):
+            world_id = world
         else:
-            self._widget.page().runJavaScript(code, callback)
+            world_id = _JS_WORLD_MAP[world]
 
-    def run_js_blocking(self, code):
-        unset = object()
-        loop = qtutils.EventLoop()
-        js_ret = unset
-
-        def js_cb(val):
-            """Handle return value from JS and stop blocking."""
-            nonlocal js_ret
-            js_ret = val
-            loop.quit()
-
-        self.run_js_async(code, js_cb)
-        loop.exec_()  # blocks until loop.quit() in js_cb
-        assert js_ret is not unset
-
-        return js_ret
+        if callback is None:
+            self._widget.page().runJavaScript(code, world_id)
+        else:
+            self._widget.page().runJavaScript(code, world_id, callback)
 
     def shutdown(self):
-        log.stub()
+        self.shutting_down.emit()
+        # WORKAROUND for
+        # https://bugreports.qt.io/browse/QTBUG-58563
+        self.search.clear()
+        self._widget.shutdown()
 
     def reload(self, *, force=False):
         if force:
@@ -393,35 +575,119 @@ class WebEngineTab(browsertab.AbstractTab):
         return self._widget.title()
 
     def icon(self):
-        try:
-            return self._widget.icon()
-        except AttributeError:
-            log.stub('on Qt < 5.7')
-            return QIcon()
+        return self._widget.icon()
 
-    def set_html(self, html, base_url):
+    def set_html(self, html, base_url=None):
         # FIXME:qtwebengine
         # check this and raise an exception if too big:
         # Warning: The content will be percent encoded before being sent to the
         # renderer via IPC. This may increase its size. The maximum size of the
         # percent encoded content is 2 megabytes minus 30 bytes.
+        if base_url is None:
+            base_url = QUrl()
         self._widget.setHtml(html, base_url)
 
+    def networkaccessmanager(self):
+        return None
+
+    def user_agent(self):
+        return None
+
     def clear_ssl_errors(self):
-        log.stub()
+        raise browsertab.UnsupportedOperationError
+
+    @pyqtSlot()
+    def _on_history_trigger(self):
+        url = self.url()
+        requested_url = self.url(requested=True)
+
+        # Don't save the title if it's generated from the URL
+        title = self.title()
+        title_url = QUrl(url)
+        title_url.setScheme('')
+        if title == title_url.toDisplayString(QUrl.RemoveScheme).strip('/'):
+            title = ""
+
+        # Don't add history entry if the URL is invalid anyways
+        if not url.isValid():
+            log.misc.debug("Ignoring invalid URL being added to history")
+            return
+
+        self.add_history_item.emit(url, requested_url, title)
+
+    @pyqtSlot(QUrl, 'QAuthenticator*')
+    def _on_authentication_required(self, url, authenticator):
+        # FIXME:qtwebengine support .netrc
+        answer = shared.authentication_required(
+            url, authenticator, abort_on=[self.shutting_down,
+                                          self.load_started])
+        if answer is None:
+            try:
+                # pylint: disable=no-member, useless-suppression
+                sip.assign(authenticator, QAuthenticator())
+            except AttributeError:
+                # WORKAROUND for
+                # https://www.riverbankcomputing.com/pipermail/pyqt/2016-December/038400.html
+                url_string = url.toDisplayString()
+                error_page = jinja.render(
+                    'error.html',
+                    title="Error loading page: {}".format(url_string),
+                    url=url_string, error="Authentication required", icon='')
+                self.set_html(error_page)
+
+    @pyqtSlot('QWebEngineFullScreenRequest')
+    def _on_fullscreen_requested(self, request):
+        request.accept()
+        on = request.toggleOn()
+        self.fullscreen_requested.emit(on)
+        if on:
+            notification = miscwidgets.FullscreenNotification(self)
+            notification.show()
+            notification.set_timeout(3000)
+
+    @pyqtSlot(QWebEnginePage.RenderProcessTerminationStatus, int)
+    def _on_render_process_terminated(self, status, exitcode):
+        """Show an error when the renderer process terminated."""
+        if (status == QWebEnginePage.AbnormalTerminationStatus and
+                exitcode == 256):
+            # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-58697
+            status = QWebEnginePage.CrashedTerminationStatus
+
+        status_map = {
+            QWebEnginePage.NormalTerminationStatus:
+                browsertab.TerminationStatus.normal,
+            QWebEnginePage.AbnormalTerminationStatus:
+                browsertab.TerminationStatus.abnormal,
+            QWebEnginePage.CrashedTerminationStatus:
+                browsertab.TerminationStatus.crashed,
+            QWebEnginePage.KilledTerminationStatus:
+                browsertab.TerminationStatus.killed,
+            -1:
+                browsertab.TerminationStatus.unknown,
+        }
+        self.renderer_process_terminated.emit(status_map[status], exitcode)
 
     def _connect_signals(self):
         view = self._widget
         page = view.page()
+
         page.windowCloseRequested.connect(self.window_close_requested)
         page.linkHovered.connect(self.link_hovered)
         page.loadProgress.connect(self._on_load_progress)
         page.loadStarted.connect(self._on_load_started)
-        view.titleChanged.connect(self.title_changed)
-        view.urlChanged.connect(self._on_url_changed)
+        page.loadFinished.connect(self._on_history_trigger)
+        page.loadFinished.connect(self._restore_zoom)
         page.loadFinished.connect(self._on_load_finished)
         page.certificate_error.connect(self._on_ssl_errors)
-        try:
-            view.iconChanged.connect(self.icon_changed)
-        except AttributeError:
-            log.stub('iconChanged, on Qt < 5.7')
+        page.authenticationRequired.connect(self._on_authentication_required)
+        page.fullScreenRequested.connect(self._on_fullscreen_requested)
+        page.contentsSizeChanged.connect(self.contents_size_changed)
+
+        view.titleChanged.connect(self.title_changed)
+        view.urlChanged.connect(self._on_url_changed)
+        view.renderProcessTerminated.connect(
+            self._on_render_process_terminated)
+        view.iconChanged.connect(self.icon_changed)
+
+    def event_target(self):
+        return self._widget.focusProxy()

qutebrowser/browser/webengine/webview.py

@@ -19,48 +19,256 @@
 
 """The main browser widget for QtWebEngine."""
 
+import functools
 
-from PyQt5.QtCore import pyqtSignal, Qt, QPoint
+from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, PYQT_VERSION
+from PyQt5.QtGui import QPalette
 # pylint: disable=no-name-in-module,import-error,useless-suppression
 from PyQt5.QtWebEngineWidgets import QWebEngineView, QWebEnginePage
 # pylint: enable=no-name-in-module,import-error,useless-suppression
 
+from qutebrowser.browser import shared
+from qutebrowser.browser.webengine import certificateerror
 from qutebrowser.config import config
-from qutebrowser.utils import log
+from qutebrowser.utils import (log, debug, usertypes, jinja, urlutils, message,
+                               objreg)
 
 
 class WebEngineView(QWebEngineView):
 
     """Custom QWebEngineView subclass with qutebrowser-specific features."""
 
-    mouse_wheel_zoom = pyqtSignal(QPoint)
-
-    def __init__(self, parent=None):
+    def __init__(self, tabdata, win_id, parent=None):
         super().__init__(parent)
-        self.setPage(WebEnginePage(self))
+        self._win_id = win_id
+        self._tabdata = tabdata
 
-    def wheelEvent(self, e):
-        """Zoom on Ctrl-Mousewheel.
+        theme_color = self.style().standardPalette().color(QPalette.Base)
+        page = WebEnginePage(theme_color=theme_color, parent=self)
+        self.setPage(page)
+
+    def shutdown(self):
+        self.page().shutdown()
+
+    def createWindow(self, wintype):
+        """Called by Qt when a page wants to create a new window.
+
+        This function is called from the createWindow() method of the
+        associated QWebEnginePage, each time the page wants to create a new
+        window of the given type. This might be the result, for example, of a
+        JavaScript request to open a document in a new window.
 
         Args:
-            e: The QWheelEvent.
+            wintype: This enum describes the types of window that can be
+                     created by the createWindow() function.
+
+                     QWebEnginePage::WebBrowserWindow:
+                         A complete web browser window.
+                     QWebEnginePage::WebBrowserTab:
+                         A web browser tab.
+                     QWebEnginePage::WebDialog:
+                         A window without decoration.
+                     QWebEnginePage::WebBrowserBackgroundTab:
+                         A web browser tab without hiding the current visible
+                         WebEngineView.
+
+        Return:
+            The new QWebEngineView object.
         """
-        if e.modifiers() & Qt.ControlModifier:
-            e.accept()
-            self.mouse_wheel_zoom.emit(e.angleDelta())
+        debug_type = debug.qenum_key(QWebEnginePage, wintype)
+        background_tabs = config.get('tabs', 'background-tabs')
+
+        log.webview.debug("createWindow with type {}, background_tabs "
+                          "{}".format(debug_type, background_tabs))
+
+        if wintype == QWebEnginePage.WebBrowserWindow:
+            # Shift-Alt-Click
+            target = usertypes.ClickTarget.window
+        elif wintype == QWebEnginePage.WebDialog:
+            log.webview.warning("{} requested, but we don't support "
+                                "that!".format(debug_type))
+            target = usertypes.ClickTarget.tab
+        elif wintype == QWebEnginePage.WebBrowserTab:
+            # Middle-click / Ctrl-Click with Shift
+            # FIXME:qtwebengine this also affects target=_blank links...
+            if background_tabs:
+                target = usertypes.ClickTarget.tab
+            else:
+                target = usertypes.ClickTarget.tab_bg
+        elif wintype == QWebEnginePage.WebBrowserBackgroundTab:
+            # Middle-click / Ctrl-Click
+            if background_tabs:
+                target = usertypes.ClickTarget.tab_bg
+            else:
+                target = usertypes.ClickTarget.tab
         else:
-            super().wheelEvent(e)
+            raise ValueError("Invalid wintype {}".format(debug_type))
+
+        tab = shared.get_tab(self._win_id, target)
+        return tab._widget  # pylint: disable=protected-access
 
 
 class WebEnginePage(QWebEnginePage):
 
-    """Custom QWebEnginePage subclass with qutebrowser-specific features."""
+    """Custom QWebEnginePage subclass with qutebrowser-specific features.
+
+    Attributes:
+        _is_shutting_down: Whether the page is currently shutting down.
+        _theme_color: The theme background color.
+
+    Signals:
+        certificate_error: Emitted on certificate errors.
+        shutting_down: Emitted when the page is shutting down.
+    """
 
     certificate_error = pyqtSignal()
+    shutting_down = pyqtSignal()
+
+    def __init__(self, theme_color, parent=None):
+        super().__init__(parent)
+        self._is_shutting_down = False
+        self.featurePermissionRequested.connect(
+            self._on_feature_permission_requested)
+        self._theme_color = theme_color
+        self._set_bg_color()
+        objreg.get('config').changed.connect(self._set_bg_color)
+
+    @config.change_filter('colors', 'webpage.bg')
+    def _set_bg_color(self):
+        col = config.get('colors', 'webpage.bg')
+        if col is None:
+            col = self._theme_color
+        self.setBackgroundColor(col)
+
+    @pyqtSlot(QUrl, 'QWebEnginePage::Feature')
+    def _on_feature_permission_requested(self, url, feature):
+        """Ask the user for approval for geolocation/media/etc.."""
+        options = {
+            QWebEnginePage.Geolocation: ('content', 'geolocation'),
+            QWebEnginePage.MediaAudioCapture: ('content', 'media-capture'),
+            QWebEnginePage.MediaVideoCapture: ('content', 'media-capture'),
+            QWebEnginePage.MediaAudioVideoCapture:
+                ('content', 'media-capture'),
+        }
+        messages = {
+            QWebEnginePage.Geolocation: 'access your location',
+            QWebEnginePage.MediaAudioCapture: 'record audio',
+            QWebEnginePage.MediaVideoCapture: 'record video',
+            QWebEnginePage.MediaAudioVideoCapture: 'record audio/video',
+        }
+        assert options.keys() == messages.keys()
+
+        if feature not in options:
+            log.webview.error("Unhandled feature permission {}".format(
+                debug.qenum_key(QWebEnginePage, feature)))
+            self.setFeaturePermission(url, feature,
+                                      QWebEnginePage.PermissionDeniedByUser)
+            return
+
+        yes_action = functools.partial(
+            self.setFeaturePermission, url, feature,
+            QWebEnginePage.PermissionGrantedByUser)
+        no_action = functools.partial(
+            self.setFeaturePermission, url, feature,
+            QWebEnginePage.PermissionDeniedByUser)
+
+        question = shared.feature_permission(
+            url=url, option=options[feature], msg=messages[feature],
+            yes_action=yes_action, no_action=no_action,
+            abort_on=[self.shutting_down, self.loadStarted])
+
+        if question is not None:
+            self.featurePermissionRequestCanceled.connect(
+                functools.partial(self._on_feature_permission_cancelled,
+                                  question, url, feature))
+
+    def _on_feature_permission_cancelled(self, question, url, feature,
+                                         cancelled_url, cancelled_feature):
+        """Slot invoked when a feature permission request was cancelled.
+
+        To be used with functools.partial.
+        """
+        if url == cancelled_url and feature == cancelled_feature:
+            try:
+                question.abort()
+            except RuntimeError:
+                # The question could already be deleted, e.g. because it was
+                # aborted after a loadStarted signal.
+                pass
+
+    def shutdown(self):
+        self._is_shutting_down = True
+        self.shutting_down.emit()
 
     def certificateError(self, error):
+        """Handle certificate errors coming from Qt."""
         self.certificate_error.emit()
-        return super().certificateError(error)
+        url = error.url()
+        error = certificateerror.CertificateErrorWrapper(error)
+        log.webview.debug("Certificate error: {}".format(error))
+
+        url_string = url.toDisplayString()
+        error_page = jinja.render(
+            'error.html', title="Error loading page: {}".format(url_string),
+            url=url_string, error=str(error), icon='')
+
+        if error.is_overridable():
+            ignore = shared.ignore_certificate_errors(
+                url, [error], abort_on=[self.loadStarted, self.shutting_down])
+        else:
+            log.webview.error("Non-overridable certificate error: "
+                              "{}".format(error))
+            ignore = False
+
+        # We can't really know when to show an error page, as the error might
+        # have happened when loading some resource.
+        # However, self.url() is not available yet and self.requestedUrl()
+        # might not match the URL we get from the error - so we just apply a
+        # heuristic here.
+        # See https://bugreports.qt.io/browse/QTBUG-56207
+        log.webview.debug("ignore {}, URL {}, requested {}".format(
+            ignore, url, self.requestedUrl()))
+        if not ignore and url.matches(self.requestedUrl(), QUrl.RemoveScheme):
+            self.setHtml(error_page)
+
+        return ignore
+
+    def javaScriptConfirm(self, url, js_msg):
+        """Override javaScriptConfirm to use qutebrowser prompts."""
+        if self._is_shutting_down:
+            return False
+        try:
+            return shared.javascript_confirm(url, js_msg,
+                                             abort_on=[self.loadStarted,
+                                                       self.shutting_down])
+        except shared.CallSuper:
+            return super().javaScriptConfirm(url, js_msg)
+
+    if PYQT_VERSION > 0x050700:
+        # WORKAROUND
+        # Can't override javaScriptPrompt with older PyQt versions
+        # https://www.riverbankcomputing.com/pipermail/pyqt/2016-November/038293.html
+        def javaScriptPrompt(self, url, js_msg, default):
+            """Override javaScriptPrompt to use qutebrowser prompts."""
+            if self._is_shutting_down:
+                return (False, "")
+            try:
+                return shared.javascript_prompt(url, js_msg, default,
+                                                abort_on=[self.loadStarted,
+                                                          self.shutting_down])
+            except shared.CallSuper:
+                return super().javaScriptPrompt(url, js_msg, default)
+
+    def javaScriptAlert(self, url, js_msg):
+        """Override javaScriptAlert to use qutebrowser prompts."""
+        if self._is_shutting_down:
+            return
+        try:
+            shared.javascript_alert(url, js_msg,
+                                    abort_on=[self.loadStarted,
+                                              self.shutting_down])
+        except shared.CallSuper:
+            super().javaScriptAlert(url, js_msg)
 
     def javaScriptConsoleMessage(self, level, msg, line, source):
         """Log javascript messages to qutebrowser's log."""
@@ -77,3 +285,23 @@ class WebEnginePage(QWebEnginePage):
         logstring = "[{}:{}] {}".format(source, line, msg)
         logger = level_to_logger[level]
         logger(logstring)
+
+    def acceptNavigationRequest(self,
+                                url: QUrl,
+                                typ: QWebEnginePage.NavigationType,
+                                is_main_frame: bool):
+        """Override acceptNavigationRequest to handle clicked links.
+
+        This only show an error on invalid links - everything else is handled
+        in createWindow.
+        """
+        log.webview.debug("navigation request: url {}, type {}, is_main_frame "
+                          "{}".format(url.toDisplayString(),
+                                      debug.qenum_key(QWebEnginePage, typ),
+                                      is_main_frame))
+        if (typ == QWebEnginePage.NavigationTypeLinkClicked and
+                not url.isValid()):
+            msg = urlutils.get_errstring(url, "Invalid link clicked")
+            message.error(msg)
+            return False
+        return True

qutebrowser/browser/webkit/cache.py

@@ -32,23 +32,14 @@ class DiskCache(QNetworkDiskCache):
 
     """Disk cache which sets correct cache dir and size.
 
-    If the cache is deactivated via the command line argument --cachedir="",
-    both attributes _cache_dir and _http_cache_dir are set to None.
-
     Attributes:
         _activated: Whether the cache should be used.
-        _cache_dir: The base directory for cache files (standarddir.cache()) or
-                    None.
-        _http_cache_dir: the HTTP subfolder in _cache_dir or None.
+        _cache_dir: The base directory for cache files (standarddir.cache())
     """
 
     def __init__(self, cache_dir, parent=None):
         super().__init__(parent)
         self._cache_dir = cache_dir
-        if cache_dir is None:
-            self._http_cache_dir = None
-        else:
-            self._http_cache_dir = os.path.join(cache_dir, 'http')
         self._maybe_activate()
         objreg.get('config').changed.connect(self.on_config_changed)
 
@@ -57,21 +48,27 @@ class DiskCache(QNetworkDiskCache):
                               maxsize=self.maximumCacheSize(),
                               path=self.cacheDirectory())
 
+    def _set_cache_size(self):
+        """Set the cache size based on the config."""
+        size = config.get('storage', 'cache-size')
+        if size is None:
+            size = 1024 * 1024 * 50  # default from QNetworkDiskCachePrivate
+        self.setMaximumCacheSize(size)
+
     def _maybe_activate(self):
         """Activate/deactivate the cache based on the config."""
-        if (config.get('general', 'private-browsing') or
-                self._cache_dir is None):
+        if config.get('general', 'private-browsing'):
             self._activated = False
         else:
             self._activated = True
-            self.setCacheDirectory(self._http_cache_dir)
-            self.setMaximumCacheSize(config.get('storage', 'cache-size'))
+            self.setCacheDirectory(os.path.join(self._cache_dir, 'http'))
+            self._set_cache_size()
 
     @pyqtSlot(str, str)
     def on_config_changed(self, section, option):
         """Update cache size/activated if the config was changed."""
         if (section, option) == ('storage', 'cache-size'):
-            self.setMaximumCacheSize(config.get('storage', 'cache-size'))
+            self._set_cache_size()
         elif (section, option) == ('general',  # pragma: no branch
                                    'private-browsing'):
             self._maybe_activate()

qutebrowser/browser/webkit/certificateerror.py

@@ -0,0 +1,52 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Wrapper over a QSslError."""
+
+
+from PyQt5.QtNetwork import QSslError
+
+from qutebrowser.utils import usertypes, utils, debug
+
+
+class CertificateErrorWrapper(usertypes.AbstractCertificateErrorWrapper):
+
+    """A wrapper over a QSslError."""
+
+    def __str__(self):
+        return self._error.errorString()
+
+    def __repr__(self):
+        return utils.get_repr(
+            self, error=debug.qenum_key(QSslError, self._error.error()),
+            string=str(self))
+
+    def __hash__(self):
+        try:
+            # Qt >= 5.4
+            return hash(self._error)
+        except TypeError:
+            return hash((self._error.certificate().toDer(),
+                         self._error.error()))
+
+    def __eq__(self, other):
+        return self._error == other._error  # pylint: disable=protected-access
+
+    def is_overridable(self):
+        return True

qutebrowser/browser/webkit/http.py

@@ -47,7 +47,8 @@ def parse_content_disposition(reply):
         # os.path.basename later.
         try:
             value = bytes(reply.rawHeader(content_disposition_header))
-            log.rfc6266.debug("Parsing Content-Disposition: {}".format(value))
+            log.rfc6266.debug("Parsing Content-Disposition: {!r}".format(
+                value))
             content_disposition = rfc6266.parse_headers(value)
             filename = content_disposition.filename()
         except (SyntaxError, UnicodeDecodeError, rfc6266.Error):

qutebrowser/browser/webkit/mhtml.py

@@ -19,6 +19,7 @@
 
 """Utils for writing an MHTML file."""
 
+import html
 import functools
 import io
 import os
@@ -31,20 +32,14 @@ import email.generator
 import email.encoders
 import email.mime.multipart
 import email.message
+import quopri
 
 from PyQt5.QtCore import QUrl
 
-from qutebrowser.browser.webkit import webelem, downloads
+from qutebrowser.browser import downloads
+from qutebrowser.browser.webkit import webkitelem
 from qutebrowser.utils import log, objreg, message, usertypes, utils, urlutils
 
-try:
-    import cssutils
-except (ImportError, re.error):
-    # Catching re.error because cssutils in earlier releases (<= 1.0) is broken
-    # on Python 3.5
-    # See https://bitbucket.org/cthedot/cssutils/issues/52
-    cssutils = None
-
 _File = collections.namedtuple('_File',
                                ['content', 'content_type', 'content_location',
                                 'transfer_encoding'])
@@ -85,6 +80,14 @@ def _get_css_imports_cssutils(data, inline=False):
         data: The content of the stylesheet to scan as string.
         inline: True if the argument is an inline HTML style attribute.
     """
+    try:
+        import cssutils
+    except (ImportError, re.error):
+        # Catching re.error because cssutils in earlier releases (<= 1.0) is
+        # broken on Python 3.5
+        # See https://bitbucket.org/cthedot/cssutils/issues/52
+        return None
+
     # We don't care about invalid CSS data, this will only litter the log
     # output with CSS errors
     parser = cssutils.CSSParser(loglevel=100,
@@ -114,10 +117,10 @@ def _get_css_imports(data, inline=False):
         data: The content of the stylesheet to scan as string.
         inline: True if the argument is an inline HTML style attribute.
     """
-    if cssutils is None:
-        return _get_css_imports_regex(data)
-    else:
-        return _get_css_imports_cssutils(data, inline)
+    imports = _get_css_imports_cssutils(data, inline)
+    if imports is None:
+        imports = _get_css_imports_regex(data)
+    return imports
 
 
 def _check_rel(element):
@@ -136,6 +139,22 @@ def _check_rel(element):
     return any(rel in rels for rel in must_have)
 
 
+def _encode_quopri_mhtml(msg):
+    """Encode the message's payload in quoted-printable.
+
+    Substitute for quopri's default 'encode_quopri' method, which needlessly
+    encodes all spaces and tabs, instead of only those at the end on the
+    line.
+
+    Args:
+        msg: Email message to quote.
+    """
+    orig = msg.get_payload(decode=True)
+    encdata = quopri.encodestring(orig, quotetabs=False)
+    msg.set_payload(encdata)
+    msg['Content-Transfer-Encoding'] = 'quoted-printable'
+
+
 MHTMLPolicy = email.policy.default.clone(linesep='\r\n', max_line_length=0)
 
 
@@ -144,7 +163,7 @@ E_BASE64 = email.encoders.encode_base64
 
 
 # Encode the file using MIME quoted-printable encoding.
-E_QUOPRI = email.encoders.encode_quopri
+E_QUOPRI = _encode_quopri_mhtml
 
 
 class MHTMLWriter:
@@ -223,7 +242,7 @@ class _Downloader:
 
     Attributes:
         tab: The AbstractTab which contains the website that will be saved.
-        dest: Destination filename.
+        target: DownloadTarget where the file should be downloaded to.
         writer: The MHTMLWriter object which is used to save the page.
         loaded_urls: A set of QUrls of finished asset downloads.
         pending_downloads: A set of unfinished (url, DownloadItem) tuples.
@@ -233,9 +252,9 @@ class _Downloader:
         _win_id: The window this downloader belongs to.
     """
 
-    def __init__(self, tab, dest):
+    def __init__(self, tab, target):
         self.tab = tab
-        self.dest = dest
+        self.target = target
         self.writer = None
         self.loaded_urls = {tab.url()}
         self.pending_downloads = set()
@@ -271,7 +290,7 @@ class _Downloader:
         elements = web_frame.findAllElements('link, script, img')
 
         for element in elements:
-            element = webelem.WebElementWrapper(element)
+            element = webkitelem.WebKitElement(element, tab=self.tab)
             # Websites are free to set whatever rel=... attribute they want.
             # We just care about stylesheets and icons.
             if not _check_rel(element):
@@ -288,7 +307,7 @@ class _Downloader:
 
         styles = web_frame.findAllElements('style')
         for style in styles:
-            style = webelem.WebElementWrapper(style)
+            style = webkitelem.WebKitElement(style, tab=self.tab)
             # The Mozilla Developer Network says:
             # type: This attribute defines the styling language as a MIME type
             # (charset should not be specified). This attribute is optional and
@@ -301,7 +320,7 @@ class _Downloader:
 
         # Search for references in inline styles
         for element in web_frame.findAllElements('[style]'):
-            element = webelem.WebElementWrapper(element)
+            element = webkitelem.WebKitElement(element, tab=self.tab)
             style = element['style']
             for element_url in _get_css_imports(style, inline=True):
                 self._fetch_url(web_url.resolved(QUrl(element_url)))
@@ -330,8 +349,8 @@ class _Downloader:
 
         # Using the download manager to download host-blocked urls might crash
         # qute, see the comments/discussion on
-        # https://github.com/The-Compiler/qutebrowser/pull/962#discussion_r40256987
-        # and https://github.com/The-Compiler/qutebrowser/issues/1053
+        # https://github.com/qutebrowser/qutebrowser/pull/962#discussion_r40256987
+        # and https://github.com/qutebrowser/qutebrowser/issues/1053
         host_blocker = objreg.get('host-blocker')
         if host_blocker.is_blocked(url):
             log.downloads.debug("Skipping {}, host-blocked".format(url))
@@ -341,9 +360,9 @@ class _Downloader:
             self.writer.add_file(urlutils.encoded_url(url), b'')
             return
 
-        download_manager = objreg.get('download-manager', scope='window',
-                                      window=self._win_id)
-        target = usertypes.FileObjDownloadTarget(_NoCloseBytesIO())
+        download_manager = objreg.get('qtnetwork-download-manager',
+                                      scope='window', window=self._win_id)
+        target = downloads.FileObjDownloadTarget(_NoCloseBytesIO())
         item = download_manager.get(url, target=target,
                                     auto_remove=True)
         self.pending_downloads.add((url, item))
@@ -443,15 +462,34 @@ class _Downloader:
             return
         self._finished_file = True
         log.downloads.debug("All assets downloaded, ready to finish off!")
+
+        if isinstance(self.target, downloads.FileDownloadTarget):
+            fobj = open(self.target.filename, 'wb')
+        elif isinstance(self.target, downloads.FileObjDownloadTarget):
+            fobj = self.target.fileobj
+        elif isinstance(self.target, downloads.OpenFileDownloadTarget):
+            try:
+                fobj = downloads.temp_download_manager.get_tmpfile(
+                    self.tab.title() + '.mhtml')
+            except OSError as exc:
+                msg = "Download error: {}".format(exc)
+                message.error(msg)
+                return
+        else:
+            raise ValueError("Invalid DownloadTarget given: {!r}"
+                             .format(self.target))
+
         try:
-            with open(self.dest, 'wb') as file_output:
-                self.writer.write_to(file_output)
+            with fobj:
+                self.writer.write_to(fobj)
         except OSError as error:
-            message.error(self._win_id,
-                          "Could not save file: {}".format(error))
+            message.error("Could not save file: {}".format(error))
             return
         log.downloads.debug("File successfully written.")
-        message.info(self._win_id, "Page saved as {}".format(self.dest))
+        message.info("Page saved as {}".format(self.target))
+
+        if isinstance(self.target, downloads.OpenFileDownloadTarget):
+            utils.open_file(fobj.name, self.target.cmdline)
 
     def _collect_zombies(self):
         """Collect done downloads and add their data to the MHTML file.
@@ -483,34 +521,37 @@ class _NoCloseBytesIO(io.BytesIO):
         super().close()
 
 
-def _start_download(dest, tab):
+def _start_download(target, tab):
     """Start downloading the current page and all assets to an MHTML file.
 
     This will overwrite dest if it already exists.
 
     Args:
-        dest: The filename where the resulting file should be saved.
+        target: The DownloadTarget where the resulting file should be saved.
         tab: Specify the tab whose page should be loaded.
     """
-    loader = _Downloader(tab, dest)
+    loader = _Downloader(tab, target)
     loader.run()
 
 
-def start_download_checked(dest, tab):
+def start_download_checked(target, tab):
     """First check if dest is already a file, then start the download.
 
     Args:
-        dest: The filename where the resulting file should be saved.
+        target: The DownloadTarget where the resulting file should be saved.
         tab: Specify the tab whose page should be loaded.
     """
-    # The default name is 'page title.mht'
+    if not isinstance(target, downloads.FileDownloadTarget):
+        _start_download(target, tab)
+        return
+    # The default name is 'page title.mhtml'
     title = tab.title()
-    default_name = utils.sanitize_filename(title + '.mht')
+    default_name = utils.sanitize_filename(title + '.mhtml')
 
     # Remove characters which cannot be expressed in the file system encoding
     encoding = sys.getfilesystemencoding()
     default_name = utils.force_encoding(default_name, encoding)
-    dest = utils.force_encoding(dest, encoding)
+    dest = utils.force_encoding(target.filename, encoding)
 
     dest = os.path.expanduser(dest)
 
@@ -528,20 +569,20 @@ def start_download_checked(dest, tab):
     # saving the file anyway.
     if not os.path.isdir(os.path.dirname(path)):
         folder = os.path.dirname(path)
-        message.error(tab.win_id,
-                      "Directory {} does not exist.".format(folder))
+        message.error("Directory {} does not exist.".format(folder))
         return
 
+    target = downloads.FileDownloadTarget(path)
     if not os.path.isfile(path):
-        _start_download(path, tab=tab)
+        _start_download(target, tab=tab)
         return
 
     q = usertypes.Question()
     q.mode = usertypes.PromptMode.yesno
-    q.text = "{} exists. Overwrite?".format(path)
+    q.title = "Overwrite existing file?"
+    q.text = "<b>{}</b> already exists. Overwrite?".format(
+        html.escape(path))
     q.completed.connect(q.deleteLater)
     q.answered_yes.connect(functools.partial(
-        _start_download, path, tab=tab))
-    message_bridge = objreg.get('message-bridge', scope='window',
-                                window=tab.win_id)
-    message_bridge.ask(q, blocking=False)
+        _start_download, target, tab=tab))
+    message.global_bridge.ask(q, blocking=False)

qutebrowser/browser/webkit/network/networkmanager.py

@@ -22,17 +22,19 @@
 import os
 import collections
 import netrc
+import html
 
 from PyQt5.QtCore import (pyqtSlot, pyqtSignal, PYQT_VERSION, QCoreApplication,
                           QUrl, QByteArray)
-from PyQt5.QtNetwork import (QNetworkAccessManager, QNetworkReply, QSslError,
-                             QSslSocket)
+from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkReply, QSslSocket
 
 from qutebrowser.config import config
 from qutebrowser.utils import (message, log, usertypes, utils, objreg, qtutils,
-                               urlutils, debug)
-from qutebrowser.browser.webkit.network import qutescheme, networkreply
-from qutebrowser.browser.webkit.network import filescheme
+                               urlutils)
+from qutebrowser.browser import shared
+from qutebrowser.browser.webkit import certificateerror
+from qutebrowser.browser.webkit.network import (webkitqutescheme, networkreply,
+                                                filescheme)
 
 
 HOSTBLOCK_ERROR_STRING = '%HOSTBLOCK%'
@@ -109,24 +111,6 @@ def init():
     QSslSocket.setDefaultCiphers(good_ciphers)
 
 
-class SslError(QSslError):
-
-    """A QSslError subclass which provides __hash__ on Qt < 5.4."""
-
-    def __hash__(self):
-        try:
-            # Qt >= 5.4
-            # pylint: disable=not-callable,useless-suppression
-            return super().__hash__()
-        except TypeError:
-            return hash((self.certificate().toDer(), self.error()))
-
-    def __repr__(self):
-        return utils.get_repr(
-            self, error=debug.qenum_key(QSslError, self.error()),
-            string=self.errorString())
-
-
 class NetworkManager(QNetworkAccessManager):
 
     """Our own QNetworkAccessManager.
@@ -163,7 +147,7 @@ class NetworkManager(QNetworkAccessManager):
         self._tab_id = tab_id
         self._requests = []
         self._scheme_handlers = {
-            'qute': qutescheme.QuteSchemeHandler(win_id),
+            'qute': webkitqutescheme.QuteSchemeHandler(win_id),
             'file': filescheme.FileSchemeHandler(win_id),
         }
         self._set_cookiejar(private=config.get('general', 'private-browsing'))
@@ -206,36 +190,18 @@ class NetworkManager(QNetworkAccessManager):
         self.setCache(cache)
         cache.setParent(app)
 
-    def _ask(self, text, mode, owner=None):
-        """Ask a blocking question in the statusbar.
-
-        Args:
-            text: The text to display to the user.
-            mode: A PromptMode.
-            owner: An object which will abort the question if destroyed, or
-                   None.
-
-        Return:
-            The answer the user gave or None if the prompt was cancelled.
-        """
-        q = usertypes.Question()
-        q.text = text
-        q.mode = mode
-        self.shutting_down.connect(q.abort)
+    def _get_abort_signals(self, owner=None):
+        """Get a list of signals which should abort a question."""
+        abort_on = [self.shutting_down]
         if owner is not None:
-            owner.destroyed.connect(q.abort)
-
+            abort_on.append(owner.destroyed)
         # This might be a generic network manager, e.g. one belonging to a
         # DownloadManager. In this case, just skip the webview thing.
         if self._tab_id is not None:
             tab = objreg.get('tab', scope='tab', window=self._win_id,
                              tab=self._tab_id)
-            tab.load_started.connect(q.abort)
-        bridge = objreg.get('message-bridge', scope='window',
-                            window=self._win_id)
-        bridge.ask(q, blocking=True)
-        q.deleteLater()
-        return q.answer
+            abort_on.append(tab.load_started)
+        return abort_on
 
     def shutdown(self):
         """Abort all running requests."""
@@ -245,7 +211,8 @@ class NetworkManager(QNetworkAccessManager):
             request.deleteLater()
         self.shutting_down.emit()
 
-    @pyqtSlot('QNetworkReply*', 'QList<QSslError>')
+    # No @pyqtSlot here, see
+    # https://github.com/qutebrowser/qutebrowser/issues/2213
     def on_ssl_errors(self, reply, errors):  # pragma: no mccabe
         """Decide if SSL errors should be ignored or not.
 
@@ -255,11 +222,9 @@ class NetworkManager(QNetworkAccessManager):
             reply: The QNetworkReply that is encountering the errors.
             errors: A list of errors.
         """
-        errors = [SslError(e) for e in errors]
-        ssl_strict = config.get('network', 'ssl-strict')
-        log.webview.debug("SSL errors {!r}, strict {}".format(
-            errors, ssl_strict))
-
+        errors = [certificateerror.CertificateErrorWrapper(e) for e in errors]
+        log.webview.debug("Certificate errors: {!r}".format(
+            ' / '.join(str(err) for err in errors)))
         try:
             host_tpl = urlutils.host_tuple(reply.url())
         except ValueError:
@@ -275,33 +240,22 @@ class NetworkManager(QNetworkAccessManager):
         log.webview.debug("Already accepted: {} / "
                           "rejected {}".format(is_accepted, is_rejected))
 
-        if (ssl_strict and ssl_strict != 'ask') or is_rejected:
+        if is_rejected:
             return
         elif is_accepted:
             reply.ignoreSslErrors()
             return
 
-        if ssl_strict == 'ask':
-            err_string = '\n'.join('- ' + err.errorString() for err in errors)
-            answer = self._ask('SSL errors - continue?\n{}'.format(err_string),
-                               mode=usertypes.PromptMode.yesno, owner=reply)
-            log.webview.debug("Asked for SSL errors, answer {}".format(answer))
-            if answer:
-                reply.ignoreSslErrors()
-                err_dict = self._accepted_ssl_errors
-            else:
-                err_dict = self._rejected_ssl_errors
-            if host_tpl is not None:
-                err_dict[host_tpl] += errors
-        else:
-            log.webview.debug("ssl-strict is False, only warning about errors")
-            for err in errors:
-                # FIXME we might want to use warn here (non-fatal error)
-                # https://github.com/The-Compiler/qutebrowser/issues/114
-                message.error(self._win_id, 'SSL error: {}'.format(
-                    err.errorString()))
+        abort_on = self._get_abort_signals(reply)
+        ignore = shared.ignore_certificate_errors(reply.url(), errors,
+                                                  abort_on=abort_on)
+        if ignore:
             reply.ignoreSslErrors()
-            self._accepted_ssl_errors[host_tpl] += errors
+            err_dict = self._accepted_ssl_errors
+        else:
+            err_dict = self._rejected_ssl_errors
+        if host_tpl is not None:
+            err_dict[host_tpl] += errors
 
     def clear_all_ssl_errors(self):
         """Clear all remembered SSL errors."""
@@ -330,7 +284,7 @@ class NetworkManager(QNetworkAccessManager):
             # altogether.
             reply.netrc_used = True
             try:
-                net = netrc.netrc()
+                net = netrc.netrc(config.get('network', 'netrc-file'))
                 authenticators = net.authenticators(reply.url().host())
                 if authenticators is not None:
                     (user, _account, password) = authenticators
@@ -341,16 +295,13 @@ class NetworkManager(QNetworkAccessManager):
             except netrc.NetrcParseError:
                 log.misc.exception("Error when parsing the netrc file")
 
-        if user is None:
-            # netrc check failed
-            answer = self._ask("Username ({}):".format(authenticator.realm()),
-                               mode=usertypes.PromptMode.user_pwd,
-                               owner=reply)
-            if answer is not None:
-                user, password = answer.user, answer.password
         if user is not None:
             authenticator.setUser(user)
             authenticator.setPassword(password)
+        else:
+            abort_on = self._get_abort_signals(reply)
+            shared.authentication_required(reply.url(), authenticator,
+                                           abort_on=abort_on)
 
     @pyqtSlot('QNetworkProxy', 'QAuthenticator*')
     def on_proxy_authentication_required(self, proxy, authenticator):
@@ -361,9 +312,13 @@ class NetworkManager(QNetworkAccessManager):
             authenticator.setUser(user)
             authenticator.setPassword(password)
         else:
-            answer = self._ask(
-                "Proxy username ({}):".format(authenticator.realm()),
-                mode=usertypes.PromptMode.user_pwd)
+            msg = '<b>{}</b> says:<br/>{}'.format(
+                html.escape(proxy.hostName()),
+                html.escape(authenticator.realm()))
+            abort_on = self._get_abort_signals()
+            answer = message.ask(
+                title="Proxy authentication required", text=msg,
+                mode=usertypes.PromptMode.user_pwd, abort_on=abort_on)
             if answer is not None:
                 authenticator.setUser(answer.user)
                 authenticator.setPassword(answer.password)
@@ -400,7 +355,7 @@ class NetworkManager(QNetworkAccessManager):
         log.downloads.debug("Adopted download, {} adopted.".format(
             self.adopted_downloads))
         download.destroyed.connect(self.on_adopted_download_destroyed)
-        download.do_retry.connect(self.adopt_download)
+        download.adopt_download.connect(self.adopt_download)
 
     def set_referer(self, req, current_url):
         """Set the referer header."""
@@ -442,6 +397,14 @@ class NetworkManager(QNetworkAccessManager):
         Return:
             A QNetworkReply.
         """
+        proxy_factory = objreg.get('proxy-factory', None)
+        if proxy_factory is not None:
+            proxy_error = proxy_factory.get_error()
+            if proxy_error is not None:
+                return networkreply.ErrorNetworkReply(
+                    req, proxy_error, QNetworkReply.UnknownProxyError,
+                    self)
+
         scheme = req.url().scheme()
         if scheme in self._scheme_handlers:
             result = self._scheme_handlers[scheme].createRequest(
@@ -449,6 +412,9 @@ class NetworkManager(QNetworkAccessManager):
             if result is not None:
                 return result
 
+        for header, value in shared.custom_headers():
+            req.setRawHeader(header, value)
+
         host_blocker = objreg.get('host-blocker')
         if (op == QNetworkAccessManager.GetOperation and
                 host_blocker.is_blocked(req.url())):
@@ -458,20 +424,6 @@ class NetworkManager(QNetworkAccessManager):
                 req, HOSTBLOCK_ERROR_STRING, QNetworkReply.ContentAccessDenied,
                 self)
 
-        if config.get('network', 'do-not-track'):
-            dnt = '1'.encode('ascii')
-        else:
-            dnt = '0'.encode('ascii')
-        req.setRawHeader('DNT'.encode('ascii'), dnt)
-        req.setRawHeader('X-Do-Not-Track'.encode('ascii'), dnt)
-
-        # Load custom headers
-        custom_headers = config.get('network', 'custom-headers')
-
-        if custom_headers is not None:
-            for header, value in custom_headers.items():
-                req.setRawHeader(header.encode('ascii'), value.encode('ascii'))
-
         # There are some scenarios where we can't figure out current_url:
         # - There's a generic NetworkManager, e.g. for downloads
         # - The download was in a tab which is now closed.
@@ -483,17 +435,13 @@ class NetworkManager(QNetworkAccessManager):
                                  tab=self._tab_id)
                 current_url = tab.url()
             except (KeyError, RuntimeError, TypeError):
-                # https://github.com/The-Compiler/qutebrowser/issues/889
+                # https://github.com/qutebrowser/qutebrowser/issues/889
                 # Catching RuntimeError and TypeError because we could be in
                 # the middle of the webpage shutdown here.
                 current_url = QUrl()
 
         self.set_referer(req, current_url)
 
-        accept_language = config.get('network', 'accept-language')
-        if accept_language is not None:
-            req.setRawHeader('Accept-Language'.encode('ascii'),
-                             accept_language.encode('ascii'))
         if PYQT_VERSION < 0x050301:
             # WORKAROUND (remove this when we bump the requirements to 5.3.1)
             #

qutebrowser/browser/webkit/network/qutescheme.py

@@ -1,275 +0,0 @@
-# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
-
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
-#
-# This file is part of qutebrowser.
-#
-# qutebrowser is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# qutebrowser is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
-
-"""Handler functions for different qute:... pages.
-
-Module attributes:
-    pyeval_output: The output of the last :pyeval command.
-"""
-
-import functools
-import configparser
-import mimetypes
-import urllib.parse
-
-from PyQt5.QtCore import pyqtSlot, QObject
-from PyQt5.QtNetwork import QNetworkReply
-
-import qutebrowser
-from qutebrowser.browser import pdfjs
-from qutebrowser.browser.webkit.network import schemehandler, networkreply
-from qutebrowser.utils import (version, utils, jinja, log, message, docutils,
-                               objreg)
-from qutebrowser.config import configexc, configdata
-
-
-pyeval_output = ":pyeval was never called"
-
-
-HANDLERS = {}
-
-
-def add_handler(name):
-    """Add a handler to the qute: scheme."""
-    def namedecorator(function):
-        HANDLERS[name] = function
-        return function
-    return namedecorator
-
-
-class QuteSchemeError(Exception):
-
-    """Exception to signal that a handler should return an ErrorReply.
-
-    Attributes correspond to the arguments in
-    networkreply.ErrorNetworkReply.
-
-    Attributes:
-        errorstring: Error string to print.
-        error: Numerical error value.
-    """
-
-    def __init__(self, errorstring, error):
-        """Constructor."""
-        self.errorstring = errorstring
-        self.error = error
-        super().__init__(errorstring)
-
-
-class QuteSchemeHandler(schemehandler.SchemeHandler):
-
-    """Scheme handler for qute: URLs."""
-
-    def createRequest(self, _op, request, _outgoing_data):
-        """Create a new request.
-
-        Args:
-             request: const QNetworkRequest & req
-             _op: Operation op
-             _outgoing_data: QIODevice * outgoingData
-
-        Return:
-            A QNetworkReply.
-        """
-        path = request.url().path()
-        host = request.url().host()
-        # A url like "qute:foo" is split as "scheme:path", not "scheme:host".
-        log.misc.debug("url: {}, path: {}, host {}".format(
-            request.url().toDisplayString(), path, host))
-        try:
-            handler = HANDLERS[path]
-        except KeyError:
-            try:
-                handler = HANDLERS[host]
-            except KeyError:
-                errorstr = "No handler found for {}!".format(
-                    request.url().toDisplayString())
-                return networkreply.ErrorNetworkReply(
-                    request, errorstr, QNetworkReply.ContentNotFoundError,
-                    self.parent())
-        try:
-            data = handler(self._win_id, request)
-        except OSError as e:
-            return networkreply.ErrorNetworkReply(
-                request, str(e), QNetworkReply.ContentNotFoundError,
-                self.parent())
-        except QuteSchemeError as e:
-            return networkreply.ErrorNetworkReply(request, e.errorstring,
-                                                  e.error, self.parent())
-        mimetype, _encoding = mimetypes.guess_type(request.url().fileName())
-        if mimetype is None:
-            mimetype = 'text/html'
-        return networkreply.FixedDataNetworkReply(request, data, mimetype,
-                                                  self.parent())
-
-
-class JSBridge(QObject):
-
-    """Javascript-bridge for special qute:... pages."""
-
-    @pyqtSlot(int, str, str, str)
-    def set(self, win_id, sectname, optname, value):
-        """Slot to set a setting from qute:settings."""
-        # https://github.com/The-Compiler/qutebrowser/issues/727
-        if ((sectname, optname) == ('content', 'allow-javascript') and
-                value == 'false'):
-            message.error(win_id, "Refusing to disable javascript via "
-                          "qute:settings as it needs javascript support.")
-            return
-        try:
-            objreg.get('config').set('conf', sectname, optname, value)
-        except (configexc.Error, configparser.Error) as e:
-            message.error(win_id, e)
-
-
-@add_handler('pyeval')
-def qute_pyeval(_win_id, _request):
-    """Handler for qute:pyeval. Return HTML content as bytes."""
-    html = jinja.render('pre.html', title='pyeval', content=pyeval_output)
-    return html.encode('UTF-8', errors='xmlcharrefreplace')
-
-
-@add_handler('version')
-@add_handler('verizon')
-def qute_version(_win_id, _request):
-    """Handler for qute:version. Return HTML content as bytes."""
-    html = jinja.render('version.html', title='Version info',
-                        version=version.version(),
-                        copyright=qutebrowser.__copyright__)
-    return html.encode('UTF-8', errors='xmlcharrefreplace')
-
-
-@add_handler('plainlog')
-def qute_plainlog(_win_id, request):
-    """Handler for qute:plainlog. Return HTML content as bytes.
-
-    An optional query parameter specifies the minimum log level to print.
-    For example, qute://log?level=warning prints warnings and errors.
-    Level can be one of: vdebug, debug, info, warning, error, critical.
-    """
-    if log.ram_handler is None:
-        text = "Log output was disabled."
-    else:
-        try:
-            level = urllib.parse.parse_qs(request.url().query())['level'][0]
-        except KeyError:
-            level = 'vdebug'
-        text = log.ram_handler.dump_log(html=False, level=level)
-    html = jinja.render('pre.html', title='log', content=text)
-    return html.encode('UTF-8', errors='xmlcharrefreplace')
-
-
-@add_handler('log')
-def qute_log(_win_id, request):
-    """Handler for qute:log. Return HTML content as bytes.
-
-    An optional query parameter specifies the minimum log level to print.
-    For example, qute://log?level=warning prints warnings and errors.
-    Level can be one of: vdebug, debug, info, warning, error, critical.
-    """
-    if log.ram_handler is None:
-        html_log = None
-    else:
-        try:
-            level = urllib.parse.parse_qs(request.url().query())['level'][0]
-        except KeyError:
-            level = 'vdebug'
-        html_log = log.ram_handler.dump_log(html=True, level=level)
-
-    html = jinja.render('log.html', title='log', content=html_log)
-    return html.encode('UTF-8', errors='xmlcharrefreplace')
-
-
-@add_handler('gpl')
-def qute_gpl(_win_id, _request):
-    """Handler for qute:gpl. Return HTML content as bytes."""
-    return utils.read_file('html/COPYING.html').encode('ASCII')
-
-
-@add_handler('help')
-def qute_help(win_id, request):
-    """Handler for qute:help. Return HTML content as bytes."""
-    try:
-        utils.read_file('html/doc/index.html')
-    except OSError:
-        html = jinja.render(
-            'error.html',
-            title="Error while loading documentation",
-            url=request.url().toDisplayString(),
-            error="This most likely means the documentation was not generated "
-                  "properly. If you are running qutebrowser from the git "
-                  "repository, please run scripts/asciidoc2html.py. "
-                  "If you're running a released version this is a bug, please "
-                  "use :report to report it.",
-            icon='')
-        return html.encode('UTF-8', errors='xmlcharrefreplace')
-    urlpath = request.url().path()
-    if not urlpath or urlpath == '/':
-        urlpath = 'index.html'
-    else:
-        urlpath = urlpath.lstrip('/')
-    if not docutils.docs_up_to_date(urlpath):
-        message.error(win_id, "Your documentation is outdated! Please re-run "
-                      "scripts/asciidoc2html.py.")
-    path = 'html/doc/{}'.format(urlpath)
-    if urlpath.endswith('.png'):
-        return utils.read_file(path, binary=True)
-    else:
-        data = utils.read_file(path)
-        return data.encode('UTF-8', errors='xmlcharrefreplace')
-
-
-@add_handler('settings')
-def qute_settings(win_id, _request):
-    """Handler for qute:settings. View/change qute configuration."""
-    config_getter = functools.partial(objreg.get('config').get, raw=True)
-    html = jinja.render('settings.html', win_id=win_id, title='settings',
-                        config=configdata, confget=config_getter)
-    return html.encode('UTF-8', errors='xmlcharrefreplace')
-
-
-@add_handler('pdfjs')
-def qute_pdfjs(_win_id, request):
-    """Handler for qute://pdfjs. Return the pdf.js viewer."""
-    urlpath = request.url().path()
-    try:
-        return pdfjs.get_pdfjs_res(urlpath)
-    except pdfjs.PDFJSNotFound as e:
-        # Logging as the error might get lost otherwise since we're not showing
-        # the error page if a single asset is missing. This way we don't lose
-        # information, as the failed pdfjs requests are still in the log.
-        log.misc.warning(
-            "pdfjs resource requested but not found: {}".format(e.path))
-        raise QuteSchemeError("Can't find pdfjs resource '{}'".format(e.path),
-                              QNetworkReply.ContentNotFoundError)
-
-
-@add_handler('bookmarks')
-def qute_bookmarks(_win_id, _request):
-    """Handler for qute:bookmarks. Display all quickmarks / bookmarks."""
-    bookmarks = sorted(objreg.get('bookmark-manager').marks.items(),
-                       key=lambda x: x[1])  # Sort by title
-    quickmarks = sorted(objreg.get('quickmark-manager').marks.items(),
-                        key=lambda x: x[0])  # Sort by name
-
-    html = jinja.render('bookmarks.html',
-                        title='Bookmarks',
-                        bookmarks=bookmarks,
-                        quickmarks=quickmarks)
-    return html.encode('UTF-8', errors='xmlcharrefreplace')

qutebrowser/browser/webkit/network/webkitqutescheme.py

@@ -0,0 +1,115 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""QtWebKit specific qute:* handlers and glue code."""
+
+import mimetypes
+import functools
+import configparser
+
+from PyQt5.QtCore import pyqtSlot, QObject
+from PyQt5.QtNetwork import QNetworkReply
+
+from qutebrowser.browser import pdfjs, qutescheme
+from qutebrowser.browser.webkit.network import schemehandler, networkreply
+from qutebrowser.utils import jinja, log, message, objreg, usertypes
+from qutebrowser.config import configexc, configdata
+
+
+class QuteSchemeHandler(schemehandler.SchemeHandler):
+
+    """Scheme handler for qute: URLs."""
+
+    def createRequest(self, _op, request, _outgoing_data):
+        """Create a new request.
+
+        Args:
+             request: const QNetworkRequest & req
+             _op: Operation op
+             _outgoing_data: QIODevice * outgoingData
+
+        Return:
+            A QNetworkReply.
+        """
+        try:
+            mimetype, data = qutescheme.data_for_url(request.url())
+        except qutescheme.NoHandlerFound:
+            errorstr = "No handler found for {}!".format(
+                request.url().toDisplayString())
+            return networkreply.ErrorNetworkReply(
+                request, errorstr, QNetworkReply.ContentNotFoundError,
+                self.parent())
+        except qutescheme.QuteSchemeOSError as e:
+            return networkreply.ErrorNetworkReply(
+                request, str(e), QNetworkReply.ContentNotFoundError,
+                self.parent())
+        except qutescheme.QuteSchemeError as e:
+            return networkreply.ErrorNetworkReply(request, e.errorstring,
+                                                  e.error, self.parent())
+
+        return networkreply.FixedDataNetworkReply(request, data, mimetype,
+                                                  self.parent())
+
+
+class JSBridge(QObject):
+
+    """Javascript-bridge for special qute:... pages."""
+
+    @pyqtSlot(str, str, str)
+    def set(self, sectname, optname, value):
+        """Slot to set a setting from qute:settings."""
+        # https://github.com/qutebrowser/qutebrowser/issues/727
+        if ((sectname, optname) == ('content', 'allow-javascript') and
+                value == 'false'):
+            message.error("Refusing to disable javascript via qute:settings "
+                          "as it needs javascript support.")
+            return
+        try:
+            objreg.get('config').set('conf', sectname, optname, value)
+        except (configexc.Error, configparser.Error) as e:
+            message.error(str(e))
+
+
+@qutescheme.add_handler('settings', backend=usertypes.Backend.QtWebKit)
+def qute_settings(_url):
+    """Handler for qute:settings. View/change qute configuration."""
+    config_getter = functools.partial(objreg.get('config').get, raw=True)
+    html = jinja.render('settings.html', title='settings', config=configdata,
+                        confget=config_getter)
+    return 'text/html', html
+
+
+@qutescheme.add_handler('pdfjs', backend=usertypes.Backend.QtWebKit)
+def qute_pdfjs(url):
+    """Handler for qute://pdfjs. Return the pdf.js viewer."""
+    try:
+        data = pdfjs.get_pdfjs_res(url.path())
+    except pdfjs.PDFJSNotFound as e:
+        # Logging as the error might get lost otherwise since we're not showing
+        # the error page if a single asset is missing. This way we don't lose
+        # information, as the failed pdfjs requests are still in the log.
+        log.misc.warning(
+            "pdfjs resource requested but not found: {}".format(e.path))
+        raise qutescheme.QuteSchemeError("Can't find pdfjs resource "
+                                         "'{}'".format(e.path),
+                                         QNetworkReply.ContentNotFoundError)
+    else:
+        mimetype, _encoding = mimetypes.guess_type(url.fileName())
+        assert mimetype is not None, url
+        return mimetype, data

qutebrowser/browser/webkit/rfc6266.py

@@ -117,7 +117,7 @@ class Language(str):
     """A language-tag (RFC 5646, Section 2.1).
 
     FIXME: This grammar is not 100% correct yet.
-    https://github.com/The-Compiler/qutebrowser/issues/105
+    https://github.com/qutebrowser/qutebrowser/issues/105
     """
 
     grammar = re.compile('[A-Za-z0-9-]+')
@@ -132,7 +132,7 @@ class ValueChars(str):
     """A value of an attribute.
 
     FIXME: Can we merge this with Value?
-    https://github.com/The-Compiler/qutebrowser/issues/105
+    https://github.com/qutebrowser/qutebrowser/issues/105
     """
 
     grammar = re.compile('({}|{})*'.format(attr_char_re, hex_digit_re))

qutebrowser/browser/webkit/tabhistory.py

@@ -21,41 +21,9 @@
 
 
 from PyQt5.QtCore import QByteArray, QDataStream, QIODevice, QUrl
+from PyQt5.QtWebKit import qWebKitVersion
 
-from qutebrowser.utils import utils, qtutils
-
-
-HISTORY_STREAM_VERSION = 2
-BACK_FORWARD_TREE_VERSION = 2
-
-
-class TabHistoryItem:
-
-    """A single item in the tab history.
-
-    Attributes:
-        url: The QUrl of this item.
-        original_url: The QUrl of this item which was originally requested.
-        title: The title as string of this item.
-        active: Whether this item is the item currently navigated to.
-        user_data: The user data for this item.
-    """
-
-    def __init__(self, url, title, *, original_url=None, active=False,
-                 user_data=None):
-        self.url = url
-        if original_url is None:
-            self.original_url = url
-        else:
-            self.original_url = original_url
-        self.title = title
-        self.active = active
-        self.user_data = user_data
-
-    def __repr__(self):
-        return utils.get_repr(self, constructor=True, url=self.url,
-                              original_url=self.original_url, title=self.title,
-                              active=self.active, user_data=self.user_data)
+from qutebrowser.utils import qtutils
 
 
 def _encode_url(url):
@@ -64,7 +32,54 @@ def _encode_url(url):
     return data.decode('ascii')
 
 
-def _serialize_item(i, item, stream):
+def _serialize_ng(items, current_idx, stream):
+    # {'currentItemIndex': 0,
+    #  'history': [{'children': [],
+    #               'documentSequenceNumber': 1485030525573123,
+    #               'documentState': [],
+    #               'formContentType': '',
+    #               'itemSequenceNumber': 1485030525573122,
+    #               'originalURLString': 'about:blank',
+    #               'pageScaleFactor': 0.0,
+    #               'referrer': '',
+    #               'scrollPosition': {'x': 0, 'y': 0},
+    #               'target': '',
+    #               'title': '',
+    #               'urlString': 'about:blank'}]}
+    data = {'currentItemIndex': current_idx, 'history': []}
+    for item in items:
+        data['history'].append(_serialize_item_ng(item))
+
+    stream.writeInt(3)  # history stream version
+    stream.writeQVariantMap(data)
+
+
+def _serialize_item_ng(item):
+    data = {
+        'originalURLString': item.original_url.toString(QUrl.FullyEncoded),
+        'scrollPosition': {'x': 0, 'y': 0},
+        'title': item.title,
+        'urlString': item.url.toString(QUrl.FullyEncoded),
+    }
+    try:
+        data['scrollPosition']['x'] = item.user_data['scroll-pos'].x()
+        data['scrollPosition']['y'] = item.user_data['scroll-pos'].y()
+    except (KeyError, TypeError):
+        pass
+    return data
+
+
+def _serialize_old(items, current_idx, stream):
+    ### Source/WebKit/qt/Api/qwebhistory.cpp operator<<
+    stream.writeInt(2)  # history stream version
+    stream.writeInt(len(items))
+    stream.writeInt(current_idx)
+
+    for i, item in enumerate(items):
+        _serialize_item_old(i, item, stream)
+
+
+def _serialize_item_old(i, item, stream):
     """Serialize a single WebHistoryItem into a QDataStream.
 
     Args:
@@ -82,7 +97,7 @@ def _serialize_item(i, item, stream):
 
     ### Source/WebCore/history/HistoryItem.cpp decodeBackForwardTree
     ## backForwardTreeEncodingVersion
-    stream.writeUInt32(BACK_FORWARD_TREE_VERSION)
+    stream.writeUInt32(2)
     ## size (recursion stack)
     stream.writeUInt64(0)
     ## node->m_documentSequenceNumber
@@ -166,14 +181,12 @@ def serialize(items):
     else:
         current_idx = 0
 
-    ### Source/WebKit/qt/Api/qwebhistory.cpp operator<<
-    stream.writeInt(HISTORY_STREAM_VERSION)
-    stream.writeInt(len(items))
-    stream.writeInt(current_idx)
+    if qtutils.is_qtwebkit_ng(qWebKitVersion()):
+        _serialize_ng(items, current_idx, stream)
+    else:
+        _serialize_old(items, current_idx, stream)
 
-    for i, item in enumerate(items):
-        _serialize_item(i, item, stream)
-        user_data.append(item.user_data)
+    user_data += [item.user_data for item in items]
 
     stream.device().reset()
     qtutils.check_qdatastream(stream)

qutebrowser/browser/webkit/webelem.py

@@ -1,494 +0,0 @@
-# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
-
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
-#
-# This file is part of qutebrowser.
-#
-# qutebrowser is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# qutebrowser is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
-
-"""Utilities related to QWebElements.
-
-Module attributes:
-    Group: Enum for different kinds of groups.
-    SELECTORS: CSS selectors for different groups of elements.
-    FILTERS: A dictionary of filter functions for the modes.
-             The filter for "links" filters javascript:-links and a-tags
-             without "href".
-"""
-
-import collections.abc
-import functools
-
-from PyQt5.QtCore import QRect, QUrl
-from PyQt5.QtWebKit import QWebElement
-
-from qutebrowser.config import config
-from qutebrowser.utils import log, usertypes, utils
-
-
-Group = usertypes.enum('Group', ['all', 'links', 'images', 'url', 'prevnext',
-                                 'focus', 'inputs'])
-
-
-SELECTORS = {
-    Group.all: ('a, area, textarea, select, input:not([type=hidden]), button, '
-                'frame, iframe, link, [onclick], [onmousedown], [role=link], '
-                '[role=option], [role=button], img'),
-    Group.links: 'a, area, link, [role=link]',
-    Group.images: 'img',
-    Group.url: '[src], [href]',
-    Group.prevnext: 'a, area, button, link, [role=button]',
-    Group.focus: '*:focus',
-    Group.inputs: ('input[type=text], input[type=email], input[type=url], '
-                   'input[type=tel], input[type=number], '
-                   'input[type=password], input[type=search], textarea'),
-}
-
-
-def filter_links(elem):
-    return 'href' in elem and QUrl(elem['href']).scheme() != 'javascript'
-
-
-FILTERS = {
-    Group.links: filter_links,
-    Group.prevnext: filter_links,
-}
-
-
-class IsNullError(Exception):
-
-    """Gets raised by WebElementWrapper if an element is null."""
-
-    pass
-
-
-class WebElementWrapper(collections.abc.MutableMapping):
-
-    """A wrapper around QWebElement to make it more intelligent."""
-
-    def __init__(self, elem):
-        if isinstance(elem, self.__class__):
-            raise TypeError("Trying to wrap a wrapper!")
-        if elem.isNull():
-            raise IsNullError('{} is a null element!'.format(elem))
-        self._elem = elem
-        for name in ['addClass', 'appendInside', 'appendOutside',
-                     'attributeNS', 'classes', 'clone', 'document',
-                     'encloseContentsWith', 'encloseWith',
-                     'evaluateJavaScript', 'findAll', 'findFirst',
-                     'firstChild', 'geometry', 'hasAttributeNS',
-                     'hasAttributes', 'hasClass', 'hasFocus', 'lastChild',
-                     'localName', 'namespaceUri', 'nextSibling', 'parent',
-                     'prefix', 'prependInside', 'prependOutside',
-                     'previousSibling', 'removeAllChildren',
-                     'removeAttributeNS', 'removeClass', 'removeFromDocument',
-                     'render', 'replace', 'setAttributeNS', 'setFocus',
-                     'setInnerXml', 'setOuterXml', 'setPlainText',
-                     'setStyleProperty', 'styleProperty', 'tagName',
-                     'takeFromDocument', 'toInnerXml', 'toOuterXml',
-                     'toggleClass', 'webFrame', '__eq__', '__ne__']:
-            # We don't wrap some methods for which we have better alternatives:
-            #   - Mapping access for attributeNames/hasAttribute/setAttribute/
-            #     attribute/removeAttribute.
-            #   - isNull is checked automagically.
-            #   - str(...) instead of toPlainText
-            # For the rest, we create a wrapper which checks if the element is
-            # null.
-
-            method = getattr(self._elem, name)
-
-            def _wrapper(meth, *args, **kwargs):
-                self._check_vanished()
-                return meth(*args, **kwargs)
-
-            wrapper = functools.partial(_wrapper, method)
-            # We used to do functools.update_wrapper here, but for some reason
-            # when using hints with many links, this accounted for nearly 50%
-            # of the time when profiling, which is unacceptable.
-            setattr(self, name, wrapper)
-
-    def __str__(self):
-        self._check_vanished()
-        return self._elem.toPlainText()
-
-    def __repr__(self):
-        try:
-            html = self.debug_text()
-        except IsNullError:
-            html = None
-        return utils.get_repr(self, html=html)
-
-    def __getitem__(self, key):
-        self._check_vanished()
-        if key not in self:
-            raise KeyError(key)
-        return self._elem.attribute(key)
-
-    def __setitem__(self, key, val):
-        self._check_vanished()
-        self._elem.setAttribute(key, val)
-
-    def __delitem__(self, key):
-        self._check_vanished()
-        if key not in self:
-            raise KeyError(key)
-        self._elem.removeAttribute(key)
-
-    def __contains__(self, key):
-        self._check_vanished()
-        return self._elem.hasAttribute(key)
-
-    def __iter__(self):
-        self._check_vanished()
-        yield from self._elem.attributeNames()
-
-    def __len__(self):
-        self._check_vanished()
-        return len(self._elem.attributeNames())
-
-    def _check_vanished(self):
-        """Raise an exception if the element vanished (is null)."""
-        if self._elem.isNull():
-            raise IsNullError('Element {} vanished!'.format(self._elem))
-
-    def is_visible(self, mainframe):
-        """Check whether the element is currently visible on the screen.
-
-        Args:
-            mainframe: The main QWebFrame.
-
-        Return:
-            True if the element is visible, False otherwise.
-        """
-        return is_visible(self._elem, mainframe)
-
-    def rect_on_view(self, **kwargs):
-        """Get the geometry of the element relative to the webview."""
-        return rect_on_view(self._elem, **kwargs)
-
-    def is_writable(self):
-        """Check whether an element is writable."""
-        self._check_vanished()
-        return not ('disabled' in self or 'readonly' in self)
-
-    def is_content_editable(self):
-        """Check if an element has a contenteditable attribute.
-
-        Args:
-            elem: The QWebElement to check.
-
-        Return:
-            True if the element has a contenteditable attribute,
-            False otherwise.
-        """
-        self._check_vanished()
-        try:
-            return self['contenteditable'].lower() not in ['false', 'inherit']
-        except KeyError:
-            return False
-
-    def _is_editable_object(self):
-        """Check if an object-element is editable."""
-        if 'type' not in self:
-            log.webview.debug("<object> without type clicked...")
-            return False
-        objtype = self['type'].lower()
-        if objtype.startswith('application/') or 'classid' in self:
-            # Let's hope flash/java stuff has an application/* mimetype OR
-            # at least a classid attribute. Oh, and let's hope images/...
-            # DON'T have a classid attribute. HTML sucks.
-            log.webview.debug("<object type='{}'> clicked.".format(objtype))
-            return config.get('input', 'insert-mode-on-plugins')
-        else:
-            # Image/Audio/...
-            return False
-
-    def _is_editable_input(self):
-        """Check if an input-element is editable.
-
-        Return:
-            True if the element is editable, False otherwise.
-        """
-        try:
-            objtype = self['type'].lower()
-        except KeyError:
-            return self.is_writable()
-        else:
-            if objtype in ['text', 'email', 'url', 'tel', 'number', 'password',
-                           'search']:
-                return self.is_writable()
-            else:
-                return False
-
-    def _is_editable_div(self):
-        """Check if a div-element is editable.
-
-        Return:
-            True if the element is editable, False otherwise.
-        """
-        # Beginnings of div-classes which are actually some kind of editor.
-        div_classes = ('CodeMirror',  # Javascript editor over a textarea
-                       'kix-',        # Google Docs editor
-                       'ace_')        # http://ace.c9.io/
-        for klass in self._elem.classes():
-            if any([klass.startswith(e) for e in div_classes]):
-                return True
-        return False
-
-    def is_editable(self, strict=False):
-        """Check whether we should switch to insert mode for this element.
-
-        Args:
-            strict: Whether to do stricter checking so only fields where we can
-                    get the value match, for use with the :editor command.
-
-        Return:
-            True if we should switch to insert mode, False otherwise.
-        """
-        self._check_vanished()
-        roles = ('combobox', 'textbox')
-        log.misc.debug("Checking if element is editable: {}".format(
-            repr(self)))
-        tag = self._elem.tagName().lower()
-        if self.is_content_editable() and self.is_writable():
-            return True
-        elif self.get('role', None) in roles and self.is_writable():
-            return True
-        elif tag == 'input':
-            return self._is_editable_input()
-        elif tag == 'textarea':
-            return self.is_writable()
-        elif tag in ['embed', 'applet']:
-            # Flash/Java/...
-            return config.get('input', 'insert-mode-on-plugins') and not strict
-        elif tag == 'object':
-            return self._is_editable_object() and not strict
-        elif tag == 'div':
-            return self._is_editable_div() and not strict
-        else:
-            return False
-
-    def is_text_input(self):
-        """Check if this element is some kind of text box."""
-        self._check_vanished()
-        roles = ('combobox', 'textbox')
-        tag = self._elem.tagName().lower()
-        return self.get('role', None) in roles or tag in ['input', 'textarea']
-
-    def remove_blank_target(self):
-        """Remove target from link."""
-        elem = self._elem
-        for _ in range(5):
-            if elem is None:
-                break
-            tag = elem.tagName().lower()
-            if tag == 'a' or tag == 'area':
-                if elem.attribute('target') == '_blank':
-                    elem.setAttribute('target', '_top')
-                break
-            elem = elem.parent()
-
-    def debug_text(self):
-        """Get a text based on an element suitable for debug output."""
-        self._check_vanished()
-        return utils.compact_text(self._elem.toOuterXml(), 500)
-
-
-def javascript_escape(text):
-    """Escape values special to javascript in strings.
-
-    With this we should be able to use something like:
-      elem.evaluateJavaScript("this.value='{}'".format(javascript_escape(...)))
-    And all values should work.
-    """
-    # This is a list of tuples because order matters, and using OrderedDict
-    # makes no sense because we don't actually need dict-like properties.
-    replacements = (
-        ('\\', r'\\'),  # First escape all literal \ signs as \\.
-        ("'", r"\'"),   # Then escape ' and " as \' and \".
-        ('"', r'\"'),   # (note it won't hurt when we escape the wrong one).
-        ('\n', r'\n'),  # We also need to escape newlines for some reason.
-        ('\r', r'\r'),
-        ('\x00', r'\x00'),
-        ('\ufeff', r'\ufeff'),
-        # http://stackoverflow.com/questions/2965293/
-        ('\u2028', r'\u2028'),
-        ('\u2029', r'\u2029'),
-    )
-    for orig, repl in replacements:
-        text = text.replace(orig, repl)
-    return text
-
-
-def get_child_frames(startframe):
-    """Get all children recursively of a given QWebFrame.
-
-    Loosely based on http://blog.nextgenetics.net/?e=64
-
-    Args:
-        startframe: The QWebFrame to start with.
-
-    Return:
-        A list of children QWebFrame, or an empty list.
-    """
-    results = []
-    frames = [startframe]
-    while frames:
-        new_frames = []
-        for frame in frames:
-            results.append(frame)
-            new_frames += frame.childFrames()
-        frames = new_frames
-    return results
-
-
-def focus_elem(frame):
-    """Get the focused element in a web frame.
-
-    Args:
-        frame: The QWebFrame to search in.
-    """
-    elem = frame.findFirstElement(SELECTORS[Group.focus])
-    return WebElementWrapper(elem)
-
-
-def rect_on_view(elem, *, elem_geometry=None, adjust_zoom=True, no_js=False):
-    """Get the geometry of the element relative to the webview.
-
-    We need this as a standalone function (as opposed to a WebElementWrapper
-    method) because we want to run is_visible before wrapping when hinting for
-    performance reasons.
-
-    Uses the getClientRects() JavaScript method to obtain the collection of
-    rectangles containing the element and returns the first rectangle which is
-    large enough (larger than 1px times 1px). If all rectangles returned by
-    getClientRects() are too small, falls back to elem.rect_on_view().
-
-    Skipping of small rectangles is due to <a> elements containing other
-    elements with "display:block" style, see
-    https://github.com/The-Compiler/qutebrowser/issues/1298
-
-    Args:
-        elem: The QWebElement to get the rect for.
-        elem_geometry: The geometry of the element, or None.
-                       Calling QWebElement::geometry is rather expensive so we
-                       want to avoid doing it twice.
-        adjust_zoom: Whether to adjust the element position based on the
-                     current zoom level.
-        no_js: Fall back to the Python implementation
-    """
-    if elem.isNull():
-        raise IsNullError("Got called on a null element!")
-
-    # First try getting the element rect via JS, as that's usually more
-    # accurate
-    if elem_geometry is None and not no_js:
-        rects = elem.evaluateJavaScript("this.getClientRects()")
-        text = utils.compact_text(elem.toOuterXml(), 500)
-        log.hints.vdebug("Client rectangles of element '{}': {}".format(text,
-                                                                        rects))
-        for i in range(int(rects.get("length", 0))):
-            rect = rects[str(i)]
-            width = rect.get("width", 0)
-            height = rect.get("height", 0)
-            if width > 1 and height > 1:
-                # fix coordinates according to zoom level
-                zoom = elem.webFrame().zoomFactor()
-                if not config.get('ui', 'zoom-text-only') and adjust_zoom:
-                    rect["left"] *= zoom
-                    rect["top"] *= zoom
-                    width *= zoom
-                    height *= zoom
-                rect = QRect(rect["left"], rect["top"], width, height)
-                frame = elem.webFrame()
-                while frame is not None:
-                    # Translate to parent frames' position
-                    # (scroll position is taken care of inside getClientRects)
-                    rect.translate(frame.geometry().topLeft())
-                    frame = frame.parentFrame()
-                return rect
-
-    # No suitable rects found via JS, try via the QWebElement API
-    if elem_geometry is None:
-        geometry = elem.geometry()
-    else:
-        geometry = elem_geometry
-    frame = elem.webFrame()
-    rect = QRect(geometry)
-    while frame is not None:
-        rect.translate(frame.geometry().topLeft())
-        rect.translate(frame.scrollPosition() * -1)
-        frame = frame.parentFrame()
-    # We deliberately always adjust the zoom here, even with adjust_zoom=False
-    if elem_geometry is None:
-        zoom = elem.webFrame().zoomFactor()
-        if not config.get('ui', 'zoom-text-only'):
-            rect.moveTo(rect.left() / zoom, rect.top() / zoom)
-            rect.setWidth(rect.width() / zoom)
-            rect.setHeight(rect.height() / zoom)
-    return rect
-
-
-def is_visible(elem, mainframe):
-    """Check if the given element is visible in the frame.
-
-    We need this as a standalone function (as opposed to a WebElementWrapper
-    method) because we want to check this before wrapping when hinting for
-    performance reasons.
-
-    Args:
-        elem: The QWebElement to check.
-        mainframe: The QWebFrame in which the element should be visible.
-    """
-    if elem.isNull():
-        raise IsNullError("Got called on a null element!")
-    # CSS attributes which hide an element
-    hidden_attributes = {
-        'visibility': 'hidden',
-        'display': 'none',
-    }
-    for k, v in hidden_attributes.items():
-        if elem.styleProperty(k, QWebElement.ComputedStyle) == v:
-            return False
-    elem_geometry = elem.geometry()
-    if not elem_geometry.isValid() and elem_geometry.x() == 0:
-        # Most likely an invisible link
-        return False
-    # First check if the element is visible on screen
-    elem_rect = rect_on_view(elem, elem_geometry=elem_geometry)
-    mainframe_geometry = mainframe.geometry()
-    if elem_rect.isValid():
-        visible_on_screen = mainframe_geometry.intersects(elem_rect)
-    else:
-        # We got an invalid rectangle (width/height 0/0 probably), but this
-        # can still be a valid link.
-        visible_on_screen = mainframe_geometry.contains(elem_rect.topLeft())
-    # Then check if it's visible in its frame if it's not in the main
-    # frame.
-    elem_frame = elem.webFrame()
-    framegeom = QRect(elem_frame.geometry())
-    if not framegeom.isValid():
-        visible_in_frame = False
-    elif elem_frame.parentFrame() is not None:
-        framegeom.moveTo(0, 0)
-        framegeom.translate(elem_frame.scrollPosition())
-        if elem_geometry.isValid():
-            visible_in_frame = framegeom.intersects(elem_geometry)
-        else:
-            # We got an invalid rectangle (width/height 0/0 probably), but
-            # this can still be a valid link.
-            visible_in_frame = framegeom.contains(elem_geometry.topLeft())
-    else:
-        visible_in_frame = visible_on_screen
-    return all([visible_on_screen, visible_in_frame])

qutebrowser/browser/webkit/webkitelem.py

@@ -0,0 +1,344 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""QtWebKit specific part of the web element API."""
+
+from PyQt5.QtCore import QRect
+from PyQt5.QtWebKit import QWebElement, QWebSettings
+
+from qutebrowser.config import config
+from qutebrowser.utils import log, utils, javascript
+from qutebrowser.browser import webelem
+
+
+class IsNullError(webelem.Error):
+
+    """Gets raised by WebKitElement if an element is null."""
+
+    pass
+
+
+class WebKitElement(webelem.AbstractWebElement):
+
+    """A wrapper around a QWebElement."""
+
+    def __init__(self, elem, tab):
+        super().__init__(tab)
+        if isinstance(elem, self.__class__):
+            raise TypeError("Trying to wrap a wrapper!")
+        if elem.isNull():
+            raise IsNullError('{} is a null element!'.format(elem))
+        self._elem = elem
+
+    def __str__(self):
+        self._check_vanished()
+        return self._elem.toPlainText()
+
+    def __eq__(self, other):
+        if not isinstance(other, WebKitElement):
+            return NotImplemented
+        return self._elem == other._elem  # pylint: disable=protected-access
+
+    def __getitem__(self, key):
+        self._check_vanished()
+        if key not in self:
+            raise KeyError(key)
+        return self._elem.attribute(key)
+
+    def __setitem__(self, key, val):
+        self._check_vanished()
+        self._elem.setAttribute(key, val)
+
+    def __delitem__(self, key):
+        self._check_vanished()
+        if key not in self:
+            raise KeyError(key)
+        self._elem.removeAttribute(key)
+
+    def __contains__(self, key):
+        self._check_vanished()
+        return self._elem.hasAttribute(key)
+
+    def __iter__(self):
+        self._check_vanished()
+        yield from self._elem.attributeNames()
+
+    def __len__(self):
+        self._check_vanished()
+        return len(self._elem.attributeNames())
+
+    def _check_vanished(self):
+        """Raise an exception if the element vanished (is null)."""
+        if self._elem.isNull():
+            raise IsNullError('Element {} vanished!'.format(self._elem))
+
+    def has_frame(self):
+        self._check_vanished()
+        return self._elem.webFrame() is not None
+
+    def geometry(self):
+        self._check_vanished()
+        return self._elem.geometry()
+
+    def style_property(self, name, *, strategy):
+        self._check_vanished()
+        strategies = {
+            # FIXME:qtwebengine which ones do we actually need?
+            'inline': QWebElement.InlineStyle,
+            'computed': QWebElement.ComputedStyle,
+        }
+        qt_strategy = strategies[strategy]
+        return self._elem.styleProperty(name, qt_strategy)
+
+    def classes(self):
+        self._check_vanished()
+        return self._elem.classes()
+
+    def tag_name(self):
+        """Get the tag name for the current element."""
+        self._check_vanished()
+        return self._elem.tagName().lower()
+
+    def outer_xml(self):
+        """Get the full HTML representation of this element."""
+        self._check_vanished()
+        return self._elem.toOuterXml()
+
+    def value(self):
+        self._check_vanished()
+        return self._elem.evaluateJavaScript('this.value')
+
+    def set_value(self, value):
+        self._check_vanished()
+        if self.is_content_editable():
+            log.webelem.debug("Filling {!r} via set_text.".format(self))
+            self._elem.setPlainText(value)
+        else:
+            log.webelem.debug("Filling {!r} via javascript.".format(self))
+            value = javascript.string_escape(value)
+            self._elem.evaluateJavaScript("this.value='{}'".format(value))
+
+    def insert_text(self, text):
+        self._check_vanished()
+        if not self.is_editable(strict=True):
+            raise webelem.Error("Element is not editable!")
+        log.webelem.debug("Inserting text into element {!r}".format(self))
+        self._elem.evaluateJavaScript("""
+            var text = "{}";
+            var event = document.createEvent("TextEvent");
+            event.initTextEvent("textInput", true, true, null, text);
+            this.dispatchEvent(event);
+        """.format(javascript.string_escape(text)))
+
+    def _parent(self):
+        """Get the parent element of this element."""
+        self._check_vanished()
+        elem = self._elem.parent()
+        if elem is None or elem.isNull():
+            return None
+        return WebKitElement(elem, tab=self._tab)
+
+    def _rect_on_view_js(self):
+        """Javascript implementation for rect_on_view."""
+        # FIXME:qtwebengine maybe we can reuse this?
+        rects = self._elem.evaluateJavaScript("this.getClientRects()")
+        if rects is None:  # pragma: no cover
+            # On e.g. Void Linux with musl libc, the stack size is too small
+            # for jsc, and running JS will fail. If that happens, fall back to
+            # the Python implementation.
+            # https://github.com/qutebrowser/qutebrowser/issues/1641
+            return None
+
+        text = utils.compact_text(self._elem.toOuterXml(), 500)
+        log.webelem.vdebug("Client rectangles of element '{}': {}".format(
+            text, rects))
+
+        for i in range(int(rects.get("length", 0))):
+            rect = rects[str(i)]
+            width = rect.get("width", 0)
+            height = rect.get("height", 0)
+            if width > 1 and height > 1:
+                # fix coordinates according to zoom level
+                zoom = self._elem.webFrame().zoomFactor()
+                if not config.get('ui', 'zoom-text-only'):
+                    rect["left"] *= zoom
+                    rect["top"] *= zoom
+                    width *= zoom
+                    height *= zoom
+                rect = QRect(rect["left"], rect["top"], width, height)
+                frame = self._elem.webFrame()
+                while frame is not None:
+                    # Translate to parent frames' position (scroll position
+                    # is taken care of inside getClientRects)
+                    rect.translate(frame.geometry().topLeft())
+                    frame = frame.parentFrame()
+                return rect
+
+        return None
+
+    def _rect_on_view_python(self, elem_geometry):
+        """Python implementation for rect_on_view."""
+        if elem_geometry is None:
+            geometry = self._elem.geometry()
+        else:
+            geometry = elem_geometry
+        frame = self._elem.webFrame()
+        rect = QRect(geometry)
+        while frame is not None:
+            rect.translate(frame.geometry().topLeft())
+            rect.translate(frame.scrollPosition() * -1)
+            frame = frame.parentFrame()
+        return rect
+
+    def rect_on_view(self, *, elem_geometry=None, no_js=False):
+        """Get the geometry of the element relative to the webview.
+
+        Uses the getClientRects() JavaScript method to obtain the collection of
+        rectangles containing the element and returns the first rectangle which
+        is large enough (larger than 1px times 1px). If all rectangles returned
+        by getClientRects() are too small, falls back to elem.rect_on_view().
+
+        Skipping of small rectangles is due to <a> elements containing other
+        elements with "display:block" style, see
+        https://github.com/qutebrowser/qutebrowser/issues/1298
+
+        Args:
+            elem_geometry: The geometry of the element, or None.
+                           Calling QWebElement::geometry is rather expensive so
+                           we want to avoid doing it twice.
+            no_js: Fall back to the Python implementation
+        """
+        self._check_vanished()
+
+        # First try getting the element rect via JS, as that's usually more
+        # accurate
+        if elem_geometry is None and not no_js:
+            rect = self._rect_on_view_js()
+            if rect is not None:
+                return rect
+
+        # No suitable rects found via JS, try via the QWebElement API
+        return self._rect_on_view_python(elem_geometry)
+
+    def _is_visible(self, mainframe):
+        """Check if the given element is visible in the given frame.
+
+        This is not public API because it can't be implemented easily here with
+        QtWebEngine, and is only used via find_css(..., only_visible=True) via
+        the tab API.
+        """
+        self._check_vanished()
+        # CSS attributes which hide an element
+        hidden_attributes = {
+            'visibility': 'hidden',
+            'display': 'none',
+            'opacity': '0',
+        }
+        for k, v in hidden_attributes.items():
+            if (self._elem.styleProperty(k, QWebElement.ComputedStyle) == v and
+                    'ace_text-input' not in self.classes()):
+                return False
+
+        elem_geometry = self._elem.geometry()
+        if not elem_geometry.isValid() and elem_geometry.x() == 0:
+            # Most likely an invisible link
+            return False
+        # First check if the element is visible on screen
+        elem_rect = self.rect_on_view(elem_geometry=elem_geometry)
+        mainframe_geometry = mainframe.geometry()
+        if elem_rect.isValid():
+            visible_on_screen = mainframe_geometry.intersects(elem_rect)
+        else:
+            # We got an invalid rectangle (width/height 0/0 probably), but this
+            # can still be a valid link.
+            visible_on_screen = mainframe_geometry.contains(
+                elem_rect.topLeft())
+        # Then check if it's visible in its frame if it's not in the main
+        # frame.
+        elem_frame = self._elem.webFrame()
+        framegeom = QRect(elem_frame.geometry())
+        if not framegeom.isValid():
+            visible_in_frame = False
+        elif elem_frame.parentFrame() is not None:
+            framegeom.moveTo(0, 0)
+            framegeom.translate(elem_frame.scrollPosition())
+            if elem_geometry.isValid():
+                visible_in_frame = framegeom.intersects(elem_geometry)
+            else:
+                # We got an invalid rectangle (width/height 0/0 probably), but
+                # this can still be a valid link.
+                visible_in_frame = framegeom.contains(elem_geometry.topLeft())
+        else:
+            visible_in_frame = visible_on_screen
+        return all([visible_on_screen, visible_in_frame])
+
+    def remove_blank_target(self):
+        elem = self
+        for _ in range(5):
+            if elem is None:
+                break
+            tag = elem.tag_name()
+            if tag == 'a' or tag == 'area':
+                if elem.get('target', None) == '_blank':
+                    elem['target'] = '_top'
+                break
+            elem = elem._parent()  # pylint: disable=protected-access
+
+    def _click_editable(self, click_target):
+        ok = self._elem.evaluateJavaScript('this.focus(); true;')
+        if not ok:
+            log.webelem.debug("Failed to focus via JS, falling back to event")
+            self._click_fake_event(click_target)
+
+    def _click_js(self, click_target):
+        settings = QWebSettings.globalSettings()
+        attribute = QWebSettings.JavascriptCanOpenWindows
+        could_open_windows = settings.testAttribute(attribute)
+        settings.setAttribute(attribute, True)
+        ok = self._elem.evaluateJavaScript('this.click(); true;')
+        settings.setAttribute(attribute, could_open_windows)
+        if not ok:
+            log.webelem.debug("Failed to click via JS, falling back to event")
+            self._click_fake_event(click_target)
+
+    def _click_fake_event(self, click_target):
+        self._tab.data.override_target = click_target
+        super()._click_fake_event(click_target)
+
+
+def get_child_frames(startframe):
+    """Get all children recursively of a given QWebFrame.
+
+    Loosely based on http://blog.nextgenetics.net/?e=64
+
+    Args:
+        startframe: The QWebFrame to start with.
+
+    Return:
+        A list of children QWebFrame, or an empty list.
+    """
+    results = []
+    frames = [startframe]
+    while frames:
+        new_frames = []
+        for frame in frames:
+            results.append(frame)
+            new_frames += frame.childFrames()
+        frames = new_frames
+    return results

qutebrowser/browser/webkit/webkithistory.py

@@ -0,0 +1,61 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""QtWebKit specific part of history."""
+
+
+from PyQt5.QtWebKit import QWebHistoryInterface
+
+
+class WebHistoryInterface(QWebHistoryInterface):
+
+    """Glue code between WebHistory and Qt's QWebHistoryInterface.
+
+    Attributes:
+        _history: The WebHistory object.
+    """
+
+    def __init__(self, webhistory, parent=None):
+        super().__init__(parent)
+        self._history = webhistory
+
+    def addHistoryEntry(self, url_string):
+        """Required for a QWebHistoryInterface impl, obsoleted by add_url."""
+        pass
+
+    def historyContains(self, url_string):
+        """Called by WebKit to determine if a URL is contained in the history.
+
+        Args:
+            url_string: The URL (as string) to check for.
+
+        Return:
+            True if the url is in the history, False otherwise.
+        """
+        return url_string in self._history.history_dict
+
+
+def init(history):
+    """Initialize the QWebHistoryInterface.
+
+    Args:
+        history: The WebHistory object.
+    """
+    interface = WebHistoryInterface(history, parent=history)
+    QWebHistoryInterface.setDefaultInterface(interface)