Remove unused import

https://github.com/pyinstaller/pyinstaller/pull/2519 was merged.
Fixes #2880

(cherry picked from commit ba04822388)

.appveyor.yml

@@ -5,13 +5,15 @@ cache:
 build: off
 environment:
     PYTHONUNBUFFERED: 1
+    PYTHON: C:\Python36\python.exe
     matrix:
-    - TESTENV: py34
-    - TESTENV: unittests-frozen
+    - TESTENV: py36-pyqt58
     - TESTENV: pylint
 
 install:
-  - C:\Python27\python -u scripts\dev\ci\appveyor_install.py
+  - '%PYTHON% -m pip install -U pip'
+  - '%PYTHON% -m pip install -r misc\requirements\requirements-tox.txt'
+  - 'set PATH=%PATH%;C:\Python36'
 
 test_script:
-  - C:\Python34\Scripts\tox -e %TESTENV%
+  - '%PYTHON% -m tox -e %TESTENV%'

.flake8

@@ -35,9 +35,9 @@ max-complexity = 12
 putty-auto-ignore = True
 putty-ignore =
     /# pylint: disable=invalid-name/ : +N801,N806
-    /# pylint: disable=wildcard-import/ : +F403
     /# pragma: no mccabe/ : +C901
     tests/*/test_*.py : +D100,D101,D401
+    tests/conftest.py : +F403
     tests/unit/browser/webkit/test_history.py : +N806
     tests/helpers/fixtures.py : +N806
     tests/unit/browser/webkit/http/test_content_disposition.py : +D400

.github/ISSUE_TEMPLATE.md

@@ -1,2 +1,2 @@
 <!-- If this is a bug report, please remember to mention your version info from
-the `qute:version` page or `qutebrowser --version` -->
+`:open qute:version` or `qutebrowser --version` -->

.pylintrc

@@ -30,15 +30,12 @@ disable=no-self-use,
         broad-except,
         bare-except,
         eval-used,
-        exec-used,
-        file-ignored,
-        wrong-import-order,
         ungrouped-imports,
-        redefined-variable-type,
         suppressed-message,
         too-many-return-statements,
         duplicate-code,
-        wrong-import-position
+        wrong-import-position,
+        no-else-return
 
 [BASIC]
 function-rgx=[a-z_][a-z0-9_]{2,50}$
@@ -52,12 +49,9 @@ no-docstring-rgx=(^_|^main$)
 
 [FORMAT]
 max-line-length=79
-ignore-long-lines=(<?https?://|^# Copyright 201\d|# (pylint|flake8): disable=)
+ignore-long-lines=(<?https?://|^# Copyright 201\d)
 expected-line-ending-format=LF
 
-[SIMILARITIES]
-min-similarity-lines=8
-
 [VARIABLES]
 dummy-variables-rgx=_.*
 
@@ -68,12 +62,10 @@ max-args=10
 valid-metaclass-classmethod-first-arg=cls
 
 [TYPECHECK]
-# WORKAROUND for https://github.com/PyCQA/astroid/pull/357
-ignored-modules=pytest
-# MsgType added as WORKAROUND for
-# https://bitbucket.org/logilab/pylint/issues/690/
-# UnsetObject because pylint infers any objreg.get(...) as UnsetObject.
-ignored-classes=qutebrowser.utils.objreg.UnsetObject,
-                qutebrowser.browser.webkit.webelem.WebElementWrapper,
-                scripts.dev.check_coverage.MsgType,
-                qutebrowser.browser.downloads.UnsupportedAttribute
+ignored-modules=PyQt5,PyQt5.QtWebKit
+
+[IMPORTS]
+# WORKAROUND
+# For some reason, pylint doesn't know about some Python 3 modules on
+# AppVeyor...
+known-standard-library=faulthandler,http,enum,tokenize,posixpath,importlib,types

.pyup.yml

@@ -0,0 +1 @@
+schedule: "every week"

.travis.yml

@@ -1,6 +1,7 @@
 sudo: required
 dist: trusty
 language: generic
+group: edge
 
 matrix:
   include:
@@ -15,9 +16,6 @@ matrix:
     - os: linux
       env: DOCKER=archlinux-webengine QUTE_BDD_WEBENGINE=true
       services: docker
-    - os: linux
-      env: DOCKER=archlinux-ng
-      services: docker
     - os: linux
       env: DOCKER=ubuntu-xenial
       services: docker
@@ -25,14 +23,18 @@ matrix:
       language: python
       python: 3.6
       env: TESTENV=py36-pyqt571
+    - os: linux
+      language: python
+      python: 3.6
+      env: TESTENV=py36-pyqt58
     - os: linux
       language: python
       python: 3.5
-      env: TESTENV=py35-pyqt58
+      env: TESTENV=py35-pyqt59
     - os: linux
       language: python
       python: 3.6
-      env: TESTENV=py36-pyqt58
+      env: TESTENV=py36-pyqt59
     - os: osx
       env: TESTENV=py36 OSX=elcapitan
       osx_image: xcode7.3
@@ -41,7 +43,9 @@ matrix:
     #   env: TESTENV=py35 OSX=yosemite
     #   osx_image: xcode6.4
     - os: linux
-      env: TESTENV=pylint
+      env: TESTENV=pylint PYTHON=python3.6
+      language: python
+      python: 3.6
     - os: linux
       env: TESTENV=flake8
     - os: linux
@@ -73,6 +77,7 @@ before_install:
 
 install:
   - bash scripts/dev/ci/travis_install.sh
+  - ulimit -c unlimited
 
 script:
   - bash scripts/dev/ci/travis_run.sh
@@ -80,6 +85,9 @@ script:
 after_success:
   - '[[ $TESTENV == *-cov ]] && codecov -e TESTENV -X gcov'
 
+after_failure:
+  - bash scripts/dev/ci/travis_backtrace.sh
+
 notifications:
   webhooks:
     - https://buildtimetrend.herokuapp.com/travis

CHANGELOG.asciidoc

@@ -14,6 +14,167 @@ This project adheres to http://semver.org/[Semantic Versioning].
 // `Fixed` for any bug fixes.
 // `Security` to invite users to upgrade in case of vulnerabilities.
 
+v0.11.1
+-------
+
+Fixes
+~~~~~
+
+- Fixed empty space being shown after tabs in the tabbar in some cases.
+- Fixed `:restart` in private browsing mode.
+- Fixed printing on macOS.
+- Closing a pinned tab via mouse now also prompts for confirmation.
+- The "try again" button on error pages works correctly again.
+- :spawn -u -d is now disallowed.
+- :spawn -d shows error messages correctly now.
+
+v0.11.0
+-------
+
+New dependencies
+~~~~~~~~~~~~~~~~
+
+- New dependency on `PyQt5.QtOpenGL` if QtWebEngine is used. QtWebEngine depends
+  on QtOpenGL already, but on distributions packaging split PyQt5 wrappers, the
+  wrappers for QtOpenGL are now required.
+- New dependency on `PyOpenGL` if QtWebEngine is used.
+
+Added
+~~~~~
+
+- Private browsing is now implemented for QtWebEngine, *and changed its
+  behavior*: The `general -> private-browsing` setting now only applies to newly
+  opened windows, and you can use the `-p` flag to `:open` to open a private
+  window.
+- New "pinned tabs" feature, with a new `:tab-pin` command (bound
+  to `<Ctrl-p>` by default).
+- (QtWebEngine) Implemented `:follow-selected`.
+- New `:clear-messages` command to clear shown messages.
+- New `ui -> keyhint-delay` setting to configure the delay until
+  the keyhint overlay pops up.
+- New `-s` option for `:open` to force a HTTPS scheme.
+- `:debug-log-filter` now accepts `none` as an argument to clear any log
+  filters.
+- New `--debug-flag` argument which replaces `--debug-exit` and
+  `--pdb-postmortem`.
+- New `tabs -> favicon-scale` option to scale up/down favicons.
+- `colors -> statusbar.bg/fg.private` and `.command.private` to
+  customize statusbar colors for private windows.
+- New `{private}` field displaying `[Private Mode]` for
+  `ui -> window-title-format` and `tabs -> title-format`.
+- (QtWebEngine) Proxy support with Qt 5.7.1 (already was supported for 5.8 and
+  newer)
+
+Changed
+~~~~~~~
+
+- To prevent elaborate phishing attacks, the Punycode version (`xn--*`) is now
+  shown in addition to the decoded version for international domain names
+  (IDN).
+- Starting with legacy QtWebKit now shows a warning message.
+  *With the next release, support for it will be removed.*
+- The Windows releases are redone from scratch, which means:
+  - They now use the new QtWebEngine backend
+  - The bundled Qt is updated from 5.5 to 5.9
+  - The bundled Python is updated from 3.4 to 3.6
+  - They are now generated with PyInstaller instead of cx_Freeze
+  - The installer is now generated using NSIS instead of being a MSI
+- Improved `qute://history` page (with lazy loading)
+- Crash reports are not public anymore.
+- Paths like `C:` are now treated as absolute paths on Windows for downloads,
+  and invalid paths are handled properly.
+- Comments in the config file are now placed before the individual options
+  instead of being before sections.
+- Messages are now hidden when clicked.
+- stdin is now closed immediately for processes spawned from qutebrowser.
+- When `ui -> message-timeout` is set to 0, messages are now never cleared.
+- Middle/right-clicking the blank parts of the tab bar (when vertical) now
+  closes the current tab.
+- The adblocker now also blocks non-GET requests (e.g. POST).
+- `javascript:` links can now be hinted.
+- `:view-source`, `:tab-clone` and `:navigate --tab` now don't open the tab as
+  "explicit" anymore, i.e. (with the default settings) open it next to the
+  active tab.
+- `qute:*` pages now use `qute://*` instead (e.g. `qute://version` instead of
+  `qute:version`), but the old versions are automatically redirected.
+- Texts in prompts are now selectable.
+- The default level for `:messages` is now `info`, not `error`
+- Trying to focus the currently focused tab with `:tab-focus` now focuses the
+  last viewed tab.
+- (QtWebEngine) With Qt 5.9, `content -> cookies-store` can now be set without
+  a restart.
+- (QtWebEngine) With Qt 5.9, better error messages are now shown for failed
+  downloads.
+- (QtWebEngine) The underlying Chromium version is now shown in the version
+  info.
+- (QtWebKit) Renderer process crashes now show an error page on Qt 5.9 or newer.
+- (QtWebKit) storage -> offline-web-application-storage` got renamed to `...-cache`
+- (QtWebKit) PAC now supports SOCKS5 as type.
+
+Fixed
+~~~~~
+
+- The macOS .dmg is now built against Qt 5.9 which fixes various
+  important issues (such as not being able to type dead keys).
+- Fixed crash with `:download` on PyQt 5.9.
+- Cloning a page without history doesn't crash anymore.
+- When a download results in a HTTP error, it now shows the error correctly
+  instead of crashing.
+- Pressing ctrl-c while a config error is shown works as intended now.
+- When the key config isn't writable, we now show an error instead of crashing.
+- Fixed crash when unbinding an unbound key in the key config.
+- Fixed crash when using `:debug-log-filter` when `--filter` wasn't given on startup.
+- Fixed crash with some invalid setting values.
+- Continuing a search after clearing it now works correctly.
+- The tabbar and completion should now be more consistently and correctly
+  styled with various system styles.
+- Applying styiles in `qt5ct` now shouldn't crash anymore.
+- The validation for colors in stylesheets is now less strict,
+  allowing for all valid Qt values.
+- `data:` URLs now aren't added to the history anymore.
+- Accidentally starting with Python 2 now shows a proper error message again.
+- For some people, running some userscripts crashed - this should now be fixed.
+- Various other rare crashes should now be fixed.
+- The settings documentation was truncated with v0.10.1 which should now be
+  fixed.
+- Scrolling to an anchor in a background tab now works correctly, and javascript
+  gets the correct window size for background tabs.
+- (QtWebEngine) Added a workaround for a black screen with some setups
+- (QtWebEngine) Starting with Nouveau graphics now shows an error message
+  instead of crashing in Qt.
+- (QtWebEngine) Retrying downloads now shows an error instead of crashing.
+- (QtWebEngine) Cloning a view-source tab now doesn't crash anymore.
+- (QtWebEngine) `window.navigator.userAgent` is now set correctly when
+  customizing the user agent.
+- (QtWebEngine) HTML fullscreen is now tracked for each tab separately, which
+  means it's not possible anymore to accidentally get stuck in fullscreen state
+  by closing a tab with a fullscreen video.
+- (QtWebEngine) `:scroll-page` with `--bottom-navigate` now works correctly.
+- (QtWebKit) The HTTP cache is disabled on Qt 5.7.1 and 5.8 now as it leads to
+  frequent crashes due to a Qt bug.
+- (QtWebKit) Fixed Crash when a PAC file returns an invalid value.
+
+v0.10.1
+-------
+
+Changed
+~~~~~~~
+
+- `--qt-arg` and `--qt-flag` can now also be used to pass arguments to Chromium when using QtWebEngine.
+
+Fixed
+~~~~~
+
+- URLs are now redacted properly (username/password, and path/query for HTTPS) when using Proxy Autoconfig with QtWebKit
+- Crash when updating adblock lists with invalid UTF8-chars in them
+- Fixed the web inspector with QtWebEngine
+- Version checks when starting qutebrowser now also take the Qt version PyQt was compiled against into account
+- Hinting a input now doesn't select existing text anymore with QtWebKit
+- The cursor now moves to the end when input elements are selected with QtWebEngine
+- Download suffixes like (1) are now correctly stripped with QtWebEngine
+- Crash when trying to print a tab which was closed in the meantime
+- Crash when trying to open a file twice on Windows
+
 v0.10.0
 -------
 
@@ -26,7 +187,7 @@ Added
 - Various new functionality with the QtWebEngine backend:
     * Printing support with Qt >= 5.8
     * Proxy support with Qt >= 5.8
-    * Tor the `general -> print-element-backgrounds` option with Qt >= 5.8
+    * The `general -> print-element-backgrounds` option with Qt >= 5.8
     * The `content -> cookies-store` option
     * The `storage -> cache-size` option
     * The `colors -> webpage.bg` option
@@ -36,14 +197,14 @@ Added
 - Open tabs are now auto-saved on each successful load and restored in case of a crash
 - `:jseval` now has a `--file` flag so you can pass a javascript file
 - `:session-save` now has a `--only-active-window` flag to only save the active window
-- OS X builds are back, and built with QtWebEngine
+- macOS builds are back, and built with QtWebEngine
 
 Changed
 ~~~~~~~
 
 - PyQt 5.7/Qt 5.7.1 is now required for the QtWebEngine backend
 - Scrolling with the scrollwheel while holding shift now scrolls sideways
-- New way of clicking hints with which solves various small issues
+- New way of clicking hints which solves various small issues
 - When yanking a mailto: link via hints, the mailto: prefix is now stripped
 - Zoom level messages are now not stacked on top of each other anymore
 - qutebrowser now automatically uses QtWebEngine if QtWebKit is unavailable
@@ -52,12 +213,24 @@ Changed
 - `network -> proxy` can also be set to `pac+file://...` now to
   use a local proxy autoconfig file (on QtWebKit)
 
+Removed
+~~~~~~~
+
+- (QtWebKit) Various rarely customized settings were removed:
+  - `ui -> css-media-type` (defaults to desktop)
+  - `general -> site-specific-quirks` (now always turned on)
+  - `storage -> offline-storage-default-quota` (defaults to 5MB)
+  - `storage -> offline-web-application-cache-quota` (defaults to no quota)
+  - `storage -> object-cache-capacities` (default depends on disk space)
+  - `content -> css-regions` (now always turned off)
+  - `storage -> offline-storage-database` (merged into `storage -> local-storage`)
+
 Fixed
 ~~~~~
 
 - Various bugs with Qt 5.8 and QtWebEngine:
     * Segfault when closing a window
-    * Segfault with when closing a tab with a search active
+    * Segfault when closing a tab with a search active
     * Fixed various mouse actions (like automatically entering insert mode) not working
     * Fixed hints sometimes not working
     * Segfault when opening a URL after a QtWebEngine renderer process crash
@@ -326,7 +499,7 @@ Fixed
 - Fix crash when pressing enter without a command
 - Adjust error message to point out QtWebEngine is unsupported with the OS
   X .app currently.
-- Hide Harfbuzz warning with the OS X .app
+- Hide Harfbuzz warning with the macOS .app
 
 v0.8.0
 ------
@@ -689,7 +862,7 @@ Fixed
 - Fixed scrolling to the very left/right with `:scroll-perc`.
 - Using an external editor should now work correctly with some funny chars
   (U+2028/U+2029/BOM).
-- Movements in caret mode now should work correctly on OS X and Windows.
+- Movements in caret mode now should work correctly on macOS and Windows.
 - Fixed upgrade from earlier config versions.
 - Fixed crash when killing a running userscript.
 - Fixed characters being passed through when shifted with
@@ -764,7 +937,7 @@ Changed
 - The completion widget doesn't show a border anymore.
 - The tabbar doesn't display ugly arrows anymore if there isn't enough space
   for all tabs.
-- Some insignificant Qt warnings which were printed on OS X are now hidden.
+- Some insignificant Qt warnings which were printed on macOS are now hidden.
 - Better support for Qt 5.5 and Python 3.5.
 
 Fixed
@@ -875,7 +1048,7 @@ Fixed
 - Fixed AssertionError when closing many windows quickly.
 - Various fixes for deprecated key bindings and auto-migrations.
 - Workaround for qutebrowser not starting when there are NUL-bytes in the history (because of a currently unknown bug).
-- Fixed handling of keybindings containing Ctrl/Meta on OS X.
+- Fixed handling of keybindings containing Ctrl/Meta on macOS.
 - Fixed crash when downloading a URL without filename (e.g. magnet links) via "Save as...".
 - Fixed exception when starting qutebrowser with `:set` as argument.
 - Fixed horrible completion performance when the `shrink` option was set.
@@ -973,7 +1146,7 @@ Changed
 - Add a `:search` command in addition to `/foo` so it's more visible and can be used from scripts.
 - Various improvements to documentation, logging, and the crash reporter.
 - Expand `~` to the users home directory with `:run-userscript`.
-- Improve the userscript runner on Linux/OS X by using `QSocketNotifier`.
+- Improve the userscript runner on Linux/macOS by using `QSocketNotifier`.
 - Add luakit-like `gt`/`gT` keybindings to cycle through tabs.
 - Show default value for config values in the completion.
 - Clone tab icon, tab text and zoom level when cloning tabs.
@@ -993,7 +1166,7 @@ Changed
     * `init_venv.py` and `run_checks.py` have been replaced by http://tox.readthedocs.org/[tox]. Install tox and run `tox -e mkvenv` instead.
     * The tests now use http://pytest.org/[pytest]
     * Many new tests added
-    * Mac Mini buildbot to run the tests on OS X.
+    * Mac Mini buildbot to run the tests on macOS.
     * Coverage recording via http://nedbatchelder.com/code/coverage/[coverage.py].
     * New `--pdb-postmortem argument` to drop into the pdb debugger on exceptions.
     * Use https://github.com/ionelmc/python-hunter[hunter] for line tracing instead of a selfmade solution.
@@ -1129,7 +1302,7 @@ Fixed
 
 * Fix rare exception when a key is pressed shortly after opening a window
 * Fix exception with certain invalid URLs like `http:foo:0`
-* Work around Qt bug which renders checkboxes on OS X unusable
+* Work around Qt bug which renders checkboxes on macOS unusable
 * Fix exception when a local files can't be read in `:adblock-update`
 * Hide 2 more Qt warnings.
 * Add `!important` to hint CSS so websites don't override the hint look
@@ -1165,7 +1338,7 @@ Changes
 * Set zoom to default instead of 100% with `:zoom`/`=`.
 * Adjust page zoom if default zoom changed.
 * Force tabs to be focused on `:undo`.
-* Replace manual installation instructions on OS X with homebrew/macports.
+* Replace manual installation instructions on macOS with homebrew/macports.
 * Allow min-/maximizing of print preview on Windows.
 * Various documentation improvements.
 * Various other small improvements and cleanups.

CONTRIBUTING.asciidoc

@@ -42,6 +42,12 @@ be easy to solve]
 * https://github.com/qutebrowser/qutebrowser/labels/not%20code[Issues which
 require little/no coding]
 
+If you prefer C++ or Javascript to Python, see the relevant issues which involve
+work in those languages:
+
+* https://github.com/qutebrowser/qutebrowser/issues?utf8=%E2%9C%93&q=is%3Aopen%20is%3Aissue%20label%3Ac%2B%2B[C++] (mostly work on Qt, the library behind qutebrowser)
+* https://github.com/qutebrowser/qutebrowser/issues?q=is%3Aopen+is%3Aissue+label%3Ajavascript[JavaScript]
+
 There are also some things to do if you don't want to write code:
 
 * Help the community, e.g., on the mailinglist and the IRC channel.
@@ -214,7 +220,7 @@ Documentation of used Python libraries:
 * http://pygments.org/docs/[pygments]
 * http://fdik.org/pyPEG/index.html[pyPEG2]
 * http://pythonhosted.org/setuptools/[setuptools]
-* http://cx-freeze.readthedocs.org/en/latest/overview.html[cx_Freeze]
+* http://www.pyinstaller.org/[PyInstaller]
 * https://pypi.python.org/pypi/colorama[colorama]
 
 Related RFCs and standards:
@@ -546,6 +552,28 @@ Rebuilding the website
 
 If you want to rebuild the website, run `./scripts/asciidoc2html.py --website <outputdir>`.
 
+Chrome URLs
+~~~~~~~~~~~
+
+With the QtWebEngine backend, qutebrowser supports several chrome:// urls which
+can be useful for debugging:
+
+- chrome://appcache-internals/
+- chrome://blob-internals/
+- chrome://gpu/
+- chrome://histograms/
+- chrome://indexeddb-internals/
+- chrome://media-internals/
+- chrome://network-errors/
+- chrome://serviceworker-internals/
+- chrome://webrtc-internals/
+- chrome://crash/ (crashes the current renderer process!)
+- chrome://kill/ (kills the current renderer process!)
+- chrome://gpucrash/ (crashes qutebrowser!)
+- chrome://gpuhang/ (hangs qutebrowser!)
+- chrome://gpuclean/ (crashes the current renderer process!)
+- chrome://ppapiflashcrash/
+- chrome://ppapiflashhang/
 
 Style conventions
 -----------------
@@ -654,8 +682,9 @@ qutebrowser release
 
 * Add newest config to `tests/unit/config/old_configs` and update `test_upgrade_version`
     - `python -m qutebrowser --basedir conf :quit`
-    - `sed '/^#/d'  conf/config/qutebrowser.conf > tests/unit/config/old_configs/qutebrowser-v0.x.y.conf`
+    - `sed '/^#/d'  conf/config/qutebrowser.conf > tests/unit/config/old_configs/qutebrowser-v0.$x.$y.conf`
     - `rm -r conf`
+    - git add
     - commit
 * Adjust `__version_info__` in `qutebrowser/__init__.py`.
 * Update changelog (remove *(unreleased)*)
@@ -670,7 +699,7 @@ qutebrowser release
 as closed.
 
 * Linux: Run `python3 scripts/dev/build_release.py --upload v0.$x.$y`
-* Windows: Run `C:\Python34_x32\python scripts\dev\build_release.py --asciidoc C:\Python27\python C:\asciidoc-8.6.9\asciidoc.py --upload v0.X.Y` (replace X/Y by hand)
+* Windows: Run `C:\Python36-32\python scripts\dev\build_release.py --asciidoc C:\Python27\python C:\asciidoc-8.6.9\asciidoc.py --upload v0.X.Y` (replace X/Y by hand)
 * OS X: Run `python3 scripts/dev/build_release.py --upload v0.X.Y` (replace X/Y by hand)
 * On server: Run `python3 scripts/dev/download_release.py v0.X.Y` (replace X/Y by hand)
 * Update `qutebrowser-git` PKGBUILD if dependencies/install changed

FAQ.asciidoc

@@ -72,8 +72,8 @@ Is there an adblocker?::
     http://www.reddit.com/r/programming/comments/25j41u/adblock_pluss_effect_on_firefoxs_memory_usage/chhpomw[big
     impact] on browsing speed and
     https://blog.mozilla.org/nnethercote/2014/05/14/adblock-pluss-effect-on-firefoxs-memory-usage/[RAM
-    usage], so implementing it properly might take some time and won't be done
-    for v0.1 if at all.
+    usage], so implementing support for AdBlockPlus-like lists is currently not
+    a priority.
 
 How do I play Youtube videos with mpv?::
     You can easily add a key binding to play youtube videos inside a real video
@@ -124,6 +124,53 @@ When using quickmark, you can give them all names, like
 `:open foodrecipes`, you will see a list of all the food recipe sites,
 without having to remember the exact website title or address.
 
+How do I use spell checking?::
+    Qutebrowser's support for spell checking is somewhat limited at the moment
+    (see https://github.com/qutebrowser/qutebrowser/issues/700[#700]), but it
+    can be done.
++
+For QtWebKit:
+
+. Install https://github.com/QupZilla/qtwebkit-plugins[qtwebkit-plugins].
+  . Note: with QtWebKit reloaded you may experience some issues. See
+    https://github.com/QupZilla/qtwebkit-plugins/issues/10[#10].
+. The dictionary to use is taken from the `DICTIONARY` environment variable.
+  The default is `en_US`. For example to use Dutch spell check set `DICTIONARY`
+  to `nl_NL`; you can't use multiple dictionaries or change them at runtime at
+  the moment.
+  (also see the README file for `qtwebkit-plugins`).
+. Remember to install the hunspell dictionaries if you don't have them already
+  (most distros should have packages for this).
+
++
+For QtWebEngine:
+
+. Not yet supported unfortunately :-( +
+  Adding it shouldn't be too hard though, since QtWebEngine 5.8 added an API for
+  this (see
+  https://github.com/qutebrowser/qutebrowser/issues/700#issuecomment-290780706[this
+  comment for a basic example]), so what are you waiting for and why aren't you
+  hacking qutebrowser yet?
+
+How do I use Tor with qutebrowser?::
+    Start tor on your machine, and do `:set network proxy socks://localhost:9050/`
+    in qutebrowser. Note this won't give you the same amount of fingerprinting
+    protection that the Tor Browser does, but it's useful to be able to access
+    `.onion` sites.
+
+Why does J move to the next (right) tab, and K to the previous (left) one?::
+    One reason is because https://bitbucket.org/portix/dwb[dwb] did it that way,
+    and qutebrowser's keybindings are designed to be compatible with dwb's.
+    The rationale behind it is that J is "down" in vim, and K is "up", which
+    corresponds nicely to "next"/"previous". It also makes much more sense with
+    vertical tabs (e.g. `:set tabs position left`).
+
+What's the difference between insert and passthrough mode?::
+    They are quite similar, but insert mode has some bindings (like `Ctrl-e` to
+    open an editor) while passthrough mode only has escape bound. It might also
+    be useful to rebind escape to something else in passthrough mode only, to be
+    able to send an escape keypress to the website.
+
 == Troubleshooting
 
 Configuration not saved after modifying config.::

INSTALL.asciidoc

@@ -1,6 +1,8 @@
 Installing qutebrowser
 ======================
 
+toc::[]
+
 On Debian / Ubuntu
 ------------------
 
@@ -15,6 +17,10 @@ still relatively easy!
 
 You can use packages that are built for every release or build it yourself from git.
 
+On Ubuntu 16.04 and 16.10 it's recommended to <<tox,install qutebrowser via tox>>
+instead in order to be able to use the new QtWebEngine backend. Newer versions
+have a QtWebEngine package in the repositories.
+
 Using the packages
 ~~~~~~~~~~~~~~~~~~
 
@@ -24,6 +30,12 @@ Install the dependencies via apt-get:
 # apt-get install python3-lxml python-tox python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python3-sip python3-jinja2 python3-pygments python3-yaml
 ----
 
+On Debian Stretch or Ubuntu 17.04 or later, it's also recommended to use the
+newer QtWebEngine backend.
+
+To do so, install `python3-pyqt5.qtwebengine` and `python3-pyqt5.qtopengl`, then
+start qutebrowser with `--backend webengine`.
+
 Get the qutebrowser package from the
 https://github.com/qutebrowser/qutebrowser/releases[release page] and download
 the https://qutebrowser.org/python3-pypeg2_2.15.2-1_all.deb[PyPEG2 package].
@@ -40,43 +52,14 @@ Build it from git
 
 Install the dependencies via apt-get:
 
-[NOTE]
-==========================
-On Debian, it's recommended to install the Qt packages from the
-https://wiki.debian.org/DebianExperimental[experimental] repository as those
-are a much newer version of Qt which is more stable.
-
-Add the following line to your `/etc/apt/sources.list`:
-
-----
-deb http://ftp.debian.org/debian experimental main
-----
-
-Then install the packages like this:
-
-----
-# apt-get update
-# apt-get install -t experimental python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python3-sip python3-dev
-# apt-get install python-tox
-----
-
-It's also recommended to pin those packages to receive updates by creating a
-file `/etc/apt/preferences.d/qutebrowser` with the following contents:
-
-----
-Package: python3-pyqt5* libqt5*
-Pin: release a=experimental
-Pin-Priority: 800
-----
-==========================
-
-For distributions other than Debian or if you prefer to not use the
-experimental repo:
-
 ----
 # apt-get install python3-pyqt5 python3-pyqt5.qtwebkit python3-pyqt5.qtquick python-tox python3-sip python3-dev
 ----
 
+On Debian Stretch or Ubuntu 17.04 or later, it's also recommended to install
+`python3-pyqt5.qtwebengine` and start qutebrowser with `--backend webengine` in
+order to use the new backend.
+
 To generate the documentation for the `:help` command, when using the git
 repository (rather than a release):
 
@@ -102,6 +85,9 @@ qutebrowser is available in the official repositories for Fedora 22 and newer.
 # dnf install qutebrowser
 ----
 
+It's also recommended to install `qt5-qtwebengine` and start with `--backend
+webengine` to use the new backend.
+
 On Archlinux
 ------------
 
@@ -111,6 +97,10 @@ qutebrowser is available in the official [community] repository.
 # pacman -S qutebrowser
 ----
 
+Archlinux packages an updated `qt5-webkit` package by default. If you want to
+use the QtWebEngine backend instead, install `qt5-webengine` and start with
+`--backend webengine`.
+
 There is also a -git version available in the AUR:
 https://aur.archlinux.org/packages/qutebrowser-git/[qutebrowser-git].
 
@@ -135,16 +125,50 @@ If video or sound don't seem to work, try installing the gstreamer plugins:
 On Gentoo
 ---------
 
-qutebrowser is available in the main repository and can be installed with:
+A version of qutebrowser is available in the main repository and can be installed with:
 
 ----
 # emerge -av qutebrowser
 ----
 
+However it is suggested to install the Live version (-9999) to take advantage
+of the newest features introduced.
+
+First of all you need to edit your package.accept_keywords file to accept the live
+version:
+
+----
+# nano /etc/portage/package.accept_keywords
+----
+
+And add the following line to it:
+
+ =www-client/qutebrowser-9999 **
+
+Save the file and then install qutebrowser via
+
+----
+# emerge -av qutebrowser
+----
+
+Or rebuild your system if you already installed it.
+
+To update to the last Live version, remember to do
+
+----
+# emerge -uDNav @live-rebuild @world
+----
+
+To include qutebrowser among the updates.
+
 Make sure you have `python3_4` in your `PYTHON_TARGETS`
 (`/etc/portage/make.conf`) and rebuild your system (`emerge -uDNav @world`) if
 necessary.
 
+It's also recommended to install QtWebKit-NG via
+https://gist.github.com/annulen/309569fb61e5d64a703c055c1e726f71[this ebuild],
+or install Qt >= 5.7.1 with QtWebEngine in order to use an up-to-date backend.
+
 If video or sound don't seem to work, try installing the gstreamer plugins:
 
 ----
@@ -162,6 +186,10 @@ with:
 # xbps-install qutebrowser
 ----
 
+It's currently recommended to install `python3-PyQt5-webengine` and
+`python3-PyQt5-opengl`, then start with `--backend webengine` to use the new
+backend.
+
 On NixOS
 --------
 
@@ -172,6 +200,9 @@ it with:
 $ nix-env -i qutebrowser
 ----
 
+It's recommended to install `qt5.qtwebengine` and start with
+`--backend webengine` to use the new backend.
+
 On openSUSE
 -----------
 
@@ -192,19 +223,19 @@ On OpenBSD
 
 qutebrowser is in http://cvsweb.openbsd.org/cgi-bin/cvsweb/ports/www/qutebrowser/[OpenBSD ports].
 
-Manual install:
+Install the package:
+
+----
+# pkg_add qutebrowser
+----
+
+Or alternatively, use the ports system :
 
 ----
 # cd /usr/ports/www/qutebrowser
 # make install
 ----
 
-Or alternatively if you're using `-current` (or OpenBSD 6.1 once it's been released):
-
-----
-# pkg_add qutebrowser
-----
-
 On Windows
 ----------
 
@@ -213,7 +244,7 @@ There are different ways to install qutebrowser on Windows:
 Prebuilt binaries
 ~~~~~~~~~~~~~~~~~
 
-Prebuilt standalone packages and MSI installers
+Prebuilt standalone packages and installers
 https://github.com/qutebrowser/qutebrowser/releases[are built] for every
 release.
 
@@ -314,17 +345,23 @@ Then run tox inside the qutebrowser repository to set up a
 https://docs.python.org/3/library/venv.html[virtual environment]:
 
 ----
-$ tox -e mkvenv
+$ tox -e mkvenv-pypi
 ----
 
-On Windows, run tox with the 'mkvenv-win' option, however make sure that ONLY Python3 is in your PATH before running tox.
+If your distribution uses OpenSSL 1.1 (like Debian Stretch or Archlinux), you'll
+need to set `LD_LIBRARY_PATH` to the OpenSSL 1.0 directory
+(`export LD_LIBRARY_PATH=/usr/lib/openssl-1.0` on Archlinux) before starting
+qutebrowser.
 
-----
-$ tox -e mkvenv-win
-----
+Alternatively, you can use `tox -e mkvenv` (without `-pypi`) to symlink your
+local Qt install instead of installing PyQt in the virtualenv. However, unless
+you have QtWebKit-NG or QtWebEngine available, qutebrowser will use the legacy
+QtWebKit backend.
 
-This installs all needed Python dependencies in a `.venv` subfolder. The
-system-wide Qt5/PyQt5 installations are symlinked into the virtual environment.
+On Windows, run `tox -e 'mkvenv-win' instead, however make sure that ONLY
+Python3 is in your PATH before running tox.
+
+This installs all needed Python dependencies in a `.venv` subfolder.
 
 You can then create a simple wrapper script to start qutebrowser somewhere in
 your `$PATH` (e.g. `/usr/local/bin/qutebrowser` or `~/bin/qutebrowser`):
@@ -334,14 +371,6 @@ your `$PATH` (e.g. `/usr/local/bin/qutebrowser` or `~/bin/qutebrowser`):
 ~/path/to/qutebrowser/.venv/bin/python3 -m qutebrowser "$@"
 ----
 
-If you are developing on qutebrowser, you may want to redirect it to a local
-config:
-
-----
-#!/bin/bash
-~/path/to/qutebrowser/.venv/bin/python3 -m qutebrowser -c .qutebrowser-local "$@"
-----
-
 Updating
 ~~~~~~~~
 
@@ -352,5 +381,5 @@ virtualenv. Thus it's recommended to run the following command to recreate the
 virtualenv:
 
 ----
-$ tox -r -e mkvenv
+$ tox -r -e mkvenv-pypi
 ----

MANIFEST.in

@@ -30,20 +30,15 @@ prune tests
 prune qutebrowser/3rdparty
 prune misc/requirements
 prune misc/docker
-exclude .editorconfig
 exclude pytest.ini
 exclude qutebrowser.rcc
-exclude .coveragerc
-exclude .pylintrc
 exclude qutebrowser/javascript/.eslintrc.yaml
 exclude qutebrowser/javascript/.eslintignore
 exclude doc/help
-exclude .appveyor.yml
-exclude .travis.yml
+exclude .*
 exclude codecov.yml
-exclude .pydocstylerc
 exclude misc/appveyor_install.py
 exclude misc/qutebrowser.spec
-exclude .flake8
+exclude misc/qutebrowser.nsi
 
 global-exclude __pycache__ *.pyc *.pyo

README.asciidoc

@@ -71,7 +71,8 @@ https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser[mailinglist] at
 mailto:qutebrowser@lists.qutebrowser.org[].
 
 There's also a https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser-announce[announce-only mailinglist]
-at mailto:qutebrowser-announce@lists.qutebrowser.org[].
+at mailto:qutebrowser-announce@lists.qutebrowser.org[] (the announcements also
+get sent to the general qutebrowser@ list).
 
 Contributions / Bugs
 --------------------
@@ -97,11 +98,11 @@ Requirements
 
 The following software and libraries are required to run qutebrowser:
 
-* http://www.python.org/[Python] 3.4 or newer
-* http://qt.io/[Qt] 5.2.0 or newer (5.5.1 recommended)
+* http://www.python.org/[Python] 3.4 or newer (3.5 recommended)
+* http://qt.io/[Qt] 5.2.0 or newer (5.9 recommended)
 * QtWebKit (old or link:https://github.com/annulen/webkit/wiki[reloaded]/NG) or QtWebEngine
 * http://www.riverbankcomputing.com/software/pyqt/intro[PyQt] 5.2.0 or newer
-(5.5.1 recommended) for Python 3
+(5.9 recommended) for Python 3
 * https://pypi.python.org/pypi/setuptools/[pkg_resources/setuptools]
 * http://fdik.org/pyPEG/[pyPEG2]
 * http://jinja.pocoo.org/[jinja2]
@@ -152,44 +153,49 @@ Contributors, sorted by the number of commits in descending order:
 * Lamar Pavel
 * Marshall Lochbaum
 * Bruno Oliveira
+* thuck
+* Martin Tournoij
+* Imran Sobir
 * Alexander Cogneau
 * Felix Van der Jeugt
 * Daniel Karbach
-* Martin Tournoij
 * Kevin Velghe
 * Raphael Pierzina
 * Joel Torstensson
 * Patric Schmitz
 * Tarcisio Fedrizzi
+* Jay Kamat
 * Claude
+* Philipp Hansch
+* Fritz Reichwald
 * Corentin Julé
 * meles5
-* Philipp Hansch
-* Imran Sobir
 * Panagiotis Ktistakis
 * Artur Shaik
 * Nathan Isom
 * Thorsten Wißmann
 * Austin Anderson
-* Fritz Reichwald
 * Jimmy
 * Niklas Haas
 * Maciej Wołczyk
-* Spreadyy
+* Clayton Craft
+* sandrosc
 * Alexey "Averrin" Nabrodov
+* pkill9
 * nanjekyejoannah
 * avk
 * ZDarian
 * Milan Svoboda
 * John ShaggyTwoDope Jenkins
-* Clayton Craft
 * Peter Vilim
+* Jacob Sword
 * knaggita
 * Oliver Caldwell
+* Nikolay Amiantov
+* Marius
 * Julian Weigt
 * Tomasz Kramkowski
 * Sebastian Frysztak
-* Nikolay Amiantov
 * Julie Engel
 * Jonas Schürmann
 * error800
@@ -211,35 +217,38 @@ Contributors, sorted by the number of commits in descending order:
 * Michał Góral
 * Michael Ilsaas
 * Martin Zimmermann
+* Link
 * Jussi Timperi
 * Cosmin Popescu
 * Brian Jackson
-* thuck
 * sbinix
 * rsteube
 * neeasade
 * jnphilipp
+* Yannis Rohloff
 * Tobias Patzl
 * Stefan Tatschner
 * Samuel Loury
 * Peter Michely
 * Panashe M. Fundira
 * Lucas Hoffmann
-* Link
 * Larry Hynes
 * Kirill A. Shutemov
 * Johannes Altmanninger
 * Jeremy Kaplan
 * Ismail
+* Iordanis Grigoriou
 * Edgar Hipp
 * Daryl Finlay
-* pkill9
 * arza
 * adam
 * Samir Benmendil
 * Regina Hug
+* Penaz
+* Matthias Lisin
 * Mathias Fussenegger
 * Marcelo Santos
+* Marcel Schilling
 * Joel Bradshaw
 * Jean-Louis Fuchs
 * Franz Fellner
@@ -252,26 +261,28 @@ Contributors, sorted by the number of commits in descending order:
 * haxwithaxe
 * evan
 * dylan araps
+* caveman
 * addictedtoflames
 * Xitian9
+* Vasilij Schneidermann
 * Tomas Orsava
 * Tom Janson
 * Tobias Werth
 * Tim Harder
 * Thiago Barroso Perrotta
+* Steve Peak
 * Sorokin Alexei
 * Simon Désaulniers
 * Rok Mandeljc
 * Noah Huesser
 * Moez Bouhlel
-* Matthias Lisin
-* Marcel Schilling
+* MikeinRealLife
 * Lazlow Carmichael
 * Kevin Wang
 * Ján Kobezda
+* Justin Partain
 * Johannes Martinsson
 * Jean-Christophe Petkovich
-* Jay Kamat
 * Helen Sherwood-Taylor
 * HalosGhost
 * Gregor Pohl
@@ -279,9 +290,12 @@ Contributors, sorted by the number of commits in descending order:
 * Dietrich Daroch
 * Derek Sivers
 * Daniel Lu
+* Daniel Jakots
 * Arseniy Seroka
+* Anton Grensjö
 * Andy Balaam
 * Andreas Fischer
+* Amos Bird
 * Akselmo
 // QUTE_AUTHORS_END
 

codecov.yml

@@ -1,9 +1,7 @@
-status:
-  project:
-    enabled: no
-  patch:
-    enabled: no
-  changes:
-    enabled: no
+coverage:
+  status:
+    project: off
+    patch: off
+    changes: off
 
 comment: off

doc/backers.asciidoc

@@ -1,13 +1,135 @@
 Crowdfunding backers
 ====================
 
+2017
+----
+
+Mid-2017, qutebrowser had its
+https://www.kickstarter.com/projects/the-compiler/qutebrowser-v10-with-per-domain-settings[second crowdfunding]
+with the goal of implementing the new config system and releasing v1.0.
+
+Thanks a lot to the following people who contributed to it:
+
+Gold sponsors
+~~~~~~~~~~~~~
+
+TODO
+
+Silver sponsors
+~~~~~~~~~~~~~~~
+
+TODO
+
+Other sponsors
+~~~~~~~~~~~~~~
+
+TODO: people with t-shirts or higher pledge levels
+
+- 7scan
+- Alex Suykov
+- Alexey Zhikhartsev
+- Allan Nordhøy
+- Anirudh Sanjeev
+- Anssi Puustinen
+- Benedikt Steindorf
+- Bernardo Kuri
+- Blaise Duszynski
+- Bostan
+- Bruno Oliveira
+- Colin Jacobs
+- Daniel Andersson
+- Danilo
+- David Beley
+- David Hollings
+- David Parrish
+- Derin Yarsuvat
+- Dmytro Kostiuchenko
+- Frederik Thorøe
+- G4v4g4i
+- Gyula Teleki
+- H
+- Hosaka
+- Iordanis Grigoriou
+- Isaac Sandaljian
+- Jakub Podeszwik
+- Jamie Anderson
+- Jasper Woudenberg
+- Jens Højgaard
+- Johannes
+- John Baber-Lucero
+- Jonas Schürmann
+- Kenichiro Ito
+- Kenny Low
+- Lars Ivar Igesund
+- Lucas Aride Moulin
+- Ludovic Chabant
+- Lukas Gierth
+- Marulkan
+- Matthew Chun-Lum
+- Matthew Cronen
+- Matthew Quigley
+- Michael Schönwälder
+- Mika Kutila
+- Mitchell Stokes
+- Nathan Howell
+- Nathan Schlehlein
+- Noël Zindel
+- Obri
+- Patrik Peng
+- Peter DiMarco
+- Peter Rice
+- Philipp Middendorf
+- Pkill9
+- Prescott
+- Robotichead
+- Roshless
+- Ryan Ellis
+- Ryan P Deslandes
+- Sam Doshi
+- Sam Stone
+- Sean Herman
+- Sebastian Frysztak
+- Shelby Cruver
+- SirCmpwn
+- Soham Pal
+- Stewart Webb
+- Sven Reinecke
+- Tom Bass
+- Tomas Slusny
+- Tomasz Kramkowski
+- Tommy Thomas
+- Vasilij Schneidermann
+- Vlaaaaaaad
+- beanieuptop
+- demure
+- evenorbert
+- fishss
+- gsnewmark
+- guillermohs9
+- hubcaps
+- lobachevsky
+- neodarz
+- nihlaeth
+- notbenh
+- patrick suwanvithaya
+- pyratebeard
+- randm_dave
+- sabreman
+- toml
+- vimja
+- wiz
+- 43 Anonymous
+
+2016
+----
+
 Mid-2016, qutebrowser did run a http://igg.me/at/qutebrowser[crowdfunding] for
 QtWebEngine support in qutebrowser.
 
 Thanks a lot to the following people who contributed to it:
 
 Gold sponsors
--------------
+~~~~~~~~~~~~~
 
 - Chris Salzberg
 - Clayton Craft
@@ -16,7 +138,7 @@ Gold sponsors
 - 1 Anonymous
 
 Day sponsors
-------------
+~~~~~~~~~~~~
 
 - Agent 42
 - Iggy Jackson
@@ -28,7 +150,7 @@ Day sponsors
 - 4 Anonymous
 
 Other sponsors
---------------
+~~~~~~~~~~~~~~
 
 - AP M
 - Alessandro Balzano

doc/help/commands.asciidoc

@@ -82,9 +82,10 @@ It is possible to run or bind multiple commands by separating them with `;;`.
 |<<tab-move,tab-move>>|Move the current tab according to the argument and [count].
 |<<tab-next,tab-next>>|Switch to the next tab, or switch [count] tabs forward.
 |<<tab-only,tab-only>>|Close all tabs except for the current one.
+|<<tab-pin,tab-pin>>|Pin/Unpin the current/[count]th tab.
 |<<tab-prev,tab-prev>>|Switch to the previous tab, or switch [count] tabs back.
 |<<unbind,unbind>>|Unbind a keychain.
-|<<undo,undo>>|Re-open a closed tab (optionally skipping [count] closed tabs).
+|<<undo,undo>>|Re-open a closed tab.
 |<<view-source,view-source>>|Show the source of the current page in a new tab.
 |<<window-only,window-only>>|Close all windows except for the current one.
 |<<wq,wq>>|Save open pages and quit.
@@ -544,7 +545,8 @@ For `increment` and `decrement`, the number to change the URL by. For `up`, the
 
 [[open]]
 === open
-Syntax: +:open [*--implicit*] [*--bg*] [*--tab*] [*--window*] ['url']+
+Syntax: +:open [*--implicit*] [*--bg*] [*--tab*] [*--window*] [*--secure*] [*--private*]
+     ['url']+
 
 Open a URL in the current/[count]th tab.
 
@@ -559,6 +561,8 @@ If the URL contains newlines, each line gets opened in its own tab.
 * +*-b*+, +*--bg*+: Open in a new background tab.
 * +*-t*+, +*--tab*+: Open in a new tab.
 * +*-w*+, +*--window*+: Open in a new window.
+* +*-s*+, +*--secure*+: Force HTTPS.
+* +*-p*+, +*--private*+: Open a new window in private browsing mode.
 
 ==== count
 The tab index to open the URL in.
@@ -742,6 +746,7 @@ Load a session.
 [[session-save]]
 === session-save
 Syntax: +:session-save [*--current*] [*--quiet*] [*--force*] [*--only-active-window*]
+             [*--with-private*]
              ['name']+
 
 Save a session.
@@ -755,6 +760,7 @@ Save a session.
 * +*-q*+, +*--quiet*+: Don't show confirmation message.
 * +*-f*+, +*--force*+: Force saving internal sessions (starting with an underline).
 * +*-o*+, +*--only-active-window*+: Saves only tabs of the currently active window.
+* +*-p*+, +*--with-private*+: Include private windows.
 
 [[set]]
 === set
@@ -830,7 +836,7 @@ Duplicate the current tab.
 
 [[tab-close]]
 === tab-close
-Syntax: +:tab-close [*--prev*] [*--next*] [*--opposite*]+
+Syntax: +:tab-close [*--prev*] [*--next*] [*--opposite*] [*--force*]+
 
 Close the current/[count]th tab.
 
@@ -839,6 +845,7 @@ Close the current/[count]th tab.
 * +*-n*+, +*--next*+: Force selecting the tab after the current tab.
 * +*-o*+, +*--opposite*+: Force selecting the tab in the opposite direction of what's configured in 'tabs->select-on-remove'.
 
+* +*-f*+, +*--force*+: Avoid confirmation for pinned tabs.
 
 ==== count
 The tab index to close
@@ -891,13 +898,23 @@ How many tabs to switch forward.
 
 [[tab-only]]
 === tab-only
-Syntax: +:tab-only [*--prev*] [*--next*]+
+Syntax: +:tab-only [*--prev*] [*--next*] [*--force*]+
 
 Close all tabs except for the current one.
 
 ==== optional arguments
 * +*-p*+, +*--prev*+: Keep tabs before the current.
 * +*-n*+, +*--next*+: Keep tabs after the current.
+* +*-f*+, +*--force*+: Avoid confirmation for pinned tabs.
+
+[[tab-pin]]
+=== tab-pin
+Pin/Unpin the current/[count]th tab.
+
+Pinning a tab shrinks it to tabs->pinned-width size. Attempting to close a pinned tab will cause a confirmation, unless --force is passed.
+
+==== count
+The tab index to pin or unpin
 
 [[tab-prev]]
 === tab-prev
@@ -919,7 +936,7 @@ Unbind a keychain.
 
 [[undo]]
 === undo
-Re-open a closed tab (optionally skipping [count] closed tabs).
+Re-open a closed tab.
 
 [[view-source]]
 === view-source
@@ -995,6 +1012,7 @@ How many steps to zoom out.
 |==============
 |Command|Description
 |<<clear-keychain,clear-keychain>>|Clear the currently entered key chain.
+|<<clear-messages,clear-messages>>|Clear all message notifications.
 |<<click-element,click-element>>|Click the element matching the given filter.
 |<<command-accept,command-accept>>|Execute the command currently in the commandline.
 |<<command-history-next,command-history-next>>|Go forward in the commandline history.
@@ -1059,6 +1077,10 @@ How many steps to zoom out.
 === clear-keychain
 Clear the currently entered key chain.
 
+[[clear-messages]]
+=== clear-messages
+Clear all message notifications.
+
 [[click-element]]
 === click-element
 Syntax: +:click-element [*--target* 'target'] [*--force-event*] 'filter' 'value'+
@@ -1433,6 +1455,8 @@ Syntax: +:scroll 'direction'+
 
 Scroll the current tab in the given direction.
 
+Note you can use `:run-with-count` to have a keybinding with a bigger scroll increment.
+
 ==== positional arguments
 * +'direction'+: In which direction to scroll (up/down/left/right/top/bottom).
 
@@ -1591,7 +1615,8 @@ Syntax: +:debug-log-filter 'filters'+
 Change the log filter for console logging.
 
 ==== positional arguments
-* +'filters'+: A comma separated list of logger names.
+* +'filters'+: A comma separated list of logger names. Can also be "none" to clear any existing filters.
+
 
 [[debug-log-level]]
 === debug-log-level
@@ -1646,7 +1671,7 @@ Syntax: +:debug-webaction 'action'+
 
 Execute a webaction.
 
-See http://doc.qt.io/qt-5/qwebpage.html#WebAction-enum for the available actions.
+Available actions: http://doc.qt.io/archives/qt-5.5/qwebpage.html#WebAction-enum (WebKit) http://doc.qt.io/qt-5/qwebenginepage.html#WebAction-enum (WebEngine)
 
 ==== positional arguments
 * +'action'+: The action to execute, e.g. MoveToNextChar.

doc/help/settings.asciidoc

@@ -18,11 +18,10 @@
 |<<general-auto-save-interval,auto-save-interval>>|How often (in milliseconds) to auto-save config/cookies/etc.
 |<<general-editor,editor>>|The editor (and arguments) to use for the `open-editor` command.
 |<<general-editor-encoding,editor-encoding>>|Encoding to use for editor.
-|<<general-private-browsing,private-browsing>>|Do not record visited pages in the history or store web page icons.
+|<<general-private-browsing,private-browsing>>|Open new windows in private browsing mode which does not record visited pages.
 |<<general-developer-extras,developer-extras>>|Enable extra tools for Web developers.
 |<<general-print-element-backgrounds,print-element-backgrounds>>|Whether the background color and images are also drawn when the page is printed.
 |<<general-xss-auditing,xss-auditing>>|Whether load requests should be monitored for cross-site scripting attempts.
-|<<general-site-specific-quirks,site-specific-quirks>>|Enable QtWebKit workarounds for broken sites.
 |<<general-default-encoding,default-encoding>>|Default encoding to use for websites.
 |<<general-new-instance-open-target,new-instance-open-target>>|How to open links in an existing instance if a new one is launched.
 |<<general-new-instance-open-target.window,new-instance-open-target.window>>|Which window to choose when opening links as new tabs.
@@ -36,6 +35,7 @@
 [options="header",width="75%",cols="25%,75%"]
 |==============
 |Setting|Description
+|<<ui-history-session-interval,history-session-interval>>|The maximum time in minutes between two history items for them to be considered being from the same session. Use -1 to disable separation.
 |<<ui-zoom-levels,zoom-levels>>|The available zoom levels, separated by commas.
 |<<ui-default-zoom,default-zoom>>|The default zoom level.
 |<<ui-downloads-position,downloads-position>>|Where to show the downloaded files.
@@ -47,8 +47,7 @@
 |<<ui-frame-flattening,frame-flattening>>|Whether to  expand each subframe to its contents.
 |<<ui-user-stylesheet,user-stylesheet>>|User stylesheet to use (absolute filename or filename relative to the config directory). Will expand environment variables.
 |<<ui-hide-scrollbar,hide-scrollbar>>|Hide the main scrollbar.
-|<<ui-css-media-type,css-media-type>>|Set the CSS media type.
-|<<ui-smooth-scrolling,smooth-scrolling>>|Whether to enable smooth scrolling for webpages.
+|<<ui-smooth-scrolling,smooth-scrolling>>|Whether to enable smooth scrolling for web pages. Note smooth scrolling does not work with the :scroll-px command.
 |<<ui-remove-finished-downloads,remove-finished-downloads>>|Number of milliseconds to wait before removing finished downloads. Will not be removed if value is -1.
 |<<ui-hide-statusbar,hide-statusbar>>|Whether to hide the statusbar unless a message is shown.
 |<<ui-statusbar-padding,statusbar-padding>>|Padding for statusbar (top, bottom, left, right).
@@ -56,6 +55,7 @@
 |<<ui-modal-js-dialog,modal-js-dialog>>|Use standard JavaScript modal dialog for alert() and confirm()
 |<<ui-hide-wayland-decoration,hide-wayland-decoration>>|Hide the window decoration when using wayland (requires restart)
 |<<ui-keyhint-blacklist,keyhint-blacklist>>|Keychains that shouldn't be shown in the keyhint dialog
+|<<ui-keyhint-delay,keyhint-delay>>|Time from pressing a key to seeing the keyhint dialog (ms)
 |<<ui-prompt-radius,prompt-radius>>|The rounding radius for the edges of prompts.
 |<<ui-prompt-filebrowser,prompt-filebrowser>>|Show a filebrowser in upload/download prompts.
 |==============
@@ -124,10 +124,13 @@
 |<<tabs-close-mouse-button,close-mouse-button>>|On which mouse button to close tabs.
 |<<tabs-position,position>>|The position of the tab bar.
 |<<tabs-show-favicons,show-favicons>>|Whether to show favicons in the tab bar.
+|<<tabs-favicon-scale,favicon-scale>>|Scale for favicons in the tab bar. The tab size is unchanged, so big favicons also require extra `tabs->padding`.
 |<<tabs-width,width>>|The width of the tab bar if it's vertical, in px or as percentage of the window.
+|<<tabs-pinned-width,pinned-width>>|The width for pinned tabs with a horizontal tabbar, in px.
 |<<tabs-indicator-width,indicator-width>>|Width of the progress indicator (0 to disable).
 |<<tabs-tabs-are-windows,tabs-are-windows>>|Whether to open windows instead of tabs.
 |<<tabs-title-format,title-format>>|The format to use for the tab title. The following placeholders are defined:
+|<<tabs-title-format-pinned,title-format-pinned>>|The format to use for the tab title for pinned tabs. The same placeholders like for title-format are defined.
 |<<tabs-title-alignment,title-alignment>>|Alignment of the text inside of tabs
 |<<tabs-mousewheel-tab-switching,mousewheel-tab-switching>>|Switch between tabs using the mouse wheel.
 |<<tabs-padding,padding>>|Padding for tabs (top, bottom, left, right).
@@ -142,12 +145,8 @@
 |<<storage-prompt-download-directory,prompt-download-directory>>|Whether to prompt the user for the download location.
 |<<storage-remember-download-directory,remember-download-directory>>|Whether to remember the last used download directory.
 |<<storage-maximum-pages-in-cache,maximum-pages-in-cache>>|The maximum number of pages to hold in the global memory page cache.
-|<<storage-object-cache-capacities,object-cache-capacities>>|The capacities for the global memory cache for dead objects such as stylesheets or scripts. Syntax: cacheMinDeadCapacity, cacheMaxDead, totalCapacity.
-|<<storage-offline-storage-default-quota,offline-storage-default-quota>>|Default quota for new offline storage databases.
-|<<storage-offline-web-application-cache-quota,offline-web-application-cache-quota>>|Quota for the offline web application cache.
-|<<storage-offline-storage-database,offline-storage-database>>|Whether support for the HTML 5 offline storage feature is enabled.
-|<<storage-offline-web-application-storage,offline-web-application-storage>>|Whether support for the HTML 5 web application cache feature is enabled.
-|<<storage-local-storage,local-storage>>|Whether support for the HTML 5 local storage feature is enabled.
+|<<storage-offline-web-application-cache,offline-web-application-cache>>|Whether support for the HTML 5 web application cache feature is enabled.
+|<<storage-local-storage,local-storage>>|Whether support for HTML 5 local storage and Web SQL is enabled.
 |<<storage-cache-size,cache-size>>|Size of the HTTP network cache. Empty to use the default value.
 |==============
 
@@ -159,7 +158,6 @@
 |<<content-allow-javascript,allow-javascript>>|Enables or disables the running of JavaScript programs.
 |<<content-allow-plugins,allow-plugins>>|Enables or disables plugins in Web pages.
 |<<content-webgl,webgl>>|Enables or disables WebGL.
-|<<content-css-regions,css-regions>>|Enable or disable support for CSS regions.
 |<<content-hyperlink-auditing,hyperlink-auditing>>|Enable or disable hyperlink auditing (<a ping>).
 |<<content-geolocation,geolocation>>|Allow websites to request geolocations.
 |<<content-notifications,notifications>>|Allow websites to show notifications.
@@ -172,7 +170,7 @@
 |<<content-local-content-can-access-remote-urls,local-content-can-access-remote-urls>>|Whether locally loaded documents are allowed to access remote urls.
 |<<content-local-content-can-access-file-urls,local-content-can-access-file-urls>>|Whether locally loaded documents are allowed to access other local urls.
 |<<content-cookies-accept,cookies-accept>>|Control which cookies to accept.
-|<<content-cookies-store,cookies-store>>|Whether to store cookies. Note this option needs a restart with QtWebEngine.
+|<<content-cookies-store,cookies-store>>|Whether to store cookies. Note this option needs a restart with QtWebEngine on Qt < 5.9.
 |<<content-host-block-lists,host-block-lists>>|List of URLs of lists which contain hosts to block.
 |<<content-host-blocking-enabled,host-blocking-enabled>>|Whether host blocking is enabled.
 |<<content-host-blocking-whitelist,host-blocking-whitelist>>|List of domains that should always be loaded, despite being ad-blocked.
@@ -218,10 +216,14 @@
 |<<colors-completion.scrollbar.bg,completion.scrollbar.bg>>|Color of the scrollbar in completion view
 |<<colors-statusbar.fg,statusbar.fg>>|Foreground color of the statusbar.
 |<<colors-statusbar.bg,statusbar.bg>>|Background color of the statusbar.
+|<<colors-statusbar.fg.private,statusbar.fg.private>>|Foreground color of the statusbar in private browsing mode.
+|<<colors-statusbar.bg.private,statusbar.bg.private>>|Background color of the statusbar in private browsing mode.
 |<<colors-statusbar.fg.insert,statusbar.fg.insert>>|Foreground color of the statusbar in insert mode.
 |<<colors-statusbar.bg.insert,statusbar.bg.insert>>|Background color of the statusbar in insert mode.
 |<<colors-statusbar.fg.command,statusbar.fg.command>>|Foreground color of the statusbar in command mode.
 |<<colors-statusbar.bg.command,statusbar.bg.command>>|Background color of the statusbar in command mode.
+|<<colors-statusbar.fg.command.private,statusbar.fg.command.private>>|Foreground color of the statusbar in private browsing + command mode.
+|<<colors-statusbar.bg.command.private,statusbar.bg.command.private>>|Background color of the statusbar in private browsing + command mode.
 |<<colors-statusbar.fg.caret,statusbar.fg.caret>>|Foreground color of the statusbar in caret mode.
 |<<colors-statusbar.bg.caret,statusbar.bg.caret>>|Background color of the statusbar in caret mode.
 |<<colors-statusbar.fg.caret-selection,statusbar.fg.caret-selection>>|Foreground color of the statusbar in caret mode with a selection
@@ -392,7 +394,7 @@ Default: +pass:[utf-8]+
 
 [[general-private-browsing]]
 === private-browsing
-Do not record visited pages in the history or store web page icons.
+Open new windows in private browsing mode which does not record visited pages.
 
 Valid values:
 
@@ -401,8 +403,6 @@ Valid values:
 
 Default: +pass:[false]+
 
-This setting is only available with the QtWebKit backend.
-
 [[general-developer-extras]]
 === developer-extras
 Enable extra tools for Web developers.
@@ -443,26 +443,13 @@ Valid values:
 
 Default: +pass:[false]+
 
-[[general-site-specific-quirks]]
-=== site-specific-quirks
-Enable QtWebKit workarounds for broken sites.
-
-Valid values:
-
- * +true+
- * +false+
-
-Default: +pass:[true]+
-
-This setting is only available with the QtWebKit backend.
-
 [[general-default-encoding]]
 === default-encoding
 Default encoding to use for websites.
 
-The encoding must be a string describing an encoding such as _utf-8_, _iso-8859-1_, etc. If left empty a default value will be used.
+The encoding must be a string describing an encoding such as _utf-8_, _iso-8859-1_, etc.
 
-Default: empty
+Default: +pass:[iso-8859-1]+
 
 [[general-new-instance-open-target]]
 === new-instance-open-target
@@ -536,6 +523,12 @@ Default: +pass:[path,query]+
 == ui
 General options related to the user interface.
 
+[[ui-history-session-interval]]
+=== history-session-interval
+The maximum time in minutes between two history items for them to be considered being from the same session. Use -1 to disable separation.
+
+Default: +pass:[30]+
+
 [[ui-zoom-levels]]
 === zoom-levels
 The available zoom levels, separated by commas.
@@ -573,6 +566,7 @@ Default: +pass:[bottom]+
 [[ui-message-timeout]]
 === message-timeout
 Time (in ms) to show messages in the statusbar for.
+Set to 0 to never clear messages.
 
 Default: +pass:[2000]+
 
@@ -645,17 +639,9 @@ Valid values:
 
 Default: +pass:[true]+
 
-[[ui-css-media-type]]
-=== css-media-type
-Set the CSS media type.
-
-Default: empty
-
-This setting is only available with the QtWebKit backend.
-
 [[ui-smooth-scrolling]]
 === smooth-scrolling
-Whether to enable smooth scrolling for webpages.
+Whether to enable smooth scrolling for web pages. Note smooth scrolling does not work with the :scroll-px command.
 
 Valid values:
 
@@ -699,6 +685,8 @@ The format to use for the window title. The following placeholders are defined:
 * `{scroll_pos}`: The page scroll position.
 * `{host}`: The host of the current web page.
 * `{backend}`: Either 'webkit' or 'webengine'
+* `{private}` : Indicates when private mode is enabled.
+
 
 Default: +pass:[{perc}{title}{title_sep}qutebrowser]+
 
@@ -732,6 +720,12 @@ Globs are supported, so ';*' will blacklist all keychainsstarting with ';'. Use
 
 Default: empty
 
+[[ui-keyhint-delay]]
+=== keyhint-delay
+Time from pressing a key to seeing the keyhint dialog (ms)
+
+Default: +pass:[500]+
+
 [[ui-prompt-radius]]
 === prompt-radius
 The rounding radius for the edges of prompts.
@@ -795,8 +789,6 @@ The proxy to use.
 
 In addition to the listed values, you can use a `socks://...` or `http://...` URL.
 
-This setting only works with Qt 5.8 or newer when using the QtWebEngine backend.
-
 Valid values:
 
  * +system+: Use the system wide proxy.
@@ -1191,12 +1183,24 @@ Valid values:
 
 Default: +pass:[true]+
 
+[[tabs-favicon-scale]]
+=== favicon-scale
+Scale for favicons in the tab bar. The tab size is unchanged, so big favicons also require extra `tabs->padding`.
+
+Default: +pass:[1.0]+
+
 [[tabs-width]]
 === width
 The width of the tab bar if it's vertical, in px or as percentage of the window.
 
 Default: +pass:[20%]+
 
+[[tabs-pinned-width]]
+=== pinned-width
+The width for pinned tabs with a horizontal tabbar, in px.
+
+Default: +pass:[43]+
+
 [[tabs-indicator-width]]
 === indicator-width
 Width of the progress indicator (0 to disable).
@@ -1227,9 +1231,17 @@ The format to use for the tab title. The following placeholders are defined:
 * `{scroll_pos}`: The page scroll position.
 * `{host}`: The host of the current web page.
 * `{backend}`: Either 'webkit' or 'webengine'
+* `{private}` : Indicates when private mode is enabled.
+
 
 Default: +pass:[{index}: {title}]+
 
+[[tabs-title-format-pinned]]
+=== title-format-pinned
+The format to use for the tab title for pinned tabs. The same placeholders like for title-format are defined.
+
+Default: +pass:[{index}]+
+
 [[tabs-title-alignment]]
 === title-alignment
 Alignment of the text inside of tabs
@@ -1305,55 +1317,12 @@ The Page Cache allows for a nicer user experience when navigating forth or back
 
 For more information about the feature, please refer to: http://webkit.org/blog/427/webkit-page-cache-i-the-basics/
 
-Default: empty
+Default: +pass:[0]+
 
 This setting is only available with the QtWebKit backend.
 
-[[storage-object-cache-capacities]]
-=== object-cache-capacities
-The capacities for the global memory cache for dead objects such as stylesheets or scripts. Syntax: cacheMinDeadCapacity, cacheMaxDead, totalCapacity.
-
-The _cacheMinDeadCapacity_ specifies the minimum number of bytes that dead objects should consume when the cache is under pressure.
-
-_cacheMaxDead_ is the maximum number of bytes that dead objects should consume when the cache is *not* under pressure.
-
-_totalCapacity_ specifies the maximum number of bytes that the cache should consume *overall*.
-
-Default: empty
-
-This setting is only available with the QtWebKit backend.
-
-[[storage-offline-storage-default-quota]]
-=== offline-storage-default-quota
-Default quota for new offline storage databases.
-
-Default: empty
-
-This setting is only available with the QtWebKit backend.
-
-[[storage-offline-web-application-cache-quota]]
-=== offline-web-application-cache-quota
-Quota for the offline web application cache.
-
-Default: empty
-
-This setting is only available with the QtWebKit backend.
-
-[[storage-offline-storage-database]]
-=== offline-storage-database
-Whether support for the HTML 5 offline storage feature is enabled.
-
-Valid values:
-
- * +true+
- * +false+
-
-Default: +pass:[true]+
-
-This setting is only available with the QtWebKit backend.
-
-[[storage-offline-web-application-storage]]
-=== offline-web-application-storage
+[[storage-offline-web-application-cache]]
+=== offline-web-application-cache
 Whether support for the HTML 5 web application cache feature is enabled.
 
 An application cache acts like an HTTP cache in some sense. For documents that use the application cache via JavaScript, the loader engine will first ask the application cache for the contents, before hitting the network.
@@ -1371,7 +1340,7 @@ This setting is only available with the QtWebKit backend.
 
 [[storage-local-storage]]
 === local-storage
-Whether support for the HTML 5 local storage feature is enabled.
+Whether support for HTML 5 local storage and Web SQL is enabled.
 
 Valid values:
 
@@ -1435,19 +1404,6 @@ Valid values:
 
 Default: +pass:[true]+
 
-[[content-css-regions]]
-=== css-regions
-Enable or disable support for CSS regions.
-
-Valid values:
-
- * +true+
- * +false+
-
-Default: +pass:[true]+
-
-This setting is only available with the QtWebKit backend.
-
 [[content-hyperlink-auditing]]
 === hyperlink-auditing
 Enable or disable hyperlink auditing (<a ping>).
@@ -1524,6 +1480,7 @@ This setting is only available with the QtWebKit backend.
 [[content-javascript-can-access-clipboard]]
 === javascript-can-access-clipboard
 Whether JavaScript programs can read or write to the clipboard.
+With QtWebEngine, writing the clipboard as response to a user interaction is always allowed.
 
 Valid values:
 
@@ -1593,7 +1550,7 @@ This setting is only available with the QtWebKit backend.
 
 [[content-cookies-store]]
 === cookies-store
-Whether to store cookies. Note this option needs a restart with QtWebEngine.
+Whether to store cookies. Note this option needs a restart with QtWebEngine on Qt < 5.9.
 
 Valid values:
 
@@ -1883,6 +1840,18 @@ Background color of the statusbar.
 
 Default: +pass:[black]+
 
+[[colors-statusbar.fg.private]]
+=== statusbar.fg.private
+Foreground color of the statusbar in private browsing mode.
+
+Default: +pass:[${statusbar.fg}]+
+
+[[colors-statusbar.bg.private]]
+=== statusbar.bg.private
+Background color of the statusbar in private browsing mode.
+
+Default: +pass:[#666666]+
+
 [[colors-statusbar.fg.insert]]
 === statusbar.fg.insert
 Foreground color of the statusbar in insert mode.
@@ -1907,6 +1876,18 @@ Background color of the statusbar in command mode.
 
 Default: +pass:[${statusbar.bg}]+
 
+[[colors-statusbar.fg.command.private]]
+=== statusbar.fg.command.private
+Foreground color of the statusbar in private browsing + command mode.
+
+Default: +pass:[${statusbar.fg.private}]+
+
+[[colors-statusbar.bg.command.private]]
+=== statusbar.bg.command.private
+Background color of the statusbar in private browsing + command mode.
+
+Default: +pass:[${statusbar.bg.private}]+
+
 [[colors-statusbar.fg.caret]]
 === statusbar.fg.caret
 Foreground color of the statusbar in caret mode.
@@ -2251,7 +2232,7 @@ Fonts used for the UI, with optional style/weight/size.
 === _monospace
 Default monospace fonts.
 
-Default: +pass:[Terminus, Monospace, &quot;DejaVu Sans Mono&quot;, Monaco, &quot;Bitstream Vera Sans Mono&quot;, &quot;Andale Mono&quot;, &quot;Courier New&quot;, Courier, &quot;Liberation Mono&quot;, monospace, Fixed, Consolas, Terminal]+
+Default: +pass:[xos4 Terminus, Terminus, Monospace, &quot;DejaVu Sans Mono&quot;, Monaco, &quot;Bitstream Vera Sans Mono&quot;, &quot;Andale Mono&quot;, &quot;Courier New&quot;, Courier, &quot;Liberation Mono&quot;, monospace, Fixed, Consolas, Terminal]+
 
 [[fonts-completion]]
 === completion
@@ -2335,25 +2316,25 @@ Default: empty
 === web-size-minimum
 The hard minimum font size.
 
-Default: empty
+Default: +pass:[0]+
 
 [[fonts-web-size-minimum-logical]]
 === web-size-minimum-logical
 The minimum logical font size that is applied when zooming out.
 
-Default: empty
+Default: +pass:[6]+
 
 [[fonts-web-size-default]]
 === web-size-default
 The default font size for regular text.
 
-Default: empty
+Default: +pass:[16]+
 
 [[fonts-web-size-default-fixed]]
 === web-size-default-fixed
 The default font size for fixed-pitch text.
 
-Default: empty
+Default: +pass:[13]+
 
 [[fonts-keyhint]]
 === keyhint

doc/quickstart.asciidoc

@@ -31,7 +31,7 @@ image:http://qutebrowser.org/img/cheatsheet-small.png["qutebrowser key binding c
 * Run `:adblock-update` to download adblock lists and activate adblocking.
 * If you just cloned the repository, you'll need to run
 `scripts/asciidoc2html.py` to generate the documentation.
-* Go to the link:qute://settings[settings page] to set up qutebrowser the way you want it.
+* Go to the link:qute://settings[settings page] to set up qutebrowser the way you want it. (Currently not available with the QtWebEngine backend and on the OS X build - use the `:set` command instead)
 * Subscribe to
 https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser[the mailinglist] or
 https://lists.schokokeks.org/mailman/listinfo.cgi/qutebrowser-announce[the announce-only mailinglist].

doc/qutebrowser.1.asciidoc

@@ -57,7 +57,7 @@ show it.
     How URLs should be opened if there is already a qutebrowser instance running.
 
 *--backend* '{webkit,webengine}'::
-    Which backend to use (webengine backend is EXPERIMENTAL!).
+    Which backend to use.
 
 *--enable-webengine-inspector*::
     Enable the web inspector for QtWebEngine. Note that this is a SECURITY RISK and you should not visit untrusted websites with the inspector turned on. See https://bugreports.qt.io/browse/QTBUG-50725 for more details.
@@ -93,12 +93,6 @@ show it.
 *--nowindow*::
     Don't show the main window.
 
-*--debug-exit*::
-    Turn on debugging of late exit.
-
-*--pdb-postmortem*::
-    Drop into pdb on exceptions.
-
 *--temp-basedir*::
     Use a temporary basedir.
 
@@ -110,6 +104,9 @@ show it.
 
 *--qt-flag* 'QT_FLAG'::
     Pass an argument to Qt as flag.
+
+*--debug-flag* 'DEBUG_FLAGS'::
+    Pass name of debugging feature to be turned on.
 // QUTE_OPTIONS_END
 
 == FILES

misc/qutebrowser.nsi

@@ -0,0 +1,77 @@
+Name "qutebrowser"
+
+Unicode true
+RequestExecutionLevel admin
+SetCompressor /solid lzma
+
+!ifdef X64
+  OutFile "..\dist\qutebrowser-${VERSION}-amd64.exe"
+  InstallDir "$ProgramFiles64\qutebrowser"
+!else
+  OutFile "..\dist\qutebrowser-${VERSION}-win32.exe"
+  InstallDir "$ProgramFiles\qutebrowser"
+!endif
+
+;Default installation folder
+  
+!include "MUI2.nsh"
+;!include "MultiUser.nsh"
+
+!define MUI_ABORTWARNING
+;!define MULTIUSER_MUI
+;!define MULTIUSER_INSTALLMODE_COMMANDLINE
+!define MUI_ICON "../icons/qutebrowser.ico"
+!define MUI_UNICON "../icons/qutebrowser.ico"
+
+!insertmacro MUI_PAGE_LICENSE "..\COPYING"
+!insertmacro MUI_PAGE_DIRECTORY
+!insertmacro MUI_PAGE_INSTFILES
+!insertmacro MUI_UNPAGE_CONFIRM
+!insertmacro MUI_UNPAGE_INSTFILES
+
+!insertmacro MUI_LANGUAGE "English"
+
+; depends on admin status
+;SetShellVarContext current
+
+
+Section "Install"
+
+  ; Uninstall old versions
+  ExecWait 'MsiExec.exe /quiet /qn /norestart /X{633F41F9-FE9B-42D1-9CC4-718CBD01EE11}'
+  ExecWait 'MsiExec.exe /quiet /qn /norestart /X{9331D947-AC86-4542-A755-A833429C6E69}'
+
+  SetOutPath "$INSTDIR"
+  
+  !ifdef X64
+	file /r "..\dist\qutebrowser-${VERSION}-x64\*.*"
+  !else
+	file /r "..\dist\qutebrowser-${VERSION}-x86\*.*"
+  !endif
+
+  SetShellVarContext all
+  CreateShortCut "$SMPROGRAMS\qutebrowser.lnk" "$INSTDIR\qutebrowser.exe"
+  
+  ;Create uninstaller
+  WriteUninstaller "$INSTDIR\uninst.exe"
+
+  WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\qutebrowser" "DisplayName" "qutebrowser"
+  WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\qutebrowser" "UninstallString" '"$INSTDIR\uninst.exe"'
+  WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\qutebrowser" "QuietUninstallString" '"$INSTDIR\uninst.exe" /S'
+
+SectionEnd
+
+;--------------------------------
+;Uninstaller Section
+
+Section "Uninstall"
+
+  SetShellVarContext all
+  Delete "$SMPROGRAMS\qutebrowser.lnk"
+
+  RMDir /r "$INSTDIR\*.*"
+  RMDir "$INSTDIR"
+
+  DeleteRegKey HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\qutebrowser"
+
+SectionEnd

misc/qutebrowser.spec

@@ -18,10 +18,10 @@ def get_data_files():
         ('../qutebrowser/git-commit-id', '')
     ]
 
-    if os.path.exists(os.path.join('qutebrowser', '3rdparty', 'pdfjs')):
-        data_files.append(('../qutebrowser/3rdparty/pdfjs', '3rdparty/pdfjs'))
-    else:
-        print("Warning: excluding pdfjs as it's not present!")
+    # if os.path.exists(os.path.join('qutebrowser', '3rdparty', 'pdfjs')):
+    #     data_files.append(('../qutebrowser/3rdparty/pdfjs', '3rdparty/pdfjs'))
+    # else:
+    #     print("Warning: excluding pdfjs as it's not present!")
 
     return data_files
 
@@ -41,10 +41,10 @@ a = Analysis(['../qutebrowser/__main__.py'],
              pathex=['misc'],
              binaries=None,
              datas=get_data_files(),
-             hiddenimports=[],
+             hiddenimports=['PyQt5.QtOpenGL', 'PyQt5._QOpenGLFunctions_2_0'],
              hookspath=[],
              runtime_hooks=[],
-             excludes=[],
+             excludes=['tkinter'],
              win_no_prefer_redirects=False,
              win_private_assemblies=False,
              cipher=block_cipher)

misc/requirements/requirements-codecov.txt

@@ -1,5 +1,9 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-codecov==2.0.5
-coverage==4.3.4
-requests==2.13.0
+certifi==2017.4.17
+chardet==3.0.4
+codecov==2.0.9
+coverage==4.4.1
+idna==2.5
+requests==2.18.1
+urllib3==1.21.1

misc/requirements/requirements-cxfreeze.txt-raw

@@ -1,5 +0,0 @@
-cx-Freeze < 5.0.0
-
-# We'll probably switch to PyInstaller soon, and 5.x doesn't install without a
-# compiler?
-#@ filter: cx-Freeze < 5.0.0

misc/requirements/requirements-flake8.txt

@@ -3,18 +3,21 @@
 flake8==2.6.2  # rq.filter: < 3.0.0
 flake8-copyright==0.2.0
 flake8-debugger==1.4.0  # rq.filter: != 2.0.0
-flake8-deprecated==1.1
-flake8-docstrings==1.0.3
+flake8-deprecated==1.2
+flake8-docstrings==1.0.3  # rq.filter: < 1.1.0
 flake8-future-import==0.4.3
 flake8-mock==0.3
-flake8-pep3101==1.0
+flake8-pep3101==1.0  # rq.filter: < 1.1
 flake8-polyfill==1.0.1
 flake8-putty==0.4.0
 flake8-string-format==0.2.3
-flake8-tidy-imports==1.0.5
-flake8-tuple==0.2.12
+flake8-tidy-imports==1.0.6
+flake8-tuple==0.2.13
 mccabe==0.6.1
+packaging==16.8
 pep8-naming==0.4.1
 pycodestyle==2.3.1
-pydocstyle==1.1.1
+pydocstyle==1.1.1  # rq.filter: < 2.0.0
 pyflakes==1.5.0
+pyparsing==2.2.0
+six==1.10.0

misc/requirements/requirements-flake8.txt-raw

@@ -2,16 +2,16 @@ flake8<3.0.0
 flake8-copyright
 flake8-debugger!=2.0.0
 flake8-deprecated
-flake8-docstrings
+flake8-docstrings<1.1.0
 flake8-future-import
 flake8-mock
-flake8-pep3101
+flake8-pep3101<1.1
 flake8-putty
 flake8-string-format
 flake8-tidy-imports
 flake8-tuple
 pep8-naming
-pydocstyle
+pydocstyle<2.0.0
 pyflakes
 
 # Pinned to 2.0.0 otherwise
@@ -21,6 +21,9 @@ mccabe==0.6.1
 
 # Waiting until flake8-putty updated
 #@ filter: flake8 < 3.0.0
+#@ filter: pydocstyle < 2.0.0
+#@ filter: flake8-docstrings < 1.1.0
+#@ filter: flake8-pep3101 < 1.1
 
 # https://github.com/JBKahn/flake8-debugger/issues/5
 #@ filter: flake8-debugger != 2.0.0

misc/requirements/requirements-pip.txt

@@ -1,8 +1,8 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-appdirs==1.4.2
+appdirs==1.4.3
 packaging==16.8
-pyparsing==2.1.10
-setuptools==34.3.0
+pyparsing==2.2.0
+setuptools==36.0.1
 six==1.10.0
 wheel==0.29.0

misc/requirements/requirements-pyinstaller.txt

@@ -1,5 +1,3 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
 -e git+https://github.com/pyinstaller/pyinstaller.git@develop#egg=PyInstaller
-PyQt5==5.8
-sip==4.19.1

misc/requirements/requirements-pyinstaller.txt-raw

@@ -1,5 +1,4 @@
 -e git+https://github.com/pyinstaller/pyinstaller.git@develop#egg=PyInstaller
-PyQt5
 
 # remove @commit-id for scm installs
-#@ replace: @.*# @develop#
+#@ replace: @.*# @develop#

misc/requirements/requirements-pylint-master.txt

@@ -1,11 +1,18 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
 -e git+https://github.com/PyCQA/astroid.git#egg=astroid
-editdistance==0.3.1
-isort==4.2.5
-lazy-object-proxy==1.2.2
+certifi==2017.4.17
+chardet==3.0.4
+github3.py==0.9.6
+idna==2.5
+isort==4.2.15
+lazy-object-proxy==1.3.1
 mccabe==0.6.1
 -e git+https://github.com/PyCQA/pylint.git#egg=pylint
 ./scripts/dev/pylint_checkers
-requests==2.13.0
-wrapt==1.10.8
+requests==2.18.1
+six==1.10.0
+uritemplate==3.0.0
+uritemplate.py==3.0.2
+urllib3==1.21.1
+wrapt==1.10.10

misc/requirements/requirements-pylint-master.txt-raw

@@ -2,6 +2,7 @@
 -e git+https://github.com/PyCQA/pylint.git#egg=pylint
 ./scripts/dev/pylint_checkers
 requests
+github3.py
 
 # remove @commit-id for scm installs
 #@ replace: @.*# #

misc/requirements/requirements-pylint.txt

@@ -1,13 +1,18 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-astroid==1.4.9
+astroid==1.5.3
+certifi==2017.4.17
+chardet==3.0.4
 github3.py==0.9.6
-isort==4.2.5
-lazy-object-proxy==1.2.2
+idna==2.5
+isort==4.2.15
+lazy-object-proxy==1.3.1
 mccabe==0.6.1
-pylint==1.6.5
+pylint==1.7.2
 ./scripts/dev/pylint_checkers
-requests==2.13.0
+requests==2.18.1
+six==1.10.0
 uritemplate==3.0.0
 uritemplate.py==3.0.2
-wrapt==1.10.8
+urllib3==1.21.1
+wrapt==1.10.10

misc/requirements/requirements-pyqt.txt

@@ -1,3 +1,4 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-cx-Freeze==4.3.4  # rq.filter: < 5.0.0
+PyQt5==5.9
+sip==4.19.3

misc/requirements/requirements-pyqt.txt-raw

@@ -0,0 +1 @@
+PyQt5

misc/requirements/requirements-tests-git.txt

@@ -27,7 +27,6 @@ git+https://github.com/pytest-dev/pytest-qt.git
 git+https://github.com/pytest-dev/pytest-repeat.git
 git+https://github.com/pytest-dev/pytest-rerunfailures.git
 git+https://github.com/abusalimov/pytest-travis-fold.git
-git+https://github.com/fschulze/pytest-warnings.git
 git+https://github.com/The-Compiler/pytest-xvfb.git
 hg+https://bitbucket.org/gutworth/six
 hg+https://bitbucket.org/jendrikseipp/vulture

misc/requirements/requirements-tests.txt

@@ -1,36 +1,39 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-beautifulsoup4==4.5.3
-cheroot==5.1.0
+beautifulsoup4==4.6.0
+cheroot==5.7.0
 click==6.7
-coverage==4.3.4
+# colorama==0.3.9
+coverage==4.4.1
 decorator==4.0.11
 EasyProcess==0.2.3
-Flask==0.12
+fields==5.0.0
+Flask==0.12.2
 glob2==0.5
 httpbin==0.5.0
-hypothesis==3.6.1
+hunter==1.4.1
+hypothesis==3.11.6
 itsdangerous==0.24
-# Jinja2==2.9.5
+# Jinja2==2.9.6
 Mako==1.0.6
-# MarkupSafe==0.23
-parse==1.6.6
+# MarkupSafe==1.0
+parse==1.8.2
 parse-type==0.3.4
-py==1.4.32
-pytest==3.0.6
-pytest-bdd==2.18.1
+py==1.4.34
+pytest==3.1.2
+pytest-bdd==2.18.2
 pytest-benchmark==3.0.0
 pytest-catchlog==1.2.2
-pytest-cov==2.4.0
+pytest-cov==2.5.1
 pytest-faulthandler==1.3.1
 pytest-instafail==0.3.0
-pytest-mock==1.5.0
+pytest-mock==1.6.0
 pytest-qt==2.1.0
 pytest-repeat==0.4.1
-pytest-rerunfailures==2.1.0
+pytest-rerunfailures==2.2
 pytest-travis-fold==1.2.0
-pytest-warnings==0.2.0
 pytest-xvfb==1.0.0
 PyVirtualDisplay==0.2.1
-vulture==0.12
-Werkzeug==0.11.15
+six==1.10.0
+vulture==0.14
+Werkzeug==0.12.2

misc/requirements/requirements-tests.txt-raw

@@ -2,6 +2,7 @@ beautifulsoup4
 cheroot
 coverage
 Flask
+hunter
 httpbin
 hypothesis
 pytest
@@ -16,8 +17,7 @@ pytest-qt
 pytest-repeat
 pytest-rerunfailures
 pytest-travis-fold
-pytest-warnings
 pytest-xvfb
 vulture
 
-#@ ignore: Jinja2, MarkupSafe
+#@ ignore: Jinja2, MarkupSafe, colorama

misc/requirements/requirements-tox.txt

@@ -1,6 +1,6 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
 pluggy==0.4.0
-py==1.4.32
-tox==2.6.0
+py==1.4.34
+tox==2.7.0
 virtualenv==15.1.0

misc/requirements/requirements-vulture.txt

@@ -1,3 +1,3 @@
 # This file is automatically generated by scripts/dev/recompile_requirements.py
 
-vulture==0.12
+vulture==0.14

misc/userscripts/dmenu_qutebrowser

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 # Copyright 2015 Zach-Button <zachrey.button@gmail.com>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

misc/userscripts/openfeeds

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 # Copyright 2015 jnphilipp <me@jnphilipp.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

misc/userscripts/password_fill

@@ -9,7 +9,7 @@ directly ask me via IRC (nickname thorsten\`) in #qutebrowser on freenode.
 
   $blink!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!$reset
   WARNING: the passwords are stored in qutebrowser's
-           debug log reachable via the url qute:log
+           debug log reachable via the url qute://log
   $blink!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!$reset
 
 Usage: run as a userscript form qutebrowser, e.g.:

misc/userscripts/qutebrowser_viewsource

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 # Copyright 2015 Zach-Button <zachrey.button@gmail.com>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

misc/userscripts/readability

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 #
 # Executes python-readability on current page and opens the summary as new tab.
 #

misc/userscripts/ripbang

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 #
 # Adds DuckDuckGo bang as searchengine.
 #
@@ -8,14 +8,21 @@
 # Example:
 #   :spawn --userscript ripbang amazon maps
 #
-import os, re, requests, sys, urllib
+
+from __future__ import print_function
+import os, re, requests, sys
+
+try:
+    from urllib.parse import unquote
+except ImportError:
+    from urllib import unquote
 
 for argument in sys.argv[1:]:
     bang = '!' + argument
     r = requests.get('https://duckduckgo.com/',
                      params={'q': bang + ' SEARCHTEXT'})
 
-    searchengine = urllib.unquote(re.search("url=[^']+", r.text).group(0))
+    searchengine = unquote(re.search("url=[^']+", r.text).group(0))
     searchengine = searchengine.replace('url=', '')
     searchengine = searchengine.replace('/l/?kh=-1&uddg=', '')
     searchengine = searchengine.replace('SEARCHTEXT', '{}')
@@ -24,4 +31,4 @@ for argument in sys.argv[1:]:
         with open(os.environ['QUTE_FIFO'], 'w') as fifo:
             fifo.write('set searchengines %s %s' % (bang, searchengine))
     else:
-        print '%s %s' % (bang, searchengine)
+        print('%s %s' % (bang, searchengine))

pytest.ini

@@ -24,11 +24,12 @@ markers =
     js_prompt: Tests needing to display a javascript prompt
     this: Used to mark tests during development
     no_invalid_lines: Don't fail on unparseable lines in end2end tests
+    issue2478: Tests which are broken on Windows with QtWebEngine, https://github.com/qutebrowser/qutebrowser/issues/2478
 qt_log_level_fail = WARNING
 qt_log_ignore =
     ^SpellCheck: .*
     ^SetProcessDpiAwareness failed: .*
-    ^QWindowsWindow::setGeometryDp: Unable to set geometry .*
+    ^QWindowsWindow::setGeometry(Dp)?: Unable to set geometry .*
     ^QProcess: Destroyed while process .* is still running\.
     ^"Method "GetAll" with signature "s" on interface "org\.freedesktop\.DBus\.Properties" doesn't exist
     ^"Method \\"GetAll\\" with signature \\"s\\" on interface \\"org\.freedesktop\.DBus\.Properties\\" doesn't exist\\n"
@@ -44,6 +45,8 @@ qt_log_ignore =
     ^QStandardPaths: XDG_RUNTIME_DIR not set, defaulting to .*
     ^QGeoclueMaster error creating GeoclueMasterClient\.
     ^Geoclue error: Process org\.freedesktop\.Geoclue\.Master exited with status 127
+    ^QDBusConnection: name 'org.freedesktop.Geoclue.Master' had owner '' but we thought it was ':1.1'
+    ^Failed to create Geoclue client interface. Geoclue error: org\.freedesktop\.DBus\.Error\.Disconnected
     ^QObject::connect: Cannot connect \(null\)::stateChanged\(QNetworkSession::State\) to QNetworkReplyHttpImpl::_q_networkSessionStateChanged\(QNetworkSession::State\)
     ^QXcbClipboard: Cannot transfer data, no data available
     ^load glyph failed
@@ -51,4 +54,6 @@ qt_log_ignore =
     ^Image of format '' blocked because it is not considered safe. If you are sure it is safe to do so, you can white-list the format by setting the environment variable QTWEBKIT_IMAGEFORMAT_WHITELIST=
     ^QPainter::end: Painter ended with \d+ saved states
     ^QSslSocket: cannot resolve SSLv[23]_(client|server)_method
+    ^QQuickWidget::invalidateRenderControl could not make context current
+    ^libpng warning: iCCP: known incorrect sRGB profile
 xfail_strict = true

qutebrowser/__init__.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -22,11 +22,11 @@
 import os.path
 
 __author__ = "Florian Bruhin"
-__copyright__ = "Copyright 2014-2016 Florian Bruhin (The Compiler)"
+__copyright__ = "Copyright 2014-2017 Florian Bruhin (The Compiler)"
 __license__ = "GPL"
 __maintainer__ = __author__
 __email__ = "mail@qutebrowser.org"
-__version_info__ = (0, 10, 0)
+__version_info__ = (0, 11, 1)
 __version__ = '.'.join(str(e) for e in __version_info__)
 __description__ = "A keyboard-driven, vim-like browser based on PyQt5."
 

qutebrowser/__main__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/app.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -53,7 +53,7 @@ from qutebrowser.browser.webkit.network import networkmanager
 from qutebrowser.keyinput import macros
 from qutebrowser.mainwindow import mainwindow, prompt
 from qutebrowser.misc import (readline, ipc, savemanager, sessions,
-                              crashsignal, earlyinit)
+                              crashsignal, earlyinit, objects)
 from qutebrowser.misc import utilcmds  # pylint: disable=unused-import
 from qutebrowser.utils import (log, version, message, utils, qtutils, urlutils,
                                objreg, usertypes, standarddir, error, debug)
@@ -133,18 +133,14 @@ def init(args, crash_handler):
     log.init.debug("Starting init...")
     qApp.setQuitOnLastWindowClosed(False)
     _init_icon()
-    utils.actute_warning()
 
     try:
         _init_modules(args, crash_handler)
-    except (OSError, UnicodeDecodeError) as e:
+    except (OSError, UnicodeDecodeError, browsertab.WebTabError) as e:
         error.handle_fatal_exc(e, args, "Error while initializing!",
                                pre_text="Error while initializing")
         sys.exit(usertypes.Exit.err_init)
 
-    QTimer.singleShot(0, functools.partial(_process_args, args))
-    QTimer.singleShot(10, functools.partial(_init_late_modules, args))
-
     log.init.debug("Initializing eventfilter...")
     event_filter = EventFilter(qApp)
     qApp.installEventFilter(event_filter)
@@ -155,11 +151,13 @@ def init(args, crash_handler):
     config_obj.style_changed.connect(style.get_stylesheet.cache_clear)
     qApp.focusChanged.connect(on_focus_changed)
 
+    _process_args(args)
+
     QDesktopServices.setUrlHandler('http', open_desktopservices_url)
     QDesktopServices.setUrlHandler('https', open_desktopservices_url)
     QDesktopServices.setUrlHandler('qute', open_desktopservices_url)
 
-    macros.init()
+    QTimer.singleShot(10, functools.partial(_init_late_modules, args))
 
     log.init.debug("Init done!")
     crash_handler.raise_crashdlg()
@@ -172,12 +170,15 @@ def _init_icon():
     for size in [16, 24, 32, 48, 64, 96, 128, 256, 512]:
         filename = ':/icons/qutebrowser-{}x{}.png'.format(size, size)
         pixmap = QPixmap(filename)
-        qtutils.ensure_not_null(pixmap)
-        fallback_icon.addPixmap(pixmap)
-    qtutils.ensure_not_null(fallback_icon)
+        if pixmap.isNull():
+            log.init.warning("Failed to load {}".format(filename))
+        else:
+            fallback_icon.addPixmap(pixmap)
     icon = QIcon.fromTheme('qutebrowser', fallback_icon)
-    qtutils.ensure_not_null(icon)
-    qApp.setWindowIcon(icon)
+    if icon.isNull():
+        log.init.warning("Failed to load icon")
+    else:
+        qApp.setWindowIcon(icon)
 
 
 def _process_args(args):
@@ -194,14 +195,14 @@ def _process_args(args):
     session_manager = objreg.get('session-manager')
     if not session_manager.did_load:
         log.init.debug("Initializing main window...")
-        window = mainwindow.MainWindow()
+        window = mainwindow.MainWindow(private=None)
         if not args.nowindow:
             window.show()
         qApp.setActiveWindow(window)
 
     process_pos_args(args.command)
     _open_startpage()
-    _open_quickstart(args)
+    _open_special_pages(args)
 
     delta = datetime.datetime.now() - earlyinit.START_TIME
     log.init.debug("Init finished after {}s".format(delta.total_seconds()))
@@ -303,7 +304,7 @@ def _open_startpage(win_id=None):
         window_ids = [win_id]
     else:
         window_ids = objreg.window_registry
-    for cur_win_id in window_ids:
+    for cur_win_id in list(window_ids):  # Copying as the dict could change
         tabbed_browser = objreg.get('tabbed-browser', scope='window',
                                     window=cur_win_id)
         if tabbed_browser.count() == 0:
@@ -318,23 +319,40 @@ def _open_startpage(win_id=None):
                     tabbed_browser.tabopen(url)
 
 
-def _open_quickstart(args):
-    """Open quickstart if it's the first start.
+def _open_special_pages(args):
+    """Open special notification pages which are only shown once.
+
+    Currently this is:
+      - Quickstart page if it's the first start.
+      - Legacy QtWebKit warning if needed.
 
     Args:
         args: The argparse namespace.
     """
     if args.basedir is not None:
-        # With --basedir given, don't open quickstart.
+        # With --basedir given, don't open anything.
         return
+
     state_config = objreg.get('state-config')
-    try:
-        quickstart_done = state_config['general']['quickstart-done'] == '1'
-    except KeyError:
-        quickstart_done = False
+    tabbed_browser = objreg.get('tabbed-browser', scope='window',
+                                window='last-focused')
+
+    # Legacy QtWebKit warning
+
+    needs_warning = (objects.backend == usertypes.Backend.QtWebKit and
+                     not qtutils.is_qtwebkit_ng())
+    warning_shown = state_config['general'].get('backend-warning-shown') == '1'
+
+    if not warning_shown and needs_warning:
+        tabbed_browser.tabopen(QUrl('qute://backend-warning'),
+                               background=False)
+        state_config['general']['backend-warning-shown'] = '1'
+
+    # Quickstart page
+
+    quickstart_done = state_config['general'].get('quickstart-done') == '1'
+
     if not quickstart_done:
-        tabbed_browser = objreg.get('tabbed-browser', scope='window',
-                                    window='last-focused')
         tabbed_browser.tabopen(
             QUrl('https://www.qutebrowser.org/quickstart.html'))
         state_config['general']['quickstart-done'] = '1'
@@ -342,8 +360,9 @@ def _open_quickstart(args):
 
 def _save_version():
     """Save the current version to the state config."""
-    state_config = objreg.get('state-config')
-    state_config['general']['version'] = qutebrowser.__version__
+    state_config = objreg.get('state-config', None)
+    if state_config is not None:
+        state_config['general']['version'] = qutebrowser.__version__
 
 
 def on_focus_changed(_old, new):
@@ -391,10 +410,8 @@ def _init_modules(args, crash_handler):
     log.init.debug("Initializing network...")
     networkmanager.init()
 
-    if qtutils.version_check('5.8'):
-        # Otherwise we can only initialize it for QtWebKit because of crashes
-        log.init.debug("Initializing proxy...")
-        proxy.init()
+    log.init.debug("Initializing proxy...")
+    proxy.init()
 
     log.init.debug("Initializing readline-bridge...")
     readline_bridge = readline.ReadlineBridge()
@@ -448,6 +465,7 @@ def _init_modules(args, crash_handler):
         os.environ['QT_WAYLAND_DISABLE_WINDOWDECORATION'] = '1'
     else:
         os.environ.pop('QT_WAYLAND_DISABLE_WINDOWDECORATION', None)
+    macros.init()
     # Init backend-specific stuff
     browsertab.init()
 
@@ -616,7 +634,7 @@ class Quitter:
         # Save the session if one is given.
         if session is not None:
             session_manager = objreg.get('session-manager')
-            session_manager.save(session)
+            session_manager.save(session, with_private=True)
         # Open a new process and immediately shutdown the existing one
         try:
             args, cwd = self._get_restart_args(pages, session)
@@ -648,14 +666,14 @@ class Quitter:
         self._shutting_down = True
         log.destroy.debug("Shutting down with status {}, session {}...".format(
             status, session))
-
-        session_manager = objreg.get('session-manager')
-        if session is not None:
-            session_manager.save(session, last_window=last_window,
-                                 load_next_time=True)
-        elif config.get('general', 'save-session'):
-            session_manager.save(sessions.default, last_window=last_window,
-                                 load_next_time=True)
+        session_manager = objreg.get('session-manager', None)
+        if session_manager is not None:
+            if session is not None:
+                session_manager.save(session, last_window=last_window,
+                                     load_next_time=True)
+            elif config.get('general', 'save-session'):
+                session_manager.save(sessions.default, last_window=last_window,
+                                     load_next_time=True)
 
         if prompt.prompt_queue.shutdown():
             # If shutdown was called while we were asking a question, we're in
@@ -672,7 +690,7 @@ class Quitter:
             # event loop, so we can shut down immediately.
             self._shutdown(status, restart=restart)
 
-    def _shutdown(self, status, restart):
+    def _shutdown(self, status, restart):  # noqa
         """Second stage of shutdown."""
         log.destroy.debug("Stage 2 of shutting down...")
         if qApp is None:
@@ -681,7 +699,9 @@ class Quitter:
         # Remove eventfilter
         try:
             log.destroy.debug("Removing eventfilter...")
-            qApp.removeEventFilter(objreg.get('event-filter'))
+            event_filter = objreg.get('event-filter', None)
+            if event_filter is not None:
+                qApp.removeEventFilter(event_filter)
         except AttributeError:
             pass
         # Close all windows
@@ -723,7 +743,9 @@ class Quitter:
         # Now we can hopefully quit without segfaults
         log.destroy.debug("Deferring QApplication::exit...")
         objreg.get('signal-handler').deactivate()
-        objreg.get('session-manager').delete_autosave()
+        session_manager = objreg.get('session-manager', None)
+        if session_manager is not None:
+            session_manager.delete_autosave()
         # We use a singleshot timer to exit here to minimize the likelihood of
         # segfaults.
         QTimer.singleShot(0, functools.partial(qApp.exit, status))
@@ -785,7 +807,7 @@ class Application(QApplication):
     def exit(self, status):
         """Extend QApplication::exit to log the event."""
         log.destroy.debug("Now calling QApplication::exit.")
-        if self._args.debug_exit:
+        if 'debug-exit' in self._args.debug_flags:
             if hunter is None:
                 print("Not logging late shutdown because hunter could not be "
                       "imported!", file=sys.stderr)

qutebrowser/browser/__init__.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/adblock.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -58,7 +58,7 @@ def get_fileobj(byte_io):
         byte_io = zf.open(filename, mode='r')
     else:
         byte_io.seek(0)  # rewind what zipfile.is_zipfile did
-    return io.TextIOWrapper(byte_io, encoding='utf-8')
+    return byte_io
 
 
 def is_whitelisted_host(host):
@@ -147,7 +147,7 @@ class HostBlocker:
             with open(filename, 'r', encoding='utf-8') as f:
                 for line in f:
                     target.add(line.strip())
-        except OSError:
+        except (OSError, UnicodeDecodeError):
             log.misc.exception("Failed to read host blocklist!")
 
         return True
@@ -165,7 +165,8 @@ class HostBlocker:
         if not found:
             args = objreg.get('args')
             if (config.get('content', 'host-block-lists') is not None and
-                    args.basedir is None):
+                    args.basedir is None and
+                    config.get('content', 'host-blocking-enabled')):
                 message.info("Run :adblock-update to get adblock lists.")
 
     @cmdutils.register(instance='host-blocker')
@@ -205,6 +206,54 @@ class HostBlocker:
                 download.finished.connect(
                     functools.partial(self.on_download_finished, download))
 
+    def _parse_line(self, line):
+        """Parse a line from a host file.
+
+        Args:
+            line: The bytes object to parse.
+
+        Returns:
+            True if parsing succeeded, False otherwise.
+        """
+        if line.startswith(b'#'):
+            # Ignoring comments early so we don't have to care about
+            # encoding errors in them.
+            return True
+
+        try:
+            line = line.decode('utf-8')
+        except UnicodeDecodeError:
+            log.misc.error("Failed to decode: {!r}".format(line))
+            return False
+
+        # Remove comments
+        try:
+            hash_idx = line.index('#')
+            line = line[:hash_idx]
+        except ValueError:
+            pass
+
+        line = line.strip()
+        # Skip empty lines
+        if not line:
+            return True
+
+        parts = line.split()
+        if len(parts) == 1:
+            # "one host per line" format
+            host = parts[0]
+        elif len(parts) == 2:
+            # /etc/hosts format
+            host = parts[1]
+        else:
+            log.misc.error("Failed to parse: {!r}".format(line))
+            return False
+
+        if host not in self.WHITELISTED:
+            self._blocked_hosts.add(host)
+
+        return True
+
     def _merge_file(self, byte_io):
         """Read and merge host files.
 
@@ -218,35 +267,18 @@ class HostBlocker:
         line_count = 0
         try:
             f = get_fileobj(byte_io)
-        except (OSError, UnicodeDecodeError, zipfile.BadZipFile,
-                zipfile.LargeZipFile, LookupError) as e:
+        except (OSError, zipfile.BadZipFile, zipfile.LargeZipFile,
+                LookupError) as e:
             message.error("adblock: Error while reading {}: {} - {}".format(
                 byte_io.name, e.__class__.__name__, e))
             return
+
         for line in f:
             line_count += 1
-            # Remove comments
-            try:
-                hash_idx = line.index('#')
-                line = line[:hash_idx]
-            except ValueError:
-                pass
-            line = line.strip()
-            # Skip empty lines
-            if not line:
-                continue
-            parts = line.split()
-            if len(parts) == 1:
-                # "one host per line" format
-                host = parts[0]
-            elif len(parts) == 2:
-                # /etc/hosts format
-                host = parts[1]
-            else:
+            ok = self._parse_line(line)
+            if not ok:
                 error_count += 1
-                continue
-            if host not in self.WHITELISTED:
-                self._blocked_hosts.add(host)
+
         log.misc.debug("{}: read {} lines".format(byte_io.name, line_count))
         if error_count > 0:
             message.error("adblock: {} read errors for {}".format(

qutebrowser/browser/browsertab.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -21,7 +21,7 @@
 
 import itertools
 
-from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, QObject, QSizeF
+from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, QObject, QSizeF, Qt
 from PyQt5.QtGui import QIcon
 from PyQt5.QtWidgets import QWidget, QApplication
 
@@ -35,11 +35,12 @@ from qutebrowser.browser import mouse, hints
 tab_id_gen = itertools.count(0)
 
 
-def create(win_id, parent=None):
+def create(win_id, private, parent=None):
     """Get a QtWebKit/QtWebEngine tab object.
 
     Args:
         win_id: The window ID where the tab will be shown.
+        private: Whether the tab is a private/off the record tab.
         parent: The Qt parent to set.
     """
     # Importing modules here so we don't depend on QtWebEngine without the
@@ -51,7 +52,8 @@ def create(win_id, parent=None):
     else:
         from qutebrowser.browser.webkit import webkittab
         tab_class = webkittab.WebKitTab
-    return tab_class(win_id=win_id, mode_manager=mode_manager, parent=parent)
+    return tab_class(win_id=win_id, mode_manager=mode_manager, private=private,
+                     parent=parent)
 
 
 def init():
@@ -94,6 +96,8 @@ class TabData:
         viewing_source: Set if we're currently showing a source view.
         override_target: Override for open_target for fake clicks (like hints).
                          Only used for QtWebKit.
+        pinned: Flag to pin the tab.
+        fullscreen: Whether the tab has a video shown fullscreen currently.
     """
 
     def __init__(self):
@@ -101,11 +105,21 @@ class TabData:
         self.viewing_source = False
         self.inspector = None
         self.override_target = None
+        self.pinned = False
+        self.fullscreen = False
 
 
 class AbstractAction:
 
-    """Attribute of AbstractTab for Qt WebActions."""
+    """Attribute of AbstractTab for Qt WebActions.
+
+    Class attributes (overridden by subclasses):
+        action_class: The class actions are defined on (QWeb{Engine,}Page)
+        action_base: The type of the actions (QWeb{Engine,}Page.WebAction)
+    """
+
+    action_class = None
+    action_base = None
 
     def __init__(self):
         self._widget = None
@@ -118,6 +132,13 @@ class AbstractAction:
         """Save the current page."""
         raise NotImplementedError
 
+    def run_string(self, name):
+        """Run a webaction based on its name."""
+        member = getattr(self.action_class, name, None)
+        if not isinstance(member, self.action_base):
+            raise WebTabError("{} is not a valid web action!".format(name))
+        self._widget.triggerPageAction(member)
+
 
 class AbstractPrinting:
 
@@ -155,6 +176,8 @@ class AbstractSearch(QObject):
 
     Attributes:
         text: The last thing this view was searched for.
+        search_displayed: Whether we're currently displaying search results in
+                          this view.
         _flags: The flags of the last search (needs to be set by subclasses).
         _widget: The underlying WebView widget.
     """
@@ -163,6 +186,7 @@ class AbstractSearch(QObject):
         super().__init__(parent)
         self._widget = None
         self.text = None
+        self.search_displayed = False
 
     def search(self, text, *, ignore_case=False, reverse=False,
                result_cb=None):
@@ -524,6 +548,7 @@ class AbstractTab(QWidget):
     Attributes:
         history: The AbstractHistory for the current tab.
         registry: The ObjectRegistry associated with this tab.
+        private: Whether private browsing is turned on for this tab.
 
         _load_status: loading status of this page
                       Accessible via load_status() method.
@@ -563,7 +588,8 @@ class AbstractTab(QWidget):
     fullscreen_requested = pyqtSignal(bool)
     renderer_process_terminated = pyqtSignal(TerminationStatus, int)
 
-    def __init__(self, win_id, mode_manager, parent=None):
+    def __init__(self, *, win_id, mode_manager, private, parent=None):
+        self.private = private
         self.win_id = win_id
         self.tab_id = next(tab_id_gen)
         super().__init__(parent)
@@ -740,6 +766,10 @@ class AbstractTab(QWidget):
     def clear_ssl_errors(self):
         raise NotImplementedError
 
+    def key_press(self, key, modifier=Qt.NoModifier):
+        """Send a fake key event to this tab."""
+        raise NotImplementedError
+
     def dump_async(self, callback, *, plain=False):
         """Dump the current page to a file ascync.
 
@@ -771,7 +801,7 @@ class AbstractTab(QWidget):
     def icon(self):
         raise NotImplementedError
 
-    def set_html(self, html, base_url):
+    def set_html(self, html, base_url=QUrl()):
         raise NotImplementedError
 
     def networkaccessmanager(self):

qutebrowser/browser/commands.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -20,22 +20,15 @@
 """Command dispatcher for TabbedBrowser."""
 
 import os
+import sys
 import os.path
 import shlex
 import functools
 
-from PyQt5.QtWidgets import QApplication, QTabBar
+from PyQt5.QtWidgets import QApplication, QTabBar, QDialog
 from PyQt5.QtCore import Qt, QUrl, QEvent, QUrlQuery
 from PyQt5.QtGui import QKeyEvent
 from PyQt5.QtPrintSupport import QPrintDialog, QPrintPreviewDialog
-try:
-    from PyQt5.QtWebKitWidgets import QWebPage
-except ImportError:
-    QWebPage = None
-try:
-    from PyQt5.QtWebEngineWidgets import QWebEnginePage
-except ImportError:
-    QWebEnginePage = None
 import pygments
 import pygments.lexers
 import pygments.formatters
@@ -75,10 +68,10 @@ class CommandDispatcher:
     def __repr__(self):
         return utils.get_repr(self)
 
-    def _new_tabbed_browser(self):
+    def _new_tabbed_browser(self, private):
         """Get a tabbed-browser from a new window."""
         from qutebrowser.mainwindow import mainwindow
-        new_window = mainwindow.MainWindow()
+        new_window = mainwindow.MainWindow(private=private)
         new_window.show()
         return new_window.tabbed_browser
 
@@ -118,7 +111,7 @@ class CommandDispatcher:
         return widget
 
     def _open(self, url, tab=False, background=False, window=False,
-              explicit=True):
+              explicit=True, private=None):
         """Helper function to open a page.
 
         Args:
@@ -126,12 +119,17 @@ class CommandDispatcher:
             tab: Whether to open in a new tab.
             background: Whether to open in the background.
             window: Whether to open in a new window
+            private: If opening a new window, open it in private browsing mode.
+                     If not given, inherit the current window's mode.
         """
         urlutils.raise_cmdexc_if_invalid(url)
         tabbed_browser = self._tabbed_browser
-        cmdutils.check_exclusive((tab, background, window), 'tbw')
-        if window:
-            tabbed_browser = self._new_tabbed_browser()
+        cmdutils.check_exclusive((tab, background, window, private), 'tbwp')
+        if window and private is None:
+            private = self._tabbed_browser.private
+
+        if window or private:
+            tabbed_browser = self._new_tabbed_browser(private)
             tabbed_browser.tabopen(url)
         elif tab:
             tabbed_browser.tabopen(url, background=False, explicit=explicit)
@@ -160,12 +158,14 @@ class CommandDispatcher:
         else:
             return None
 
-    def _tab_focus_last(self):
+    def _tab_focus_last(self, *, show_error=True):
         """Select the tab which was last focused."""
         try:
             tab = objreg.get('last-focused-tab', scope='window',
                              window=self._win_id)
         except KeyError:
+            if not show_error:
+                return
             raise cmdexc.CommandError("No last focused tab!")
         idx = self._tabbed_browser.indexOf(tab)
         if idx == -1:
@@ -205,24 +205,21 @@ class CommandDispatcher:
                                  "{!r}!".format(conf_selection))
         return None
 
-    @cmdutils.register(instance='command-dispatcher', scope='window')
-    @cmdutils.argument('count', count=True)
-    def tab_close(self, prev=False, next_=False, opposite=False, count=None):
-        """Close the current/[count]th tab.
+    def _tab_close(self, tab, prev=False, next_=False, opposite=False):
+        """Helper function for tab_close be able to handle message.async.
 
         Args:
+            tab: Tab object to select be closed.
             prev: Force selecting the tab before the current tab.
             next_: Force selecting the tab after the current tab.
             opposite: Force selecting the tab in the opposite direction of
                       what's configured in 'tabs->select-on-remove'.
             count: The tab index to close, or None
         """
-        tab = self._cntwidget(count)
-        if tab is None:
-            return
         tabbar = self._tabbed_browser.tabBar()
         selection_override = self._get_selection_override(prev, next_,
                                                           opposite)
+
         if selection_override is None:
             self._tabbed_browser.close_tab(tab)
         else:
@@ -231,12 +228,55 @@ class CommandDispatcher:
             self._tabbed_browser.close_tab(tab)
             tabbar.setSelectionBehaviorOnRemove(old_selection_behavior)
 
+    @cmdutils.register(instance='command-dispatcher', scope='window')
+    @cmdutils.argument('count', count=True)
+    def tab_close(self, prev=False, next_=False, opposite=False,
+                  force=False, count=None):
+        """Close the current/[count]th tab.
+
+        Args:
+            prev: Force selecting the tab before the current tab.
+            next_: Force selecting the tab after the current tab.
+            opposite: Force selecting the tab in the opposite direction of
+                      what's configured in 'tabs->select-on-remove'.
+            force: Avoid confirmation for pinned tabs.
+            count: The tab index to close, or None
+        """
+        tab = self._cntwidget(count)
+        if tab is None:
+            return
+        close = functools.partial(self._tab_close, tab, prev,
+                                  next_, opposite)
+
+        self._tabbed_browser.tab_close_prompt_if_pinned(tab, force, close)
+
+    @cmdutils.register(instance='command-dispatcher', scope='window',
+                       name='tab-pin')
+    @cmdutils.argument('count', count=True)
+    def tab_pin(self, count=None):
+        """Pin/Unpin the current/[count]th tab.
+
+        Pinning a tab shrinks it to tabs->pinned-width size.
+        Attempting to close a pinned tab will cause a confirmation,
+        unless --force is passed.
+
+        Args:
+            count: The tab index to pin or unpin, or None
+        """
+        tab = self._cntwidget(count)
+        if tab is None:
+            return
+
+        to_pin = not tab.data.pinned
+        self._tabbed_browser.set_tab_pinned(tab, to_pin)
+
     @cmdutils.register(instance='command-dispatcher', name='open',
                        maxsplit=0, scope='window')
     @cmdutils.argument('url', completion=usertypes.Completion.url)
     @cmdutils.argument('count', count=True)
     def openurl(self, url=None, implicit=False,
-                bg=False, tab=False, window=False, count=None):
+                bg=False, tab=False, window=False, count=None, secure=False,
+                private=False):
         """Open a URL in the current/[count]th tab.
 
         If the URL contains newlines, each line gets opened in its own tab.
@@ -249,6 +289,8 @@ class CommandDispatcher:
             implicit: If opening a new tab, treat the tab as implicit (like
                       clicking on a link).
             count: The tab index to open the URL in, or None.
+            secure: Force HTTPS.
+            private: Open a new window in private browsing mode.
         """
         if url is None:
             urls = [config.get('general', 'default-page')]
@@ -256,11 +298,15 @@ class CommandDispatcher:
             urls = self._parse_url_input(url)
 
         for i, cur_url in enumerate(urls):
+            if secure:
+                cur_url.setScheme('https')
             if not window and i > 0:
                 tab = False
                 bg = True
-            if tab or bg or window:
-                self._open(cur_url, tab, bg, window, not implicit)
+
+            if tab or bg or window or private:
+                self._open(cur_url, tab, bg, window, explicit=not implicit,
+                           private=private)
             else:
                 curtab = self._cntwidget(count)
                 if curtab is None:
@@ -271,6 +317,8 @@ class CommandDispatcher:
                     else:
                         # Explicit count with a tab that doesn't exist.
                         return
+                elif curtab.data.pinned:
+                    message.info("Tab is pinned!")
                 else:
                     curtab.openurl(cur_url)
 
@@ -350,7 +398,7 @@ class CommandDispatcher:
                 message.error("Printing failed!")
 
         tab.printing.check_preview_support()
-        diag = QPrintPreviewDialog()
+        diag = QPrintPreviewDialog(tab)
         diag.setAttribute(Qt.WA_DeleteOnClose)
         diag.setWindowFlags(diag.windowFlags() | Qt.WindowMaximizeButtonHint |
                             Qt.WindowMinimizeButtonHint)
@@ -376,9 +424,18 @@ class CommandDispatcher:
                 message.error("Printing failed!")
             diag.deleteLater()
 
-        diag = QPrintDialog()
-        diag.open(lambda: tab.printing.to_printer(diag.printer(),
-                                                  print_callback))
+        def do_print():
+            """Called when the dialog was closed."""
+            tab.printing.to_printer(diag.printer(), print_callback)
+
+        diag = QPrintDialog(tab)
+        if sys.platform == 'darwin':
+            # For some reason we get a segfault when using open() on macOS
+            ret = diag.exec_()
+            if ret == QDialog.Accepted:
+                do_print()
+        else:
+            diag.open(do_print)
 
     @cmdutils.register(instance='command-dispatcher', name='print',
                        scope='window')
@@ -435,10 +492,11 @@ class CommandDispatcher:
         # The new tab could be in a new tabbed_browser (e.g. because of
         # tabs-are-windows being set)
         if window:
-            new_tabbed_browser = self._new_tabbed_browser()
+            new_tabbed_browser = self._new_tabbed_browser(
+                private=self._tabbed_browser.private)
         else:
             new_tabbed_browser = self._tabbed_browser
-        newtab = new_tabbed_browser.tabopen(background=bg, explicit=True)
+        newtab = new_tabbed_browser.tabopen(background=bg)
         new_tabbed_browser = objreg.get('tabbed-browser', scope='window',
                                         window=newtab.win_id)
         idx = new_tabbed_browser.indexOf(newtab)
@@ -452,6 +510,7 @@ class CommandDispatcher:
         newtab.data.keep_icon = True
         newtab.history.deserialize(history)
         newtab.zoom.set_factor(curtab.zoom.factor())
+        new_tabbed_browser.set_tab_pinned(newtab, curtab.data.pinned)
         return newtab
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
@@ -565,7 +624,7 @@ class CommandDispatcher:
                         tab=tab, background=bg, window=window)
             elif where in ['up', 'increment', 'decrement']:
                 new_url = handlers[where](url, count)
-                self._open(new_url, tab, bg, window)
+                self._open(new_url, tab, bg, window, explicit=False)
             else:  # pragma: no cover
                 raise ValueError("Got called with invalid value {} for "
                                 "`where'.".format(where))
@@ -595,6 +654,9 @@ class CommandDispatcher:
     def scroll(self, direction: typing.Union[str, int], count=1):
         """Scroll the current tab in the given direction.
 
+        Note you can use `:run-with-count` to have a keybinding with a bigger
+        scroll increment.
+
         Args:
             direction: In which direction to scroll
                        (up/down/left/right/top/bottom).
@@ -628,7 +690,7 @@ class CommandDispatcher:
                        scope='window')
     @cmdutils.argument('count', count=True)
     @cmdutils.argument('horizontal', flag='x')
-    def scroll_perc(self, perc: float=None, horizontal=False, count=None):
+    def scroll_perc(self, perc: float = None, horizontal=False, count=None):
         """Scroll to a specific percentage of the page.
 
         The percentage can be given either as argument or as count.
@@ -664,7 +726,7 @@ class CommandDispatcher:
     @cmdutils.argument('bottom_navigate', metavar='ACTION',
                        choices=('next', 'increment'))
     def scroll_page(self, x: float, y: float, *,
-                    top_navigate: str=None, bottom_navigate: str=None,
+                    top_navigate: str = None, bottom_navigate: str = None,
                     count=1):
         """Scroll the frame page-wise.
 
@@ -801,7 +863,7 @@ class CommandDispatcher:
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
     @cmdutils.argument('count', count=True)
-    def zoom(self, zoom: int=None, count=None):
+    def zoom(self, zoom=None, count=None):
         """Set the zoom level for the current tab.
 
         The zoom can be given as argument or as [count]. If neither is
@@ -812,6 +874,13 @@ class CommandDispatcher:
             zoom: The zoom percentage to set.
             count: The zoom percentage to set.
         """
+        if zoom is not None:
+            try:
+                zoom = int(zoom.rstrip('%'))
+            except ValueError:
+                raise cmdexc.CommandError("zoom: Invalid int value {}"
+                                          .format(zoom))
+
         level = count if count is not None else zoom
         if level is None:
             level = config.get('ui', 'default-zoom')
@@ -824,27 +893,42 @@ class CommandDispatcher:
         message.info("Zoom level: {}%".format(level), replace=True)
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
-    def tab_only(self, prev=False, next_=False):
+    def tab_only(self, prev=False, next_=False, force=False):
         """Close all tabs except for the current one.
 
         Args:
             prev: Keep tabs before the current.
             next_: Keep tabs after the current.
+            force: Avoid confirmation for pinned tabs.
         """
         cmdutils.check_exclusive((prev, next_), 'pn')
         cur_idx = self._tabbed_browser.currentIndex()
         assert cur_idx != -1
 
+        def _to_close(i):
+            """Helper method to check if a tab should be closed or not."""
+            return not (i == cur_idx or
+                        (prev and i < cur_idx) or
+                        (next_ and i > cur_idx))
+
+        # Check to see if we are closing any pinned tabs
+        if not force:
+            for i, tab in enumerate(self._tabbed_browser.widgets()):
+                if _to_close(i) and tab.data.pinned:
+                    self._tabbed_browser.tab_close_prompt_if_pinned(
+                        tab,
+                        force,
+                        lambda: self.tab_only(
+                            prev=prev, next_=next_, force=True))
+                    return
+
         for i, tab in enumerate(self._tabbed_browser.widgets()):
-            if (i == cur_idx or (prev and i < cur_idx) or
-                    (next_ and i > cur_idx)):
-                continue
-            else:
+            if _to_close(i):
                 self._tabbed_browser.close_tab(tab)
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
     def undo(self):
-        """Re-open a closed tab (optionally skipping [count] closed tabs)."""
+        """Re-open a closed tab."""
         try:
             self._tabbed_browser.undo()
         except IndexError:
@@ -997,12 +1081,15 @@ class CommandDispatcher:
                    last tab.
             count: The tab index to focus, starting with 1.
         """
+        index = count if count is not None else index
+
         if index == 'last':
             self._tab_focus_last()
             return
-        index = count if count is not None else index
-
-        if index is None:
+        elif index == self._current_index() + 1:
+            self._tab_focus_last(show_error=False)
+            return
+        elif index is None:
             self.tab_next()
             return
 
@@ -1076,6 +1163,7 @@ class CommandDispatcher:
             detach: Whether the command should be detached from qutebrowser.
             cmdline: The commandline to execute.
         """
+        cmdutils.check_exclusive((userscript, detach), 'ud')
         try:
             cmd, *args = shlex.split(cmdline)
         except ValueError as e:
@@ -1227,7 +1315,7 @@ class CommandDispatcher:
         except urlmarks.Error as e:
             raise cmdexc.CommandError(str(e))
         else:
-            msg = "Bookmarked {}!" if was_added else "Removed bookmark {}!"
+            msg = "Bookmarked {}" if was_added else "Removed bookmark {}"
             message.info(msg.format(url.toDisplayString()))
 
     @cmdutils.register(instance='command-dispatcher', scope='window',
@@ -1331,6 +1419,9 @@ class CommandDispatcher:
                                       scope='window', window=self._win_id)
         target = None
         if dest is not None:
+            dest = downloads.transform_path(dest)
+            if dest is None:
+                raise cmdexc.CommandError("Invalid target filename")
             target = downloads.FileDownloadTarget(dest)
 
         tab = self._current_widget()
@@ -1368,19 +1459,22 @@ class CommandDispatcher:
         if tab.data.viewing_source:
             raise cmdexc.CommandError("Already viewing source!")
 
+        try:
+            current_url = self._current_url()
+        except cmdexc.CommandError as e:
+            message.error(str(e))
+            return
+
         def show_source_cb(source):
             """Show source as soon as it's ready."""
             lexer = pygments.lexers.HtmlLexer()
-            formatter = pygments.formatters.HtmlFormatter(full=True,
-                                                          linenos='table')
+            formatter = pygments.formatters.HtmlFormatter(
+                full=True, linenos='table',
+                title='Source for {}'.format(current_url.toDisplayString()))
             highlighted = pygments.highlight(source, lexer, formatter)
-            try:
-                current_url = self._current_url()
-            except cmdexc.CommandError as e:
-                message.error(str(e))
-                return
-            new_tab = self._tabbed_browser.tabopen(explicit=True)
-            new_tab.set_html(highlighted, current_url)
+
+            new_tab = self._tabbed_browser.tabopen()
+            new_tab.set_html(highlighted)
             new_tab.data.viewing_source = True
 
         tab.dump_async(show_source_cb)
@@ -1463,7 +1557,7 @@ class CommandDispatcher:
         self._open(url, tab, bg, window)
 
     @cmdutils.register(instance='command-dispatcher', scope='window')
-    def messages(self, level='error', plain=False, tab=False, bg=False,
+    def messages(self, level='info', plain=False, tab=False, bg=False,
                  window=False):
         """Show a log of past messages.
 
@@ -1496,6 +1590,7 @@ class CommandDispatcher:
         if text is None:
             message.error("Could not get text from the focused element.")
             return
+        assert isinstance(text, str), text
 
         ed = editor.ExternalEditor(self._tabbed_browser)
         ed.editing_finished.connect(functools.partial(
@@ -1533,10 +1628,7 @@ class CommandDispatcher:
                        backend=usertypes.Backend.QtWebKit)
     def paste_primary(self):
         """Paste the primary selection at cursor position."""
-        try:
-            self.insert_text(utils.get_clipboard(selection=True))
-        except utils.SelectionUnsupportedError:
-            self.insert_text(utils.get_clipboard())
+        self.insert_text(utils.get_clipboard(selection=True, fallback=True))
 
     @cmdutils.register(instance='command-dispatcher', maxsplit=0,
                        scope='window')
@@ -1647,21 +1739,22 @@ class CommandDispatcher:
         tab = self._current_widget()
         tab.search.clear()
 
+        if not text:
+            return
+
         options = {
             'ignore_case': config.get('general', 'ignore-case'),
             'reverse': reverse,
         }
+
         self._tabbed_browser.search_text = text
         self._tabbed_browser.search_options = dict(options)
 
-        if text:
-            cb = functools.partial(self._search_cb, tab=tab,
-                                   old_scroll_pos=tab.scroller.pos_px(),
-                                   options=options, text=text, prev=False)
-        else:
-            cb = None
-
+        cb = functools.partial(self._search_cb, tab=tab,
+                               old_scroll_pos=tab.scroller.pos_px(),
+                               options=options, text=text, prev=False)
         options['result_cb'] = cb
+
         tab.search.search(text, **options)
 
     @cmdutils.register(instance='command-dispatcher', hide=True,
@@ -1897,33 +1990,20 @@ class CommandDispatcher:
     def debug_webaction(self, action, count=1):
         """Execute a webaction.
 
-        See http://doc.qt.io/qt-5/qwebpage.html#WebAction-enum for the
-        available actions.
+        Available actions:
+        http://doc.qt.io/archives/qt-5.5/qwebpage.html#WebAction-enum (WebKit)
+        http://doc.qt.io/qt-5/qwebenginepage.html#WebAction-enum (WebEngine)
 
         Args:
             action: The action to execute, e.g. MoveToNextChar.
             count: How many times to repeat the action.
         """
         tab = self._current_widget()
-
-        if tab.backend == usertypes.Backend.QtWebKit:
-            assert QWebPage is not None
-            member = getattr(QWebPage, action, None)
-            base = QWebPage.WebAction
-        elif tab.backend == usertypes.Backend.QtWebEngine:
-            assert QWebEnginePage is not None
-            member = getattr(QWebEnginePage, action, None)
-            base = QWebEnginePage.WebAction
-
-        if not isinstance(member, base):
-            raise cmdexc.CommandError("{} is not a valid web action!".format(
-                action))
-
         for _ in range(count):
-            # This whole command is backend-specific anyways, so it makes no
-            # sense to introduce some API for this.
-            # pylint: disable=protected-access
-            tab._widget.triggerPageAction(member)
+            try:
+                tab.action.run_string(action)
+            except browsertab.WebTabError as e:
+                raise cmdexc.CommandError(str(e))
 
     @cmdutils.register(instance='command-dispatcher', scope='window',
                        maxsplit=0, no_cmd_split=True)
@@ -2001,11 +2081,6 @@ class CommandDispatcher:
                 QApplication.postEvent(window, press_event)
                 QApplication.postEvent(window, release_event)
             else:
-                try:
-                    tab = objreg.get('tab', scope='tab', tab='current')
-                except objreg.RegistryUnavailableError:
-                    raise cmdexc.CommandError("No focused webview!")
-
                 tab = self._current_widget()
                 tab.send_event(press_event)
                 tab.send_event(release_event)

qutebrowser/browser/downloads.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -19,11 +19,13 @@
 
 """Shared QtWebKit/QtWebEngine code for downloads."""
 
+import re
 import sys
 import html
 import os.path
 import collections
 import functools
+import pathlib
 import tempfile
 
 import sip
@@ -161,6 +163,25 @@ def get_filename_question(*, suggested_filename, url, parent=None):
     return q
 
 
+def transform_path(path):
+    r"""Do platform-specific transformations, like changing E: to E:\.
+
+    Returns None if the path is invalid on the current platform.
+    """
+    if sys.platform != "win32":
+        return path
+    path = utils.expand_windows_drive(path)
+    # Drive dependent working directories are not supported, e.g.
+    # E:filename is invalid
+    if re.match(r'[A-Z]:[^\\]', path, re.IGNORECASE):
+        return None
+    # Paths like COM1, ...
+    # See https://github.com/qutebrowser/qutebrowser/issues/82
+    if pathlib.Path(path).is_reserved():
+        return None
+    return path
+
+
 class NoFilenameError(Exception):
 
     """Raised when we can't find out a filename in DownloadTarget."""
@@ -507,6 +528,14 @@ class AbstractDownloadItem(QObject):
         """Retry a failed download."""
         raise NotImplementedError
 
+    @pyqtSlot()
+    def try_retry(self):
+        """Try to retry a download and show an error if it's unsupported."""
+        try:
+            self.retry()
+        except UnsupportedOperationError as e:
+            message.error(str(e))
+
     def _get_open_filename(self):
         """Get the filename to open a download.
 
@@ -529,7 +558,11 @@ class AbstractDownloadItem(QObject):
         if filename is None:  # pragma: no cover
             log.downloads.error("No filename to open the download!")
             return
-        utils.open_file(filename, cmdline)
+        # By using a singleshot timer, we ensure that we return fast. This
+        # is important on systems where process creation takes long, as
+        # otherwise the prompt might hang around and cause bugs
+        # (see issue #2296)
+        QTimer.singleShot(0, lambda: utils.open_file(filename, cmdline))
 
     def _ensure_can_set_filename(self, filename):
         """Make sure we can still set a filename."""
@@ -919,7 +952,7 @@ class DownloadModel(QAbstractListModel):
 
     @cmdutils.register(instance='download-model', scope='window', maxsplit=0)
     @cmdutils.argument('count', count=True)
-    def download_open(self, cmdline: str=None, count=0):
+    def download_open(self, cmdline: str = None, count=0):
         """Open the last/[count]th download.
 
         If no specific command is given, this will use the system's default
@@ -964,7 +997,7 @@ class DownloadModel(QAbstractListModel):
                 raise cmdexc.CommandError("No failed downloads!")
             else:
                 download = to_retry[0]
-        download.retry()
+        download.try_retry()
 
     def can_clear(self):
         """Check if there are finished downloads to clear."""

qutebrowser/browser/downloadview.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -23,7 +23,7 @@ import functools
 
 import sip
 from PyQt5.QtCore import pyqtSlot, QSize, Qt, QTimer
-from PyQt5.QtWidgets import QListView, QSizePolicy, QMenu
+from PyQt5.QtWidgets import QListView, QSizePolicy, QMenu, QStyleFactory
 
 from qutebrowser.browser import downloads
 from qutebrowser.config import style
@@ -75,6 +75,7 @@ class DownloadView(QListView):
 
     def __init__(self, win_id, parent=None):
         super().__init__(parent)
+        self.setStyle(QStyleFactory.create('Fusion'))
         style.set_register_stylesheet(self)
         self.setResizeMode(QListView.Adjust)
         self.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
@@ -134,7 +135,7 @@ class DownloadView(QListView):
             if item.successful:
                 actions.append(("Open", item.open_file))
             else:
-                actions.append(("Retry", item.retry))
+                actions.append(("Retry", item.try_retry))
             actions.append(("Remove", item.remove))
         else:
             actions.append(("Cancel", item.cancel))

qutebrowser/browser/hints.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -579,12 +579,10 @@ class HintManager(QObject):
         if elems is None:
             message.error("There was an error while getting hint elements")
             return
-
-        filterfunc = webelem.FILTERS.get(self._context.group, lambda e: True)
-        elems = [e for e in elems if filterfunc(e)]
         if not elems:
             message.error("No elements found.")
             return
+
         strings = self._hint_strings(elems)
         log.hints.debug("hints: {}".format(', '.join(strings)))
 

qutebrowser/browser/history.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -27,7 +27,6 @@ from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, QObject
 from qutebrowser.commands import cmdutils
 from qutebrowser.utils import (utils, objreg, standarddir, log, qtutils,
                                usertypes, message)
-from qutebrowser.config import config
 from qutebrowser.misc import lineparser, objects
 
 
@@ -257,6 +256,12 @@ class WebHistory(QObject):
     @pyqtSlot(QUrl, QUrl, str)
     def add_from_tab(self, url, requested_url, title):
         """Add a new history entry as slot, called from a BrowserTab."""
+        if url.scheme() == 'data' or requested_url.scheme() == 'data':
+            return
+        if url.isEmpty():
+            # things set via setHtml
+            return
+
         no_formatting = QUrl.UrlFormattingOption(0)
         if (requested_url.isValid() and
                 not requested_url.matches(url, no_formatting)):
@@ -274,9 +279,9 @@ class WebHistory(QObject):
                       (hidden in completion)
             atime: Override the atime used to add the entry
         """
-        if config.get('general', 'private-browsing'):
-            return
-        if not url.isValid():
+        if not url.isValid():  # pragma: no cover
+            # the no cover pragma is a WORKAROUND for this not being covered in
+            # old Qt versions.
             log.misc.warning("Ignoring invalid URL being added to history")
             return
 

qutebrowser/browser/inspector.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/mouse.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/navigate.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -79,8 +79,7 @@ def _find_prevnext(prev, elems):
             return e
 
     # Then check for regular links/buttons.
-    filterfunc = webelem.FILTERS[webelem.Group.prevnext]
-    elems = [e for e in elems if e.tag_name() != 'link' and filterfunc(e)]
+    elems = [e for e in elems if e.tag_name() != 'link']
     option = 'prev-regexes' if prev else 'next-regexes'
     if not elems:
         return None
@@ -128,20 +127,21 @@ def prevnext(*, browsertab, win_id, baseurl, prev=False,
             return
         qtutils.ensure_valid(url)
 
+        cur_tabbed_browser = objreg.get('tabbed-browser', scope='window',
+                                        window=win_id)
+
         if window:
             from qutebrowser.mainwindow import mainwindow
-            new_window = mainwindow.MainWindow()
+            new_window = mainwindow.MainWindow(
+                private=cur_tabbed_browser.private)
             new_window.show()
             tabbed_browser = objreg.get('tabbed-browser', scope='window',
                                         window=new_window.win_id)
             tabbed_browser.tabopen(url, background=False)
         elif tab:
-            tabbed_browser = objreg.get('tabbed-browser', scope='window',
-                                        window=win_id)
-            tabbed_browser.tabopen(url, background=background)
+            cur_tabbed_browser.tabopen(url, background=background)
         else:
             browsertab.openurl(url)
 
-    selector = ', '.join([webelem.SELECTORS[webelem.Group.links],
-                          webelem.SELECTORS[webelem.Group.prevnext]])
-    browsertab.elements.find_css(selector, _prevnext_cb)
+    browsertab.elements.find_css(webelem.SELECTORS[webelem.Group.links],
+                                 _prevnext_cb)

qutebrowser/browser/network/pac.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -22,7 +22,7 @@
 import sys
 import functools
 
-from PyQt5.QtCore import (QObject, pyqtSignal, pyqtSlot)
+from PyQt5.QtCore import QObject, pyqtSignal, pyqtSlot, QUrl
 from PyQt5.QtNetwork import (QNetworkProxy, QNetworkRequest, QHostInfo,
                              QNetworkReply, QNetworkAccessManager,
                              QHostAddress)
@@ -155,7 +155,7 @@ class PACResolver:
                 raise ParseProxyError("Invalid number of parameters for PROXY")
             host, port = PACResolver._parse_proxy_host(config[1])
             return QNetworkProxy(QNetworkProxy.HttpProxy, host, port)
-        elif config[0] == "SOCKS":
+        elif config[0] in ["SOCKS", "SOCKS5"]:
             if len(config) != 2:
                 raise ParseProxyError("Invalid number of parameters for SOCKS")
             host, port = PACResolver._parse_proxy_host(config[1])
@@ -199,16 +199,24 @@ class PACResolver:
             err = "Cannot resolve FindProxyForURL function, got '{}' instead"
             raise EvalProxyError(err.format(self._resolver.toString()))
 
-    def resolve(self, query):
+    def resolve(self, query, from_file=False):
         """Resolve a proxy via PAC.
 
         Args:
             query: QNetworkProxyQuery.
+            from_file: Whether the proxy info is coming from a file.
 
         Return:
             A list of QNetworkProxy objects in order of preference.
         """
-        result = self._resolver.call([query.url().toString(),
+        if from_file:
+            string_flags = QUrl.PrettyDecoded
+        else:
+            string_flags = QUrl.RemoveUserInfo
+            if query.url().scheme() == 'https':
+                string_flags |= QUrl.RemovePath | QUrl.RemoveQuery
+
+        result = self._resolver.call([query.url().toString(string_flags),
                                       query.peerHostName()])
         result_str = result.toString()
         if not result.isString():
@@ -236,6 +244,7 @@ class PACFetcher(QObject):
         assert url.scheme().startswith(pac_prefix)
         url.setScheme(url.scheme()[len(pac_prefix):])
 
+        self._pac_url = url
         self._manager = QNetworkAccessManager()
         self._manager.setProxy(QNetworkProxy(QNetworkProxy.NoProxy))
         self._reply = self._manager.get(QNetworkRequest(url))
@@ -292,8 +301,9 @@ class PACFetcher(QObject):
         Return a list of QNetworkProxy objects in order of preference.
         """
         self._wait()
+        from_file = self._pac_url.scheme() == 'file'
         try:
-            return self._pac.resolve(query)
+            return self._pac.resolve(query, from_file=from_file)
         except (EvalProxyError, ParseProxyError) as e:
             log.network.exception("Error in PAC resolution: {}.".format(e))
             # .invalid is guaranteed to be inaccessible in RFC 6761.
@@ -302,4 +312,4 @@ class PACFetcher(QObject):
             # Later NetworkManager.createRequest will detect this and display
             # an error message.
             error_host = "pac-resolve-error.qutebrowser.invalid"
-            return QNetworkProxy(QNetworkProxy.HttpProxy, error_host, 9)
+            return [QNetworkProxy(QNetworkProxy.HttpProxy, error_host, 9)]

qutebrowser/browser/network/proxy.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/pdfjs.py

@@ -1,6 +1,7 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
 # Copyright 2015 Daniel Schadt
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/qtnetworkdownloads.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -27,6 +27,7 @@ import collections
 from PyQt5.QtCore import pyqtSlot, pyqtSignal, QTimer
 from PyQt5.QtNetwork import QNetworkRequest, QNetworkReply
 
+from qutebrowser.config import config
 from qutebrowser.utils import message, usertypes, log, urlutils, utils
 from qutebrowser.browser import downloads
 from qutebrowser.browser.webkit import http
@@ -110,6 +111,9 @@ class DownloadItem(downloads.AbstractDownloadItem):
     def _do_die(self):
         """Abort the download and emit an error."""
         self._read_timer.stop()
+        if self._reply is None:
+            log.downloads.debug("Reply gone while dying")
+            return
         self._reply.downloadProgress.disconnect()
         self._reply.finished.disconnect()
         self._reply.error.disconnect()
@@ -270,7 +274,7 @@ class DownloadItem(downloads.AbstractDownloadItem):
         if self.fileobj is None or self._reply is None:
             # No filename has been set yet (so we don't empty the buffer) or we
             # got a readyRead after the reply was finished (which happens on
-            # qute:log for example).
+            # qute://log for example).
             return
         if not self._reply.isOpen():
             raise OSError("Reply is closed!")
@@ -363,7 +367,8 @@ class DownloadManager(downloads.AbstractDownloadManager):
     def __init__(self, win_id, parent=None):
         super().__init__(parent)
         self._networkmanager = networkmanager.NetworkManager(
-            win_id, None, self)
+            win_id=win_id, tab_id=None,
+            private=config.get('general', 'private-browsing'), parent=self)
 
     @pyqtSlot('QUrl')
     def get(self, url, *, user_agent=None, **kwargs):

qutebrowser/browser/qutescheme.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -17,23 +17,27 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Backend-independent qute:* code.
+"""Backend-independent qute://* code.
 
 Module attributes:
     pyeval_output: The output of the last :pyeval command.
     _HANDLERS: The handlers registered via decorators.
 """
 
+import json
+import os
 import sys
 import time
-import datetime
 import urllib.parse
+import datetime
+import pkg_resources
 
-from PyQt5.QtCore import QUrlQuery
+from PyQt5.QtCore import QUrlQuery, QUrl
 
 import qutebrowser
+from qutebrowser.config import config
 from qutebrowser.utils import (version, utils, jinja, log, message, docutils,
-                               objreg)
+                               objreg, usertypes, qtutils)
 from qutebrowser.misc import objects
 
 
@@ -75,12 +79,25 @@ class QuteSchemeError(Exception):
         super().__init__(errorstring)
 
 
-class add_handler:  # pylint: disable=invalid-name
+class Redirect(Exception):
 
-    """Decorator to register a qute:* URL handler.
+    """Exception to signal a redirect should happen.
 
     Attributes:
-        _name: The 'foo' part of qute:foo
+        url: The URL to redirect to, as a QUrl.
+    """
+
+    def __init__(self, url):
+        super().__init__(url.toDisplayString())
+        self.url = url
+
+
+class add_handler:  # pylint: disable=invalid-name
+
+    """Decorator to register a qute://* URL handler.
+
+    Attributes:
+        _name: The 'foo' part of qute://foo
         backend: Limit which backends the handler can run with.
     """
 
@@ -103,7 +120,7 @@ class add_handler:  # pylint: disable=invalid-name
     def wrong_backend_handler(self, url):
         """Show an error page about using the invalid backend."""
         html = jinja.render('error.html',
-                            title="Error while opening qute:url",
+                            title="Error while opening qute://url",
                             url=url.toDisplayString(),
                             error='{} is not available with this '
                                   'backend'.format(url.toDisplayString()),
@@ -125,13 +142,19 @@ def data_for_url(url):
     # A url like "qute:foo" is split as "scheme:path", not "scheme:host".
     log.misc.debug("url: {}, path: {}, host {}".format(
         url.toDisplayString(), path, host))
+    if path and not host:
+        new_url = QUrl()
+        new_url.setScheme('qute')
+        new_url.setHost(path)
+        new_url.setPath('/')
+        if new_url.host():  # path was a valid host
+            raise Redirect(new_url)
+
     try:
-        handler = _HANDLERS[path]
+        handler = _HANDLERS[host]
     except KeyError:
-        try:
-            handler = _HANDLERS[host]
-        except KeyError:
-            raise NoHandlerFound(url)
+        raise NoHandlerFound(url)
+
     try:
         mimetype, data = handler(url)
     except OSError as e:
@@ -150,7 +173,7 @@ def data_for_url(url):
 
 @add_handler('bookmarks')
 def qute_bookmarks(_url):
-    """Handler for qute:bookmarks. Display all quickmarks / bookmarks."""
+    """Handler for qute://bookmarks. Display all quickmarks / bookmarks."""
     bookmarks = sorted(objreg.get('bookmark-manager').marks.items(),
                        key=lambda x: x[1])  # Sort by title
     quickmarks = sorted(objreg.get('quickmark-manager').marks.items(),
@@ -163,90 +186,160 @@ def qute_bookmarks(_url):
     return 'text/html', html
 
 
-@add_handler('history')  # noqa
-def qute_history(url):
-    """Handler for qute:history. Display history."""
-    # Get current date from query parameter, if not given choose today.
-    curr_date = datetime.date.today()
-    try:
-        query_date = QUrlQuery(url).queryItemValue("date")
-        if query_date:
-            curr_date = datetime.datetime.strptime(query_date, "%Y-%m-%d")
-            curr_date = curr_date.date()
-    except ValueError:
-        log.misc.debug("Invalid date passed to qute:history: " + query_date)
+def history_data(start_time):  # noqa
+    """Return history data
 
-    one_day = datetime.timedelta(days=1)
-    next_date = curr_date + one_day
-    prev_date = curr_date - one_day
+    Arguments:
+        start_time -- select history starting from this timestamp.
+    """
+    def history_iter(start_time, reverse=False):
+        """Iterate through the history and get items we're interested.
 
-    def history_iter(reverse):
-        """Iterate through the history and get items we're interested in."""
-        curr_timestamp = time.mktime(curr_date.timetuple())
+        Arguments:
+            reverse -- whether to reverse the history_dict before iterating.
+        """
         history = objreg.get('web-history').history_dict.values()
         if reverse:
             history = reversed(history)
 
+        # when history_dict is not reversed, we need to keep track of last item
+        # so that we can yield its atime
+        last_item = None
+
+        # end is 24hrs earlier than start
+        end_time = start_time - 24*60*60
+
         for item in history:
-            # If we can't apply the reverse performance trick below,
-            # at least continue as early as possible with old items.
-            # This gets us down from 550ms to 123ms with 500k old items on my
-            # machine.
-            if item.atime < curr_timestamp and not reverse:
-                continue
-
-            # Convert timestamp
-            try:
-                item_atime = datetime.datetime.fromtimestamp(item.atime)
-            except (ValueError, OSError, OverflowError):
-                log.misc.debug("Invalid timestamp {}.".format(item.atime))
-                continue
-
-            if reverse and item_atime.date() < curr_date:
-                # If we could reverse the history in-place, and this entry is
-                # older than today, only older entries will follow, so we can
-                # abort here.
-                return
-
-            # Skip items not on curr_date
             # Skip redirects
             # Skip qute:// links
-            is_internal = item.url.scheme() == 'qute'
-            is_not_today = item_atime.date() != curr_date
-            if item.redirect or is_internal or is_not_today:
+            if item.redirect or item.url.scheme() == 'qute':
                 continue
 
+            # Skip items out of time window
+            item_newer = item.atime > start_time
+            item_older = item.atime <= end_time
+            if reverse:
+                # history_dict is reversed, we are going back in history.
+                # so:
+                #     abort if item is older than start_time+24hr
+                #     skip if item is newer than start
+                if item_older:
+                    yield {"next": int(item.atime)}
+                    return
+                if item_newer:
+                    continue
+            else:
+                # history_dict isn't reversed, we are going forward in history.
+                # so:
+                #     abort if item is newer than start_time
+                #     skip if item is older than start_time+24hrs
+                if item_older:
+                    last_item = item
+                    continue
+                if item_newer:
+                    yield {"next": int(last_item.atime if last_item else -1)}
+                    return
+
             # Use item's url as title if there's no title.
             item_url = item.url.toDisplayString()
             item_title = item.title if item.title else item_url
-            display_atime = item_atime.strftime("%X")
+            item_time = int(item.atime * 1000)
 
-            yield (item_url, item_title, display_atime)
+            yield {"url": item_url, "title": item_title, "time": item_time}
+
+        # if we reached here, we had reached the end of history
+        yield {"next": int(last_item.atime if last_item else -1)}
 
     if sys.hexversion >= 0x03050000:
         # On Python >= 3.5 we can reverse the ordereddict in-place and thus
         # apply an additional performance improvement in history_iter.
         # On my machine, this gets us down from 550ms to 72us with 500k old
         # items.
-        history = list(history_iter(reverse=True))
+        history = history_iter(start_time, reverse=True)
     else:
         # On Python 3.4, we can't do that, so we'd need to copy the entire
         # history to a list. There, filter first and then reverse it here.
-        history = reversed(list(history_iter(reverse=False)))
+        history = reversed(list(history_iter(start_time, reverse=False)))
 
-    html = jinja.render('history.html',
-                        title='History',
-                        history=history,
-                        curr_date=curr_date,
-                        next_date=next_date,
-                        prev_date=prev_date,
-                        today=datetime.date.today())
-    return 'text/html', html
+    return list(history)
+
+
+@add_handler('history')
+def qute_history(url):
+    """Handler for qute://history. Display and serve history."""
+    if url.path() == '/data':
+        # Use start_time in query or current time.
+        try:
+            start_time = QUrlQuery(url).queryItemValue("start_time")
+            start_time = float(start_time) if start_time else time.time()
+        except ValueError as e:
+            raise QuteSchemeError("Query parameter start_time is invalid", e)
+
+        return 'text/html', json.dumps(history_data(start_time))
+    else:
+        if (
+            config.get('content', 'allow-javascript') and
+            (objects.backend == usertypes.Backend.QtWebEngine or
+             qtutils.is_qtwebkit_ng())
+        ):
+            return 'text/html', jinja.render(
+                'history.html',
+                title='History',
+                session_interval=config.get('ui', 'history-session-interval')
+            )
+        else:
+            # Get current date from query parameter, if not given choose today.
+            curr_date = datetime.date.today()
+            try:
+                query_date = QUrlQuery(url).queryItemValue("date")
+                if query_date:
+                    curr_date = datetime.datetime.strptime(query_date,
+                        "%Y-%m-%d").date()
+            except ValueError:
+                log.misc.debug("Invalid date passed to qute:history: " +
+                    query_date)
+
+            one_day = datetime.timedelta(days=1)
+            next_date = curr_date + one_day
+            prev_date = curr_date - one_day
+
+            # start_time is the last second of curr_date
+            start_time = time.mktime(next_date.timetuple()) - 1
+            history = [
+                (i["url"], i["title"],
+                 datetime.datetime.fromtimestamp(i["time"]/1000),
+                 QUrl(i["url"]).host())
+                for i in history_data(start_time) if "next" not in i
+            ]
+
+            return 'text/html', jinja.render(
+                'history_nojs.html',
+                title='History',
+                history=history,
+                curr_date=curr_date,
+                next_date=next_date,
+                prev_date=prev_date,
+                today=datetime.date.today(),
+            )
+
+
+@add_handler('javascript')
+def qute_javascript(url):
+    """Handler for qute://javascript.
+
+    Return content of file given as query parameter.
+    """
+    path = url.path()
+    if path:
+        path = "javascript" + os.sep.join(path.split('/'))
+        return 'text/html', utils.read_file(path, binary=False)
+    else:
+        raise QuteSchemeError("No file specified", ValueError())
 
 
 @add_handler('pyeval')
 def qute_pyeval(_url):
-    """Handler for qute:pyeval."""
+    """Handler for qute://pyeval."""
     html = jinja.render('pre.html', title='pyeval', content=pyeval_output)
     return 'text/html', html
 
@@ -254,7 +347,7 @@ def qute_pyeval(_url):
 @add_handler('version')
 @add_handler('verizon')
 def qute_version(_url):
-    """Handler for qute:version."""
+    """Handler for qute://version."""
     html = jinja.render('version.html', title='Version info',
                         version=version.version(),
                         copyright=qutebrowser.__copyright__)
@@ -263,7 +356,7 @@ def qute_version(_url):
 
 @add_handler('plainlog')
 def qute_plainlog(url):
-    """Handler for qute:plainlog.
+    """Handler for qute://plainlog.
 
     An optional query parameter specifies the minimum log level to print.
     For example, qute://log?level=warning prints warnings and errors.
@@ -283,7 +376,7 @@ def qute_plainlog(url):
 
 @add_handler('log')
 def qute_log(url):
-    """Handler for qute:log.
+    """Handler for qute://log.
 
     An optional query parameter specifies the minimum log level to print.
     For example, qute://log?level=warning prints warnings and errors.
@@ -304,13 +397,13 @@ def qute_log(url):
 
 @add_handler('gpl')
 def qute_gpl(_url):
-    """Handler for qute:gpl. Return HTML content as string."""
+    """Handler for qute://gpl. Return HTML content as string."""
     return 'text/html', utils.read_file('html/COPYING.html')
 
 
 @add_handler('help')
 def qute_help(url):
-    """Handler for qute:help."""
+    """Handler for qute://help."""
     try:
         utils.read_file('html/doc/index.html')
     except OSError:
@@ -339,3 +432,14 @@ def qute_help(url):
     else:
         data = utils.read_file(path)
         return 'text/html', data
+
+
+@add_handler('backend-warning')
+def qute_backend_warning(_url):
+    """Handler for qute://backend-warning."""
+    html = jinja.render('backend-warning.html',
+                        distribution=version.distribution(),
+                        Distribution=version.Distribution,
+                        version=pkg_resources.parse_version,
+                        title="Legacy backend warning")
+    return 'text/html', html

qutebrowser/browser/shared.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -216,8 +216,10 @@ def get_tab(win_id, target):
         win_id = win_id
         bg_tab = True
     elif target == usertypes.ClickTarget.window:
+        tabbed_browser = objreg.get('tabbed-browser', scope='window',
+                                    window=win_id)
         from qutebrowser.mainwindow import mainwindow
-        window = mainwindow.MainWindow()
+        window = mainwindow.MainWindow(private=tabbed_browser.private)
         window.show()
         win_id = window.win_id
         bg_tab = False

qutebrowser/browser/signalfilter.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/urlmarks.py

@@ -1,7 +1,7 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
-# Copyright 2015-2016 Antoni Boucher <bouanto@zoho.com>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Antoni Boucher <bouanto@zoho.com>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webelem.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -22,9 +22,6 @@
 Module attributes:
     Group: Enum for different kinds of groups.
     SELECTORS: CSS selectors for different groups of elements.
-    FILTERS: A dictionary of filter functions for the modes.
-             The filter for "links" filters javascript:-links and a-tags
-             without "href".
 """
 
 import collections.abc
@@ -37,18 +34,16 @@ from qutebrowser.keyinput import modeman
 from qutebrowser.utils import log, usertypes, utils, qtutils, objreg
 
 
-Group = usertypes.enum('Group', ['all', 'links', 'images', 'url', 'prevnext',
-                                 'inputs'])
+Group = usertypes.enum('Group', ['all', 'links', 'images', 'url', 'inputs'])
 
 
 SELECTORS = {
     Group.all: ('a, area, textarea, select, input:not([type=hidden]), button, '
                 'frame, iframe, link, [onclick], [onmousedown], [role=link], '
                 '[role=option], [role=button], img'),
-    Group.links: 'a, area, link, [role=link]',
+    Group.links: 'a[href], area[href], link[href], [role=link][href]',
     Group.images: 'img',
     Group.url: '[src], [href]',
-    Group.prevnext: 'a, area, button, link, [role=button]',
     Group.inputs: ('input[type=text], input[type=email], input[type=url], '
                    'input[type=tel], input[type=number], '
                    'input[type=password], input[type=search], '
@@ -56,16 +51,6 @@ SELECTORS = {
 }
 
 
-def filter_links(elem):
-    return 'href' in elem and QUrl(elem['href']).scheme() != 'javascript'
-
-
-FILTERS = {
-    Group.links: filter_links,
-    Group.prevnext: filter_links,
-}
-
-
 class Error(Exception):
 
     """Base class for WebElement errors."""
@@ -120,10 +105,6 @@ class AbstractWebElement(collections.abc.MutableMapping):
         """Get the geometry for this element."""
         raise NotImplementedError
 
-    def style_property(self, name, *, strategy):
-        """Get the element style resolved with the given strategy."""
-        raise NotImplementedError
-
     def classes(self):
         """Get a list of classes assigned to this element."""
         raise NotImplementedError
@@ -310,6 +291,11 @@ class AbstractWebElement(collections.abc.MutableMapping):
         qtutils.ensure_valid(url)
         return url
 
+    def is_link(self):
+        """Return True if this AbstractWebElement is a link."""
+        href_tags = ['a', 'area', 'link']
+        return self.tag_name() in href_tags and 'href' in self
+
     def _mouse_pos(self):
         """Get the position to click/hover."""
         # Click the center of the largest square fitting into the top/left
@@ -326,6 +312,10 @@ class AbstractWebElement(collections.abc.MutableMapping):
             raise Error("Element position is out of view!")
         return pos
 
+    def _move_text_cursor(self):
+        """Move cursor to end after clicking."""
+        raise NotImplementedError
+
     def _click_fake_event(self, click_target):
         """Send a fake click event to the element."""
         pos = self._mouse_pos()
@@ -356,11 +346,7 @@ class AbstractWebElement(collections.abc.MutableMapping):
         for evt in events:
             self._tab.send_event(evt)
 
-        def after_click():
-            """Move cursor to end after clicking."""
-            if self.is_text_input() and self.is_editable():
-                self._tab.caret.move_to_end_of_document()
-        QTimer.singleShot(0, after_click)
+        QTimer.singleShot(0, self._move_text_cursor)
 
     def _click_editable(self, click_target):
         """Fake a click on an editable input field."""
@@ -378,15 +364,16 @@ class AbstractWebElement(collections.abc.MutableMapping):
             self._click_fake_event(click_target)
             return
 
+        tabbed_browser = objreg.get('tabbed-browser', scope='window',
+                                    window=self._tab.win_id)
+
         if click_target in [usertypes.ClickTarget.tab,
                             usertypes.ClickTarget.tab_bg]:
             background = click_target == usertypes.ClickTarget.tab_bg
-            tabbed_browser = objreg.get('tabbed-browser', scope='window',
-                                        window=self._tab.win_id)
             tabbed_browser.tabopen(url, background=background)
         elif click_target == usertypes.ClickTarget.window:
             from qutebrowser.mainwindow import mainwindow
-            window = mainwindow.MainWindow()
+            window = mainwindow.MainWindow(private=tabbed_browser.private)
             window.show()
             window.tabbed_browser.tabopen(url)
         else:
@@ -407,9 +394,8 @@ class AbstractWebElement(collections.abc.MutableMapping):
             self._click_fake_event(click_target)
             return
 
-        href_tags = ['a', 'area', 'link']
         if click_target == usertypes.ClickTarget.normal:
-            if self.tag_name() in href_tags:
+            if self.is_link():
                 log.webelem.debug("Clicking via JS click()")
                 self._click_js(click_target)
             elif self.is_editable(strict=True):
@@ -422,7 +408,7 @@ class AbstractWebElement(collections.abc.MutableMapping):
         elif click_target in [usertypes.ClickTarget.tab,
                               usertypes.ClickTarget.tab_bg,
                               usertypes.ClickTarget.window]:
-            if self.tag_name() in href_tags:
+            if self.is_link():
                 self._click_href(click_target)
             else:
                 self._click_fake_event(click_target)

qutebrowser/browser/webengine/__init__.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webengine/certificateerror.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -19,9 +19,7 @@
 
 """Wrapper over a QWebEngineCertificateError."""
 
-# pylint: disable=no-name-in-module,import-error,useless-suppression
 from PyQt5.QtWebEngineWidgets import QWebEngineCertificateError
-# pylint: enable=no-name-in-module,import-error,useless-suppression
 
 from qutebrowser.utils import usertypes, utils, debug
 

qutebrowser/browser/webengine/interceptor.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -19,9 +19,7 @@
 
 """A request interceptor taking care of adblocking and custom headers."""
 
-# pylint: disable=no-name-in-module,import-error,useless-suppression
 from PyQt5.QtWebEngineCore import QWebEngineUrlRequestInterceptor
-# pylint: enable=no-name-in-module,import-error,useless-suppression
 
 from qutebrowser.config import config
 from qutebrowser.browser import shared
@@ -57,8 +55,7 @@ class RequestInterceptor(QWebEngineUrlRequestInterceptor):
             info: QWebEngineUrlRequestInfo &info
         """
         # FIXME:qtwebengine only block ads for NavigationTypeOther?
-        if (bytes(info.requestMethod()) == b'GET' and
-                self._host_blocker.is_blocked(info.requestUrl())):
+        if self._host_blocker.is_blocked(info.requestUrl()):
             log.webview.info("Request to {} blocked by host blocker.".format(
                 info.requestUrl().host()))
             info.block(True)

qutebrowser/browser/webengine/tabhistory.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webengine/webenginedownloads.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -25,9 +25,7 @@ import urllib
 import functools
 
 from PyQt5.QtCore import pyqtSlot, Qt
-# pylint: disable=no-name-in-module,import-error,useless-suppression
 from PyQt5.QtWebEngineWidgets import QWebEngineDownloadItem
-# pylint: enable=no-name-in-module,import-error,useless-suppression
 
 from qutebrowser.browser import downloads
 from qutebrowser.utils import debug, usertypes, message, log, qtutils
@@ -78,32 +76,39 @@ class DownloadItem(downloads.AbstractDownloadItem):
             self.stats.finish()
         elif state == QWebEngineDownloadItem.DownloadInterrupted:
             self.successful = False
-            self.done = True
             # https://bugreports.qt.io/browse/QTBUG-56839
-            self.error.emit("Download failed")
-            self.stats.finish()
+            try:
+                reason = self._qt_item.interruptReasonString()
+            except AttributeError:
+                # Qt < 5.9
+                reason = "Download failed"
+            self._die(reason)
         else:
             raise ValueError("_on_state_changed was called with unknown state "
                              "{}".format(state_name))
 
     def _do_die(self):
         self._qt_item.downloadProgress.disconnect()
-        self._qt_item.cancel()
+        if self._qt_item.state() != QWebEngineDownloadItem.DownloadInterrupted:
+            self._qt_item.cancel()
 
     def _do_cancel(self):
         self._qt_item.cancel()
 
     def retry(self):
         # https://bugreports.qt.io/browse/QTBUG-56840
-        raise downloads.UnsupportedOperationError
+        raise downloads.UnsupportedOperationError(
+            "Retrying downloads is unsupported with QtWebEngine")
 
     def _get_open_filename(self):
         return self._filename
 
-    def _set_fileobj(self, fileobj):
+    def _set_fileobj(self, fileobj, *,
+                     autoclose=True):  # pylint: disable=unused-argument
         raise downloads.UnsupportedOperationError
 
     def _set_tempfile(self, fileobj):
+        fileobj.close()
         self._set_filename(fileobj.name, force_overwrite=True,
                            remember_directory=False)
 
@@ -144,8 +149,8 @@ def _get_suggested_filename(path):
     See https://bugreports.qt.io/browse/QTBUG-56978
     """
     filename = os.path.basename(path)
-    filename = re.sub(r'\([0-9]+\)$', '', filename)
-    if not qtutils.version_check('5.8.1'):
+    filename = re.sub(r'\([0-9]+\)(?=\.|$)', '', filename)
+    if not qtutils.version_check('5.9'):
         # https://bugreports.qt.io/browse/QTBUG-58155
         filename = urllib.parse.unquote(filename)
         # Doing basename a *second* time because there could be a %2F in

qutebrowser/browser/webengine/webengineelem.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -18,16 +18,14 @@
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
 # FIXME:qtwebengine remove this once the stubs are gone
-# pylint: disable=unused-variable
+# pylint: disable=unused-argument
 
 """QtWebEngine specific part of the web element API."""
 
 from PyQt5.QtCore import QRect, Qt, QPoint, QEventLoop
 from PyQt5.QtGui import QMouseEvent
 from PyQt5.QtWidgets import QApplication
-# pylint: disable=no-name-in-module,import-error,useless-suppression
 from PyQt5.QtWebEngineWidgets import QWebEngineSettings
-# pylint: enable=no-name-in-module,import-error,useless-suppression
 
 from qutebrowser.utils import log, javascript
 from qutebrowser.browser import webelem
@@ -39,6 +37,38 @@ class WebEngineElement(webelem.AbstractWebElement):
 
     def __init__(self, js_dict, tab):
         super().__init__(tab)
+        # Do some sanity checks on the data we get from JS
+        js_dict_types = {
+            'id': int,
+            'text': str,
+            'value': (str, int, float),
+            'tag_name': str,
+            'outer_xml': str,
+            'class_name': str,
+            'rects': list,
+            'attributes': dict,
+        }
+        assert set(js_dict.keys()).issubset(js_dict_types.keys())
+        for name, typ in js_dict_types.items():
+            if name in js_dict and not isinstance(js_dict[name], typ):
+                raise TypeError("Got {} for {} from JS but expected {}: "
+                                "{}".format(type(js_dict[name]), name, typ,
+                                            js_dict))
+        for name, value in js_dict['attributes'].items():
+            if not isinstance(name, str):
+                raise TypeError("Got {} ({}) for attribute name from JS: "
+                                "{}".format(name, type(name), js_dict))
+            if not isinstance(value, str):
+                raise TypeError("Got {} ({}) for attribute {} from JS: "
+                                "{}".format(value, type(value), name, js_dict))
+        for rect in js_dict['rects']:
+            assert set(rect.keys()) == {'top', 'right', 'bottom', 'left',
+                                        'height', 'width'}, rect.keys()
+            for value in rect.values():
+                if not isinstance(value, (int, float)):
+                    raise TypeError("Got {} ({}) for rect from JS: "
+                                    "{}".format(value, type(value), js_dict))
+
         self._id = js_dict['id']
         self._js_dict = js_dict
 
@@ -56,9 +86,7 @@ class WebEngineElement(webelem.AbstractWebElement):
 
     def __setitem__(self, key, val):
         self._js_dict['attributes'][key] = val
-        js_code = javascript.assemble('webelem', 'set_attribute', self._id,
-            key, val)
-        self._tab.run_js_async(js_code)
+        self._js_call('set_attribute', key, val)
 
     def __delitem__(self, key):
         log.stub()
@@ -69,6 +97,11 @@ class WebEngineElement(webelem.AbstractWebElement):
     def __len__(self):
         return len(self._js_dict['attributes'])
 
+    def _js_call(self, name, *args, callback=None):
+        """Wrapper to run stuff from webelem.js."""
+        js_code = javascript.assemble('webelem', name, self._id, *args)
+        self._tab.run_js_async(js_code, callback=callback)
+
     def has_frame(self):
         return True
 
@@ -76,10 +109,6 @@ class WebEngineElement(webelem.AbstractWebElement):
         log.stub()
         return QRect()
 
-    def style_property(self, name, *, strategy):
-        log.stub()
-        return ''
-
     def classes(self):
         """Get a list of classes assigned to this element."""
         return self._js_dict['class_name'].split()
@@ -89,7 +118,9 @@ class WebEngineElement(webelem.AbstractWebElement):
 
         The returned name will always be lower-case.
         """
-        return self._js_dict['tag_name'].lower()
+        tag = self._js_dict['tag_name']
+        assert isinstance(tag, str), tag
+        return tag.lower()
 
     def outer_xml(self):
         """Get the full HTML representation of this element."""
@@ -99,15 +130,13 @@ class WebEngineElement(webelem.AbstractWebElement):
         return self._js_dict.get('value', None)
 
     def set_value(self, value):
-        js_code = javascript.assemble('webelem', 'set_value', self._id, value)
-        self._tab.run_js_async(js_code)
+        self._js_call('set_value', value)
 
     def insert_text(self, text):
         if not self.is_editable(strict=True):
             raise webelem.Error("Element is not editable!")
         log.webelem.debug("Inserting text into element {!r}".format(self))
-        js_code = javascript.assemble('webelem', 'insert_text', self._id, text)
-        self._tab.run_js_async(js_code)
+        self._js_call('insert_text', text)
 
     def rect_on_view(self, *, elem_geometry=None, no_js=False):
         """Get the geometry of the element relative to the webview.
@@ -153,27 +182,27 @@ class WebEngineElement(webelem.AbstractWebElement):
     def remove_blank_target(self):
         if self._js_dict['attributes'].get('target') == '_blank':
             self._js_dict['attributes']['target'] = '_top'
-        js_code = javascript.assemble('webelem', 'remove_blank_target',
-            self._id)
-        self._tab.run_js_async(js_code)
+        self._js_call('remove_blank_target')
+
+    def _move_text_cursor(self):
+        if self.is_text_input() and self.is_editable():
+            self._js_call('move_cursor_to_end')
 
     def _click_editable(self, click_target):
         # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-58515
-        # pylint doesn't know about Qt.MouseEventSynthesizedBySystem
-        # because it was added in Qt 5.6, but we can be sure we use that with
-        # QtWebEngine.
-        # pylint: disable=no-member
         ev = QMouseEvent(QMouseEvent.MouseButtonPress, QPoint(0, 0),
                          QPoint(0, 0), QPoint(0, 0), Qt.NoButton, Qt.NoButton,
                          Qt.NoModifier, Qt.MouseEventSynthesizedBySystem)
-        # pylint: enable=no-member
         self._tab.send_event(ev)
         # This actually "clicks" the element by calling focus() on it in JS.
-        js_code = javascript.assemble('webelem', 'focus', self._id)
-        self._tab.run_js_async(js_code)
+        self._js_call('focus')
+        self._move_text_cursor()
 
     def _click_js(self, _click_target):
-        settings = QWebEngineSettings.globalSettings()
+        # FIXME:qtwebengine Have a proper API for this
+        # pylint: disable=protected-access
+        settings = self._tab._widget.settings()
+        # pylint: enable=protected-access
         attribute = QWebEngineSettings.JavascriptCanOpenWindows
         could_open_windows = settings.testAttribute(attribute)
         settings.setAttribute(attribute, True)
@@ -189,5 +218,4 @@ class WebEngineElement(webelem.AbstractWebElement):
         def reset_setting(_arg):
             settings.setAttribute(attribute, could_open_windows)
 
-        js_code = javascript.assemble('webelem', 'click', self._id)
-        self._tab.run_js_async(js_code, reset_setting)
+        self._js_call('click', callback=reset_setting)

qutebrowser/browser/webengine/webengineinspector.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -22,9 +22,7 @@
 import os
 
 from PyQt5.QtCore import QUrl
-# pylint: disable=no-name-in-module,import-error,useless-suppression
 from PyQt5.QtWebEngineWidgets import QWebEngineView
-# pylint: enable=no-name-in-module,import-error,useless-suppression
 
 from qutebrowser.browser import inspector
 

qutebrowser/browser/webengine/webenginequtescheme.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -17,24 +17,22 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-"""QtWebEngine specific qute:* handlers and glue code."""
+"""QtWebEngine specific qute://* handlers and glue code."""
 
 from PyQt5.QtCore import QBuffer, QIODevice
-# pylint: disable=no-name-in-module,import-error,useless-suppression
 from PyQt5.QtWebEngineCore import (QWebEngineUrlSchemeHandler,
                                    QWebEngineUrlRequestJob)
-# pylint: enable=no-name-in-module,import-error,useless-suppression
 
 from qutebrowser.browser import qutescheme
-from qutebrowser.utils import log
+from qutebrowser.utils import log, qtutils
 
 
 class QuteSchemeHandler(QWebEngineUrlSchemeHandler):
 
-    """Handle qute:* requests on QtWebEngine."""
+    """Handle qute://* requests on QtWebEngine."""
 
     def install(self, profile):
-        """Install the handler for qute: URLs on the given profile."""
+        """Install the handler for qute:// URLs on the given profile."""
         profile.installUrlSchemeHandler(b'qute', self)
 
     def requestStarted(self, job):
@@ -58,12 +56,15 @@ class QuteSchemeHandler(QWebEngineUrlSchemeHandler):
             job.fail(QWebEngineUrlRequestJob.UrlNotFound)
         except qutescheme.QuteSchemeOSError:
             # FIXME:qtwebengine how do we show a better error here?
-            log.misc.exception("OSError while handling qute:* URL")
+            log.misc.exception("OSError while handling qute://* URL")
             job.fail(QWebEngineUrlRequestJob.UrlNotFound)
         except qutescheme.QuteSchemeError:
             # FIXME:qtwebengine how do we show a better error here?
-            log.misc.exception("Error while handling qute:* URL")
+            log.misc.exception("Error while handling qute://* URL")
             job.fail(QWebEngineUrlRequestJob.RequestFailed)
+        except qutescheme.Redirect as e:
+            qtutils.ensure_valid(e.url)
+            job.redirect(e.url)
         else:
             log.misc.debug("Returning {} data".format(mimetype))
 

qutebrowser/browser/webengine/webenginesettings.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -17,6 +17,9 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
+# We get various "abstract but not overridden" warnings
+# pylint: disable=abstract-method
+
 """Bridge from QWebEngineSettings to our own settings.
 
 Module attributes:
@@ -25,80 +28,101 @@ Module attributes:
 """
 
 import os
+import sys
+import ctypes
+import ctypes.util
 
-# pylint: disable=no-name-in-module,import-error,useless-suppression
+from PyQt5.QtGui import QFont
 from PyQt5.QtWebEngineWidgets import (QWebEngineSettings, QWebEngineProfile,
                                       QWebEngineScript)
-# pylint: enable=no-name-in-module,import-error,useless-suppression
 
 from qutebrowser.browser import shared
 from qutebrowser.config import config, websettings
-from qutebrowser.utils import objreg, utils, standarddir, javascript
+from qutebrowser.utils import objreg, utils, standarddir, javascript, qtutils
 
 
-class Attribute(websettings.Attribute):
+# The default QWebEngineProfile
+default_profile = None
+# The QWebEngineProfile used for private (off-the-record) windows
+private_profile = None
+
+
+class Base(websettings.Base):
+
+    """Base settings class with appropriate _get_global_settings."""
+
+    def _get_global_settings(self):
+        return [default_profile.settings(), private_profile.settings()]
+
+
+class Attribute(Base, websettings.Attribute):
 
     """A setting set via QWebEngineSettings::setAttribute."""
 
-    GLOBAL_SETTINGS = QWebEngineSettings.globalSettings
     ENUM_BASE = QWebEngineSettings
 
 
-class Setter(websettings.Setter):
+class Setter(Base, websettings.Setter):
 
-    """A setting set via QWebEngineSettings getter/setter methods."""
+    """A setting set via a QWebEngineSettings setter method."""
 
-    GLOBAL_SETTINGS = QWebEngineSettings.globalSettings
+    pass
 
 
-class NullStringSetter(websettings.NullStringSetter):
+class FontFamilySetter(Base, websettings.FontFamilySetter):
 
-    """A setter for settings requiring a null QString as default."""
+    """A setter for a font family.
 
-    GLOBAL_SETTINGS = QWebEngineSettings.globalSettings
+    Gets the default value from QFont.
+    """
+
+    def __init__(self, font):
+        # Mapping from WebEngineSettings::initDefaults in
+        # qtwebengine/src/core/web_engine_settings.cpp
+        font_to_qfont = {
+            QWebEngineSettings.StandardFont: QFont.Serif,
+            QWebEngineSettings.FixedFont: QFont.Monospace,
+            QWebEngineSettings.SerifFont: QFont.Serif,
+            QWebEngineSettings.SansSerifFont: QFont.SansSerif,
+            QWebEngineSettings.CursiveFont: QFont.Cursive,
+            QWebEngineSettings.FantasyFont: QFont.Fantasy,
+        }
+        super().__init__(setter=QWebEngineSettings.setFontFamily, font=font,
+                         qfont=font_to_qfont[font])
 
 
-class StaticSetter(websettings.StaticSetter):
-
-    """A setting set via static QWebEngineSettings getter/setter methods."""
-
-    GLOBAL_SETTINGS = QWebEngineSettings.globalSettings
-
-
-class ProfileSetter(websettings.Base):
+class DefaultProfileSetter(websettings.Base):
 
     """A setting set on the QWebEngineProfile."""
 
-    def __init__(self, getter, setter):
-        super().__init__()
-        profile = QWebEngineProfile.defaultProfile()
-        self._getter = getattr(profile, getter)
-        self._setter = getattr(profile, setter)
+    def __init__(self, setter, default=websettings.UNSET):
+        super().__init__(default)
+        self._setter = setter
 
-    def get(self, settings=None):
-        utils.unused(settings)
-        return self._getter()
+    def __repr__(self):
+        return utils.get_repr(self, setter=self._setter, constructor=True)
 
     def _set(self, value, settings=None):
-        utils.unused(settings)
-        self._setter(value)
+        if settings is not None:
+            raise ValueError("'settings' may not be set with "
+                             "DefaultProfileSetters!")
+        setter = getattr(default_profile, self._setter)
+        setter(value)
 
 
-class PersistentCookiePolicy(ProfileSetter):
+class PersistentCookiePolicy(DefaultProfileSetter):
 
     """The cookies -> store setting is different from other settings."""
 
     def __init__(self):
-        super().__init__(getter='persistentCookiesPolicy',
-                         setter='setPersistentCookiesPolicy')
-
-    def get(self, settings=None):
-        utils.unused(settings)
-        return config.get('content', 'cookies-store')
+        super().__init__('setPersistentCookiesPolicy')
 
     def _set(self, value, settings=None):
-        utils.unused(settings)
-        self._setter(
+        if settings is not None:
+            raise ValueError("'settings' may not be set with "
+                             "PersistentCookiePolicy!")
+        setter = getattr(QWebEngineProfile.defaultProfile(), self._setter)
+        setter(
             QWebEngineProfile.AllowPersistentCookies if value else
             QWebEngineProfile.NoPersistentCookies
         )
@@ -110,9 +134,6 @@ def _init_stylesheet(profile):
     Mostly inspired by QupZilla:
     https://github.com/QupZilla/qupzilla/blob/v2.0/src/lib/app/mainapplication.cpp#L1063-L1101
     https://github.com/QupZilla/qupzilla/blob/v2.0/src/lib/tools/scripts.cpp#L119-L132
-
-    FIXME:qtwebengine Use QWebEngineStyleSheet once that's available
-    https://codereview.qt-project.org/#/c/148671/
     """
     old_script = profile.scripts().findScript('_qute_stylesheet')
     if not old_script.isNull():
@@ -137,19 +158,43 @@ def _init_stylesheet(profile):
     profile.scripts().insert(script)
 
 
-def _init_profile(profile):
-    """Initialize settings set on the QWebEngineProfile."""
-    profile.setCachePath(os.path.join(standarddir.cache(), 'webengine'))
-    profile.setPersistentStoragePath(
-        os.path.join(standarddir.data(), 'webengine'))
+def _set_user_agent(profile):
+    """Set the user agent for the given profile.
+
+    We override this per request in the URL interceptor (to allow for
+    per-domain user agents), but this one still gets used for things like
+    window.navigator.userAgent in JS.
+    """
+    user_agent = config.get('network', 'user-agent')
+    profile.setHttpUserAgent(user_agent)
 
 
 def update_settings(section, option):
     """Update global settings when qwebsettings changed."""
     websettings.update_mappings(MAPPINGS, section, option)
-    profile = QWebEngineProfile.defaultProfile()
     if section == 'ui' and option in ['hide-scrollbar', 'user-stylesheet']:
-        _init_stylesheet(profile)
+        _init_stylesheet(default_profile)
+        _init_stylesheet(private_profile)
+    elif section == 'network' and option == 'user-agent':
+        _set_user_agent(default_profile)
+        _set_user_agent(private_profile)
+
+
+def _init_profiles():
+    """Init the two used QWebEngineProfiles."""
+    global default_profile, private_profile
+    default_profile = QWebEngineProfile.defaultProfile()
+    default_profile.setCachePath(
+        os.path.join(standarddir.cache(), 'webengine'))
+    default_profile.setPersistentStoragePath(
+        os.path.join(standarddir.data(), 'webengine'))
+    _init_stylesheet(default_profile)
+    _set_user_agent(default_profile)
+
+    private_profile = QWebEngineProfile()
+    assert private_profile.isOffTheRecord()
+    _init_stylesheet(private_profile)
+    _set_user_agent(private_profile)
 
 
 def init(args):
@@ -157,13 +202,17 @@ def init(args):
     if args.enable_webengine_inspector:
         os.environ['QTWEBENGINE_REMOTE_DEBUGGING'] = str(utils.random_port())
 
-    profile = QWebEngineProfile.defaultProfile()
-    _init_profile(profile)
-    _init_stylesheet(profile)
+    # WORKAROUND for
+    # https://bugs.launchpad.net/ubuntu/+source/python-qt4/+bug/941826
+    if sys.platform == 'linux':
+        ctypes.CDLL(ctypes.util.find_library("GL"), mode=ctypes.RTLD_GLOBAL)
+
+    _init_profiles()
+
     # We need to do this here as a WORKAROUND for
     # https://bugreports.qt.io/browse/QTBUG-58650
-    PersistentCookiePolicy().set(config.get('content', 'cookies-store'))
-
+    if not qtutils.version_check('5.9'):
+        PersistentCookiePolicy().set(config.get('content', 'cookies-store'))
     Attribute(QWebEngineSettings.FullScreenSupportEnabled).set(True)
 
     websettings.init_mappings(MAPPINGS)
@@ -184,7 +233,6 @@ def shutdown():
 # - AllowRunningInsecureContent (5.8)
 #
 # Missing QtWebEngine fonts:
-# - FantasyFont
 # - PictographFont
 
 
@@ -206,9 +254,6 @@ MAPPINGS = {
             Attribute(QWebEngineSettings.LocalContentCanAccessRemoteUrls),
         'local-content-can-access-file-urls':
             Attribute(QWebEngineSettings.LocalContentCanAccessFileUrls),
-        # https://bugreports.qt.io/browse/QTBUG-58650
-        # 'cookies-store':
-        #     PersistentCookiePolicy(),
         'webgl':
             Attribute(QWebEngineSettings.WebGLEnabled),
     },
@@ -220,44 +265,28 @@ MAPPINGS = {
     },
     'fonts': {
         'web-family-standard':
-            Setter(getter=QWebEngineSettings.fontFamily,
-                   setter=QWebEngineSettings.setFontFamily,
-                   args=[QWebEngineSettings.StandardFont]),
+            FontFamilySetter(QWebEngineSettings.StandardFont),
         'web-family-fixed':
-            Setter(getter=QWebEngineSettings.fontFamily,
-                   setter=QWebEngineSettings.setFontFamily,
-                   args=[QWebEngineSettings.FixedFont]),
+            FontFamilySetter(QWebEngineSettings.FixedFont),
         'web-family-serif':
-            Setter(getter=QWebEngineSettings.fontFamily,
-                   setter=QWebEngineSettings.setFontFamily,
-                   args=[QWebEngineSettings.SerifFont]),
+            FontFamilySetter(QWebEngineSettings.SerifFont),
         'web-family-sans-serif':
-            Setter(getter=QWebEngineSettings.fontFamily,
-                   setter=QWebEngineSettings.setFontFamily,
-                   args=[QWebEngineSettings.SansSerifFont]),
+            FontFamilySetter(QWebEngineSettings.SansSerifFont),
         'web-family-cursive':
-            Setter(getter=QWebEngineSettings.fontFamily,
-                   setter=QWebEngineSettings.setFontFamily,
-                   args=[QWebEngineSettings.CursiveFont]),
+            FontFamilySetter(QWebEngineSettings.CursiveFont),
         'web-family-fantasy':
-            Setter(getter=QWebEngineSettings.fontFamily,
-                   setter=QWebEngineSettings.setFontFamily,
-                   args=[QWebEngineSettings.FantasyFont]),
+            FontFamilySetter(QWebEngineSettings.FantasyFont),
         'web-size-minimum':
-            Setter(getter=QWebEngineSettings.fontSize,
-                   setter=QWebEngineSettings.setFontSize,
+            Setter(QWebEngineSettings.setFontSize,
                    args=[QWebEngineSettings.MinimumFontSize]),
         'web-size-minimum-logical':
-            Setter(getter=QWebEngineSettings.fontSize,
-                   setter=QWebEngineSettings.setFontSize,
+            Setter(QWebEngineSettings.setFontSize,
                    args=[QWebEngineSettings.MinimumLogicalFontSize]),
         'web-size-default':
-            Setter(getter=QWebEngineSettings.fontSize,
-                   setter=QWebEngineSettings.setFontSize,
+            Setter(QWebEngineSettings.setFontSize,
                    args=[QWebEngineSettings.DefaultFontSize]),
         'web-size-default-fixed':
-            Setter(getter=QWebEngineSettings.fontSize,
-                   setter=QWebEngineSettings.setFontSize,
+            Setter(QWebEngineSettings.setFontSize,
                    args=[QWebEngineSettings.DefaultFixedFontSize]),
     },
     'ui': {
@@ -268,15 +297,14 @@ MAPPINGS = {
         'local-storage':
             Attribute(QWebEngineSettings.LocalStorageEnabled),
         'cache-size':
-            ProfileSetter(getter='httpCacheMaximumSize',
-                          setter='setHttpCacheMaximumSize')
+            # 0: automatically managed by QtWebEngine
+            DefaultProfileSetter('setHttpCacheMaximumSize', default=0),
     },
     'general': {
         'xss-auditing':
             Attribute(QWebEngineSettings.XSSAuditingEnabled),
         'default-encoding':
-            Setter(getter=QWebEngineSettings.defaultTextEncoding,
-                   setter=QWebEngineSettings.setDefaultTextEncoding),
+            Setter(QWebEngineSettings.setDefaultTextEncoding),
     }
 }
 
@@ -286,3 +314,8 @@ try:
 except AttributeError:
     # Added in Qt 5.8
     pass
+
+
+if qtutils.version_check('5.9'):
+    # https://bugreports.qt.io/browse/QTBUG-58650
+    MAPPINGS['content']['cookies-store'] = PersistentCookiePolicy()

qutebrowser/browser/webengine/webenginetab.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -17,30 +17,27 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-# FIXME:qtwebengine remove this once the stubs are gone
-# pylint: disable=unused-variable
-
 """Wrapper over a QWebEngineView."""
 
+import os
+import math
 import functools
 
 import sip
 from PyQt5.QtCore import pyqtSlot, Qt, QEvent, QPoint, QUrl, QTimer
 from PyQt5.QtGui import QKeyEvent
 from PyQt5.QtNetwork import QAuthenticator
-# pylint: disable=no-name-in-module,import-error,useless-suppression
 from PyQt5.QtWidgets import QApplication
-from PyQt5.QtWebEngineWidgets import (QWebEnginePage, QWebEngineScript,
-                                      QWebEngineProfile)
-# pylint: enable=no-name-in-module,import-error,useless-suppression
+from PyQt5.QtWebEngineWidgets import QWebEnginePage, QWebEngineScript
 
 from qutebrowser.browser import browsertab, mouse, shared
 from qutebrowser.browser.webengine import (webview, webengineelem, tabhistory,
                                            interceptor, webenginequtescheme,
-                                           webenginedownloads)
+                                           webenginedownloads,
+                                           webenginesettings)
 from qutebrowser.misc import miscwidgets
 from qutebrowser.utils import (usertypes, qtutils, log, javascript, utils,
-                               objreg, jinja)
+                               objreg, jinja, debug, version)
 
 
 _qute_scheme_handler = None
@@ -53,21 +50,31 @@ def init():
     # https://www.riverbankcomputing.com/pipermail/pyqt/2016-September/038075.html
     global _qute_scheme_handler
     app = QApplication.instance()
-    profile = QWebEngineProfile.defaultProfile()
 
-    log.init.debug("Initializing qute:* handler...")
+    software_rendering = (os.environ.get('LIBGL_ALWAYS_SOFTWARE') == '1' or
+                          'QT_XCB_FORCE_SOFTWARE_OPENGL' in os.environ)
+    if version.opengl_vendor() == 'nouveau' and not software_rendering:
+        # FIXME:qtwebengine display something more sophisticated here
+        raise browsertab.WebTabError(
+            "QtWebEngine is not supported with Nouveau graphics (unless "
+            "QT_XCB_FORCE_SOFTWARE_OPENGL is set as environment variable).")
+
+    log.init.debug("Initializing qute://* handler...")
     _qute_scheme_handler = webenginequtescheme.QuteSchemeHandler(parent=app)
-    _qute_scheme_handler.install(profile)
+    _qute_scheme_handler.install(webenginesettings.default_profile)
+    _qute_scheme_handler.install(webenginesettings.private_profile)
 
     log.init.debug("Initializing request interceptor...")
     host_blocker = objreg.get('host-blocker')
     req_interceptor = interceptor.RequestInterceptor(
         host_blocker, parent=app)
-    req_interceptor.install(profile)
+    req_interceptor.install(webenginesettings.default_profile)
+    req_interceptor.install(webenginesettings.private_profile)
 
     log.init.debug("Initializing QtWebEngine downloads...")
     download_manager = webenginedownloads.DownloadManager(parent=app)
-    download_manager.install(profile)
+    download_manager.install(webenginesettings.default_profile)
+    download_manager.install(webenginesettings.private_profile)
     objreg.register('webengine-download-manager', download_manager)
 
 
@@ -82,17 +89,17 @@ _JS_WORLD_MAP = {
 
 class WebEngineAction(browsertab.AbstractAction):
 
-    """QtWebKit implementations related to web actions."""
+    """QtWebEngine implementations related to web actions."""
 
-    def _action(self, action):
-        self._widget.triggerPageAction(action)
+    action_class = QWebEnginePage
+    action_base = QWebEnginePage.WebAction
 
     def exit_fullscreen(self):
-        self._action(QWebEnginePage.ExitFullScreen)
+        self._widget.triggerPageAction(QWebEnginePage.ExitFullScreen)
 
     def save_page(self):
         """Save the current page."""
-        self._action(QWebEnginePage.SavePage)
+        self._widget.triggerPageAction(QWebEnginePage.SavePage)
 
 
 class WebEnginePrinting(browsertab.AbstractPrinting):
@@ -128,12 +135,23 @@ class WebEngineSearch(browsertab.AbstractSearch):
         super().__init__(parent)
         self._flags = QWebEnginePage.FindFlags(0)
 
-    def _find(self, text, flags, cb=None):
-        """Call findText on the widget with optional callback."""
-        if cb is None:
-            self._widget.findText(text, flags)
-        else:
-            self._widget.findText(text, flags, cb)
+    def _find(self, text, flags, callback, caller):
+        """Call findText on the widget."""
+        self.search_displayed = True
+
+        def wrapped_callback(found):
+            """Wrap the callback to do debug logging."""
+            found_text = 'found' if found else "didn't find"
+            if flags:
+                flag_text = 'with flags {}'.format(debug.qflags_key(
+                    QWebEnginePage, flags, klass=QWebEnginePage.FindFlag))
+            else:
+                flag_text = ''
+            log.webview.debug(' '.join([caller, found_text, text, flag_text])
+                              .strip())
+            if callback is not None:
+                callback(found)
+        self._widget.findText(text, flags, wrapped_callback)
 
     def search(self, text, *, ignore_case=False, reverse=False,
                result_cb=None):
@@ -148,9 +166,10 @@ class WebEngineSearch(browsertab.AbstractSearch):
 
         self.text = text
         self._flags = flags
-        self._find(text, flags, result_cb)
+        self._find(text, flags, result_cb, 'search')
 
     def clear(self):
+        self.search_displayed = False
         self._widget.findText('')
 
     def prev_result(self, *, result_cb=None):
@@ -160,10 +179,10 @@ class WebEngineSearch(browsertab.AbstractSearch):
             flags &= ~QWebEnginePage.FindBackward
         else:
             flags |= QWebEnginePage.FindBackward
-        self._find(self.text, flags, result_cb)
+        self._find(self.text, flags, result_cb, 'prev_result')
 
     def next_result(self, *, result_cb=None):
-        self._find(self.text, self._flags, result_cb)
+        self._find(self.text, self._flags, result_cb, 'next_result')
 
 
 class WebEngineCaret(browsertab.AbstractCaret):
@@ -237,8 +256,47 @@ class WebEngineCaret(browsertab.AbstractCaret):
             raise browsertab.UnsupportedOperationError
         return self._widget.selectedText()
 
+    def _follow_selected_cb(self, js_elem, tab=False):
+        """Callback for javascript which clicks the selected element.
+
+        Args:
+            js_elem: The element serialized from javascript.
+            tab: Open in a new tab.
+        """
+        if js_elem is None:
+            return
+        assert isinstance(js_elem, dict), js_elem
+        elem = webengineelem.WebEngineElement(js_elem, tab=self._tab)
+        if tab:
+            click_type = usertypes.ClickTarget.tab
+        else:
+            click_type = usertypes.ClickTarget.normal
+
+        # Only click if we see a link
+        if elem.is_link():
+            log.webview.debug("Found link in selection, clicking. ClickTarget "
+                              "{}, elem {}".format(click_type, elem))
+            elem.click(click_type)
+
     def follow_selected(self, *, tab=False):
-        log.stub()
+        if self._tab.search.search_displayed:
+            # We are currently in search mode.
+            # let's click the link via a fake-click
+            # https://bugreports.qt.io/browse/QTBUG-60673
+            self._tab.search.clear()
+
+            log.webview.debug("Clicking a searched link via fake key press.")
+            # send a fake enter, clicking the orange selection box
+            if tab:
+                self._tab.key_press(Qt.Key_Enter, modifier=Qt.ControlModifier)
+            else:
+                self._tab.key_press(Qt.Key_Enter)
+
+        else:
+            # click an existing blue selection
+            js_code = javascript.assemble('webelem', 'find_selected_link')
+            self._tab.run_js_async(js_code, lambda jsret:
+                                   self._follow_selected_cb(jsret, tab))
 
 
 class WebEngineScroller(browsertab.AbstractScroller):
@@ -256,13 +314,10 @@ class WebEngineScroller(browsertab.AbstractScroller):
         page = widget.page()
         page.scrollPositionChanged.connect(self._update_pos)
 
-    def _key_press(self, key, count=1):
+    def _repeated_key_press(self, key, count=1, modifier=Qt.NoModifier):
+        """Send count fake key presses to this scroller's WebEngineTab."""
         for _ in range(min(count, 5000)):
-            press_evt = QKeyEvent(QEvent.KeyPress, key, Qt.NoModifier, 0, 0, 0)
-            release_evt = QKeyEvent(QEvent.KeyRelease, key, Qt.NoModifier,
-                                    0, 0, 0)
-            self._tab.send_event(press_evt)
-            self._tab.send_event(release_evt)
+            self._tab.key_press(key, modifier)
 
     @pyqtSlot()
     def _update_pos(self):
@@ -289,7 +344,7 @@ class WebEngineScroller(browsertab.AbstractScroller):
             else:
                 perc_y = min(100, round(100 / dy * jsret['px']['y']))
 
-            self._at_bottom = dy >= jsret['px']['y']
+            self._at_bottom = math.ceil(jsret['px']['y']) >= dy
             self._pos_perc = perc_x, perc_y
 
             self.perc_changed.emit(*self._pos_perc)
@@ -319,28 +374,28 @@ class WebEngineScroller(browsertab.AbstractScroller):
         self._tab.run_js_async(js_code)
 
     def up(self, count=1):
-        self._key_press(Qt.Key_Up, count)
+        self._repeated_key_press(Qt.Key_Up, count)
 
     def down(self, count=1):
-        self._key_press(Qt.Key_Down, count)
+        self._repeated_key_press(Qt.Key_Down, count)
 
     def left(self, count=1):
-        self._key_press(Qt.Key_Left, count)
+        self._repeated_key_press(Qt.Key_Left, count)
 
     def right(self, count=1):
-        self._key_press(Qt.Key_Right, count)
+        self._repeated_key_press(Qt.Key_Right, count)
 
     def top(self):
-        self._key_press(Qt.Key_Home)
+        self._tab.key_press(Qt.Key_Home)
 
     def bottom(self):
-        self._key_press(Qt.Key_End)
+        self._tab.key_press(Qt.Key_End)
 
     def page_up(self, count=1):
-        self._key_press(Qt.Key_PageUp, count)
+        self._repeated_key_press(Qt.Key_PageUp, count)
 
     def page_down(self, count=1):
-        self._key_press(Qt.Key_PageDown, count)
+        self._repeated_key_press(Qt.Key_PageDown, count)
 
     def at_top(self):
         return self.pos_px().y() == 0
@@ -369,11 +424,15 @@ class WebEngineHistory(browsertab.AbstractHistory):
         return self._history.canGoForward()
 
     def serialize(self):
-        # WORKAROUND for https://github.com/qutebrowser/qutebrowser/issues/2289
-        # FIXME:qtwebengine can we get rid of this with Qt 5.8.1?
-        scheme = self._history.currentItem().url().scheme()
-        if scheme in ['view-source', 'chrome']:
-            raise browsertab.WebTabError("Can't serialize special URL!")
+        if not qtutils.version_check('5.9'):
+            # WORKAROUND for
+            # https://github.com/qutebrowser/qutebrowser/issues/2289
+            # Don't use the history's currentItem here, because of
+            # https://bugreports.qt.io/browse/QTBUG-59599 and because it doesn't
+            # contain view-source.
+            scheme = self._tab.url().scheme()
+            if scheme in ['view-source', 'chrome']:
+                raise browsertab.WebTabError("Can't serialize special URL!")
         return qtutils.serialize(self._history)
 
     def deserialize(self, data):
@@ -443,18 +502,18 @@ class WebEngineElements(browsertab.AbstractElements):
             callback(elem)
 
     def find_css(self, selector, callback, *, only_visible=False):
-        js_code = javascript.assemble('webelem', 'find_all', selector,
+        js_code = javascript.assemble('webelem', 'find_css', selector,
                                       only_visible)
         js_cb = functools.partial(self._js_cb_multiple, callback)
         self._tab.run_js_async(js_code, js_cb)
 
     def find_id(self, elem_id, callback):
-        js_code = javascript.assemble('webelem', 'element_by_id', elem_id)
+        js_code = javascript.assemble('webelem', 'find_id', elem_id)
         js_cb = functools.partial(self._js_cb_single, callback)
         self._tab.run_js_async(js_code, js_cb)
 
     def find_focused(self, callback):
-        js_code = javascript.assemble('webelem', 'focus_element')
+        js_code = javascript.assemble('webelem', 'find_focused')
         js_cb = functools.partial(self._js_cb_single, callback)
         self._tab.run_js_async(js_code, js_cb)
 
@@ -462,7 +521,7 @@ class WebEngineElements(browsertab.AbstractElements):
         assert pos.x() >= 0
         assert pos.y() >= 0
         pos /= self._tab.zoom.factor()
-        js_code = javascript.assemble('webelem', 'element_at_pos',
+        js_code = javascript.assemble('webelem', 'find_at_pos',
                                       pos.x(), pos.y())
         js_cb = functools.partial(self._js_cb_single, callback)
         self._tab.run_js_async(js_code, js_cb)
@@ -472,10 +531,11 @@ class WebEngineTab(browsertab.AbstractTab):
 
     """A QtWebEngine tab in the browser."""
 
-    def __init__(self, win_id, mode_manager, parent=None):
+    def __init__(self, *, win_id, mode_manager, private, parent=None):
         super().__init__(win_id=win_id, mode_manager=mode_manager,
-                         parent=parent)
-        widget = webview.WebEngineView(tabdata=self.data, win_id=win_id)
+                         private=private, parent=parent)
+        widget = webview.WebEngineView(tabdata=self.data, win_id=win_id,
+                                       private=private)
         self.history = WebEngineHistory(self)
         self.scroller = WebEngineScroller(self, parent=self)
         self.caret = WebEngineCaret(win_id=win_id, mode_manager=mode_manager,
@@ -556,9 +616,10 @@ class WebEngineTab(browsertab.AbstractTab):
 
     def shutdown(self):
         self.shutting_down.emit()
-        # WORKAROUND for
-        # https://bugreports.qt.io/browse/QTBUG-58563
-        self.search.clear()
+        if qtutils.version_check('5.8', exact=True):
+            # WORKAROUND for
+            # https://bugreports.qt.io/browse/QTBUG-58563
+            self.search.clear()
         self._widget.shutdown()
 
     def reload(self, *, force=False):
@@ -577,14 +638,12 @@ class WebEngineTab(browsertab.AbstractTab):
     def icon(self):
         return self._widget.icon()
 
-    def set_html(self, html, base_url=None):
+    def set_html(self, html, base_url=QUrl()):
         # FIXME:qtwebengine
         # check this and raise an exception if too big:
         # Warning: The content will be percent encoded before being sent to the
         # renderer via IPC. This may increase its size. The maximum size of the
         # percent encoded content is 2 megabytes minus 30 bytes.
-        if base_url is None:
-            base_url = QUrl()
         self._widget.setHtml(html, base_url)
 
     def networkaccessmanager(self):
@@ -596,6 +655,13 @@ class WebEngineTab(browsertab.AbstractTab):
     def clear_ssl_errors(self):
         raise browsertab.UnsupportedOperationError
 
+    def key_press(self, key, modifier=Qt.NoModifier):
+        press_evt = QKeyEvent(QEvent.KeyPress, key, modifier, 0, 0, 0)
+        release_evt = QKeyEvent(QEvent.KeyRelease, key, modifier,
+                                0, 0, 0)
+        self.send_event(press_evt)
+        self.send_event(release_evt)
+
     @pyqtSlot()
     def _on_history_trigger(self):
         url = self.url()
@@ -639,12 +705,23 @@ class WebEngineTab(browsertab.AbstractTab):
     def _on_fullscreen_requested(self, request):
         request.accept()
         on = request.toggleOn()
+
+        self.data.fullscreen = on
         self.fullscreen_requested.emit(on)
         if on:
             notification = miscwidgets.FullscreenNotification(self)
             notification.show()
             notification.set_timeout(3000)
 
+    @pyqtSlot()
+    def _on_load_started(self):
+        """Clear search when a new load is started if needed."""
+        if qtutils.version_check('5.9'):
+            # WORKAROUND for
+            # https://bugreports.qt.io/browse/QTBUG-61506
+            self.search.clear()
+        super()._on_load_started()
+
     @pyqtSlot(QWebEnginePage.RenderProcessTerminationStatus, int)
     def _on_render_process_terminated(self, status, exitcode):
         """Show an error when the renderer process terminated."""

qutebrowser/browser/webengine/webview.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -23,12 +23,10 @@ import functools
 
 from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, PYQT_VERSION
 from PyQt5.QtGui import QPalette
-# pylint: disable=no-name-in-module,import-error,useless-suppression
 from PyQt5.QtWebEngineWidgets import QWebEngineView, QWebEnginePage
-# pylint: enable=no-name-in-module,import-error,useless-suppression
 
 from qutebrowser.browser import shared
-from qutebrowser.browser.webengine import certificateerror
+from qutebrowser.browser.webengine import certificateerror, webenginesettings
 from qutebrowser.config import config
 from qutebrowser.utils import (log, debug, usertypes, jinja, urlutils, message,
                                objreg)
@@ -38,13 +36,19 @@ class WebEngineView(QWebEngineView):
 
     """Custom QWebEngineView subclass with qutebrowser-specific features."""
 
-    def __init__(self, tabdata, win_id, parent=None):
+    def __init__(self, *, tabdata, win_id, private, parent=None):
         super().__init__(parent)
         self._win_id = win_id
         self._tabdata = tabdata
 
         theme_color = self.style().standardPalette().color(QPalette.Base)
-        page = WebEnginePage(theme_color=theme_color, parent=self)
+        if private:
+            profile = webenginesettings.private_profile
+            assert profile.isOffTheRecord()
+        else:
+            profile = webenginesettings.default_profile
+        page = WebEnginePage(theme_color=theme_color, profile=profile,
+                             parent=self)
         self.setPage(page)
 
     def shutdown(self):
@@ -124,8 +128,8 @@ class WebEnginePage(QWebEnginePage):
     certificate_error = pyqtSignal()
     shutting_down = pyqtSignal()
 
-    def __init__(self, theme_color, parent=None):
-        super().__init__(parent)
+    def __init__(self, *, theme_color, profile, parent=None):
+        super().__init__(profile, parent)
         self._is_shutting_down = False
         self.featurePermissionRequested.connect(
             self._on_feature_permission_requested)

qutebrowser/browser/webkit/__init__.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webkit/cache.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -21,161 +21,35 @@
 
 import os.path
 
-from PyQt5.QtCore import pyqtSlot
-from PyQt5.QtNetwork import QNetworkDiskCache, QNetworkCacheMetaData
+from PyQt5.QtNetwork import QNetworkDiskCache
 
 from qutebrowser.config import config
-from qutebrowser.utils import utils, objreg
+from qutebrowser.utils import utils, objreg, qtutils
 
 
 class DiskCache(QNetworkDiskCache):
 
-    """Disk cache which sets correct cache dir and size.
-
-    Attributes:
-        _activated: Whether the cache should be used.
-        _cache_dir: The base directory for cache files (standarddir.cache())
-    """
+    """Disk cache which sets correct cache dir and size."""
 
     def __init__(self, cache_dir, parent=None):
         super().__init__(parent)
-        self._cache_dir = cache_dir
-        self._maybe_activate()
-        objreg.get('config').changed.connect(self.on_config_changed)
+        self.setCacheDirectory(os.path.join(cache_dir, 'http'))
+        self._set_cache_size()
+        objreg.get('config').changed.connect(self._set_cache_size)
 
     def __repr__(self):
         return utils.get_repr(self, size=self.cacheSize(),
                               maxsize=self.maximumCacheSize(),
                               path=self.cacheDirectory())
 
+    @config.change_filter('storage', 'cache-size')
     def _set_cache_size(self):
         """Set the cache size based on the config."""
         size = config.get('storage', 'cache-size')
         if size is None:
             size = 1024 * 1024 * 50  # default from QNetworkDiskCachePrivate
+        # WORKAROUND for https://bugreports.qt.io/browse/QTBUG-59909
+        if (qtutils.version_check('5.7.1') and
+                not qtutils.version_check('5.9')):  # pragma: no cover
+            size = 0
         self.setMaximumCacheSize(size)
-
-    def _maybe_activate(self):
-        """Activate/deactivate the cache based on the config."""
-        if config.get('general', 'private-browsing'):
-            self._activated = False
-        else:
-            self._activated = True
-            self.setCacheDirectory(os.path.join(self._cache_dir, 'http'))
-            self._set_cache_size()
-
-    @pyqtSlot(str, str)
-    def on_config_changed(self, section, option):
-        """Update cache size/activated if the config was changed."""
-        if (section, option) == ('storage', 'cache-size'):
-            self._set_cache_size()
-        elif (section, option) == ('general',  # pragma: no branch
-                                   'private-browsing'):
-            self._maybe_activate()
-
-    def cacheSize(self):
-        """Return the current size taken up by the cache.
-
-        Return:
-            An int.
-        """
-        if self._activated:
-            return super().cacheSize()
-        else:
-            return 0
-
-    def fileMetaData(self, filename):
-        """Return the QNetworkCacheMetaData for the cache file filename.
-
-        Args:
-            filename: The file name as a string.
-
-        Return:
-            A QNetworkCacheMetaData object.
-        """
-        if self._activated:
-            return super().fileMetaData(filename)
-        else:
-            return QNetworkCacheMetaData()
-
-    def data(self, url):
-        """Return the data associated with url.
-
-        Args:
-            url: A QUrl.
-
-        return:
-            A QIODevice or None.
-        """
-        if self._activated:
-            return super().data(url)
-        else:
-            return None
-
-    def insert(self, device):
-        """Insert the data in device and the prepared meta data into the cache.
-
-        Args:
-            device: A QIODevice.
-        """
-        if self._activated:
-            super().insert(device)
-        else:
-            return None
-
-    def metaData(self, url):
-        """Return the meta data for the url url.
-
-        Args:
-            url: A QUrl.
-
-        Return:
-            A QNetworkCacheMetaData object.
-        """
-        if self._activated:
-            return super().metaData(url)
-        else:
-            return QNetworkCacheMetaData()
-
-    def prepare(self, meta_data):
-        """Return the device that should be populated with the data.
-
-        Args:
-            meta_data: A QNetworkCacheMetaData object.
-
-        Return:
-            A QIODevice or None.
-        """
-        if self._activated:
-            return super().prepare(meta_data)
-        else:
-            return None
-
-    def remove(self, url):
-        """Remove the cache entry for url.
-
-        Return:
-            True on success, False otherwise.
-        """
-        if self._activated:
-            return super().remove(url)
-        else:
-            return False
-
-    def updateMetaData(self, meta_data):
-        """Update the cache meta date for the meta_data's url to meta_data.
-
-        Args:
-            meta_data: A QNetworkCacheMetaData object.
-        """
-        if self._activated:
-            super().updateMetaData(meta_data)
-        else:
-            return
-
-    def clear(self):
-        """Remove all items from the cache."""
-        if self._activated:
-            super().clear()
-        else:
-            return

qutebrowser/browser/webkit/certificateerror.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -41,7 +41,7 @@ class CertificateErrorWrapper(usertypes.AbstractCertificateErrorWrapper):
         try:
             # Qt >= 5.4
             return hash(self._error)
-        except TypeError:
+        except TypeError:  # pragma: no cover
             return hash((self._error.certificate().toDer(),
                          self._error.error()))
 

qutebrowser/browser/webkit/cookies.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webkit/http.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webkit/mhtml.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2016 Daniel Schadt
+# Copyright 2015-2017 Daniel Schadt
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webkit/network/filescheme.py

@@ -1,7 +1,7 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
-# Copyright 2015-2016 Antoni Boucher (antoyo) <bouanto@zoho.com>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Antoni Boucher (antoyo) <bouanto@zoho.com>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webkit/network/networkmanager.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -121,13 +121,13 @@ class NetworkManager(QNetworkAccessManager):
                            reparented to the DownloadManager. This counts the
                            still running downloads, so the QNAM can clean
                            itself up when this reaches zero again.
-        _requests: Pending requests.
         _scheme_handlers: A dictionary (scheme -> handler) of supported custom
                           schemes.
         _win_id: The window ID this NetworkManager is associated with.
         _tab_id: The tab ID this NetworkManager is associated with.
         _rejected_ssl_errors: A {QUrl: [SslError]} dict of rejected errors.
         _accepted_ssl_errors: A {QUrl: [SslError]} dict of accepted errors.
+        _private: Whether we're in private browsing mode.
 
     Signals:
         shutting_down: Emitted when the QNAM is shutting down.
@@ -135,7 +135,7 @@ class NetworkManager(QNetworkAccessManager):
 
     shutting_down = pyqtSignal()
 
-    def __init__(self, win_id, tab_id, parent=None):
+    def __init__(self, *, win_id, tab_id, private, parent=None):
         log.init.debug("Initializing NetworkManager")
         with log.disable_qt_msghandler():
             # WORKAROUND for a hang when a message is printed - See:
@@ -145,12 +145,12 @@ class NetworkManager(QNetworkAccessManager):
         self.adopted_downloads = 0
         self._win_id = win_id
         self._tab_id = tab_id
-        self._requests = []
+        self._private = private
         self._scheme_handlers = {
             'qute': webkitqutescheme.QuteSchemeHandler(win_id),
             'file': filescheme.FileSchemeHandler(win_id),
         }
-        self._set_cookiejar(private=config.get('general', 'private-browsing'))
+        self._set_cookiejar()
         self._set_cache()
         self.sslErrors.connect(self.on_ssl_errors)
         self._rejected_ssl_errors = collections.defaultdict(list)
@@ -158,15 +158,10 @@ class NetworkManager(QNetworkAccessManager):
         self.authenticationRequired.connect(self.on_authentication_required)
         self.proxyAuthenticationRequired.connect(
             self.on_proxy_authentication_required)
-        objreg.get('config').changed.connect(self.on_config_changed)
 
-    def _set_cookiejar(self, private=False):
-        """Set the cookie jar of the NetworkManager correctly.
-
-        Args:
-            private: Whether we're currently in private browsing mode.
-        """
-        if private:
+    def _set_cookiejar(self):
+        """Set the cookie jar of the NetworkManager correctly."""
+        if self._private:
             cookie_jar = objreg.get('ram-cookie-jar')
         else:
             cookie_jar = objreg.get('cookie-jar')
@@ -178,11 +173,9 @@ class NetworkManager(QNetworkAccessManager):
         cookie_jar.setParent(app)
 
     def _set_cache(self):
-        """Set the cache of the NetworkManager correctly.
-
-        We can't switch the whole cache in private mode because QNAM would
-        delete the old cache.
-        """
+        """Set the cache of the NetworkManager correctly."""
+        if self._private:
+            return
         # We have a shared cache - we restore its parent so we don't take
         # ownership of it.
         app = QCoreApplication.instance()
@@ -206,9 +199,6 @@ class NetworkManager(QNetworkAccessManager):
     def shutdown(self):
         """Abort all running requests."""
         self.setNetworkAccessible(QNetworkAccessManager.NotAccessible)
-        for request in self._requests:
-            request.abort()
-            request.deleteLater()
         self.shutting_down.emit()
 
     # No @pyqtSlot here, see
@@ -324,17 +314,6 @@ class NetworkManager(QNetworkAccessManager):
                 authenticator.setPassword(answer.password)
                 _proxy_auth_cache[proxy_id] = answer
 
-    @config.change_filter('general', 'private-browsing')
-    def on_config_changed(self):
-        """Set cookie jar when entering/leaving private browsing mode."""
-        private_browsing = config.get('general', 'private-browsing')
-        if private_browsing:
-            # switched from normal mode to private mode
-            self._set_cookiejar(private=True)
-        else:
-            # switched from private mode to normal mode
-            self._set_cookiejar()
-
     @pyqtSlot()
     def on_adopted_download_destroyed(self):
         """Check if we can clean up if an adopted download was destroyed.
@@ -386,9 +365,6 @@ class NetworkManager(QNetworkAccessManager):
     def createRequest(self, op, req, outgoing_data):
         """Return a new QNetworkReply object.
 
-        Extend QNetworkAccessManager::createRequest to save requests in
-        self._requests and handle custom schemes.
-
         Args:
              op: Operation op
              req: const QNetworkRequest & req
@@ -416,8 +392,7 @@ class NetworkManager(QNetworkAccessManager):
             req.setRawHeader(header, value)
 
         host_blocker = objreg.get('host-blocker')
-        if (op == QNetworkAccessManager.GetOperation and
-                host_blocker.is_blocked(req.url())):
+        if host_blocker.is_blocked(req.url()):
             log.webview.info("Request to {} blocked by host blocker.".format(
                 req.url().host()))
             return networkreply.ErrorNetworkReply(
@@ -454,6 +429,4 @@ class NetworkManager(QNetworkAccessManager):
                 reply = super().createRequest(op, req, outgoing_data)
         else:
             reply = super().createRequest(op, req, outgoing_data)
-        self._requests.append(reply)
-        reply.destroyed.connect(self._requests.remove)
         return reply

qutebrowser/browser/webkit/network/networkreply.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # Based on the Eric5 helpviewer,
 # Copyright (c) 2009 - 2014 Detlev Offenbach <detlev@die-offenbachs.de>
@@ -19,6 +19,10 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+#
+# For some reason, a segfault will be triggered if the unnecessary lambdas in
+# this file aren't there.
+# pylint: disable=unnecessary-lambda
 
 """Special network replies.."""
 
@@ -114,9 +118,6 @@ class ErrorNetworkReply(QNetworkReply):
         # the device to avoid getting a warning.
         self.setOpenMode(QIODevice.ReadOnly)
         self.setError(error, errorstring)
-        # For some reason, a segfault will be triggered if these lambdas aren't
-        # there.
-        # pylint: disable=unnecessary-lambda
         QTimer.singleShot(0, lambda: self.error.emit(error))
         QTimer.singleShot(0, lambda: self.finished.emit())
 
@@ -137,3 +138,20 @@ class ErrorNetworkReply(QNetworkReply):
 
     def isRunning(self):
         return False
+
+
+class RedirectNetworkReply(QNetworkReply):
+
+    """A reply which redirects to the given URL."""
+
+    def __init__(self, new_url, parent=None):
+        super().__init__(parent)
+        self.setAttribute(QNetworkRequest.RedirectionTargetAttribute, new_url)
+        QTimer.singleShot(0, lambda: self.finished.emit())
+
+    def abort(self):
+        """Called when there's e.g. a redirection limit."""
+        pass
+
+    def readData(self, _maxlen):
+        return bytes()

qutebrowser/browser/webkit/network/schemehandler.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # Based on the Eric5 helpviewer,
 # Copyright (c) 2009 - 2014 Detlev Offenbach <detlev@die-offenbachs.de>

qutebrowser/browser/webkit/network/webkitqutescheme.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-"""QtWebKit specific qute:* handlers and glue code."""
+"""QtWebKit specific qute://* handlers and glue code."""
 
 import mimetypes
 import functools
@@ -28,13 +28,13 @@ from PyQt5.QtNetwork import QNetworkReply
 
 from qutebrowser.browser import pdfjs, qutescheme
 from qutebrowser.browser.webkit.network import schemehandler, networkreply
-from qutebrowser.utils import jinja, log, message, objreg, usertypes
+from qutebrowser.utils import jinja, log, message, objreg, usertypes, qtutils
 from qutebrowser.config import configexc, configdata
 
 
 class QuteSchemeHandler(schemehandler.SchemeHandler):
 
-    """Scheme handler for qute: URLs."""
+    """Scheme handler for qute:// URLs."""
 
     def createRequest(self, _op, request, _outgoing_data):
         """Create a new request.
@@ -62,6 +62,9 @@ class QuteSchemeHandler(schemehandler.SchemeHandler):
         except qutescheme.QuteSchemeError as e:
             return networkreply.ErrorNetworkReply(request, e.errorstring,
                                                   e.error, self.parent())
+        except qutescheme.Redirect as e:
+            qtutils.ensure_valid(e.url)
+            return networkreply.RedirectNetworkReply(e.url, self.parent())
 
         return networkreply.FixedDataNetworkReply(request, data, mimetype,
                                                   self.parent())
@@ -69,15 +72,15 @@ class QuteSchemeHandler(schemehandler.SchemeHandler):
 
 class JSBridge(QObject):
 
-    """Javascript-bridge for special qute:... pages."""
+    """Javascript-bridge for special qute://... pages."""
 
     @pyqtSlot(str, str, str)
     def set(self, sectname, optname, value):
-        """Slot to set a setting from qute:settings."""
+        """Slot to set a setting from qute://settings."""
         # https://github.com/qutebrowser/qutebrowser/issues/727
         if ((sectname, optname) == ('content', 'allow-javascript') and
                 value == 'false'):
-            message.error("Refusing to disable javascript via qute:settings "
+            message.error("Refusing to disable javascript via qute://settings "
                           "as it needs javascript support.")
             return
         try:
@@ -88,7 +91,7 @@ class JSBridge(QObject):
 
 @qutescheme.add_handler('settings', backend=usertypes.Backend.QtWebKit)
 def qute_settings(_url):
-    """Handler for qute:settings. View/change qute configuration."""
+    """Handler for qute://settings. View/change qute configuration."""
     config_getter = functools.partial(objreg.get('config').get, raw=True)
     html = jinja.render('settings.html', title='settings', config=configdata,
                         confget=config_getter)

qutebrowser/browser/webkit/rfc6266.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -286,9 +286,6 @@ def normalize_ws(text):
 
 def parse_headers(content_disposition):
     """Build a _ContentDisposition from header values."""
-    # WORKAROUND for https://bitbucket.org/logilab/pylint/issue/492/
-    # pylint: disable=no-member
-
     # We allow non-ascii here (it will only be parsed inside of qdtext, and
     # rejected by the grammar if it appears in other places), although parsing
     # it can be ambiguous.  Parsing it ensures that a non-ambiguous filename*

qutebrowser/browser/webkit/tabhistory.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -21,7 +21,6 @@
 
 
 from PyQt5.QtCore import QByteArray, QDataStream, QIODevice, QUrl
-from PyQt5.QtWebKit import qWebKitVersion
 
 from qutebrowser.utils import qtutils
 
@@ -181,7 +180,7 @@ def serialize(items):
     else:
         current_idx = 0
 
-    if qtutils.is_qtwebkit_ng(qWebKitVersion()):
+    if qtutils.is_qtwebkit_ng():
         _serialize_ng(items, current_idx, stream)
     else:
         _serialize_old(items, current_idx, stream)

qutebrowser/browser/webkit/webkitelem.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2014-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -96,16 +96,6 @@ class WebKitElement(webelem.AbstractWebElement):
         self._check_vanished()
         return self._elem.geometry()
 
-    def style_property(self, name, *, strategy):
-        self._check_vanished()
-        strategies = {
-            # FIXME:qtwebengine which ones do we actually need?
-            'inline': QWebElement.InlineStyle,
-            'computed': QWebElement.ComputedStyle,
-        }
-        qt_strategy = strategies[strategy]
-        return self._elem.styleProperty(name, qt_strategy)
-
     def classes(self):
         self._check_vanished()
         return self._elem.classes()
@@ -122,7 +112,9 @@ class WebKitElement(webelem.AbstractWebElement):
 
     def value(self):
         self._check_vanished()
-        return self._elem.evaluateJavaScript('this.value')
+        val = self._elem.evaluateJavaScript('this.value')
+        assert isinstance(val, (int, float, str, type(None))), val
+        return val
 
     def set_value(self, value):
         self._check_vanished()
@@ -293,16 +285,24 @@ class WebKitElement(webelem.AbstractWebElement):
         for _ in range(5):
             if elem is None:
                 break
-            tag = elem.tag_name()
-            if tag == 'a' or tag == 'area':
+            if elem.is_link():
                 if elem.get('target', None) == '_blank':
                     elem['target'] = '_top'
                 break
             elem = elem._parent()  # pylint: disable=protected-access
 
+    def _move_text_cursor(self):
+        if self is None:
+            # old PyQt versions call the slot after the element is deleted.
+            return
+        if self.is_text_input() and self.is_editable():
+            self._tab.caret.move_to_end_of_document()
+
     def _click_editable(self, click_target):
         ok = self._elem.evaluateJavaScript('this.focus(); true;')
-        if not ok:
+        if ok:
+            self._move_text_cursor()
+        else:
             log.webelem.debug("Failed to focus via JS, falling back to event")
             self._click_fake_event(click_target)
 

qutebrowser/browser/webkit/webkithistory.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webkit/webkitinspector.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2015-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/webkit/webkitsettings.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -17,6 +17,9 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
+# We get various "abstract but not overridden" warnings
+# pylint: disable=abstract-method
+
 """Bridge from QWebSettings to our own settings.
 
 Module attributes:
@@ -26,43 +29,66 @@ Module attributes:
 
 import os.path
 
-from PyQt5.QtWebKit import QWebSettings, qWebKitVersion
+from PyQt5.QtGui import QFont
+from PyQt5.QtWebKit import QWebSettings
 
 from qutebrowser.config import config, websettings
-from qutebrowser.utils import standarddir, objreg, urlutils, qtutils, message
+from qutebrowser.utils import standarddir, objreg, urlutils, qtutils
 from qutebrowser.browser import shared
 
 
-class Attribute(websettings.Attribute):
+class Base(websettings.Base):
+
+    """Base settings class with appropriate _get_global_settings."""
+
+    def _get_global_settings(self):
+        return [QWebSettings.globalSettings()]
+
+
+class Attribute(Base, websettings.Attribute):
 
     """A setting set via QWebSettings::setAttribute."""
 
-    GLOBAL_SETTINGS = QWebSettings.globalSettings
     ENUM_BASE = QWebSettings
 
 
-class Setter(websettings.Setter):
+class Setter(Base, websettings.Setter):
 
-    """A setting set via QWebSettings getter/setter methods."""
+    """A setting set via a QWebSettings setter method."""
 
-    GLOBAL_SETTINGS = QWebSettings.globalSettings
+    pass
 
 
-class NullStringSetter(websettings.NullStringSetter):
+class StaticSetter(Base, websettings.StaticSetter):
 
-    """A setter for settings requiring a null QString as default."""
+    """A setting set via a static QWebSettings setter method."""
 
-    GLOBAL_SETTINGS = QWebSettings.globalSettings
+    pass
 
 
-class StaticSetter(websettings.StaticSetter):
+class FontFamilySetter(Base, websettings.FontFamilySetter):
 
-    """A setting set via static QWebSettings getter/setter methods."""
+    """A setter for a font family.
 
-    GLOBAL_SETTINGS = QWebSettings.globalSettings
+    Gets the default value from QFont.
+    """
+
+    def __init__(self, font):
+        # Mapping from QWebSettings::QWebSettings() in
+        # qtwebkit/Source/WebKit/qt/Api/qwebsettings.cpp
+        font_to_qfont = {
+            QWebSettings.StandardFont: QFont.Serif,
+            QWebSettings.FixedFont: QFont.Monospace,
+            QWebSettings.SerifFont: QFont.Serif,
+            QWebSettings.SansSerifFont: QFont.SansSerif,
+            QWebSettings.CursiveFont: QFont.Cursive,
+            QWebSettings.FantasyFont: QFont.Fantasy,
+        }
+        super().__init__(setter=QWebSettings.setFontFamily, font=font,
+                         qfont=font_to_qfont[font])
 
 
-class CookiePolicy(websettings.Base):
+class CookiePolicy(Base):
 
     """The ThirdPartyCookiePolicy setting is different from other settings."""
 
@@ -73,12 +99,9 @@ class CookiePolicy(websettings.Base):
         'no-unknown-3rdparty': QWebSettings.AllowThirdPartyWithExistingCookies,
     }
 
-    def get(self, settings=None):
-        return config.get('content', 'cookies-accept')
-
     def _set(self, value, settings=None):
-        QWebSettings.globalSettings().setThirdPartyCookiePolicy(
-            self.MAPPING[value])
+        for obj in self._get_settings(settings):
+            obj.setThirdPartyCookiePolicy(self.MAPPING[value])
 
 
 def _set_user_stylesheet():
@@ -88,21 +111,9 @@ def _set_user_stylesheet():
     QWebSettings.globalSettings().setUserStyleSheetUrl(url)
 
 
-def _init_private_browsing():
-    if config.get('general', 'private-browsing'):
-        if qtutils.is_qtwebkit_ng(qWebKitVersion()):
-            message.warning("Private browsing is not fully implemented by "
-                            "QtWebKit-NG!")
-        QWebSettings.setIconDatabasePath('')
-    else:
-        QWebSettings.setIconDatabasePath(standarddir.cache())
-
-
 def update_settings(section, option):
     """Update global settings when qwebsettings changed."""
-    if (section, option) == ('general', 'private-browsing'):
-        _init_private_browsing()
-    elif section == 'ui' and option in ['hide-scrollbar', 'user-stylesheet']:
+    if section == 'ui' and option in ['hide-scrollbar', 'user-stylesheet']:
         _set_user_stylesheet()
 
     websettings.update_mappings(MAPPINGS, section, option)
@@ -113,8 +124,7 @@ def init(_args):
     cache_path = standarddir.cache()
     data_path = standarddir.data()
 
-    _init_private_browsing()
-
+    QWebSettings.setIconDatabasePath(standarddir.cache())
     QWebSettings.setOfflineWebApplicationCachePath(
         os.path.join(cache_path, 'application-cache'))
     QWebSettings.globalSettings().setLocalStoragePath(
@@ -122,6 +132,13 @@ def init(_args):
     QWebSettings.setOfflineStoragePath(
         os.path.join(data_path, 'offline-storage'))
 
+    if (config.get('general', 'private-browsing') and
+            not qtutils.version_check('5.4.2')):
+        # WORKAROUND for https://codereview.qt-project.org/#/c/108936/
+        # Won't work when private browsing is not enabled globally, but that's
+        # the best we can do...
+        QWebSettings.setIconDatabasePath('')
+
     websettings.init_mappings(MAPPINGS)
     _set_user_stylesheet()
     objreg.get('config').changed.connect(update_settings)
@@ -146,14 +163,10 @@ MAPPINGS = {
             Attribute(QWebSettings.JavascriptCanCloseWindows),
         'javascript-can-access-clipboard':
             Attribute(QWebSettings.JavascriptCanAccessClipboard),
-        #'allow-java':
-        #   Attribute(QWebSettings.JavaEnabled),
         'allow-plugins':
             Attribute(QWebSettings.PluginsEnabled),
         'webgl':
             Attribute(QWebSettings.WebGLEnabled),
-        'css-regions':
-            Attribute(QWebSettings.CSSRegionsEnabled),
         'hyperlink-auditing':
             Attribute(QWebSettings.HyperlinkAuditingEnabled),
         'local-content-can-access-remote-urls':
@@ -175,44 +188,28 @@ MAPPINGS = {
     },
     'fonts': {
         'web-family-standard':
-            Setter(getter=QWebSettings.fontFamily,
-                   setter=QWebSettings.setFontFamily,
-                   args=[QWebSettings.StandardFont]),
+            FontFamilySetter(QWebSettings.StandardFont),
         'web-family-fixed':
-            Setter(getter=QWebSettings.fontFamily,
-                   setter=QWebSettings.setFontFamily,
-                   args=[QWebSettings.FixedFont]),
+            FontFamilySetter(QWebSettings.FixedFont),
         'web-family-serif':
-            Setter(getter=QWebSettings.fontFamily,
-                   setter=QWebSettings.setFontFamily,
-                   args=[QWebSettings.SerifFont]),
+            FontFamilySetter(QWebSettings.SerifFont),
         'web-family-sans-serif':
-            Setter(getter=QWebSettings.fontFamily,
-                   setter=QWebSettings.setFontFamily,
-                   args=[QWebSettings.SansSerifFont]),
+            FontFamilySetter(QWebSettings.SansSerifFont),
         'web-family-cursive':
-            Setter(getter=QWebSettings.fontFamily,
-                   setter=QWebSettings.setFontFamily,
-                   args=[QWebSettings.CursiveFont]),
+            FontFamilySetter(QWebSettings.CursiveFont),
         'web-family-fantasy':
-            Setter(getter=QWebSettings.fontFamily,
-                   setter=QWebSettings.setFontFamily,
-                   args=[QWebSettings.FantasyFont]),
+            FontFamilySetter(QWebSettings.FantasyFont),
         'web-size-minimum':
-            Setter(getter=QWebSettings.fontSize,
-                   setter=QWebSettings.setFontSize,
+            Setter(QWebSettings.setFontSize,
                    args=[QWebSettings.MinimumFontSize]),
         'web-size-minimum-logical':
-            Setter(getter=QWebSettings.fontSize,
-                   setter=QWebSettings.setFontSize,
+            Setter(QWebSettings.setFontSize,
                    args=[QWebSettings.MinimumLogicalFontSize]),
         'web-size-default':
-            Setter(getter=QWebSettings.fontSize,
-                   setter=QWebSettings.setFontSize,
+            Setter(QWebSettings.setFontSize,
                    args=[QWebSettings.DefaultFontSize]),
         'web-size-default-fixed':
-            Setter(getter=QWebSettings.fontSize,
-                   setter=QWebSettings.setFontSize,
+            Setter(QWebSettings.setFontSize,
                    args=[QWebSettings.DefaultFixedFontSize]),
     },
     'ui': {
@@ -221,9 +218,6 @@ MAPPINGS = {
         'frame-flattening':
             Attribute(QWebSettings.FrameFlatteningEnabled),
         # user-stylesheet is handled separately
-        'css-media-type':
-            NullStringSetter(getter=QWebSettings.cssMediaType,
-                             setter=QWebSettings.setCSSMediaType),
         'smooth-scrolling':
             Attribute(QWebSettings.ScrollAnimatorEnabled),
         #'accelerated-compositing':
@@ -232,40 +226,22 @@ MAPPINGS = {
         #   Attribute(QWebSettings.TiledBackingStoreEnabled),
     },
     'storage': {
-        'offline-storage-database':
-            Attribute(QWebSettings.OfflineStorageDatabaseEnabled),
-        'offline-web-application-storage':
+        'offline-web-application-cache':
             Attribute(QWebSettings.OfflineWebApplicationCacheEnabled),
         'local-storage':
-            Attribute(QWebSettings.LocalStorageEnabled),
+            Attribute(QWebSettings.LocalStorageEnabled,
+                      QWebSettings.OfflineStorageDatabaseEnabled),
         'maximum-pages-in-cache':
-            StaticSetter(getter=QWebSettings.maximumPagesInCache,
-                         setter=QWebSettings.setMaximumPagesInCache),
-        'object-cache-capacities':
-            StaticSetter(getter=None,
-                         setter=QWebSettings.setObjectCacheCapacities,
-                         unpack=True),
-        'offline-storage-default-quota':
-            StaticSetter(getter=QWebSettings.offlineStorageDefaultQuota,
-                         setter=QWebSettings.setOfflineStorageDefaultQuota),
-        'offline-web-application-cache-quota':
-            StaticSetter(
-                getter=QWebSettings.offlineWebApplicationCacheQuota,
-                setter=QWebSettings.setOfflineWebApplicationCacheQuota),
+            StaticSetter(QWebSettings.setMaximumPagesInCache),
     },
     'general': {
-        'private-browsing':
-            Attribute(QWebSettings.PrivateBrowsingEnabled),
         'developer-extras':
             Attribute(QWebSettings.DeveloperExtrasEnabled),
         'print-element-backgrounds':
             Attribute(QWebSettings.PrintElementBackgrounds),
         'xss-auditing':
             Attribute(QWebSettings.XSSAuditingEnabled),
-        'site-specific-quirks':
-            Attribute(QWebSettings.SiteSpecificQuirksEnabled),
         'default-encoding':
-            Setter(getter=QWebSettings.defaultTextEncoding,
-                   setter=QWebSettings.setDefaultTextEncoding),
+            Setter(QWebSettings.setDefaultTextEncoding),
     }
 }

qutebrowser/browser/webkit/webkittab.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016-2017 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #
@@ -23,6 +23,7 @@ import sys
 import functools
 import xml.etree.ElementTree
 
+import sip
 from PyQt5.QtCore import (pyqtSlot, Qt, QEvent, QUrl, QPoint, QTimer, QSizeF,
                           QSize)
 from PyQt5.QtGui import QKeyEvent
@@ -32,21 +33,14 @@ from PyQt5.QtWebKit import QWebSettings
 from PyQt5.QtPrintSupport import QPrinter
 
 from qutebrowser.browser import browsertab
-from qutebrowser.browser.network import proxy
 from qutebrowser.browser.webkit import webview, tabhistory, webkitelem
 from qutebrowser.browser.webkit.network import webkitqutescheme
-from qutebrowser.utils import qtutils, objreg, usertypes, utils, log
+from qutebrowser.utils import qtutils, objreg, usertypes, utils, log, debug
 
 
 def init():
     """Initialize QtWebKit-specific modules."""
     qapp = QApplication.instance()
-
-    if not qtutils.version_check('5.8'):
-        # Otherwise we initialize it globally in app.py
-        log.init.debug("Initializing proxy...")
-        proxy.init()
-
     log.init.debug("Initializing js-bridge...")
     js_bridge = webkitqutescheme.JSBridge(qapp)
     objreg.register('js-bridge', js_bridge)
@@ -56,6 +50,9 @@ class WebKitAction(browsertab.AbstractAction):
 
     """QtWebKit implementations related to web actions."""
 
+    action_class = QWebPage
+    action_base = QWebPage.WebAction
+
     def exit_fullscreen(self):
         raise browsertab.UnsupportedOperationError
 
@@ -103,7 +100,7 @@ class WebKitSearch(browsertab.AbstractSearch):
         super().__init__(parent)
         self._flags = QWebPage.FindFlags(0)
 
-    def _call_cb(self, callback, found):
+    def _call_cb(self, callback, found, text, flags, caller):
         """Call the given callback if it's non-None.
 
         Delays the call via a QTimer so the website is re-rendered in between.
@@ -111,17 +108,34 @@ class WebKitSearch(browsertab.AbstractSearch):
         Args:
             callback: What to call
             found: If the text was found
+            text: The text searched for
+            flags: The flags searched with
+            caller: Name of the caller.
         """
+        found_text = 'found' if found else "didn't find"
+        # Removing FindWrapsAroundDocument to get the same logging as with
+        # QtWebEngine
+        debug_flags = debug.qflags_key(
+            QWebPage, flags & ~QWebPage.FindWrapsAroundDocument,
+            klass=QWebPage.FindFlag)
+        if debug_flags != '0x0000':
+            flag_text = 'with flags {}'.format(debug_flags)
+        else:
+            flag_text = ''
+        log.webview.debug(' '.join([caller, found_text, text, flag_text])
+                          .strip())
         if callback is not None:
             QTimer.singleShot(0, functools.partial(callback, found))
 
     def clear(self):
+        self.search_displayed = False
         # We first clear the marked text, then the highlights
         self._widget.findText('')
         self._widget.findText('', QWebPage.HighlightAllOccurrences)
 
     def search(self, text, *, ignore_case=False, reverse=False,
                result_cb=None):
+        self.search_displayed = True
         flags = QWebPage.FindWrapsAroundDocument
         if ignore_case == 'smart':
             if not text.islower():
@@ -136,13 +150,15 @@ class WebKitSearch(browsertab.AbstractSearch):
         self._widget.findText(text, flags | QWebPage.HighlightAllOccurrences)
         self.text = text
         self._flags = flags
-        self._call_cb(result_cb, found)
+        self._call_cb(result_cb, found, text, flags, 'search')
 
     def next_result(self, *, result_cb=None):
+        self.search_displayed = True
         found = self._widget.findText(self.text, self._flags)
-        self._call_cb(result_cb, found)
+        self._call_cb(result_cb, found, self.text, self._flags, 'next_result')
 
     def prev_result(self, *, result_cb=None):
+        self.search_displayed = True
         # The int() here makes sure we get a copy of the flags.
         flags = QWebPage.FindFlags(int(self._flags))
         if flags & QWebPage.FindBackward:
@@ -150,7 +166,7 @@ class WebKitSearch(browsertab.AbstractSearch):
         else:
             flags |= QWebPage.FindBackward
         found = self._widget.findText(self.text, flags)
-        self._call_cb(result_cb, found)
+        self._call_cb(result_cb, found, self.text, flags, 'prev_result')
 
 
 class WebKitCaret(browsertab.AbstractCaret):
@@ -444,15 +460,11 @@ class WebKitScroller(browsertab.AbstractScroller):
         # self._widget.setFocus()
 
         for _ in range(min(count, 5000)):
-            press_evt = QKeyEvent(QEvent.KeyPress, key, Qt.NoModifier, 0, 0, 0)
-            release_evt = QKeyEvent(QEvent.KeyRelease, key, Qt.NoModifier,
-                                    0, 0, 0)
             # Abort scrolling if the minimum/maximum was reached.
             if (getter is not None and
                     frame.scrollBarValue(direction) == getter(direction)):
                 return
-            self._widget.keyPressEvent(press_evt)
-            self._widget.keyReleaseEvent(release_evt)
+            self._tab.key_press(key)
 
     def up(self, count=1):
         self._key_press(Qt.Key_Up, count, 'scrollBarMinimum', Qt.Vertical)
@@ -610,10 +622,13 @@ class WebKitTab(browsertab.AbstractTab):
 
     """A QtWebKit tab in the browser."""
 
-    def __init__(self, win_id, mode_manager, parent=None):
+    def __init__(self, *, win_id, mode_manager, private, parent=None):
         super().__init__(win_id=win_id, mode_manager=mode_manager,
-                         parent=parent)
-        widget = webview.WebView(win_id, self.tab_id, tab=self)
+                         private=private, parent=parent)
+        widget = webview.WebView(win_id=win_id, tab_id=self.tab_id,
+                                 private=private, tab=self)
+        if private:
+            self._make_private(widget)
         self.history = WebKitHistory(self)
         self.scroller = WebKitScroller(self, parent=self)
         self.caret = WebKitCaret(win_id=win_id, mode_manager=mode_manager,
@@ -630,6 +645,10 @@ class WebKitTab(browsertab.AbstractTab):
     def _install_event_filter(self):
         self._widget.installEventFilter(self._mouse_event_filter)
 
+    def _make_private(self, widget):
+        settings = widget.settings()
+        settings.setAttribute(QWebSettings.PrivateBrowsingEnabled, True)
+
     def openurl(self, url):
         self._openurl_prepare(url)
         self._widget.openurl(url)
@@ -678,13 +697,20 @@ class WebKitTab(browsertab.AbstractTab):
     def clear_ssl_errors(self):
         self.networkaccessmanager().clear_all_ssl_errors()
 
+    def key_press(self, key, modifier=Qt.NoModifier):
+        press_evt = QKeyEvent(QEvent.KeyPress, key, modifier, 0, 0, 0)
+        release_evt = QKeyEvent(QEvent.KeyRelease, key, modifier,
+                                0, 0, 0)
+        self.send_event(press_evt)
+        self.send_event(release_evt)
+
     @pyqtSlot()
     def _on_history_trigger(self):
         url = self.url()
         requested_url = self.url(requested=True)
         self.add_history_item.emit(url, requested_url, self.title())
 
-    def set_html(self, html, base_url):
+    def set_html(self, html, base_url=QUrl()):
         self._widget.setHtml(html, base_url)
 
     def networkaccessmanager(self):
@@ -707,6 +733,9 @@ class WebKitTab(browsertab.AbstractTab):
     @pyqtSlot()
     def _on_webkit_icon_changed(self):
         """Emit iconChanged with a QIcon like QWebEngineView does."""
+        if sip.isdeleted(self._widget):
+            log.webview.debug("Got _on_webkit_icon_changed for deleted view!")
+            return
         self.icon_changed.emit(self._widget.icon())
 
     @pyqtSlot(QWebFrame)