Release v0.8.4

They renamed PDFView to PDFViewerApplication, which we need to account
for in our pdfjs scripts.

Also, it seems like the actual viewer is now only created when the DOM
has been loaded. This means that at the time when our script is
executed, the viewer does not yet exist. Thus we need to delay the open
request too by registering a DOMContentLoaded handler.

.appveyor.yml

@@ -11,7 +11,7 @@ environment:
     - TESTENV: pylint
 
 install:
-  - C:\Python27\python -u scripts\dev\ci\install.py
+  - C:\Python27\python -u scripts\dev\ci\appveyor_install.py
 
 test_script:
   - C:\Python34\Scripts\tox -e %TESTENV%

.editorconfig

@@ -0,0 +1,17 @@
+[*]
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+charset = utf-8
+
+max_line_length = 79
+
+indent_style = space
+indent_size = 4
+
+[*.yml]
+indent_size = 2
+
+[*.feature]
+max_line_length = 9999
+

.flake8

@@ -1,5 +1,5 @@
 [flake8]
-exclude = .venv,.hypothesis,.git,__pycache__,resources.py
+exclude = .*,__pycache__,resources.py
 # E128: continuation line under-indented for visual indent
 # E226: missing whitespace around arithmetic operator
 # E265: Block comment should start with '#'
@@ -8,17 +8,6 @@ exclude = .venv,.hypothesis,.git,__pycache__,resources.py
 # E266: too many leading '#' for block comment
 # F401: Unused import
 # N802: function name should be lowercase
-# L101: The __init__ method of classes must not have a docstring
-# L102: A docstring was incorrectly formatted.
-# L103: A test docstring must not start with any form of the words "test", ...
-# L201: Container literals must have a trailing comma
-# L202: print is not allowed except for debugging.
-# L203: pdb and compatible modules are not allowed except for debugging.
-# L204: Implicit string literal concatenation is only allowed if every string
-#       being concatenated is parenthesize
-# L207: pass is only necessary in non-optional suites containing no other
-#       statements.
-# L302: The line was too long.
 # P101: format string does contain unindexed parameters
 # P102: docstring does contain unindexed parameters
 # P103: other string does contain unindexed parameters
@@ -30,27 +19,30 @@ exclude = .venv,.hypothesis,.git,__pycache__,resources.py
 # D211: No blank lines allowed before class docstring
 #       (PEP257 got changed, but let's stick to the old standard)
 # D402: First line should not be function's signature (false-positives)
+# D403: First word of the first line should be properly capitalized
+#       (false-positives)
+# H101: Use TODO(NAME)
+# H201: bare except
+# H238: Use new-stule classes
+# H301: one import per line
+# H306: imports not in alphabetical order
 ignore =
     E128,E226,E265,E501,E402,E266,
     F401,
     N802,
-    L101,L102,L103,L201,L202,L203,L204,L207,L302,
     P101,P102,P103,
-    D102,D103,D104,D105,D209,D211,D402
+    D102,D103,D104,D105,D209,D211,D402,D403,
+    H101,H201,H238,H301,H306
+min-version = 3.4.0
 max-complexity = 12
+putty-auto-ignore = True
 putty-ignore =
     /# pylint: disable=invalid-name/ : +N801,N806
     /# pylint: disable=wildcard-import/ : +F403
     /# pragma: no mccabe/ : +C901
-    /# flake8: disable=E131/ : +E131
-    /# flake8: disable=N803/ : +N803
-    /# flake8: disable=T002/ : +T002
-    /# flake8: disable=F841/ : +F841
-    /# flake8: disable=S001/ : +S001
-    tests/*/*/test_*.py : +D100,D101,D401
     tests/*/test_*.py : +D100,D101,D401
-    tests/unit/browser/http/test_content_disposition.py : +D400
-    scripts/dev/ci/install.py : +C901
+    tests/unit/browser/webkit/http/test_content_disposition.py : +D400
+    scripts/dev/ci/appveyor_install.py : +FI53
 copyright-check = True
 copyright-regexp = # Copyright [\d-]+ .*
 copyright-min-file-size = 110

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,4 @@
+Please remember to mention your version info (qutebrowser, Qt, PyQt,
+OS/distribution) from the `qute:version` page or `qutebrowser --version`
+
+---

.gitignore

@@ -1,4 +1,5 @@
 __pycache__
+*.py~
 *.pyc
 *.swp
 /build
@@ -30,4 +31,9 @@ __pycache__
 /.testmondata
 /.hypothesis
 /prof
+/venv
 TODO
+/scripts/testbrowser_cpp/Makefile
+/scripts/testbrowser_cpp/main.o
+/scripts/testbrowser_cpp/testbrowser
+/scripts/dev/pylint_checkers/qute_pylint.egg-info

.pylintrc

@@ -6,7 +6,10 @@ extension-pkg-whitelist=PyQt5,sip
 load-plugins=qute_pylint.config,
              qute_pylint.modeline,
              qute_pylint.openencoding,
-             qute_pylint.settrace
+             qute_pylint.settrace,
+             pylint.extensions.bad_builtin,
+             pylint.extensions.docstyle
+persistent=n
 
 [MESSAGES CONTROL]
 enable=all
@@ -14,6 +17,7 @@ disable=no-self-use,
         fixme,
         global-statement,
         locally-disabled,
+        locally-enabled,
         too-many-ancestors,
         too-few-public-methods,
         too-many-public-methods,
@@ -31,12 +35,15 @@ disable=no-self-use,
         wrong-import-order,
         ungrouped-imports,
         redefined-variable-type,
-        suppressed-message
+        suppressed-message,
+        too-many-return-statements,
+        duplicate-code,
+        wrong-import-position
 
 [BASIC]
 function-rgx=[a-z_][a-z0-9_]{2,50}$
 const-rgx=[A-Za-z_][A-Za-z0-9_]{0,30}$
-method-rgx=[a-z_][A-Za-z0-9_]{2,50}$
+method-rgx=[a-z_][A-Za-z0-9_]{1,50}$
 attr-rgx=[a-z_][a-z0-9_]{0,30}$
 argument-rgx=[a-z_][a-z0-9_]{0,30}$
 variable-rgx=[a-z_][a-z0-9_]{0,30}$
@@ -57,10 +64,13 @@ dummy-variables-rgx=_.*
 [DESIGN]
 max-args=10
 
+[CLASSES]
+valid-metaclass-classmethod-first-arg=cls
+
 [TYPECHECK]
 # MsgType added as WORKAROUND for
 # https://bitbucket.org/logilab/pylint/issues/690/
 # UnsetObject because pylint infers any objreg.get(...) as UnsetObject.
 ignored-classes=qutebrowser.utils.objreg.UnsetObject,
-                qutebrowser.browser.webelem.WebElementWrapper,
+                qutebrowser.browser.webkit.webelem.WebElementWrapper,
                 scripts.dev.check_coverage.MsgType

.travis.yml

@@ -1,17 +1,11 @@
 sudo: required
 dist: trusty
-env:
-  global:
-    - PATH=/home/travis/bin:/home/travis/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-# Not really, but this is here so we can do stuff by hand.
-language: c
+language: generic
 
 matrix:
   include:
     - os: linux
       env: TESTENV=py34-cov
-    - os: linux
-      env: TESTENV=unittests-nodisp
     - os: linux
       env: DOCKER=debian-jessie
       services: docker
@@ -19,7 +13,7 @@ matrix:
       env: DOCKER=archlinux
       services: docker
     - os: linux
-      env: DOCKER=ubuntu-wily
+      env: DOCKER=ubuntu-xenial
       services: docker
     - os: osx
       env: TESTENV=py35
@@ -48,8 +42,12 @@ cache:
     - $HOME/.cache/pip
     - $HOME/build/The-Compiler/qutebrowser/.cache
 
+before_install:
+  # We need to do this so we pick up the system-wide python properly
+  - 'export PATH="/usr/bin:$PATH"'
+
 install:
-  - python scripts/dev/ci/install.py
+  - bash scripts/dev/ci/travis_install.sh
 
 script:
   - bash scripts/dev/ci/travis_run.sh
@@ -62,8 +60,8 @@ notifications:
     - https://buildtimetrend.herokuapp.com/travis
   irc:
     channels:
-      - "chat.freenode.net#qutebrowser"
-    on_success: change
+      - "chat.freenode.net#qutebrowser-dev"
+    on_success: always
     on_failure: always
     skip_join: true
     template:

CHANGELOG.asciidoc

@@ -14,6 +14,228 @@ This project adheres to http://semver.org/[Semantic Versioning].
 // `Fixed` for any bug fixes.
 // `Security` to invite users to upgrade in case of vulnerabilities.
 
+v0.8.3
+------
+
+Fixed
+~~~~~
+
+- Fixed crash when doing `:<space><enter>`, another corner-case introduced in v0.8.0
+- Fixed `:open-editor` (`<Ctrl-e>`) on Windows
+- Fixed crash when setting `general -> auto-save-interval` to a too big value.
+- Fixed crash when using hints on Void Linux.
+- Fixed compatibility with Python 3.5.2+ on Debian unstable
+- Compatibility with pdfjs v1.6.210
+- `:bind` can now be used to bind to an alias (binding by editing `keys.conf`
+  already worked before)
+- The command completion now updates correctly when changing aliases
+- The tabbar now displays correctly with the Adwaita Qt theme
+- The default `sk` keybinding now sets the commandline to `:bind` correctly
+- Fixed crash when closing a window without focusing it
+
+v0.8.2
+------
+
+Fixed
+~~~~~
+
+- Fixed `general -> private-browsing` not being set correctly until a restart
+  (which caused e.g. local storage to be enabled).
+- When hinting input fields (`:t`), also consider input elements without a type.
+- Fixed crash when opening an invalid URL with a percent-encoded and a real @ in it
+- Fixed default `;o` and `;O` bindings
+- Fixed local storage not working (and possible other bugs) when using a
+  relative path with `--basedir`.
+- Fixed crash when deleting a quickmark with Ctrl-D
+- Fixed HTML5 video playback on Windows
+- Fixed crash when using `:prompt-open-download` with a file with chars not
+  encodable with the OS' filesystem encoding (e.g. with `LC_ALL=C`)
+- Fixed `:prompt-open-download` with a too long filename (> 255 bytes)
+- Fixed crash when cancelling a download after doing `:prompt-open-download`
+- Fixed crash when writing a download to disk fails with
+  `:prompt-open-download`.
+- Fixed `:restart` deleting the basedir when it was given with `--basedir`.
+
+v0.8.1
+------
+
+Fixed
+~~~~~
+
+- Fix crash when pressing enter without a command
+- Adjust error message to point out QtWebEngine is unsupported with the OS
+  X .app currently.
+- Hide Harfbuzz warning with the OS X .app
+
+v0.8.0
+------
+
+Added
+~~~~~
+
+- New `:repeat-command` command (mapped to `.`) to repeat the last command.
+  Note that two former default bundings conflict with that binding, unbinding
+  them via `:unbind .i` and `:unbind .o` is recommended.
+- New `qute:bookmarks` page which displays all bookmarks and quickmarks.
+- New `:prompt-open-download` (bound to `Ctrl-X`) which can be used to open a
+  download directly when getting the filename prompt.
+- New `{host}` replacement for tab- and window titles which evaluates
+  to the current host.
+- New default binding `;t` for `:hint input`.
+- New variables `$QUTE_CONFIG_DIR`, `$QUTE_DATA_DIR` and
+  `$QUTE_DOWNLOAD_DIR` available for userscripts.
+- New option `ui` -> `status-position` to configure the position of the
+  status bar (top/bottom).
+- New `--pdf <filename>` argument for `:print` WHICH can be used to generate a
+  PDF without a dialog.
+
+Changed
+~~~~~~~
+
+- `:scroll-perc` now prefers a count over the argument given to it, which means
+  `gg` can be used with a count.
+- Aliases can now use `;;` to have an alias which executed multiple commands.
+- `:edit-url` now does nothing if the URL isn't changed in the spawned editor.
+- `:bookmark-add` can now be passed a URL and title to add that as a bookmark
+  rather than the current page.
+- New `taskadd` userscript to add a taskwarrior task annotated with the
+  current URL.
+- `:bookmark-del` and `:quickmark-del` now delete the current page's URL if none
+  is given.
+
+Fixed
+-----
+
+- Compatibility with PyQt 5.7
+- Fixed some configuration values being lost when a config option gets removed
+  from qutebrowser's code.
+- Fix crash when downloading with a full disk
+- Using `:jump-mark` (e.g. `''`) when the current URL is invalid doesn't crash
+  anymore.
+
+Removed
+-------
+
+- The ability to display status messages from webpages, as well as the related
+  `ui ->  display-statusbar-messages` setting.
+- The `general -> wrap-search` setting as searches now always wrap.
+  According to a quick straw poll and prior crash logs, almost nobody is using
+  `wrap-search = false`, and turning off wrapping is not possible with
+  QtWebEngine.
+- `:edit-url` now doesn't accept a count anymore as its behavior was confusing
+  and it doesn't make much sense to add a count.
+
+v0.7.0
+------
+
+Added
+~~~~~
+
+- New `:edit-url` command to edit the URL in an external editor.
+- New `network -> custom-headers` setting to send custom headers with every request.
+- New `{url:pretty}` commandline replacement which gets replaced by the decoded URL.
+- New marks to remember a scroll position:
+    - New `:jump-mark` command to jump to a mark, bound to `'`
+    - New `:set-mark` command to set a mark, bound to ```(backtick)
+    - The `'` mark gets set when moving away (hinting link with anchor, searching, etc.) so you can move back with `''`
+- New `--force-color` argument to force colored logging even if stdout is not a
+  terminal
+- New `:messages` command to show error messages
+- New pop-up showing possible keybinding when the first key of a keychain is
+  pressed. This can be turned off using `:set ui keyhint-blacklist *`.
+- New `hints -> auto-follow-timeout` setting to ignore keypresses after
+  following a hint when filtering in number mode.
+- New `:history-clear` command to clear the entire history
+- New `hints -> find-implementation` to select which implementation (JS/Python)
+  should be used to find hints on a page. The `javascript` implementation is
+  better, but slower.
+- New `inputs` group for `:hint` to hint text input fields.
+
+Changed
+~~~~~~~
+
+- qutebrowser got a new (slightly updated) logo
+- `:tab-focus` can now take a negative index to focus the nth tab counted from
+  the right.
+- `:yank` can now yank the pretty/decoded URL by adding `--pretty`
+- `:navigate` now clears the URL fragment
+- `:completion-item-del` (`Ctrl-D`) can now be used in `:buffer` completion to
+  close a tab
+- Various SSL ciphers are now disabled by default. With recent Qt/OpenSSL
+  versions those already all are disabled, but with older versions they might
+  not be.
+- Show favicons as window icon with `tabs-are-windows` set.
+- `:bind <key>` without a command now shows the existing binding
+- The optional `colorlog` dependency got removed, as qutebrowser now displays
+  colored logs without it.
+- URLs are now shown decoded when hovering.
+- Keybindings are now shown in the command completion
+- Improved behavior when pasting multiple lines
+- Rapid hints can now also be used for the `normal` hint target, which can be
+  useful with javascript click handlers or checkboxes which don't actually open
+  a new page.
+- `:zoom-in` or `:zoom-out` (`+`/`-`) with a too large count now zooms to the
+  smallest/largest zoom instead of doing nothing.
+- The commandline now accepts partially typed commands if they're unique.
+- Number hints are now kept filtered after following a hint in rapid mode.
+- Number hints are now renumbered after filtering
+- Number hints can now be filtered with multiple space-separated search terms
+- `hints -> scatter` is now ignored for number hints
+- Better history implementation which also stores titles.
+  As a consequence, URLs which redirect to another URL are now added to the
+  history too, marked with a `-r` suffix to the timestamp field.
+
+Fixed
+-----
+
+- Fixed using `:hint links spawn` with flags - you can now use things like the
+  `-v` argument for `:spawn` or pass flags to the spawned commands.
+- Various fixes for hinting corner-cases where following a link didn't work or
+  the hint was drawn at the wrong position.
+- Fixed crash when downloading from a URL with SSL errors
+- Close file handles correctly when a download failed
+- Fixed crash when using `;Y` (`:hint links yank-primary`) on a system without
+  primary selection
+- Don't display quit confirmation with finished downloads
+- Fixed updating the tab index in the statusbar when opening a background tab
+- Fixed a crash when entering `:-- ` in the commandline
+- Fixed `:debug-console` with PyQt 5.6
+- Fixed qutebrowser not starting when `sys.stderr` is `None`
+- Fixed crash when cancelling a download which belongs to an MHTML download
+- Fixed rebinding of keybindings being case-sensitive
+- Fix for tab indicators getting lost when moving tabs
+- Fixed handling of backspace in number hinting mode
+- Fixed `FileNotFoundError` when starting in some cases on old Qt versions
+- Fixed sharing of cookies between tabs when `private-browsing` is enabled
+- Toggling values with `:set` now uses lower-case values
+- Hints now work with (non-standard) links with spaces around the URL
+- Strip off trailing spaces for history entries with no title
+
+v0.6.2
+------
+
+Fixed
+~~~~~
+
+- Fixed crash when using `:tab-{prev,next,focus}` right after closing the last
+  tab with `last-close` set to `close`.
+- Fixed crash when doing `:undo` in a new instance with `tabs -> last-close` set
+  to `default-page`.
+- Fixed crash when starting with --cachedir=""
+- Fixed crash in some circumstances when using dictionary hints
+- Fixed various crashes related to PyQt 5.6
+
+v0.6.1
+-----
+
+Fixed
+~~~~~~
+
+- Fixed broken cheatsheet image which was missing from package
+- Fixed occasional crash when switching/disconnecting monitors
+- Fixed crash when downloading non-ascii files with a broken locale (`LC_ALL=C`)
+- Added workaround for a Qt/PyQt bug which is too weird to describe here
+
 v0.6.0
 ------
 
@@ -381,7 +603,7 @@ Fixed
 ~~~~~
 
 - Scrolling should now work more reliably on some pages where arrow keys worked but `hjkl` didn't.
-- Small improvements when checking if an input is an URL or not.
+- Small improvements when checking if an input is a URL or not.
 - Fixed wrong cursor position when completing the first item in the completion.
 - Fixed exception when using search engines with {foo} in their name.
 - Fixed a bug where the same title was shown for all tabs on some systems.
@@ -393,7 +615,7 @@ Fixed
 - Various fixes for deprecated key bindings and auto-migrations.
 - Workaround for qutebrowser not starting when there are NUL-bytes in the history (because of a currently unknown bug).
 - Fixed handling of keybindings containing Ctrl/Meta on OS X.
-- Fixed crash when downloading an URL without filename (e.g. magnet links) via "Save as...".
+- Fixed crash when downloading a URL without filename (e.g. magnet links) via "Save as...".
 - Fixed exception when starting qutebrowser with `:set` as argument.
 - Fixed horrible completion performance when the `shrink` option was set.
 - Sessions now store zoom/scroll-position correctly.
@@ -544,7 +766,7 @@ Fixed
 - Scroll completion to top when showing it.
 - Handle unencodable file paths in config types correctly.
 - Fix for crash when executing a delayed command (because of a shadowed keybinding) and then unfocusing the window.
-- Fix for crash when hinting on a page which doesn't have an URL yet.
+- Fix for crash when hinting on a page which doesn't have a URL yet.
 - Fix exception when using `:set-cmd-text` with an empty argument.
 - Add a timeout to pastebin HTTP replies.
 - Various other fixes for small/rare bugs.

CONTRIBUTING.asciidoc

@@ -158,11 +158,11 @@ Examples:
 # run only pytest tests which failed in last run:
 tox -e py35 -- --lf
 
-# run only the integration feature tests:
-tox -e py35 -- tests/integration/features
+# run only the end2end feature tests:
+tox -e py35 -- tests/end2end/features
 
 # run everything with undo in the generated name, based on the scenario text
-tox -e py35 -- tests/integration/features/test_tabs.py -k undo
+tox -e py35 -- tests/end2end/features/test_tabs_bdd.py -k undo
 
 # run coverage test for specific file (updates htmlcov/index.html)
 tox -e py35-cov -- tests/unit/browser/test_webelem.py
@@ -174,10 +174,14 @@ Profiling
 In the _scripts/_ subfolder there's a `run_profile.py` which profiles the code
 and shows a graphical representation of what takes how much time.
 
-It needs https://pypi.python.org/pypi/pyprof2calltree/[pyprof2calltree] and
-http://kcachegrind.sourceforge.net/html/Home.html[KCacheGrind]. It uses the
-built-in Python https://docs.python.org/3.4/library/profile.html[cProfile]
-module.
+It uses the built-in Python
+https://docs.python.org/3.4/library/profile.html[cProfile] module and can show
+the output in four different ways:
+
+* Raw profile file (`--profile-tool=none`)
+* https://pypi.python.org/pypi/pyprof2calltree/[pyprof2calltree] and http://kcachegrind.sourceforge.net/html/Home.html[KCacheGrind] (`--profile-tool=kcachegrind`)
+* https://jiffyclub.github.io/snakeviz/[SnakeViz] (`--profile-tool=snakeviz`)
+* https://github.com/jrfonseca/gprof2dot[gprof2dot] (needs `dot` from http://graphviz.org/[Graphviz] and http://feh.finalrewind.org/[feh])
 
 Debugging
 ~~~~~~~~~
@@ -212,7 +216,6 @@ Documentation of used Python libraries:
 * http://pythonhosted.org/setuptools/[setuptools]
 * http://cx-freeze.readthedocs.org/en/latest/overview.html[cx_Freeze]
 * https://pypi.python.org/pypi/colorama[colorama]
-* https://pypi.python.org/pypi/colorlog[colorlog]
 
 Related RFCs and standards:
 
@@ -371,7 +374,7 @@ The following logging levels are available for every logger:
 
 [width="75%",cols="25%,75%"]
 |=======================================================================
-|criticial |Critical issue, qutebrowser can't continue to run.
+|critical  |Critical issue, qutebrowser can't continue to run.
 |error     |There was an issue and some kind of operation was abandoned.
 |warning   |There was an issue but the operation can continue running.
 |info      |General informational messages.
@@ -417,35 +420,59 @@ The types of the function arguments are inferred based on their default values,
 e.g. an argument `foo=True` will be converted to a flag `-f`/`--foo` in
 qutebrowser's commandline.
 
-This behavior can be overridden using Python's
-http://legacy.python.org/dev/peps/pep-3107/[function annotations]. The
-annotation should always be a `dict`, like this:
+The type can be overridden using Python's
+http://legacy.python.org/dev/peps/pep-3107/[function annotations]:
 
 [source,python]
 ----
 @cmdutils.register(...)
-def foo(bar: {'type': int}, baz=True):
+def foo(bar: int, baz=True):
     ...
 ----
 
-The following keys are supported in the dict:
-
-* `type`: The type this value should have. The value entered by the user is
-then automatically checked. Possible values:
+Possible values:
     - A callable (`int`, `float`, etc.): Gets called to validate/convert the
       value.
-    - A string: The value must match exactly (mainly useful with tuples to get
-      a choice of values, see below).
     - A python enum type: All members of the enum are possible values.
-    - A tuple of multiple types above: Any of these types are valid values,
-      e.g. `('foo', 'bar')` or `(int, 'foo')`.
-* `flag`: The flag to be used, as 1-char string (default: First char of the
-long name).
-* `nargs`: Gets passed to argparse, see
-https://docs.python.org/dev/library/argparse.html#nargs[its documentation].
+    - A `typing.Union` of multiple types above: Any of these types are valid
+      values, e.g. `typing.Union[str, int]`
+
+You can customize how an argument is handled using the `@cmdutils.argument`
+decorator *after* `@cmdutils.register`. This can e.g. be used to customize the
+flag an argument should get:
+
+[source,python]
+----
+@cmdutils.register(...)
+@cmdutils.argument('bar', flag='c')
+def foo(bar):
+    ...
+----
+
+For a `str` argument, you can restrict the allowed strings using `choices`:
+
+[source,python]
+----
+@cmdutils.register(...)
+@cmdutils.argument('bar', choices=['val1', 'val2'])
+def foo(bar: str):
+    ...
+----
+
+For `typing.Union` types, the given `choices` are only checked if other types
+(like `int`) don't match.
+
+The following arguments are supported for `@cmdutils.argument`:
+
+- `flag`: Customize the short flag (`-x`) the argument will get.
+- `win_id=True`: Mark the argument as special window ID argument
+- `count=True`: Mark the argument as special count argument
+- `hide=True`: Hide the argument from the documentation
+- `completion`: A `usertypes.Completion` member to use as completion.
+- `choices`: The allowed string choices for the argument.
 
 The name of an argument will always be the parameter name, with any trailing
-underscores stripped.
+underscores stripped and underscores replaced by dashes.
 
 [[handling-urls]]
 Handling URLs
@@ -568,10 +595,7 @@ Return:
 * The layout of a class should be like this:
   - docstring
   - `__magic__` methods
-  - properties
-  - _private methods
-  - public methods
-  - `on_*` methods
+  - other methods
   - overrides of Qt methods
 
 Checklists
@@ -593,6 +617,14 @@ workaround.
 * Check relevant
 https://github.com/The-Compiler/qutebrowser/issues?q=is%3Aopen+is%3Aissue+label%3Aqt[qutebrowser
 bugs] and check if they're fixed.
+* As soon as Homebrew updated, update the custom OS X bottle:
+  - Update https://github.com/The-Compiler/homebrew-qt5-webkit/blob/master/Formula/qt5.rb[qt5.rb]
+  - `brew install --build-from-source --build-bottle --verbose qt5.rb`
+  - `brew bottle qt5.rb`
+  - `brew install --build-from-source --build-bottle --verbose pyqt5`
+  - `brew bottle pyqt5`
+  - Upload bottles to github
+  - Adjust `scripts/dev/ci/travis_install.sh`
 
 New PyQt release
 ~~~~~~~~~~~~~~~~
@@ -606,6 +638,11 @@ qutebrowser release
 
 * Make sure there are no unstaged changes and the tests are green.
 
+* Add newest config to `tests/unit/config/old_configs` and update `test_upgrade_version`
+    - `python -m qutebrowser --basedir conf :quit`
+    - `sed '/^#/d'  conf/config/qutebrowser.conf > tests/unit/config/old_configs/qutebrowser-v0.x.y.conf`
+    - `rm -r conf`
+    - commit
 * Adjust `__version_info__` in `qutebrowser/__init__.py`.
 * Remove *(unreleased)* from changelog.
 * Run tests again
@@ -613,7 +650,6 @@ qutebrowser release
 * Commit
 
 * Create annotated git tag (`git tag -s "v0.X.Y" -m "Release v0.X.Y"`)
-* If it's a new minor, create git branch `v0.X.x`
 * If committing on minor branch, cherry-pick release commit to master.
 * `git push origin`; `git push origin v0.X.Y`
 * Create release on github
@@ -622,12 +658,16 @@ as closed.
 
 * Run `scripts/dev/build_release.py` on Linux to build an sdist
 * Upload to PyPI: `twine upload dist/foo{,.asc}`
-* Create Windows packages via `scripts/dev/build_release.py` and upload.
+* Create Windows packages via `C:\Python34_x32\python scripts\dev\build_release.py --asciidoc C:\Python27\python C:\asciidoc-8.6.9\asciidoc.py`
+* Upload to github
 
 * Upload to qutebrowser.org with checksum/GPG
     - On server: `sudo mkdir -p /srv/http/qutebrowser/releases/v0.X.Y/windows`
     - `rsync -avPh dist/ tonks:`
     - On server: `sudo mv qutebrowser-0.X.Y.tar.gz* /srv/http/qutebrowser/releases/v0.X.Y`
-
-* Update AUR package
+    - Upload windows release:
+           - `scp bb-win8:proj/qutebrowser/qutebrowser-0.X.Y-windows.zip .`
+           - `aunpack qutebrowser-0.X.Y-windows.zip`
+           - `sudo mv qutebrowser-0.X.Y-windows/* /srv/http/qutebrowser/releases/v0.X.Y/windows`
+* Update `qutebrowser-git` PKGBUILD if dependencies/install changed
 * Announce to qutebrowser mailinglist

FAQ.asciidoc

@@ -80,13 +80,27 @@ How do I play Youtube videos with mpv?::
     player - optionally even with hinting for links:
 +
 ----
-:bind x spawn mpv {url}
-:bind ;x hint links spawn mpv {hint-url}
+:bind m spawn mpv {url}
+:bind M hint links spawn mpv {hint-url}
 ----
 +
 Note that you might need an additional package (e.g.
 https://www.archlinux.org/packages/community/any/youtube-dl/[youtube-dl] on
 Archlinux) to play web videos with mpv.
++
+There is a very useful script for mpv, which emulates "unique application"
+functionality. This way you can add links to the mpv playlist instead of
+playing them all at once.
++
+You can find the script here: https://github.com/mpv-player/mpv/blob/master/TOOLS/umpv
++
+It also works nicely with rapid hints:
++
+----
+:bind m spawn umpv {url}
+:bind M hint links spawn umpv {hint-url}
+:bind ;M hint --rapid links spawn umpv {hint-url}
+----
 
 How do I use qutebrowser with mutt?::
     Due to a Qt limitation, local files without `.html` extensions are

INSTALL.asciidoc

@@ -10,9 +10,31 @@ qutebrowser should run on these systems:
 * Ubuntu Trusty (14.04 LTS) or newer
 * Any other distribution based on these (e.g. Linux Mint 17+)
 
-Unfortunately there is no Debian package yet, but installing qutebrowser is
-still relatively easy! If you want to help packaging it for Debian, please
-https://github.com/The-Compiler/qutebrowser/issues/582[get in touch]!
+Unfortunately there is no Debian package in the official repos yet, but installing qutebrowser is
+still relatively easy!
+
+You can use packages that are built for every release or build it yourself from git.
+
+Using the packages
+~~~~~~~~~~~~~~~~~~
+
+Install the dependencies via apt-get:
+
+----
+# apt-get install python3-lxml python-tox python3-pyqt5 python3-pyqt5.qtwebkit python3-sip python3-jinja2 python3-pygments python3-yaml
+----
+
+Get the packages from the https://github.com/The-Compiler/qutebrowser/releases[release page]
+
+Install the packages:
+
+----
+# dpkg -i python3-pypeg2_*_all.deb
+# dpkg -i qutebrowser_*_all.deb
+----
+
+Build it from git
+~~~~~~~~~~~~~~~~~
 
 Install the dependencies via apt-get:
 
@@ -121,6 +143,13 @@ Make sure you have `python3_4` in your `PYTHON_TARGETS`
 (`/etc/portage/make.conf`) and rebuild your system (`emerge -uDNav @world`) if
 necessary.
 
+If video or sound don't seem to work, try installing the gstreamer plugins:
+
+----
+# emerge -av gst-plugins-{base,good,bad,ugly,libav}
+----
+
+
 On Void Linux
 -------------
 
@@ -200,21 +229,31 @@ Then <<tox,install qutebrowser via tox>>.
 On OS X
 -------
 
-*Using qutebrowser with Homebrew on OS X is currently broken, as Homebrew
-dropped QtWebKit support with Qt 5.6. I'm working on building a standalone
-`.app` for OS X instead, but it'll still take a few days until it's ready.*
+The easiest way to install qutebrowser on OS X is to use the prebuilt `.app`
+files from the
+https://github.com/The-Compiler/qutebrowser/releases[release page].
 
-To install qutebrowser on OS X, you'll want a package manager, e.g.
-http://brew.sh/[Homebrew] or https://www.macports.org/[MacPorts]. Also make
-sure, you have https://itunes.apple.com/en/app/xcode/id497799835[XCode]
-installed to compile PyQt5 in a later step.
+Alternatively, you can install the dependencies via a package manager (like
+http://brew.sh/[Homebrew] or https://www.macports.org/[MacPorts]) and run
+qutebrowser from source.
+
+For Homebrew, a few extra steps are necessary since Homebrew dropped QtWebKit
+from Qt 5.6 - however, some users reported this didn't work for them, so using
+the `.app` is strongly encouraged.
+
+This installs a Qt 5.5 and symlinks it so PyQt5 will work with it instead of Qt
+5.6. This requires that `qt5` is not installed via Homebrew:
 
 ----
-$ brew install python3 pyqt5
+$ brew install python3 d-bus mysql sip xz
+$ brew install homebrew/versions/qt55
+$ brew install --ignore-dependencies pyqt5
+$ ln -s /usr/local/opt/qt55 /usr/local/opt/qt5
+
 $ pip3.5 install qutebrowser
 ----
 
-if you are using Homebrew. For MacPorts, run:
+For MacPorts, run:
 
 ----
 $ sudo port install python34 py34-jinja2 asciidoc py34-pygments py34-pyqt5

MANIFEST.in

@@ -1,12 +1,13 @@
 recursive-include qutebrowser *.py
-recursive-include qutebrowser/html *.html
 recursive-include qutebrowser/img *.svg *.png
 recursive-include qutebrowser/test *.py
 recursive-include qutebrowser/javascript *.js
+graft qutebrowser/html
 graft qutebrowser/3rdparty
 graft icons
 graft doc/img
-graft misc
+graft misc/apparmor
+graft misc/userscripts
 recursive-include scripts *.py
 include qutebrowser/utils/testfile
 include qutebrowser/git-commit-id
@@ -15,15 +16,21 @@ include qutebrowser.desktop
 include requirements.txt
 include tox.ini
 include qutebrowser.py
+include misc/cheatsheet.svg
 
 prune www
 prune scripts/dev
+prune scripts/testbrowser_cpp
+prune .github
 exclude scripts/asciidoc2html.py
 exclude doc/notes
 recursive-exclude doc *.asciidoc
 include doc/qutebrowser.1.asciidoc
 prune tests
 prune qutebrowser/3rdparty
+prune misc/requirements
+prune misc/docker
+exclude .editorconfig
 exclude pytest.ini
 exclude qutebrowser.rcc
 exclude .coveragerc
@@ -33,7 +40,10 @@ exclude .eslintignore
 exclude doc/help
 exclude .appveyor.yml
 exclude .travis.yml
+exclude codecov.yml
 exclude .pydocstylerc
 exclude misc/appveyor_install.py
+exclude misc/qutebrowser.spec
+exclude .flake8
 
 global-exclude __pycache__ *.pyc *.pyo

README.asciidoc

@@ -15,6 +15,8 @@ image:https://requires.io/github/The-Compiler/qutebrowser/requirements.svg?branc
 image:https://travis-ci.org/The-Compiler/qutebrowser.svg?branch=master["Build Status", link="https://travis-ci.org/The-Compiler/qutebrowser"]
 image:https://ci.appveyor.com/api/projects/status/9gmnuip6i1oq7046?svg=true["AppVeyor build status", link="https://ci.appveyor.com/project/The-Compiler/qutebrowser"]
 image:https://codecov.io/github/The-Compiler/qutebrowser/coverage.svg?branch=master["coverage badge",link="https://codecov.io/github/The-Compiler/qutebrowser?branch=master"]
+
+link:http://www.qutebrowser.org[website] | link:http://blog.qutebrowser.org[blog] | link:https://github.com/The-Compiler/qutebrowser/releases[releases]
 // QUTE_WEB_HIDE_END
 
 qutebrowser is a keyboard-focused browser with a minimal GUI. It's based
@@ -22,12 +24,6 @@ on Python, PyQt5 and QtWebKit and free software, licensed under the GPL.
 
 It was inspired by other browsers/addons like dwb and Vimperator/Pentadactyl.
 
-// QUTE_WEB_HIDE
-**qutebrowser is currently running a crowdfunding campaign to add support for
-the QtWebEngine backend, which fixes many issues. Please
-link:http://igg.me/at/qutebrowser[check it out]!**
-// QUTE_WEB_HIDE_END
-
 Screenshots
 -----------
 
@@ -115,11 +111,8 @@ The following libraries are optional and provide a better user experience:
 To generate the documentation for the `:help` command, when using the git
 repository (rather than a release), http://asciidoc.org/[asciidoc] is needed.
 
-The following libraries are optional and provide colored logging in the
-console:
-
-* https://pypi.python.org/pypi/colorlog/[colorlog]
-* On Windows: https://pypi.python.org/pypi/colorama/[colorama]
+On Windows, https://pypi.python.org/pypi/colorama/[colorama] is needed to
+display colored log output.
 
 See link:INSTALL.asciidoc[INSTALL] for directions on how to install qutebrowser
 and its dependencies.
@@ -148,81 +141,104 @@ Contributors, sorted by the number of commits in descending order:
 // QUTE_AUTHORS_START
 * Florian Bruhin
 * Daniel Schadt
+* Ryan Roden-Corrent
 * Antoni Boucher
 * Lamar Pavel
 * Bruno Oliveira
 * Alexander Cogneau
-* Martin Tournoij
 * Felix Van der Jeugt
+* Martin Tournoij
+* Jakub Klinkovský
 * Raphael Pierzina
 * Joel Torstensson
+* Jan Verbeek
+* Tarcisio Fedrizzi
 * Patric Schmitz
 * Claude
+* Corentin Julé
 * meles5
-* Tarcisio Fedrizzi
+* Philipp Hansch
+* Panagiotis Ktistakis
 * Artur Shaik
 * Nathan Isom
 * Thorsten Wißmann
-* Philipp Hansch
 * Kevin Velghe
 * Austin Anderson
+* Jimmy
+* Marshall Lochbaum
 * Alexey "Averrin" Nabrodov
 * avk
 * ZDarian
 * Milan Svoboda
 * John ShaggyTwoDope Jenkins
-* Jimmy
 * Peter Vilim
 * Clayton Craft
 * Oliver Caldwell
 * Jonas Schürmann
-* Panagiotis Ktistakis
-* Jakub Klinkovský
-* skinnay
 * error800
+* Liam BEGUIN
+* skinnay
 * Zach-Button
+* Tomasz Kramkowski
+* Ismail S
 * Halfwit
+* David Vogt
 * rikn00
+* kanikaa1234
+* haitaka
+* Nick Ginther
+* Michał Góral
 * Michael Ilsaas
 * Martin Zimmermann
+* Fritz Reichwald
 * Brian Jackson
 * sbinix
 * neeasade
 * jnphilipp
 * Tobias Patzl
+* Stefan Tatschner
+* Samuel Loury
 * Peter Michely
+* Panashe M. Fundira
 * Link
 * Larry Hynes
 * Johannes Altmanninger
+* Jeremy Kaplan
+* Ismail
+* Edgar Hipp
+* Daryl Finlay
+* adam
 * Samir Benmendil
-* Ryan Roden-Corrent
 * Regina Hug
 * Mathias Fussenegger
 * Marcelo Santos
+* Jean-Louis Fuchs
 * Fritz V155 Reichwald
 * Franz Fellner
-* Corentin Jule
 * zwarag
 * xd1le
+* oniondreams
 * issue
 * haxwithaxe
 * evan
 * dylan araps
-* Tomasz Kramkowski
+* Xitian9
 * Tomas Orsava
 * Tobias Werth
 * Tim Harder
 * Thiago Barroso Perrotta
-* Stefan Tatschner
 * Sorokin Alexei
-* Samuel Loury
+* Noah Huesser
 * Matthias Lisin
 * Marcel Schilling
+* Johannes Martinsson
 * Jean-Christophe Petkovich
+* Jay Kamat
 * Helen Sherwood-Taylor
 * HalosGhost
 * Gregor Pohl
 * Eivind Uggedal
+* Dietrich Daroch
 * Daniel Lu
 * Arseniy Seroka
 * Andy Balaam
@@ -231,7 +247,8 @@ Contributors, sorted by the number of commits in descending order:
 
 The following people have contributed graphics:
 
-* WOFall (icon)
+* Jad/link:http://yelostudio.com[yelo] (new icon)
+* WOFall (original icon)
 * regines (key binding cheatsheet)
 
 Thanks / Similar projects
@@ -276,6 +293,7 @@ problems and helpful hints:
 
 Also, thanks to:
 
+* Everyone contributing to the link:doc/backers.asciidoc[crowdfunding].
 * Everyone who had the patience to test qutebrowser before v0.1.
 * Everyone triaging/fixing my bugs in the
 https://bugreports.qt.io/secure/Dashboard.jspa[Qt bugtracker]

codecov.yml

@@ -0,0 +1,9 @@
+status:
+  project:
+    enabled: no
+  patch:
+    enabled: no
+  changes:
+    enabled: no
+
+comment: off

doc/backers.asciidoc

@@ -0,0 +1,172 @@
+Crowdfunding backers
+====================
+
+Mid-2016, qutebrowser did run a http://igg.me/at/qutebrowser[crowdfunding] for
+QtWebEngine support in qutebrowser.
+
+Thanks a lot to the following people who contributed to it:
+
+Gold sponsors
+-------------
+
+- Chris Salzberg
+- Clayton Craft
+- Jean-Louis Fuchs
+- Matthias Lisin
+- 1 Anonymous
+
+Day sponsors
+------------
+
+- Agent 42
+- Iggy Jackson
+- James B
+- Rudi Seitz
+- Tim „Das MooL“ Wegener
+- amd1212
+- gavtroy
+- 4 Anonymous
+
+Other sponsors
+--------------
+
+- AP M
+- Alessandro Balzano
+- Allan Nordhøy
+- Andor Uhlar
+- Andreas Leppert
+- Andreas Saga Romsdal
+- Andrew Rogers / tuxlovesyou
+- André Glüpker
+- Arian Sanusi
+- Arin Lares
+- Assaf Lavie
+- Baptiste Wicht
+- Benjamin Richter
+- Benjamin Schnitzler
+- Bernardo Kuri
+- Boris Kourtoukov
+- Brian Buccola
+- Bruno Oliveira
+- Bryan Gilbert
+- Cassandra Rebecca Ruppen
+- Charles Saternos
+- Chris H
+- Christian Karl
+- Christian Lange
+- Christian Strasser
+- Colin O'Brien
+- Corsin Pfister
+- Cosmin Popescu
+- Daniel Andersson
+- David Wilson
+- Demure Demeanor
+- Doug Stone-Weaver
+- Eero Kari
+- Enric Morales
+- Eric Krohn
+- Eskild Hustvedt
+- Federico Panico
+- Felix Van der Jeugt
+- Francis Tseng
+- Geir Isene
+- George Voronin
+- German Correa
+- Grady Martin
+- Gregor Böhl
+- Guilherme Stein
+- Hannes Doyle
+- Hasan Soydabas
+- Ian Scott
+- Jacob Boldman
+- Jacob Wikmark
+- Jan Verbeek
+- Jarrod Seccombe
+- Joel Bradshaw
+- Johannes Martinsson
+- Jonas Schürmann
+- Josh Medeiros
+- José Alberto Orejuela García
+- Julie Engel
+- Jörg Behrmann
+- Jørgen Skancke
+- Kevin Velghe
+- Konstantin Shmelkov
+- Kyle Frazer
+- Lukas Gierth
+- Mar v Leeuwaarde
+- Marek Roszman
+- Marius Betz
+- Marius Krämer
+- Markus Schmidinger
+- Martin Gabelmann
+- Martin Zimmermann
+- Mathias Fußenegger
+- Maxime Wack
+- Michał Góral
+- Nathan Isom
+- Nathanael Philipp
+- Nils Stål
+- Oliver Hope
+- Oskar Nyberg
+- Pablo Navarro
+- Panashe M. Fundira
+- Patric Schmitz
+- Pete M
+- Peter Smith
+- Phil Collins
+- Philipp Hansch
+- Philipp Kuhnz
+- Raphael Khaiat
+- Raphael Pierzina
+- Renan Guilherme
+- Rick Losie
+- Robert Cross
+- Roy Van Ginneken
+- Rupus Reinefjord
+- Ryan Roden-Corrent
+- Samir Benmendil
+- Simon Giotta
+- Stephen England
+- Sverrir H Steindorsson
+- Tarcisio Fedrizzi
+- Thorsten Wißmann
+- Timon Stampfli
+- Tjelvar Olsson
+- Tomasz Kramkowski
+- Tsukiko Tsutsukakushi
+- Vasilij Schneidermann
+- Vinney Cavallo
+- Wesly Grefrath
+- Will Ware
+- Yousaf Khurshid
+- Zach Schultz
+- averrin
+- ben hengst
+- colin
+- craigtski47
+- dag.robole
+- daniel.m.kao
+- diepfann3
+- eamonn oneil
+- esakaforever
+- francois47
+- glspisso
+- gmccoy4242
+- gtcee3
+- jonathf
+- lapinski.maciej
+- lauri.hakko
+- ljanzen
+- mutilx9
+- nussgipfel
+- oed
+- p p
+- r.c.bruno.andre
+- robert.perce
+- sghctoma
+- targy
+- freelancer
+- pupu
+- regines
+- 37 Anonymous

doc/help/commands.asciidoc

@@ -8,7 +8,7 @@
 |<<adblock-update,adblock-update>>|Update the adblock block lists.
 |<<back,back>>|Go back in the history of the current tab.
 |<<bind,bind>>|Bind a key to a command.
-|<<bookmark-add,bookmark-add>>|Save the current page as a bookmark.
+|<<bookmark-add,bookmark-add>>|Save the current page as a bookmark, or a specific url.
 |<<bookmark-del,bookmark-del>>|Delete a bookmark.
 |<<bookmark-load,bookmark-load>>|Load a bookmark.
 |<<buffer,buffer>>|Select tab by index or url/title best match.
@@ -20,15 +20,19 @@
 |<<download-open,download-open>>|Open the last/[count]th download.
 |<<download-remove,download-remove>>|Remove the last/[count]th download from the list.
 |<<download-retry,download-retry>>|Retry the first failed/[count]th download.
+|<<edit-url,edit-url>>|Navigate to a url formed in an external editor.
 |<<fake-key,fake-key>>|Send a fake keypress or key string to the website or qutebrowser.
 |<<forward,forward>>|Go forward in the history of the current tab.
 |<<fullscreen,fullscreen>>|Toggle fullscreen mode.
 |<<help,help>>|Show help about a command or setting.
 |<<hint,hint>>|Start hinting.
+|<<history-clear,history-clear>>|Clear all browsing history.
 |<<home,home>>|Open main startpage in current tab.
 |<<inspector,inspector>>|Toggle the web inspector.
 |<<jseval,jseval>>|Evaluate a JavaScript string.
+|<<jump-mark,jump-mark>>|Jump to the mark named by `key`.
 |<<later,later>>|Execute a command after some time.
+|<<messages,messages>>|Show a log of past messages.
 |<<navigate,navigate>>|Open typical prev/next links or navigate using the URL path.
 |<<open,open>>|Open a URL in the current/[count]th tab.
 |<<paste,paste>>|Open a page from the clipboard.
@@ -49,6 +53,7 @@
 |<<session-save,session-save>>|Save a session.
 |<<set,set>>|Set an option.
 |<<set-cmd-text,set-cmd-text>>|Preset the statusbar to some text.
+|<<set-mark,set-mark>>|Set a mark at the current scroll position in the current tab.
 |<<spawn,spawn>>|Spawn a command in a shell.
 |<<stop,stop>>|Stop loading in the current/[count]th tab.
 |<<tab-clone,tab-clone>>|Duplicate the current tab.
@@ -91,13 +96,14 @@ How many pages to go back.
 
 [[bind]]
 === bind
-Syntax: +:bind [*--mode* 'MODE'] [*--force*] 'key' 'command'+
+Syntax: +:bind [*--mode* 'mode'] [*--force*] 'key' ['command']+
 
 Bind a key to a command.
 
 ==== positional arguments
 * +'key'+: The keychain or special key (inside `<...>`) to bind.
-* +'command'+: The command to execute, with optional args.
+* +'command'+: The command to execute, with optional args, or not given to print the current binding.
+
 
 ==== optional arguments
 * +*-m*+, +*--mode*+: A comma-separated list of modes to bind the key in (default: `normal`).
@@ -110,20 +116,28 @@ Bind a key to a command.
 
 [[bookmark-add]]
 === bookmark-add
-Save the current page as a bookmark.
+Syntax: +:bookmark-add ['url'] ['title']+
+
+Save the current page as a bookmark, or a specific url.
+
+If no url and title are provided, then save the current page as a bookmark. If a url and title have been provided, then save the given url as a bookmark with the provided title. You can view all saved bookmarks on the link:qute://bookmarks[bookmarks page].
+
+==== positional arguments
+* +'url'+: url to save as a bookmark. If None, use url of current page.
+* +'title'+: title of the new bookmark.
 
 [[bookmark-del]]
 === bookmark-del
-Syntax: +:bookmark-del 'url'+
+Syntax: +:bookmark-del ['url']+
 
 Delete a bookmark.
 
 ==== positional arguments
-* +'url'+: The URL of the bookmark to delete.
+* +'url'+: The url of the bookmark to delete. If not given, use the current page's url.
+
 
 ==== note
 * This command does not split arguments after the last argument and handles quotes literally.
-* With this command, +;;+ is interpreted literally instead of splitting off a second command.
 
 [[bookmark-load]]
 === bookmark-load
@@ -141,7 +155,6 @@ Load a bookmark.
 
 ==== note
 * This command does not split arguments after the last argument and handles quotes literally.
-* With this command, +;;+ is interpreted literally instead of splitting off a second command.
 
 [[buffer]]
 === buffer
@@ -161,7 +174,7 @@ Close the current window.
 
 [[download]]
 === download
-Syntax: +:download [*--mhtml*] [*--dest* 'DEST'] ['url'] ['dest-old']+
+Syntax: +:download [*--mhtml*] [*--dest* 'dest'] ['url'] ['dest-old']+
 
 Download a given URL, or current page if no URL given.
 
@@ -223,6 +236,22 @@ Retry the first failed/[count]th download.
 ==== count
 The index of the download to retry.
 
+[[edit-url]]
+=== edit-url
+Syntax: +:edit-url [*--bg*] [*--tab*] [*--window*] ['url']+
+
+Navigate to a url formed in an external editor.
+
+The editor which should be launched can be configured via the `general -> editor` config option.
+
+==== positional arguments
+* +'url'+: URL to edit; defaults to the current page url.
+
+==== optional arguments
+* +*-b*+, +*--bg*+: Open in a new background tab.
+* +*-t*+, +*--tab*+: Open in a new tab.
+* +*-w*+, +*--window*+: Open in a new window.
+
 [[fake-key]]
 === fake-key
 Syntax: +:fake-key [*--global*] 'keystring'+
@@ -285,6 +314,7 @@ Start hinting.
  - `all`: All clickable elements.
  - `links`: Only links.
  - `images`: Only images.
+ - `inputs`: Only input fields.
  
 
 
@@ -331,6 +361,15 @@ Start hinting.
  `window`, `run`, `hover`, `userscript` and `spawn`.
 
 
+==== note
+* This command does not split arguments after the last argument and handles quotes literally.
+
+[[history-clear]]
+=== history-clear
+Clear all browsing history.
+
+Note this only clears the global history (e.g. `~/.local/share/qutebrowser/history` on Linux) but not cookies, the back/forward history of a tab, cache or other persistent data.
+
 [[home]]
 === home
 Open main startpage in current tab.
@@ -357,6 +396,15 @@ Evaluate a JavaScript string.
 * This command does not split arguments after the last argument and handles quotes literally.
 * With this command, +;;+ is interpreted literally instead of splitting off a second command.
 
+[[jump-mark]]
+=== jump-mark
+Syntax: +:jump-mark 'key'+
+
+Jump to the mark named by `key`.
+
+==== positional arguments
+* +'key'+: mark identifier; capital indicates a global mark
+
 [[later]]
 === later
 Syntax: +:later 'ms' 'command'+
@@ -371,6 +419,22 @@ Execute a command after some time.
 * This command does not split arguments after the last argument and handles quotes literally.
 * With this command, +;;+ is interpreted literally instead of splitting off a second command.
 
+[[messages]]
+=== messages
+Syntax: +:messages [*--plain*] [*--tab*] [*--bg*] [*--window*] ['level']+
+
+Show a log of past messages.
+
+==== positional arguments
+* +'level'+: Include messages with `level` or higher severity. Valid values: vdebug, debug, info, warning, error, critical.
+
+
+==== optional arguments
+* +*-p*+, +*--plain*+: Whether to show plaintext (as opposed to html).
+* +*-t*+, +*--tab*+: Open in a new tab.
+* +*-b*+, +*--bg*+: Open in a background tab.
+* +*-w*+, +*--window*+: Open in a new window.
+
 [[navigate]]
 === navigate
 Syntax: +:navigate [*--tab*] [*--bg*] [*--window*] 'where'+
@@ -415,7 +479,6 @@ The tab index to open the URL in.
 
 ==== note
 * This command does not split arguments after the last argument and handles quotes literally.
-* With this command, +;;+ is interpreted literally instead of splitting off a second command.
 
 [[paste]]
 === paste
@@ -433,12 +496,13 @@ If the pasted text contains newlines, each line gets opened in its own tab.
 
 [[print]]
 === print
-Syntax: +:print [*--preview*]+
+Syntax: +:print [*--preview*] [*--pdf* 'file']+
 
 Print the current/[count]th tab.
 
 ==== optional arguments
 * +*-p*+, +*--preview*+: Show preview instead of printing.
+* +*-f*+, +*--pdf*+: The file path to write the PDF to.
 
 ==== count
 The tab index to print.
@@ -449,22 +513,25 @@ Syntax: +:quickmark-add 'url' 'name'+
 
 Add a new quickmark.
 
+You can view all saved quickmarks on the link:qute://bookmarks[bookmarks page].
+
 ==== positional arguments
 * +'url'+: The url to add as quickmark.
 * +'name'+: The name for the new quickmark.
 
 [[quickmark-del]]
 === quickmark-del
-Syntax: +:quickmark-del 'name'+
+Syntax: +:quickmark-del ['name']+
 
 Delete a quickmark.
 
 ==== positional arguments
-* +'name'+: The name of the quickmark to delete.
+* +'name'+: The name of the quickmark to delete. If not given, delete the quickmark for the current page (choosing one arbitrarily
+ if there are more than one).
+
 
 ==== note
 * This command does not split arguments after the last argument and handles quotes literally.
-* With this command, +;;+ is interpreted literally instead of splitting off a second command.
 
 [[quickmark-load]]
 === quickmark-load
@@ -482,7 +549,6 @@ Load a quickmark.
 
 ==== note
 * This command does not split arguments after the last argument and handles quotes literally.
-* With this command, +;;+ is interpreted literally instead of splitting off a second command.
 
 [[quickmark-save]]
 === quickmark-save
@@ -550,7 +616,6 @@ Search for a text on the current page. With no text, clear results.
 
 ==== note
 * This command does not split arguments after the last argument and handles quotes literally.
-* With this command, +;;+ is interpreted literally instead of splitting off a second command.
 
 [[session-delete]]
 === session-delete
@@ -618,6 +683,8 @@ Syntax: +:set-cmd-text [*--space*] [*--append*] 'text'+
 
 Preset the statusbar to some text.
 
+You can use the `{url}` and `{url:pretty}` variables here which will get replaced by the encoded/decoded URL.
+
 ==== positional arguments
 * +'text'+: The commandline to set.
 
@@ -627,7 +694,15 @@ Preset the statusbar to some text.
 
 ==== note
 * This command does not split arguments after the last argument and handles quotes literally.
-* With this command, +;;+ is interpreted literally instead of splitting off a second command.
+
+[[set-mark]]
+=== set-mark
+Syntax: +:set-mark 'key'+
+
+Set a mark at the current scroll position in the current tab.
+
+==== positional arguments
+* +'key'+: mark identifier; capital indicates a global mark
 
 [[spawn]]
 === spawn
@@ -635,7 +710,7 @@ Syntax: +:spawn [*--userscript*] [*--verbose*] [*--detach*] 'cmdline'+
 
 Spawn a command in a shell.
 
-Note the {url} variable which gets replaced by the current URL might be useful here.
+Note the `{url}` and `{url:pretty}` variables might be useful here. `{url}` gets replaced by the URL in fully encoded format and `{url:pretty}` uses a "pretty form" with most percent-encoded characters decoded.
 
 ==== positional arguments
 * +'cmdline'+: The commandline to execute.
@@ -652,7 +727,6 @@ Note the {url} variable which gets replaced by the current URL might be useful h
 
 ==== note
 * This command does not split arguments after the last argument and handles quotes literally.
-* With this command, +;;+ is interpreted literally instead of splitting off a second command.
 
 [[stop]]
 === stop
@@ -699,7 +773,8 @@ Select the tab given as argument/[count].
 If neither count nor index are given, it behaves like tab-next.
 
 ==== positional arguments
-* +'index'+: The tab index to focus, starting with 1. The special value `last` focuses the last focused tab.
+* +'index'+: The tab index to focus, starting with 1. The special value `last` focuses the last focused tab. Negative indexes
+ counts from the end, such that -1 is the last tab.
 
 
 ==== count
@@ -773,7 +848,7 @@ Save open pages and quit.
 
 [[yank]]
 === yank
-Syntax: +:yank [*--title*] [*--sel*] [*--domain*]+
+Syntax: +:yank [*--title*] [*--sel*] [*--domain*] [*--pretty*]+
 
 Yank the current URL/title to the clipboard or primary selection.
 
@@ -781,6 +856,7 @@ Yank the current URL/title to the clipboard or primary selection.
 * +*-t*+, +*--title*+: Yank the title instead of the URL.
 * +*-s*+, +*--sel*+: Use the primary selection instead of the clipboard.
 * +*-d*+, +*--domain*+: Yank only the scheme, domain, and port number.
+* +*-p*+, +*--pretty*+: Yank the URL in pretty decoded form.
 
 [[yank-selected]]
 === yank-selected
@@ -860,7 +936,9 @@ How many steps to zoom out.
 |<<paste-primary,paste-primary>>|Paste the primary selection at cursor position.
 |<<prompt-accept,prompt-accept>>|Accept the current prompt.
 |<<prompt-no,prompt-no>>|Answer no to a yes/no prompt.
+|<<prompt-open-download,prompt-open-download>>|Immediately open a download.
 |<<prompt-yes,prompt-yes>>|Answer yes to a yes/no prompt.
+|<<repeat-command,repeat-command>>|Repeat the last executed command.
 |<<rl-backward-char,rl-backward-char>>|Move back a character.
 |<<rl-backward-delete-char,rl-backward-delete-char>>|Delete the character before the cursor.
 |<<rl-backward-word,rl-backward-word>>|Move back to the start of the current or previous word.
@@ -1083,10 +1161,21 @@ Accept the current prompt.
 === prompt-no
 Answer no to a yes/no prompt.
 
+[[prompt-open-download]]
+=== prompt-open-download
+Immediately open a download.
+
 [[prompt-yes]]
 === prompt-yes
 Answer yes to a yes/no prompt.
 
+[[repeat-command]]
+=== repeat-command
+Repeat the last executed command.
+
+==== count
+Which count to pass the command.
+
 [[rl-backward-char]]
 === rl-backward-char
 Move back a character.
@@ -1222,7 +1311,7 @@ Scroll the current tab by 'count * dx/dy' pixels.
 
 ==== positional arguments
 * +'dx'+: How much to scroll in x-direction.
-* +'dy'+: How much to scroll in x-direction.
+* +'dy'+: How much to scroll in y-direction.
 
 ==== count
 multiplier

doc/help/settings.asciidoc

@@ -5,7 +5,6 @@
 |==============
 |Setting|Description
 |<<general-ignore-case,ignore-case>>|Whether to find text on a page case-insensitively.
-|<<general-wrap-search,wrap-search>>|Whether to wrap finding text to the top when arriving at the end.
 |<<general-startpage,startpage>>|The default page(s) to open at the start, separated by commas.
 |<<general-default-page,default-page>>|The page to open if :open -t/-b/-w is used without URL. Use `about:blank` for a blank page.
 |<<general-auto-search,auto-search>>|Whether to start a search when something else than a URL is entered.
@@ -33,10 +32,10 @@
 |<<ui-zoom-levels,zoom-levels>>|The available zoom levels, separated by commas.
 |<<ui-default-zoom,default-zoom>>|The default zoom level.
 |<<ui-downloads-position,downloads-position>>|Where to show the downloaded files.
+|<<ui-status-position,status-position>>|The position of the status bar.
 |<<ui-message-timeout,message-timeout>>|Time (in ms) to show messages in the statusbar for.
 |<<ui-message-unfocused,message-unfocused>>|Whether to show messages in unfocused windows.
 |<<ui-confirm-quit,confirm-quit>>|Whether to confirm quitting the application.
-|<<ui-display-statusbar-messages,display-statusbar-messages>>|Whether to display javascript statusbar messages.
 |<<ui-zoom-text-only,zoom-text-only>>|Whether the zoom factor on a frame applies only to the text or to all content.
 |<<ui-frame-flattening,frame-flattening>>|Whether to  expand each subframe to its contents.
 |<<ui-user-stylesheet,user-stylesheet>>|User stylesheet to use (absolute filename, filename relative to the config directory or CSS string). Will expand environment variables.
@@ -49,6 +48,7 @@
 |<<ui-hide-mouse-cursor,hide-mouse-cursor>>|Whether to hide the mouse cursor.
 |<<ui-modal-js-dialog,modal-js-dialog>>|Use standard JavaScript modal dialog for alert() and confirm()
 |<<ui-hide-wayland-decoration,hide-wayland-decoration>>|Hide the window decoration when using wayland (requires restart)
+|<<ui-keyhint-blacklist,keyhint-blacklist>>|Keychains that shouldn't be shown in the keyhint dialog
 |==============
 
 .Quick reference for section ``network''
@@ -63,6 +63,7 @@
 |<<network-proxy-dns-requests,proxy-dns-requests>>|Whether to send DNS requests over the configured proxy.
 |<<network-ssl-strict,ssl-strict>>|Whether to validate SSL handshakes.
 |<<network-dns-prefetch,dns-prefetch>>|Whether to try to pre-fetch DNS entries to speed up browsing.
+|<<network-custom-headers,custom-headers>>|Set custom headers for qutebrowser HTTP requests.
 |==============
 
 .Quick reference for section ``completion''
@@ -177,12 +178,14 @@
 |<<hints-mode,mode>>|Mode to use for hints.
 |<<hints-chars,chars>>|Chars used for hint strings.
 |<<hints-min-chars,min-chars>>|Minimum number of chars used for hint strings.
-|<<hints-scatter,scatter>>|Whether to scatter hint key chains (like Vimium) or not (like dwb).
+|<<hints-scatter,scatter>>|Whether to scatter hint key chains (like Vimium) or not (like dwb). Ignored for number hints.
 |<<hints-uppercase,uppercase>>|Make chars in hint strings uppercase.
 |<<hints-dictionary,dictionary>>|The dictionary file to be used by the word hints.
 |<<hints-auto-follow,auto-follow>>|Follow a hint immediately when the hint text is completely matched.
+|<<hints-auto-follow-timeout,auto-follow-timeout>>|A timeout to inhibit normal-mode key bindings after a successfulauto-follow.
 |<<hints-next-regexes,next-regexes>>|A comma-separated list of regexes to use for 'next' links.
 |<<hints-prev-regexes,prev-regexes>>|A comma-separated list of regexes to use for 'prev' links.
+|<<hints-find-implementation,find-implementation>>|Which implementation to use to find elements to hint.
 |==============
 
 .Quick reference for section ``colors''
@@ -252,6 +255,9 @@
 |<<colors-downloads.fg.error,downloads.fg.error>>|Foreground color for downloads with errors.
 |<<colors-downloads.bg.error,downloads.bg.error>>|Background color for downloads with errors.
 |<<colors-webpage.bg,webpage.bg>>|Background color for webpages if unset (or empty to use the theme's color)
+|<<colors-keyhint.fg,keyhint.fg>>|Text color for the keyhint widget.
+|<<colors-keyhint.fg.suffix,keyhint.fg.suffix>>|Highlight color for keys to complete the current keychain
+|<<colors-keyhint.bg,keyhint.bg>>|Background color of the keyhint widget.
 |==============
 
 .Quick reference for section ``fonts''
@@ -275,6 +281,7 @@
 |<<fonts-web-size-minimum-logical,web-size-minimum-logical>>|The minimum logical font size that is applied when zooming out.
 |<<fonts-web-size-default,web-size-default>>|The default font size for regular text.
 |<<fonts-web-size-default-fixed,web-size-default-fixed>>|The default font size for fixed-pitch text.
+|<<fonts-keyhint,keyhint>>|Font used in the keyhint widget.
 |==============
 
 == general
@@ -292,17 +299,6 @@ Valid values:
 
 Default: +pass:[smart]+
 
-[[general-wrap-search]]
-=== wrap-search
-Whether to wrap finding text to the top when arriving at the end.
-
-Valid values:
-
- * +true+
- * +false+
-
-Default: +pass:[true]+
-
 [[general-startpage]]
 === startpage
 The default page(s) to open at the start, separated by commas.
@@ -508,6 +504,17 @@ Valid values:
 
 Default: +pass:[top]+
 
+[[ui-status-position]]
+=== status-position
+The position of the status bar.
+
+Valid values:
+
+ * +top+
+ * +bottom+
+
+Default: +pass:[bottom]+
+
 [[ui-message-timeout]]
 === message-timeout
 Time (in ms) to show messages in the statusbar for.
@@ -538,17 +545,6 @@ Valid values:
 
 Default: +pass:[never]+
 
-[[ui-display-statusbar-messages]]
-=== display-statusbar-messages
-Whether to display javascript statusbar messages.
-
-Valid values:
-
- * +true+
- * +false+
-
-Default: +pass:[false]+
-
 [[ui-zoom-text-only]]
 === zoom-text-only
 Whether the zoom factor on a frame applies only to the text or to all content.
@@ -629,6 +625,7 @@ The format to use for the window title. The following placeholders are defined:
 * `{title_sep}`: The string ` - ` if a title is set, empty otherwise.
 * `{id}`: The internal window ID of this window.
 * `{scroll_pos}`: The page scroll position.
+* `{host}`: The host of the current web page.
 
 Default: +pass:[{perc}{title}{title_sep}qutebrowser]+
 
@@ -665,6 +662,14 @@ Valid values:
 
 Default: +pass:[false]+
 
+[[ui-keyhint-blacklist]]
+=== keyhint-blacklist
+Keychains that shouldn't be shown in the keyhint dialog
+
+Globs are supported, so ';*' will blacklist all keychainsstarting with ';'. Use '*' to disable keyhints
+
+Default: empty
+
 == network
 Settings related to the network.
 
@@ -750,6 +755,12 @@ Valid values:
 
 Default: +pass:[true]+
 
+[[network-custom-headers]]
+=== custom-headers
+Set custom headers for qutebrowser HTTP requests.
+
+Default: empty
+
 == completion
 Options related to completion and command history.
 
@@ -856,13 +867,17 @@ Options related to input modes.
 === timeout
 Timeout for ambiguous key bindings.
 
+If the current input forms both a complete match and a partial match, the complete match will be executed after this time.
+
 Default: +pass:[500]+
 
 [[input-partial-timeout]]
 === partial-timeout
 Timeout for partially typed key bindings.
 
-Default: +pass:[1000]+
+If the current input forms only partial matches, the keystring will be cleared after this time.
+
+Default: +pass:[5000]+
 
 [[input-insert-mode-on-plugins]]
 === insert-mode-on-plugins
@@ -1127,6 +1142,7 @@ The format to use for the tab title. The following placeholders are defined:
 * `{index}`: The index of this tab.
 * `{id}`: The internal tab ID of this tab.
 * `{scroll_pos}`: The page scroll position.
+* `{host}`: The host of the current web page.
 
 Default: +pass:[{index}: {title}]+
 
@@ -1321,7 +1337,7 @@ Valid values:
  * +true+
  * +false+
 
-Default: +pass:[true]+
+Default: +pass:[false]+
 
 [[content-css-regions]]
 === css-regions
@@ -1557,7 +1573,7 @@ Default: +pass:[1]+
 
 [[hints-scatter]]
 === scatter
-Whether to scatter hint key chains (like Vimium) or not (like dwb).
+Whether to scatter hint key chains (like Vimium) or not (like dwb). Ignored for number hints.
 
 Valid values:
 
@@ -1594,6 +1610,12 @@ Valid values:
 
 Default: +pass:[true]+
 
+[[hints-auto-follow-timeout]]
+=== auto-follow-timeout
+A timeout to inhibit normal-mode key bindings after a successfulauto-follow.
+
+Default: +pass:[0]+
+
 [[hints-next-regexes]]
 === next-regexes
 A comma-separated list of regexes to use for 'next' links.
@@ -1606,6 +1628,17 @@ A comma-separated list of regexes to use for 'prev' links.
 
 Default: +pass:[\bprev(ious)?\b,\bback\b,\bolder\b,\b[&lt;←≪]\b,\b(&lt;&lt;|«)\b]+
 
+[[hints-find-implementation]]
+=== find-implementation
+Which implementation to use to find elements to hint.
+
+Valid values:
+
+ * +javascript+: Better but slower
+ * +python+: Slightly worse but faster
+
+Default: +pass:[javascript]+
+
 == searchengines
 Definitions of search engines which can be used via the address bar.
 The searchengine named `DEFAULT` is used when `general -> auto-search` is true and something else than a URL was entered to be opened. Other search engines can be used by prepending the search engine name to the search term, e.g. `:open google qutebrowser`. The string `{}` will be replaced by the search term, use `{{` and `}}` for literal `{`/`}` signs.
@@ -1621,7 +1654,7 @@ Colors used in the UI.
 A value can be in one of the following format:
 
  * `#RGB`/`#RRGGBB`/`#RRRGGGBBB`/`#RRRRGGGGBBBB`
- * A SVG color name as specified in http://www.w3.org/TR/SVG/types.html#ColorKeywords[the W3C specification].
+ * An SVG color name as specified in http://www.w3.org/TR/SVG/types.html#ColorKeywords[the W3C specification].
  * transparent (no color)
  * `rgb(r, g, b)` / `rgba(r, g, b, a)` (values 0-255 or percentages)
  * `hsv(h, s, v)` / `hsva(h, s, v, a)` (values 0-255, hue 0-359)
@@ -2030,6 +2063,24 @@ Background color for webpages if unset (or empty to use the theme's color)
 
 Default: +pass:[white]+
 
+[[colors-keyhint.fg]]
+=== keyhint.fg
+Text color for the keyhint widget.
+
+Default: +pass:[#FFFFFF]+
+
+[[colors-keyhint.fg.suffix]]
+=== keyhint.fg.suffix
+Highlight color for keys to complete the current keychain
+
+Default: +pass:[#FFFF00]+
+
+[[colors-keyhint.bg]]
+=== keyhint.bg
+Background color of the keyhint widget.
+
+Default: +pass:[rgba(0, 0, 0, 80%)]+
+
 == fonts
 Fonts used for the UI, with optional style/weight/size.
 
@@ -2138,3 +2189,9 @@ Default: empty
 The default font size for fixed-pitch text.
 
 Default: empty
+
+[[fonts-keyhint]]
+=== keyhint
+Font used in the keyhint widget.
+
+Default: +pass:[8pt ${_monospace}]+

doc/quickstart.asciidoc

@@ -5,10 +5,23 @@ The Compiler <mail@qutebrowser.org>
 NOTE: This page will only appear on the first start. To view it at a later
 time, use the `:help` command.
 
+Basic keybindings to get you started
+------------------------------------
+
+* Use the arrow keys or `hjkl` to move around a webpage (vim-like syntax is used in quite a few places)
+* To go to a new webpage, press `o`, then type a url, then press Enter (Use `O` to open the url in a new tab). If what you've typed isn't a url, then a search engine will be used instead (DuckDuckGo, by default)
+* To switch between tabs, use `J` (next tab) and `K` (previous tab), or press `<Alt-num>`, where `num` is the position of the tab to switch to
+* To close the current tab, press `d` (and press `u` to undo closing a tab)
+* Use `H` and `L` to go back and forth in the history
+* To click on something without using the mouse, press `f` to show the hints, then type the keys next to what you want to click on (if that sounds weird, then just try pressing `f` and see what happens)
+* Press `:` to show the commandline
+* To search in a page, press `/`, type the phrase to search for, then press Enter. Use `n` and `N` to go back and forth through the matches, and press Esc to stop doing the search.
+* To close qutebrowser, press `Alt-F4`, or `:q`, or `:wq` to save the currently open tabs and quit (note that in the settings you can make qutebrowser always save the currently open tabs)
+
 What to do now
 --------------
 
-* View the http://qutebrowser.org/img/cheatsheet-big.png[key binding cheatsheet]
+* View the link:http://qutebrowser.org/img/cheatsheet-big.png[key binding cheatsheet]
 to make yourself familiar with the key bindings: +
 image:http://qutebrowser.org/img/cheatsheet-small.png["qutebrowser key binding cheatsheet",link="http://qutebrowser.org/img/cheatsheet-big.png"]
 * Run `:adblock-update` to download adblock lists and activate adblocking.

doc/qutebrowser.1.asciidoc

@@ -65,6 +65,9 @@ show it.
 *--target* '{auto,tab,tab-bg,tab-silent,tab-bg-silent,window}'::
     How URLs should be opened if there is already a qutebrowser instance running.
 
+*--backend* '{webkit,webengine}'::
+    Which backend to use (webengine backend is EXPERIMENTAL!).
+
 === debug arguments
 *-l* '{critical,error,warning,info,debug,vdebug}', *--loglevel* '{critical,error,warning,info,debug,vdebug}'::
     Set loglevel
@@ -78,9 +81,15 @@ show it.
 *--debug*::
     Turn on debugging options.
 
+*--json-logging*::
+    Output log lines in JSON format (one object per line).
+
 *--nocolor*::
     Turn off colored logging.
 
+*--force-color*::
+    Force colored logging
+
 *--harfbuzz* '{old,new,system,auto}'::
     HarfBuzz engine version to use. Default: auto.
 

doc/stacktrace.asciidoc

@@ -50,7 +50,7 @@ $ git checkout symbols
 $ export DEBUG_CFLAGS='-ggdb3 -fvar-tracking-assignments -Og'
 $ export DEBUG_CXXFLAGS='-ggdb3 -fvar-tracking-assignments -Og'
 $ cd qt5
-$ makepkg -si --pkg qt5-base-debug,qt5-webkit-debug
+$ makepkg -si --pkg qt5-base-debug,qt5-webkit-debug,qt5-webengine-debug
 $ cd ../pyqt5
 $ makepkg -si --pkg pyqt5-common-debug,python-pyqt5-debug
 ----
@@ -76,7 +76,7 @@ Server = http://qutebrowser.org/qt-debug/$arch
 Then install the packages:
 
 ----
-# pacman -Suy pyqt5-common-debug python-pyqt5-debug qt5-base-debug qt5-webkit-debug
+# pacman -Suy pyqt5-common-debug python-pyqt5-debug qt5-base-debug qt5-webkit-debug,qt5-webengine-debug
 ----
 
 The `-debug` packages conflict with the non-debug variants - it's safe to

doc/userscripts.asciidoc

@@ -33,6 +33,9 @@ The following environment variables will be set when a userscript is launched:
 - `QUTE_FIFO`: The FIFO or file to write commands to.
 - `QUTE_HTML`: Path of a file containing the HTML source of the current page.
 - `QUTE_TEXT`: Path of a file containing the plaintext of the current page.
+- `QUTE_CONFIG_DIR`: Path of the directory containing qutebrowser's configuration.
+- `QUTE_DATA_DIR`: Path of the directory containing qutebrowser's data.
+- `QUTE_DOWNLOAD_DIR`: Path of the downloads directory.
 
 In `command` mode:
 

icons/qutebrowser.xpm

@@ -0,0 +1,207 @@
+/* XPM */
+static char *qutebrowser[] = {
+/* columns rows colors chars-per-pixel */
+"32 32 169 2 ",
+"   c #0A396E",
+".  c #0B3C72",
+"X  c #0B4077",
+"o  c #0C437B",
+"O  c #134175",
+"+  c #15467C",
+"@  c #18477B",
+"#  c #1A497D",
+"$  c #0D4B86",
+"%  c #0F4E8D",
+"&  c #124A80",
+"*  c #1F4F83",
+"=  c #0E518C",
+"-  c #1F5084",
+";  c #11508C",
+":  c #0F5193",
+">  c #115799",
+",  c #115B9C",
+"<  c #204F83",
+"1  c #245287",
+"2  c #2A598C",
+"3  c #325E8F",
+"4  c #11609F",
+"5  c #346496",
+"6  c #3B6898",
+"7  c #115CA1",
+"8  c #115EAC",
+"9  c #1263A3",
+"0  c #1260AD",
+"q  c #136BAC",
+"w  c #136BB2",
+"e  c #1366BA",
+"r  c #196BB2",
+"t  c #157ABB",
+"y  c #1577BB",
+"u  c #2E6DB0",
+"i  c #387FB1",
+"p  c #456E9A",
+"a  c #4873A1",
+"s  c #4375AA",
+"d  c #507AA6",
+"f  c #597EA4",
+"g  c #4D7EB3",
+"h  c #156FCB",
+"j  c #167AC5",
+"k  c #1675CA",
+"l  c #177BCE",
+"z  c #1777D8",
+"x  c #1476E4",
+"c  c #167BE6",
+"v  c #167DE8",
+"b  c #197EEF",
+"n  c #1A7FF0",
+"m  c #1A80BE",
+"M  c #5F87AF",
+"N  c #5D8BBA",
+"B  c #5A84B1",
+"V  c #6C8FB3",
+"C  c #6F96BE",
+"Z  c #1886CC",
+"A  c #1883D7",
+"S  c #198DD5",
+"D  c #1987D9",
+"F  c #198ADC",
+"G  c #1A96DC",
+"H  c #3090D9",
+"J  c #1682E9",
+"K  c #1983ED",
+"L  c #1689E9",
+"P  c #1A8DEE",
+"I  c #1B95ED",
+"U  c #1C9EEA",
+"Y  c #1B97E4",
+"T  c #1A84F2",
+"R  c #1A8BF2",
+"E  c #1C94F4",
+"W  c #1D9CF5",
+"Q  c #3388E6",
+"!  c #3D90E9",
+"~  c #228EF3",
+"^  c #229FF6",
+"/  c #3294F4",
+"(  c #3D9FF6",
+")  c #339CF4",
+"_  c #1CA2E5",
+"`  c #1DABEE",
+"'  c #1DA4F6",
+"]  c #1EA9F7",
+"[  c #1EADF8",
+"{  c #1FB4F9",
+"}  c #1FB9FA",
+"|  c #20ACF8",
+" . c #27A4F6",
+".. c #3DA9F6",
+"X. c #20B9FA",
+"o. c #2EB6F9",
+"O. c #458DC9",
+"+. c #5C8DC1",
+"@. c #5795C6",
+"#. c #709DCB",
+"$. c #74A8DD",
+"%. c #4A97EA",
+"&. c #4896EA",
+"*. c #559EEA",
+"=. c #439AF5",
+"-. c #46A3F6",
+";. c #5FA9F6",
+":. c #5EA6F3",
+">. c #47BCF9",
+",. c #51B5F8",
+"<. c #58BDF8",
+"1. c #68ABEF",
+"2. c #7DB9E7",
+"3. c #63AEF7",
+"4. c #6FB1F7",
+"5. c #66B9F8",
+"6. c #61B2F6",
+"7. c #71B4F7",
+"8. c #78B7F4",
+"9. c #72BFF9",
+"0. c #3BC0FA",
+"q. c #6FCEFB",
+"w. c #6CC5FA",
+"e. c #7BCAF9",
+"r. c #89A7C3",
+"t. c #83A2C1",
+"y. c #98B6D3",
+"u. c #9DB9D3",
+"i. c #89B6E4",
+"p. c #83B6E9",
+"a. c #81BDF7",
+"s. c #83BFF8",
+"d. c #9EC4E9",
+"f. c #8CC2F9",
+"g. c #85CDFB",
+"h. c #87C4F9",
+"j. c #92C6F9",
+"k. c #95CAFA",
+"l. c #9CCBFA",
+"z. c #89D7FC",
+"x. c #91D9FC",
+"c. c #9CDEFD",
+"v. c #9ED2FB",
+"b. c #A7CAEC",
+"n. c #B5CEE3",
+"m. c #A1CEFA",
+"M. c #AED0F0",
+"N. c #ACD6FA",
+"B. c #A0DFFC",
+"V. c #AFD8FC",
+"C. c #B5D9FB",
+"Z. c #BCDDFC",
+"A. c #BFDCF5",
+"S. c #ACE3FD",
+"D. c #B5E5FE",
+"F. c #BBE2FC",
+"G. c #CFE5F5",
+"H. c #C3E1FC",
+"J. c #CAE6FD",
+"K. c #CCEBFD",
+"L. c #C4EBFE",
+"P. c #D6EDFE",
+"I. c #DAEEFD",
+"U. c #DEF1FE",
+"Y. c #D6F2FE",
+"T. c #E4F4FE",
+"R. c #E9F6FE",
+"E. c #EBF8FF",
+"W. c None",
+/* pixels */
+"W.W.W.W.W.W.W.W.W.W.W.c.S.L.Y.E.E.S.X.} W.W.W.W.W.W.W.W.W.W.W.W.",
+"W.W.W.W.W.W.W.W.W.D.T.E.E.T.L.D.c.z.} } X.} } W.W.W.W.W.W.W.W.W.",
+"W.W.W.W.W.W.W.B.T.T.R.T.R.U.0.X.z.S.} } } } { { X.W.W.W.W.W.W.W.",
+"W.W.W.W.W.W.x.x.K.T.T.T.L.P.q.o.{ } } ` _ { { { { { W.W.W.W.W.W.",
+"W.W.W.W.W.c.P.D.G.u.r.i 9 Z _ { { G 4 X t { { { { { { W.W.W.W.W.",
+"W.W.W.W.K.U.n.f O             { =       t { { { { [ { { W.W.W.W.",
+"W.W.W.F.I.t..                 '         t { { [ [ [ [ [ >.W.W.W.",
+"W.W.x.P.V                     ' X       t ` [ [ [ [ [ [ o.e.W.W.",
+"W.W.J.y.        X t S Y Z $   ' .       y [ [ [ ] [ [ | Z.J.W.W.",
+"W.<.e.&       , _ ] ] [ ] U . ' .       y [ ' [ ] ] ] w.K.J.g.W.",
+"W.' S       o ' ' [ ' [ ' ] o ' .       y Y 9 = = 9 @.J.J.J.F.W.",
+"W.| ,       j ' ' ' ' ' ' ' o ' .       $             p A.J.J.g.",
+"'  ..       G ' ' ' ' ' ' ' o ' .                       M H.H.h.",
+",.2.      . W ' W ' ' ' ' W . '                         . M.A.x.",
+"N.M..     . W W W ' W W W W  .w           9 I U 0         #.Z.m.",
+" .9.O       D W W W W ' W j $         % F W W W  .5       d Z.C.",
+"W W ;       9 9.h.5...Q %         o j W W W W W W O.      3 C.N.",
+"E W 7         B b.d.a         . w E E W W W E W E A       @ C.l.",
+"I E l                         u W E W E W E E E E A .     - k.6.",
+"P E E 7                       m.o E E E E E E E E l .     = E P ",
+"L E E E >           . O       s.o E E E E E E E E 7       , E L ",
+"W.R E R ) #.5 1 6 N i.2       s.+ E E E E E E R L .       k R W.",
+"W.L R E -.m.m.m.m.m.m.2       m.@ N m.m.s.( R R %       X E J W.",
+"W.W.K R ~ a.m.l.l.l.l.2       s.+ < i.l.m.j.h %         e K W.W.",
+"W.W.J R R / l.l.l.l.k.2       s.+     * 5 +           8 R J W.W.",
+"W.W.W.v T R 3.k.k.j.k.2     2 j.&                 . 8 R v W.W.W.",
+"W.W.W.W.J T ~ 7.j.j.j.g +.p.j.s.+.  .         . : z T v W.W.W.W.",
+"W.W.W.W.W.c T T =.f.j.j.s.j.j.j.j.$.g s u e h b T T v W.W.W.W.W.",
+"W.W.W.W.W.W.c b n 4.f.f.s.m.s.s.s.j.s.j./ T n T b c W.W.W.W.W.W.",
+"W.W.W.W.W.W.W.c x 1.s.s.s.s.s.s.s.s.4.=.n T n c c W.W.W.W.W.W.W.",
+"W.W.W.W.W.W.W.W.W.&.*.1.a.s.s.s.s.3.n n v x x W.W.W.W.W.W.W.W.W.",
+"W.W.W.W.W.W.W.W.W.W.W.%.%.%.%.*.*.Q x x x W.W.W.W.W.W.W.W.W.W.W."
+};

misc/docker/debian-jessie/Dockerfile

@@ -5,9 +5,18 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get -y update && \
     apt-get -y dist-upgrade && \
-    apt-get -y install python3-pyqt5 python3-pyqt5.qtwebkit python-tox \
-                       python3-sip xvfb git python3-setuptools wget \
-                       herbstluftwm locales
+    apt-get -y install --no-install-recommends \
+            python3-pyqt5 \
+            python3-pyqt5.qtwebkit \
+            python-tox \
+            python3-sip \
+            xvfb \
+            git \
+            python3-setuptools \
+            wget \
+            herbstluftwm \
+            locales \
+            libjs-pdf
 RUN echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen && locale-gen
 
 RUN useradd user && mkdir /home/user && chown -R user:users /home/user

misc/docker/ubuntu-xenial/Dockerfile

@@ -1,13 +1,25 @@
-FROM ubuntu:wily
+FROM ubuntu:xenial
 MAINTAINER Florian Bruhin <me@the-compiler.org>
 
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get -y update && \
     apt-get -y dist-upgrade && \
-    apt-get -y install python3-pyqt5 python3-pyqt5.qtwebkit python-tox \
-                       python3-sip xvfb git python3-setuptools wget \
-                       herbstluftwm language-pack-en
+    apt-get -y install --no-install-recommends \
+            python3-pyqt5 \
+            python3-pyqt5.qtwebkit \
+            python-tox \
+            python3-sip \
+            xvfb \
+            git \
+            python3-setuptools \
+            wget \
+            herbstluftwm \
+            language-pack-en \
+            libjs-pdf \
+            dbus
+
+RUN dbus-uuidgen --ensure
 
 RUN useradd user && mkdir /home/user && chown -R user:users /home/user
 USER user
@@ -22,4 +34,4 @@ CMD Xvfb -screen 0 800x600x24 :0 & \
     herbstluftwm & \
     git clone /outside qutebrowser.git && \
     cd qutebrowser.git && \
-    tox -e py34
+    tox -e py35

misc/qt_menu.nib/README

@@ -1,7 +0,0 @@
-These files are copied from Qt's source tree in
-src/plugins/platforms/cocoa/qt_menu.nib at revision
-b8246f08e49eb672974fd3d3d972a5ff13c1524d.
-
-http://code.qt.io/cgit/qt/qtbase.git/tree/src/plugins/platforms/cocoa/qt_menu.nib
-
-They are needed for cx_Freeze and don't seem to be bundled with Qt anymore.

misc/qt_menu.nib/classes.nib

@@ -1,59 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-<plist version="1.0">
-<dict>
-	<key>IBClasses</key>
-	<array>
-		<dict>
-			<key>ACTIONS</key>
-			<dict>
-				<key>hide</key>
-				<string>id</string>
-				<key>hideOtherApplications</key>
-				<string>id</string>
-				<key>orderFrontStandardAboutPanel</key>
-				<string>id</string>
-                <key>qtDispatcherToQPAMenuItem</key>
-				<string>id</string>
-				<key>terminate</key>
-				<string>id</string>
-				<key>unhideAllApplications</key>
-				<string>id</string>
-			</dict>
-			<key>CLASS</key>
-			<string>QCocoaMenuLoader</string>
-			<key>LANGUAGE</key>
-			<string>ObjC</string>
-			<key>OUTLETS</key>
-			<dict>
-				<key>aboutItem</key>
-				<string>NSMenuItem</string>
-				<key>aboutQtItem</key>
-				<string>NSMenuItem</string>
-				<key>appMenu</key>
-				<string>NSMenu</string>
-				<key>hideItem</key>
-				<string>NSMenuItem</string>
-				<key>preferencesItem</key>
-				<string>NSMenuItem</string>
-				<key>quitItem</key>
-				<string>NSMenuItem</string>
-				<key>theMenu</key>
-				<string>NSMenu</string>
-			</dict>
-			<key>SUPERCLASS</key>
-			<string>NSResponder</string>
-		</dict>
-		<dict>
-			<key>CLASS</key>
-			<string>FirstResponder</string>
-			<key>LANGUAGE</key>
-			<string>ObjC</string>
-			<key>SUPERCLASS</key>
-			<string>NSObject</string>
-		</dict>
-	</array>
-	<key>IBVersion</key>
-	<string>1</string>
-</dict>
-</plist>

misc/qt_menu.nib/info.nib

@@ -1,18 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-<plist version="1.0">
-<dict>
-	<key>IBFramework Version</key>
-	<string>672</string>
-	<key>IBOldestOS</key>
-	<integer>5</integer>
-	<key>IBOpenObjects</key>
-	<array>
-		<integer>57</integer>
-	</array>
-	<key>IBSystem Version</key>
-	<string>9L31a</string>
-	<key>targetFramework</key>
-	<string>IBCocoaFramework</string>
-</dict>
-</plist>

misc/qutebrowser.spec

@@ -0,0 +1,74 @@
+# -*- mode: python -*-
+
+import sys
+import os
+
+sys.path.insert(0, os.getcwd())
+from scripts import setupcommon
+
+block_cipher = None
+
+
+def get_data_files():
+    data_files = [
+        ('../qutebrowser/html', 'html'),
+        ('../qutebrowser/img', 'img'),
+        ('../qutebrowser/javascript', 'javascript'),
+        ('../qutebrowser/html/doc', 'html/doc'),
+        ('../qutebrowser/git-commit-id', '')
+    ]
+
+    if os.path.exists(os.path.join('qutebrowser', '3rdparty', 'pdfjs')):
+        data_files.append(('../qutebrowser/3rdparty/pdfjs', '3rdparty/pdfjs'))
+    else:
+        print("Warning: excluding pdfjs as it's not present!")
+
+    return data_files
+
+
+setupcommon.write_git_file()
+
+
+if os.name == 'nt':
+    icon = 'icons/qutebrowser.ico'
+elif sys.platform == 'darwin':
+    icon = 'icons/qutebrowser.icns'
+else:
+    icon = None
+
+
+a = Analysis(['../qutebrowser/__main__.py'],
+             pathex=['misc'],
+             binaries=None,
+             datas=get_data_files(),
+             hiddenimports=[],
+             hookspath=[],
+             runtime_hooks=[],
+             excludes=[],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher)
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+exe = EXE(pyz,
+          a.scripts,
+          exclude_binaries=True,
+          name='qutebrowser',
+          icon=icon,
+          debug=False,
+          strip=False,
+          upx=True,
+          console=False )
+coll = COLLECT(exe,
+               a.binaries,
+               a.zipfiles,
+               a.datas,
+               strip=False,
+               upx=True,
+               name='qutebrowser')
+
+app = BUNDLE(coll,
+             name='qutebrowser.app',
+             icon=icon,
+             info_plist={'NSHighResolutionCapable': 'True'},
+             bundle_identifier=None)

misc/requirements/README.md

@@ -0,0 +1,20 @@
+This directory contains various `requirements` files which are used by `tox` to
+have reproducable tests with pinned versions.
+
+The files are generated based on unpinned requirements in `*.txt-raw` files.
+
+Those files can also contain some special commands:
+
+- Add an additional comment to a line: `#@ comment: <package> <comment here>`
+- Filter a line for requirements.io: `#@ filter: <package> <filter>`
+- Don't include a package in the output: `#@ ignore: <package>` (or multiple packages)
+- Replace a part of a frozen package specification with another: `#@ replace <regex> <replacement>`
+
+Some examples:
+
+```
+#@ comment: mypkg blah blub
+#@ filter: mypkg != 1.0.0
+#@ ignore: mypkg, otherpkg
+#@ replace: foo bar
+```

misc/requirements/requirements-check-manifest.txt

@@ -0,0 +1,3 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
+check-manifest==0.31

misc/requirements/requirements-check-manifest.txt-raw

@@ -0,0 +1 @@
+check-manifest

misc/requirements/requirements-codecov.txt

@@ -0,0 +1,5 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
+codecov==2.0.5
+coverage==4.1
+requests==2.10.0

misc/requirements/requirements-codecov.txt-raw

@@ -0,0 +1 @@
+codecov

misc/requirements/requirements-cxfreeze.txt

@@ -0,0 +1,3 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
+cx-Freeze==4.3.4

misc/requirements/requirements-cxfreeze.txt-raw

@@ -0,0 +1 @@
+cx_Freeze

misc/requirements/requirements-flake8.txt

@@ -0,0 +1,26 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
+flake8==2.6.2  # rq.filter: < 3.0.0
+flake8-copyright==0.2.0
+flake8-debugger==1.4.0
+flake8-deprecated==1.0
+flake8-docstrings==0.2.8
+flake8-future-import==0.4.3
+flake8-mock==0.2
+flake8-pep3101==0.4
+flake8-putty==0.4.0
+flake8-string-format==0.2.2
+flake8-tidy-imports==1.0.2
+flake8-tuple==0.2.12
+hacking==0.11.0
+mccabe==0.5.0
+packaging==16.7
+pbr==1.10.0
+pep257==0.7.0  # still needed by flake8-docstrings but ignored
+pep8==1.7.0
+pep8-naming==0.4.1
+pycodestyle==2.0.0
+pydocstyle==1.0.0
+pyflakes==1.2.3
+pyparsing==2.1.5
+six==1.10.0

misc/requirements/requirements-flake8.txt-raw

@@ -0,0 +1,23 @@
+flake8<3.0.0
+flake8-copyright
+flake8-debugger
+flake8-deprecated
+flake8-docstrings
+flake8-future-import
+flake8-mock
+flake8-pep3101
+flake8-putty
+flake8-string-format
+flake8-tidy-imports
+flake8-tuple
+hacking
+pep8-naming
+pydocstyle
+pyflakes
+
+pep8==1.7.0
+
+#@ comment: pep257 still needed by flake8-docstrings but ignored
+
+# Waiting until hacking/flake8-tuple are updated
+#@ filter: flake8 < 3.0.0

misc/requirements/requirements-pip.txt

@@ -0,0 +1 @@
+pip==8.1.2

misc/requirements/requirements-pyinstaller.txt

@@ -0,0 +1,3 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
+PyInstaller==3.2

misc/requirements/requirements-pyinstaller.txt-raw

@@ -0,0 +1 @@
+PyInstaller

misc/requirements/requirements-pylint-master.txt

@@ -0,0 +1,11 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
+-e git+https://github.com/PyCQA/astroid.git#egg=astroid
+isort==4.2.5
+lazy-object-proxy==1.2.2
+mccabe==0.5.0
+-e git+https://github.com/PyCQA/pylint.git#egg=pylint
+./scripts/dev/pylint_checkers
+requests==2.10.0
+six==1.10.0
+wrapt==1.10.8

misc/requirements/requirements-pylint-master.txt-raw

@@ -0,0 +1,13 @@
+-e git+https://github.com/PyCQA/astroid.git#egg=astroid
+-e git+https://github.com/PyCQA/pylint.git#egg=pylint
+./scripts/dev/pylint_checkers
+requests
+
+# https://github.com/PyCQA/pylint/issues/932
+mccabe==0.5.0
+
+# remove @commit-id for scm installs
+#@ replace: @.*# #
+
+# fix qute-pylint location
+#@ replace: qute-pylint==.* ./scripts/dev/pylint_checkers

misc/requirements/requirements-pylint.txt

@@ -0,0 +1,11 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
+astroid==1.4.7
+isort==4.2.5
+lazy-object-proxy==1.2.2
+mccabe==0.5.0
+pylint==1.6.4
+./scripts/dev/pylint_checkers
+requests==2.10.0
+six==1.10.0
+wrapt==1.10.8

misc/requirements/requirements-pylint.txt-raw

@@ -0,0 +1,6 @@
+pylint
+./scripts/dev/pylint_checkers
+requests
+
+# fix qute-pylint location
+#@ replace: qute-pylint==.* ./scripts/dev/pylint_checkers

misc/requirements/requirements-pyroma.txt

@@ -0,0 +1,4 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
+docutils==0.12
+pyroma==2.0.2

misc/requirements/requirements-pyroma.txt-raw

@@ -0,0 +1 @@
+pyroma

misc/requirements/requirements-qutebrowser.txt-raw

@@ -0,0 +1,6 @@
+Jinja2
+Pygments
+pyPEG2
+PyYAML
+colorama
+cssutils

misc/requirements/requirements-tests.txt

@@ -0,0 +1,32 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
+beautifulsoup4==4.5.0
+CherryPy==7.1.0
+coverage==4.1
+decorator==4.0.10
+Flask==0.10.1  # rq.filter: < 0.11.0
+glob2==0.4.1
+httpbin==0.4.1
+hypothesis==3.4.2
+itsdangerous==0.24
+# Jinja2==2.8
+Mako==1.0.4
+# MarkupSafe==0.23
+parse==1.6.6
+parse-type==0.3.4
+py==1.4.31
+pytest==2.9.2
+pytest-bdd==2.17.0
+pytest-catchlog==1.2.2
+pytest-cov==2.3.0
+pytest-faulthandler==1.3.0
+pytest-instafail==0.3.0
+pytest-mock==1.1
+pytest-qt==1.11.0
+pytest-repeat==0.3.0
+pytest-rerunfailures==2.0.0
+pytest-travis-fold==1.2.0
+pytest-xvfb==0.2.0
+six==1.10.0
+vulture==0.10
+Werkzeug==0.11.10

misc/requirements/requirements-tests.txt-raw

@@ -0,0 +1,22 @@
+beautifulsoup4
+CherryPy
+coverage
+Flask==0.10.1
+httpbin
+hypothesis
+pytest
+pytest-bdd
+pytest-catchlog
+pytest-cov
+pytest-faulthandler
+pytest-instafail
+pytest-mock
+pytest-qt
+pytest-repeat
+pytest-rerunfailures
+pytest-travis-fold
+pytest-xvfb
+vulture
+
+#@ filter: Flask < 0.11.0
+#@ ignore: Jinja2, MarkupSafe

misc/requirements/requirements-tox.txt

@@ -0,0 +1,6 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
+pluggy==0.3.1
+py==1.4.31
+tox==2.3.1
+virtualenv==15.0.2

misc/requirements/requirements-tox.txt-raw

@@ -0,0 +1 @@
+tox

misc/requirements/requirements-vulture.txt

@@ -0,0 +1,3 @@
+# This file is automatically generated by scripts/dev/recompile_requirements.py
+
+vulture==0.10

misc/requirements/requirements-vulture.txt-raw

@@ -0,0 +1 @@
+vulture

misc/userscripts/dmenu_qutebrowser

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Copyright 2015 Zach-Button <zachrey.button@gmail.com>
 #
@@ -39,10 +39,9 @@
 
 [ -z "$QUTE_URL" ] && QUTE_URL='http://google.com'
 
-url=$(echo "$QUTE_URL" | cat - ~/.config/qutebrowser/quickmarks ~/.local/share/qutebrowser/history | dmenu -l 15 -p qutebrowser)
-url=$(echo $url | sed -E 's/[^ ]+ +//g' | egrep "https?:" || echo $url)
+url=$(echo "$QUTE_URL" | cat - "$QUTE_CONFIG_DIR/quickmarks" "$QUTE_DATA_DIR/history" | dmenu -l 15 -p qutebrowser)
+url=$(echo "$url" | sed -E 's/[^ ]+ +//g' | egrep "https?:" || echo "$url")
 
 [ -z "${url// }" ] && exit
 
 echo "open $url" >> "$QUTE_FIFO" || qutebrowser "$url"
-

misc/userscripts/open_download

@@ -1,4 +1,4 @@
-#!/bin/bash -e
+#!/usr/bin/env bash
 # Both standalone script and qutebrowser userscript that opens a rofi menu with
 # all files from the download director and opens the selected file. It works
 # both as a userscript and a standalone script that is called from outside of
@@ -18,7 +18,10 @@
 # Thorsten Wißmann, 2015 (thorsten` on freenode)
 # Any feedback is welcome!
 
+set -e
+
 # open a file from the download directory using rofi
+DOWNLOAD_DIR=${DOWNLOAD_DIR:-$QUTE_DOWNLOAD_DIR}
 DOWNLOAD_DIR=${DOWNLOAD_DIR:-$HOME/Downloads}
 # the name of the rofi command
 ROFI_CMD=${ROFI_CMD:-rofi}
@@ -49,7 +52,7 @@ die() {
 if ! [ -d "$DOWNLOAD_DIR" ] ; then
     die "Download directory »$DOWNLOAD_DIR« not found!"
 fi
-if ! $(which "${ROFI_CMD}" > /dev/null ) ; then
+if ! which "${ROFI_CMD}" > /dev/null ; then
     die "Rofi command »${ROFI_CMD}« not found in PATH!"
 fi
 
@@ -109,5 +112,3 @@ fi
 msg info "Opening »$file« (of type $filetype) with ${application%.desktop}"
 
 xdg-open "$path" &
-
-

misc/userscripts/password_fill

@@ -1,4 +1,4 @@
-#!/bin/bash -e
+#!/usr/bin/env bash
 help() {
     blink=$'\e[1;31m' reset=$'\e[0m'
 cat <<EOF
@@ -22,7 +22,7 @@ Pass backend: (see also passwordstore.org)
   must be contained in a later line beginning with "user:", "login:", or
   "username:" (configurable by the user_pattern variable).
 
-Behaviour:
+Behavior:
   It will try to find a username/password entry in the configured backend
   (currently only pass) for the current website and will load that pair of
   username and password to any form on the current page that has some password
@@ -39,6 +39,7 @@ Configuration:
 EOF
 }
 
+set -o errexit
 set -o pipefail
 shopt -s nocasematch # make regexp matching in bash case insensitive
 
@@ -61,7 +62,7 @@ die() {
 }
 
 javascript_escape() {
-    # print the first argument in a escaped way, such that it can savely
+    # print the first argument in an escaped way, such that it can safely
     # be used within javascripts double quotes
     sed "s,[\\\'\"],\\\&,g" <<< "$1"
 }
@@ -80,7 +81,7 @@ javascript_escape() {
 # between different paths on the same domain.
 
 simplify_url() {
-    simple_url="${1##*://}" # remove protocoll specification
+    simple_url="${1##*://}" # remove protocol specification
     simple_url="${simple_url%%\?*}" # remove GET parameters
     simple_url="${simple_url%%/*}" # remove directory path
     simple_url="${simple_url%:*}" # remove port
@@ -89,7 +90,7 @@ simplify_url() {
 
 # no_entries_found() is called if the first query_entries() call did not find
 # any matching entries. Multiple implementations are possible:
-# The easiest behaviour is to quit:
+# The easiest behavior is to quit:
 #no_entries_found() {
 #    if [ 0 -eq "${#files[@]}" ] ; then
 #        die "No entry found for »$simple_url«"
@@ -106,7 +107,7 @@ simplify_url() {
 #     fi
 # }
 
-# Another beahviour is to drop another level of subdomains until search hits
+# Another behavior is to drop another level of subdomains until search hits
 # are found:
 no_entries_found() {
     while [ 0 -eq "${#files[@]}" ] && [ -n "$simple_url" ]; do
@@ -170,7 +171,7 @@ choose_entry_random() {
 # dmenu or zenity or even qutebrowser completion in future?) which entry to
 # pick
 MENU_COMMAND=( head -n 1 )
-# whether to show the menu if there is only one entrie in it
+# whether to show the menu if there is only one entry in it
 menu_if_one_entry=0
 choose_entry_menu() {
     local nr=${#files[@]}
@@ -245,7 +246,7 @@ pass_backend() {
             done < <(find -L "$PREFIX" -iname '*.gpg' -print0)
         fi
         if ((match_filename)) ; then
-            # add entries wth matching filepath
+            # add entries with matching filepath
             while read -r passfile ; do
                 passfile="${passfile#$PREFIX}"
                 passfile="${passfile#/}"

misc/userscripts/qutebrowser_viewsource

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Copyright 2015 Zach-Button <zachrey.button@gmail.com>
 #
@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-# 
+#
 # This script fetches the unprocessed HTML source for a page and opens it in vim.
 #	:bind gf spawn --userscript qutebrowser_viewsource
 #

misc/userscripts/qutedmenu

@@ -7,7 +7,6 @@
 # If you would like to set a custom colorscheme/font use these dirs.
 # https://github.com/halfwit/dotfiles/blob/master/.config/dmenu/bemenucolors
 readonly confdir=${XDG_CONFIG_HOME:-$HOME/.config}
-readonly datadir=${XDG_DATA_HOME:-$HOME/.local/share}
 
 readonly optsfile=$confdir/dmenu/bemenucolors
 
@@ -15,17 +14,17 @@ create_menu() {
     # Check quickmarks
     while read -r url; do
         printf -- '%s\n' "$url"
-    done < "$confdir"/qutebrowser/quickmarks
+    done < "$QUTE_CONFIG_DIR"/quickmarks
 
     # Next bookmarks
     while read -r url _; do
         printf -- '%s\n' "$url"
-    done < "$confdir"/qutebrowser/bookmarks/urls
+    done < "$QUTE_CONFIG_DIR"/bookmarks/urls
 
     # Finally history
     while read -r _ url; do
         printf -- '%s\n' "$url"
-    done < "$datadir"/qutebrowser/history
+    done < "$QUTE_DATA_DIR"/history
     }
 
 get_selection() {

misc/userscripts/taskadd

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+#
+# Behavior:
+#   Userscript for qutebrowser which adds a task to taskwarrior.
+#   If run as a command (:spawn --userscript taskadd), it creates a new task
+#   with the description equal to the current page title and annotates it with
+#   the current page url. Additional arguments are passed along so you can add
+#   mods to the task (e.g. priority, due date, tags).
+#
+#   Example:
+#       :spawn --userscript taskadd due:eod pri:H
+#
+#   To enable passing along extra args, I suggest using a mapping like:
+#       :bind <somekey> set-cmd-text -s :spawn --userscript taskadd
+#
+#   If run from hint mode, it uses the selected hint text as the description
+#   and the selected hint url as the annotation.
+#
+# Ryan Roden-Corrent (rcorre), 2016
+# Any feedback is welcome!
+#
+# For more info on Taskwarrior, see http://taskwarrior.org/
+
+# use either the current page title or the hint text as the task description
+[[ $QUTE_MODE == 'hints' ]] && title=$QUTE_SELECTED_TEXT || title=$QUTE_TITLE
+
+# try to add the task and grab the output
+msg="$(task add $title $@ 2>&1)"
+
+if [[ $? == 0 ]]; then
+    # annotate the new task with the url, send the output back to the browser
+    task +LATEST annotate "$QUTE_URL"
+    echo "message-info '$msg'" >> $QUTE_FIFO
+else
+    echo "message-error '$msg'" >> $QUTE_FIFO
+fi

misc/userscripts/view_in_mpv

@@ -1,6 +1,6 @@
-#!/bin/bash -e
+#!/usr/bin/env bash
 #
-# Behaviour:
+# Behavior:
 #   Userscript for qutebrowser which views the current web page in mpv using
 #   sensible mpv-flags. While viewing the page in MPV, all <video>, <embed>,
 #   and <object> tags in the original page are temporarily removed. Clicking on
@@ -24,6 +24,8 @@
 # Thorsten Wißmann, 2015 (thorsten` on freenode)
 # Any feedback is welcome!
 
+set -e
+
 if [ -z "$QUTE_FIFO" ] ; then
     cat 1>&2 <<EOF
 Error: $0 can not be run as a standalone script.

pytest.ini

@@ -1,5 +1,4 @@
 [pytest]
-norecursedirs = .tox .venv
 addopts = --strict -rfEw --faulthandler-timeout=70 --instafail
 markers =
     gui: Tests using the GUI (e.g. spawning widgets)
@@ -11,12 +10,11 @@ markers =
     not_frozen: Tests which can't be run if sys.frozen is True.
     no_xvfb: Tests which can't be run with Xvfb.
     frozen: Tests which can only be run if sys.frozen is True.
-    integration: Tests which test a bigger portion of code, run without coverage.
-    skip: Always skipped test.
-    pyqt531_or_newer: Needs PyQt 5.3.1 or newer.
+    integration: Tests which test a bigger portion of code
+    end2end: End to end tests which run qutebrowser as subprocess
     xfail_norun: xfail the test with out running it
     ci: Tests which should only run on CI.
-    flaky: Tests which are flaky and should be rerun
+    flaky_once: Try to rerun this test once if it fails
 qt_log_level_fail = WARNING
 qt_log_ignore =
     ^SpellCheck: .*
@@ -37,5 +35,6 @@ qt_log_ignore =
     ^Geoclue error: Process org\.freedesktop\.Geoclue\.Master exited with status 127
     ^QObject::connect: Cannot connect \(null\)::stateChanged\(QNetworkSession::State\) to QNetworkReplyHttpImpl::_q_networkSessionStateChanged\(QNetworkSession::State\)
     ^QXcbClipboard: Cannot transfer data, no data available
+    ^load glyph failed
 qt_wait_signal_raising = true
 xfail_strict = true

qutebrowser/__init__.py

@@ -28,7 +28,7 @@ __copyright__ = "Copyright 2014-2016 Florian Bruhin (The Compiler)"
 __license__ = "GPL"
 __maintainer__ = __author__
 __email__ = "mail@qutebrowser.org"
-__version_info__ = (0, 6, 0)
+__version_info__ = (0, 8, 4)
 __version__ = '.'.join(str(e) for e in __version_info__)
 __description__ = "A keyboard-driven, vim-like browser based on PyQt5 and QtWebKit."
 

qutebrowser/app.py

@@ -31,7 +31,7 @@ import atexit
 import datetime
 import tokenize
 
-from PyQt5.QtWidgets import QApplication
+from PyQt5.QtWidgets import QApplication, QWidget
 from PyQt5.QtWebKit import QWebSettings
 from PyQt5.QtGui import QDesktopServices, QPixmap, QIcon, QCursor, QWindow
 from PyQt5.QtCore import (pyqtSlot, qInstallMessageHandler, QTimer, QUrl,
@@ -46,8 +46,10 @@ import qutebrowser.resources
 from qutebrowser.completion.models import instances as completionmodels
 from qutebrowser.commands import cmdutils, runners, cmdexc
 from qutebrowser.config import style, config, websettings, configexc
-from qutebrowser.browser import urlmarks, cookies, cache, adblock, history
-from qutebrowser.browser.network import qutescheme, proxy, networkmanager
+from qutebrowser.browser import urlmarks, adblock
+from qutebrowser.browser.webkit import cookies, cache, history, downloads
+from qutebrowser.browser.webkit.network import (qutescheme, proxy,
+                                                networkmanager)
 from qutebrowser.mainwindow import mainwindow
 from qutebrowser.misc import readline, ipc, savemanager, sessions, crashsignal
 from qutebrowser.misc import utilcmds  # pylint: disable=unused-import
@@ -62,12 +64,7 @@ qApp = None
 def run(args):
     """Initialize everything and run the application."""
     if args.version:
-        print(version.version(short=True))
-        print()
-        print()
-        print(qutebrowser.__copyright__)
-        print()
-        print(version.GPL_BOILERPLATE.strip())
+        print(version.version())
         sys.exit(usertypes.Exit.ok)
 
     if args.temp_basedir:
@@ -166,7 +163,7 @@ def _init_icon():
     """Initialize the icon of qutebrowser."""
     icon = QIcon()
     fallback_icon = QIcon()
-    for size in (16, 24, 32, 48, 64, 96, 128, 256, 512):
+    for size in [16, 24, 32, 48, 64, 96, 128, 256, 512]:
         filename = ':/icons/qutebrowser-{}x{}.png'.format(size, size)
         pixmap = QPixmap(filename)
         qtutils.ensure_not_null(pixmap)
@@ -244,12 +241,7 @@ def process_pos_args(args, via_ipc=False, cwd=None, target_arg=None):
         cwd: The cwd to use for fuzzy_url.
         target_arg: Command line argument received by a running instance via
                     ipc. If the --target argument was not specified, target_arg
-                    will be an empty string instead of None. This behavior is
-                    caused by the PyQt signal
-                    ``got_args = pyqtSignal(list, str, str)``
-                    used in the misc.ipc.IPCServer class. PyQt converts the
-                    None value into a null QString and then back to an empty
-                    python string
+                    will be an empty string.
     """
     if via_ipc and not args:
         win_id = mainwindow.get_window(via_ipc, force_window=True)
@@ -275,14 +267,17 @@ def process_pos_args(args, via_ipc=False, cwd=None, target_arg=None):
             tabbed_browser = objreg.get('tabbed-browser', scope='window',
                                         window=win_id)
             log.init.debug("Startup URL {}".format(cmd))
+            if not cwd:  # could also be an empty string due to the PyQt signal
+                cwd = None
             try:
                 url = urlutils.fuzzy_url(cmd, cwd, relative=True)
             except urlutils.InvalidUrlError as e:
                 message.error('current', "Error in startup argument '{}': "
                               "{}".format(cmd, e))
             else:
-                background = open_target in ('tab-bg', 'tab-bg-silent')
-                tabbed_browser.tabopen(url, background=background)
+                background = open_target in ['tab-bg', 'tab-bg-silent']
+                tabbed_browser.tabopen(url, background=background,
+                                       explicit=True)
 
 
 def _open_startpage(win_id=None):
@@ -344,24 +339,23 @@ def _save_version():
 
 def on_focus_changed(_old, new):
     """Register currently focused main window in the object registry."""
-    if new is None:
-        window = None
-    else:
-        window = new.window()
-    if window is None or not isinstance(window, mainwindow.MainWindow):
+    if not isinstance(new, QWidget):
+        log.misc.debug("on_focus_changed called with non-QWidget {!r}".format(
+            new))
+
+    if new is None or not isinstance(new, mainwindow.MainWindow):
         try:
             objreg.delete('last-focused-main-window')
         except KeyError:
             pass
         qApp.restoreOverrideCursor()
     else:
-        objreg.register('last-focused-main-window', window, update=True)
+        objreg.register('last-focused-main-window', new.window(), update=True)
         _maybe_hide_mouse_cursor()
 
 
-@pyqtSlot(QUrl)
 def open_desktopservices_url(url):
-    """Handler to open an URL via QDesktopServices."""
+    """Handler to open a URL via QDesktopServices."""
     win_id = mainwindow.get_window(via_ipc=True, force_window=False)
     tabbed_browser = objreg.get('tabbed-browser', scope='window',
                                 window=win_id)
@@ -427,7 +421,9 @@ def _init_modules(args, crash_handler):
     proxy.init()
     log.init.debug("Initializing cookies...")
     cookie_jar = cookies.CookieJar(qApp)
+    ram_cookie_jar = cookies.RAMCookieJar(qApp)
     objreg.register('cookie-jar', cookie_jar)
+    objreg.register('ram-cookie-jar', ram_cookie_jar)
     log.init.debug("Initializing cache...")
     diskcache = cache.DiskCache(standarddir.cache(), parent=qApp)
     objreg.register('cache', diskcache)
@@ -440,6 +436,8 @@ def _init_modules(args, crash_handler):
         os.environ.pop('QT_WAYLAND_DISABLE_WINDOWDECORATION', None)
     _maybe_hide_mouse_cursor()
     objreg.get('config').changed.connect(_maybe_hide_mouse_cursor)
+    temp_downloads = downloads.TempDownloadManager(qApp)
+    objreg.register('temporary-downloads', temp_downloads)
 
 
 def _init_late_modules(args):
@@ -478,7 +476,6 @@ class Quitter:
         self._shutting_down = False
         self._args = args
 
-    @pyqtSlot()
     def on_last_window_closed(self):
         """Slot which gets invoked when the last window was closed."""
         self.shutdown(last_window=True)
@@ -493,12 +490,12 @@ class Quitter:
         else:
             path = os.path.abspath(os.path.dirname(qutebrowser.__file__))
             if not os.path.isdir(path):
-                # Probably running from an python egg.
+                # Probably running from a python egg.
                 return
 
         for dirpath, _dirnames, filenames in os.walk(path):
             for fn in filenames:
-                if os.path.splitext(fn)[1] == '.py':
+                if os.path.splitext(fn)[1] == '.py' and os.path.isfile(fn):
                     with tokenize.open(os.path.join(dirpath, fn)) as f:
                         compile(f.read(), fn, 'exec')
 
@@ -526,7 +523,7 @@ class Quitter:
             cwd = os.path.join(os.path.abspath(os.path.dirname(
                                qutebrowser.__file__)), '..')
             if not os.path.isdir(cwd):
-                # Probably running from an python egg. Let's fallback to
+                # Probably running from a python egg. Let's fallback to
                 # cwd=None and see if that works out.
                 # See https://github.com/The-Compiler/qutebrowser/issues/323
                 cwd = None
@@ -554,8 +551,9 @@ class Quitter:
             argdict['session'] = session
             argdict['override_restore'] = False
         # Ensure :restart works with --temp-basedir
-        argdict['temp_basedir'] = False
-        argdict['temp_basedir_restarted'] = True
+        if self._args.temp_basedir:
+            argdict['temp_basedir'] = False
+            argdict['temp_basedir_restarted'] = True
 
         # Dump the data
         data = json.dumps(argdict)
@@ -713,6 +711,8 @@ class Quitter:
                 not restart):
             atexit.register(shutil.rmtree, self._args.basedir,
                             ignore_errors=True)
+        # Delete temp download dir
+        objreg.get('temporary-downloads').cleanup()
         # If we don't kill our custom handler here we might get segfaults
         log.destroy.debug("Deactivating message handler...")
         qInstallMessageHandler(None)
@@ -723,8 +723,8 @@ class Quitter:
         # segfaults.
         QTimer.singleShot(0, functools.partial(qApp.exit, status))
 
-    @cmdutils.register(instance='quitter', name='wq',
-                       completion=[usertypes.Completion.sessions])
+    @cmdutils.register(instance='quitter', name='wq')
+    @cmdutils.argument('name', completion=usertypes.Completion.sessions)
     def save_and_quit(self, name=sessions.default):
         """Save open pages and quit.
 

qutebrowser/browser/__init__.py

@@ -1,6 +1,6 @@
 # vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
 
-# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
 #
 # This file is part of qutebrowser.
 #

qutebrowser/browser/adblock.py

@@ -27,7 +27,7 @@ import zipfile
 import fnmatch
 
 from qutebrowser.config import config
-from qutebrowser.utils import objreg, standarddir, log, message
+from qutebrowser.utils import objreg, standarddir, log, message, usertypes
 from qutebrowser.commands import cmdutils, cmdexc
 
 
@@ -48,7 +48,7 @@ def guess_zip_filename(zf):
 
 
 def get_fileobj(byte_io):
-    """Get an usable file object to read the hosts file from."""
+    """Get a usable file object to read the hosts file from."""
     byte_io.seek(0)  # rewind downloaded file
     if zipfile.is_zipfile(byte_io):
         byte_io.seek(0)  # rewind what zipfile.is_zipfile did
@@ -116,6 +116,7 @@ class HostBlocker:
             self._local_hosts_file = None
         else:
             self._local_hosts_file = os.path.join(data_dir, 'blocked-hosts')
+        self.on_config_changed()
 
         config_dir = standarddir.config()
         if config_dir is None:
@@ -176,7 +177,8 @@ class HostBlocker:
                 message.info('current',
                              "Run :adblock-update to get adblock lists.")
 
-    @cmdutils.register(instance='host-blocker', win_id='win_id')
+    @cmdutils.register(instance='host-blocker')
+    @cmdutils.argument('win_id', win_id=True)
     def adblock_update(self, win_id):
         """Update the adblock block lists.
 
@@ -208,7 +210,8 @@ class HostBlocker:
             else:
                 fobj = io.BytesIO()
                 fobj.name = 'adblock: ' + url.host()
-                download = download_manager.get(url, fileobj=fobj,
+                target = usertypes.FileObjDownloadTarget(fobj)
+                download = download_manager.get(url, target=target,
                                                 auto_remove=True)
                 self._in_progress.append(download)
                 download.finished.connect(
@@ -273,11 +276,13 @@ class HostBlocker:
     def on_config_changed(self):
         """Update files when the config changed."""
         urls = config.get('content', 'host-block-lists')
-        if urls is None:
+        if urls is None and self._local_hosts_file is not None:
             try:
                 os.remove(self._local_hosts_file)
-            except OSError:
-                log.misc.exception("Failed to delete hosts file.")
+            except FileNotFoundError:
+                pass
+            except OSError as e:
+                log.misc.exception("Failed to delete hosts file: {}".format(e))
 
     def on_download_finished(self, download):
         """Check if all downloads are finished and if so, trigger reading.

qutebrowser/browser/browsertab.py

@@ -0,0 +1,641 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Base class for a wrapper over QWebView/QWebEngineView."""
+
+import itertools
+
+from PyQt5.QtCore import pyqtSignal, pyqtSlot, QUrl, QObject, QPoint
+from PyQt5.QtGui import QIcon
+from PyQt5.QtWidgets import QWidget, QLayout
+
+from qutebrowser.keyinput import modeman
+from qutebrowser.config import config
+from qutebrowser.utils import utils, objreg, usertypes, message, log, qtutils
+
+
+tab_id_gen = itertools.count(0)
+
+
+def create(win_id, parent=None):
+    """Get a QtWebKit/QtWebEngine tab object.
+
+    Args:
+        win_id: The window ID where the tab will be shown.
+        parent: The Qt parent to set.
+    """
+    # Importing modules here so we don't depend on QtWebEngine without the
+    # argument and to avoid circular imports.
+    mode_manager = modeman.instance(win_id)
+    if objreg.get('args').backend == 'webengine':
+        from qutebrowser.browser.webengine import webenginetab
+        tab_class = webenginetab.WebEngineTab
+    else:
+        from qutebrowser.browser.webkit import webkittab
+        tab_class = webkittab.WebKitTab
+    return tab_class(win_id=win_id, mode_manager=mode_manager, parent=parent)
+
+
+class WebTabError(Exception):
+
+    """Base class for various errors."""
+
+
+class WrapperLayout(QLayout):
+
+    """A Qt layout which simply wraps a single widget.
+
+    This is used so the widget is hidden behind a AbstractTab API and can't
+    easily be accidentally accessed.
+    """
+
+    def __init__(self, widget, parent=None):
+        super().__init__(parent)
+        self._widget = widget
+
+    def addItem(self, _widget):
+        raise AssertionError("Should never be called!")
+
+    def sizeHint(self):
+        return self._widget.sizeHint()
+
+    def itemAt(self, _index):  # pragma: no cover
+        # For some reason this sometimes gets called by Qt.
+        return None
+
+    def takeAt(self, _index):
+        raise AssertionError("Should never be called!")
+
+    def setGeometry(self, rect):
+        self._widget.setGeometry(rect)
+
+
+class TabData:
+
+    """A simple namespace with a fixed set of attributes.
+
+    Attributes:
+        keep_icon: Whether the (e.g. cloned) icon should not be cleared on page
+                   load.
+        inspector: The QWebInspector used for this webview.
+        viewing_source: Set if we're currently showing a source view.
+    """
+
+    __slots__ = ['keep_icon', 'viewing_source', 'inspector']
+
+    def __init__(self):
+        self.keep_icon = False
+        self.viewing_source = False
+        self.inspector = None
+
+
+class AbstractPrinting:
+
+    """Attribute of AbstractTab for printing the page."""
+
+    def __init__(self):
+        self._widget = None
+
+    def check_pdf_support(self):
+        raise NotImplementedError
+
+    def check_printer_support(self):
+        raise NotImplementedError
+
+    def to_pdf(self, filename):
+        raise NotImplementedError
+
+    def to_printer(self, printer):
+        raise NotImplementedError
+
+
+class AbstractSearch(QObject):
+
+    """Attribute of AbstractTab for doing searches.
+
+    Attributes:
+        text: The last thing this view was searched for.
+        _flags: The flags of the last search (needs to be set by subclasses).
+        _widget: The underlying WebView widget.
+    """
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._widget = None
+        self.text = None
+
+    def search(self, text, *, ignore_case=False, reverse=False,
+               result_cb=None):
+        """Find the given text on the page.
+
+        Args:
+            text: The text to search for.
+            ignore_case: Search case-insensitively. (True/False/'smart')
+            reverse: Reverse search direction.
+            result_cb: Called with a bool indicating whether a match was found.
+        """
+        raise NotImplementedError
+
+    def clear(self):
+        """Clear the current search."""
+        raise NotImplementedError
+
+    def prev_result(self, *, result_cb=None):
+        """Go to the previous result of the current search.
+
+        Args:
+            result_cb: Called with a bool indicating whether a match was found.
+        """
+        raise NotImplementedError
+
+    def next_result(self, *, result_cb=None):
+        """Go to the next result of the current search.
+
+        Args:
+            result_cb: Called with a bool indicating whether a match was found.
+        """
+        raise NotImplementedError
+
+
+class AbstractZoom(QObject):
+
+    """Attribute of AbstractTab for controlling zoom.
+
+    Attributes:
+        _neighborlist: A NeighborList with the zoom levels.
+        _default_zoom_changed: Whether the zoom was changed from the default.
+    """
+
+    def __init__(self, win_id, parent=None):
+        super().__init__(parent)
+        self._widget = None
+        self._win_id = win_id
+        self._default_zoom_changed = False
+        self._init_neighborlist()
+        objreg.get('config').changed.connect(self._on_config_changed)
+
+        # # FIXME:qtwebengine is this needed?
+        # # For some reason, this signal doesn't get disconnected automatically
+        # # when the WebView is destroyed on older PyQt versions.
+        # # See https://github.com/The-Compiler/qutebrowser/issues/390
+        # self.destroyed.connect(functools.partial(
+        #     cfg.changed.disconnect, self.init_neighborlist))
+
+    @pyqtSlot(str, str)
+    def _on_config_changed(self, section, option):
+        if section == 'ui' and option in ['zoom-levels', 'default-zoom']:
+            if not self._default_zoom_changed:
+                factor = float(config.get('ui', 'default-zoom')) / 100
+                self._set_factor_internal(factor)
+            self._default_zoom_changed = False
+            self._init_neighborlist()
+
+    def _init_neighborlist(self):
+        """Initialize self._neighborlist."""
+        levels = config.get('ui', 'zoom-levels')
+        self._neighborlist = usertypes.NeighborList(
+            levels, mode=usertypes.NeighborList.Modes.edge)
+        self._neighborlist.fuzzyval = config.get('ui', 'default-zoom')
+
+    def offset(self, offset):
+        """Increase/Decrease the zoom level by the given offset.
+
+        Args:
+            offset: The offset in the zoom level list.
+
+        Return:
+            The new zoom percentage.
+        """
+        level = self._neighborlist.getitem(offset)
+        self.set_factor(float(level) / 100, fuzzyval=False)
+        return level
+
+    def set_factor(self, factor, *, fuzzyval=True):
+        """Zoom to a given zoom factor.
+
+        Args:
+            factor: The zoom factor as float.
+            fuzzyval: Whether to set the NeighborLists fuzzyval.
+        """
+        if fuzzyval:
+            self._neighborlist.fuzzyval = int(factor * 100)
+        if factor < 0:
+            raise ValueError("Can't zoom to factor {}!".format(factor))
+        self._default_zoom_changed = True
+        self._set_factor_internal(factor)
+
+    def factor(self):
+        raise NotImplementedError
+
+    def set_default(self):
+        default_zoom = config.get('ui', 'default-zoom')
+        self._set_factor_internal(float(default_zoom) / 100)
+
+    @pyqtSlot(QPoint)
+    def _on_mouse_wheel_zoom(self, delta):
+        """Handle zooming via mousewheel requested by the web view."""
+        divider = config.get('input', 'mouse-zoom-divider')
+        factor = self.factor() + delta.y() / divider
+        if factor < 0:
+            return
+        perc = int(100 * factor)
+        message.info(self._win_id, "Zoom level: {}%".format(perc))
+        self._neighborlist.fuzzyval = perc
+        self._set_factor_internal(factor)
+        self._default_zoom_changed = True
+
+
+class AbstractCaret(QObject):
+
+    """Attribute of AbstractTab for caret browsing."""
+
+    def __init__(self, win_id, tab, mode_manager, parent=None):
+        super().__init__(parent)
+        self._tab = tab
+        self._win_id = win_id
+        self._widget = None
+        self.selection_enabled = False
+        mode_manager.entered.connect(self._on_mode_entered)
+        mode_manager.left.connect(self._on_mode_left)
+
+    def _on_mode_entered(self, mode):
+        raise NotImplementedError
+
+    def _on_mode_left(self):
+        raise NotImplementedError
+
+    def move_to_next_line(self, count=1):
+        raise NotImplementedError
+
+    def move_to_prev_line(self, count=1):
+        raise NotImplementedError
+
+    def move_to_next_char(self, count=1):
+        raise NotImplementedError
+
+    def move_to_prev_char(self, count=1):
+        raise NotImplementedError
+
+    def move_to_end_of_word(self, count=1):
+        raise NotImplementedError
+
+    def move_to_next_word(self, count=1):
+        raise NotImplementedError
+
+    def move_to_prev_word(self, count=1):
+        raise NotImplementedError
+
+    def move_to_start_of_line(self):
+        raise NotImplementedError
+
+    def move_to_end_of_line(self):
+        raise NotImplementedError
+
+    def move_to_start_of_next_block(self, count=1):
+        raise NotImplementedError
+
+    def move_to_start_of_prev_block(self, count=1):
+        raise NotImplementedError
+
+    def move_to_end_of_next_block(self, count=1):
+        raise NotImplementedError
+
+    def move_to_end_of_prev_block(self, count=1):
+        raise NotImplementedError
+
+    def move_to_start_of_document(self):
+        raise NotImplementedError
+
+    def move_to_end_of_document(self):
+        raise NotImplementedError
+
+    def toggle_selection(self):
+        raise NotImplementedError
+
+    def drop_selection(self):
+        raise NotImplementedError
+
+    def has_selection(self):
+        raise NotImplementedError
+
+    def selection(self, html=False):
+        raise NotImplementedError
+
+    def follow_selected(self, *, tab=False):
+        raise NotImplementedError
+
+
+class AbstractScroller(QObject):
+
+    """Attribute of AbstractTab to manage scroll position."""
+
+    perc_changed = pyqtSignal(int, int)
+
+    def __init__(self, tab, parent=None):
+        super().__init__(parent)
+        self._tab = tab
+        self._widget = None
+
+    def _init_widget(self, widget):
+        self._widget = widget
+
+    def pos_px(self):
+        raise NotImplementedError
+
+    def pos_perc(self):
+        raise NotImplementedError
+
+    def to_perc(self, x=None, y=None):
+        raise NotImplementedError
+
+    def to_point(self, point):
+        raise NotImplementedError
+
+    def delta(self, x=0, y=0):
+        raise NotImplementedError
+
+    def delta_page(self, x=0, y=0):
+        raise NotImplementedError
+
+    def up(self, count=1):
+        raise NotImplementedError
+
+    def down(self, count=1):
+        raise NotImplementedError
+
+    def left(self, count=1):
+        raise NotImplementedError
+
+    def right(self, count=1):
+        raise NotImplementedError
+
+    def top(self):
+        raise NotImplementedError
+
+    def bottom(self):
+        raise NotImplementedError
+
+    def page_up(self, count=1):
+        raise NotImplementedError
+
+    def page_down(self, count=1):
+        raise NotImplementedError
+
+    def at_top(self):
+        raise NotImplementedError
+
+    def at_bottom(self):
+        raise NotImplementedError
+
+
+class AbstractHistory:
+
+    """The history attribute of a AbstractTab."""
+
+    def __init__(self, tab):
+        self._tab = tab
+        self._history = None
+
+    def __len__(self):
+        return len(self._history)
+
+    def __iter__(self):
+        return iter(self._history.items())
+
+    def current_idx(self):
+        raise NotImplementedError
+
+    def back(self):
+        raise NotImplementedError
+
+    def forward(self):
+        raise NotImplementedError
+
+    def can_go_back(self):
+        raise NotImplementedError
+
+    def can_go_forward(self):
+        raise NotImplementedError
+
+    def serialize(self):
+        """Serialize into an opaque format understood by self.deserialize."""
+        raise NotImplementedError
+
+    def deserialize(self, data):
+        """Serialize from a format produced by self.serialize."""
+        raise NotImplementedError
+
+    def load_items(self, items):
+        """Deserialize from a list of WebHistoryItems."""
+        raise NotImplementedError
+
+
+class AbstractTab(QWidget):
+
+    """A wrapper over the given widget to hide its API and expose another one.
+
+    We use this to unify QWebView and QWebEngineView.
+
+    Attributes:
+        history: The AbstractHistory for the current tab.
+        registry: The ObjectRegistry associated with this tab.
+
+        _load_status: loading status of this page
+                      Accessible via load_status() method.
+        _has_ssl_errors: Whether SSL errors happened.
+                         Needs to be set by subclasses.
+
+        for properties, see WebView/WebEngineView docs.
+
+    Signals:
+        See related Qt signals.
+
+        new_tab_requested: Emitted when a new tab should be opened with the
+                           given URL.
+        load_status_changed: The loading status changed
+    """
+
+    window_close_requested = pyqtSignal()
+    link_hovered = pyqtSignal(str)
+    load_started = pyqtSignal()
+    load_progress = pyqtSignal(int)
+    load_finished = pyqtSignal(bool)
+    icon_changed = pyqtSignal(QIcon)
+    title_changed = pyqtSignal(str)
+    load_status_changed = pyqtSignal(str)
+    new_tab_requested = pyqtSignal(QUrl)
+    url_changed = pyqtSignal(QUrl)
+    shutting_down = pyqtSignal()
+
+    def __init__(self, win_id, parent=None):
+        self.win_id = win_id
+        self.tab_id = next(tab_id_gen)
+        super().__init__(parent)
+
+        self.registry = objreg.ObjectRegistry()
+        tab_registry = objreg.get('tab-registry', scope='window',
+                                  window=win_id)
+        tab_registry[self.tab_id] = self
+        objreg.register('tab', self, registry=self.registry)
+
+        # self.history = AbstractHistory(self)
+        # self.scroller = AbstractScroller(self, parent=self)
+        # self.caret = AbstractCaret(win_id=win_id, tab=self, mode_manager=...,
+        #                            parent=self)
+        # self.zoom = AbstractZoom(win_id=win_id)
+        # self.search = AbstractSearch(parent=self)
+        # self.printing = AbstractPrinting()
+        self.data = TabData()
+        self._layout = None
+        self._widget = None
+        self._progress = 0
+        self._has_ssl_errors = False
+        self._load_status = usertypes.LoadStatus.none
+        self.backend = None
+
+    def _set_widget(self, widget):
+        # pylint: disable=protected-access
+        self._layout = WrapperLayout(widget, self)
+        self._widget = widget
+        self.history._history = widget.history()
+        self.scroller._init_widget(widget)
+        self.caret._widget = widget
+        self.zoom._widget = widget
+        self.search._widget = widget
+        self.printing._widget = widget
+        widget.mouse_wheel_zoom.connect(self.zoom._on_mouse_wheel_zoom)
+        widget.setParent(self)
+        self.setFocusProxy(widget)
+
+    def _set_load_status(self, val):
+        """Setter for load_status."""
+        if not isinstance(val, usertypes.LoadStatus):
+            raise TypeError("Type {} is no LoadStatus member!".format(val))
+        log.webview.debug("load status for {}: {}".format(repr(self), val))
+        self._load_status = val
+        self.load_status_changed.emit(val.name)
+
+    @pyqtSlot(QUrl)
+    def _on_url_changed(self, url):
+        """Update title when URL has changed and no title is available."""
+        if url.isValid() and not self.title():
+            self.title_changed.emit(url.toDisplayString())
+        self.url_changed.emit(url)
+
+    @pyqtSlot()
+    def _on_load_started(self):
+        self._progress = 0
+        self._has_ssl_errors = False
+        self.data.viewing_source = False
+        self._set_load_status(usertypes.LoadStatus.loading)
+        self.load_started.emit()
+
+    @pyqtSlot(bool)
+    def _on_load_finished(self, ok):
+        if ok and not self._has_ssl_errors:
+            if self.url().scheme() == 'https':
+                self._set_load_status(usertypes.LoadStatus.success_https)
+            else:
+                self._set_load_status(usertypes.LoadStatus.success)
+
+        elif ok:
+            self._set_load_status(usertypes.LoadStatus.warn)
+        else:
+            self._set_load_status(usertypes.LoadStatus.error)
+        self.load_finished.emit(ok)
+        if not self.title():
+            self.title_changed.emit(self.url().toDisplayString())
+
+    @pyqtSlot(int)
+    def _on_load_progress(self, perc):
+        self._progress = perc
+        self.load_progress.emit(perc)
+
+    @pyqtSlot()
+    def _on_ssl_errors(self):
+        self._has_ssl_errors = True
+
+    def url(self):
+        raise NotImplementedError
+
+    def progress(self):
+        return self._progress
+
+    def load_status(self):
+        return self._load_status
+
+    def _openurl_prepare(self, url):
+        qtutils.ensure_valid(url)
+        self.title_changed.emit(url.toDisplayString())
+
+    def openurl(self, url):
+        raise NotImplementedError
+
+    def reload(self, *, force=False):
+        raise NotImplementedError
+
+    def stop(self):
+        raise NotImplementedError
+
+    def clear_ssl_errors(self):
+        raise NotImplementedError
+
+    def dump_async(self, callback, *, plain=False):
+        """Dump the current page to a file ascync.
+
+        The given callback will be called with the result when dumping is
+        complete.
+        """
+        raise NotImplementedError
+
+    def run_js_async(self, code, callback=None):
+        """Run javascript async.
+
+        The given callback will be called with the result when running JS is
+        complete.
+        """
+        raise NotImplementedError
+
+    def run_js_blocking(self, code):
+        """Run javascript and block.
+
+        This returns the result to the caller. Its use should be avoided when
+        possible as it runs a local event loop for QtWebEngine.
+        """
+        raise NotImplementedError
+
+    def shutdown(self):
+        raise NotImplementedError
+
+    def title(self):
+        raise NotImplementedError
+
+    def icon(self):
+        raise NotImplementedError
+
+    def set_html(self, html, base_url):
+        raise NotImplementedError
+
+    def __repr__(self):
+        try:
+            url = utils.elide(self.url().toDisplayString(QUrl.EncodeUnicode),
+                              100)
+        except AttributeError:
+            url = '<AttributeError>'
+        return utils.get_repr(self, tab_id=self.tab_id, url=url)

qutebrowser/browser/downloadview.py

@@ -25,7 +25,7 @@ import sip
 from PyQt5.QtCore import pyqtSlot, QSize, Qt, QTimer
 from PyQt5.QtWidgets import QListView, QSizePolicy, QMenu
 
-from qutebrowser.browser import downloads
+from qutebrowser.browser.webkit import downloads
 from qutebrowser.config import style
 from qutebrowser.utils import qtutils, utils, objreg
 

qutebrowser/browser/hints.py

@@ -23,7 +23,7 @@ import collections
 import functools
 import math
 import re
-import string
+from string import ascii_lowercase
 
 from PyQt5.QtCore import (pyqtSignal, pyqtSlot, QObject, QEvent, Qt, QUrl,
                           QTimer)
@@ -33,10 +33,9 @@ from PyQt5.QtWebKitWidgets import QWebPage
 
 from qutebrowser.config import config
 from qutebrowser.keyinput import modeman, modeparsers
-from qutebrowser.browser import webelem
+from qutebrowser.browser.webkit import webelem
 from qutebrowser.commands import userscripts, cmdexc, cmdutils, runners
 from qutebrowser.utils import usertypes, log, qtutils, message, objreg, utils
-from qutebrowser.misc import guiprocess
 
 
 ElemTuple = collections.namedtuple('ElemTuple', ['elem', 'label'])
@@ -53,7 +52,6 @@ class WordHintingError(Exception):
     """Exception raised on errors during word hinting."""
 
 
-@pyqtSlot(usertypes.KeyMode)
 def on_mode_entered(mode, win_id):
     """Stop hinting when insert mode was entered."""
     if mode == usertypes.KeyMode.insert:
@@ -68,7 +66,9 @@ class HintContext:
         frames: The QWebFrames to use.
         destroyed_frames: id()'s of QWebFrames which have been destroyed.
                           (Workaround for https://github.com/The-Compiler/qutebrowser/issues/152)
+        all_elems: A list of all (elem, label) namedtuples ever created.
         elems: A mapping from key strings to (elem, label) namedtuples.
+               May contain less elements than `all_elems` due to filtering.
         baseurl: The URL of the current page.
         target: What to do with the opened links.
                 normal/current/tab/tab_fg/tab_bg/window: Get passed to
@@ -83,10 +83,12 @@ class HintContext:
         args: Custom arguments for userscript/spawn
         rapid: Whether to do rapid hinting.
         mainframe: The main QWebFrame where we started hinting in.
+        tab: The WebTab object we started hinting in.
         group: The group of web elements to hint.
     """
 
     def __init__(self):
+        self.all_elems = []
         self.elems = {}
         self.target = None
         self.baseurl = None
@@ -96,6 +98,7 @@ class HintContext:
         self.destroyed_frames = []
         self.args = []
         self.mainframe = None
+        self.tab = None
         self.group = None
 
     def get_args(self, urlstr):
@@ -118,6 +121,7 @@ class HintManager(QObject):
         _context: The HintContext for the current invocation.
         _win_id: The window ID this HintManager is associated with.
         _tab_id: The tab ID this HintManager is associated with.
+        _filterstr: Used to save the filter string for restoring in rapid mode.
 
     Signals:
         mouse_event: Mouse event to be posted in the web view.
@@ -154,6 +158,7 @@ class HintManager(QObject):
         self._win_id = win_id
         self._tab_id = tab_id
         self._context = None
+        self._filterstr = None
         self._word_hinter = WordHinter()
         mode_manager = objreg.get('mode-manager', scope='window',
                                   window=win_id)
@@ -169,7 +174,7 @@ class HintManager(QObject):
 
     def _cleanup(self):
         """Clean up after hinting."""
-        for elem in self._context.elems.values():
+        for elem in self._context.all_elems:
             try:
                 elem.label.removeFromDocument()
             except webelem.IsNullError:
@@ -195,6 +200,7 @@ class HintManager(QObject):
                                     window=self._win_id)
         message_bridge.maybe_reset_text(text)
         self._context = None
+        self._filterstr = None
 
     def _hint_strings(self, elems):
         """Calculate the hint strings for elems.
@@ -219,7 +225,7 @@ class HintManager(QObject):
         else:
             chars = config.get('hints', 'chars')
         min_chars = config.get('hints', 'min-chars')
-        if config.get('hints', 'scatter'):
+        if config.get('hints', 'scatter') and hint_mode != 'number':
             return self._hint_scattered(min_chars, chars, elems)
         else:
             return self._hint_linear(min_chars, chars, elems)
@@ -351,7 +357,7 @@ class HintManager(QObject):
             ('display', 'inline !important'),
             ('z-index', '{} !important'.format(int(2 ** 32 / 2 - 1))),
             ('pointer-events', 'none !important'),
-            ('position', 'absolute !important'),
+            ('position', 'fixed !important'),
             ('color', config.get('colors', 'hints.fg') + ' !important'),
             ('background', config.get('colors', 'hints.bg') + ' !important'),
             ('font', config.get('fonts', 'hints') + ' !important'),
@@ -377,19 +383,16 @@ class HintManager(QObject):
             elem: The QWebElement to set the style attributes for.
             label: The label QWebElement.
         """
-        rect = elem.geometry()
+        no_js = config.get('hints', 'find-implementation') != 'javascript'
+        rect = elem.rect_on_view(adjust_zoom=False, no_js=no_js)
         left = rect.x()
         top = rect.y()
-        zoom = elem.webFrame().zoomFactor()
-        if not config.get('ui', 'zoom-text-only'):
-            left /= zoom
-            top /= zoom
-        log.hints.vdebug("Drawing label '{!r}' at {}/{} for element '{!r}', "
-                         "zoom level {}".format(label, left, top, elem, zoom))
+        log.hints.vdebug("Drawing label '{!r}' at {}/{} for element '{!r}' "
+                         "(no_js: {})".format(label, left, top, elem, no_js))
         label.setStyleProperty('left', '{}px !important'.format(left))
         label.setStyleProperty('top', '{}px !important'.format(top))
 
-    def _draw_label(self, elem, text):
+    def _draw_label(self, elem, string):
         """Draw a hint label over an element.
 
         Args:
@@ -414,7 +417,7 @@ class HintManager(QObject):
         label = webelem.WebElementWrapper(parent.lastChild())
         label['class'] = 'qutehint'
         self._set_style_properties(elem, label)
-        label.setPlainText(text)
+        label.setPlainText(string)
         return label
 
     def _show_url_error(self):
@@ -441,14 +444,22 @@ class HintManager(QObject):
             target_mapping[Target.tab] = usertypes.ClickTarget.tab_bg
         else:
             target_mapping[Target.tab] = usertypes.ClickTarget.tab
-        # FIXME Instead of clicking the center, we could have nicer heuristics.
-        # e.g. parse (-webkit-)border-radius correctly and click text fields at
-        # the bottom right, and everything else on the top left or so.
-        # https://github.com/The-Compiler/qutebrowser/issues/70
-        pos = elem.rect_on_view().center()
+
+        # Click the center of the largest square fitting into the top/left
+        # corner of the rectangle, this will help if part of the <a> element
+        # is hidden behind other elements
+        # https://github.com/The-Compiler/qutebrowser/issues/1005
+        rect = elem.rect_on_view()
+        if rect.width() > rect.height():
+            rect.setWidth(rect.height())
+        else:
+            rect.setHeight(rect.width())
+        pos = rect.center()
+
         action = "Hovering" if context.target == Target.hover else "Clicking"
-        log.hints.debug("{} on '{}' at {}/{}".format(
-            action, elem, pos.x(), pos.y()))
+        log.hints.debug("{} on '{}' at position {}".format(
+            action, elem.debug_text(), pos))
+
         self.start_hinting.emit(target_mapping[context.target])
         if context.target in [Target.tab, Target.tab_fg, Target.tab_bg,
                               Target.window]:
@@ -466,6 +477,13 @@ class HintManager(QObject):
                 QMouseEvent(QEvent.MouseButtonRelease, pos, Qt.LeftButton,
                             Qt.NoButton, modifiers),
             ]
+
+        if context.target in [Target.normal, Target.current]:
+            # Set the pre-jump mark ', so we can jump back here after following
+            tabbed_browser = objreg.get('tabbed-browser', scope='window',
+                                        window=self._win_id)
+            tabbed_browser.set_mark("'")
+
         if context.target == Target.current:
             elem.remove_blank_target()
         for evt in events:
@@ -483,7 +501,9 @@ class HintManager(QObject):
             url: The URL to open as a QUrl.
             context: The HintContext to use.
         """
-        sel = context.target == Target.yank_primary
+        sel = (context.target == Target.yank_primary and
+               utils.supports_selection())
+
         urlstr = url.toString(QUrl.FullyEncoded | QUrl.RemovePassword)
         utils.set_clipboard(urlstr, selection=sel)
 
@@ -551,7 +571,6 @@ class HintManager(QObject):
         """
         cmd = context.args[0]
         args = context.args[1:]
-        frame = context.mainframe
         env = {
             'QUTE_MODE': 'hints',
             'QUTE_SELECTED_TEXT': str(elem),
@@ -560,8 +579,12 @@ class HintManager(QObject):
         url = self._resolve_url(elem, context.baseurl)
         if url is not None:
             env['QUTE_URL'] = url.toString(QUrl.FullyEncoded)
-        env.update(userscripts.store_source(frame))
-        userscripts.run(cmd, *args, win_id=self._win_id, env=env)
+
+        try:
+            userscripts.run_async(context.tab, cmd, *args, win_id=self._win_id,
+                                  env=env)
+        except userscripts.UnsupportedError as e:
+            message.error(self._win_id, str(e), immediately=True)
 
     def _spawn(self, url, context):
         """Spawn a simple command from a hint.
@@ -572,9 +595,8 @@ class HintManager(QObject):
         """
         urlstr = url.toString(QUrl.FullyEncoded | QUrl.RemovePassword)
         args = context.get_args(urlstr)
-        cmd, *args = args
-        proc = guiprocess.GUIProcess(self._win_id, what='command', parent=self)
-        proc.start(cmd, args)
+        commandrunner = runners.CommandRunner(self._win_id)
+        commandrunner.run_safely('spawn ' + ' '.join(args))
 
     def _resolve_url(self, elem, baseurl):
         """Resolve a URL and check if we want to keep it.
@@ -586,9 +608,9 @@ class HintManager(QObject):
         Return:
             A QUrl with the absolute URL, or None.
         """
-        for attr in ('href', 'src'):
+        for attr in ['href', 'src']:
             if attr in elem:
-                text = elem[attr]
+                text = elem[attr].strip()
                 break
         else:
             return None
@@ -604,8 +626,7 @@ class HintManager(QObject):
     def _find_prevnext(self, frame, prev=False):
         """Find a prev/next element in frame."""
         # First check for <link rel="prev(ious)|next">
-        elems = frame.findAllElements(
-            webelem.SELECTORS[webelem.Group.links])
+        elems = frame.findAllElements(webelem.SELECTORS[webelem.Group.links])
         rel_values = ('prev', 'previous') if prev else ('next')
         for e in elems:
             e = webelem.WebElementWrapper(e)
@@ -656,8 +677,8 @@ class HintManager(QObject):
         """
         if not isinstance(target, Target):
             raise TypeError("Target {} is no Target member!".format(target))
-        if target in (Target.userscript, Target.spawn, Target.run,
-                      Target.fill):
+        if target in [Target.userscript, Target.spawn, Target.run,
+                      Target.fill]:
             if not args:
                 raise cmdexc.CommandError(
                     "'args' is required with target userscript/spawn/run/"
@@ -681,15 +702,27 @@ class HintManager(QObject):
         elems = [e for e in elems if filterfunc(e)]
         if not elems:
             raise cmdexc.CommandError("No elements found.")
-        hints = self._hint_strings(elems)
-        log.hints.debug("hints: {}".format(', '.join(hints)))
-        for e, hint in zip(elems, hints):
-            label = self._draw_label(e, hint)
-            self._context.elems[hint] = ElemTuple(e, label)
+        strings = self._hint_strings(elems)
+        log.hints.debug("hints: {}".format(', '.join(strings)))
+        for e, string in zip(elems, strings):
+            label = self._draw_label(e, string)
+            elem = ElemTuple(e, label)
+            self._context.all_elems.append(elem)
+            self._context.elems[string] = elem
         keyparsers = objreg.get('keyparsers', scope='window',
                                 window=self._win_id)
         keyparser = keyparsers[usertypes.KeyMode.hint]
-        keyparser.update_bindings(hints)
+        keyparser.update_bindings(strings)
+
+    def _filter_matches(self, filterstr, elemstr):
+        """Return True if `filterstr` matches `elemstr`."""
+        # Empty string and None always match
+        if not filterstr:
+            return True
+        filterstr = filterstr.casefold()
+        elemstr = elemstr.casefold()
+        # Do multi-word matching
+        return all(word in elemstr for word in filterstr.split())
 
     def follow_prevnext(self, frame, baseurl, prev=False, tab=False,
                         background=False, window=False):
@@ -724,14 +757,16 @@ class HintManager(QObject):
                                         window=self._win_id)
             tabbed_browser.tabopen(url, background=background)
         else:
-            webview = objreg.get('webview', scope='tab', window=self._win_id,
-                                 tab=self._tab_id)
-            webview.openurl(url)
+            tab = objreg.get('tab', scope='tab', window=self._win_id,
+                             tab=self._tab_id)
+            tab.openurl(url)
 
     @cmdutils.register(instance='hintmanager', scope='tab', name='hint',
-                       win_id='win_id')
+                       star_args_optional=True, maxsplit=2,
+                       backend=usertypes.Backend.QtWebKit)
+    @cmdutils.argument('win_id', win_id=True)
     def start(self, rapid=False, group=webelem.Group.all, target=Target.normal,
-              *args: {'nargs': '*'}, win_id):
+              *args, win_id):
         """Start hinting.
 
         Args:
@@ -743,6 +778,7 @@ class HintManager(QObject):
                 - `all`: All clickable elements.
                 - `links`: Only links.
                 - `images`: Only images.
+                - `inputs`: Only input fields.
 
             target: What to do with the selected element.
 
@@ -781,10 +817,12 @@ class HintManager(QObject):
         """
         tabbed_browser = objreg.get('tabbed-browser', scope='window',
                                     window=self._win_id)
-        widget = tabbed_browser.currentWidget()
-        if widget is None:
+        tab = tabbed_browser.currentWidget()
+        if tab is None:
             raise cmdexc.CommandError("No WebView available yet!")
-        mainframe = widget.page().mainFrame()
+        # FIXME:qtwebengine have a proper API for this
+        page = tab._widget.page()  # pylint: disable=protected-access
+        mainframe = page.mainFrame()
         if mainframe is None:
             raise cmdexc.CommandError("No frame focused!")
         mode_manager = objreg.get('mode-manager', scope='window',
@@ -795,7 +833,7 @@ class HintManager(QObject):
         if rapid:
             if target in [Target.tab_bg, Target.window, Target.run,
                           Target.hover, Target.userscript, Target.spawn,
-                          Target.download]:
+                          Target.download, Target.normal, Target.current]:
                 pass
             elif (target == Target.tab and
                   config.get('tabs', 'background-tabs')):
@@ -807,6 +845,7 @@ class HintManager(QObject):
 
         self._check_args(target, *args)
         self._context = HintContext()
+        self._context.tab = tab
         self._context.target = target
         self._context.rapid = rapid
         try:
@@ -833,53 +872,115 @@ class HintManager(QObject):
     def handle_partial_key(self, keystr):
         """Handle a new partial keypress."""
         log.hints.debug("Handling new keystring: '{}'".format(keystr))
-        for (text, elems) in self._context.elems.items():
+        for string, elem in self._context.elems.items():
             try:
-                if text.startswith(keystr):
-                    matched = text[:len(keystr)]
-                    rest = text[len(keystr):]
+                if string.startswith(keystr):
+                    matched = string[:len(keystr)]
+                    rest = string[len(keystr):]
                     match_color = config.get('colors', 'hints.fg.match')
-                    elems.label.setInnerXml(
+                    elem.label.setInnerXml(
                         '<font color="{}">{}</font>{}'.format(
                             match_color, matched, rest))
-                    if self._is_hidden(elems.label):
+                    if self._is_hidden(elem.label):
                         # hidden element which matches again -> show it
-                        self._show_elem(elems.label)
+                        self._show_elem(elem.label)
                 else:
                     # element doesn't match anymore -> hide it
-                    self._hide_elem(elems.label)
+                    self._hide_elem(elem.label)
             except webelem.IsNullError:
                 pass
 
+    def _filter_number_hints(self):
+        """Apply filters for numbered hints and renumber them.
+
+        Return:
+            Elements which are still visible
+        """
+        # renumber filtered hints
+        elems = []
+        for e in self._context.all_elems:
+            try:
+                if not self._is_hidden(e.label):
+                    elems.append(e)
+            except webelem.IsNullError:
+                pass
+        if not elems:
+            # Whoops, filtered all hints
+            modeman.leave(self._win_id, usertypes.KeyMode.hint,
+                          'all filtered')
+            return {}
+
+        strings = self._hint_strings(elems)
+        self._context.elems = {}
+        for elem, string in zip(elems, strings):
+            elem.label.setInnerXml(string)
+            self._context.elems[string] = elem
+        keyparsers = objreg.get('keyparsers', scope='window',
+                                window=self._win_id)
+        keyparser = keyparsers[usertypes.KeyMode.hint]
+        keyparser.update_bindings(strings, preserve_filter=True)
+
+        return self._context.elems
+
+    def _filter_non_number_hints(self):
+        """Apply filters for letter/word hints.
+
+        Return:
+            Elements which are still visible
+        """
+        visible = {}
+        for string, elem in self._context.elems.items():
+            try:
+                if not self._is_hidden(elem.label):
+                    visible[string] = elem
+            except webelem.IsNullError:
+                pass
+        if not visible:
+            # Whoops, filtered all hints
+            modeman.leave(self._win_id, usertypes.KeyMode.hint,
+                          'all filtered')
+        return visible
+
     def filter_hints(self, filterstr):
         """Filter displayed hints according to a text.
 
         Args:
-            filterstr: The string to filter with, or None to show all.
+            filterstr: The string to filter with, or None to use the filter
+                       from previous call (saved in `self._filterstr`). If
+                       `filterstr` is an empty string or if both `filterstr`
+                       and `self._filterstr` are None, all hints are shown.
         """
-        for elems in self._context.elems.values():
+        if filterstr is None:
+            filterstr = self._filterstr
+        else:
+            self._filterstr = filterstr
+
+        for elem in self._context.all_elems:
             try:
-                if (filterstr is None or
-                        filterstr.casefold() in str(elems.elem).casefold()):
-                    if self._is_hidden(elems.label):
+                if self._filter_matches(filterstr, str(elem.elem)):
+                    if self._is_hidden(elem.label):
                         # hidden element which matches again -> show it
-                        self._show_elem(elems.label)
+                        self._show_elem(elem.label)
                 else:
                     # element doesn't match anymore -> hide it
-                    self._hide_elem(elems.label)
+                    self._hide_elem(elem.label)
             except webelem.IsNullError:
                 pass
-        visible = {}
-        for k, e in self._context.elems.items():
-            try:
-                if not self._is_hidden(e.label):
-                    visible[k] = e
-            except webelem.IsNullError:
-                pass
-        if not visible:
-            # Whoops, filtered all hints
-            modeman.leave(self._win_id, usertypes.KeyMode.hint, 'all filtered')
-        elif len(visible) == 1 and config.get('hints', 'auto-follow'):
+
+        if config.get('hints', 'mode') == 'number':
+            visible = self._filter_number_hints()
+        else:
+            visible = self._filter_non_number_hints()
+
+        if (len(visible) == 1 and
+                config.get('hints', 'auto-follow') and
+                filterstr is not None):
+            # apply auto-follow-timeout
+            timeout = config.get('hints', 'auto-follow-timeout')
+            keyparsers = objreg.get('keyparsers', scope='window',
+                                    window=self._win_id)
+            normal_parser = keyparsers[usertypes.KeyMode.normal]
+            normal_parser.set_inhibited_timeout(timeout)
             # unpacking gets us the first (and only) key in the dict.
             self.fire(*visible)
 
@@ -917,30 +1018,31 @@ class HintManager(QObject):
         }
         elem = self._context.elems[keystr].elem
         if elem.webFrame() is None:
-            message.error(self._win_id, "This element has no webframe.",
+            message.error(self._win_id,
+                          "This element has no webframe.",
                           immediately=True)
             return
         if self._context.target in elem_handlers:
-            handler = functools.partial(
-                elem_handlers[self._context.target], elem, self._context)
+            handler = functools.partial(elem_handlers[self._context.target],
+                                        elem, self._context)
         elif self._context.target in url_handlers:
             url = self._resolve_url(elem, self._context.baseurl)
             if url is None:
                 self._show_url_error()
                 return
-            handler = functools.partial(
-                url_handlers[self._context.target], url, self._context)
+            handler = functools.partial(url_handlers[self._context.target],
+                                        url, self._context)
         else:
             raise ValueError("No suitable handler found!")
         if not self._context.rapid:
             modeman.maybe_leave(self._win_id, usertypes.KeyMode.hint,
                                 'followed')
         else:
-            # Show all hints again
+            # Reset filtering
             self.filter_hints(None)
             # Undo keystring highlighting
-            for (text, elems) in self._context.elems.items():
-                elems.label.setInnerXml(text)
+            for string, elem in self._context.elems.items():
+                elem.label.setInnerXml(string)
         handler()
 
     @cmdutils.register(instance='hintmanager', scope='tab', hide=True,
@@ -964,13 +1066,13 @@ class HintManager(QObject):
     def on_contents_size_changed(self, _size):
         """Reposition hints if contents size changed."""
         log.hints.debug("Contents size changed...!")
-        for elems in self._context.elems.values():
+        for e in self._context.all_elems:
             try:
-                if elems.elem.webFrame() is None:
+                if e.elem.webFrame() is None:
                     # This sometimes happens for some reason...
-                    elems.label.removeFromDocument()
+                    e.label.removeFromDocument()
                     continue
-                self._set_style_position(elems.elem, elems.label)
+                self._set_style_position(e.elem, e.label)
             except webelem.IsNullError:
                 pass
 
@@ -997,14 +1099,17 @@ class WordHinter:
     def __init__(self):
         # will be initialized on first use.
         self.words = set()
+        self.dictionary = None
 
     def ensure_initialized(self):
-        """Generate the used words if yet uninialized."""
-        if not self.words:
-            dictionary = config.get("hints", "dictionary")
+        """Generate the used words if yet uninitialized."""
+        dictionary = config.get("hints", "dictionary")
+        if not self.words or self.dictionary != dictionary:
+            self.words.clear()
+            self.dictionary = dictionary
             try:
                 with open(dictionary, encoding="UTF-8") as wordfile:
-                    alphabet = set(string.ascii_lowercase)
+                    alphabet = set(ascii_lowercase)
                     hints = set()
                     lines = (line.rstrip().lower() for line in wordfile)
                     for word in lines:
@@ -1034,13 +1139,11 @@ class WordHinter:
             "text": str,
         }
 
-        extractable_attrs = collections.defaultdict(
-            list, {
-                "IMG": ["alt", "title", "src"],
-                "A": ["title", "href", "text"],
-                "INPUT": ["name"]
-            }
-        )
+        extractable_attrs = collections.defaultdict(list, {
+            "IMG": ["alt", "title", "src"],
+            "A": ["title", "href", "text"],
+            "INPUT": ["name"]
+        })
 
         return (attr_extractors[attr](elem)
                 for attr in extractable_attrs[elem.tagName()]
@@ -1058,15 +1161,19 @@ class WordHinter:
                 yield candidate[match.start():match.end()].lower()
 
     def any_prefix(self, hint, existing):
-        return any(hint.startswith(e) or e.startswith(hint)
-                   for e in existing)
+        return any(hint.startswith(e) or e.startswith(hint) for e in existing)
 
-    def new_hint_for(self, elem, existing):
+    def filter_prefixes(self, hints, existing):
+        return (h for h in hints if not self.any_prefix(h, existing))
+
+    def new_hint_for(self, elem, existing, fallback):
         """Return a hint for elem, not conflicting with the existing."""
         new = self.tag_words_to_hints(self.extract_tag_words(elem))
-        no_prefixes = (h for h in new if not self.any_prefix(h, existing))
+        new_no_prefixes = self.filter_prefixes(new, existing)
+        fallback_no_prefixes = self.filter_prefixes(fallback, existing)
         # either the first good, or None
-        return next(no_prefixes, None)
+        return (next(new_no_prefixes, None) or
+                next(fallback_no_prefixes, None))
 
     def hint(self, elems):
         """Produce hint labels based on the html tags.
@@ -1086,7 +1193,9 @@ class WordHinter:
         used_hints = set()
         words = iter(self.words)
         for elem in elems:
-            hint = self.new_hint_for(elem, used_hints) or next(words)
+            hint = self.new_hint_for(elem, used_hints, words)
+            if not hint:
+                raise WordHintingError("Not enough words in the dictionary.")
             used_hints.add(hint)
             hints.append(hint)
         return hints

qutebrowser/browser/history.py

@@ -1,211 +0,0 @@
-# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
-
-# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
-#
-# This file is part of qutebrowser.
-#
-# qutebrowser is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# qutebrowser is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
-
-"""Simple history which gets written to disk."""
-
-import time
-import collections
-
-from PyQt5.QtCore import pyqtSignal, QUrl
-from PyQt5.QtWebKit import QWebHistoryInterface
-
-from qutebrowser.utils import utils, objreg, standarddir, log
-from qutebrowser.config import config
-from qutebrowser.misc import lineparser
-
-
-class HistoryEntry:
-
-    """A single entry in the web history.
-
-    Attributes:
-        atime: The time the page was accessed.
-        url: The URL which was accessed as QUrl.
-        url_string: The URL which was accessed as string.
-    """
-
-    def __init__(self, atime, url):
-        self.atime = float(atime)
-        self.url = QUrl(url)
-        self.url_string = url
-
-    def __repr__(self):
-        return utils.get_repr(self, constructor=True, atime=self.atime,
-                              url=self.url.toDisplayString())
-
-    def __str__(self):
-        return '{} {}'.format(int(self.atime), self.url_string)
-
-
-class WebHistory(QWebHistoryInterface):
-
-    """A QWebHistoryInterface which supports being written to disk.
-
-    Attributes:
-        _lineparser: The AppendLineParser used to save the history.
-        _history_dict: An OrderedDict of URLs read from the on-disk history.
-        _new_history: A list of HistoryEntry items of the current session.
-        _saved_count: How many HistoryEntries have been written to disk.
-        _initial_read_started: Whether async_read was called.
-        _initial_read_done: Whether async_read has completed.
-        _temp_history: OrderedDict of temporary history entries before
-                       async_read was called.
-
-    Signals:
-        add_completion_item: Emitted before a new HistoryEntry is added.
-                             arg: The new HistoryEntry.
-        item_added: Emitted after a new HistoryEntry is added.
-                    arg: The new HistoryEntry.
-    """
-
-    add_completion_item = pyqtSignal(HistoryEntry)
-    item_added = pyqtSignal(HistoryEntry)
-    async_read_done = pyqtSignal()
-
-    def __init__(self, parent=None):
-        super().__init__(parent)
-        self._initial_read_started = False
-        self._initial_read_done = False
-        self._lineparser = lineparser.AppendLineParser(
-            standarddir.data(), 'history', parent=self)
-        self._history_dict = collections.OrderedDict()
-        self._temp_history = collections.OrderedDict()
-        self._new_history = []
-        self._saved_count = 0
-        objreg.get('save-manager').add_saveable(
-            'history', self.save, self.item_added)
-
-    def __repr__(self):
-        return utils.get_repr(self, length=len(self))
-
-    def __getitem__(self, key):
-        return self._new_history[key]
-
-    def __iter__(self):
-        return iter(self._history_dict.values())
-
-    def __len__(self):
-        return len(self._history_dict)
-
-    def async_read(self):
-        """Read the initial history."""
-        if self._initial_read_started:
-            log.init.debug("Ignoring async_read() because reading is started.")
-            return
-        self._initial_read_started = True
-
-        if standarddir.data() is None:
-            self._initial_read_done = True
-            self.async_read_done.emit()
-            return
-
-        with self._lineparser.open():
-            for line in self._lineparser:
-                yield
-                data = line.rstrip().split(maxsplit=1)
-                if not data:
-                    # empty line
-                    continue
-                elif len(data) != 2:
-                    # other malformed line
-                    log.init.warning("Invalid history entry {!r}!".format(
-                        line))
-                    continue
-                atime, url = data
-                if atime.startswith('\0'):
-                    log.init.warning(
-                        "Removing NUL bytes from entry {!r} - see "
-                        "https://github.com/The-Compiler/qutebrowser/issues/"
-                        "670".format(data))
-                    atime = atime.lstrip('\0')
-                # This de-duplicates history entries; only the latest
-                # entry for each URL is kept. If you want to keep
-                # information about previous hits change the items in
-                # old_urls to be lists or change HistoryEntry to have a
-                # list of atimes.
-                entry = HistoryEntry(atime, url)
-                self._add_entry(entry)
-
-        self._initial_read_done = True
-        self.async_read_done.emit()
-
-        for url, entry in self._temp_history.items():
-            self._new_history.append(entry)
-            self._add_entry(entry)
-            self.add_completion_item.emit(entry)
-
-    def _add_entry(self, entry, target=None):
-        """Add an entry to self._history_dict or another given OrderedDict."""
-        if target is None:
-            target = self._history_dict
-        target[entry.url_string] = entry
-        target.move_to_end(entry.url_string)
-
-    def get_recent(self):
-        """Get the most recent history entries."""
-        old = self._lineparser.get_recent()
-        return old + [str(e) for e in self._new_history]
-
-    def save(self):
-        """Save the history to disk."""
-        new = (str(e) for e in self._new_history[self._saved_count:])
-        self._lineparser.new_data = new
-        self._lineparser.save()
-        self._saved_count = len(self._new_history)
-
-    def addHistoryEntry(self, url_string):
-        """Called by WebKit when an URL should be added to the history.
-
-        Args:
-            url_string: An url as string to add to the history.
-        """
-        if not url_string:
-            return
-        if config.get('general', 'private-browsing'):
-            return
-        entry = HistoryEntry(time.time(), url_string)
-        if self._initial_read_done:
-            self.add_completion_item.emit(entry)
-            self._new_history.append(entry)
-            self._add_entry(entry)
-            self.item_added.emit(entry)
-        else:
-            self._add_entry(entry, target=self._temp_history)
-
-    def historyContains(self, url_string):
-        """Called by WebKit to determine if an URL is contained in the history.
-
-        Args:
-            url_string: The URL (as string) to check for.
-
-        Return:
-            True if the url is in the history, False otherwise.
-        """
-        return url_string in self._history_dict
-
-
-def init(parent=None):
-    """Initialize the web history.
-
-    Args:
-        parent: The parent to use for WebHistory.
-    """
-    history = WebHistory(parent)
-    objreg.register('web-history', history)
-    QWebHistoryInterface.setDefaultInterface(history)

qutebrowser/browser/pdfjs.py

@@ -23,7 +23,7 @@ import os
 
 from PyQt5.QtCore import QUrl
 
-from qutebrowser.browser import webelem
+from qutebrowser.browser.webkit import webelem
 from qutebrowser.utils import utils
 
 
@@ -51,9 +51,8 @@ def generate_pdfjs_page(url):
     """
     viewer = get_pdfjs_res('web/viewer.html').decode('utf-8')
     script = _generate_pdfjs_script(url)
-    html_page = viewer.replace(
-        '</body>', '</body><script>{}</script>'.format(script)
-    )
+    html_page = viewer.replace('</body>',
+                               '</body><script>{}</script>'.format(script))
     return html_page
 
 
@@ -64,13 +63,15 @@ def _generate_pdfjs_script(url):
         url: The url of the pdf page as QUrl.
     """
     return (
-        'PDFJS.verbosity = PDFJS.VERBOSITY_LEVELS.info;\n'
-        'PDFView.open("{url}");\n'
+        'document.addEventListener("DOMContentLoaded", function() {{\n'
+        '  PDFJS.verbosity = PDFJS.VERBOSITY_LEVELS.info;\n'
+        '  (window.PDFView || window.PDFViewerApplication).open("{url}");\n'
+        '}});\n'
     ).format(url=webelem.javascript_escape(url.toString(QUrl.FullyEncoded)))
 
 
 def fix_urls(asset):
-    """Take a html page and replace each relative URL with an absolute.
+    """Take an html page and replace each relative URL with an absolute.
 
     This is specialized for pdf.js files and not a general purpose function.
 

qutebrowser/browser/signalfilter.py

@@ -40,8 +40,7 @@ class SignalFilter(QObject):
         BLACKLIST: List of signal names which should not be logged.
     """
 
-    BLACKLIST = ['cur_scroll_perc_changed', 'cur_progress',
-                 'cur_statusbar_message', 'cur_link_hovered']
+    BLACKLIST = ['cur_scroll_perc_changed', 'cur_progress', 'cur_link_hovered']
 
     def __init__(self, win_id, parent=None):
         super().__init__(parent)

qutebrowser/browser/urlmarks.py

@@ -32,8 +32,9 @@ import collections
 
 from PyQt5.QtCore import pyqtSignal, QUrl, QObject
 
-from qutebrowser.utils import message, usertypes, urlutils, standarddir, objreg
-from qutebrowser.commands import cmdexc, cmdutils
+from qutebrowser.utils import (message, usertypes, qtutils, urlutils,
+                               standarddir, objreg)
+from qutebrowser.commands import cmdutils
 from qutebrowser.misc import lineparser
 
 
@@ -154,8 +155,7 @@ class QuickmarkManager(UrlMarkManager):
         try:
             key, url = line.rsplit(maxsplit=1)
         except ValueError:
-            message.error('current', "Invalid quickmark '{}'".format(
-                line))
+            message.error('current', "Invalid quickmark '{}'".format(line))
         else:
             self.marks[key] = url
 
@@ -174,10 +174,14 @@ class QuickmarkManager(UrlMarkManager):
             win_id, "Add quickmark:", usertypes.PromptMode.text,
             functools.partial(self.quickmark_add, win_id, urlstr))
 
-    @cmdutils.register(instance='quickmark-manager', win_id='win_id')
+    @cmdutils.register(instance='quickmark-manager')
+    @cmdutils.argument('win_id', win_id=True)
     def quickmark_add(self, win_id, url, name):
         """Add a new quickmark.
 
+        You can view all saved quickmarks on the
+        link:qute://bookmarks[bookmarks page].
+
         Args:
             win_id: The window ID to display the errors in.
             url: The url to add as quickmark.
@@ -204,18 +208,22 @@ class QuickmarkManager(UrlMarkManager):
         else:
             set_mark()
 
-    @cmdutils.register(instance='quickmark-manager', maxsplit=0,
-                       completion=[usertypes.Completion.quickmark_by_name])
-    def quickmark_del(self, name):
-        """Delete a quickmark.
+    def get_by_qurl(self, url):
+        """Look up a quickmark by QUrl, returning its name.
 
-        Args:
-            name: The name of the quickmark to delete.
+        Takes O(n) time, where n is the number of quickmarks.
+        Use a name instead where possible.
         """
+        qtutils.ensure_valid(url)
+        urlstr = url.toString(QUrl.RemovePassword | QUrl.FullyEncoded)
+
         try:
-            self.delete(name)
-        except KeyError:
-            raise cmdexc.CommandError("Quickmark '{}' not found!".format(name))
+            index = list(self.marks.values()).index(urlstr)
+            key = list(self.marks.keys())[index]
+        except ValueError:
+            raise DoesNotExistError(
+                "Quickmark for '{}' not found!".format(urlstr))
+        return key
 
     def get(self, name):
         """Get the URL of the quickmark named name as a QUrl."""
@@ -283,16 +291,3 @@ class BookmarkManager(UrlMarkManager):
             self.marks[urlstr] = title
             self.changed.emit()
             self.added.emit(title, urlstr)
-
-    @cmdutils.register(instance='bookmark-manager', maxsplit=0,
-                       completion=[usertypes.Completion.bookmark_by_url])
-    def bookmark_del(self, url):
-        """Delete a bookmark.
-
-        Args:
-            url: The URL of the bookmark to delete.
-        """
-        try:
-            self.delete(url)
-        except KeyError:
-            raise cmdexc.CommandError("Bookmark '{}' not found!".format(url))

qutebrowser/browser/webengine/__init__.py

@@ -0,0 +1,20 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Classes related to the browser widgets for QtWebEngine."""

qutebrowser/browser/webengine/webenginetab.py

@@ -0,0 +1,431 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+# FIXME:qtwebengine remove this once the stubs are gone
+# pylint: disable=unused-variable
+
+"""Wrapper over a QWebEngineView."""
+
+from PyQt5.QtCore import pyqtSlot, Qt, QEvent, QPoint
+from PyQt5.QtGui import QKeyEvent, QIcon
+from PyQt5.QtWidgets import QApplication
+# pylint: disable=no-name-in-module,import-error,useless-suppression
+from PyQt5.QtWebEngineWidgets import QWebEnginePage
+# pylint: enable=no-name-in-module,import-error,useless-suppression
+
+from qutebrowser.browser import browsertab
+from qutebrowser.browser.webengine import webview
+from qutebrowser.utils import usertypes, qtutils, log, utils
+
+
+class WebEnginePrinting(browsertab.AbstractPrinting):
+
+    """QtWebEngine implementations related to printing."""
+
+    def check_pdf_support(self):
+        if not hasattr(self._widget.page(), 'printToPdf'):
+            raise browsertab.WebTabError(
+                "Printing to PDF is unsupported with QtWebEngine on Qt > 5.7")
+
+    def check_printer_support(self):
+        raise browsertab.WebTabError(
+            "Printing is unsupported with QtWebEngine")
+
+    def to_pdf(self, filename):
+        self._widget.page().printToPdf(filename)
+
+    def to_printer(self, printer):
+        # Should never be called
+        assert False
+
+
+class WebEngineSearch(browsertab.AbstractSearch):
+
+    """QtWebEngine implementations related to searching on the page."""
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self._flags = QWebEnginePage.FindFlags(0)
+
+    def _find(self, text, flags, cb=None):
+        """Call findText on the widget with optional callback."""
+        if cb is None:
+            self._widget.findText(text, flags)
+        else:
+            self._widget.findText(text, flags, cb)
+
+    def search(self, text, *, ignore_case=False, reverse=False,
+               result_cb=None):
+        flags = QWebEnginePage.FindFlags(0)
+        if ignore_case == 'smart':
+            if not text.islower():
+                flags |= QWebEnginePage.FindCaseSensitively
+        elif not ignore_case:
+            flags |= QWebEnginePage.FindCaseSensitively
+        if reverse:
+            flags |= QWebEnginePage.FindBackward
+
+        self.text = text
+        self._flags = flags
+        self._find(text, flags, result_cb)
+
+    def clear(self):
+        self._widget.findText('')
+
+    def prev_result(self, *, result_cb=None):
+        # The int() here makes sure we get a copy of the flags.
+        flags = QWebEnginePage.FindFlags(int(self._flags))
+        if flags & QWebEnginePage.FindBackward:
+            flags &= ~QWebEnginePage.FindBackward
+        else:
+            flags |= QWebEnginePage.FindBackward
+        self._find(self.text, self._flags, result_cb)
+
+    def next_result(self, *, result_cb=None):
+        self._find(self.text, self._flags, result_cb)
+
+
+class WebEngineCaret(browsertab.AbstractCaret):
+
+    """QtWebEngine implementations related to moving the cursor/selection."""
+
+    @pyqtSlot(usertypes.KeyMode)
+    def _on_mode_entered(self, mode):
+        log.stub()
+
+    @pyqtSlot(usertypes.KeyMode)
+    def _on_mode_left(self):
+        log.stub()
+
+    def move_to_next_line(self, count=1):
+        log.stub()
+
+    def move_to_prev_line(self, count=1):
+        log.stub()
+
+    def move_to_next_char(self, count=1):
+        log.stub()
+
+    def move_to_prev_char(self, count=1):
+        log.stub()
+
+    def move_to_end_of_word(self, count=1):
+        log.stub()
+
+    def move_to_next_word(self, count=1):
+        log.stub()
+
+    def move_to_prev_word(self, count=1):
+        log.stub()
+
+    def move_to_start_of_line(self):
+        log.stub()
+
+    def move_to_end_of_line(self):
+        log.stub()
+
+    def move_to_start_of_next_block(self, count=1):
+        log.stub()
+
+    def move_to_start_of_prev_block(self, count=1):
+        log.stub()
+
+    def move_to_end_of_next_block(self, count=1):
+        log.stub()
+
+    def move_to_end_of_prev_block(self, count=1):
+        log.stub()
+
+    def move_to_start_of_document(self):
+        log.stub()
+
+    def move_to_end_of_document(self):
+        log.stub()
+
+    def toggle_selection(self):
+        log.stub()
+
+    def drop_selection(self):
+        log.stub()
+
+    def has_selection(self):
+        return self._widget.hasSelection()
+
+    def selection(self, html=False):
+        if html:
+            raise NotImplementedError
+        return self._widget.selectedText()
+
+    def follow_selected(self, *, tab=False):
+        log.stub()
+
+
+class WebEngineScroller(browsertab.AbstractScroller):
+
+    """QtWebEngine implementations related to scrolling."""
+
+    def __init__(self, tab, parent=None):
+        super().__init__(tab, parent)
+        self._pos_perc = (None, None)
+        self._pos_px = QPoint()
+
+    def _init_widget(self, widget):
+        super()._init_widget(widget)
+        page = widget.page()
+        try:
+            page.scrollPositionChanged.connect(
+                self._on_scroll_pos_changed)
+        except AttributeError:
+            log.stub('scrollPositionChanged, on Qt < 5.7')
+        self._on_scroll_pos_changed()
+
+    def _key_press(self, key, count=1):
+        # FIXME:qtwebengine Abort scrolling if the minimum/maximum was reached.
+        press_evt = QKeyEvent(QEvent.KeyPress, key, Qt.NoModifier, 0, 0, 0)
+        release_evt = QKeyEvent(QEvent.KeyRelease, key, Qt.NoModifier, 0, 0, 0)
+        recipient = self._widget.focusProxy()
+        for _ in range(count):
+            # If we get a segfault here, we might want to try sendEvent
+            # instead.
+            QApplication.postEvent(recipient, press_evt)
+            QApplication.postEvent(recipient, release_evt)
+
+    @pyqtSlot()
+    def _on_scroll_pos_changed(self):
+        """Update the scroll position attributes when it changed."""
+        def update_scroll_pos(jsret):
+            """Callback after getting scroll position via JS."""
+            if jsret is None:
+                # This can happen when the callback would get called after
+                # shutting down a tab
+                return
+            assert isinstance(jsret, dict), jsret
+            self._pos_perc = (jsret['perc']['x'], jsret['perc']['y'])
+            self._pos_px = QPoint(jsret['px']['x'], jsret['px']['y'])
+            self.perc_changed.emit(*self._pos_perc)
+
+        js_code = """
+            {scroll_js}
+            scroll_pos();
+        """.format(scroll_js=utils.read_file('javascript/scroll.js'))
+        self._tab.run_js_async(js_code, update_scroll_pos)
+
+    def pos_px(self):
+        return self._pos_px
+
+    def pos_perc(self):
+        return self._pos_perc
+
+    def to_perc(self, x=None, y=None):
+        js_code = """
+            {scroll_js}
+            scroll_to_perc({x}, {y});
+        """.format(scroll_js=utils.read_file('javascript/scroll.js'),
+                   x='undefined' if x is None else x,
+                   y='undefined' if y is None else y)
+        self._tab.run_js_async(js_code)
+
+    def to_point(self, point):
+        self._tab.run_js_async("window.scroll({x}, {y});".format(
+            x=point.x(), y=point.y()))
+
+    def delta(self, x=0, y=0):
+        self._tab.run_js_async("window.scrollBy({x}, {y});".format(x=x, y=y))
+
+    def delta_page(self, x=0, y=0):
+        js_code = """
+            {scroll_js}
+            scroll_delta_page({x}, {y});
+        """.format(scroll_js=utils.read_file('javascript/scroll.js'), x=x, y=y)
+        self._tab.run_js_async(js_code)
+
+    def up(self, count=1):
+        self._key_press(Qt.Key_Up, count)
+
+    def down(self, count=1):
+        self._key_press(Qt.Key_Down, count)
+
+    def left(self, count=1):
+        self._key_press(Qt.Key_Left, count)
+
+    def right(self, count=1):
+        self._key_press(Qt.Key_Right, count)
+
+    def top(self):
+        self._key_press(Qt.Key_Home)
+
+    def bottom(self):
+        self._key_press(Qt.Key_End)
+
+    def page_up(self, count=1):
+        self._key_press(Qt.Key_PageUp, count)
+
+    def page_down(self, count=1):
+        self._key_press(Qt.Key_PageDown, count)
+
+    def at_top(self):
+        return self.pos_px().y() == 0
+
+    def at_bottom(self):
+        log.stub()
+
+
+class WebEngineHistory(browsertab.AbstractHistory):
+
+    """QtWebEngine implementations related to page history."""
+
+    def current_idx(self):
+        return self._history.currentItemIndex()
+
+    def back(self):
+        self._history.back()
+
+    def forward(self):
+        self._history.forward()
+
+    def can_go_back(self):
+        return self._history.canGoBack()
+
+    def can_go_forward(self):
+        return self._history.canGoForward()
+
+    def serialize(self):
+        return qtutils.serialize(self._history)
+
+    def deserialize(self, data):
+        return qtutils.deserialize(data, self._history)
+
+    def load_items(self, items):
+        log.stub()
+
+
+class WebEngineZoom(browsertab.AbstractZoom):
+
+    """QtWebEngine implementations related to zooming."""
+
+    def _set_factor_internal(self, factor):
+        self._widget.setZoomFactor(factor)
+
+    def factor(self):
+        return self._widget.zoomFactor()
+
+
+class WebEngineTab(browsertab.AbstractTab):
+
+    """A QtWebEngine tab in the browser."""
+
+    def __init__(self, win_id, mode_manager, parent=None):
+        super().__init__(win_id)
+        widget = webview.WebEngineView()
+        self.history = WebEngineHistory(self)
+        self.scroller = WebEngineScroller(self, parent=self)
+        self.caret = WebEngineCaret(win_id=win_id, mode_manager=mode_manager,
+                                    tab=self, parent=self)
+        self.zoom = WebEngineZoom(win_id=win_id, parent=self)
+        self.search = WebEngineSearch(parent=self)
+        self.printing = WebEnginePrinting()
+        self._set_widget(widget)
+        self._connect_signals()
+        self.backend = usertypes.Backend.QtWebEngine
+
+    def openurl(self, url):
+        self._openurl_prepare(url)
+        self._widget.load(url)
+
+    def url(self):
+        return self._widget.url()
+
+    def dump_async(self, callback, *, plain=False):
+        if plain:
+            self._widget.page().toPlainText(callback)
+        else:
+            self._widget.page().toHtml(callback)
+
+    def run_js_async(self, code, callback=None):
+        if callback is None:
+            self._widget.page().runJavaScript(code)
+        else:
+            self._widget.page().runJavaScript(code, callback)
+
+    def run_js_blocking(self, code):
+        unset = object()
+        loop = qtutils.EventLoop()
+        js_ret = unset
+
+        def js_cb(val):
+            """Handle return value from JS and stop blocking."""
+            nonlocal js_ret
+            js_ret = val
+            loop.quit()
+
+        self.run_js_async(code, js_cb)
+        loop.exec_()  # blocks until loop.quit() in js_cb
+        assert js_ret is not unset
+
+        return js_ret
+
+    def shutdown(self):
+        log.stub()
+
+    def reload(self, *, force=False):
+        if force:
+            action = QWebEnginePage.ReloadAndBypassCache
+        else:
+            action = QWebEnginePage.Reload
+        self._widget.triggerPageAction(action)
+
+    def stop(self):
+        self._widget.stop()
+
+    def title(self):
+        return self._widget.title()
+
+    def icon(self):
+        try:
+            return self._widget.icon()
+        except AttributeError:
+            log.stub('on Qt < 5.7')
+            return QIcon()
+
+    def set_html(self, html, base_url):
+        # FIXME:qtwebengine
+        # check this and raise an exception if too big:
+        # Warning: The content will be percent encoded before being sent to the
+        # renderer via IPC. This may increase its size. The maximum size of the
+        # percent encoded content is 2 megabytes minus 30 bytes.
+        self._widget.setHtml(html, base_url)
+
+    def clear_ssl_errors(self):
+        log.stub()
+
+    def _connect_signals(self):
+        view = self._widget
+        page = view.page()
+        page.windowCloseRequested.connect(self.window_close_requested)
+        page.linkHovered.connect(self.link_hovered)
+        page.loadProgress.connect(self._on_load_progress)
+        page.loadStarted.connect(self._on_load_started)
+        view.titleChanged.connect(self.title_changed)
+        view.urlChanged.connect(self._on_url_changed)
+        page.loadFinished.connect(self._on_load_finished)
+        page.certificate_error.connect(self._on_ssl_errors)
+        try:
+            view.iconChanged.connect(self.icon_changed)
+        except AttributeError:
+            log.stub('iconChanged, on Qt < 5.7')

qutebrowser/browser/webengine/webview.py

@@ -0,0 +1,79 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""The main browser widget for QtWebEngine."""
+
+
+from PyQt5.QtCore import pyqtSignal, Qt, QPoint
+# pylint: disable=no-name-in-module,import-error,useless-suppression
+from PyQt5.QtWebEngineWidgets import QWebEngineView, QWebEnginePage
+# pylint: enable=no-name-in-module,import-error,useless-suppression
+
+from qutebrowser.config import config
+from qutebrowser.utils import log
+
+
+class WebEngineView(QWebEngineView):
+
+    """Custom QWebEngineView subclass with qutebrowser-specific features."""
+
+    mouse_wheel_zoom = pyqtSignal(QPoint)
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self.setPage(WebEnginePage(self))
+
+    def wheelEvent(self, e):
+        """Zoom on Ctrl-Mousewheel.
+
+        Args:
+            e: The QWheelEvent.
+        """
+        if e.modifiers() & Qt.ControlModifier:
+            e.accept()
+            self.mouse_wheel_zoom.emit(e.angleDelta())
+        else:
+            super().wheelEvent(e)
+
+
+class WebEnginePage(QWebEnginePage):
+
+    """Custom QWebEnginePage subclass with qutebrowser-specific features."""
+
+    certificate_error = pyqtSignal()
+
+    def certificateError(self, error):
+        self.certificate_error.emit()
+        return super().certificateError(error)
+
+    def javaScriptConsoleMessage(self, level, msg, line, source):
+        """Log javascript messages to qutebrowser's log."""
+        # FIXME:qtwebengine maybe unify this in the tab api somehow?
+        setting = config.get('general', 'log-javascript-console')
+        if setting == 'none':
+            return
+
+        level_to_logger = {
+            QWebEnginePage.InfoMessageLevel: log.js.info,
+            QWebEnginePage.WarningMessageLevel: log.js.warning,
+            QWebEnginePage.ErrorMessageLevel: log.js.error,
+        }
+        logstring = "[{}:{}] {}".format(source, line, msg)
+        logger = level_to_logger[level]
+        logger(logstring)

qutebrowser/browser/webkit/__init__.py

@@ -0,0 +1,20 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2014-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Classes related to the browser widgets for QtWebKit."""

qutebrowser/browser/webkit/cache.py

@@ -32,16 +32,23 @@ class DiskCache(QNetworkDiskCache):
 
     """Disk cache which sets correct cache dir and size.
 
+    If the cache is deactivated via the command line argument --cachedir="",
+    both attributes _cache_dir and _http_cache_dir are set to None.
+
     Attributes:
         _activated: Whether the cache should be used.
-        _cache_dir: The base directory for cache files (standarddir.cache())
-        _http_cache_dir: the HTTP subfolder in _cache_dir.
+        _cache_dir: The base directory for cache files (standarddir.cache()) or
+                    None.
+        _http_cache_dir: the HTTP subfolder in _cache_dir or None.
     """
 
     def __init__(self, cache_dir, parent=None):
         super().__init__(parent)
         self._cache_dir = cache_dir
-        self._http_cache_dir = os.path.join(cache_dir, 'http')
+        if cache_dir is None:
+            self._http_cache_dir = None
+        else:
+            self._http_cache_dir = os.path.join(cache_dir, 'http')
         self._maybe_activate()
         objreg.get('config').changed.connect(self.on_config_changed)
 
@@ -65,7 +72,7 @@ class DiskCache(QNetworkDiskCache):
         """Update cache size/activated if the config was changed."""
         if (section, option) == ('storage', 'cache-size'):
             self.setMaximumCacheSize(config.get('storage', 'cache-size'))
-        elif (section, option) == ('general',   # pragma: no branch
+        elif (section, option) == ('general',  # pragma: no branch
                                    'private-browsing'):
             self._maybe_activate()
 

qutebrowser/browser/webkit/cookies.py

@@ -37,9 +37,6 @@ class RAMCookieJar(QNetworkCookieJar):
 
     changed = pyqtSignal()
 
-    def __init__(self, parent=None):
-        super().__init__(parent)
-
     def __repr__(self):
         return utils.get_repr(self, count=len(self.allCookies()))
 

qutebrowser/browser/webkit/downloads.py

@@ -25,6 +25,7 @@ import sys
 import os.path
 import shutil
 import functools
+import tempfile
 import collections
 
 import sip
@@ -39,8 +40,8 @@ from qutebrowser.config import config
 from qutebrowser.commands import cmdexc, cmdutils
 from qutebrowser.utils import (message, usertypes, log, utils, urlutils,
                                objreg, standarddir, qtutils)
-from qutebrowser.browser import http
-from qutebrowser.browser.network import networkmanager
+from qutebrowser.browser.webkit import http
+from qutebrowser.browser.webkit.network import networkmanager
 
 
 ModelRole = usertypes.enum('ModelRole', ['item'], start=Qt.UserRole,
@@ -49,10 +50,7 @@ ModelRole = usertypes.enum('ModelRole', ['item'], start=Qt.UserRole,
 
 RetryInfo = collections.namedtuple('RetryInfo', ['request', 'manager'])
 
-
-DownloadPath = collections.namedtuple('DownloadPath', ['filename',
-                                                       'question'])
-
+DownloadPath = collections.namedtuple('DownloadPath', ['filename', 'question'])
 
 # Remember the last used directory
 last_used_directory = None
@@ -90,7 +88,7 @@ def path_suggestion(filename):
         return filename
     elif suggestion == 'both':
         return os.path.join(download_dir(), filename)
-    else:
+    else:  # pragma: no cover
         raise ValueError("Invalid suggestion value {}!".format(suggestion))
 
 
@@ -104,6 +102,11 @@ def create_full_filename(basename, filename):
     Return:
         The full absolute path, or None if filename creation was not possible.
     """
+    # Remove chars which can't be encoded in the filename encoding.
+    # See https://github.com/The-Compiler/qutebrowser/issues/427
+    encoding = sys.getfilesystemencoding()
+    filename = utils.force_encoding(filename, encoding)
+    basename = utils.force_encoding(basename, encoding)
     if os.path.isabs(filename) and os.path.isdir(filename):
         # We got an absolute directory from the user, so we save it under
         # the default filename in that directory.
@@ -143,8 +146,7 @@ def ask_for_filename(suggested_filename, win_id, *, parent=None,
         return DownloadPath(filename=download_dir(), question=None)
 
     encoding = sys.getfilesystemencoding()
-    suggested_filename = utils.force_encoding(suggested_filename,
-                                              encoding)
+    suggested_filename = utils.force_encoding(suggested_filename, encoding)
 
     q = usertypes.Question(parent)
     q.text = "Save file to:"
@@ -226,7 +228,7 @@ class DownloadItemStats(QObject):
         else:
             return remaining_bytes / avg
 
-    @pyqtSlot(int, int)
+    @pyqtSlot('qint64', 'qint64')
     def on_download_progress(self, bytes_done, bytes_total):
         """Update local variables when the download progress changed.
 
@@ -279,6 +281,7 @@ class DownloadItem(QObject):
         _read_timer: A Timer which reads the QNetworkReply into self._buffer
                      periodically.
         _win_id: The window ID the DownloadItem runs in.
+        _dead: Whether the Download has _die()'d.
 
     Signals:
         data_changed: The downloads metadata changed.
@@ -327,6 +330,7 @@ class DownloadItem(QObject):
         self.init_reply(reply)
         self._win_id = win_id
         self.raw_headers = {}
+        self._dead = False
 
     def __repr__(self):
         return utils.get_repr(self, basename=self.basename)
@@ -344,7 +348,7 @@ class DownloadItem(QObject):
             errmsg = ""
         else:
             errmsg = " - {}".format(self.error_msg)
-        if all(e is None for e in (perc, remaining, self.stats.total)):
+        if all(e is None for e in [perc, remaining, self.stats.total]):
             return ('{index}: {name} [{speed:>10}|{down}]{errmsg}'.format(
                 index=self.index, name=self.basename, speed=speed,
                 down=down, errmsg=errmsg))
@@ -394,6 +398,21 @@ class DownloadItem(QObject):
     def _die(self, msg):
         """Abort the download and emit an error."""
         assert not self.successful
+        # Prevent actions if calling _die() twice. This might happen if the
+        # error handler correctly connects, and the error occurs in init_reply
+        # between reply.error.connect and the reply.error() check. In this
+        # case, the connected error handlers will be called twice, once via the
+        # direct error.emit() and once here in _die(). The stacks look like
+        # this then:
+        #   <networkmanager error.emit> -> on_reply_error -> _die ->
+        #   self.error.emit()
+        # and
+        #   [init_reply -> <single shot timer> ->] <lambda in init_reply> ->
+        #   self.error.emit()
+        # which may lead to duplicate error messages (and failing tests)
+        if self._dead:
+            return
+        self._dead = True
         self._read_timer.stop()
         self.reply.downloadProgress.disconnect()
         self.reply.finished.disconnect()
@@ -411,6 +430,11 @@ class DownloadItem(QObject):
         self.reply = None
         self.done = True
         self.data_changed.emit()
+        if self.fileobj is not None:
+            try:
+                self.fileobj.close()
+            except OSError:
+                log.downloads.exception("Error while closing file object")
 
     def init_reply(self, reply):
         """Set a new reply and connect its signals.
@@ -435,7 +459,7 @@ class DownloadItem(QObject):
         # Here no signals are connected to the DownloadItem yet, so we use a
         # singleShot QTimer to emit them after they are connected.
         if reply.error() != QNetworkReply.NoError:
-            QTimer.singleShot(0, lambda: self.error.emit(reply.errorString()))
+            QTimer.singleShot(0, lambda: self._die(reply.errorString()))
 
     def get_status_color(self, position):
         """Choose an appropriate color for presenting the download's status.
@@ -445,7 +469,7 @@ class DownloadItem(QObject):
         """
         # pylint: disable=bad-config-call
         # WORKAROUND for https://bitbucket.org/logilab/astroid/issue/104/
-        assert position in ("fg", "bg")
+        assert position in ["fg", "bg"]
         start = config.get('colors', 'downloads.{}.start'.format(position))
         stop = config.get('colors', 'downloads.{}.stop'.format(position))
         system = config.get('colors', 'downloads.{}.system'.format(position))
@@ -456,8 +480,8 @@ class DownloadItem(QObject):
         elif self.stats.percentage() is None:
             return start
         else:
-            return utils.interpolate_color(
-                start, stop, self.stats.percentage(), system)
+            return utils.interpolate_color(start, stop,
+                                           self.stats.percentage(), system)
 
     @pyqtSlot()
     def cancel(self, remove_data=True):
@@ -507,7 +531,13 @@ class DownloadItem(QObject):
     def open_file(self):
         """Open the downloaded file."""
         assert self.successful
-        url = QUrl.fromLocalFile(self._filename)
+        filename = self._filename
+        if filename is None:
+            filename = getattr(self.fileobj, 'name', None)
+        if filename is None:
+            log.downloads.error("No filename to open the download!")
+            return
+        url = QUrl.fromLocalFile(filename)
         QDesktopServices.openUrl(url)
 
     def set_filename(self, filename):
@@ -518,15 +548,11 @@ class DownloadItem(QObject):
                       None: special value to stop the download.
         """
         global last_used_directory
-        if self.fileobj is not None:
+        if self.fileobj is not None:  # pragma: no cover
             raise ValueError("fileobj was already set! filename: {}, "
                              "existing: {}, fileobj {}".format(
                                  filename, self._filename, self.fileobj))
         filename = os.path.expanduser(filename)
-        # Remove chars which can't be encoded in the filename encoding.
-        # See https://github.com/The-Compiler/qutebrowser/issues/427
-        encoding = sys.getfilesystemencoding()
-        filename = utils.force_encoding(filename, encoding)
         self._filename = create_full_filename(self.basename, filename)
         if self._filename is None:
             # We only got a filename (without directory) or a relative path
@@ -535,7 +561,7 @@ class DownloadItem(QObject):
             self._filename = create_full_filename(
                 self.basename, os.path.join(download_dir(), filename))
 
-        # At this point, we have a misconfigured XDG_DOWNLOAd_DIR, as
+        # At this point, we have a misconfigured XDG_DOWNLOAD_DIR, as
         # download_dir() + filename is still no absolute path.
         # The config value is checked for "absoluteness", but
         # ~/.config/user-dirs.dirs may be misconfigured and a non-absolute path
@@ -561,8 +587,8 @@ class DownloadItem(QObject):
             txt = self._filename + " already exists. Overwrite?"
             self._ask_confirm_question(txt)
         # FIFO, device node, etc. Make sure we want to do this
-        elif (os.path.exists(self._filename) and not
-                os.path.isdir(self._filename)):
+        elif (os.path.exists(self._filename) and
+              not os.path.isdir(self._filename)):
             txt = (self._filename + " already exists and is a special file. "
                    "Write to this?")
             self._ask_confirm_question(txt)
@@ -575,7 +601,7 @@ class DownloadItem(QObject):
         Args:
             fileobj: A file-like object.
         """
-        if self.fileobj is not None:
+        if self.fileobj is not None:  # pragma: no cover
             raise ValueError("fileobj was already set! Old: {}, new: "
                              "{}".format(self.fileobj, fileobj))
         self.fileobj = fileobj
@@ -649,7 +675,7 @@ class DownloadItem(QObject):
         except OSError as e:
             self._die(e.strerror)
 
-    @pyqtSlot(int)
+    @pyqtSlot('QNetworkReply::NetworkError')
     def on_reply_error(self, code):
         """Handle QNetworkReply errors."""
         if code == QNetworkReply.OperationCanceledError:
@@ -676,7 +702,7 @@ class DownloadItem(QObject):
             self.raw_headers[bytes(key)] = bytes(value)
 
     def _handle_redirect(self):
-        """Handle a HTTP redirect.
+        """Handle an HTTP redirect.
 
         Return:
             True if the download was redirected, False otherwise.
@@ -736,6 +762,9 @@ class DownloadManager(QAbstractListModel):
     def _postprocess_question(self, q):
         """Postprocess a Question object that is asked."""
         q.destroyed.connect(functools.partial(self.questions.remove, q))
+        # We set the mode here so that other code that uses ask_for_filename
+        # doesn't need to handle the special download mode.
+        q.mode = usertypes.PromptMode.download
         self.questions.append(q)
 
     @pyqtSlot()
@@ -755,10 +784,7 @@ class DownloadManager(QAbstractListModel):
             **kwargs: passed to get_request().
 
         Return:
-            If the download could start immediately, (fileobj/filename given),
-            the created DownloadItem.
-
-            If not, None.
+            The created DownloadItem.
         """
         if not url.isValid():
             urlutils.invalid_url_error(self._win_id, url, "start download")
@@ -766,27 +792,17 @@ class DownloadManager(QAbstractListModel):
         req = QNetworkRequest(url)
         return self.get_request(req, **kwargs)
 
-    def get_request(self, request, *, fileobj=None, filename=None,
-                    prompt_download_directory=None, **kwargs):
+    def get_request(self, request, *, target=None, **kwargs):
         """Start a download with a QNetworkRequest.
 
         Args:
             request: The QNetworkRequest to download.
-            fileobj: The file object to write the answer to.
-            filename: A path to write the data to.
-            prompt_download_directory: Whether to prompt for the download dir
-                                       or automatically download. If None, the
-                                       config is used.
+            target: Where to save the download as usertypes.DownloadTarget.
             **kwargs: Passed to fetch_request.
 
         Return:
-            If the download could start immediately, (fileobj/filename given),
-            the created DownloadItem.
-
-            If not, None.
+            The created DownloadItem.
         """
-        if fileobj is not None and filename is not None:
-            raise TypeError("Only one of fileobj/filename may be given!")
         # WORKAROUND for Qt corrupting data loaded from cache:
         # https://bugreports.qt.io/browse/QTBUG-42757
         request.setAttribute(QNetworkRequest.CacheLoadControlAttribute,
@@ -814,27 +830,10 @@ class DownloadManager(QAbstractListModel):
         if suggested_fn is None:
             suggested_fn = 'qutebrowser-download'
 
-        # We won't need a question if a filename or fileobj is already given
-        if fileobj is None and filename is None:
-            filename, q = ask_for_filename(
-                suggested_fn, self._win_id, parent=self,
-                prompt_download_directory=prompt_download_directory
-            )
-
-        if fileobj is not None or filename is not None:
-            return self.fetch_request(request,
-                                      fileobj=fileobj,
-                                      filename=filename,
-                                      suggested_filename=suggested_fn,
-                                      **kwargs)
-        q.answered.connect(
-            lambda fn: self.fetch_request(request,
-                                          filename=fn,
-                                          suggested_filename=suggested_fn,
-                                          **kwargs))
-        self._postprocess_question(q)
-        q.ask()
-        return None
+        return self.fetch_request(request,
+                                  target=target,
+                                  suggested_filename=suggested_fn,
+                                  **kwargs)
 
     def fetch_request(self, request, *, page=None, **kwargs):
         """Download a QNetworkRequest to disk.
@@ -855,27 +854,25 @@ class DownloadManager(QAbstractListModel):
         return self.fetch(reply, **kwargs)
 
     @pyqtSlot('QNetworkReply')
-    def fetch(self, reply, *, fileobj=None, filename=None, auto_remove=False,
+    def fetch(self, reply, *, target=None, auto_remove=False,
               suggested_filename=None, prompt_download_directory=None):
         """Download a QNetworkReply to disk.
 
         Args:
             reply: The QNetworkReply to download.
-            fileobj: The file object to write the answer to.
-            filename: A path to write the data to.
+            target: Where to save the download as usertypes.DownloadTarget.
             auto_remove: Whether to remove the download even if
                          ui -> remove-finished-downloads is set to -1.
 
         Return:
             The created DownloadItem.
         """
-        if fileobj is not None and filename is not None:
-            raise TypeError("Only one of fileobj/filename may be given!")
         if not suggested_filename:
-            if filename is not None:
-                suggested_filename = os.path.basename(filename)
-            elif fileobj is not None and getattr(fileobj, 'name', None):
-                suggested_filename = fileobj.name
+            if isinstance(target, usertypes.FileDownloadTarget):
+                suggested_filename = os.path.basename(target.filename)
+            elif (isinstance(target, usertypes.FileObjDownloadTarget) and
+                  getattr(target.fileobj, 'name', None)):
+                suggested_filename = target.fileobj.name
             else:
                 _, suggested_filename = http.parse_content_disposition(reply)
         log.downloads.debug("fetch: {} -> {}".format(reply.url(),
@@ -898,8 +895,8 @@ class DownloadManager(QAbstractListModel):
         download.redirected.connect(
             functools.partial(self.on_redirect, download))
         download.basename = suggested_filename
-        idx = len(self.downloads) + 1
-        download.index = idx
+        idx = len(self.downloads)
+        download.index = idx + 1  # "Human readable" index
         self.beginInsertRows(QModelIndex(), idx, idx)
         self.downloads.append(download)
         self.endInsertRows()
@@ -907,13 +904,8 @@ class DownloadManager(QAbstractListModel):
         if not self._update_timer.isActive():
             self._update_timer.start()
 
-        if fileobj is not None:
-            download.set_fileobj(fileobj)
-            download.autoclose = False
-            return download
-
-        if filename is not None:
-            download.set_filename(filename)
+        if target is not None:
+            self._set_download_target(download, suggested_filename, target)
             return download
 
         # Neither filename nor fileobj were given, prepare a question
@@ -924,12 +916,15 @@ class DownloadManager(QAbstractListModel):
 
         # User doesn't want to be asked, so just use the download_dir
         if filename is not None:
-            download.set_filename(filename)
+            target = usertypes.FileDownloadTarget(filename)
+            self._set_download_target(download, suggested_filename, target)
             return download
 
         # Ask the user for a filename
         self._postprocess_question(q)
-        q.answered.connect(download.set_filename)
+        q.answered.connect(
+            functools.partial(self._set_download_target, download,
+                              suggested_filename))
         q.cancelled.connect(download.cancel)
         download.cancelled.connect(q.abort)
         download.error.connect(q.abort)
@@ -937,6 +932,43 @@ class DownloadManager(QAbstractListModel):
 
         return download
 
+    def _set_download_target(self, download, suggested_filename, target):
+        """Set the target for a given download.
+
+        Args:
+            download: The download to set the filename for.
+            suggested_filename: The suggested filename.
+            target: The usertypes.DownloadTarget for this download.
+        """
+        if isinstance(target, usertypes.FileObjDownloadTarget):
+            download.set_fileobj(target.fileobj)
+            download.autoclose = False
+        elif isinstance(target, usertypes.FileDownloadTarget):
+            download.set_filename(target.filename)
+        elif isinstance(target, usertypes.OpenFileDownloadTarget):
+            tmp_manager = objreg.get('temporary-downloads')
+            try:
+                fobj = tmp_manager.get_tmpfile(suggested_filename)
+            except OSError as exc:
+                msg = "Download error: {}".format(exc)
+                message.error(self._win_id, msg)
+                download.cancel()
+                return
+            download.finished.connect(
+                functools.partial(self._open_download, download))
+            download.autoclose = True
+            download.set_fileobj(fobj)
+        else:
+            log.downloads.error("Unknown download target: {}".format(target))
+
+    def _open_download(self, download):
+        """Open the given download but only if it was successful."""
+        if download.successful:
+            download.open_file()
+        else:
+            log.downloads.debug("{} finished but not successful, not opening!"
+                                .format(download))
+
     def raise_no_download(self, count):
         """Raise an exception that the download doesn't exist.
 
@@ -947,8 +979,8 @@ class DownloadManager(QAbstractListModel):
             raise cmdexc.CommandError("There's no download!")
         raise cmdexc.CommandError("There's no download {}!".format(count))
 
-    @cmdutils.register(instance='download-manager', scope='window',
-                       count='count')
+    @cmdutils.register(instance='download-manager', scope='window')
+    @cmdutils.argument('count', count=True)
     def download_cancel(self, all_=False, count=0):
         """Cancel the last/[count]th download.
 
@@ -974,8 +1006,8 @@ class DownloadManager(QAbstractListModel):
                                         .format(count))
             download.cancel()
 
-    @cmdutils.register(instance='download-manager', scope='window',
-                       count='count')
+    @cmdutils.register(instance='download-manager', scope='window')
+    @cmdutils.argument('count', count=True)
     def download_delete(self, count=0):
         """Delete the last/[count]th download from disk.
 
@@ -992,9 +1024,10 @@ class DownloadManager(QAbstractListModel):
             raise cmdexc.CommandError("Download {} is not done!".format(count))
         download.delete()
         self.remove_item(download)
+        log.downloads.debug("deleted download {}".format(download))
 
-    @cmdutils.register(instance='download-manager', scope='window',
-                       count='count')
+    @cmdutils.register(instance='download-manager', scope='window')
+    @cmdutils.argument('count', count=True)
     def download_open(self, count=0):
         """Open the last/[count]th download.
 
@@ -1011,8 +1044,8 @@ class DownloadManager(QAbstractListModel):
             raise cmdexc.CommandError("Download {} is not done!".format(count))
         download.open_file()
 
-    @cmdutils.register(instance='download-manager', scope='window',
-                       count='count')
+    @cmdutils.register(instance='download-manager', scope='window')
+    @cmdutils.argument('count', count=True)
     def download_retry(self, count=0):
         """Retry the first failed/[count]th download.
 
@@ -1038,7 +1071,7 @@ class DownloadManager(QAbstractListModel):
 
     @pyqtSlot(QNetworkRequest, QNetworkReply)
     def on_redirect(self, download, request, reply):
-        """Handle a HTTP redirect of a download.
+        """Handle an HTTP redirect of a download.
 
         Args:
             download: The old DownloadItem.
@@ -1097,8 +1130,8 @@ class DownloadManager(QAbstractListModel):
         finished_items = [d for d in self.downloads if d.done]
         self.remove_items(finished_items)
 
-    @cmdutils.register(instance='download-manager', scope='window',
-                       count='count')
+    @cmdutils.register(instance='download-manager', scope='window')
+    @cmdutils.argument('count', count=True)
     def download_remove(self, all_=False, count=0):
         """Remove the last/[count]th download from the list.
 
@@ -1228,7 +1261,8 @@ class DownloadManager(QAbstractListModel):
     def flags(self, _index):
         """Override flags so items aren't selectable.
 
-        The default would be Qt.ItemIsEnabled | Qt.ItemIsSelectable."""
+        The default would be Qt.ItemIsEnabled | Qt.ItemIsSelectable.
+        """
         return Qt.ItemIsEnabled | Qt.ItemNeverHasChildren
 
     def rowCount(self, parent=QModelIndex()):
@@ -1237,3 +1271,72 @@ class DownloadManager(QAbstractListModel):
             # We don't have children
             return 0
         return len(self.downloads)
+
+    def running_downloads(self):
+        """Return the amount of still running downloads.
+
+        Return:
+            The number of unfinished downloads.
+        """
+        return sum(1 for download in self.downloads if not download.done)
+
+
+class TempDownloadManager(QObject):
+
+    """Manager to handle temporary download files.
+
+    The downloads are downloaded to a temporary location and then openened with
+    the system standard application. The temporary files are deleted when
+    qutebrowser is shutdown.
+
+    Attributes:
+        files: A list of NamedTemporaryFiles of downloaded items.
+    """
+
+    def __init__(self, parent=None):
+        super().__init__(parent)
+        self.files = []
+        self._tmpdir = None
+
+    def cleanup(self):
+        """Clean up any temporary files."""
+        if self._tmpdir is not None:
+            self._tmpdir.cleanup()
+            self._tmpdir = None
+
+    def _get_tmpdir(self):
+        """Return the temporary directory that is used for downloads.
+
+        The directory is created lazily on first access.
+
+        Return:
+            The tempfile.TemporaryDirectory that is used.
+        """
+        if self._tmpdir is None:
+            self._tmpdir = tempfile.TemporaryDirectory(
+                prefix='qutebrowser-downloads-')
+        return self._tmpdir
+
+    def get_tmpfile(self, suggested_name):
+        """Return a temporary file in the temporary downloads directory.
+
+        The files are kept as long as qutebrowser is running and automatically
+        cleaned up at program exit.
+
+        Args:
+            suggested_name: str of the "suggested"/original filename. Used as a
+                            suffix, so any file extenions are preserved.
+
+        Return:
+            A tempfile.NamedTemporaryFile that should be used to save the file.
+        """
+        tmpdir = self._get_tmpdir()
+        encoding = sys.getfilesystemencoding()
+        suggested_name = utils.force_encoding(suggested_name, encoding)
+        # Make sure that the filename is not too long
+        if len(suggested_name) > 50:
+            suggested_name = suggested_name[:25] + '...' + suggested_name[-25:]
+        fobj = tempfile.NamedTemporaryFile(dir=tmpdir.name, delete=False,
+                                           suffix=suggested_name)
+        self.files.append(fobj)
+        return fobj

qutebrowser/browser/webkit/history.py

@@ -0,0 +1,326 @@
+# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
+
+# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
+#
+# This file is part of qutebrowser.
+#
+# qutebrowser is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# qutebrowser is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Simple history which gets written to disk."""
+
+import time
+import collections
+
+from PyQt5.QtCore import pyqtSignal, QUrl, QObject
+from PyQt5.QtWebKit import QWebHistoryInterface
+
+from qutebrowser.commands import cmdutils
+from qutebrowser.utils import utils, objreg, standarddir, log, qtutils
+from qutebrowser.config import config
+from qutebrowser.misc import lineparser
+
+
+class Entry:
+
+    """A single entry in the web history.
+
+    Attributes:
+        atime: The time the page was accessed.
+        url: The URL which was accessed as QUrl.
+        redirect: If True, don't save this entry to disk
+    """
+
+    def __init__(self, atime, url, title, redirect=False):
+        self.atime = float(atime)
+        self.url = url
+        self.title = title
+        self.redirect = redirect
+        qtutils.ensure_valid(url)
+
+    def __repr__(self):
+        return utils.get_repr(self, constructor=True, atime=self.atime,
+                              url=self.url_str(), title=self.title,
+                              redirect=self.redirect)
+
+    def __str__(self):
+        atime = str(int(self.atime))
+        if self.redirect:
+            atime += '-r'  # redirect flag
+        elems = [atime, self.url_str()]
+        if self.title:
+            elems.append(self.title)
+        return ' '.join(elems)
+
+    def __eq__(self, other):
+        return (self.atime == other.atime and
+                self.title == other.title and
+                self.url == other.url and
+                self.redirect == other.redirect)
+
+    def url_str(self):
+        """Get the URL as a lossless string."""
+        return self.url.toString(QUrl.FullyEncoded | QUrl.RemovePassword)
+
+    @classmethod
+    def from_str(cls, line):
+        """Parse a history line like '12345 http://example.com title'."""
+        data = line.split(maxsplit=2)
+        if len(data) == 2:
+            atime, url = data
+            title = ""
+        elif len(data) == 3:
+            atime, url, title = data
+        else:
+            raise ValueError("2 or 3 fields expected")
+
+        url = QUrl(url)
+        if not url.isValid():
+            raise ValueError("Invalid URL: {}".format(url.errorString()))
+
+        if atime.startswith('\0'):
+            log.init.debug(
+                "Removing NUL bytes from entry {!r} - see "
+                "https://github.com/The-Compiler/qutebrowser/issues/"
+                "670".format(data))
+            atime = atime.lstrip('\0')
+
+        if '-' in atime:
+            atime, flags = atime.split('-')
+        else:
+            flags = ''
+
+        if not set(flags).issubset('r'):
+            raise ValueError("Invalid flags {!r}".format(flags))
+
+        redirect = 'r' in flags
+
+        return cls(atime, url, title, redirect=redirect)
+
+
+class WebHistoryInterface(QWebHistoryInterface):
+
+    """Glue code between WebHistory and Qt's QWebHistoryInterface.
+
+    Attributes:
+        _history: The WebHistory object.
+    """
+
+    def __init__(self, webhistory, parent=None):
+        super().__init__(parent)
+        self._history = webhistory
+
+    def addHistoryEntry(self, url_string):
+        """Required for a QWebHistoryInterface impl, obsoleted by add_url."""
+        pass
+
+    def historyContains(self, url_string):
+        """Called by WebKit to determine if a URL is contained in the history.
+
+        Args:
+            url_string: The URL (as string) to check for.
+
+        Return:
+            True if the url is in the history, False otherwise.
+        """
+        return url_string in self._history.history_dict
+
+
+class WebHistory(QObject):
+
+    """The global history of visited pages.
+
+    This is a little more complex as you'd expect so the history can be read
+    from disk async while new history is already arriving.
+
+    self.history_dict is the main place where the history is stored, in an
+    OrderedDict (sorted by time) of URL strings mapped to Entry objects.
+
+    While reading from disk is still ongoing, the history is saved in
+    self._temp_history instead, and then appended to self.history_dict once
+    that's fully populated.
+
+    All history which is new in this session (rather than read from disk from a
+    previous browsing session) is also stored in self._new_history.
+    self._saved_count tracks how many of those entries were already written to
+    disk, so we can always append to the existing data.
+
+    Attributes:
+        history_dict: An OrderedDict of URLs read from the on-disk history.
+        _hist_dir: The directory to store the history in
+        _lineparser: The AppendLineParser used to save the history.
+        _new_history: A list of Entry items of the current session.
+        _saved_count: How many HistoryEntries have been written to disk.
+        _initial_read_started: Whether async_read was called.
+        _initial_read_done: Whether async_read has completed.
+        _temp_history: OrderedDict of temporary history entries before
+                       async_read was called.
+
+    Signals:
+        add_completion_item: Emitted before a new Entry is added.
+                             Used to sync with the completion.
+                             arg: The new Entry.
+        item_added: Emitted after a new Entry is added.
+                    Used to tell the savemanager that the history is dirty.
+                    arg: The new Entry.
+        cleared: Emitted after the history is cleared.
+    """
+
+    add_completion_item = pyqtSignal(Entry)
+    item_added = pyqtSignal(Entry)
+    cleared = pyqtSignal()
+    async_read_done = pyqtSignal()
+
+    def __init__(self, hist_dir, hist_name, parent=None):
+        super().__init__(parent)
+        self._initial_read_started = False
+        self._initial_read_done = False
+        self._hist_dir = hist_dir
+        self._lineparser = lineparser.AppendLineParser(hist_dir, hist_name,
+                                                       parent=self)
+        self.history_dict = collections.OrderedDict()
+        self._temp_history = collections.OrderedDict()
+        self._new_history = []
+        self._saved_count = 0
+        objreg.get('save-manager').add_saveable(
+            'history', self.save, self.item_added)
+
+    def __repr__(self):
+        return utils.get_repr(self, length=len(self))
+
+    def __iter__(self):
+        return iter(self.history_dict.values())
+
+    def __len__(self):
+        return len(self.history_dict)
+
+    def async_read(self):
+        """Read the initial history."""
+        if self._initial_read_started:
+            log.init.debug("Ignoring async_read() because reading is started.")
+            return
+        self._initial_read_started = True
+
+        if self._hist_dir is None:
+            self._initial_read_done = True
+            self.async_read_done.emit()
+            assert not self._temp_history
+            return
+
+        with self._lineparser.open():
+            for line in self._lineparser:
+                yield
+
+                line = line.rstrip()
+                if not line:
+                    continue
+
+                try:
+                    entry = Entry.from_str(line)
+                except ValueError as e:
+                    log.init.warning("Invalid history entry {!r}: {}!".format(
+                        line, e))
+                    continue
+
+                # This de-duplicates history entries; only the latest
+                # entry for each URL is kept. If you want to keep
+                # information about previous hits change the items in
+                # old_urls to be lists or change Entry to have a
+                # list of atimes.
+                self._add_entry(entry)
+
+        self._initial_read_done = True
+        self.async_read_done.emit()
+
+        for entry in self._temp_history.values():
+            self._add_entry(entry)
+            self._new_history.append(entry)
+            if not entry.redirect:
+                self.add_completion_item.emit(entry)
+        self._temp_history.clear()
+
+    def _add_entry(self, entry, target=None):
+        """Add an entry to self.history_dict or another given OrderedDict."""
+        if target is None:
+            target = self.history_dict
+        url_str = entry.url_str()
+        target[url_str] = entry
+        target.move_to_end(url_str)
+
+    def get_recent(self):
+        """Get the most recent history entries."""
+        old = self._lineparser.get_recent()
+        return old + [str(e) for e in self._new_history]
+
+    def save(self):
+        """Save the history to disk."""
+        new = (str(e) for e in self._new_history[self._saved_count:])
+        self._lineparser.new_data = new
+        self._lineparser.save()
+        self._saved_count = len(self._new_history)
+
+    @cmdutils.register(name='history-clear', instance='web-history')
+    def clear(self):
+        """Clear all browsing history.
+
+        Note this only clears the global history
+        (e.g. `~/.local/share/qutebrowser/history` on Linux) but not cookies,
+        the back/forward history of a tab, cache or other persistent data.
+        """
+        self._lineparser.clear()
+        self.history_dict.clear()
+        self._temp_history.clear()
+        self._new_history.clear()
+        self._saved_count = 0
+        self.cleared.emit()
+
+    def add_url(self, url, title="", *, redirect=False, atime=None):
+        """Called by WebKit when a URL should be added to the history.
+
+        Args:
+            url: A url (as QUrl) to add to the history.
+            redirect: Whether the entry was redirected to another URL
+                      (hidden in completion)
+            atime: Override the atime used to add the entry
+        """
+        if config.get('general', 'private-browsing'):
+            return
+        if not url.isValid():
+            log.misc.warning("Ignoring invalid URL being added to history")
+            return
+
+        if atime is None:
+            atime = time.time()
+        entry = Entry(atime, url, title, redirect=redirect)
+        if self._initial_read_done:
+            self._add_entry(entry)
+            self._new_history.append(entry)
+            self.item_added.emit(entry)
+            if not entry.redirect:
+                self.add_completion_item.emit(entry)
+        else:
+            self._add_entry(entry, target=self._temp_history)
+
+
+def init(parent=None):
+    """Initialize the web history.
+
+    Args:
+        parent: The parent to use for WebHistory.
+    """
+    history = WebHistory(hist_dir=standarddir.data(), hist_name='history',
+                         parent=parent)
+    objreg.register('web-history', history)
+
+    interface = WebHistoryInterface(history, parent=history)
+    QWebHistoryInterface.setDefaultInterface(interface)

qutebrowser/browser/webkit/http.py

@@ -23,7 +23,7 @@
 import os.path
 
 from qutebrowser.utils import log
-from qutebrowser.browser import rfc6266
+from qutebrowser.browser.webkit import rfc6266
 
 from PyQt5.QtNetwork import QNetworkRequest
 

qutebrowser/browser/webkit/mhtml.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with qutebrowser.  If not, see <http://www.gnu.org/licenses/>.
 
-"""Utils for writing a MHTML file."""
+"""Utils for writing an MHTML file."""
 
 import functools
 import io
@@ -34,7 +34,7 @@ import email.message
 
 from PyQt5.QtCore import QUrl
 
-from qutebrowser.browser import webelem, downloads
+from qutebrowser.browser.webkit import webelem, downloads
 from qutebrowser.utils import log, objreg, message, usertypes, utils, urlutils
 
 try:
@@ -83,7 +83,7 @@ def _get_css_imports_cssutils(data, inline=False):
 
     Args:
         data: The content of the stylesheet to scan as string.
-        inline: True if the argument is a inline HTML style attribute.
+        inline: True if the argument is an inline HTML style attribute.
     """
     # We don't care about invalid CSS data, this will only litter the log
     # output with CSS errors
@@ -112,7 +112,7 @@ def _get_css_imports(data, inline=False):
 
     Args:
         data: The content of the stylesheet to scan as string.
-        inline: True if the argument is a inline HTML style attribute.
+        inline: True if the argument is an inline HTML style attribute.
     """
     if cssutils is None:
         return _get_css_imports_regex(data)
@@ -149,7 +149,7 @@ E_QUOPRI = email.encoders.encode_quopri
 
 class MHTMLWriter:
 
-    """A class for outputting multiple files to a MHTML document.
+    """A class for outputting multiple files to an MHTML document.
 
     Attributes:
         root_content: The root content as bytes.
@@ -222,7 +222,7 @@ class _Downloader:
     """A class to download whole websites.
 
     Attributes:
-        web_view: The QWebView which contains the website that will be saved.
+        tab: The AbstractTab which contains the website that will be saved.
         dest: Destination filename.
         writer: The MHTMLWriter object which is used to save the page.
         loaded_urls: A set of QUrls of finished asset downloads.
@@ -233,15 +233,15 @@ class _Downloader:
         _win_id: The window this downloader belongs to.
     """
 
-    def __init__(self, web_view, dest):
-        self.web_view = web_view
+    def __init__(self, tab, dest):
+        self.tab = tab
         self.dest = dest
         self.writer = None
-        self.loaded_urls = {web_view.url()}
+        self.loaded_urls = {tab.url()}
         self.pending_downloads = set()
         self._finished_file = False
         self._used = False
-        self._win_id = web_view.win_id
+        self._win_id = tab.win_id
 
     def run(self):
         """Download and save the page.
@@ -252,8 +252,11 @@ class _Downloader:
         if self._used:
             raise ValueError("Downloader already used")
         self._used = True
-        web_url = self.web_view.url()
-        web_frame = self.web_view.page().mainFrame()
+        web_url = self.tab.url()
+
+        # FIXME:qtwebengine have a proper API for this
+        page = self.tab._widget.page()  # pylint: disable=protected-access
+        web_frame = page.mainFrame()
 
         self.writer = MHTMLWriter(
             web_frame.toHtml().encode('utf-8'),
@@ -316,7 +319,7 @@ class _Downloader:
         Args:
             url: The file to download as QUrl.
         """
-        if url.scheme() not in {'http', 'https'}:
+        if url.scheme() not in ['http', 'https']:
             return
         # Prevent loading an asset twice
         if url in self.loaded_urls:
@@ -340,15 +343,13 @@ class _Downloader:
 
         download_manager = objreg.get('download-manager', scope='window',
                                       window=self._win_id)
-        item = download_manager.get(url, fileobj=_NoCloseBytesIO(),
+        target = usertypes.FileObjDownloadTarget(_NoCloseBytesIO())
+        item = download_manager.get(url, target=target,
                                     auto_remove=True)
         self.pending_downloads.add((url, item))
-        item.finished.connect(
-            functools.partial(self._finished, url, item))
-        item.error.connect(
-            functools.partial(self._error, url, item))
-        item.cancelled.connect(
-            functools.partial(self._error, url, item))
+        item.finished.connect(functools.partial(self._finished, url, item))
+        item.error.connect(functools.partial(self._error, url, item))
+        item.cancelled.connect(functools.partial(self._cancelled, url, item))
 
     def _finished(self, url, item):
         """Callback when a single asset is downloaded.
@@ -404,7 +405,7 @@ class _Downloader:
         """Callback when a download error occurred.
 
         Args:
-            url: The orignal url of the asset as QUrl.
+            url: The original url of the asset as QUrl.
             item: The DownloadItem given by the DownloadManager.
         """
         try:
@@ -421,6 +422,20 @@ class _Downloader:
             return
         self._finish_file()
 
+    def _cancelled(self, url, item):
+        """Callback when a download is cancelled by the user.
+
+        Args:
+            url: The original url of the asset as QUrl.
+            item: The DownloadItem given by the DownloadManager.
+        """
+        # This callback is called before _finished, so there's no need to
+        # remove the item or close the fileobject.
+        log.downloads.debug("MHTML download cancelled by user: {}".format(url))
+        # Write an empty file instead
+        item.fileobj.seek(0)
+        item.fileobj.truncate()
+
     def _finish_file(self):
         """Save the file to the filename given in __init__."""
         if self._finished_file:
@@ -468,28 +483,28 @@ class _NoCloseBytesIO(io.BytesIO):
         super().close()
 
 
-def _start_download(dest, web_view):
-    """Start downloading the current page and all assets to a MHTML file.
+def _start_download(dest, tab):
+    """Start downloading the current page and all assets to an MHTML file.
 
     This will overwrite dest if it already exists.
 
     Args:
         dest: The filename where the resulting file should be saved.
-        web_view: Specify the webview whose page should be loaded.
+        tab: Specify the tab whose page should be loaded.
     """
-    loader = _Downloader(web_view, dest)
+    loader = _Downloader(tab, dest)
     loader.run()
 
 
-def start_download_checked(dest, web_view):
+def start_download_checked(dest, tab):
     """First check if dest is already a file, then start the download.
 
     Args:
         dest: The filename where the resulting file should be saved.
-        web_view: Specify the webview whose page should be loaded.
+        tab: Specify the tab whose page should be loaded.
     """
     # The default name is 'page title.mht'
-    title = web_view.title()
+    title = tab.title()
     default_name = utils.sanitize_filename(title + '.mht')
 
     # Remove characters which cannot be expressed in the file system encoding
@@ -513,12 +528,12 @@ def start_download_checked(dest, web_view):
     # saving the file anyway.
     if not os.path.isdir(os.path.dirname(path)):
         folder = os.path.dirname(path)
-        message.error(web_view.win_id,
+        message.error(tab.win_id,
                       "Directory {} does not exist.".format(folder))
         return
 
     if not os.path.isfile(path):
-        _start_download(path, web_view=web_view)
+        _start_download(path, tab=tab)
         return
 
     q = usertypes.Question()
@@ -526,7 +541,7 @@ def start_download_checked(dest, web_view):
     q.text = "{} exists. Overwrite?".format(path)
     q.completed.connect(q.deleteLater)
     q.answered_yes.connect(functools.partial(
-        _start_download, path, web_view=web_view))
+        _start_download, path, tab=tab))
     message_bridge = objreg.get('message-bridge', scope='window',
-                                window=web_view.win_id)
+                                window=tab.win_id)
     message_bridge.ask(q, blocking=False)