diff --git a/.gitignore b/.gitignore index 4ad4a98b884b..fd3a35592543 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ modules.builtin Module.symvers *.dwo +*.su # # Top-level generic files @@ -44,6 +45,7 @@ Module.symvers /TAGS /linux /vmlinux +/vmlinux.32 /vmlinux-gdb.py /vmlinuz /System.map @@ -89,6 +91,9 @@ GRTAGS GSYMS GTAGS +# id-utils files +ID + *.orig *~ \#*# @@ -97,6 +102,7 @@ GTAGS # Leavings from module signing # extra_certificates +signing_key.pem signing_key.priv signing_key.x509 x509.genkey diff --git a/CREDITS b/CREDITS index 1d616640bbf6..8207cc62ee9d 100644 --- a/CREDITS +++ b/CREDITS @@ -20,6 +20,10 @@ D: One of assisting postmasters for vger.kernel.org's lists S: (ask for current address) S: Finland +N: Thomas Abraham +E: thomas.ab@samsung.com +D: Samsung pin controller driver + N: Dragos Acostachioaie E: dragos@iname.com W: http://www.arbornet.org/~dragos @@ -2988,6 +2992,10 @@ S: 2200 Mission College Blvd S: Santa Clara, CA 95052 S: USA +N: Anil Ravindranath +E: anil_ravindranath@pmc-sierra.com +D: PMC-Sierra MaxRAID driver + N: Eric S. Raymond E: esr@thyrsus.com W: http://www.tuxedo.org/~esr/ @@ -3219,6 +3227,11 @@ S: 69 rue Dunois S: 75013 Paris S: France +N: Aleksa Sarai +E: cyphar@cyphar.com +W: https://www.cyphar.com/ +D: `pids` cgroup subsystem + N: Dipankar Sarma E: dipankar@in.ibm.com D: RCU diff --git a/Documentation/ABI/testing/sysfs-class-cxl b/Documentation/ABI/testing/sysfs-class-cxl index acfe9df83139..b07e86d4597f 100644 --- a/Documentation/ABI/testing/sysfs-class-cxl +++ b/Documentation/ABI/testing/sysfs-class-cxl @@ -223,3 +223,13 @@ Description: write only Writing 1 will issue a PERST to card which may cause the card to reload the FPGA depending on load_image_on_perst. Users: https://github.com/ibm-capi/libcxl + +What: /sys/class/cxl//perst_reloads_same_image +Date: July 2015 +Contact: linuxppc-dev@lists.ozlabs.org +Description: read/write + Trust that when an image is reloaded via PERST, it will not + have changed. + 0 = don't trust, the image may be different (default) + 1 = trust that the image will not change. +Users: https://github.com/ibm-capi/libcxl diff --git a/Documentation/ABI/testing/sysfs-driver-wacom b/Documentation/ABI/testing/sysfs-driver-wacom index c4f0fed64a6e..dca429340772 100644 --- a/Documentation/ABI/testing/sysfs-driver-wacom +++ b/Documentation/ABI/testing/sysfs-driver-wacom @@ -77,3 +77,22 @@ Description: The format is also scrambled, like in the USB mode, and it can be summarized by converting 76543210 into GECA6420. HGFEDCBA HFDB7531 + +What: /sys/bus/hid/devices/::./wacom_remote/unpair_remote +Date: July 2015 +Contact: linux-input@vger.kernel.org +Description: + Writing the character sequence '*' followed by a newline to + this file will delete all of the current pairings on the + device. Other character sequences are reserved. This file is + write only. + +What: /sys/bus/hid/devices/::./wacom_remote//remote_mode +Date: July 2015 +Contact: linux-input@vger.kernel.org +Description: + Reading from this file reports the mode status of the + remote as indicated by the LED lights on the device. If no + reports have been received from the paired device, reading + from this file will report '-1'. The mode is read-only + and cannot be set through the driver. diff --git a/Documentation/ABI/testing/sysfs-gpio b/Documentation/ABI/testing/sysfs-gpio index 80f4c94c7bef..55ffa2df1c10 100644 --- a/Documentation/ABI/testing/sysfs-gpio +++ b/Documentation/ABI/testing/sysfs-gpio @@ -16,7 +16,8 @@ Description: /sys/class/gpio /export ... asks the kernel to export a GPIO to userspace /unexport ... to return a GPIO to the kernel - /gpioN ... for each exported GPIO #N + /gpioN ... for each exported GPIO #N OR + / ... for a properly named GPIO line /value ... always readable, writes fail for input GPIOs /direction ... r/w as: in, out (default low); write: high, low /edge ... r/w as: none, falling, rising, both diff --git a/Documentation/ABI/testing/sysfs-hypervisor-pmu b/Documentation/ABI/testing/sysfs-hypervisor-pmu new file mode 100644 index 000000000000..224faa105e18 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-hypervisor-pmu @@ -0,0 +1,23 @@ +What: /sys/hypervisor/pmu/pmu_mode +Date: August 2015 +KernelVersion: 4.3 +Contact: Boris Ostrovsky +Description: + Describes mode that Xen's performance-monitoring unit (PMU) + uses. Accepted values are + "off" -- PMU is disabled + "self" -- The guest can profile itself + "hv" -- The guest can profile itself and, if it is + privileged (e.g. dom0), the hypervisor + "all" -- The guest can profile itself, the hypervisor + and all other guests. Only available to + privileged guests. + +What: /sys/hypervisor/pmu/pmu_features +Date: August 2015 +KernelVersion: 4.3 +Contact: Boris Ostrovsky +Description: + Describes Xen PMU features (as an integer). A set bit indicates + that the corresponding feature is enabled. See + include/xen/interface/xenpmu.h for available features diff --git a/Documentation/Changes b/Documentation/Changes index 646cdaa6e9d1..6d8863004858 100644 --- a/Documentation/Changes +++ b/Documentation/Changes @@ -43,6 +43,7 @@ o udev 081 # udevd --version o grub 0.93 # grub --version || grub-install --version o mcelog 0.6 # mcelog --version o iptables 1.4.2 # iptables -V +o openssl & libcrypto 1.0.1k # openssl version Kernel compilation @@ -79,6 +80,17 @@ BC You will need bc to build kernels 3.10 and higher +OpenSSL +------- + +Module signing and external certificate handling use the OpenSSL program and +crypto library to do key creation and signature generation. + +You will need openssl to build kernels 3.7 and higher if module signing is +enabled. You will also need openssl development packages to build kernels 4.3 +and higher. + + System utilities ================ @@ -295,6 +307,10 @@ Binutils -------- o +OpenSSL +------- +o + System utilities **************** @@ -392,4 +408,3 @@ o NFS-Utils --------- o - diff --git a/Documentation/DMA-API.txt b/Documentation/DMA-API.txt index 7eba542eff7c..edccacd4f048 100644 --- a/Documentation/DMA-API.txt +++ b/Documentation/DMA-API.txt @@ -104,6 +104,13 @@ crossing restrictions, pass 0 for alloc; passing 4096 says memory allocated from this pool must not cross 4KByte boundaries. + void *dma_pool_zalloc(struct dma_pool *pool, gfp_t mem_flags, + dma_addr_t *handle) + +Wraps dma_pool_alloc() and also zeroes the returned memory if the +allocation attempt succeeded. + + void *dma_pool_alloc(struct dma_pool *pool, gfp_t gfp_flags, dma_addr_t *dma_handle); diff --git a/Documentation/DocBook/alsa-driver-api.tmpl b/Documentation/DocBook/alsa-driver-api.tmpl index 71f9246127ec..e94a10bb4a9e 100644 --- a/Documentation/DocBook/alsa-driver-api.tmpl +++ b/Documentation/DocBook/alsa-driver-api.tmpl @@ -108,7 +108,7 @@ ASoC Core API !Iinclude/sound/soc.h !Esound/soc/soc-core.c -!Esound/soc/soc-cache.c + !Esound/soc/soc-devres.c !Esound/soc/soc-io.c !Esound/soc/soc-pcm.c diff --git a/Documentation/DocBook/device-drivers.tmpl b/Documentation/DocBook/device-drivers.tmpl index bbc1d7ee9c76..1d6008d51b55 100644 --- a/Documentation/DocBook/device-drivers.tmpl +++ b/Documentation/DocBook/device-drivers.tmpl @@ -217,6 +217,40 @@ X!Isound/sound_firmware.c --> + + Media Devices + + Video2Linux devices +!Iinclude/media/v4l2-async.h +!Iinclude/media/v4l2-ctrls.h +!Iinclude/media/v4l2-dv-timings.h +!Iinclude/media/v4l2-event.h +!Iinclude/media/v4l2-flash-led-class.h +!Iinclude/media/v4l2-mediabus.h +!Iinclude/media/v4l2-mem2mem.h +!Iinclude/media/v4l2-of.h +!Iinclude/media/v4l2-subdev.h +!Iinclude/media/videobuf2-core.h +!Iinclude/media/videobuf2-memops.h + + Digital TV (DVB) devices +!Idrivers/media/dvb-core/dvb_ca_en50221.h +!Idrivers/media/dvb-core/dvb_frontend.h +!Idrivers/media/dvb-core/dvb_math.h +!Idrivers/media/dvb-core/dvb_ringbuffer.h +!Idrivers/media/dvb-core/dvbdev.h + + Remote Controller devices +!Iinclude/media/rc-core.h + + Media Controller devices +!Iinclude/media/media-device.h +!Iinclude/media/media-devnode.h +!Iinclude/media/media-entity.h + + + + 16x50 UART Driver !Edrivers/tty/serial/serial_core.c @@ -456,4 +490,31 @@ X!Ilib/fonts/fonts.c !Edrivers/hsi/hsi.c + + Pulse-Width Modulation (PWM) + + Pulse-width modulation is a modulation technique primarily used to + control power supplied to electrical devices. + + + The PWM framework provides an abstraction for providers and consumers + of PWM signals. A controller that provides one or more PWM signals is + registered as struct pwm_chip. Providers are + expected to embed this structure in a driver-specific structure. This + structure contains fields that describe a particular chip. + + + A chip exposes one or more PWM signal sources, each of which exposed + as a struct pwm_device. Operations can be + performed on PWM devices to control the period, duty cycle, polarity + and active state of the signal. + + + Note that PWM devices are exclusive resources: they can always only be + used by one consumer at a time. + +!Iinclude/linux/pwm.h +!Edrivers/pwm/core.c + + diff --git a/Documentation/DocBook/drm.tmpl b/Documentation/DocBook/drm.tmpl index 2fb9a5457522..9ddf8c6cb887 100644 --- a/Documentation/DocBook/drm.tmpl +++ b/Documentation/DocBook/drm.tmpl @@ -3982,7 +3982,6 @@ int num_ioctls; Interrupt Handling !Pdrivers/gpu/drm/i915/i915_irq.c interrupt handling !Fdrivers/gpu/drm/i915/i915_irq.c intel_irq_init intel_irq_init_hw intel_hpd_init -!Fdrivers/gpu/drm/i915/i915_irq.c intel_irq_fini !Fdrivers/gpu/drm/i915/i915_irq.c intel_runtime_pm_disable_interrupts !Fdrivers/gpu/drm/i915/i915_irq.c intel_runtime_pm_enable_interrupts @@ -4012,7 +4011,6 @@ int num_ioctls; Frontbuffer Tracking !Pdrivers/gpu/drm/i915/intel_frontbuffer.c frontbuffer tracking !Idrivers/gpu/drm/i915/intel_frontbuffer.c -!Fdrivers/gpu/drm/i915/intel_drv.h intel_frontbuffer_flip !Fdrivers/gpu/drm/i915/i915_gem.c i915_gem_track_fb @@ -4044,6 +4042,11 @@ int num_ioctls; probing, so those sections fully apply. + + Hotplug +!Pdrivers/gpu/drm/i915/intel_hotplug.c Hotplug +!Idrivers/gpu/drm/i915/intel_hotplug.c + High Definition Audio !Pdrivers/gpu/drm/i915/intel_audio.c High Definition Audio over HDMI and Display Port @@ -4193,6 +4196,23 @@ int num_ioctls; Global GTT views !Pdrivers/gpu/drm/i915/i915_gem_gtt.c Global GTT views !Idrivers/gpu/drm/i915/i915_gem_gtt.c + + + GTT Fences and Swizzling +!Idrivers/gpu/drm/i915/i915_gem_fence.c + + Global GTT Fence Handling +!Pdrivers/gpu/drm/i915/i915_gem_fence.c fence register handling + + + Hardware Tiling and Swizzling Details +!Pdrivers/gpu/drm/i915/i915_gem_fence.c tiling swizzling details + + + + Object Tiling IOCTLs +!Idrivers/gpu/drm/i915/i915_gem_tiling.c +!Pdrivers/gpu/drm/i915/i915_gem_tiling.c buffer object tiling Buffer Object Eviction diff --git a/Documentation/DocBook/filesystems.tmpl b/Documentation/DocBook/filesystems.tmpl index bcdfdb9a9277..6006b6358c86 100644 --- a/Documentation/DocBook/filesystems.tmpl +++ b/Documentation/DocBook/filesystems.tmpl @@ -146,36 +146,30 @@ The journalling layer is easy to use. You need to first of all create a journal_t data structure. There are two calls to do this dependent on how you decide to allocate the physical -media on which the journal resides. The journal_init_inode() call -is for journals stored in filesystem inodes, or the journal_init_dev() -call can be use for journal stored on a raw device (in a continuous range +media on which the journal resides. The jbd2_journal_init_inode() call +is for journals stored in filesystem inodes, or the jbd2_journal_init_dev() +call can be used for journal stored on a raw device (in a continuous range of blocks). A journal_t is a typedef for a struct pointer, so when -you are finally finished make sure you call journal_destroy() on it +you are finally finished make sure you call jbd2_journal_destroy() on it to free up any used kernel memory. Once you have got your journal_t object you need to 'mount' or load the journal -file, unless of course you haven't initialised it yet - in which case you -need to call journal_create(). +file. The journalling layer expects the space for the journal was already +allocated and initialized properly by the userspace tools. When loading the +journal you must call jbd2_journal_load() to process journal contents. If the +client file system detects the journal contents does not need to be processed +(or even need not have valid contents), it may call jbd2_journal_wipe() to +clear the journal contents before calling jbd2_journal_load(). -Most of the time however your journal file will already have been created, but -before you load it you must call journal_wipe() to empty the journal file. -Hang on, you say , what if the filesystem wasn't cleanly umount()'d . Well, it is the -job of the client file system to detect this and skip the call to journal_wipe(). - - - -In either case the next call should be to journal_load() which prepares the -journal file for use. Note that journal_wipe(..,0) calls journal_skip_recovery() -for you if it detects any outstanding transactions in the journal and similarly -journal_load() will call journal_recover() if necessary. -I would advise reading fs/ext3/super.c for examples on this stage. -[RGG: Why is the journal_wipe() call necessary - doesn't this needlessly -complicate the API. Or isn't a good idea for the journal layer to hide -dirty mounts from the client fs] +Note that jbd2_journal_wipe(..,0) calls jbd2_journal_skip_recovery() for you if +it detects any outstanding transactions in the journal and similarly +jbd2_journal_load() will call jbd2_journal_recover() if necessary. I would +advise reading ext4_load_journal() in fs/ext4/super.c for examples on this +stage. @@ -189,41 +183,41 @@ You still need to actually journal your filesystem changes, this is done by wrapping them into transactions. Additionally you also need to wrap the modification of each of the buffers with calls to the journal layer, so it knows what the modifications -you are actually making are. To do this use journal_start() which +you are actually making are. To do this use jbd2_journal_start() which returns a transaction handle. -journal_start() -and its counterpart journal_stop(), which indicates the end of a transaction -are nestable calls, so you can reenter a transaction if necessary, -but remember you must call journal_stop() the same number of times as -journal_start() before the transaction is completed (or more accurately -leaves the update phase). Ext3/VFS makes use of this feature to simplify -quota support. +jbd2_journal_start() +and its counterpart jbd2_journal_stop(), which indicates the end of a +transaction are nestable calls, so you can reenter a transaction if necessary, +but remember you must call jbd2_journal_stop() the same number of times as +jbd2_journal_start() before the transaction is completed (or more accurately +leaves the update phase). Ext4/VFS makes use of this feature to simplify +handling of inode dirtying, quota support, etc. Inside each transaction you need to wrap the modifications to the individual buffers (blocks). Before you start to modify a buffer you -need to call journal_get_{create,write,undo}_access() as appropriate, +need to call jbd2_journal_get_{create,write,undo}_access() as appropriate, this allows the journalling layer to copy the unmodified data if it needs to. After all the buffer may be part of a previously uncommitted transaction. At this point you are at last ready to modify a buffer, and once -you are have done so you need to call journal_dirty_{meta,}data(). +you are have done so you need to call jbd2_journal_dirty_{meta,}data(). Or if you've asked for access to a buffer you now know is now longer -required to be pushed back on the device you can call journal_forget() +required to be pushed back on the device you can call jbd2_journal_forget() in much the same way as you might have used bforget() in the past. -A journal_flush() may be called at any time to commit and checkpoint +A jbd2_journal_flush() may be called at any time to commit and checkpoint all your transactions. -Then at umount time , in your put_super() you can then call journal_destroy() +Then at umount time , in your put_super() you can then call jbd2_journal_destroy() to clean up your in-core journal object. @@ -231,82 +225,74 @@ to clean up your in-core journal object. Unfortunately there a couple of ways the journal layer can cause a deadlock. The first thing to note is that each task can only have a single outstanding transaction at any one time, remember nothing -commits until the outermost journal_stop(). This means +commits until the outermost jbd2_journal_stop(). This means you must complete the transaction at the end of each file/inode/address etc. operation you perform, so that the journalling system isn't re-entered on another journal. Since transactions can't be nested/batched across differing journals, and another filesystem other than -yours (say ext3) may be modified in a later syscall. +yours (say ext4) may be modified in a later syscall. -The second case to bear in mind is that journal_start() can +The second case to bear in mind is that jbd2_journal_start() can block if there isn't enough space in the journal for your transaction (based on the passed nblocks param) - when it blocks it merely(!) needs to wait for transactions to complete and be committed from other tasks, -so essentially we are waiting for journal_stop(). So to avoid -deadlocks you must treat journal_start/stop() as if they +so essentially we are waiting for jbd2_journal_stop(). So to avoid +deadlocks you must treat jbd2_journal_start/stop() as if they were semaphores and include them in your semaphore ordering rules to prevent -deadlocks. Note that journal_extend() has similar blocking behaviour to -journal_start() so you can deadlock here just as easily as on journal_start(). +deadlocks. Note that jbd2_journal_extend() has similar blocking behaviour to +jbd2_journal_start() so you can deadlock here just as easily as on +jbd2_journal_start(). Try to reserve the right number of blocks the first time. ;-). This will be the maximum number of blocks you are going to touch in this transaction. -I advise having a look at at least ext3_jbd.h to see the basis on which -ext3 uses to make these decisions. +I advise having a look at at least ext4_jbd.h to see the basis on which +ext4 uses to make these decisions. Another wriggle to watch out for is your on-disk block allocation strategy. -why? Because, if you undo a delete, you need to ensure you haven't reused any -of the freed blocks in a later transaction. One simple way of doing this -is make sure any blocks you allocate only have checkpointed transactions -listed against them. Ext3 does this in ext3_test_allocatable(). +Why? Because, if you do a delete, you need to ensure you haven't reused any +of the freed blocks until the transaction freeing these blocks commits. If you +reused these blocks and crash happens, there is no way to restore the contents +of the reallocated blocks at the end of the last fully committed transaction. + +One simple way of doing this is to mark blocks as free in internal in-memory +block allocation structures only after the transaction freeing them commits. +Ext4 uses journal commit callback for this purpose. -Lock is also providing through journal_{un,}lock_updates(), -ext3 uses this when it wants a window with a clean and stable fs for a moment. -eg. +With journal commit callbacks you can ask the journalling layer to call a +callback function when the transaction is finally committed to disk, so that +you can do some of your own management. You ask the journalling layer for +calling the callback by simply setting journal->j_commit_callback function +pointer and that function is called after each transaction commit. You can also +use transaction->t_private_list for attaching entries to a transaction that +need processing when the transaction commits. + + + +JBD2 also provides a way to block all transaction updates via +jbd2_journal_{un,}lock_updates(). Ext4 uses this when it wants a window with a +clean and stable fs for a moment. E.g. - journal_lock_updates() //stop new stuff happening.. - journal_flush() // checkpoint everything. + jbd2_journal_lock_updates() //stop new stuff happening.. + jbd2_journal_flush() // checkpoint everything. ..do stuff on stable fs - journal_unlock_updates() // carry on with filesystem use. + jbd2_journal_unlock_updates() // carry on with filesystem use. The opportunities for abuse and DOS attacks with this should be obvious, if you allow unprivileged userspace to trigger codepaths containing these calls. - - - -A new feature of jbd since 2.5.25 is commit callbacks with the new -journal_callback_set() function you can now ask the journalling layer -to call you back when the transaction is finally committed to disk, so that -you can do some of your own management. The key to this is the journal_callback -struct, this maintains the internal callback information but you can -extend it like this:- - - - struct myfs_callback_s { - //Data structure element required by jbd.. - struct journal_callback for_jbd; - // Stuff for myfs allocated together. - myfs_inode* i_commited; - - } - - - -this would be useful if you needed to know when data was committed to a -particular inode. @@ -319,36 +305,6 @@ being each mount, each modification (transaction) and each changed buffer to tell the journalling layer about them. - -Here is a some pseudo code to give you an idea of how it works, as -an example. - - - - journal_t* my_jnrl = journal_create(); - journal_init_{dev,inode}(jnrl,...) - if (clean) journal_wipe(); - journal_load(); - - foreach(transaction) { /*transactions must be - completed before - a syscall returns to - userspace*/ - - handle_t * xct=journal_start(my_jnrl); - foreach(bh) { - journal_get_{create,write,undo}_access(xact,bh); - if ( myfs_modify(bh) ) { /* returns true - if makes changes */ - journal_dirty_{meta,}data(xact,bh); - } else { - journal_forget(bh); - } - } - journal_stop(xct); - } - journal_destroy(my_jrnl); - @@ -357,13 +313,13 @@ an example. Data Types The journalling layer uses typedefs to 'hide' the concrete definitions - of the structures used. As a client of the JBD layer you can + of the structures used. As a client of the JBD2 layer you can just rely on the using the pointer as a magic cookie of some sort. Obviously the hiding is not enforced as this is 'C'. Structures -!Iinclude/linux/jbd.h +!Iinclude/linux/jbd2.h @@ -375,11 +331,11 @@ an example. manage transactions Journal Level -!Efs/jbd/journal.c -!Ifs/jbd/recovery.c +!Efs/jbd2/journal.c +!Ifs/jbd2/recovery.c Transasction Level -!Efs/jbd/transaction.c +!Efs/jbd2/transaction.c diff --git a/Documentation/DocBook/media/Makefile b/Documentation/DocBook/media/Makefile index 23996f88cd58..08527e7ea4d0 100644 --- a/Documentation/DocBook/media/Makefile +++ b/Documentation/DocBook/media/Makefile @@ -199,7 +199,8 @@ DVB_DOCUMENTED = \ # install_media_images = \ - $(Q)-cp $(OBJIMGFILES) $(MEDIA_SRC_DIR)/*.svg $(MEDIA_SRC_DIR)/v4l/*.svg $(MEDIA_OBJ_DIR)/media_api + $(Q)-mkdir $(MEDIA_OBJ_DIR)/media_api; \ + cp $(OBJIMGFILES) $(MEDIA_SRC_DIR)/*.svg $(MEDIA_SRC_DIR)/v4l/*.svg $(MEDIA_OBJ_DIR)/media_api $(MEDIA_OBJ_DIR)/%: $(MEDIA_SRC_DIR)/%.b64 $(Q)base64 -d $< >$@ diff --git a/Documentation/DocBook/media/dvb/intro.xml b/Documentation/DocBook/media/dvb/intro.xml index bcc72c216402..51db15648099 100644 --- a/Documentation/DocBook/media/dvb/intro.xml +++ b/Documentation/DocBook/media/dvb/intro.xml @@ -163,9 +163,8 @@ are called: where N enumerates the DVB PCI cards in a system starting from 0, and M enumerates the devices of each type within each adapter, starting from 0, too. We will omit the “ -/dev/dvb/adapterN/” in the further dicussion -of these devices. The naming scheme for the devices is the same wheter -devfs is used or not. +/dev/dvb/adapterN/” in the further discussion +of these devices. More details about the data structures and function calls of all the devices are described in the following chapters. diff --git a/Documentation/DocBook/media/v4l/controls.xml b/Documentation/DocBook/media/v4l/controls.xml index 6e1667b5f3eb..33aece541880 100644 --- a/Documentation/DocBook/media/v4l/controls.xml +++ b/Documentation/DocBook/media/v4l/controls.xml @@ -3414,7 +3414,7 @@ giving priority to the center of the metered area. V4L2_EXPOSURE_METERING_MATRIX  A multi-zone metering. The light intensity is measured -in several points of the frame and the the results are combined. The +in several points of the frame and the results are combined. The algorithm of the zones selection and their significance in calculating the final value is device dependent. diff --git a/Documentation/DocBook/media/v4l/media-ioc-device-info.xml b/Documentation/DocBook/media/v4l/media-ioc-device-info.xml index 2ce521419e67..b0a21ac300b8 100644 --- a/Documentation/DocBook/media/v4l/media-ioc-device-info.xml +++ b/Documentation/DocBook/media/v4l/media-ioc-device-info.xml @@ -102,7 +102,7 @@ __u32 - media_version + driver_version Media device driver version, formatted with the KERNEL_VERSION() macro. Together with the driver field this identifies a particular diff --git a/Documentation/DocBook/media/v4l/vidioc-expbuf.xml b/Documentation/DocBook/media/v4l/vidioc-expbuf.xml index a78c9207422f..0ae0b6a915d0 100644 --- a/Documentation/DocBook/media/v4l/vidioc-expbuf.xml +++ b/Documentation/DocBook/media/v4l/vidioc-expbuf.xml @@ -62,28 +62,28 @@ buffer as a DMABUF file at any time after buffers have been allocated with the &VIDIOC-REQBUFS; ioctl. To export a buffer, applications fill &v4l2-exportbuffer;. The - type field is set to the same buffer type as was -previously used with &v4l2-requestbuffers; type . -Applications must also set the index field. Valid +type field is set to the same buffer type as was +previously used with &v4l2-requestbuffers; type. +Applications must also set the index field. Valid index numbers range from zero to the number of buffers allocated with -&VIDIOC-REQBUFS; (&v4l2-requestbuffers; count ) -minus one. For the multi-planar API, applications set the plane - field to the index of the plane to be exported. Valid planes +&VIDIOC-REQBUFS; (&v4l2-requestbuffers; count) +minus one. For the multi-planar API, applications set the plane +field to the index of the plane to be exported. Valid planes range from zero to the maximal number of valid planes for the currently active -format. For the single-planar API, applications must set plane - to zero. Additional flags may be posted in the -flags field. Refer to a manual for open() for details. +format. For the single-planar API, applications must set plane +to zero. Additional flags may be posted in the flags +field. Refer to a manual for open() for details. Currently only O_CLOEXEC, O_RDONLY, O_WRONLY, and O_RDWR are supported. All other fields must be set to zero. In the case of multi-planar API, every plane is exported separately using -multiple VIDIOC_EXPBUF calls. +multiple VIDIOC_EXPBUF calls. - After calling VIDIOC_EXPBUF the fd - field will be set by a driver. This is a DMABUF file +After calling VIDIOC_EXPBUF the fd +field will be set by a driver. This is a DMABUF file descriptor. The application may pass it to other DMABUF-aware devices. Refer to DMABUF importing for details about importing DMABUF files into V4L2 nodes. It is recommended to close a DMABUF file when it -is no longer used to allow the associated memory to be reclaimed. +is no longer used to allow the associated memory to be reclaimed. @@ -170,9 +170,9 @@ multi-planar API. Otherwise this value must be set to zero. __u32 flags - Flags for the newly created file, currently only -O_CLOEXEC , O_RDONLY, O_WRONLY -, and O_RDWR are supported, refer to the manual + Flags for the newly created file, currently only +O_CLOEXEC, O_RDONLY, O_WRONLY, +and O_RDWR are supported, refer to the manual of open() for more details. @@ -200,9 +200,9 @@ set the array to zero. EINVAL A queue is not in MMAP mode or DMABUF exporting is not -supported or flags or type - or index or plane - fields are invalid. +supported or flags or type +or index or plane fields +are invalid. diff --git a/Documentation/DocBook/media/v4l/vidioc-g-parm.xml b/Documentation/DocBook/media/v4l/vidioc-g-parm.xml index f4e28e7d4751..721728745407 100644 --- a/Documentation/DocBook/media/v4l/vidioc-g-parm.xml +++ b/Documentation/DocBook/media/v4l/vidioc-g-parm.xml @@ -267,7 +267,7 @@ is intended for still imaging applications. The idea is to get the best possible image quality that the hardware can deliver. It is not defined how the driver writer may achieve that; it will depend on the hardware and the ingenuity of the driver writer. High quality mode is -a different mode from the the regular motion video capture modes. In +a different mode from the regular motion video capture modes. In high quality mode: The driver may be able to capture higher diff --git a/Documentation/DocBook/media/v4l/vidioc-queryctrl.xml b/Documentation/DocBook/media/v4l/vidioc-queryctrl.xml index dc83ad70f8dc..6ec39c698baf 100644 --- a/Documentation/DocBook/media/v4l/vidioc-queryctrl.xml +++ b/Documentation/DocBook/media/v4l/vidioc-queryctrl.xml @@ -616,7 +616,7 @@ pointer to memory containing the payload of the control. V4L2_CTRL_FLAG_EXECUTE_ON_WRITE 0x0200 The value provided to the control will be propagated to the driver -even if remains constant. This is required when the control represents an action +even if it remains constant. This is required when the control represents an action on the hardware. For example: clearing an error flag or triggering the flash. All the controls of the type V4L2_CTRL_TYPE_BUTTON have this flag set. diff --git a/Documentation/SubmittingPatches b/Documentation/SubmittingPatches index fa596d8a3ab0..fd89b04d34f0 100644 --- a/Documentation/SubmittingPatches +++ b/Documentation/SubmittingPatches @@ -291,7 +291,7 @@ sending him e-mail. If you have a patch that fixes an exploitable security bug, send that patch to security@kernel.org. For severe bugs, a short embargo may be considered -to allow distrbutors to get the patch out to users; in such cases, +to allow distributors to get the patch out to users; in such cases, obviously, the patch should not be sent to any public lists. Patches that fix a severe bug in a released kernel should be directed diff --git a/Documentation/acpi/method-tracing.txt b/Documentation/acpi/method-tracing.txt index f6efb1ea559a..c2505eefc878 100644 --- a/Documentation/acpi/method-tracing.txt +++ b/Documentation/acpi/method-tracing.txt @@ -1,26 +1,192 @@ -/sys/module/acpi/parameters/: +ACPICA Trace Facility -trace_method_name - The AML method name that the user wants to trace +Copyright (C) 2015, Intel Corporation +Author: Lv Zheng -trace_debug_layer - The temporary debug_layer used when tracing the method. - Using 0xffffffff by default if it is 0. -trace_debug_level - The temporary debug_level used when tracing the method. - Using 0x00ffffff by default if it is 0. +Abstract: -trace_state +This document describes the functions and the interfaces of the method +tracing facility. + +1. Functionalities and usage examples: + + ACPICA provides method tracing capability. And two functions are + currently implemented using this capability. + + A. Log reducer + ACPICA subsystem provides debugging outputs when CONFIG_ACPI_DEBUG is + enabled. The debugging messages which are deployed via + ACPI_DEBUG_PRINT() macro can be reduced at 2 levels - per-component + level (known as debug layer, configured via + /sys/module/acpi/parameters/debug_layer) and per-type level (known as + debug level, configured via /sys/module/acpi/parameters/debug_level). + + But when the particular layer/level is applied to the control method + evaluations, the quantity of the debugging outputs may still be too + large to be put into the kernel log buffer. The idea thus is worked out + to only enable the particular debug layer/level (normally more detailed) + logs when the control method evaluation is started, and disable the + detailed logging when the control method evaluation is stopped. + + The following command examples illustrate the usage of the "log reducer" + functionality: + a. Filter out the debug layer/level matched logs when control methods + are being evaluated: + # cd /sys/module/acpi/parameters + # echo "0xXXXXXXXX" > trace_debug_layer + # echo "0xYYYYYYYY" > trace_debug_level + # echo "enable" > trace_state + b. Filter out the debug layer/level matched logs when the specified + control method is being evaluated: + # cd /sys/module/acpi/parameters + # echo "0xXXXXXXXX" > trace_debug_layer + # echo "0xYYYYYYYY" > trace_debug_level + # echo "\PPPP.AAAA.TTTT.HHHH" > trace_method_name + # echo "method" > /sys/module/acpi/parameters/trace_state + c. Filter out the debug layer/level matched logs when the specified + control method is being evaluated for the first time: + # cd /sys/module/acpi/parameters + # echo "0xXXXXXXXX" > trace_debug_layer + # echo "0xYYYYYYYY" > trace_debug_level + # echo "\PPPP.AAAA.TTTT.HHHH" > trace_method_name + # echo "method-once" > /sys/module/acpi/parameters/trace_state + Where: + 0xXXXXXXXX/0xYYYYYYYY: Refer to Documentation/acpi/debug.txt for + possible debug layer/level masking values. + \PPPP.AAAA.TTTT.HHHH: Full path of a control method that can be found + in the ACPI namespace. It needn't be an entry + of a control method evaluation. + + B. AML tracer + + There are special log entries added by the method tracing facility at + the "trace points" the AML interpreter starts/stops to execute a control + method, or an AML opcode. Note that the format of the log entries are + subject to change: + [ 0.186427] exdebug-0398 ex_trace_point : Method Begin [0xf58394d8:\_SB.PCI0.LPCB.ECOK] execution. + [ 0.186630] exdebug-0398 ex_trace_point : Opcode Begin [0xf5905c88:If] execution. + [ 0.186820] exdebug-0398 ex_trace_point : Opcode Begin [0xf5905cc0:LEqual] execution. + [ 0.187010] exdebug-0398 ex_trace_point : Opcode Begin [0xf5905a20:-NamePath-] execution. + [ 0.187214] exdebug-0398 ex_trace_point : Opcode End [0xf5905a20:-NamePath-] execution. + [ 0.187407] exdebug-0398 ex_trace_point : Opcode Begin [0xf5905f60:One] execution. + [ 0.187594] exdebug-0398 ex_trace_point : Opcode End [0xf5905f60:One] execution. + [ 0.187789] exdebug-0398 ex_trace_point : Opcode End [0xf5905cc0:LEqual] execution. + [ 0.187980] exdebug-0398 ex_trace_point : Opcode Begin [0xf5905cc0:Return] execution. + [ 0.188146] exdebug-0398 ex_trace_point : Opcode Begin [0xf5905f60:One] execution. + [ 0.188334] exdebug-0398 ex_trace_point : Opcode End [0xf5905f60:One] execution. + [ 0.188524] exdebug-0398 ex_trace_point : Opcode End [0xf5905cc0:Return] execution. + [ 0.188712] exdebug-0398 ex_trace_point : Opcode End [0xf5905c88:If] execution. + [ 0.188903] exdebug-0398 ex_trace_point : Method End [0xf58394d8:\_SB.PCI0.LPCB.ECOK] execution. + + Developers can utilize these special log entries to track the AML + interpretion, thus can aid issue debugging and performance tuning. Note + that, as the "AML tracer" logs are implemented via ACPI_DEBUG_PRINT() + macro, CONFIG_ACPI_DEBUG is also required to be enabled for enabling + "AML tracer" logs. + + The following command examples illustrate the usage of the "AML tracer" + functionality: + a. Filter out the method start/stop "AML tracer" logs when control + methods are being evaluated: + # cd /sys/module/acpi/parameters + # echo "0x80" > trace_debug_layer + # echo "0x10" > trace_debug_level + # echo "enable" > trace_state + b. Filter out the method start/stop "AML tracer" when the specified + control method is being evaluated: + # cd /sys/module/acpi/parameters + # echo "0x80" > trace_debug_layer + # echo "0x10" > trace_debug_level + # echo "\PPPP.AAAA.TTTT.HHHH" > trace_method_name + # echo "method" > trace_state + c. Filter out the method start/stop "AML tracer" logs when the specified + control method is being evaluated for the first time: + # cd /sys/module/acpi/parameters + # echo "0x80" > trace_debug_layer + # echo "0x10" > trace_debug_level + # echo "\PPPP.AAAA.TTTT.HHHH" > trace_method_name + # echo "method-once" > trace_state + d. Filter out the method/opcode start/stop "AML tracer" when the + specified control method is being evaluated: + # cd /sys/module/acpi/parameters + # echo "0x80" > trace_debug_layer + # echo "0x10" > trace_debug_level + # echo "\PPPP.AAAA.TTTT.HHHH" > trace_method_name + # echo "opcode" > trace_state + e. Filter out the method/opcode start/stop "AML tracer" when the + specified control method is being evaluated for the first time: + # cd /sys/module/acpi/parameters + # echo "0x80" > trace_debug_layer + # echo "0x10" > trace_debug_level + # echo "\PPPP.AAAA.TTTT.HHHH" > trace_method_name + # echo "opcode-opcode" > trace_state + + Note that all above method tracing facility related module parameters can + be used as the boot parameters, for example: + acpi.trace_debug_layer=0x80 acpi.trace_debug_level=0x10 \ + acpi.trace_method_name=\_SB.LID0._LID acpi.trace_state=opcode-once + +2. Interface descriptions: + + All method tracing functions can be configured via ACPI module + parameters that are accessible at /sys/module/acpi/parameters/: + + trace_method_name + The full path of the AML method that the user wants to trace. + Note that the full path shouldn't contain the trailing "_"s in its + name segments but may contain "\" to form an absolute path. + + trace_debug_layer + The temporary debug_layer used when the tracing feature is enabled. + Using ACPI_EXECUTER (0x80) by default, which is the debug_layer + used to match all "AML tracer" logs. + + trace_debug_level + The temporary debug_level used when the tracing feature is enabled. + Using ACPI_LV_TRACE_POINT (0x10) by default, which is the + debug_level used to match all "AML tracer" logs. + + trace_state The status of the tracing feature. - - "enabled" means this feature is enabled - and the AML method is traced every time it's executed. - - "1" means this feature is enabled and the AML method - will only be traced during the next execution. - - "disabled" means this feature is disabled. - Users can enable/disable this debug tracing feature by - "echo string > /sys/module/acpi/parameters/trace_state". - "string" should be one of "enable", "disable" and "1". + Users can enable/disable this debug tracing feature by executing + the following command: + # echo string > /sys/module/acpi/parameters/trace_state + Where "string" should be one of the followings: + "disable" + Disable the method tracing feature. + "enable" + Enable the method tracing feature. + ACPICA debugging messages matching + "trace_debug_layer/trace_debug_level" during any method + execution will be logged. + "method" + Enable the method tracing feature. + ACPICA debugging messages matching + "trace_debug_layer/trace_debug_level" during method execution + of "trace_method_name" will be logged. + "method-once" + Enable the method tracing feature. + ACPICA debugging messages matching + "trace_debug_layer/trace_debug_level" during method execution + of "trace_method_name" will be logged only once. + "opcode" + Enable the method tracing feature. + ACPICA debugging messages matching + "trace_debug_layer/trace_debug_level" during method/opcode + execution of "trace_method_name" will be logged. + "opcode-once" + Enable the method tracing feature. + ACPICA debugging messages matching + "trace_debug_layer/trace_debug_level" during method/opcode + execution of "trace_method_name" will be logged only once. + Note that, the difference between the "enable" and other feature + enabling options are: + 1. When "enable" is specified, since + "trace_debug_layer/trace_debug_level" shall apply to all control + method evaluations, after configuring "trace_state" to "enable", + "trace_method_name" will be reset to NULL. + 2. When "method/opcode" is specified, if + "trace_method_name" is NULL when "trace_state" is configured to + these options, the "trace_debug_layer/trace_debug_level" will + apply to all control method evaluations. diff --git a/Documentation/arm64/booting.txt b/Documentation/arm64/booting.txt index 1690350f16e7..7d9d3c2286b2 100644 --- a/Documentation/arm64/booting.txt +++ b/Documentation/arm64/booting.txt @@ -81,7 +81,7 @@ The decompressed kernel image contains a 64-byte header as follows: u64 res3 = 0; /* reserved */ u64 res4 = 0; /* reserved */ u32 magic = 0x644d5241; /* Magic number, little endian, "ARM\x64" */ - u32 res5; /* reserved (used for PE COFF offset) */ + u32 res5; /* reserved (used for PE COFF offset) */ Header notes: @@ -103,7 +103,7 @@ Header notes: - The flags field (introduced in v3.17) is a little-endian 64-bit field composed as follows: - Bit 0: Kernel endianness. 1 if BE, 0 if LE. + Bit 0: Kernel endianness. 1 if BE, 0 if LE. Bits 1-63: Reserved. - When image_size is zero, a bootloader should attempt to keep as much @@ -115,11 +115,14 @@ The Image must be placed text_offset bytes from a 2MB aligned base address near the start of usable system RAM and called there. Memory below that base address is currently unusable by Linux, and therefore it is strongly recommended that this location is the start of system RAM. +The region between the 2 MB aligned base address and the start of the +image has no special significance to the kernel, and may be used for +other purposes. At least image_size bytes from the start of the image must be free for use by the kernel. -Any memory described to the kernel (even that below the 2MB aligned base -address) which is not marked as reserved from the kernel e.g. with a +Any memory described to the kernel (even that below the start of the +image) which is not marked as reserved from the kernel (e.g., with a memreserve region in the device tree) will be considered as available to the kernel. diff --git a/Documentation/atomic_ops.txt b/Documentation/atomic_ops.txt index dab6da3382d9..b19fc34efdb1 100644 --- a/Documentation/atomic_ops.txt +++ b/Documentation/atomic_ops.txt @@ -266,7 +266,9 @@ with the given old and new values. Like all atomic_xxx operations, atomic_cmpxchg will only satisfy its atomicity semantics as long as all other accesses of *v are performed through atomic_xxx operations. -atomic_cmpxchg must provide explicit memory barriers around the operation. +atomic_cmpxchg must provide explicit memory barriers around the operation, +although if the comparison fails then no memory ordering guarantees are +required. The semantics for atomic_cmpxchg are the same as those defined for 'cas' below. diff --git a/Documentation/block/biodoc.txt b/Documentation/block/biodoc.txt index fd12c0d835fd..5be8a7f4cc7f 100644 --- a/Documentation/block/biodoc.txt +++ b/Documentation/block/biodoc.txt @@ -1109,7 +1109,7 @@ it will loop and handle as many sectors (on a bio-segment granularity) as specified. Now bh->b_end_io is replaced by bio->bi_end_io, but most of the time the -right thing to use is bio_endio(bio, uptodate) instead. +right thing to use is bio_endio(bio) instead. If the driver is dropping the io_request_lock from its request_fn strategy, then it just needs to replace that with q->queue_lock instead. diff --git a/Documentation/block/biovecs.txt b/Documentation/block/biovecs.txt index 74a32ad52f53..25689584e6e0 100644 --- a/Documentation/block/biovecs.txt +++ b/Documentation/block/biovecs.txt @@ -24,7 +24,7 @@ particular, presenting the illusion of partially completed biovecs so that normal code doesn't have to deal with bi_bvec_done. * Driver code should no longer refer to biovecs directly; we now have - bio_iovec() and bio_iovec_iter() macros that return literal struct biovecs, + bio_iovec() and bio_iter_iovec() macros that return literal struct biovecs, constructed from the raw biovecs but taking into account bi_bvec_done and bi_size. @@ -109,3 +109,11 @@ Other implications: over all the biovecs in the new bio - which is silly as it's not needed. So, don't use bi_vcnt anymore. + + * The current interface allows the block layer to split bios as needed, so we + could eliminate a lot of complexity particularly in stacked drivers. Code + that creates bios can then create whatever size bios are convenient, and + more importantly stacked drivers don't have to deal with both their own bio + size limitations and the limitations of the underlying devices. Thus + there's no need to define ->merge_bvec_fn() callbacks for individual block + drivers. diff --git a/Documentation/block/queue-sysfs.txt b/Documentation/block/queue-sysfs.txt index 3a29f8914df9..e5d914845be6 100644 --- a/Documentation/block/queue-sysfs.txt +++ b/Documentation/block/queue-sysfs.txt @@ -20,7 +20,7 @@ This shows the size of internal allocation of the device in bytes, if reported by the device. A value of '0' means device does not support the discard functionality. -discard_max_bytes (RO) +discard_max_hw_bytes (RO) ---------------------- Devices that support discard functionality may have internal limits on the number of bytes that can be trimmed or unmapped in a single operation. @@ -29,6 +29,14 @@ number of bytes that can be discarded in a single operation. Discard requests issued to the device must not exceed this limit. A discard_max_bytes value of 0 means that the device does not support discard functionality. +discard_max_bytes (RW) +---------------------- +While discard_max_hw_bytes is the hardware limit for the device, this +setting is the software limit. Some devices exhibit large latencies when +large discards are issued, setting this value lower will make Linux issue +smaller discards and potentially help reduce latencies induced by large +discard operations. + discard_zeroes_data (RO) ------------------------ When read, this file will show if the discarded block are zeroed by the diff --git a/Documentation/blockdev/zram.txt b/Documentation/blockdev/zram.txt index c4de576093af..62435bb25266 100644 --- a/Documentation/blockdev/zram.txt +++ b/Documentation/blockdev/zram.txt @@ -144,7 +144,8 @@ mem_used_max RW the maximum amount memory zram have consumed to store compressed data mem_limit RW the maximum amount of memory ZRAM can use to store the compressed data -num_migrated RO the number of objects migrated migrated by compaction +pages_compacted RO the number of pages freed during compaction + (available only via zram/mm_stat node) compact WO trigger memory compaction WARNING diff --git a/Documentation/cgroups/00-INDEX b/Documentation/cgroups/00-INDEX index 96ce071a3633..3f5a40f57d4a 100644 --- a/Documentation/cgroups/00-INDEX +++ b/Documentation/cgroups/00-INDEX @@ -22,6 +22,8 @@ net_cls.txt - Network classifier cgroups details and usages. net_prio.txt - Network priority cgroups details and usages. +pids.txt + - Process number cgroups details and usages. resource_counter.txt - Resource Counter API. unified-hierarchy.txt diff --git a/Documentation/cgroups/blkio-controller.txt b/Documentation/cgroups/blkio-controller.txt index 68b6a6a470b0..12686bec37b9 100644 --- a/Documentation/cgroups/blkio-controller.txt +++ b/Documentation/cgroups/blkio-controller.txt @@ -201,7 +201,7 @@ Proportional weight policy files specifies the number of bytes. - blkio.io_serviced - - Number of IOs completed to/from the disk by the group. These + - Number of IOs (bio) issued to the disk by the group. These are further divided by the type of operation - read or write, sync or async. First two fields specify the major and minor number of the device, third field specifies the operation type and the fourth field @@ -327,18 +327,11 @@ Note: If both BW and IOPS rules are specified for a device, then IO is subjected to both the constraints. - blkio.throttle.io_serviced - - Number of IOs (bio) completed to/from the disk by the group (as - seen by throttling policy). These are further divided by the type - of operation - read or write, sync or async. First two fields specify - the major and minor number of the device, third field specifies the - operation type and the fourth field specifies the number of IOs. - - blkio.io_serviced does accounting as seen by CFQ and counts are in - number of requests (struct request). On the other hand, - blkio.throttle.io_serviced counts number of IO in terms of number - of bios as seen by throttling policy. These bios can later be - merged by elevator and total number of requests completed can be - lesser. + - Number of IOs (bio) issued to the disk by the group. These + are further divided by the type of operation - read or write, sync + or async. First two fields specify the major and minor number of the + device, third field specifies the operation type and the fourth field + specifies the number of IOs. - blkio.throttle.io_service_bytes - Number of bytes transferred to/from the disk by the group. These @@ -347,11 +340,6 @@ Note: If both BW and IOPS rules are specified for a device, then IO is device, third field specifies the operation type and the fourth field specifies the number of bytes. - These numbers should roughly be same as blkio.io_service_bytes as - updated by CFQ. The difference between two is that - blkio.io_service_bytes will not be updated if CFQ is not operating - on request queue. - Common files among various policies ----------------------------------- - blkio.reset_stats diff --git a/Documentation/cgroups/pids.txt b/Documentation/cgroups/pids.txt new file mode 100644 index 000000000000..1a078b5d281a --- /dev/null +++ b/Documentation/cgroups/pids.txt @@ -0,0 +1,85 @@ + Process Number Controller + ========================= + +Abstract +-------- + +The process number controller is used to allow a cgroup hierarchy to stop any +new tasks from being fork()'d or clone()'d after a certain limit is reached. + +Since it is trivial to hit the task limit without hitting any kmemcg limits in +place, PIDs are a fundamental resource. As such, PID exhaustion must be +preventable in the scope of a cgroup hierarchy by allowing resource limiting of +the number of tasks in a cgroup. + +Usage +----- + +In order to use the `pids` controller, set the maximum number of tasks in +pids.max (this is not available in the root cgroup for obvious reasons). The +number of processes currently in the cgroup is given by pids.current. + +Organisational operations are not blocked by cgroup policies, so it is possible +to have pids.current > pids.max. This can be done by either setting the limit to +be smaller than pids.current, or attaching enough processes to the cgroup such +that pids.current > pids.max. However, it is not possible to violate a cgroup +policy through fork() or clone(). fork() and clone() will return -EAGAIN if the +creation of a new process would cause a cgroup policy to be violated. + +To set a cgroup to have no limit, set pids.max to "max". This is the default for +all new cgroups (N.B. that PID limits are hierarchical, so the most stringent +limit in the hierarchy is followed). + +pids.current tracks all child cgroup hierarchies, so parent/pids.current is a +superset of parent/child/pids.current. + +Example +------- + +First, we mount the pids controller: +# mkdir -p /sys/fs/cgroup/pids +# mount -t cgroup -o pids none /sys/fs/cgroup/pids + +Then we create a hierarchy, set limits and attach processes to it: +# mkdir -p /sys/fs/cgroup/pids/parent/child +# echo 2 > /sys/fs/cgroup/pids/parent/pids.max +# echo $$ > /sys/fs/cgroup/pids/parent/cgroup.procs +# cat /sys/fs/cgroup/pids/parent/pids.current +2 +# + +It should be noted that attempts to overcome the set limit (2 in this case) will +fail: + +# cat /sys/fs/cgroup/pids/parent/pids.current +2 +# ( /bin/echo "Here's some processes for you." | cat ) +sh: fork: Resource temporary unavailable +# + +Even if we migrate to a child cgroup (which doesn't have a set limit), we will +not be able to overcome the most stringent limit in the hierarchy (in this case, +parent's): + +# echo $$ > /sys/fs/cgroup/pids/parent/child/cgroup.procs +# cat /sys/fs/cgroup/pids/parent/pids.current +2 +# cat /sys/fs/cgroup/pids/parent/child/pids.current +2 +# cat /sys/fs/cgroup/pids/parent/child/pids.max +max +# ( /bin/echo "Here's some processes for you." | cat ) +sh: fork: Resource temporary unavailable +# + +We can set a limit that is smaller than pids.current, which will stop any new +processes from being forked at all (note that the shell itself counts towards +pids.current): + +# echo 1 > /sys/fs/cgroup/pids/parent/pids.max +# /bin/echo "We can't even spawn a single process now." +sh: fork: Resource temporary unavailable +# echo 0 > /sys/fs/cgroup/pids/parent/pids.max +# /bin/echo "We can't even spawn a single process now." +sh: fork: Resource temporary unavailable +# diff --git a/Documentation/cgroups/unified-hierarchy.txt b/Documentation/cgroups/unified-hierarchy.txt index 86847a7647ab..e0975c2cf03d 100644 --- a/Documentation/cgroups/unified-hierarchy.txt +++ b/Documentation/cgroups/unified-hierarchy.txt @@ -23,10 +23,13 @@ CONTENTS 5. Other Changes 5-1. [Un]populated Notification 5-2. Other Core Changes - 5-3. Per-Controller Changes - 5-3-1. blkio - 5-3-2. cpuset - 5-3-3. memory + 5-3. Controller File Conventions + 5-3-1. Format + 5-3-2. Control Knobs + 5-4. Per-Controller Changes + 5-4-1. io + 5-4-2. cpuset + 5-4-3. memory 6. Planned Changes 6-1. CAP for resource control @@ -200,7 +203,7 @@ other issues. The mapping from nice level to weight isn't obvious or universal, and there are various other knobs which simply aren't available for tasks. -The blkio controller implicitly creates a hidden leaf node for each +The io controller implicitly creates a hidden leaf node for each cgroup to host the tasks. The hidden leaf has its own copies of all the knobs with "leaf_" prefixed. While this allows equivalent control over internal tasks, it's with serious drawbacks. It always adds an @@ -372,14 +375,128 @@ supported and the interface files "release_agent" and - The "cgroup.clone_children" file is removed. -5-3. Per-Controller Changes +5-3. Controller File Conventions -5-3-1. blkio +5-3-1. Format -- blk-throttle becomes properly hierarchical. +In general, all controller files should be in one of the following +formats whenever possible. + +- Values only files + + VAL0 VAL1...\n + +- Flat keyed files + + KEY0 VAL0\n + KEY1 VAL1\n + ... + +- Nested keyed files + + KEY0 SUB_KEY0=VAL00 SUB_KEY1=VAL01... + KEY1 SUB_KEY0=VAL10 SUB_KEY1=VAL11... + ... + +For a writeable file, the format for writing should generally match +reading; however, controllers may allow omitting later fields or +implement restricted shortcuts for most common use cases. + +For both flat and nested keyed files, only the values for a single key +can be written at a time. For nested keyed files, the sub key pairs +may be specified in any order and not all pairs have to be specified. -5-3-2. cpuset +5-3-2. Control Knobs + +- Settings for a single feature should generally be implemented in a + single file. + +- In general, the root cgroup should be exempt from resource control + and thus shouldn't have resource control knobs. + +- If a controller implements ratio based resource distribution, the + control knob should be named "weight" and have the range [1, 10000] + and 100 should be the default value. The values are chosen to allow + enough and symmetric bias in both directions while keeping it + intuitive (the default is 100%). + +- If a controller implements an absolute resource guarantee and/or + limit, the control knobs should be named "min" and "max" + respectively. If a controller implements best effort resource + gurantee and/or limit, the control knobs should be named "low" and + "high" respectively. + + In the above four control files, the special token "max" should be + used to represent upward infinity for both reading and writing. + +- If a setting has configurable default value and specific overrides, + the default settings should be keyed with "default" and appear as + the first entry in the file. Specific entries can use "default" as + its value to indicate inheritance of the default value. + + +5-4. Per-Controller Changes + +5-4-1. io + +- blkio is renamed to io. The interface is overhauled anyway. The + new name is more in line with the other two major controllers, cpu + and memory, and better suited given that it may be used for cgroup + writeback without involving block layer. + +- Everything including stat is always hierarchical making separate + recursive stat files pointless and, as no internal node can have + tasks, leaf weights are meaningless. The operation model is + simplified and the interface is overhauled accordingly. + + io.stat + + The stat file. The reported stats are from the point where + bio's are issued to request_queue. The stats are counted + independent of which policies are enabled. Each line in the + file follows the following format. More fields may later be + added at the end. + + $MAJ:$MIN rbytes=$RBYTES wbytes=$WBYTES rios=$RIOS wrios=$WIOS + + io.weight + + The weight setting, currently only available and effective if + cfq-iosched is in use for the target device. The weight is + between 1 and 10000 and defaults to 100. The first line + always contains the default weight in the following format to + use when per-device setting is missing. + + default $WEIGHT + + Subsequent lines list per-device weights of the following + format. + + $MAJ:$MIN $WEIGHT + + Writing "$WEIGHT" or "default $WEIGHT" changes the default + setting. Writing "$MAJ:$MIN $WEIGHT" sets per-device weight + while "$MAJ:$MIN default" clears it. + + This file is available only on non-root cgroups. + + io.max + + The maximum bandwidth and/or iops setting, only available if + blk-throttle is enabled. The file is of the following format. + + $MAJ:$MIN rbps=$RBPS wbps=$WBPS riops=$RIOPS wiops=$WIOPS + + ${R|W}BPS are read/write bytes per second and ${R|W}IOPS are + read/write IOs per second. "max" indicates no limit. Writing + to the file follows the same format but the individual + settings may be ommitted or specified in any order. + + This file is available only on non-root cgroups. + + +5-4-2. cpuset - Tasks are kept in empty cpusets after hotplug and take on the masks of the nearest non-empty ancestor, instead of being moved to it. @@ -388,7 +505,7 @@ supported and the interface files "release_agent" and masks of the nearest non-empty ancestor. -5-3-3. memory +5-4-3. memory - use_hierarchy is on by default and the cgroup file for the flag is not created. diff --git a/Documentation/cpu-freq/core.txt b/Documentation/cpu-freq/core.txt index 70933eadc308..ba78e7c2a069 100644 --- a/Documentation/cpu-freq/core.txt +++ b/Documentation/cpu-freq/core.txt @@ -55,16 +55,13 @@ transition notifiers. ---------------------------- These are notified when a new policy is intended to be set. Each -CPUFreq policy notifier is called three times for a policy transition: +CPUFreq policy notifier is called twice for a policy transition: 1.) During CPUFREQ_ADJUST all CPUFreq notifiers may change the limit if they see a need for this - may it be thermal considerations or hardware limitations. -2.) During CPUFREQ_INCOMPATIBLE only changes may be done in order to avoid - hardware failure. - -3.) And during CPUFREQ_NOTIFY all notifiers are informed of the new policy +2.) And during CPUFREQ_NOTIFY all notifiers are informed of the new policy - if two hardware drivers failed to agree on a new policy before this stage, the incompatible hardware shall be shut down, and the user informed of this. diff --git a/Documentation/device-mapper/dm-raid.txt b/Documentation/device-mapper/dm-raid.txt index cb12af3b51c2..df2d636b6088 100644 --- a/Documentation/device-mapper/dm-raid.txt +++ b/Documentation/device-mapper/dm-raid.txt @@ -209,6 +209,37 @@ include: "repair" - Initiate a repair of the array. "reshape"- Currently unsupported (-EINVAL). + +Discard Support +--------------- +The implementation of discard support among hardware vendors varies. +When a block is discarded, some storage devices will return zeroes when +the block is read. These devices set the 'discard_zeroes_data' +attribute. Other devices will return random data. Confusingly, some +devices that advertise 'discard_zeroes_data' will not reliably return +zeroes when discarded blocks are read! Since RAID 4/5/6 uses blocks +from a number of devices to calculate parity blocks and (for performance +reasons) relies on 'discard_zeroes_data' being reliable, it is important +that the devices be consistent. Blocks may be discarded in the middle +of a RAID 4/5/6 stripe and if subsequent read results are not +consistent, the parity blocks may be calculated differently at any time; +making the parity blocks useless for redundancy. It is important to +understand how your hardware behaves with discards if you are going to +enable discards with RAID 4/5/6. + +Since the behavior of storage devices is unreliable in this respect, +even when reporting 'discard_zeroes_data', by default RAID 4/5/6 +discard support is disabled -- this ensures data integrity at the +expense of losing some performance. + +Storage devices that properly support 'discard_zeroes_data' are +increasingly whitelisted in the kernel and can thus be trusted. + +For trusted devices, the following dm-raid module parameter can be set +to safely enable discard support for RAID 4/5/6: + 'devices_handle_discards_safely' + + Version History --------------- 1.0.0 Initial version. Support for RAID 4/5/6 diff --git a/Documentation/device-mapper/statistics.txt b/Documentation/device-mapper/statistics.txt index 4919b2dfd1b3..6f5ef944ca4c 100644 --- a/Documentation/device-mapper/statistics.txt +++ b/Documentation/device-mapper/statistics.txt @@ -121,6 +121,10 @@ Messages Output format: : + + precise_timestamps histogram:n1,n2,n3,... + + The strings "precise_timestamps" and "histogram" are printed only + if they were specified when creating the region. @stats_print [ ] diff --git a/Documentation/devicetree/bindings/arm/bcm/ns2.txt b/Documentation/devicetree/bindings/arm/bcm/ns2.txt new file mode 100644 index 000000000000..35f056f4a1c3 --- /dev/null +++ b/Documentation/devicetree/bindings/arm/bcm/ns2.txt @@ -0,0 +1,9 @@ +Broadcom North Star 2 (NS2) device tree bindings +------------------------------------------------ + +Boards with NS2 shall have the following properties: + +Required root node property: + +NS2 SVK board +compatible = "brcm,ns2-svk", "brcm,ns2"; diff --git a/Documentation/devicetree/bindings/arm/l2cc.txt b/Documentation/devicetree/bindings/arm/l2cc.txt index 2251dccb141e..06c88a4d28ac 100644 --- a/Documentation/devicetree/bindings/arm/l2cc.txt +++ b/Documentation/devicetree/bindings/arm/l2cc.txt @@ -67,6 +67,12 @@ Optional properties: disable if zero. - arm,prefetch-offset : Override prefetch offset value. Valid values are 0-7, 15, 23, and 31. +- arm,shared-override : The default behavior of the pl310 cache controller with + respect to the shareable attribute is to transform "normal memory + non-cacheable transactions" into "cacheable no allocate" (for reads) or + "write through no write allocate" (for writes). + On systems where this may cause DMA buffer corruption, this property must be + specified to indicate that such transforms are precluded. - prefetch-data : Data prefetch. Value: <0> (forcibly disable), <1> (forcibly enable), property absent (retain settings set by firmware) - prefetch-instr : Instruction prefetch. Value: <0> (forcibly disable), diff --git a/Documentation/devicetree/bindings/arm/mediatek.txt b/Documentation/devicetree/bindings/arm/mediatek.txt index 2daa424a13a4..618a91994a18 100644 --- a/Documentation/devicetree/bindings/arm/mediatek.txt +++ b/Documentation/devicetree/bindings/arm/mediatek.txt @@ -1,6 +1,7 @@ -MediaTek mt65xx & mt81xx Platforms Device Tree Bindings +MediaTek mt65xx, mt67xx & mt81xx Platforms Device Tree Bindings -Boards with a MediaTek mt65xx/mt81xx SoC shall have the following property: +Boards with a MediaTek mt65xx/mt67xx/mt81xx SoC shall have the +following property: Required root node property: @@ -8,6 +9,7 @@ compatible: Must contain one of "mediatek,mt6580" "mediatek,mt6589" "mediatek,mt6592" + "mediatek,mt6795" "mediatek,mt8127" "mediatek,mt8135" "mediatek,mt8173" @@ -24,6 +26,9 @@ Supported boards: - Evaluation board for MT6592: Required root node properties: - compatible = "mediatek,mt6592-evb", "mediatek,mt6592"; +- Evaluation board for MT6795(Helio X10): + Required root node properties: + - compatible = "mediatek,mt6795-evb", "mediatek,mt6795"; - MTK mt8127 tablet moose EVB: Required root node properties: - compatible = "mediatek,mt8127-moose", "mediatek,mt8127"; diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,sysirq.txt b/Documentation/devicetree/bindings/arm/mediatek/mediatek,sysirq.txt index 3c9c3a7f3d25..afef6a85ac51 100644 --- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,sysirq.txt +++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,sysirq.txt @@ -1,4 +1,4 @@ -Mediatek 65xx/81xx sysirq ++Mediatek 65xx/67xx/81xx sysirq Mediatek SOCs sysirq support controllable irq inverter for each GIC SPI interrupt. @@ -8,6 +8,7 @@ Required properties: "mediatek,mt8173-sysirq" "mediatek,mt8135-sysirq" "mediatek,mt8127-sysirq" + "mediatek,mt6795-sysirq" "mediatek,mt6592-sysirq" "mediatek,mt6589-sysirq" "mediatek,mt6582-sysirq" diff --git a/Documentation/devicetree/bindings/arm/pmu.txt b/Documentation/devicetree/bindings/arm/pmu.txt index 3b5f5d1088c6..435251fa9ce0 100644 --- a/Documentation/devicetree/bindings/arm/pmu.txt +++ b/Documentation/devicetree/bindings/arm/pmu.txt @@ -26,13 +26,19 @@ Required properties: Optional properties: -- interrupt-affinity : Valid only when using SPIs, specifies a list of phandles - to CPU nodes corresponding directly to the affinity of +- interrupt-affinity : When using SPIs, specifies a list of phandles to CPU + nodes corresponding directly to the affinity of the SPIs listed in the interrupts property. - This property should be present when there is more than + When using a PPI, specifies a list of phandles to CPU + nodes corresponding to the set of CPUs which have + a PMU of this type signalling the PPI listed in the + interrupts property. + + This property should be present when there is more than a single SPI. + - qcom,no-pc-write : Indicates that this PMU doesn't support the 0xc and 0xd events. diff --git a/Documentation/devicetree/bindings/clock/mt8173-cpu-dvfs.txt b/Documentation/devicetree/bindings/clock/mt8173-cpu-dvfs.txt new file mode 100644 index 000000000000..52b457c23eed --- /dev/null +++ b/Documentation/devicetree/bindings/clock/mt8173-cpu-dvfs.txt @@ -0,0 +1,83 @@ +Device Tree Clock bindins for CPU DVFS of Mediatek MT8173 SoC + +Required properties: +- clocks: A list of phandle + clock-specifier pairs for the clocks listed in clock names. +- clock-names: Should contain the following: + "cpu" - The multiplexer for clock input of CPU cluster. + "intermediate" - A parent of "cpu" clock which is used as "intermediate" clock + source (usually MAINPLL) when the original CPU PLL is under + transition and not stable yet. + Please refer to Documentation/devicetree/bindings/clk/clock-bindings.txt for + generic clock consumer properties. +- proc-supply: Regulator for Vproc of CPU cluster. + +Optional properties: +- sram-supply: Regulator for Vsram of CPU cluster. When present, the cpufreq driver + needs to do "voltage tracking" to step by step scale up/down Vproc and + Vsram to fit SoC specific needs. When absent, the voltage scaling + flow is handled by hardware, hence no software "voltage tracking" is + needed. + +Example: +-------- + cpu0: cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a53"; + reg = <0x000>; + enable-method = "psci"; + cpu-idle-states = <&CPU_SLEEP_0>; + clocks = <&infracfg CLK_INFRA_CA53SEL>, + <&apmixedsys CLK_APMIXED_MAINPLL>; + clock-names = "cpu", "intermediate"; + }; + + cpu1: cpu@1 { + device_type = "cpu"; + compatible = "arm,cortex-a53"; + reg = <0x001>; + enable-method = "psci"; + cpu-idle-states = <&CPU_SLEEP_0>; + clocks = <&infracfg CLK_INFRA_CA53SEL>, + <&apmixedsys CLK_APMIXED_MAINPLL>; + clock-names = "cpu", "intermediate"; + }; + + cpu2: cpu@100 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x100>; + enable-method = "psci"; + cpu-idle-states = <&CPU_SLEEP_0>; + clocks = <&infracfg CLK_INFRA_CA57SEL>, + <&apmixedsys CLK_APMIXED_MAINPLL>; + clock-names = "cpu", "intermediate"; + }; + + cpu3: cpu@101 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x101>; + enable-method = "psci"; + cpu-idle-states = <&CPU_SLEEP_0>; + clocks = <&infracfg CLK_INFRA_CA57SEL>, + <&apmixedsys CLK_APMIXED_MAINPLL>; + clock-names = "cpu", "intermediate"; + }; + + &cpu0 { + proc-supply = <&mt6397_vpca15_reg>; + }; + + &cpu1 { + proc-supply = <&mt6397_vpca15_reg>; + }; + + &cpu2 { + proc-supply = <&da9211_vcpu_reg>; + sram-supply = <&mt6397_vsramca7_reg>; + }; + + &cpu3 { + proc-supply = <&da9211_vcpu_reg>; + sram-supply = <&mt6397_vsramca7_reg>; + }; diff --git a/Documentation/devicetree/bindings/devfreq/event/exynos-ppmu.txt b/Documentation/devicetree/bindings/devfreq/event/exynos-ppmu.txt index b54bf3a2ff57..3e36c1d11386 100644 --- a/Documentation/devicetree/bindings/devfreq/event/exynos-ppmu.txt +++ b/Documentation/devicetree/bindings/devfreq/event/exynos-ppmu.txt @@ -11,15 +11,14 @@ to various devfreq devices. The devfreq devices would use the event data when derterming the current state of each IP. Required properties: -- compatible: Should be "samsung,exynos-ppmu". +- compatible: Should be "samsung,exynos-ppmu" or "samsung,exynos-ppmu-v2. - reg: physical base address of each PPMU and length of memory mapped region. Optional properties: - clock-names : the name of clock used by the PPMU, "ppmu" - clocks : phandles for clock specified in "clock-names" property -- #clock-cells: should be 1. -Example1 : PPMU nodes in exynos3250.dtsi are listed below. +Example1 : PPMUv1 nodes in exynos3250.dtsi are listed below. ppmu_dmc0: ppmu_dmc0@106a0000 { compatible = "samsung,exynos-ppmu"; @@ -108,3 +107,41 @@ Example2 : Events of each PPMU node in exynos3250-rinato.dts are listed below. }; }; }; + +Example3 : PPMUv2 nodes in exynos5433.dtsi are listed below. + + ppmu_d0_cpu: ppmu_d0_cpu@10480000 { + compatible = "samsung,exynos-ppmu-v2"; + reg = <0x10480000 0x2000>; + status = "disabled"; + }; + + ppmu_d0_general: ppmu_d0_general@10490000 { + compatible = "samsung,exynos-ppmu-v2"; + reg = <0x10490000 0x2000>; + status = "disabled"; + }; + + ppmu_d0_rt: ppmu_d0_rt@104a0000 { + compatible = "samsung,exynos-ppmu-v2"; + reg = <0x104a0000 0x2000>; + status = "disabled"; + }; + + ppmu_d1_cpu: ppmu_d1_cpu@104b0000 { + compatible = "samsung,exynos-ppmu-v2"; + reg = <0x104b0000 0x2000>; + status = "disabled"; + }; + + ppmu_d1_general: ppmu_d1_general@104c0000 { + compatible = "samsung,exynos-ppmu-v2"; + reg = <0x104c0000 0x2000>; + status = "disabled"; + }; + + ppmu_d1_rt: ppmu_d1_rt@104d0000 { + compatible = "samsung,exynos-ppmu-v2"; + reg = <0x104d0000 0x2000>; + status = "disabled"; + }; diff --git a/Documentation/devicetree/bindings/dma/adi,axi-dmac.txt b/Documentation/devicetree/bindings/dma/adi,axi-dmac.txt new file mode 100644 index 000000000000..47cb1d14b690 --- /dev/null +++ b/Documentation/devicetree/bindings/dma/adi,axi-dmac.txt @@ -0,0 +1,61 @@ +Analog Device AXI-DMAC DMA controller + +Required properties: + - compatible: Must be "adi,axi-dmac-1.00.a". + - reg: Specification for the controllers memory mapped register map. + - interrupts: Specification for the controllers interrupt. + - clocks: Phandle and specifier to the controllers AXI interface clock + - #dma-cells: Must be 1. + +Required sub-nodes: + - adi,channels: This sub-node must contain a sub-node for each DMA channel. For + the channel sub-nodes the following bindings apply. They must match the + configuration options of the peripheral as it was instantiated. + +Required properties for adi,channels sub-node: + - #size-cells: Must be 0 + - #address-cells: Must be 1 + +Required channel sub-node properties: + - reg: Which channel this node refers to. + - adi,length-width: Width of the DMA transfer length register. + - adi,source-bus-width, + adi,destination-bus-width: Width of the source or destination bus in bits. + - adi,source-bus-type, + adi,destination-bus-type: Type of the source or destination bus. Must be one + of the following: + 0 (AXI_DMAC_TYPE_AXI_MM): Memory mapped AXI interface + 1 (AXI_DMAC_TYPE_AXI_STREAM): Streaming AXI interface + 2 (AXI_DMAC_TYPE_AXI_FIFO): FIFO interface + +Optional channel properties: + - adi,cyclic: Must be set if the channel supports hardware cyclic DMA + transfers. + - adi,2d: Must be set if the channel supports hardware 2D DMA transfers. + +DMA clients connected to the AXI-DMAC DMA controller must use the format +described in the dma.txt file using a one-cell specifier. The value of the +specifier refers to the DMA channel index. + +Example: + +dma: dma@7c420000 { + compatible = "adi,axi-dmac-1.00.a"; + reg = <0x7c420000 0x10000>; + interrupts = <0 57 0>; + clocks = <&clkc 16>; + #dma-cells = <1>; + + adi,channels { + #size-cells = <0>; + #address-cells = <1>; + + dma-channel@0 { + reg = <0>; + adi,source-bus-width = <32>; + adi,source-bus-type = ; + adi,destination-bus-width = <64>; + adi,destination-bus-type = ; + }; + }; +}; diff --git a/Documentation/devicetree/bindings/dma/arm-pl08x.txt b/Documentation/devicetree/bindings/dma/arm-pl08x.txt new file mode 100644 index 000000000000..8a0097a029d3 --- /dev/null +++ b/Documentation/devicetree/bindings/dma/arm-pl08x.txt @@ -0,0 +1,54 @@ +* ARM PrimeCells PL080 and PL081 and derivatives DMA controller + +Required properties: +- compatible: "arm,pl080", "arm,primecell"; + "arm,pl081", "arm,primecell"; +- reg: Address range of the PL08x registers +- interrupt: The PL08x interrupt number +- clocks: The clock running the IP core clock +- clock-names: Must contain "apb_pclk" +- lli-bus-interface-ahb1: if AHB master 1 is eligible for fetching LLIs +- lli-bus-interface-ahb2: if AHB master 2 is eligible for fetching LLIs +- mem-bus-interface-ahb1: if AHB master 1 is eligible for fetching memory contents +- mem-bus-interface-ahb2: if AHB master 2 is eligible for fetching memory contents +- #dma-cells: must be <2>. First cell should contain the DMA request, + second cell should contain either 1 or 2 depending on + which AHB master that is used. + +Optional properties: +- dma-channels: contains the total number of DMA channels supported by the DMAC +- dma-requests: contains the total number of DMA requests supported by the DMAC +- memcpy-burst-size: the size of the bursts for memcpy: 1, 4, 8, 16, 32 + 64, 128 or 256 bytes are legal values +- memcpy-bus-width: the bus width used for memcpy: 8, 16 or 32 are legal + values + +Clients +Required properties: +- dmas: List of DMA controller phandle, request channel and AHB master id +- dma-names: Names of the aforementioned requested channels + +Example: + +dmac0: dma-controller@10130000 { + compatible = "arm,pl080", "arm,primecell"; + reg = <0x10130000 0x1000>; + interrupt-parent = <&vica>; + interrupts = <15>; + clocks = <&hclkdma0>; + clock-names = "apb_pclk"; + lli-bus-interface-ahb1; + lli-bus-interface-ahb2; + mem-bus-interface-ahb2; + memcpy-burst-size = <256>; + memcpy-bus-width = <32>; + #dma-cells = <2>; +}; + +device@40008000 { + ... + dmas = <&dmac0 0 2 + &dmac0 1 2>; + dma-names = "tx", "rx"; + ... +}; diff --git a/Documentation/devicetree/bindings/dma/lpc1850-dmamux.txt b/Documentation/devicetree/bindings/dma/lpc1850-dmamux.txt new file mode 100644 index 000000000000..87740adb2995 --- /dev/null +++ b/Documentation/devicetree/bindings/dma/lpc1850-dmamux.txt @@ -0,0 +1,54 @@ +NXP LPC18xx/43xx DMA MUX (DMA request router) + +Required properties: +- compatible: "nxp,lpc1850-dmamux" +- reg: Memory map for accessing module +- #dma-cells: Should be set to <3>. + * 1st cell contain the master dma request signal + * 2nd cell contain the mux value (0-3) for the peripheral + * 3rd cell contain either 1 or 2 depending on the AHB + master used. +- dma-requests: Number of DMA requests for the mux +- dma-masters: phandle pointing to the DMA controller + +The DMA controller node need to have the following poroperties: +- dma-requests: Number of DMA requests the controller can handle + +Example: + +dmac: dma@40002000 { + compatible = "nxp,lpc1850-gpdma", "arm,pl080", "arm,primecell"; + arm,primecell-periphid = <0x00041080>; + reg = <0x40002000 0x1000>; + interrupts = <2>; + clocks = <&ccu1 CLK_CPU_DMA>; + clock-names = "apb_pclk"; + #dma-cells = <2>; + dma-channels = <8>; + dma-requests = <16>; + lli-bus-interface-ahb1; + lli-bus-interface-ahb2; + mem-bus-interface-ahb1; + mem-bus-interface-ahb2; + memcpy-burst-size = <256>; + memcpy-bus-width = <32>; +}; + +dmamux: dma-mux { + compatible = "nxp,lpc1850-dmamux"; + #dma-cells = <3>; + dma-requests = <64>; + dma-masters = <&dmac>; +}; + +uart0: serial@40081000 { + compatible = "nxp,lpc1850-uart", "ns16550a"; + reg = <0x40081000 0x1000>; + reg-shift = <2>; + interrupts = <24>; + clocks = <&ccu2 CLK_APB0_UART0>, <&ccu1 CLK_CPU_UART0>; + clock-names = "uartclk", "reg"; + dmas = <&dmamux 1 1 2 + &dmamux 2 1 2>; + dma-names = "tx", "rx"; +}; diff --git a/Documentation/devicetree/bindings/dma/mv-xor.txt b/Documentation/devicetree/bindings/dma/mv-xor.txt index cc29c35266e2..276ef815ef32 100644 --- a/Documentation/devicetree/bindings/dma/mv-xor.txt +++ b/Documentation/devicetree/bindings/dma/mv-xor.txt @@ -12,10 +12,13 @@ XOR engine has. Those sub-nodes have the following required properties: - interrupts: interrupt of the XOR channel -And the following optional properties: +The sub-nodes used to contain one or several of the following +properties, but they are now deprecated: - dmacap,memcpy to indicate that the XOR channel is capable of memcpy operations - dmacap,memset to indicate that the XOR channel is capable of memset operations - dmacap,xor to indicate that the XOR channel is capable of xor operations +- dmacap,interrupt to indicate that the XOR channel is capable of + generating interrupts Example: @@ -28,13 +31,8 @@ xor@d0060900 { xor00 { interrupts = <51>; - dmacap,memcpy; - dmacap,xor; }; xor01 { interrupts = <52>; - dmacap,memcpy; - dmacap,xor; - dmacap,memset; }; }; diff --git a/Documentation/devicetree/bindings/dma/sun4i-dma.txt b/Documentation/devicetree/bindings/dma/sun4i-dma.txt new file mode 100644 index 000000000000..f1634a27a830 --- /dev/null +++ b/Documentation/devicetree/bindings/dma/sun4i-dma.txt @@ -0,0 +1,46 @@ +Allwinner A10 DMA Controller + +This driver follows the generic DMA bindings defined in dma.txt. + +Required properties: + +- compatible: Must be "allwinner,sun4i-a10-dma" +- reg: Should contain the registers base address and length +- interrupts: Should contain a reference to the interrupt used by this device +- clocks: Should contain a reference to the parent AHB clock +- #dma-cells : Should be 2, first cell denoting normal or dedicated dma, + second cell holding the request line number. + +Example: + dma: dma-controller@01c02000 { + compatible = "allwinner,sun4i-a10-dma"; + reg = <0x01c02000 0x1000>; + interrupts = <27>; + clocks = <&ahb_gates 6>; + #dma-cells = <2>; + }; + +Clients: + +DMA clients connected to the Allwinner A10 DMA controller must use the +format described in the dma.txt file, using a three-cell specifier for +each channel: a phandle plus two integer cells. +The three cells in order are: + +1. A phandle pointing to the DMA controller. +2. Whether it is using normal (0) or dedicated (1) channels +3. The port ID as specified in the datasheet + +Example: + spi2: spi@01c17000 { + compatible = "allwinner,sun4i-a10-spi"; + reg = <0x01c17000 0x1000>; + interrupts = <0 12 4>; + clocks = <&ahb_gates 22>, <&spi2_clk>; + clock-names = "ahb", "mod"; + dmas = <&dma 1 29>, <&dma 1 28>; + dma-names = "rx", "tx"; + status = "disabled"; + #address-cells = <1>; + #size-cells = <0>; + }; diff --git a/Documentation/devicetree/bindings/dma/zxdma.txt b/Documentation/devicetree/bindings/dma/zxdma.txt new file mode 100644 index 000000000000..3207ceb04d0b --- /dev/null +++ b/Documentation/devicetree/bindings/dma/zxdma.txt @@ -0,0 +1,38 @@ +* ZTE ZX296702 DMA controller + +Required properties: +- compatible: Should be "zte,zx296702-dma" +- reg: Should contain DMA registers location and length. +- interrupts: Should contain one interrupt shared by all channel +- #dma-cells: see dma.txt, should be 1, para number +- dma-channels: physical channels supported +- dma-requests: virtual channels supported, each virtual channel + have specific request line +- clocks: clock required + +Example: + +Controller: + dma: dma-controller@0x09c00000{ + compatible = "zte,zx296702-dma"; + reg = <0x09c00000 0x1000>; + clocks = <&topclk ZX296702_DMA_ACLK>; + interrupts = ; + #dma-cells = <1>; + dma-channels = <24>; + dma-requests = <24>; + }; + +Client: +Use specific request line passing from dmax +For example, spdif0 tx channel request line is 4 + spdif0: spdif0@0b004000 { + #sound-dai-cells = <0>; + compatible = "zte,zx296702-spdif"; + reg = <0x0b004000 0x1000>; + clocks = <&lsp0clk ZX296702_SPDIF0_DIV>; + clock-names = "tx"; + interrupts = ; + dmas = <&dma 4>; + dma-names = "tx"; + } diff --git a/Documentation/devicetree/bindings/drm/msm/dsi.txt b/Documentation/devicetree/bindings/drm/msm/dsi.txt index cd8fe6cf536c..d56923cd5590 100644 --- a/Documentation/devicetree/bindings/drm/msm/dsi.txt +++ b/Documentation/devicetree/bindings/drm/msm/dsi.txt @@ -30,20 +30,27 @@ Optional properties: - panel@0: Node of panel connected to this DSI controller. See files in Documentation/devicetree/bindings/panel/ for each supported panel. -- qcom,dual-panel-mode: Boolean value indicating if the DSI controller is +- qcom,dual-dsi-mode: Boolean value indicating if the DSI controller is driving a panel which needs 2 DSI links. -- qcom,master-panel: Boolean value indicating if the DSI controller is driving +- qcom,master-dsi: Boolean value indicating if the DSI controller is driving the master link of the 2-DSI panel. -- qcom,sync-dual-panel: Boolean value indicating if the DSI controller is +- qcom,sync-dual-dsi: Boolean value indicating if the DSI controller is driving a 2-DSI panel whose 2 links need receive command simultaneously. - interrupt-parent: phandle to the MDP block if the interrupt signal is routed through MDP block +- pinctrl-names: the pin control state names; should contain "default" +- pinctrl-0: the default pinctrl state (active) +- pinctrl-n: the "sleep" pinctrl state +- port: DSI controller output port. This contains one endpoint subnode, with its + remote-endpoint set to the phandle of the connected panel's endpoint. + See Documentation/devicetree/bindings/graph.txt for device graph info. DSI PHY: Required properties: - compatible: Could be the following * "qcom,dsi-phy-28nm-hpm" * "qcom,dsi-phy-28nm-lp" + * "qcom,dsi-phy-20nm" - reg: Physical base address and length of the registers of PLL, PHY and PHY regulator - reg-names: The names of register regions. The following regions are required: @@ -59,6 +66,10 @@ Required properties: * "iface_clk" - vddio-supply: phandle to vdd-io regulator device node +Optional properties: +- qcom,dsi-phy-regulator-ldo-mode: Boolean value indicating if the LDO mode PHY + regulator is wanted. + Example: mdss_dsi0: qcom,mdss_dsi@fd922800 { compatible = "qcom,mdss-dsi-ctrl"; @@ -90,9 +101,13 @@ Example: qcom,dsi-phy = <&mdss_dsi_phy0>; - qcom,dual-panel-mode; - qcom,master-panel; - qcom,sync-dual-panel; + qcom,dual-dsi-mode; + qcom,master-dsi; + qcom,sync-dual-dsi; + + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&mdss_dsi_active>; + pinctrl-1 = <&mdss_dsi_suspend>; panel: panel@0 { compatible = "sharp,lq101r1sx01"; @@ -101,6 +116,18 @@ Example: power-supply = <...>; backlight = <...>; + + port { + panel_in: endpoint { + remote-endpoint = <&dsi0_out>; + }; + }; + }; + + port { + dsi0_out: endpoint { + remote-endpoint = <&panel_in>; + }; }; }; @@ -117,4 +144,6 @@ Example: clock-names = "iface_clk"; clocks = <&mmcc MDSS_AHB_CLK>; vddio-supply = <&pma8084_l12>; + + qcom,dsi-phy-regulator-ldo-mode; }; diff --git a/Documentation/devicetree/bindings/drm/msm/hdmi.txt b/Documentation/devicetree/bindings/drm/msm/hdmi.txt index c43aa53debed..e926239e1101 100644 --- a/Documentation/devicetree/bindings/drm/msm/hdmi.txt +++ b/Documentation/devicetree/bindings/drm/msm/hdmi.txt @@ -2,8 +2,9 @@ Qualcomm adreno/snapdragon hdmi output Required properties: - compatible: one of the following + * "qcom,hdmi-tx-8994" * "qcom,hdmi-tx-8084" - * "qcom,hdmi-tx-8074" + * "qcom,hdmi-tx-8974" * "qcom,hdmi-tx-8660" * "qcom,hdmi-tx-8960" - reg: Physical base address and length of the controller's registers diff --git a/Documentation/devicetree/bindings/gpio/brcm,brcmstb-gpio.txt b/Documentation/devicetree/bindings/gpio/brcm,brcmstb-gpio.txt index 435f1bcca341..b405b4410bfb 100644 --- a/Documentation/devicetree/bindings/gpio/brcm,brcmstb-gpio.txt +++ b/Documentation/devicetree/bindings/gpio/brcm,brcmstb-gpio.txt @@ -33,6 +33,13 @@ Optional properties: - interrupt-parent: phandle of the parent interrupt controller +- interrupts-extended: + Alternate form of specifying interrupts and parents that allows for + multiple parents. This takes precedence over 'interrupts' and + 'interrupt-parent'. Wakeup-capable GPIO controllers often route their + wakeup interrupt lines through a different interrupt controller than the + primary interrupt line, making this property necessary. + - #interrupt-cells: Should be <2>. The first cell is the GPIO number, the second should specify flags. The following subset of flags is supported: @@ -47,19 +54,33 @@ Optional properties: - interrupt-controller: Marks the device node as an interrupt controller -- interrupt-names: - The name of the IRQ resource used by this controller +- wakeup-source: + GPIOs for this controller can be used as a wakeup source Example: upg_gio: gpio@f040a700 { - #gpio-cells = <0x2>; - #interrupt-cells = <0x2>; + #gpio-cells = <2>; + #interrupt-cells = <2>; compatible = "brcm,bcm7445-gpio", "brcm,brcmstb-gpio"; gpio-controller; interrupt-controller; reg = <0xf040a700 0x80>; - interrupt-parent = <0xf>; + interrupt-parent = <&irq0_intc>; interrupts = <0x6>; - interrupt-names = "upg_gio"; - brcm,gpio-bank-widths = <0x20 0x20 0x20 0x18>; + brcm,gpio-bank-widths = <32 32 32 24>; + }; + + upg_gio_aon: gpio@f04172c0 { + #gpio-cells = <2>; + #interrupt-cells = <2>; + compatible = "brcm,bcm7445-gpio", "brcm,brcmstb-gpio"; + gpio-controller; + interrupt-controller; + reg = <0xf04172c0 0x40>; + interrupt-parent = <&irq0_aon_intc>; + interrupts = <0x6>; + interrupts-extended = <&irq0_aon_intc 0x6>, + <&aon_pm_l2_intc 0x5>; + wakeup-source; + brcm,gpio-bank-widths = <18 4>; }; diff --git a/Documentation/devicetree/bindings/gpio/gpio-etraxfs.txt b/Documentation/devicetree/bindings/gpio/gpio-etraxfs.txt index abf4db736c6e..170194af3027 100644 --- a/Documentation/devicetree/bindings/gpio/gpio-etraxfs.txt +++ b/Documentation/devicetree/bindings/gpio/gpio-etraxfs.txt @@ -2,8 +2,9 @@ Axis ETRAX FS General I/O controller bindings Required properties: -- compatible: +- compatible: one of: - "axis,etraxfs-gio" + - "axis,artpec3-gio" - reg: Physical base address and length of the controller's registers. - #gpio-cells: Should be 3 - The first cell is the gpio offset number. diff --git a/Documentation/devicetree/bindings/gpio/gpio-mpc8xxx.txt b/Documentation/devicetree/bindings/gpio/gpio-mpc8xxx.txt new file mode 100644 index 000000000000..805ddcd79a57 --- /dev/null +++ b/Documentation/devicetree/bindings/gpio/gpio-mpc8xxx.txt @@ -0,0 +1,22 @@ +* Freescale MPC512x/MPC8xxx GPIO controller + +Required properties: +- compatible : Should be "fsl,-gpio" + The following s are known to be supported: + mpc5121, mpc5125, mpc8349, mpc8572, mpc8610, pq3, qoriq +- reg : Address and length of the register set for the device +- interrupts : Should be the port interrupt shared by all 32 pins. +- #gpio-cells : Should be two. The first cell is the pin number and + the second cell is used to specify the gpio polarity: + 0 = active high + 1 = active low + +Example: + +gpio0: gpio@1100 { + compatible = "fsl,mpc5125-gpio"; + #gpio-cells = <2>; + reg = <0x1100 0x080>; + interrupts = <78 0x8>; + status = "okay"; +}; diff --git a/Documentation/devicetree/bindings/gpio/renesas,gpio-rcar.txt b/Documentation/devicetree/bindings/gpio/renesas,gpio-rcar.txt index 38fb86f28ba2..f60e2f477e93 100644 --- a/Documentation/devicetree/bindings/gpio/renesas,gpio-rcar.txt +++ b/Documentation/devicetree/bindings/gpio/renesas,gpio-rcar.txt @@ -9,6 +9,7 @@ Required Properties: - "renesas,gpio-r8a7791": for R8A7791 (R-Car M2-W) compatible GPIO controller. - "renesas,gpio-r8a7793": for R8A7793 (R-Car M2-N) compatible GPIO controller. - "renesas,gpio-r8a7794": for R8A7794 (R-Car E2) compatible GPIO controller. + - "renesas,gpio-r8a7795": for R8A7795 (R-Car H3) compatible GPIO controller. - "renesas,gpio-rcar": for generic R-Car GPIO controller. - reg: Base address and length of each memory resource used by the GPIO diff --git a/Documentation/devicetree/bindings/gpio/zx296702-gpio.txt b/Documentation/devicetree/bindings/gpio/zx296702-gpio.txt new file mode 100644 index 000000000000..0dab156fcf41 --- /dev/null +++ b/Documentation/devicetree/bindings/gpio/zx296702-gpio.txt @@ -0,0 +1,24 @@ +ZTE ZX296702 GPIO controller + +Required properties: +- compatible : "zte,zx296702-gpio" +- #gpio-cells : Should be two. The first cell is the pin number and the + second cell is used to specify optional parameters: + - bit 0 specifies polarity (0 for normal, 1 for inverted) +- gpio-controller : Marks the device node as a GPIO controller. +- interrupts : Interrupt mapping for GPIO IRQ. +- gpio-ranges : Interaction with the PINCTRL subsystem. + +gpio1: gpio@b008040 { + compatible = "zte,zx296702-gpio"; + reg = <0xb008040 0x40>; + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = < &pmx0 0 54 2 &pmx0 2 59 14>; + interrupts = ; + interrupt-parent = <&intc>; + interrupt-controller; + #interrupt-cells = <2>; + clock-names = "gpio_pclk"; + clocks = <&lsp0clk ZX296702_GPIO_CLK>; +}; diff --git a/Documentation/devicetree/bindings/gpu/nvidia,tegra20-host1x.txt b/Documentation/devicetree/bindings/gpu/nvidia,tegra20-host1x.txt index 009f4bfa1590..e685610d38e2 100644 --- a/Documentation/devicetree/bindings/gpu/nvidia,tegra20-host1x.txt +++ b/Documentation/devicetree/bindings/gpu/nvidia,tegra20-host1x.txt @@ -197,9 +197,11 @@ of the following host1x client modules: - sor: serial output resource Required properties: - - compatible: For Tegra124, must contain "nvidia,tegra124-sor". Otherwise, - must contain '"nvidia,-sor", "nvidia,tegra124-sor"', where - is tegra132. + - compatible: Should be: + - "nvidia,tegra124-sor": for Tegra124 and Tegra132 + - "nvidia,tegra132-sor": for Tegra132 + - "nvidia,tegra210-sor": for Tegra210 + - "nvidia,tegra210-sor1": for Tegra210 - reg: Physical base address and length of the controller's registers. - interrupts: The interrupt outputs from the controller. - clocks: Must contain an entry for each entry in clock-names. diff --git a/Documentation/devicetree/bindings/gpu/st,stih4xx.txt b/Documentation/devicetree/bindings/gpu/st,stih4xx.txt index 6b1d75f1a529..a36dfce0032e 100644 --- a/Documentation/devicetree/bindings/gpu/st,stih4xx.txt +++ b/Documentation/devicetree/bindings/gpu/st,stih4xx.txt @@ -52,10 +52,9 @@ STMicroelectronics stih4xx platforms See ../reset/reset.txt for details. - reset-names: names of the resets listed in resets property in the same order. - - ranges: to allow probing of subdevices - sti-hdmi: hdmi output block - must be a child of sti-tvout + must be a child of sti-display-subsystem Required properties: - compatible: "st,stih-hdmi"; - reg: Physical base address of the IP registers and length of memory mapped region. @@ -72,7 +71,7 @@ STMicroelectronics stih4xx platforms sti-hda: Required properties: - must be a child of sti-tvout + must be a child of sti-display-subsystem - compatible: "st,stih-hda" - reg: Physical base address of the IP registers and length of memory mapped region. - reg-names: names of the mapped memory regions listed in regs property in @@ -85,7 +84,7 @@ sti-hda: sti-dvo: Required properties: - must be a child of sti-tvout + must be a child of sti-display-subsystem - compatible: "st,stih-dvo" - reg: Physical base address of the IP registers and length of memory mapped region. - reg-names: names of the mapped memory regions listed in regs property in @@ -195,38 +194,37 @@ Example: reg-names = "tvout-reg", "hda-reg", "syscfg"; reset-names = "tvout"; resets = <&softreset STIH416_HDTVOUT_SOFTRESET>; - ranges; + }; - sti-hdmi@fe85c000 { - compatible = "st,stih416-hdmi"; - reg = <0xfe85c000 0x1000>, <0xfe830000 0x10000>; - reg-names = "hdmi-reg", "syscfg"; - interrupts = ; - interrupt-names = "irq"; - clock-names = "pix", "tmds", "phy", "audio"; - clocks = <&clockgen_c_vcc CLK_S_PIX_HDMI>, <&clockgen_c_vcc CLK_S_TMDS_HDMI>, <&clockgen_c_vcc CLK_S_HDMI_REJECT_PLL>, <&clockgen_b1 CLK_S_PCM_0>; - }; + sti-hdmi@fe85c000 { + compatible = "st,stih416-hdmi"; + reg = <0xfe85c000 0x1000>, <0xfe830000 0x10000>; + reg-names = "hdmi-reg", "syscfg"; + interrupts = ; + interrupt-names = "irq"; + clock-names = "pix", "tmds", "phy", "audio"; + clocks = <&clockgen_c_vcc CLK_S_PIX_HDMI>, <&clockgen_c_vcc CLK_S_TMDS_HDMI>, <&clockgen_c_vcc CLK_S_HDMI_REJECT_PLL>, <&clockgen_b1 CLK_S_PCM_0>; + }; - sti-hda@fe85a000 { - compatible = "st,stih416-hda"; - reg = <0xfe85a000 0x400>, <0xfe83085c 0x4>; - reg-names = "hda-reg", "video-dacs-ctrl"; - clock-names = "pix", "hddac"; - clocks = <&clockgen_c_vcc CLK_S_PIX_HD>, <&clockgen_c_vcc CLK_S_HDDAC>; - }; + sti-hda@fe85a000 { + compatible = "st,stih416-hda"; + reg = <0xfe85a000 0x400>, <0xfe83085c 0x4>; + reg-names = "hda-reg", "video-dacs-ctrl"; + clock-names = "pix", "hddac"; + clocks = <&clockgen_c_vcc CLK_S_PIX_HD>, <&clockgen_c_vcc CLK_S_HDDAC>; + }; - sti-dvo@8d00400 { - compatible = "st,stih407-dvo"; - reg = <0x8d00400 0x200>; - reg-names = "dvo-reg"; - clock-names = "dvo_pix", "dvo", - "main_parent", "aux_parent"; - clocks = <&clk_s_d2_flexgen CLK_PIX_DVO>, <&clk_s_d2_flexgen CLK_DVO>, - <&clk_s_d2_quadfs 0>, <&clk_s_d2_quadfs 1>; - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_dvo>; - sti,panel = <&panel_dvo>; - }; + sti-dvo@8d00400 { + compatible = "st,stih407-dvo"; + reg = <0x8d00400 0x200>; + reg-names = "dvo-reg"; + clock-names = "dvo_pix", "dvo", + "main_parent", "aux_parent"; + clocks = <&clk_s_d2_flexgen CLK_PIX_DVO>, <&clk_s_d2_flexgen CLK_DVO>, + <&clk_s_d2_quadfs 0>, <&clk_s_d2_quadfs 1>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_dvo>; + sti,panel = <&panel_dvo>; }; sti-hqvdp@9c000000 { @@ -237,7 +235,7 @@ Example: reset-names = "hqvdp"; resets = <&softreset STIH407_HDQVDP_SOFTRESET>; st,vtg = <&vtg_main>; - }; + }; }; ... }; diff --git a/Documentation/devicetree/bindings/i2c/ina209.txt b/Documentation/devicetree/bindings/hwmon/ina209.txt similarity index 100% rename from Documentation/devicetree/bindings/i2c/ina209.txt rename to Documentation/devicetree/bindings/hwmon/ina209.txt diff --git a/Documentation/devicetree/bindings/i2c/ina2xx.txt b/Documentation/devicetree/bindings/hwmon/ina2xx.txt similarity index 100% rename from Documentation/devicetree/bindings/i2c/ina2xx.txt rename to Documentation/devicetree/bindings/hwmon/ina2xx.txt diff --git a/Documentation/devicetree/bindings/i2c/max6697.txt b/Documentation/devicetree/bindings/hwmon/max6697.txt similarity index 100% rename from Documentation/devicetree/bindings/i2c/max6697.txt rename to Documentation/devicetree/bindings/hwmon/max6697.txt diff --git a/Documentation/devicetree/bindings/i2c/i2c-cadence.txt b/Documentation/devicetree/bindings/i2c/i2c-cadence.txt index 7cb0b5608f49..ebaa90c58c8e 100644 --- a/Documentation/devicetree/bindings/i2c/i2c-cadence.txt +++ b/Documentation/devicetree/bindings/i2c/i2c-cadence.txt @@ -2,7 +2,11 @@ Binding for the Cadence I2C controller Required properties: - reg: Physical base address and size of the controller's register area. - - compatible: Compatibility string. Must be 'cdns,i2c-r1p10'. + - compatible: Should contain one of: + * "cdns,i2c-r1p10" + Note: Use this when cadence i2c controller version 1.0 is used. + * "cdns,i2c-r1p14" + Note: Use this when cadence i2c controller version 1.4 is used. - clocks: Input clock specifier. Refer to common clock bindings. - interrupts: Interrupt specifier. Refer to interrupt bindings. - #address-cells: Should be 1. diff --git a/Documentation/devicetree/bindings/i2c/i2c-emev2.txt b/Documentation/devicetree/bindings/i2c/i2c-emev2.txt new file mode 100644 index 000000000000..5ed1ea1c7e14 --- /dev/null +++ b/Documentation/devicetree/bindings/i2c/i2c-emev2.txt @@ -0,0 +1,22 @@ +Device tree configuration for Renesas EMEV2 IIC controller + +Required properties: +- compatible : "renesas,iic-emev2" +- reg : address start and address range size of device +- interrupts : specifier for the IIC controller interrupt +- clocks : phandle to the IP core SCLK +- clock-names : must be "sclk" +- #address-cells : should be <1> +- #size-cells : should be <0> + +Example: + + iic0: i2c@e0070000 { + #address-cells = <1>; + #size-cells = <0>; + compatible = "renesas,iic-emev2"; + reg = <0xe0070000 0x28>; + interrupts = <0 32 IRQ_TYPE_EDGE_RISING>; + clocks = <&iic0_sclk>; + clock-names = "sclk"; + }; diff --git a/Documentation/devicetree/bindings/i2c/i2c-lpc2k.txt b/Documentation/devicetree/bindings/i2c/i2c-lpc2k.txt new file mode 100644 index 000000000000..4101aa621ad4 --- /dev/null +++ b/Documentation/devicetree/bindings/i2c/i2c-lpc2k.txt @@ -0,0 +1,33 @@ +NXP I2C controller for LPC2xxx/178x/18xx/43xx + +Required properties: + - compatible: must be "nxp,lpc1788-i2c" + - reg: physical address and length of the device registers + - interrupts: a single interrupt specifier + - clocks: clock for the device + - #address-cells: should be <1> + - #size-cells: should be <0> + +Optional properties: +- clock-frequency: the desired I2C bus clock frequency in Hz; in + absence of this property the default value is used (100 kHz). + +Example: +i2c0: i2c@400a1000 { + compatible = "nxp,lpc1788-i2c"; + reg = <0x400a1000 0x1000>; + interrupts = <18>; + clocks = <&ccu1 CLK_APB1_I2C0>; + #address-cells = <1>; + #size-cells = <0>; +}; + +&i2c0 { + clock-frequency = <400000>; + + lm75@48 { + compatible = "nxp,lm75"; + reg = <0x48>; + }; +}; + diff --git a/Documentation/devicetree/bindings/i2c/i2c-mux-reg.txt b/Documentation/devicetree/bindings/i2c/i2c-mux-reg.txt new file mode 100644 index 000000000000..688783fbe696 --- /dev/null +++ b/Documentation/devicetree/bindings/i2c/i2c-mux-reg.txt @@ -0,0 +1,74 @@ +Register-based I2C Bus Mux + +This binding describes an I2C bus multiplexer that uses a single register +to route the I2C signals. + +Required properties: +- compatible: i2c-mux-reg +- i2c-parent: The phandle of the I2C bus that this multiplexer's master-side + port is connected to. +* Standard I2C mux properties. See mux.txt in this directory. +* I2C child bus nodes. See mux.txt in this directory. + +Optional properties: +- reg: this pair of specifies the register to control the mux. + The depends on its parent node. It can be any memory-mapped + address. The size must be either 1, 2, or 4 bytes. If reg is omitted, the + resource of this device will be used. +- little-endian: The existence indicates the register is in little endian. +- big-endian: The existence indicates the register is in big endian. + If both little-endian and big-endian are omitted, the endianness of the + CPU will be used. +- write-only: The existence indicates the register is write-only. +- idle-state: value to set the muxer to when idle. When no value is + given, it defaults to the last value used. + +Whenever an access is made to a device on a child bus, the value set +in the revelant node's reg property will be output to the register. + +If an idle state is defined, using the idle-state (optional) property, +whenever an access is not being made to a device on a child bus, the +register will be set according to the idle value. + +If an idle state is not defined, the most recently used value will be +left programmed into the register. + +Example of a mux on PCIe card, the host is a powerpc SoC (big endian): + + i2c-mux { + /* the depends on the address translation + * of the parent device. If omitted, device resource + * will be used instead. The size is to determine + * whether iowrite32, iowrite16, or iowrite8 will be used. + */ + reg = <0x6028 0x4>; + little-endian; /* little endian register on PCIe */ + compatible = "i2c-mux-reg"; + #address-cells = <1>; + #size-cells = <0>; + i2c-parent = <&i2c1>; + i2c@0 { + reg = <0>; + #address-cells = <1>; + #size-cells = <0>; + + si5338: clock-generator@70 { + compatible = "silabs,si5338"; + reg = <0x70>; + /* other stuff */ + }; + }; + + i2c@1 { + /* data is written using iowrite32 */ + reg = <1>; + #address-cells = <1>; + #size-cells = <0>; + + si5338: clock-generator@70 { + compatible = "silabs,si5338"; + reg = <0x70>; + /* other stuff */ + }; + }; + }; diff --git a/Documentation/devicetree/bindings/i2c/i2c.txt b/Documentation/devicetree/bindings/i2c/i2c.txt new file mode 100644 index 000000000000..8a99150ac3a7 --- /dev/null +++ b/Documentation/devicetree/bindings/i2c/i2c.txt @@ -0,0 +1,45 @@ +Generic device tree bindings for I2C busses +=========================================== + +This document describes generic bindings which can be used to describe I2C +busses in a device tree. + +Required properties +------------------- + +- #address-cells - should be <1>. Read more about addresses below. +- #size-cells - should be <0>. +- compatible - name of I2C bus controller following generic names + recommended practice. + +For other required properties e.g. to describe register sets, +clocks, etc. check the binding documentation of the specific driver. + +The cells properties above define that an address of children of an I2C bus +are described by a single value. This is usually a 7 bit address. However, +flags can be attached to the address. I2C_TEN_BIT_ADDRESS is used to mark a 10 +bit address. It is needed to avoid the ambiguity between e.g. a 7 bit address +of 0x50 and a 10 bit address of 0x050 which, in theory, can be on the same bus. +Another flag is I2C_OWN_SLAVE_ADDRESS to mark addresses on which we listen to +be devices ourselves. + +Optional properties +------------------- + +These properties may not be supported by all drivers. However, if a driver +wants to support one of the below features, it should adapt the bindings below. + +- clock-frequency - frequency of bus clock in Hz. +- wakeup-source - device can be used as a wakeup source. + +- interrupts - interrupts used by the device. +- interrupt-names - "irq" and "wakeup" names are recognized by I2C core, + other names are left to individual drivers. + +Binding may contain optional "interrupts" property, describing interrupts +used by the device. I2C core will assign "irq" interrupt (or the very first +interrupt if not using interrupt names) as primary interrupt for the slave. + +Also, if device is marked as a wakeup source, I2C core will set up "wakeup" +interrupt for the device. If "wakeup" interrupt name is not present in the +binding, then primary interrupt will be used as wakeup interrupt. diff --git a/Documentation/devicetree/bindings/i2c/trivial-devices.txt b/Documentation/devicetree/bindings/i2c/trivial-devices.txt index 00f8652e193a..d77d412cbc68 100644 --- a/Documentation/devicetree/bindings/i2c/trivial-devices.txt +++ b/Documentation/devicetree/bindings/i2c/trivial-devices.txt @@ -95,6 +95,8 @@ stm,m41t00 Serial Access TIMEKEEPER stm,m41t62 Serial real-time clock (RTC) with alarm stm,m41t80 M41T80 - SERIAL ACCESS RTC WITH ALARMS taos,tsl2550 Ambient Light Sensor with SMBUS/Two Wire Serial Interface +ti,ads7828 8-Channels, 12-bit ADC +ti,ads7830 8-Channels, 8-bit ADC ti,tsc2003 I2C Touch-Screen Controller ti,tmp102 Low Power Digital Temperature Sensor with SMBUS/Two Wire Serial Interface ti,tmp103 Low Power Digital Temperature Sensor with SMBUS/Two Wire Serial Interface diff --git a/Documentation/devicetree/bindings/input/ads7846.txt b/Documentation/devicetree/bindings/input/ads7846.txt index 5f7619c22743..df8b1279491d 100644 --- a/Documentation/devicetree/bindings/input/ads7846.txt +++ b/Documentation/devicetree/bindings/input/ads7846.txt @@ -64,7 +64,7 @@ Optional properties: pendown-gpio (u32). pendown-gpio GPIO handle describing the pin the !PENIRQ line is connected to. - linux,wakeup use any event on touchscreen as wakeup event. + wakeup-source use any event on touchscreen as wakeup event. Example for a TSC2046 chip connected to an McSPI controller of an OMAP SoC:: diff --git a/Documentation/devicetree/bindings/input/cap11xx.txt b/Documentation/devicetree/bindings/input/cap11xx.txt index 7d0a3009771b..8c67a0b5058d 100644 --- a/Documentation/devicetree/bindings/input/cap11xx.txt +++ b/Documentation/devicetree/bindings/input/cap11xx.txt @@ -55,5 +55,24 @@ i2c_controller { <105>, /* KEY_LEFT */ <109>, /* KEY_PAGEDOWN */ <104>; /* KEY_PAGEUP */ + + #address-cells = <1>; + #size-cells = <0>; + + usr@0 { + label = "cap11xx:green:usr0"; + reg = <0>; + }; + + usr@1 { + label = "cap11xx:green:usr1"; + reg = <1>; + }; + + alive@2 { + label = "cap11xx:green:alive"; + reg = <2>; + linux,default_trigger = "heartbeat"; + }; }; } diff --git a/Documentation/devicetree/bindings/input/cypress,cyapa.txt b/Documentation/devicetree/bindings/input/cypress,cyapa.txt new file mode 100644 index 000000000000..635a3b036630 --- /dev/null +++ b/Documentation/devicetree/bindings/input/cypress,cyapa.txt @@ -0,0 +1,44 @@ +Cypress I2C Touchpad + +Required properties: +- compatible: must be "cypress,cyapa". +- reg: I2C address of the chip. +- interrupt-parent: a phandle for the interrupt controller (see interrupt + binding[0]). +- interrupts: interrupt to which the chip is connected (see interrupt + binding[0]). + +Optional properties: +- wakeup-source: touchpad can be used as a wakeup source. +- pinctrl-names: should be "default" (see pinctrl binding [1]). +- pinctrl-0: a phandle pointing to the pin settings for the device (see + pinctrl binding [1]). +- vcc-supply: a phandle for the regulator supplying 3.3V power. + +[0]: Documentation/devicetree/bindings/interrupt-controller/interrupts.txt +[1]: Documentation/devicetree/bindings/pinctrl/pinctrl-bindings.txt + +Example: + &i2c0 { + /* ... */ + + /* Cypress Gen3 touchpad */ + touchpad@67 { + compatible = "cypress,cyapa"; + reg = <0x24>; + interrupt-parent = <&gpio>; + interrupts = <2 IRQ_TYPE_EDGE_FALLING>; /* GPIO 2 */ + wakeup-source; + }; + + /* Cypress Gen5 and later touchpad */ + touchpad@24 { + compatible = "cypress,cyapa"; + reg = <0x24>; + interrupt-parent = <&gpio>; + interrupts = <2 IRQ_TYPE_EDGE_FALLING>; /* GPIO 2 */ + wakeup-source; + }; + + /* ... */ + }; diff --git a/Documentation/devicetree/bindings/input/elants_i2c.txt b/Documentation/devicetree/bindings/input/elants_i2c.txt index a765232e6446..8a71038f3489 100644 --- a/Documentation/devicetree/bindings/input/elants_i2c.txt +++ b/Documentation/devicetree/bindings/input/elants_i2c.txt @@ -13,6 +13,9 @@ Optional properties: - pinctrl-names: should be "default" (see pinctrl binding [1]). - pinctrl-0: a phandle pointing to the pin settings for the device (see pinctrl binding [1]). +- reset-gpios: reset gpio the chip is connected to. +- vcc33-supply: a phandle for the regulator supplying 3.3V power. +- vccio-supply: a phandle for the regulator supplying IO power. [0]: Documentation/devicetree/bindings/interrupt-controller/interrupts.txt [1]: Documentation/devicetree/bindings/pinctrl/pinctrl-bindings.txt diff --git a/Documentation/devicetree/bindings/input/gpio-keys-polled.txt b/Documentation/devicetree/bindings/input/gpio-keys-polled.txt index 313abefa37cc..5b91f5a3bd5c 100644 --- a/Documentation/devicetree/bindings/input/gpio-keys-polled.txt +++ b/Documentation/devicetree/bindings/input/gpio-keys-polled.txt @@ -20,7 +20,7 @@ Optional subnode-properties: If not specified defaults to <1> == EV_KEY. - debounce-interval: Debouncing interval time in milliseconds. If not specified defaults to 5. - - gpio-key,wakeup: Boolean, button can wake-up the system. + - wakeup-source: Boolean, button can wake-up the system. Example nodes: diff --git a/Documentation/devicetree/bindings/input/gpio-keys.txt b/Documentation/devicetree/bindings/input/gpio-keys.txt index 44b705767aca..072bf7573c37 100644 --- a/Documentation/devicetree/bindings/input/gpio-keys.txt +++ b/Documentation/devicetree/bindings/input/gpio-keys.txt @@ -23,7 +23,7 @@ Optional subnode-properties: If not specified defaults to <1> == EV_KEY. - debounce-interval: Debouncing interval time in milliseconds. If not specified defaults to 5. - - gpio-key,wakeup: Boolean, button can wake-up the system. + - wakeup-source: Boolean, button can wake-up the system. - linux,can-disable: Boolean, indicates that button is connected to dedicated (not shared) interrupt which can be disabled to suppress events from the button. diff --git a/Documentation/devicetree/bindings/input/gpio-matrix-keypad.txt b/Documentation/devicetree/bindings/input/gpio-matrix-keypad.txt index ead641c65e0a..4d86059c370c 100644 --- a/Documentation/devicetree/bindings/input/gpio-matrix-keypad.txt +++ b/Documentation/devicetree/bindings/input/gpio-matrix-keypad.txt @@ -19,7 +19,7 @@ Required Properties: Optional Properties: - linux,no-autorepeat: do no enable autorepeat feature. -- linux,wakeup: use any event on keypad as wakeup event. +- wakeup-source: use any event on keypad as wakeup event. - debounce-delay-ms: debounce interval in milliseconds - col-scan-delay-us: delay, measured in microseconds, that is needed before we can scan keypad after activating column gpio diff --git a/Documentation/devicetree/bindings/input/qcom,pm8xxx-keypad.txt b/Documentation/devicetree/bindings/input/qcom,pm8xxx-keypad.txt index 7d8cb92831d7..ee6215681182 100644 --- a/Documentation/devicetree/bindings/input/qcom,pm8xxx-keypad.txt +++ b/Documentation/devicetree/bindings/input/qcom,pm8xxx-keypad.txt @@ -33,7 +33,7 @@ PROPERTIES Value type: Definition: don't enable autorepeat feature. -- linux,keypad-wakeup: +- wakeup-source: Usage: optional Value type: Definition: use any event on keypad as wakeup event. diff --git a/Documentation/devicetree/bindings/input/samsung-keypad.txt b/Documentation/devicetree/bindings/input/samsung-keypad.txt index 942d071baaa5..863e77f619dc 100644 --- a/Documentation/devicetree/bindings/input/samsung-keypad.txt +++ b/Documentation/devicetree/bindings/input/samsung-keypad.txt @@ -36,9 +36,11 @@ Required Board Specific Properties: - pinctrl-0: Should specify pin control groups used for this controller. - pinctrl-names: Should contain only one value - "default". +Optional Properties: +- wakeup-source: use any event on keypad as wakeup event. + Optional Properties specific to linux: - linux,keypad-no-autorepeat: do no enable autorepeat feature. -- linux,keypad-wakeup: use any event on keypad as wakeup event. Example: diff --git a/Documentation/devicetree/bindings/input/touchscreen/colibri-vf50-ts.txt b/Documentation/devicetree/bindings/input/touchscreen/colibri-vf50-ts.txt new file mode 100644 index 000000000000..9d9e930f3251 --- /dev/null +++ b/Documentation/devicetree/bindings/input/touchscreen/colibri-vf50-ts.txt @@ -0,0 +1,36 @@ +* Toradex Colibri VF50 Touchscreen driver + +Required Properties: +- compatible must be toradex,vf50-touchscreen +- io-channels: adc channels being used by the Colibri VF50 module +- xp-gpios: FET gate driver for input of X+ +- xm-gpios: FET gate driver for input of X- +- yp-gpios: FET gate driver for input of Y+ +- ym-gpios: FET gate driver for input of Y- +- interrupt-parent: phandle for the interrupt controller +- interrupts: pen irq interrupt for touch detection +- pinctrl-names: "idle", "default", "gpios" +- pinctrl-0: pinctrl node for pen/touch detection state pinmux +- pinctrl-1: pinctrl node for X/Y and pressure measurement (ADC) state pinmux +- pinctrl-2: pinctrl node for gpios functioning as FET gate drivers +- vf50-ts-min-pressure: pressure level at which to stop measuring X/Y values + +Example: + + touchctrl: vf50_touchctrl { + compatible = "toradex,vf50-touchscreen"; + io-channels = <&adc1 0>,<&adc0 0>, + <&adc0 1>,<&adc1 2>; + xp-gpios = <&gpio0 13 GPIO_ACTIVE_LOW>; + xm-gpios = <&gpio2 29 GPIO_ACTIVE_HIGH>; + yp-gpios = <&gpio0 12 GPIO_ACTIVE_LOW>; + ym-gpios = <&gpio0 4 GPIO_ACTIVE_HIGH>; + interrupt-parent = <&gpio0>; + interrupts = <8 IRQ_TYPE_LEVEL_LOW>; + pinctrl-names = "idle","default","gpios"; + pinctrl-0 = <&pinctrl_touchctrl_idle>; + pinctrl-1 = <&pinctrl_touchctrl_default>; + pinctrl-2 = <&pinctrl_touchctrl_gpios>; + vf50-ts-min-pressure = <200>; + status = "disabled"; + }; diff --git a/Documentation/devicetree/bindings/input/touchscreen/imx6ul_tsc.txt b/Documentation/devicetree/bindings/input/touchscreen/imx6ul_tsc.txt new file mode 100644 index 000000000000..853dff96dd9f --- /dev/null +++ b/Documentation/devicetree/bindings/input/touchscreen/imx6ul_tsc.txt @@ -0,0 +1,36 @@ +* Freescale i.MX6UL Touch Controller + +Required properties: +- compatible: must be "fsl,imx6ul-tsc". +- reg: this touch controller address and the ADC2 address. +- interrupts: the interrupt of this touch controller and ADC2. +- clocks: the root clock of touch controller and ADC2. +- clock-names; must be "tsc" and "adc". +- xnur-gpio: the X- gpio this controller connect to. + This xnur-gpio returns to low once the finger leave the touch screen (The + last touch event the touch controller capture). + +Optional properties: +- measure-delay-time: the value of measure delay time. + Before X-axis or Y-axis measurement, the screen need some time before + even potential distribution ready. + This value depends on the touch screen. +- pre-charge-time: the touch screen need some time to precharge. + This value depends on the touch screen. + +Example: + tsc: tsc@02040000 { + compatible = "fsl,imx6ul-tsc"; + reg = <0x02040000 0x4000>, <0x0219c000 0x4000>; + interrupts = , + ; + clocks = <&clks IMX6UL_CLK_IPG>, + <&clks IMX6UL_CLK_ADC2>; + clock-names = "tsc", "adc"; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_tsc>; + xnur-gpio = <&gpio1 3 GPIO_ACTIVE_LOW>; + measure-delay-time = <0xfff>; + pre-charge-time = <0xffff>; + status = "okay"; + }; diff --git a/Documentation/devicetree/bindings/input/touchscreen/pixcir_i2c_ts.txt b/Documentation/devicetree/bindings/input/touchscreen/pixcir_i2c_ts.txt index 6e551090f465..8eb240a287c8 100644 --- a/Documentation/devicetree/bindings/input/touchscreen/pixcir_i2c_ts.txt +++ b/Documentation/devicetree/bindings/input/touchscreen/pixcir_i2c_ts.txt @@ -8,6 +8,9 @@ Required properties: - touchscreen-size-x: horizontal resolution of touchscreen (in pixels) - touchscreen-size-y: vertical resolution of touchscreen (in pixels) +Optional properties: +- reset-gpio: GPIO connected to the RESET line of the chip + Example: i2c@00000000 { diff --git a/Documentation/devicetree/bindings/input/touchscreen/zforce_ts.txt b/Documentation/devicetree/bindings/input/touchscreen/zforce_ts.txt index 80c37df940a7..e3c27c4fd9c8 100644 --- a/Documentation/devicetree/bindings/input/touchscreen/zforce_ts.txt +++ b/Documentation/devicetree/bindings/input/touchscreen/zforce_ts.txt @@ -4,12 +4,12 @@ Required properties: - compatible: must be "neonode,zforce" - reg: I2C address of the chip - interrupts: interrupt to which the chip is connected -- gpios: gpios the chip is connected to - first one is the interrupt gpio and second one the reset gpio +- reset-gpios: reset gpio the chip is connected to - x-size: horizontal resolution of touchscreen - y-size: vertical resolution of touchscreen Optional properties: +- irq-gpios : interrupt gpio the chip is connected to - vdd-supply: Regulator controlling the controller supply Example: @@ -23,8 +23,8 @@ Example: interrupts = <2 0>; vdd-supply = <®_zforce_vdd>; - gpios = <&gpio5 6 0>, /* INT */ - <&gpio5 9 0>; /* RST */ + reset-gpios = <&gpio5 9 0>; /* RST */ + irq-gpios = <&gpio5 6 0>; /* IRQ, optional */ x-size = <800>; y-size = <600>; diff --git a/Documentation/devicetree/bindings/interrupt-controller/brcm,bcm2835-armctrl-ic.txt b/Documentation/devicetree/bindings/interrupt-controller/brcm,bcm2835-armctrl-ic.txt index 7da578d72123..2d6c8bb4d827 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/brcm,bcm2835-armctrl-ic.txt +++ b/Documentation/devicetree/bindings/interrupt-controller/brcm,bcm2835-armctrl-ic.txt @@ -5,9 +5,14 @@ The BCM2835 contains a custom top-level interrupt controller, which supports controller, or the HW block containing it, is referred to occasionally as "armctrl" in the SoC documentation, hence naming of this binding. +The BCM2836 contains the same interrupt controller with the same +interrupts, but the per-CPU interrupt controller is the root, and an +interrupt there indicates that the ARMCTRL has an interrupt to handle. + Required properties: -- compatible : should be "brcm,bcm2835-armctrl-ic" +- compatible : should be "brcm,bcm2835-armctrl-ic" or + "brcm,bcm2836-armctrl-ic" - reg : Specifies base physical address and size of the registers. - interrupt-controller : Identifies the node as an interrupt controller - #interrupt-cells : Specifies the number of cells needed to encode an @@ -20,6 +25,12 @@ Required properties: The 2nd cell contains the interrupt number within the bank. Valid values are 0..7 for bank 0, and 0..31 for bank 1. +Additional required properties for brcm,bcm2836-armctrl-ic: +- interrupt-parent : Specifies the parent interrupt controller when this + controller is the second level. +- interrupts : Specifies the interrupt on the parent for this interrupt + controller to handle. + The interrupt sources are as follows: Bank 0: @@ -102,9 +113,21 @@ Bank 2: Example: +/* BCM2835, first level */ intc: interrupt-controller { compatible = "brcm,bcm2835-armctrl-ic"; reg = <0x7e00b200 0x200>; interrupt-controller; #interrupt-cells = <2>; }; + +/* BCM2836, second level */ +intc: interrupt-controller { + compatible = "brcm,bcm2836-armctrl-ic"; + reg = <0x7e00b200 0x200>; + interrupt-controller; + #interrupt-cells = <2>; + + interrupt-parent = <&local_intc>; + interrupts = <8>; +}; diff --git a/Documentation/devicetree/bindings/interrupt-controller/brcm,bcm2836-l1-intc.txt b/Documentation/devicetree/bindings/interrupt-controller/brcm,bcm2836-l1-intc.txt new file mode 100644 index 000000000000..f320dcd6e69b --- /dev/null +++ b/Documentation/devicetree/bindings/interrupt-controller/brcm,bcm2836-l1-intc.txt @@ -0,0 +1,37 @@ +BCM2836 per-CPU interrupt controller + +The BCM2836 has a per-cpu interrupt controller for the timer, PMU +events, and SMP IPIs. One of the CPUs may receive interrupts for the +peripheral (GPU) events, which chain to the BCM2835-style interrupt +controller. + +Required properties: + +- compatible: Should be "brcm,bcm2836-l1-intc" +- reg: Specifies base physical address and size of the + registers +- interrupt-controller: Identifies the node as an interrupt controller +- #interrupt-cells: Specifies the number of cells needed to encode an + interrupt source. The value shall be 1 + +Please refer to interrupts.txt in this directory for details of the common +Interrupt Controllers bindings used by client devices. + +The interrupt sources are as follows: + +0: CNTPSIRQ +1: CNTPNSIRQ +2: CNTHPIRQ +3: CNTVIRQ +8: GPU_FAST +9: PMU_FAST + +Example: + +local_intc: local_intc { + compatible = "brcm,bcm2836-l1-intc"; + reg = <0x40000000 0x100>; + interrupt-controller; + #interrupt-cells = <1>; + interrupt-parent = <&local_intc>; +}; diff --git a/Documentation/devicetree/bindings/interrupt-controller/msi.txt b/Documentation/devicetree/bindings/interrupt-controller/msi.txt new file mode 100644 index 000000000000..c60c034dcf19 --- /dev/null +++ b/Documentation/devicetree/bindings/interrupt-controller/msi.txt @@ -0,0 +1,135 @@ +This document describes the generic device tree binding for MSI controllers and +their master(s). + +Message Signaled Interrupts (MSIs) are a class of interrupts generated by a +write to an MMIO address. + +MSIs were originally specified by PCI (and are used with PCIe), but may also be +used with other busses, and hence a mechanism is required to relate devices on +those busses to the MSI controllers which they are capable of using, +potentially including additional information. + +MSIs are distinguished by some combination of: + +- The doorbell (the MMIO address written to). + + Devices may be configured by software to write to arbitrary doorbells which + they can address. An MSI controller may feature a number of doorbells. + +- The payload (the value written to the doorbell). + + Devices may be configured to write an arbitrary payload chosen by software. + MSI controllers may have restrictions on permitted payloads. + +- Sideband information accompanying the write. + + Typically this is neither configurable nor probeable, and depends on the path + taken through the memory system (i.e. it is a property of the combination of + MSI controller and device rather than a property of either in isolation). + + +MSI controllers: +================ + +An MSI controller signals interrupts to a CPU when a write is made to an MMIO +address by some master. An MSI controller may feature a number of doorbells. + +Required properties: +-------------------- + +- msi-controller: Identifies the node as an MSI controller. + +Optional properties: +-------------------- + +- #msi-cells: The number of cells in an msi-specifier, required if not zero. + + Typically this will encode information related to sideband data, and will + not encode doorbells or payloads as these can be configured dynamically. + + The meaning of the msi-specifier is defined by the device tree binding of + the specific MSI controller. + + +MSI clients +=========== + +MSI clients are devices which generate MSIs. For each MSI they wish to +generate, the doorbell and payload may be configured, though sideband +information may not be configurable. + +Required properties: +-------------------- + +- msi-parent: A list of phandle + msi-specifier pairs, one for each MSI + controller which the device is capable of using. + + This property is unordered, and MSIs may be allocated from any combination of + MSI controllers listed in the msi-parent property. + + If a device has restrictions on the allocation of MSIs, these restrictions + must be described with additional properties. + + When #msi-cells is non-zero, busses with an msi-parent will require + additional properties to describe the relationship between devices on the bus + and the set of MSIs they can potentially generate. + + +Example +======= + +/ { + #address-cells = <1>; + #size-cells = <1>; + + msi_a: msi-controller@a { + reg = <0xa 0xf00>; + compatible = "vendor-a,some-controller"; + msi-controller; + /* No sideband data, so #msi-cells omitted */ + }; + + msi_b: msi-controller@b { + reg = <0xb 0xf00>; + compatible = "vendor-b,another-controller"; + msi-controller; + /* Each device has some unique ID */ + #msi-cells = <1>; + }; + + msi_c: msi-controller@c { + reg = <0xb 0xf00>; + compatible = "vendor-b,another-controller"; + msi-controller; + /* Each device has some unique ID */ + #msi-cells = <1>; + }; + + dev@0 { + reg = <0x0 0xf00>; + compatible = "vendor-c,some-device"; + + /* Can only generate MSIs to msi_a */ + msi-parent = <&msi_a>; + }; + + dev@1 { + reg = <0x1 0xf00>; + compatible = "vendor-c,some-device"; + + /* + * Can generate MSIs to either A or B. + */ + msi-parent = <&msi_a>, <&msi_b 0x17>; + }; + + dev@2 { + reg = <0x2 0xf00>; + compatible = "vendor-c,some-device"; + /* + * Has different IDs at each MSI controller. + * Can generate MSIs to all of the MSI controllers. + */ + msi-parent = <&msi_a>, <&msi_b 0x17>, <&msi_c 0x53>; + }; +}; diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.txt b/Documentation/devicetree/bindings/iommu/arm,smmu.txt index 06760503a819..718074501fcb 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.txt +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.txt @@ -43,6 +43,12 @@ conditions. ** System MMU optional properties: +- dma-coherent : Present if page table walks made by the SMMU are + cache coherent with the CPU. + + NOTE: this only applies to the SMMU itself, not + masters connected upstream of the SMMU. + - calxeda,smmu-secure-config-access : Enable proper handling of buggy implementations that always use secure access to SMMU configuration registers. In this case non-secure diff --git a/Documentation/devicetree/bindings/iommu/ti,omap-iommu.txt b/Documentation/devicetree/bindings/iommu/ti,omap-iommu.txt index 42531dc387aa..869699925fd5 100644 --- a/Documentation/devicetree/bindings/iommu/ti,omap-iommu.txt +++ b/Documentation/devicetree/bindings/iommu/ti,omap-iommu.txt @@ -8,6 +8,11 @@ Required properties: - ti,hwmods : Name of the hwmod associated with the IOMMU instance - reg : Address space for the configuration registers - interrupts : Interrupt specifier for the IOMMU instance +- #iommu-cells : Should be 0. OMAP IOMMUs are all "single-master" devices, + and needs no additional data in the pargs specifier. Please + also refer to the generic bindings document for more info + on this property, + Documentation/devicetree/bindings/iommu/iommu.txt Optional properties: - ti,#tlb-entries : Number of entries in the translation look-aside buffer. @@ -18,6 +23,7 @@ Optional properties: Example: /* OMAP3 ISP MMU */ mmu_isp: mmu@480bd400 { + #iommu-cells = <0>; compatible = "ti,omap2-iommu"; reg = <0x480bd400 0x80>; interrupts = <24>; diff --git a/Documentation/devicetree/bindings/ipmi.txt b/Documentation/devicetree/bindings/ipmi.txt new file mode 100644 index 000000000000..d5f1a877ed3e --- /dev/null +++ b/Documentation/devicetree/bindings/ipmi.txt @@ -0,0 +1,25 @@ +IPMI device + +Required properties: +- compatible: should be one of ipmi-kcs, ipmi-smic, or ipmi-bt +- device_type: should be ipmi +- reg: Address and length of the register set for the device + +Optional properties: +- interrupts: The interrupt for the device. Without this the interface + is polled. +- reg-size - The size of the register. Defaults to 1 +- reg-spacing - The number of bytes between register starts. Defaults to 1 +- reg-shift - The amount to shift the registers to the right to get the data + into bit zero. + +Example: + +smic@fff3a000 { + compatible = "ipmi-smic"; + device_type = "ipmi"; + reg = <0xfff3a000 0x1000>; + interrupts = <0 24 4>; + reg-size = <4>; + reg-spacing = <4>; +}; diff --git a/Documentation/devicetree/bindings/leds/leds-powernv.txt b/Documentation/devicetree/bindings/leds/leds-powernv.txt new file mode 100644 index 000000000000..66655690f749 --- /dev/null +++ b/Documentation/devicetree/bindings/leds/leds-powernv.txt @@ -0,0 +1,26 @@ +Device Tree binding for LEDs on IBM Power Systems +------------------------------------------------- + +Required properties: +- compatible : Should be "ibm,opal-v3-led". +- led-mode : Should be "lightpath" or "guidinglight". + +Each location code of FRU/Enclosure must be expressed in the +form of a sub-node. + +Required properties for the sub nodes: +- led-types : Supported LED types (attention/identify/fault) provided + in the form of string array. + +Example: + +leds { + compatible = "ibm,opal-v3-led"; + led-mode = "lightpath"; + + U78C9.001.RST0027-P1-C1 { + led-types = "identify", "fault"; + }; + ... + ... +}; diff --git a/Documentation/devicetree/bindings/media/i2c/adv7604.txt b/Documentation/devicetree/bindings/media/i2c/adv7604.txt index c27cede3bd68..8337f75c75da 100644 --- a/Documentation/devicetree/bindings/media/i2c/adv7604.txt +++ b/Documentation/devicetree/bindings/media/i2c/adv7604.txt @@ -1,15 +1,17 @@ -* Analog Devices ADV7604/11 video decoder with HDMI receiver +* Analog Devices ADV7604/11/12 video decoder with HDMI receiver -The ADV7604 and ADV7611 are multiformat video decoders with an integrated HDMI -receiver. The ADV7604 has four multiplexed HDMI inputs and one analog input, -and the ADV7611 has one HDMI input and no analog input. +The ADV7604 and ADV7611/12 are multiformat video decoders with an integrated +HDMI receiver. The ADV7604 has four multiplexed HDMI inputs and one analog +input, and the ADV7611 has one HDMI input and no analog input. The 7612 is +similar to the 7611 but has 2 HDMI inputs. -These device tree bindings support the ADV7611 only at the moment. +These device tree bindings support the ADV7611/12 only at the moment. Required Properties: - compatible: Must contain one of the following - "adi,adv7611" for the ADV7611 + - "adi,adv7612" for the ADV7612 - reg: I2C slave address @@ -22,10 +24,10 @@ port, in accordance with the video interface bindings defined in Documentation/devicetree/bindings/media/video-interfaces.txt. The port nodes are numbered as follows. - Port ADV7611 + Port ADV7611 ADV7612 ------------------------------------------------------------ - HDMI 0 - Digital output 1 + HDMI 0 0, 1 + Digital output 1 2 The digital output port node must contain at least one endpoint. @@ -45,6 +47,7 @@ Optional Endpoint Properties: If none of hsync-active, vsync-active and pclk-sample is specified the endpoint will use embedded BT.656 synchronization. + - default-input: Select which input is selected after reset. Example: @@ -58,6 +61,8 @@ Example: #address-cells = <1>; #size-cells = <0>; + default-input = <0>; + port@0 { reg = <0>; }; diff --git a/Documentation/devicetree/bindings/media/i2c/tc358743.txt b/Documentation/devicetree/bindings/media/i2c/tc358743.txt new file mode 100644 index 000000000000..5218921629ed --- /dev/null +++ b/Documentation/devicetree/bindings/media/i2c/tc358743.txt @@ -0,0 +1,48 @@ +* Toshiba TC358743 HDMI-RX to MIPI CSI2-TX Bridge + +The Toshiba TC358743 HDMI-RX to MIPI CSI2-TX (H2C) is a bridge that converts +a HDMI stream to MIPI CSI-2 TX. It is programmable through I2C. + +Required Properties: + +- compatible: value should be "toshiba,tc358743" +- clocks, clock-names: should contain a phandle link to the reference clock + source, the clock input is named "refclk". + +Optional Properties: + +- reset-gpios: gpio phandle GPIO connected to the reset pin +- interrupts, interrupt-parent: GPIO connected to the interrupt pin +- data-lanes: should be <1 2 3 4> for four-lane operation, + or <1 2> for two-lane operation +- clock-lanes: should be <0> +- clock-noncontinuous: Presence of this boolean property decides whether the + MIPI CSI-2 clock is continuous or non-continuous. +- link-frequencies: List of allowed link frequencies in Hz. Each frequency is + expressed as a 64-bit big-endian integer. The frequency + is half of the bps per lane due to DDR transmission. + +For further information on the MIPI CSI-2 endpoint node properties, see +Documentation/devicetree/bindings/media/video-interfaces.txt. + +Example: + + tc358743@0f { + compatible = "toshiba,tc358743"; + reg = <0x0f>; + clocks = <&hdmi_osc>; + clock-names = "refclk"; + reset-gpios = <&gpio6 9 GPIO_ACTIVE_LOW>; + interrupt-parent = <&gpio2>; + interrupts = <5 IRQ_TYPE_LEVEL_HIGH>; + + port { + tc358743_out: endpoint { + remote-endpoint = <&mipi_csi2_in>; + data-lanes = <1 2 3 4>; + clock-lanes = <0>; + clock-noncontinuous; + link-frequencies = /bits/ 64 <297000000>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/media/renesas,jpu.txt b/Documentation/devicetree/bindings/media/renesas,jpu.txt new file mode 100644 index 000000000000..0cb94201bf92 --- /dev/null +++ b/Documentation/devicetree/bindings/media/renesas,jpu.txt @@ -0,0 +1,24 @@ +* Renesas JPEG Processing Unit + +The JPEG processing unit (JPU) incorporates the JPEG codec with an encoding +and decoding function conforming to the JPEG baseline process, so that the JPU +can encode image data and decode JPEG data quickly. + +Required properties: + - compatible: should containg one of the following: + - "renesas,jpu-r8a7790" for R-Car H2 + - "renesas,jpu-r8a7791" for R-Car M2-W + - "renesas,jpu-r8a7792" for R-Car V2H + - "renesas,jpu-r8a7793" for R-Car M2-N + + - reg: Base address and length of the registers block for the JPU. + - interrupts: JPU interrupt specifier. + - clocks: A phandle + clock-specifier pair for the JPU functional clock. + +Example: R8A7790 (R-Car H2) JPU node + jpeg-codec@fe980000 { + compatible = "renesas,jpu-r8a7790"; + reg = <0 0xfe980000 0 0x10300>; + interrupts = <0 272 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&mstp1_clks R8A7790_CLK_JPU>; + }; diff --git a/Documentation/devicetree/bindings/media/stih407-c8sectpfe.txt b/Documentation/devicetree/bindings/media/stih407-c8sectpfe.txt new file mode 100644 index 000000000000..d4def767bdfe --- /dev/null +++ b/Documentation/devicetree/bindings/media/stih407-c8sectpfe.txt @@ -0,0 +1,89 @@ +STMicroelectronics STi c8sectpfe binding +============================================ + +This document describes the c8sectpfe device bindings that is used to get transport +stream data into the SoC on the TS pins, and into DDR for further processing. + +It is typically used in conjunction with one or more demodulator and tuner devices +which converts from the RF to digital domain. Demodulators and tuners are usually +located on an external DVB frontend card connected to SoC TS input pins. + +Currently 7 TS input (tsin) channels are supported on the stih407 family SoC. + +Required properties (controller (parent) node): +- compatible : Should be "stih407-c8sectpfe" + +- reg : Address and length of register sets for each device in + "reg-names" + +- reg-names : The names of the register addresses corresponding to the + registers filled in "reg": + - c8sectpfe: c8sectpfe registers + - c8sectpfe-ram: c8sectpfe internal sram + +- clocks : phandle list of c8sectpfe clocks +- clock-names : should be "c8sectpfe" +See: Documentation/devicetree/bindings/clock/clock-bindings.txt + +- pinctrl-names : a pinctrl state named tsin%d-serial or tsin%d-parallel (where %d is tsin-num) + must be defined for each tsin child node. +- pinctrl-0 : phandle referencing pin configuration for this tsin configuration +See: Documentation/devicetree/bindings/pinctrl/pinctrl-binding.txt + + +Required properties (tsin (child) node): + +- tsin-num : tsin id of the InputBlock (must be between 0 to 6) +- i2c-bus : phandle to the I2C bus DT node which the demodulators & tuners on this tsin channel are connected. +- rst-gpio : reset gpio for this tsin channel. + +Optional properties (tsin (child) node): + +- invert-ts-clk : Bool property to control sense of ts input clock (data stored on falling edge of clk). +- serial-not-parallel : Bool property to configure input bus width (serial on ts_data<7>). +- async-not-sync : Bool property to control if data is received in asynchronous mode + (all bits/bytes with ts_valid or ts_packet asserted are valid). + +- dvb-card : Describes the NIM card connected to this tsin channel. + +Example: + +/* stih410 SoC b2120 + b2004a + stv0367-pll(NIMB) + stv0367-tda18212 (NIMA) DT example) */ + + c8sectpfe@08a20000 { + compatible = "st,stih407-c8sectpfe"; + status = "okay"; + reg = <0x08a20000 0x10000>, <0x08a00000 0x4000>; + reg-names = "stfe", "stfe-ram"; + interrupts = <0 34 0>, <0 35 0>; + interrupt-names = "stfe-error-irq", "stfe-idle-irq"; + + pinctrl-names = "tsin0-serial", "tsin0-parallel", "tsin3-serial", + "tsin4-serial", "tsin5-serial"; + + pinctrl-0 = <&pinctrl_tsin0_serial>; + pinctrl-1 = <&pinctrl_tsin0_parallel>; + pinctrl-2 = <&pinctrl_tsin3_serial>; + pinctrl-3 = <&pinctrl_tsin4_serial_alt3>; + pinctrl-4 = <&pinctrl_tsin5_serial_alt1>; + + clocks = <&clk_s_c0_flexgen CLK_PROC_STFE>; + clock-names = "stfe"; + + /* tsin0 is TSA on NIMA */ + tsin0: port@0 { + tsin-num = <0>; + serial-not-parallel; + i2c-bus = <&ssc2>; + rst-gpio = <&pio15 4 0>; + dvb-card = ; + }; + + tsin3: port@3 { + tsin-num = <3>; + serial-not-parallel; + i2c-bus = <&ssc3>; + rst-gpio = <&pio15 7 0>; + dvb-card = ; + }; + }; diff --git a/Documentation/devicetree/bindings/memory-controllers/fsl/ifc.txt b/Documentation/devicetree/bindings/memory-controllers/fsl/ifc.txt index d5e370450ac0..89427b018ba7 100644 --- a/Documentation/devicetree/bindings/memory-controllers/fsl/ifc.txt +++ b/Documentation/devicetree/bindings/memory-controllers/fsl/ifc.txt @@ -18,6 +18,8 @@ Properties: interrupt (NAND_EVTER_STAT). If there is only one, that interrupt reports both types of event. +- little-endian : If this property is absent, the big-endian mode will + be in use as default for registers. - ranges : Each range corresponds to a single chipselect, and covers the entire access window as configured. @@ -34,6 +36,7 @@ Example: #size-cells = <1>; reg = <0x0 0xffe1e000 0 0x2000>; interrupts = <16 2 19 2>; + little-endian; /* NOR, NAND Flashes and CPLD on board */ ranges = <0x0 0x0 0x0 0xee000000 0x02000000 diff --git a/Documentation/devicetree/bindings/mfd/atmel-hlcdc.txt b/Documentation/devicetree/bindings/mfd/atmel-hlcdc.txt index f64de95a8e8b..ad5d90482a0e 100644 --- a/Documentation/devicetree/bindings/mfd/atmel-hlcdc.txt +++ b/Documentation/devicetree/bindings/mfd/atmel-hlcdc.txt @@ -2,7 +2,11 @@ Device-Tree bindings for Atmel's HLCDC (High LCD Controller) MFD driver Required properties: - compatible: value should be one of the following: + "atmel,at91sam9n12-hlcdc" + "atmel,at91sam9x5-hlcdc" + "atmel,sama5d2-hlcdc" "atmel,sama5d3-hlcdc" + "atmel,sama5d4-hlcdc" - reg: base address and size of the HLCDC device registers. - clock-names: the name of the 3 clocks requested by the HLCDC device. Should contain "periph_clk", "sys_clk" and "slow_clk". diff --git a/Documentation/devicetree/bindings/mfd/axp20x.txt b/Documentation/devicetree/bindings/mfd/axp20x.txt index 753f14f46e85..41811223e5be 100644 --- a/Documentation/devicetree/bindings/mfd/axp20x.txt +++ b/Documentation/devicetree/bindings/mfd/axp20x.txt @@ -1,12 +1,14 @@ AXP family PMIC device tree bindings The axp20x family current members : +axp152 (X-Powers) axp202 (X-Powers) axp209 (X-Powers) axp221 (X-Powers) Required properties: -- compatible: "x-powers,axp202", "x-powers,axp209", "x-powers,axp221" +- compatible: "x-powers,axp152", "x-powers,axp202", "x-powers,axp209", + "x-powers,axp221" - reg: The I2C slave address for the AXP chip - interrupt-parent: The parent interrupt controller - interrupts: SoC NMI / GPIO interrupt connected to the PMIC's IRQ pin diff --git a/Documentation/devicetree/bindings/mfd/da9062.txt b/Documentation/devicetree/bindings/mfd/da9062.txt new file mode 100644 index 000000000000..38802b54d48a --- /dev/null +++ b/Documentation/devicetree/bindings/mfd/da9062.txt @@ -0,0 +1,88 @@ +* Dialog DA9062 Power Management Integrated Circuit (PMIC) + +DA9062 consists of a large and varied group of sub-devices: + +Device Supply Names Description +------ ------------ ----------- +da9062-regulator : : LDOs & BUCKs +da9062-rtc : : Real-Time Clock +da9062-watchdog : : Watchdog Timer + +====== + +Required properties: + +- compatible : Should be "dlg,da9062". +- reg : Specifies the I2C slave address (this defaults to 0x58 but it can be + modified to match the chip's OTP settings). +- interrupt-parent : Specifies the reference to the interrupt controller for + the DA9062. +- interrupts : IRQ line information. +- interrupt-controller + +See Documentation/devicetree/bindings/interrupt-controller/interrupts.txt for +further information on IRQ bindings. + +Sub-nodes: + +- regulators : This node defines the settings for the LDOs and BUCKs. The + DA9062 regulators are bound using their names listed below: + + buck1 : BUCK_1 + buck2 : BUCK_2 + buck3 : BUCK_3 + buck4 : BUCK_4 + ldo1 : LDO_1 + ldo2 : LDO_2 + ldo3 : LDO_3 + ldo4 : LDO_4 + + The component follows the standard regulator framework and the bindings + details of individual regulator device can be found in: + Documentation/devicetree/bindings/regulator/regulator.txt + + +- rtc : This node defines settings required for the Real-Time Clock associated + with the DA9062. There are currently no entries in this binding, however + compatible = "dlg,da9062-rtc" should be added if a node is created. + +- watchdog: This node defines the settings for the watchdog driver associated + with the DA9062 PMIC. The compatible = "dlg,da9062-watchdog" should be added + if a node is created. + + +Example: + + pmic0: da9062@58 { + compatible = "dlg,da9062"; + reg = <0x58>; + interrupt-parent = <&gpio6>; + interrupts = <11 IRQ_TYPE_LEVEL_LOW>; + interrupt-controller; + + rtc { + compatible = "dlg,da9062-rtc"; + }; + + watchdog { + compatible = "dlg,da9062-watchdog"; + }; + + regulators { + DA9062_BUCK1: buck1 { + regulator-name = "BUCK1"; + regulator-min-microvolt = <300000>; + regulator-max-microvolt = <1570000>; + regulator-min-microamp = <500000>; + regulator-max-microamp = <2000000>; + regulator-boot-on; + }; + DA9062_LDO1: ldo1 { + regulator-name = "LDO_1"; + regulator-min-microvolt = <900000>; + regulator-max-microvolt = <3600000>; + regulator-boot-on; + }; + }; + }; + diff --git a/Documentation/devicetree/bindings/mfd/max77686.txt b/Documentation/devicetree/bindings/mfd/max77686.txt index 163bd81a4607..741e76688cf2 100644 --- a/Documentation/devicetree/bindings/mfd/max77686.txt +++ b/Documentation/devicetree/bindings/mfd/max77686.txt @@ -7,8 +7,9 @@ different i2c slave address,presently for which we are statically creating i2c client while probing.This document describes the binding for mfd device and PMIC submodule. -Binding for the built-in 32k clock generator block is defined separately -in bindings/clk/maxim,max77686.txt file. +Bindings for the built-in 32k clock generator block and +regulators are defined in ../clk/maxim,max77686.txt and +../regulator/max77686.txt respectively. Required properties: - compatible : Must be "maxim,max77686"; @@ -16,67 +17,11 @@ Required properties: - interrupts : This i2c device has an IRQ line connected to the main SoC. - interrupt-parent : The parent interrupt controller. -Optional node: -- voltage-regulators : The regulators of max77686 have to be instantiated - under subnode named "voltage-regulators" using the following format. - - regulator_name { - regulator-compatible = LDOn/BUCKn - standard regulator constraints.... - }; - refer Documentation/devicetree/bindings/regulator/regulator.txt - - The regulator-compatible property of regulator should initialized with string -to get matched with their hardware counterparts as follow: - - -LDOn : for LDOs, where n can lie in range 1 to 26. - example: LDO1, LDO2, LDO26. - -BUCKn : for BUCKs, where n can lie in range 1 to 9. - example: BUCK1, BUCK5, BUCK9. - - Regulators which can be turned off during system suspend: - -LDOn : 2, 6-8, 10-12, 14-16, - -BUCKn : 1-4. - Use standard regulator bindings for it ('regulator-off-in-suspend'). - - LDO20, LDO21, LDO22, BUCK8 and BUCK9 can be configured to GPIO enable - control. To turn this feature on this property must be added to the regulator - sub-node: - - maxim,ena-gpios : one GPIO specifier enable control (the gpio - flags are actually ignored and always - ACTIVE_HIGH is used) - Example: - max77686@09 { + max77686: pmic@09 { compatible = "maxim,max77686"; interrupt-parent = <&wakeup_eint>; interrupts = <26 0>; reg = <0x09>; - - voltage-regulators { - ldo11_reg { - regulator-compatible = "LDO11"; - regulator-name = "vdd_ldo11"; - regulator-min-microvolt = <1900000>; - regulator-max-microvolt = <1900000>; - regulator-always-on; - }; - - buck1_reg { - regulator-compatible = "BUCK1"; - regulator-name = "vdd_mif"; - regulator-min-microvolt = <950000>; - regulator-max-microvolt = <1300000>; - regulator-always-on; - regulator-boot-on; - }; - - buck9_reg { - regulator-compatible = "BUCK9"; - regulator-name = "CAM_ISP_CORE_1.2V"; - regulator-min-microvolt = <1000000>; - regulator-max-microvolt = <1200000>; - maxim,ena-gpios = <&gpm0 3 GPIO_ACTIVE_HIGH>; - }; - } + }; diff --git a/Documentation/devicetree/bindings/mfd/max77802.txt b/Documentation/devicetree/bindings/mfd/max77802.txt new file mode 100644 index 000000000000..51fc1a60caa5 --- /dev/null +++ b/Documentation/devicetree/bindings/mfd/max77802.txt @@ -0,0 +1,26 @@ +Maxim MAX77802 multi-function device + +The Maxim MAX77802 is a Power Management IC (PMIC) that contains 10 high +efficiency Buck regulators, 32 Low-DropOut (LDO) regulators used to power +up application processors and peripherals, a 2-channel 32kHz clock outputs, +a Real-Time-Clock (RTC) and a I2C interface to program the individual +regulators, clocks outputs and the RTC. + +Bindings for the built-in 32k clock generator block and +regulators are defined in ../clk/maxim,max77802.txt and +../regulator/max77802.txt respectively. + +Required properties: +- compatible : Must be "maxim,max77802" +- reg : Specifies the I2C slave address of PMIC block. +- interrupts : I2C device IRQ line connected to the main SoC. +- interrupt-parent : The parent interrupt controller. + +Example: + + max77802: pmic@09 { + compatible = "maxim,max77802"; + interrupt-parent = <&intc>; + interrupts = <26 IRQ_TYPE_NONE>; + reg = <0x09>; + }; diff --git a/Documentation/devicetree/bindings/mfd/tc3589x.txt b/Documentation/devicetree/bindings/mfd/tc3589x.txt index 6fcedba46ae9..37bf7f1aa70a 100644 --- a/Documentation/devicetree/bindings/mfd/tc3589x.txt +++ b/Documentation/devicetree/bindings/mfd/tc3589x.txt @@ -55,7 +55,7 @@ Optional nodes: - linux,keymap: the definition can be found in bindings/input/matrix-keymap.txt - linux,no-autorepeat: do no enable autorepeat feature. - - linux,wakeup: use any event on keypad as wakeup event. + - wakeup-source: use any event on keypad as wakeup event. Example: @@ -84,7 +84,6 @@ tc35893@44 { keypad,num-columns = <8>; keypad,num-rows = <8>; linux,no-autorepeat; - linux,wakeup; linux,keymap = <0x0301006b 0x04010066 0x06040072 @@ -103,5 +102,6 @@ tc35893@44 { 0x01030039 0x07060069 0x050500d9>; + wakeup-source; }; }; diff --git a/Documentation/devicetree/bindings/mmc/arasan,sdhci.txt b/Documentation/devicetree/bindings/mmc/arasan,sdhci.txt index 7e9490313d5a..da541c3631f8 100644 --- a/Documentation/devicetree/bindings/mmc/arasan,sdhci.txt +++ b/Documentation/devicetree/bindings/mmc/arasan,sdhci.txt @@ -9,7 +9,7 @@ Device Tree Bindings for the Arasan SDHCI Controller Required Properties: - compatible: Compatibility string. Must be 'arasan,sdhci-8.9a' or - 'arasan,sdhci-4.9a' + 'arasan,sdhci-4.9a' or 'arasan,sdhci-5.1' - reg: From mmc bindings: Register location and length. - clocks: From clock bindings: Handles to clock inputs. - clock-names: From clock bindings: Tuple including "clk_xin" and "clk_ahb" diff --git a/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.txt b/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.txt index 211e7785f4d2..dca56d6248f5 100644 --- a/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.txt +++ b/Documentation/devicetree/bindings/mmc/fsl-imx-esdhc.txt @@ -15,6 +15,7 @@ Required properties: "fsl,imx6q-usdhc" "fsl,imx6sl-usdhc" "fsl,imx6sx-usdhc" + "fsl,imx7d-usdhc" Optional properties: - fsl,wp-controller : Indicate to use controller internal write protection @@ -27,6 +28,11 @@ Optional properties: transparent level shifters on the outputs of the controller. Two cells are required, first cell specifies minimum slot voltage (mV), second cell specifies maximum slot voltage (mV). Several ranges could be specified. +- fsl,tuning-step: Specify the increasing delay cell steps in tuning procedure. + The uSDHC use one delay cell as default increasing step to do tuning process. + This property allows user to change the tuning step to more than one delay + cells which is useful for some special boards or cards when the default + tuning step can't find the proper delay window within limited tuning retries. Examples: diff --git a/Documentation/devicetree/bindings/mmc/sdhci-atmel.txt b/Documentation/devicetree/bindings/mmc/sdhci-atmel.txt new file mode 100644 index 000000000000..1b662d7171a0 --- /dev/null +++ b/Documentation/devicetree/bindings/mmc/sdhci-atmel.txt @@ -0,0 +1,21 @@ +* Atmel SDHCI controller + +This file documents the differences between the core properties in +Documentation/devicetree/bindings/mmc/mmc.txt and the properties used by the +sdhci-of-at91 driver. + +Required properties: +- compatible: Must be "atmel,sama5d2-sdhci". +- clocks: Phandlers to the clocks. +- clock-names: Must be "hclock", "multclk", "baseclk"; + + +Example: + +sdmmc0: sdio-host@a0000000 { + compatible = "atmel,sama5d2-sdhci"; + reg = <0xa0000000 0x300>; + interrupts = <31 IRQ_TYPE_LEVEL_HIGH 0>; + clocks = <&sdmmc0_hclk>, <&sdmmc0_gclk>, <&main>; + clock-names = "hclock", "multclk", "baseclk"; +}; diff --git a/Documentation/devicetree/bindings/mmc/ti-omap-hsmmc.txt b/Documentation/devicetree/bindings/mmc/ti-omap-hsmmc.txt index 76bf087bc889..74166a0d460d 100644 --- a/Documentation/devicetree/bindings/mmc/ti-omap-hsmmc.txt +++ b/Documentation/devicetree/bindings/mmc/ti-omap-hsmmc.txt @@ -102,7 +102,7 @@ not every application needs SDIO irq, e.g. MMC cards. pinctrl-1 = <&mmc1_idle>; pinctrl-2 = <&mmc1_sleep>; ... - interrupts-extended = <&intc 64 &gpio2 28 0>; + interrupts-extended = <&intc 64 &gpio2 28 GPIO_ACTIVE_LOW>; }; mmc1_idle : pinmux_cirq_pin { diff --git a/Documentation/devicetree/bindings/mtd/fsl-quadspi.txt b/Documentation/devicetree/bindings/mtd/fsl-quadspi.txt index 4461dc71cb10..862aa2f8837a 100644 --- a/Documentation/devicetree/bindings/mtd/fsl-quadspi.txt +++ b/Documentation/devicetree/bindings/mtd/fsl-quadspi.txt @@ -1,7 +1,8 @@ * Freescale Quad Serial Peripheral Interface(QuadSPI) Required properties: - - compatible : Should be "fsl,vf610-qspi" or "fsl,imx6sx-qspi" + - compatible : Should be "fsl,vf610-qspi", "fsl,imx6sx-qspi", + "fsl,imx7d-qspi", "fsl,imx6ul-qspi" - reg : the first contains the register location and length, the second contains the memory mapping address and length - reg-names: Should contain the reg names "QuadSPI" and "QuadSPI-memory" diff --git a/Documentation/devicetree/bindings/mtd/nxp-spifi.txt b/Documentation/devicetree/bindings/mtd/nxp-spifi.txt new file mode 100644 index 000000000000..f8b6b250654e --- /dev/null +++ b/Documentation/devicetree/bindings/mtd/nxp-spifi.txt @@ -0,0 +1,58 @@ +* NXP SPI Flash Interface (SPIFI) + +NXP SPIFI is a specialized SPI interface for serial Flash devices. +It supports one Flash device with 1-, 2- and 4-bits width in SPI +mode 0 or 3. The controller operates in either command or memory +mode. In memory mode the Flash is accessible from the CPU as +normal memory. + +Required properties: + - compatible : Should be "nxp,lpc1773-spifi" + - reg : the first contains the register location and length, + the second contains the memory mapping address and length + - reg-names: Should contain the reg names "spifi" and "flash" + - interrupts : Should contain the interrupt for the device + - clocks : The clocks needed by the SPIFI controller + - clock-names : Should contain the clock names "spifi" and "reg" + +Optional properties: + - resets : phandle + reset specifier + +The SPI Flash must be a child of the SPIFI node and must have a +compatible property as specified in bindings/mtd/jedec,spi-nor.txt + +Optionally it can also contain the following properties. + - spi-cpol : Controller only supports mode 0 and 3 so either + both spi-cpol and spi-cpha should be present or + none of them + - spi-cpha : See above + - spi-rx-bus-width : Used to select how many pins that are used + for input on the controller + +See bindings/spi/spi-bus.txt for more information. + +Example: +spifi: spifi@40003000 { + compatible = "nxp,lpc1773-spifi"; + reg = <0x40003000 0x1000>, <0x14000000 0x4000000>; + reg-names = "spifi", "flash"; + interrupts = <30>; + clocks = <&ccu1 CLK_SPIFI>, <&ccu1 CLK_CPU_SPIFI>; + clock-names = "spifi", "reg"; + resets = <&rgu 53>; + + flash@0 { + compatible = "jedec,spi-nor"; + spi-cpol; + spi-cpha; + spi-rx-bus-width = <4>; + #address-cells = <1>; + #size-cells = <1>; + + partition@0 { + label = "data"; + reg = <0 0x200000>; + }; + }; +}; + diff --git a/Documentation/devicetree/bindings/mtd/pxa3xx-nand.txt b/Documentation/devicetree/bindings/mtd/pxa3xx-nand.txt index 4f833e3c4f51..d9b655f11048 100644 --- a/Documentation/devicetree/bindings/mtd/pxa3xx-nand.txt +++ b/Documentation/devicetree/bindings/mtd/pxa3xx-nand.txt @@ -11,6 +11,7 @@ Required properties: Optional properties: + - dmas: dma data channel, see dma.txt binding doc - marvell,nand-enable-arbiter: Set to enable the bus arbiter - marvell,nand-keep-config: Set to keep the NAND controller config as set by the bootloader @@ -32,6 +33,8 @@ Example: compatible = "marvell,pxa3xx-nand"; reg = <0x43100000 90>; interrupts = <45>; + dmas = <&pdma 97 0>; + dma-names = "data"; #address-cells = <1>; marvell,nand-enable-arbiter; diff --git a/Documentation/devicetree/bindings/net/cpsw.txt b/Documentation/devicetree/bindings/net/cpsw.txt index 33fe8462edf4..a9df21aaa154 100644 --- a/Documentation/devicetree/bindings/net/cpsw.txt +++ b/Documentation/devicetree/bindings/net/cpsw.txt @@ -2,7 +2,11 @@ TI SoC Ethernet Switch Controller Device Tree Bindings ------------------------------------------------------ Required properties: -- compatible : Should be "ti,cpsw" +- compatible : Should be one of the below:- + "ti,cpsw" for backward compatible + "ti,am335x-cpsw" for AM335x controllers + "ti,am4372-cpsw" for AM437x controllers + "ti,dra7-cpsw" for DRA7x controllers - reg : physical base address and size of the cpsw registers map - interrupts : property with a value describing the interrupt diff --git a/Documentation/devicetree/bindings/net/dsa/dsa.txt b/Documentation/devicetree/bindings/net/dsa/dsa.txt index f0b4cd72411d..04e6bef3ac3f 100644 --- a/Documentation/devicetree/bindings/net/dsa/dsa.txt +++ b/Documentation/devicetree/bindings/net/dsa/dsa.txt @@ -44,9 +44,10 @@ Note that a port labelled "dsa" will imply checking for the uplink phandle described below. Optionnal property: -- link : Should be a phandle to another switch's DSA port. +- link : Should be a list of phandles to another switch's DSA port. This property is only used when switches are being - chained/cascaded together. + chained/cascaded together. This port is used as outgoing port + towards the phandle port, which can be more than one hop away. - phy-handle : Phandle to a PHY on an external MDIO bus, not the switch internal one. See @@ -58,6 +59,10 @@ Optionnal property: Documentation/devicetree/bindings/net/ethernet.txt for details. +- mii-bus : Should be a phandle to a valid MDIO bus device node. + This mii-bus will be used in preference to the + global dsa,mii-bus defined above, for this switch. + Optional subnodes: - fixed-link : Fixed-link subnode describing a link to a non-MDIO managed entity. See @@ -96,10 +101,11 @@ Example: label = "cpu"; }; - switch0uplink: port@6 { + switch0port6: port@6 { reg = <6>; label = "dsa"; - link = <&switch1uplink>; + link = <&switch1port0 + &switch2port0>; }; }; @@ -107,11 +113,31 @@ Example: #address-cells = <1>; #size-cells = <0>; reg = <17 1>; /* MDIO address 17, switch 1 in tree */ + mii-bus = <&mii_bus1>; - switch1uplink: port@0 { + switch1port0: port@0 { reg = <0>; label = "dsa"; - link = <&switch0uplink>; + link = <&switch0port6>; + }; + switch1port1: port@1 { + reg = <1>; + label = "dsa"; + link = <&switch2port1>; + }; + }; + + switch@2 { + #address-cells = <1>; + #size-cells = <0>; + reg = <18 2>; /* MDIO address 18, switch 2 in tree */ + mii-bus = <&mii_bus1>; + + switch2port0: port@0 { + reg = <0>; + label = "dsa"; + link = <&switch1port1 + &switch0port6>; }; }; }; diff --git a/Documentation/devicetree/bindings/net/ethernet.txt b/Documentation/devicetree/bindings/net/ethernet.txt index 41b3f3f864e8..5d88f37480b6 100644 --- a/Documentation/devicetree/bindings/net/ethernet.txt +++ b/Documentation/devicetree/bindings/net/ethernet.txt @@ -25,7 +25,11 @@ The following properties are common to the Ethernet controllers: flow control thresholds. - tx-fifo-depth: the size of the controller's transmit fifo in bytes. This is used for components that can have configurable fifo sizes. +- managed: string, specifies the PHY management type. Supported values are: + "auto", "in-band-status". "auto" is the default, it usess MDIO for + management if fixed-link is not specified. Child nodes of the Ethernet controller are typically the individual PHY devices connected via the MDIO bus (sometimes the MDIO bus controller is separate). They are described in the phy.txt file in this same directory. +For non-MDIO PHY management see fixed-link.txt. diff --git a/Documentation/devicetree/bindings/net/fixed-link.txt b/Documentation/devicetree/bindings/net/fixed-link.txt index 82bf7e0f47b6..ec5d889fe3d8 100644 --- a/Documentation/devicetree/bindings/net/fixed-link.txt +++ b/Documentation/devicetree/bindings/net/fixed-link.txt @@ -17,6 +17,8 @@ properties: enabled. * 'asym-pause' (boolean, optional), to indicate that asym_pause should be enabled. +* 'link-gpios' ('gpio-list', optional), to indicate if a gpio can be read + to determine if the link is up. Old, deprecated 'fixed-link' binding: @@ -30,7 +32,7 @@ Old, deprecated 'fixed-link' binding: - e: asymmetric pause configuration: 0 for no asymmetric pause, 1 for asymmetric pause -Example: +Examples: ethernet@0 { ... @@ -40,3 +42,13 @@ ethernet@0 { }; ... }; + +ethernet@1 { + ... + fixed-link { + speed = <1000>; + pause; + link-gpios = <&gpio0 12 GPIO_ACTIVE_HIGH>; + }; + ... +}; diff --git a/Documentation/devicetree/bindings/net/keystone-netcp.txt b/Documentation/devicetree/bindings/net/keystone-netcp.txt index d0e6fa38f335..b30ab6b5cbfa 100644 --- a/Documentation/devicetree/bindings/net/keystone-netcp.txt +++ b/Documentation/devicetree/bindings/net/keystone-netcp.txt @@ -130,7 +130,11 @@ Required properties: Optional properties: - efuse-mac: If this is 1, then the MAC address for the interface is - obtained from the device efuse mac address register + obtained from the device efuse mac address register. + If this is 2, the two DWORDs occupied by the MAC address + are swapped. The netcp driver will swap the two DWORDs + back to the proper order when this property is set to 2 + when it obtains the mac address from efuse. - local-mac-address: the driver is designed to use the of_get_mac_address api only if efuse-mac is 0. When efuse-mac is 0, the MAC address is obtained from local-mac-address. If this diff --git a/Documentation/devicetree/bindings/net/nfc/s3fwrn5.txt b/Documentation/devicetree/bindings/net/nfc/s3fwrn5.txt new file mode 100644 index 000000000000..fb1e75facf1b --- /dev/null +++ b/Documentation/devicetree/bindings/net/nfc/s3fwrn5.txt @@ -0,0 +1,27 @@ +* Samsung S3FWRN5 NCI NFC Controller + +Required properties: +- compatible: Should be "samsung,s3fwrn5-i2c". +- reg: address on the bus +- interrupt-parent: phandle for the interrupt gpio controller +- interrupts: GPIO interrupt to which the chip is connected +- s3fwrn5,en-gpios: Output GPIO pin used for enabling/disabling the chip +- s3fwrn5,fw-gpios: Output GPIO pin used to enter firmware mode and + sleep/wakeup control + +Example: + +&hsi2c_4 { + status = "okay"; + s3fwrn5@27 { + compatible = "samsung,s3fwrn5-i2c"; + + reg = <0x27>; + + interrupt-parent = <&gpa1>; + interrupts = <3 0 0>; + + s3fwrn5,en-gpios = <&gpf1 4 0>; + s3fwrn5,fw-gpios = <&gpj0 2 0>; + }; +}; diff --git a/Documentation/devicetree/bindings/net/nfc/st-nci.txt b/Documentation/devicetree/bindings/net/nfc/st-nci-i2c.txt similarity index 100% rename from Documentation/devicetree/bindings/net/nfc/st-nci.txt rename to Documentation/devicetree/bindings/net/nfc/st-nci-i2c.txt diff --git a/Documentation/devicetree/bindings/net/nfc/st-nci-spi.txt b/Documentation/devicetree/bindings/net/nfc/st-nci-spi.txt new file mode 100644 index 000000000000..525681b6dc39 --- /dev/null +++ b/Documentation/devicetree/bindings/net/nfc/st-nci-spi.txt @@ -0,0 +1,31 @@ +* STMicroelectronics SAS. ST NCI NFC Controller + +Required properties: +- compatible: Should be "st,st21nfcb-spi" +- spi-max-frequency: Maximum SPI frequency (<= 10000000). +- interrupt-parent: phandle for the interrupt gpio controller +- interrupts: GPIO interrupt to which the chip is connected +- reset-gpios: Output GPIO pin used to reset the ST21NFCB + +Optional SoC Specific Properties: +- pinctrl-names: Contains only one value - "default". +- pintctrl-0: Specifies the pin control groups used for this controller. + +Example (for ARM-based BeagleBoard xM with ST21NFCB on SPI4): + +&mcspi4 { + + status = "okay"; + + st21nfcb: st21nfcb@0 { + + compatible = "st,st21nfcb-spi"; + + clock-frequency = <4000000>; + + interrupt-parent = <&gpio5>; + interrupts = <2 IRQ_TYPE_EDGE_RISING>; + + reset-gpios = <&gpio5 29 GPIO_ACTIVE_HIGH>; + }; +}; diff --git a/Documentation/devicetree/bindings/net/snps,dwc-qos-ethernet.txt b/Documentation/devicetree/bindings/net/snps,dwc-qos-ethernet.txt new file mode 100644 index 000000000000..51f8d2eba8d8 --- /dev/null +++ b/Documentation/devicetree/bindings/net/snps,dwc-qos-ethernet.txt @@ -0,0 +1,75 @@ +* Synopsys DWC Ethernet QoS IP version 4.10 driver (GMAC) + + +Required properties: +- compatible: Should be "snps,dwc-qos-ethernet-4.10" +- reg: Address and length of the register set for the device +- clocks: Phandles to the reference clock and the bus clock +- clock-names: Should be "phy_ref_clk" for the reference clock and "apb_pclk" + for the bus clock. +- interrupt-parent: Should be the phandle for the interrupt controller + that services interrupts for this device +- interrupts: Should contain the core's combined interrupt signal +- phy-mode: See ethernet.txt file in the same directory + +Optional properties: +- dma-coherent: Present if dma operations are coherent +- mac-address: See ethernet.txt in the same directory +- local-mac-address: See ethernet.txt in the same directory +- snps,en-lpi: If present it enables use of the AXI low-power interface +- snps,write-requests: Number of write requests that the AXI port can issue. + It depends on the SoC configuration. +- snps,read-requests: Number of read requests that the AXI port can issue. + It depends on the SoC configuration. +- snps,burst-map: Bitmap of allowed AXI burst lengts, with the LSB + representing 4, then 8 etc. +- snps,txpbl: DMA Programmable burst length for the TX DMA +- snps,rxpbl: DMA Programmable burst length for the RX DMA +- snps,en-tx-lpi-clockgating: Enable gating of the MAC TX clock during + TX low-power mode. +- phy-handle: See ethernet.txt file in the same directory +- mdio device tree subnode: When the GMAC has a phy connected to its local + mdio, there must be device tree subnode with the following + required properties: + - compatible: Must be "snps,dwc-qos-ethernet-mdio". + - #address-cells: Must be <1>. + - #size-cells: Must be <0>. + + For each phy on the mdio bus, there must be a node with the following + fields: + + - reg: phy id used to communicate to phy. + - device_type: Must be "ethernet-phy". + - fixed-mode device tree subnode: see fixed-link.txt in the same directory + +Examples: +ethernet2@40010000 { + clock-names = "phy_ref_clk", "apb_pclk"; + clocks = <&clkc 17>, <&clkc 15>; + compatible = "snps,dwc-qos-ethernet-4.10"; + interrupt-parent = <&intc>; + interrupts = <0x0 0x1e 0x4>; + reg = <0x40010000 0x4000>; + phy-handle = <&phy2>; + phy-mode = "gmii"; + + snps,en-tx-lpi-clockgating; + snps,en-lpi; + snps,write-requests = <2>; + snps,read-requests = <16>; + snps,burst-map = <0x7>; + snps,txpbl = <8>; + snps,rxpbl = <2>; + + dma-coherent; + + mdio { + #address-cells = <0x1>; + #size-cells = <0x0>; + phy2: phy@1 { + compatible = "ethernet-phy-ieee802.3-c22"; + device_type = "ethernet-phy"; + reg = <0x1>; + }; + }; +}; diff --git a/Documentation/devicetree/bindings/power/opp.txt b/Documentation/devicetree/bindings/opp/opp.txt similarity index 93% rename from Documentation/devicetree/bindings/power/opp.txt rename to Documentation/devicetree/bindings/opp/opp.txt index 0d5e7c978121..0cb44dc21f97 100644 --- a/Documentation/devicetree/bindings/power/opp.txt +++ b/Documentation/devicetree/bindings/opp/opp.txt @@ -88,7 +88,7 @@ This defines voltage-current-frequency combinations along with other related properties. Required properties: -- opp-hz: Frequency in Hz +- opp-hz: Frequency in Hz, expressed as a 64-bit big-endian integer. Optional properties: - opp-microvolt: voltage in micro Volts. @@ -158,20 +158,20 @@ Example 1: Single cluster Dual-core ARM cortex A9, switch DVFS states together. opp-shared; opp00 { - opp-hz = <1000000000>; + opp-hz = /bits/ 64 <1000000000>; opp-microvolt = <970000 975000 985000>; opp-microamp = <70000>; clock-latency-ns = <300000>; opp-suspend; }; opp01 { - opp-hz = <1100000000>; + opp-hz = /bits/ 64 <1100000000>; opp-microvolt = <980000 1000000 1010000>; opp-microamp = <80000>; clock-latency-ns = <310000>; }; opp02 { - opp-hz = <1200000000>; + opp-hz = /bits/ 64 <1200000000>; opp-microvolt = <1025000>; clock-latency-ns = <290000>; turbo-mode; @@ -237,20 +237,20 @@ independently. */ opp00 { - opp-hz = <1000000000>; + opp-hz = /bits/ 64 <1000000000>; opp-microvolt = <970000 975000 985000>; opp-microamp = <70000>; clock-latency-ns = <300000>; opp-suspend; }; opp01 { - opp-hz = <1100000000>; + opp-hz = /bits/ 64 <1100000000>; opp-microvolt = <980000 1000000 1010000>; opp-microamp = <80000>; clock-latency-ns = <310000>; }; opp02 { - opp-hz = <1200000000>; + opp-hz = /bits/ 64 <1200000000>; opp-microvolt = <1025000>; opp-microamp = <90000; lock-latency-ns = <290000>; @@ -313,20 +313,20 @@ DVFS state together. opp-shared; opp00 { - opp-hz = <1000000000>; + opp-hz = /bits/ 64 <1000000000>; opp-microvolt = <970000 975000 985000>; opp-microamp = <70000>; clock-latency-ns = <300000>; opp-suspend; }; opp01 { - opp-hz = <1100000000>; + opp-hz = /bits/ 64 <1100000000>; opp-microvolt = <980000 1000000 1010000>; opp-microamp = <80000>; clock-latency-ns = <310000>; }; opp02 { - opp-hz = <1200000000>; + opp-hz = /bits/ 64 <1200000000>; opp-microvolt = <1025000>; opp-microamp = <90000>; clock-latency-ns = <290000>; @@ -339,20 +339,20 @@ DVFS state together. opp-shared; opp10 { - opp-hz = <1300000000>; + opp-hz = /bits/ 64 <1300000000>; opp-microvolt = <1045000 1050000 1055000>; opp-microamp = <95000>; clock-latency-ns = <400000>; opp-suspend; }; opp11 { - opp-hz = <1400000000>; + opp-hz = /bits/ 64 <1400000000>; opp-microvolt = <1075000>; opp-microamp = <100000>; clock-latency-ns = <400000>; }; opp12 { - opp-hz = <1500000000>; + opp-hz = /bits/ 64 <1500000000>; opp-microvolt = <1010000 1100000 1110000>; opp-microamp = <95000>; clock-latency-ns = <400000>; @@ -379,7 +379,7 @@ Example 4: Handling multiple regulators opp-shared; opp00 { - opp-hz = <1000000000>; + opp-hz = /bits/ 64 <1000000000>; opp-microvolt = <970000>, /* Supply 0 */ <960000>, /* Supply 1 */ <960000>; /* Supply 2 */ @@ -392,7 +392,7 @@ Example 4: Handling multiple regulators /* OR */ opp00 { - opp-hz = <1000000000>; + opp-hz = /bits/ 64 <1000000000>; opp-microvolt = <970000 975000 985000>, /* Supply 0 */ <960000 965000 975000>, /* Supply 1 */ <960000 965000 975000>; /* Supply 2 */ @@ -405,7 +405,7 @@ Example 4: Handling multiple regulators /* OR */ opp00 { - opp-hz = <1000000000>; + opp-hz = /bits/ 64 <1000000000>; opp-microvolt = <970000 975000 985000>, /* Supply 0 */ <960000 965000 975000>, /* Supply 1 */ <960000 965000 975000>; /* Supply 2 */ @@ -437,12 +437,12 @@ Example 5: Multiple OPP tables opp-shared; opp00 { - opp-hz = <600000000>; + opp-hz = /bits/ 64 <600000000>; ... }; opp01 { - opp-hz = <800000000>; + opp-hz = /bits/ 64 <800000000>; ... }; }; @@ -453,12 +453,12 @@ Example 5: Multiple OPP tables opp-shared; opp10 { - opp-hz = <1000000000>; + opp-hz = /bits/ 64 <1000000000>; ... }; opp11 { - opp-hz = <1100000000>; + opp-hz = /bits/ 64 <1100000000>; ... }; }; diff --git a/Documentation/devicetree/bindings/panel/auo,b080uan01.txt b/Documentation/devicetree/bindings/panel/auo,b080uan01.txt new file mode 100644 index 000000000000..bae0e2b51467 --- /dev/null +++ b/Documentation/devicetree/bindings/panel/auo,b080uan01.txt @@ -0,0 +1,7 @@ +AU Optronics Corporation 8.0" WUXGA TFT LCD panel + +Required properties: +- compatible: should be "auo,b101ean01" + +This binding is compatible with the simple-panel binding, which is specified +in simple-panel.txt in this directory. diff --git a/Documentation/devicetree/bindings/panel/lg,lg4573.txt b/Documentation/devicetree/bindings/panel/lg,lg4573.txt new file mode 100644 index 000000000000..824441f4e95a --- /dev/null +++ b/Documentation/devicetree/bindings/panel/lg,lg4573.txt @@ -0,0 +1,19 @@ +LG LG4573 TFT Liquid Crystal Display with SPI control bus + +Required properties: + - compatible: "lg,lg4573" + - reg: address of the panel on the SPI bus + +The panel must obey rules for SPI slave device specified in document [1]. + +[1]: Documentation/devicetree/bindings/spi/spi-bus.txt + +Example: + + lcd_panel: display@0 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "lg,lg4573"; + spi-max-frequency = <10000000>; + reg = <0>; + }; diff --git a/Documentation/devicetree/bindings/panel/nec,nl4827hc19-05b.txt b/Documentation/devicetree/bindings/panel/nec,nl4827hc19-05b.txt new file mode 100644 index 000000000000..8e1914d1edb8 --- /dev/null +++ b/Documentation/devicetree/bindings/panel/nec,nl4827hc19-05b.txt @@ -0,0 +1,7 @@ +NEC LCD Technologies,Ltd. WQVGA TFT LCD panel + +Required properties: +- compatible: should be "nec,nl4827hc19-05b" + +This binding is compatible with the simple-panel binding, which is specified +in simple-panel.txt in this directory. diff --git a/Documentation/devicetree/bindings/panel/okaya,rs800480t-7x0gp.txt b/Documentation/devicetree/bindings/panel/okaya,rs800480t-7x0gp.txt new file mode 100644 index 000000000000..ddf8e211d382 --- /dev/null +++ b/Documentation/devicetree/bindings/panel/okaya,rs800480t-7x0gp.txt @@ -0,0 +1,7 @@ +OKAYA Electric America, Inc. RS800480T-7X0GP 7" WVGA LCD panel + +Required properties: +- compatible: should be "okaya,rs800480t-7x0gp" + +This binding is compatible with the simple-panel binding, which is specified +in simple-panel.txt in this directory. diff --git a/Documentation/devicetree/bindings/pinctrl/allwinner,sunxi-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/allwinner,sunxi-pinctrl.txt index 9462ab7ddd1f..3c821cda1ad0 100644 --- a/Documentation/devicetree/bindings/pinctrl/allwinner,sunxi-pinctrl.txt +++ b/Documentation/devicetree/bindings/pinctrl/allwinner,sunxi-pinctrl.txt @@ -48,7 +48,7 @@ Optional subnode-properties: Examples: -pinctrl@01c20800 { +pio: pinctrl@01c20800 { compatible = "allwinner,sun5i-a13-pinctrl"; reg = <0x01c20800 0x400>; #address-cells = <1>; @@ -68,3 +68,38 @@ pinctrl@01c20800 { allwinner,pull = <0>; }; }; + + +GPIO and interrupt controller +----------------------------- + +This hardware also acts as a GPIO controller and an interrupt +controller. + +Consumers that would want to refer to one or the other (or both) +should provide through the usual *-gpios and interrupts properties a +cell with 3 arguments, first the number of the bank, then the pin +inside that bank, and finally the flags for the GPIO/interrupts. + +Example: + +xio: gpio@38 { + compatible = "nxp,pcf8574a"; + reg = <0x38>; + + gpio-controller; + #gpio-cells = <2>; + + interrupt-parent = <&pio>; + interrupts = <6 0 IRQ_TYPE_EDGE_FALLING>; + interrupt-controller; + #interrupt-cells = <2>; +}; + +reg_usb1_vbus: usb1-vbus { + compatible = "regulator-fixed"; + regulator-name = "usb1-vbus"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + gpio = <&pio 7 6 GPIO_ACTIVE_HIGH>; +}; diff --git a/Documentation/devicetree/bindings/pinctrl/cnxt,cx92755-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/cnxt,cx92755-pinctrl.txt new file mode 100644 index 000000000000..23ce8dc26990 --- /dev/null +++ b/Documentation/devicetree/bindings/pinctrl/cnxt,cx92755-pinctrl.txt @@ -0,0 +1,86 @@ +Conexant Digicolor CX92755 General Purpose Pin Mapping + +This document describes the device tree binding of the pin mapping hardware +modules in the Conexant Digicolor CX92755 SoCs. The CX92755 in one of the +Digicolor series of SoCs. + +=== Pin Controller Node === + +Required Properties: + +- compatible: Must be "cnxt,cx92755-pinctrl" +- reg: Base address of the General Purpose Pin Mapping register block and the + size of the block. +- gpio-controller: Marks the device node as a GPIO controller. +- #gpio-cells: Must be <2>. The first cell is the pin number and the + second cell is used to specify flags. See include/dt-bindings/gpio/gpio.h + for possible values. + +For example, the following is the bare minimum node: + + pinctrl: pinctrl@f0000e20 { + compatible = "cnxt,cx92755-pinctrl"; + reg = <0xf0000e20 0x100>; + gpio-controller; + #gpio-cells = <2>; + }; + +As a pin controller device, in addition to the required properties, this node +should also contain the pin configuration nodes that client devices reference, +if any. + +For a general description of GPIO bindings, please refer to ../gpio/gpio.txt. + +=== Pin Configuration Node === + +Each pin configuration node is a sub-node of the pin controller node and is a +container of an arbitrary number of subnodes, called pin group nodes in this +document. + +Please refer to the pinctrl-bindings.txt in this directory for details of the +common pinctrl bindings used by client devices, including the definition of a +"pin configuration node". + +=== Pin Group Node === + +A pin group node specifies the desired pin mux for an arbitrary number of +pins. The name of the pin group node is optional and not used. + +A pin group node only affects the properties specified in the node, and has no +effect on any properties that are omitted. + +The pin group node accepts a subset of the generic pin config properties. For +details generic pin config properties, please refer to pinctrl-bindings.txt +and . + +Required Pin Group Node Properties: + +- pins: Multiple strings. Specifies the name(s) of one or more pins to be + configured by this node. The format of a pin name string is "GP_xy", where x + is an uppercase character from 'A' to 'R', and y is a digit from 0 to 7. +- function: String. Specifies the pin mux selection. Values must be one of: + "gpio", "client_a", "client_b", "client_c" + +Example: + pinctrl: pinctrl@f0000e20 { + compatible = "cnxt,cx92755-pinctrl"; + reg = <0xf0000e20 0x100>; + + uart0_default: uart0_active { + data_signals { + pins = "GP_O0", "GP_O1"; + function = "client_b"; + }; + }; + }; + + uart0: uart@f0000740 { + compatible = "cnxt,cx92755-usart"; + ... + pinctrl-0 = <&uart0_default>; + pinctrl-names = "default"; + }; + +In the example above, a single pin group configuration node defines the +"client select" for the Rx and Tx signals of uart0. The uart0 node references +that pin configuration node using the &uart0_default phandle. diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-mpp.txt b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-mpp.txt index ed19991aad35..d7803a2a94e9 100644 --- a/Documentation/devicetree/bindings/pinctrl/qcom,pmic-mpp.txt +++ b/Documentation/devicetree/bindings/pinctrl/qcom,pmic-mpp.txt @@ -7,8 +7,13 @@ of PMIC's from Qualcomm. Usage: required Value type: Definition: Should contain one of: + "qcom,pm8018-mpp", + "qcom,pm8038-mpp", + "qcom,pm8821-mpp", "qcom,pm8841-mpp", "qcom,pm8916-mpp", + "qcom,pm8917-mpp", + "qcom,pm8921-mpp", "qcom,pm8941-mpp", "qcom,pma8084-mpp", @@ -77,12 +82,9 @@ to specify in a pin configuration subnode: Value type: Definition: Specify the alternative function to be configured for the specified pins. Valid values are: - "normal", - "paired", - "dtest1", - "dtest2", - "dtest3", - "dtest4" + "digital", + "analog", + "sink" - bias-disable: Usage: optional @@ -127,12 +129,18 @@ to specify in a pin configuration subnode: Definition: Selects the power source for the specified pins. Valid power sources are defined in -- qcom,analog-mode: +- qcom,analog-level: Usage: optional - Value type: - Definition: Selects Analog mode of operation: combined with input-enable - and/or output-high, output-low MPP could operate as - Bidirectional Logic, Analog Input, Analog Output. + Value type: + Definition: Selects the source for analog output. Valued values are + defined in + PMIC_MPP_AOUT_LVL_* + +- qcom,dtest: + Usage: optional + Value type: + Definition: Selects which dtest rail to be routed in the various functions. + Valid values are 1-4 - qcom,amux-route: Usage: optional @@ -140,6 +148,10 @@ to specify in a pin configuration subnode: Definition: Selects the source for analog input. Valid values are defined in PMIC_MPP_AMUX_ROUTE_CH5, PMIC_MPP_AMUX_ROUTE_CH6... +- qcom,paired: + Usage: optional + Value type: + Definition: Indicates that the pin should be operating in paired mode. Example: @@ -156,7 +168,7 @@ Example: pm8841_default: default { gpio { pins = "mpp1", "mpp2", "mpp3", "mpp4"; - function = "normal"; + function = "digital"; input-enable; power-source = ; }; diff --git a/Documentation/devicetree/bindings/pinctrl/renesas,pfc-pinctrl.txt b/Documentation/devicetree/bindings/pinctrl/renesas,pfc-pinctrl.txt index 51cee44fc140..9496934528bd 100644 --- a/Documentation/devicetree/bindings/pinctrl/renesas,pfc-pinctrl.txt +++ b/Documentation/devicetree/bindings/pinctrl/renesas,pfc-pinctrl.txt @@ -58,12 +58,12 @@ are parsed through phandles and processed purely based on their content. Pin Configuration Node Properties: -- renesas,pins : An array of strings, each string containing the name of a pin. -- renesas,groups : An array of strings, each string containing the name of a pin +- pins : An array of strings, each string containing the name of a pin. +- groups : An array of strings, each string containing the name of a pin group. -- renesas,function: A string containing the name of the function to mux to the - pin group(s) specified by the renesas,groups property +- function: A string containing the name of the function to mux to the pin + group(s) specified by the groups property. Valid values for pin, group and function names can be found in the group and function arrays of the PFC data file corresponding to the SoC @@ -71,7 +71,9 @@ Pin Configuration Node Properties: The pin configuration parameters use the generic pinconf bindings defined in pinctrl-bindings.txt in this directory. The supported parameters are -bias-disable, bias-pull-up and bias-pull-down. +bias-disable, bias-pull-up, bias-pull-down and power-source. For pins that +have a configurable I/O voltage, the power-source value should be the +nominal I/O voltage in millivolts. GPIO @@ -141,19 +143,19 @@ Example 3: KZM-A9-GT (SH-Mobile AG5) default pin state hog and pin control maps mmcif_pins: mmcif { mux { - renesas,groups = "mmc0_data8_0", "mmc0_ctrl_0"; - renesas,function = "mmc0"; + groups = "mmc0_data8_0", "mmc0_ctrl_0"; + function = "mmc0"; }; cfg { - renesas,groups = "mmc0_data8_0"; - renesas,pins = "PORT279"; + groups = "mmc0_data8_0"; + pins = "PORT279"; bias-pull-up; }; }; scifa4_pins: scifa4 { - renesas,groups = "scifa4_data", "scifa4_ctrl"; - renesas,function = "scifa4"; + groups = "scifa4_data", "scifa4_ctrl"; + function = "scifa4"; }; }; diff --git a/Documentation/devicetree/bindings/pinctrl/ste,nomadik.txt b/Documentation/devicetree/bindings/pinctrl/ste,nomadik.txt index f63fcb3ed352..2213802435e0 100644 --- a/Documentation/devicetree/bindings/pinctrl/ste,nomadik.txt +++ b/Documentation/devicetree/bindings/pinctrl/ste,nomadik.txt @@ -3,7 +3,9 @@ ST Ericsson Nomadik pinmux controller Required properties: - compatible: "stericsson,db8500-pinctrl", "stericsson,db8540-pinctrl", "stericsson,stn8815-pinctrl" -- reg: Should contain the register physical address and length of the PRCMU. +- nomadik-gpio-chips: array of phandles to the corresponding GPIO chips + (these have the register ranges used by the pin controller). +- prcm: phandle to the PRCMU managing the back end of this pin controller Please refer to pinctrl-bindings.txt in this directory for details of the common pinctrl bindings used by client devices, including the meaning of the @@ -74,7 +76,8 @@ Example board file extract: pinctrl@80157000 { compatible = "stericsson,db8500-pinctrl"; - reg = <0x80157000 0x2000>; + nomadik-gpio-chips = <&gpio0>, <&gpio1>, <&gpio2>, <&gpio3>; + prcm = <&prcmu>; pinctrl-names = "default"; diff --git a/Documentation/devicetree/bindings/power/power_domain.txt b/Documentation/devicetree/bindings/power/power_domain.txt index 0f8ed3710c66..025b5e7df61c 100644 --- a/Documentation/devicetree/bindings/power/power_domain.txt +++ b/Documentation/devicetree/bindings/power/power_domain.txt @@ -48,7 +48,7 @@ Example 2: #power-domain-cells = <1>; }; - child: power-controller@12340000 { + child: power-controller@12341000 { compatible = "foo,power-controller"; reg = <0x12341000 0x1000>; power-domains = <&parent 0>; diff --git a/Documentation/devicetree/bindings/power/rockchip-io-domain.txt b/Documentation/devicetree/bindings/power/rockchip-io-domain.txt index 8b70db103ca7..b8627e763dba 100644 --- a/Documentation/devicetree/bindings/power/rockchip-io-domain.txt +++ b/Documentation/devicetree/bindings/power/rockchip-io-domain.txt @@ -33,6 +33,8 @@ Required properties: - compatible: should be one of: - "rockchip,rk3188-io-voltage-domain" for rk3188 - "rockchip,rk3288-io-voltage-domain" for rk3288 + - "rockchip,rk3368-io-voltage-domain" for rk3368 + - "rockchip,rk3368-pmu-io-voltage-domain" for rk3368 pmu-domains - rockchip,grf: phandle to the syscon managing the "general register files" @@ -64,6 +66,18 @@ Possible supplies for rk3288: - sdcard-supply: The supply connected to SDMMC0_VDD. - wifi-supply: The supply connected to APIO3_VDD. Also known as SDIO0. +Possible supplies for rk3368: +- audio-supply: The supply connected to APIO3_VDD. +- dvp-supply: The supply connected to DVPIO_VDD. +- flash0-supply: The supply connected to FLASH0_VDD. Typically for eMMC +- gpio30-supply: The supply connected to APIO1_VDD. +- gpio1830 The supply connected to APIO4_VDD. +- sdcard-supply: The supply connected to SDMMC0_VDD. +- wifi-supply: The supply connected to APIO2_VDD. Also known as SDIO0. + +Possible supplies for rk3368 pmu-domains: +- pmu-supply: The supply connected to PMUIO_VDD. +- vop-supply: The supply connected to LCDC_VDD. Example: diff --git a/Documentation/devicetree/bindings/powerpc/fsl/scfg.txt b/Documentation/devicetree/bindings/powerpc/fsl/scfg.txt new file mode 100644 index 000000000000..0532c46b3372 --- /dev/null +++ b/Documentation/devicetree/bindings/powerpc/fsl/scfg.txt @@ -0,0 +1,18 @@ +Freescale Supplement configuration unit (SCFG) + +SCFG is the supplemental configuration unit, that provides SoC specific +configuration and status registers for the chip. Such as getting PEX port +status. + +Required properties: + +- compatible: should be "fsl,-scfg" +- reg: should contain base address and length of SCFG memory-mapped +registers + +Example: + + scfg: global-utilities@fc000 { + compatible = "fsl,t1040-scfg"; + reg = <0xfc000 0x1000>; + }; diff --git a/Documentation/devicetree/bindings/pwm/lpc1850-sct-pwm.txt b/Documentation/devicetree/bindings/pwm/lpc1850-sct-pwm.txt new file mode 100644 index 000000000000..36e49d4325cd --- /dev/null +++ b/Documentation/devicetree/bindings/pwm/lpc1850-sct-pwm.txt @@ -0,0 +1,20 @@ +* NXP LPC18xx State Configurable Timer - Pulse Width Modulator driver + +Required properties: + - compatible: Should be "nxp,lpc1850-sct-pwm" + - reg: Should contain physical base address and length of pwm registers. + - clocks: Must contain an entry for each entry in clock-names. + See ../clock/clock-bindings.txt for details. + - clock-names: Must include the following entries. + - pwm: PWM operating clock. + - #pwm-cells: Should be 3. See pwm.txt in this directory for the description + of the cells format. + +Example: + pwm: pwm@40000000 { + compatible = "nxp,lpc1850-sct-pwm"; + reg = <0x40000000 0x1000>; + clocks =<&ccu1 CLK_CPU_SCT>; + clock-names = "pwm"; + #pwm-cells = <3>; + }; diff --git a/Documentation/devicetree/bindings/regulator/max77686.txt b/Documentation/devicetree/bindings/regulator/max77686.txt new file mode 100644 index 000000000000..0dded64d89d3 --- /dev/null +++ b/Documentation/devicetree/bindings/regulator/max77686.txt @@ -0,0 +1,71 @@ +Binding for Maxim MAX77686 regulators + +This is a part of the device tree bindings of MAX77686 multi-function device. +More information can be found in ../mfd/max77686.txt file. + +The MAX77686 PMIC has 9 high-efficiency Buck and 26 Low-DropOut (LDO) +regulators that can be controlled over I2C. + +Following properties should be present in main device node of the MFD chip. + +Optional node: +- voltage-regulators : The regulators of max77686 have to be instantiated + under subnode named "voltage-regulators" using the following format. + + regulator_name { + regulator-compatible = LDOn/BUCKn + standard regulator constraints.... + }; + refer Documentation/devicetree/bindings/regulator/regulator.txt + + The regulator node's name should be initialized with a string +to get matched with their hardware counterparts as follow: + + -LDOn : for LDOs, where n can lie in range 1 to 26. + example: LDO1, LDO2, LDO26. + -BUCKn : for BUCKs, where n can lie in range 1 to 9. + example: BUCK1, BUCK5, BUCK9. + + Regulators which can be turned off during system suspend: + -LDOn : 2, 6-8, 10-12, 14-16, + -BUCKn : 1-4. + Use standard regulator bindings for it ('regulator-off-in-suspend'). + + LDO20, LDO21, LDO22, BUCK8 and BUCK9 can be configured to GPIO enable + control. To turn this feature on this property must be added to the regulator + sub-node: + - maxim,ena-gpios : one GPIO specifier enable control (the gpio + flags are actually ignored and always + ACTIVE_HIGH is used) + +Example: + + max77686: pmic@09 { + compatible = "maxim,max77686"; + interrupt-parent = <&wakeup_eint>; + interrupts = <26 IRQ_TYPE_NONE>; + reg = <0x09>; + + voltage-regulators { + ldo11_reg: LDO11 { + regulator-name = "vdd_ldo11"; + regulator-min-microvolt = <1900000>; + regulator-max-microvolt = <1900000>; + regulator-always-on; + }; + + buck1_reg: BUCK1 { + regulator-name = "vdd_mif"; + regulator-min-microvolt = <950000>; + regulator-max-microvolt = <1300000>; + regulator-always-on; + regulator-boot-on; + }; + + buck9_reg: BUCK9 { + regulator-name = "CAM_ISP_CORE_1.2V"; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1200000>; + maxim,ena-gpios = <&gpm0 3 GPIO_ACTIVE_HIGH>; + }; + }; diff --git a/Documentation/devicetree/bindings/rtc/nxp,lpc1788-rtc.txt b/Documentation/devicetree/bindings/rtc/nxp,lpc1788-rtc.txt new file mode 100644 index 000000000000..3c97bd180592 --- /dev/null +++ b/Documentation/devicetree/bindings/rtc/nxp,lpc1788-rtc.txt @@ -0,0 +1,21 @@ +NXP LPC1788 real-time clock + +The LPC1788 RTC provides calendar and clock functionality +together with periodic tick and alarm interrupt support. + +Required properties: +- compatible : must contain "nxp,lpc1788-rtc" +- reg : Specifies base physical address and size of the registers. +- interrupts : A single interrupt specifier. +- clocks : Must contain clock specifiers for rtc and register clock +- clock-names : Must contain "rtc" and "reg" + See ../clocks/clock-bindings.txt for details. + +Example: +rtc: rtc@40046000 { + compatible = "nxp,lpc1788-rtc"; + reg = <0x40046000 0x1000>; + interrupts = <47>; + clocks = <&creg_clk 0>, <&ccu1 CLK_CPU_BUS>; + clock-names = "rtc", "reg"; +}; diff --git a/Documentation/devicetree/bindings/rtc/rtc-omap.txt b/Documentation/devicetree/bindings/rtc/rtc-omap.txt index 43a83668673a..bf7d11ae9bea 100644 --- a/Documentation/devicetree/bindings/rtc/rtc-omap.txt +++ b/Documentation/devicetree/bindings/rtc/rtc-omap.txt @@ -16,6 +16,8 @@ Required properties: Optional properties: - system-power-controller: whether the rtc is controlling the system power through pmic_power_en +- clocks: Any internal or external clocks feeding in to rtc +- clock-names: Corresponding names of the clocks Example: @@ -26,4 +28,6 @@ rtc@1c23000 { 19>; interrupt-parent = <&intc>; system-power-controller; + clocks = <&clk_32k_rtc>, <&clk_32768_ck>; + clock-names = "ext-clk", "int-clk"; }; diff --git a/Documentation/devicetree/bindings/rtc/rtc-st-lpc.txt b/Documentation/devicetree/bindings/rtc/rtc-st-lpc.txt index 73407f502e4e..daf88265df32 100644 --- a/Documentation/devicetree/bindings/rtc/rtc-st-lpc.txt +++ b/Documentation/devicetree/bindings/rtc/rtc-st-lpc.txt @@ -1,20 +1,23 @@ STMicroelectronics Low Power Controller (LPC) - RTC =================================================== -LPC currently supports Watchdog OR Real Time Clock functionality. +LPC currently supports Watchdog OR Real Time Clock OR Clocksource +functionality. [See: ../watchdog/st_lpc_wdt.txt for Watchdog options] +[See: ../timer/st,stih407-lpc for Clocksource options] Required properties -- compatible : Must be one of: "st,stih407-lpc" "st,stih416-lpc" - "st,stih415-lpc" "st,stid127-lpc" +- compatible : Must be: "st,stih407-lpc" - reg : LPC registers base address + size - interrupts : LPC interrupt line number and associated flags - clocks : Clock used by LPC device (See: ../clock/clock-bindings.txt) -- st,lpc-mode : The LPC can run either one of two modes ST_LPC_MODE_RTC [0] or - ST_LPC_MODE_WDT [1]. One (and only one) mode must be - selected. +- st,lpc-mode : The LPC can run either one of three modes: + ST_LPC_MODE_RTC [0] + ST_LPC_MODE_WDT [1] + ST_LPC_MODE_CLKSRC [2] + One (and only one) mode must be selected. Example: lpc@fde05000 { diff --git a/Documentation/devicetree/bindings/i2c/ti,bq32k.txt b/Documentation/devicetree/bindings/rtc/ti,bq32k.txt similarity index 100% rename from Documentation/devicetree/bindings/i2c/ti,bq32k.txt rename to Documentation/devicetree/bindings/rtc/ti,bq32k.txt diff --git a/Documentation/devicetree/bindings/rtc/xlnx-rtc.txt b/Documentation/devicetree/bindings/rtc/xlnx-rtc.txt new file mode 100644 index 000000000000..0df6f016b1b7 --- /dev/null +++ b/Documentation/devicetree/bindings/rtc/xlnx-rtc.txt @@ -0,0 +1,25 @@ +* Xilinx Zynq Ultrascale+ MPSoC Real Time Clock + +RTC controller for the Xilinx Zynq MPSoC Real Time Clock +Separate IRQ lines for seconds and alarm + +Required properties: +- compatible: Should be "xlnx,zynqmp-rtc" +- reg: Physical base address of the controller and length + of memory mapped region. +- interrupts: IRQ lines for the RTC. +- interrupt-names: interrupt line names eg. "sec" "alarm" + +Optional: +- calibration: calibration value for 1 sec period which will + be programmed directly to calibration register + +Example: +rtc: rtc@ffa60000 { + compatible = "xlnx,zynqmp-rtc"; + reg = <0x0 0xffa60000 0x100>; + interrupt-parent = <&gic>; + interrupts = <0 26 4>, <0 27 4>; + interrupt-names = "alarm", "sec"; + calibration = <0x198233>; +}; diff --git a/Documentation/devicetree/bindings/serial/mtk-uart.txt b/Documentation/devicetree/bindings/serial/mtk-uart.txt index a875997f2062..2d47add34765 100644 --- a/Documentation/devicetree/bindings/serial/mtk-uart.txt +++ b/Documentation/devicetree/bindings/serial/mtk-uart.txt @@ -5,11 +5,12 @@ Required properties: * "mediatek,mt8135-uart" for MT8135 compatible UARTS * "mediatek,mt8127-uart" for MT8127 compatible UARTS * "mediatek,mt8173-uart" for MT8173 compatible UARTS + * "mediatek,mt6795-uart" for MT6795 compatible UARTS * "mediatek,mt6589-uart" for MT6589 compatible UARTS * "mediatek,mt6582-uart" for MT6582 compatible UARTS * "mediatek,mt6580-uart" for MT6580 compatible UARTS - * "mediatek,mt6577-uart" for all compatible UARTS (MT8173, MT6589, MT6582, - MT6580, MT6577) + * "mediatek,mt6577-uart" for all compatible UARTS (MT8173, MT6795, + MT6589, MT6582, MT6580, MT6577) - reg: The base address of the UART register bank. diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,smd.txt b/Documentation/devicetree/bindings/soc/qcom/qcom,smd.txt index f65c76db9859..97d9b3e1bf39 100644 --- a/Documentation/devicetree/bindings/soc/qcom/qcom,smd.txt +++ b/Documentation/devicetree/bindings/soc/qcom/qcom,smd.txt @@ -37,6 +37,12 @@ The edge is described by the following properties: Definition: the identifier of the remote processor in the smd channel allocation table +- qcom,remote-pid: + Usage: optional + Value type: + Definition: the identifier for the remote processor as known by the rest + of the system. + = SMD DEVICES In turn, subnodes of the "edges" represent devices tied to SMD channels on that diff --git a/Documentation/devicetree/bindings/sound/cs4349.txt b/Documentation/devicetree/bindings/sound/cs4349.txt new file mode 100644 index 000000000000..54c117b59dba --- /dev/null +++ b/Documentation/devicetree/bindings/sound/cs4349.txt @@ -0,0 +1,19 @@ +CS4349 audio CODEC + +Required properties: + + - compatible : "cirrus,cs4349" + + - reg : the I2C address of the device for I2C + +Optional properties: + + - reset-gpios : a GPIO spec for the reset pin. + +Example: + +codec: cs4349@48 { + compatible = "cirrus,cs4349"; + reg = <0x48>; + reset-gpios = <&gpio 54 0>; +}; diff --git a/Documentation/devicetree/bindings/sound/ics43432.txt b/Documentation/devicetree/bindings/sound/ics43432.txt new file mode 100644 index 000000000000..b02e3a6c0fef --- /dev/null +++ b/Documentation/devicetree/bindings/sound/ics43432.txt @@ -0,0 +1,17 @@ +Invensense ICS-43432 MEMS microphone with I2S output. + +There are no software configuration options for this device, indeed, the only +host connection is the I2S interface. Apart from requirements on clock +frequency (460 kHz to 3.379 MHz according to the data sheet) there must be +64 clock cycles in each stereo output frame; 24 of the 32 available bits +contain audio data. A hardware pin determines if the device outputs data +on the left or right channel of the I2S frame. + +Required properties: + - compatible : Must be "invensense,ics43432" + +Example: + + ics43432: ics43432 { + compatible = "invensense,ics43432"; + }; diff --git a/Documentation/devicetree/bindings/sound/max98357a.txt b/Documentation/devicetree/bindings/sound/max98357a.txt index a7a149a236e5..28645a2ff885 100644 --- a/Documentation/devicetree/bindings/sound/max98357a.txt +++ b/Documentation/devicetree/bindings/sound/max98357a.txt @@ -4,7 +4,11 @@ This node models the Maxim MAX98357A DAC. Required properties: - compatible : "maxim,max98357a" -- sdmode-gpios : GPIO specifier for the GPIO -> DAC SDMODE pin + +Optional properties: +- sdmode-gpios : GPIO specifier for the chip's SD_MODE pin. + If this option is not specified then driver does not manage + the pin state (e.g. chip is always on). Example: diff --git a/Documentation/devicetree/bindings/sound/renesas,rsnd.txt b/Documentation/devicetree/bindings/sound/renesas,rsnd.txt index b6b3a786855f..1173395b5e5c 100644 --- a/Documentation/devicetree/bindings/sound/renesas,rsnd.txt +++ b/Documentation/devicetree/bindings/sound/renesas,rsnd.txt @@ -18,6 +18,12 @@ Required properties: - rcar_sound,src : Should contain SRC feature. The number of SRC subnode should be same as HW. see below for detail. +- rcar_sound,ctu : Should contain CTU feature. + The number of CTU subnode should be same as HW. + see below for detail. +- rcar_sound,mix : Should contain MIX feature. + The number of MIX subnode should be same as HW. + see below for detail. - rcar_sound,dvc : Should contain DVC feature. The number of DVC subnode should be same as HW. see below for detail. @@ -90,6 +96,22 @@ rcar_sound: sound@ec500000 { }; }; + rcar_sound,mix { + mix0: mix@0 { }; + mix1: mix@1 { }; + }; + + rcar_sound,ctu { + ctu00: ctu@0 { }; + ctu01: ctu@1 { }; + ctu02: ctu@2 { }; + ctu03: ctu@3 { }; + ctu10: ctu@4 { }; + ctu11: ctu@5 { }; + ctu12: ctu@6 { }; + ctu13: ctu@7 { }; + }; + rcar_sound,src { src0: src@0 { interrupts = <0 352 IRQ_TYPE_LEVEL_HIGH>; diff --git a/Documentation/devicetree/bindings/sound/renesas,rsrc-card.txt b/Documentation/devicetree/bindings/sound/renesas,rsrc-card.txt index c64155027288..962748a8d919 100644 --- a/Documentation/devicetree/bindings/sound/renesas,rsrc-card.txt +++ b/Documentation/devicetree/bindings/sound/renesas,rsrc-card.txt @@ -6,6 +6,7 @@ Required properties: - compatible : "renesas,rsrc-card," Examples with soctypes are: + - "renesas,rsrc-card" - "renesas,rsrc-card,lager" - "renesas,rsrc-card,koelsch" Optional properties: @@ -29,6 +30,12 @@ Optional subnode properties: - frame-inversion : bool property. Add this if the dai-link uses frame clock inversion. - convert-rate : platform specified sampling rate convert +- audio-prefix : see audio-routing +- audio-routing : A list of the connections between audio components. + Each entry is a pair of strings, the first being the connection's sink, + the second being the connection's source. Valid names for sources. + use audio-prefix if some components is using same sink/sources naming. + it can be used if compatible was "renesas,rsrc-card"; Required CPU/CODEC subnodes properties: diff --git a/Documentation/devicetree/bindings/sound/rockchip-max98090.txt b/Documentation/devicetree/bindings/sound/rockchip-max98090.txt new file mode 100644 index 000000000000..a805aa99ad75 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/rockchip-max98090.txt @@ -0,0 +1,19 @@ +ROCKCHIP with MAX98090 CODEC + +Required properties: +- compatible: "rockchip,rockchip-audio-max98090" +- rockchip,model: The user-visible name of this sound complex +- rockchip,i2s-controller: The phandle of the Rockchip I2S controller that's + connected to the CODEC +- rockchip,audio-codec: The phandle of the MAX98090 audio codec +- rockchip,headset-codec: The phandle of Ext chip for jack detection + +Example: + +sound { + compatible = "rockchip,rockchip-audio-max98090"; + rockchip,model = "ROCKCHIP-I2S"; + rockchip,i2s-controller = <&i2s>; + rockchip,audio-codec = <&max98090>; + rockchip,headset-codec = <&headsetcodec>; +}; diff --git a/Documentation/devicetree/bindings/sound/rockchip-rt5645.txt b/Documentation/devicetree/bindings/sound/rockchip-rt5645.txt new file mode 100644 index 000000000000..411a62b3ff41 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/rockchip-rt5645.txt @@ -0,0 +1,17 @@ +ROCKCHIP with RT5645/RT5650 CODECS + +Required properties: +- compatible: "rockchip,rockchip-audio-rt5645" +- rockchip,model: The user-visible name of this sound complex +- rockchip,i2s-controller: The phandle of the Rockchip I2S controller that's + connected to the CODEC +- rockchip,audio-codec: The phandle of the RT5645/RT5650 audio codec + +Example: + +sound { + compatible = "rockchip,rockchip-audio-rt5645"; + rockchip,model = "ROCKCHIP-I2S"; + rockchip,i2s-controller = <&i2s>; + rockchip,audio-codec = <&rt5645>; +}; diff --git a/Documentation/devicetree/bindings/sound/st,sti-asoc-card.txt b/Documentation/devicetree/bindings/sound/st,sti-asoc-card.txt new file mode 100644 index 000000000000..028fa1c82f50 --- /dev/null +++ b/Documentation/devicetree/bindings/sound/st,sti-asoc-card.txt @@ -0,0 +1,155 @@ +STMicroelectronics sti ASoC cards + +The sti ASoC Sound Card can be used, for all sti SoCs using internal sti-sas +codec or external codecs. + +sti sound drivers allows to expose sti SoC audio interface through the +generic ASoC simple card. For details about sound card declaration please refer to +Documentation/devicetree/bindings/sound/simple-card.txt. + +1) sti-uniperiph-dai: audio dai device. +--------------------------------------- + +Required properties: + - compatible: "st,sti-uni-player" or "st,sti-uni-reader" + + - st,syscfg: phandle to boot-device system configuration registers + + - clock-names: name of the clocks listed in clocks property in the same order + + - reg: CPU DAI IP Base address and size entries, listed in same + order than the CPU_DAI properties. + + - reg-names: names of the mapped memory regions listed in regs property in + the same order. + + - interrupts: CPU_DAI interrupt line, listed in the same order than the + CPU_DAI properties. + + - dma: CPU_DAI DMA controller phandle and DMA request line, listed in the same + order than the CPU_DAI properties. + + - dma-names: identifier string for each DMA request line in the dmas property. + "tx" for "st,sti-uni-player" compatibility + "rx" for "st,sti-uni-reader" compatibility + + - version: IP version integrated in SOC. + + - dai-name: DAI name that describes the IP. + +Required properties ("st,sti-uni-player" compatibility only): + - clocks: CPU_DAI IP clock source, listed in the same order than the + CPU_DAI properties. + + - uniperiph-id: internal SOC IP instance ID. + + - IP mode: IP working mode depending on associated codec. + "HDMI" connected to HDMI codec IP and IEC HDMI formats. + "SPDIF"connected to SPDIF codec and support SPDIF formats. + "PCM" PCM standard mode for I2S or TDM bus. + +Optional properties: + - pinctrl-0: defined for CPU_DAI@1 and CPU_DAI@4 to describe I2S PIOs for + external codecs connection. + + - pinctrl-names: should contain only one value - "default". + +Example: + + sti_uni_player2: sti-uni-player@2 { + compatible = "st,sti-uni-player"; + status = "okay"; + #sound-dai-cells = <0>; + st,syscfg = <&syscfg_core>; + clocks = <&clk_s_d0_flexgen CLK_PCM_2>; + reg = <0x8D82000 0x158>; + interrupts = ; + dmas = <&fdma0 4 0 1>; + dai-name = "Uni Player #1 (DAC)"; + dma-names = "tx"; + uniperiph-id = <2>; + version = <5>; + mode = "PCM"; + }; + + sti_uni_player3: sti-uni-player@3 { + compatible = "st,sti-uni-player"; + status = "okay"; + #sound-dai-cells = <0>; + st,syscfg = <&syscfg_core>; + clocks = <&clk_s_d0_flexgen CLK_SPDIFF>; + reg = <0x8D85000 0x158>; + interrupts = ; + dmas = <&fdma0 7 0 1>; + dma-names = "tx"; + dai-name = "Uni Player #1 (PIO)"; + uniperiph-id = <3>; + version = <5>; + mode = "SPDIF"; + }; + + sti_uni_reader1: sti-uni-reader@1 { + compatible = "st,sti-uni-reader"; + status = "disabled"; + #sound-dai-cells = <0>; + st,syscfg = <&syscfg_core>; + reg = <0x8D84000 0x158>; + interrupts = ; + dmas = <&fdma0 6 0 1>; + dma-names = "rx"; + dai-name = "Uni Reader #1 (HDMI RX)"; + version = <3>; + }; + +2) sti-sas-codec: internal audio codec IPs driver +------------------------------------------------- + +Required properties: + - compatible: "st,sti-sas-codec" . + Should be chip "st,stih416-sas-codec" or "st,stih407-sas-codec" + + - st,syscfg: phandle to boot-device system configuration registers. + + - pinctrl-0: SPDIF PIO description. + + - pinctrl-names: should contain only one value - "default". + +Example: + sti_sas_codec: sti-sas-codec { + compatible = "st,stih407-sas-codec"; + #sound-dai-cells = <1>; + st,reg_audio = <&syscfg_core>; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_spdif_out >; + }; + +Example of audio card declaration: + sound { + compatible = "simple-audio-card"; + simple-audio-card,name = "sti audio card"; + status = "okay"; + + simple-audio-card,dai-link@0 { + /* DAC */ + format = "i2s"; + dai-tdm-slot-width = <32>; + cpu { + sound-dai = <&sti_uni_player2>; + }; + + codec { + sound-dai = <&sti_sasg_codec 1>; + }; + }; + simple-audio-card,dai-link@1 { + /* SPDIF */ + format = "left_j"; + cpu { + sound-dai = <&sti_uni_player3>; + }; + + codec { + sound-dai = <&sti_sasg_codec 0>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/timer/img,pistachio-gptimer.txt b/Documentation/devicetree/bindings/timer/img,pistachio-gptimer.txt new file mode 100644 index 000000000000..7afce80bf6a0 --- /dev/null +++ b/Documentation/devicetree/bindings/timer/img,pistachio-gptimer.txt @@ -0,0 +1,28 @@ +* Pistachio general-purpose timer based clocksource + +Required properties: + - compatible: "img,pistachio-gptimer". + - reg: Address range of the timer registers. + - interrupts: An interrupt for each of the four timers + - clocks: Should contain a clock specifier for each entry in clock-names + - clock-names: Should contain the following entries: + "sys", interface clock + "slow", slow counter clock + "fast", fast counter clock + - img,cr-periph: Must contain a phandle to the peripheral control + syscon node. + +Example: + timer: timer@18102000 { + compatible = "img,pistachio-gptimer"; + reg = <0x18102000 0x100>; + interrupts = , + , + , + ; + clocks = <&clk_periph PERIPH_CLK_COUNTER_FAST>, + <&clk_periph PERIPH_CLK_COUNTER_SLOW>, + <&cr_periph SYS_CLK_TIMER>; + clock-names = "fast", "slow", "sys"; + img,cr-periph = <&cr_periph>; + }; diff --git a/Documentation/devicetree/bindings/timer/st,stih407-lpc b/Documentation/devicetree/bindings/timer/st,stih407-lpc new file mode 100644 index 000000000000..72acb487b856 --- /dev/null +++ b/Documentation/devicetree/bindings/timer/st,stih407-lpc @@ -0,0 +1,28 @@ +STMicroelectronics Low Power Controller (LPC) - Clocksource +=========================================================== + +LPC currently supports Watchdog OR Real Time Clock OR Clocksource +functionality. + +[See: ../watchdog/st_lpc_wdt.txt for Watchdog options] +[See: ../rtc/rtc-st-lpc.txt for RTC options] + +Required properties + +- compatible : Must be: "st,stih407-lpc" +- reg : LPC registers base address + size +- interrupts : LPC interrupt line number and associated flags +- clocks : Clock used by LPC device (See: ../clock/clock-bindings.txt) +- st,lpc-mode : The LPC can run either one of three modes: + ST_LPC_MODE_RTC [0] + ST_LPC_MODE_WDT [1] + ST_LPC_MODE_CLKSRC [2] + One (and only one) mode must be selected. + +Example: + lpc@fde05000 { + compatible = "st,stih407-lpc"; + reg = <0xfde05000 0x1000>; + clocks = <&clk_s_d3_flexgen CLK_LPC_0>; + st,lpc-mode = ; + }; diff --git a/Documentation/devicetree/bindings/vendor-prefixes.txt b/Documentation/devicetree/bindings/vendor-prefixes.txt index d3fbd1f30eb4..ac5f0c34ae00 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.txt +++ b/Documentation/devicetree/bindings/vendor-prefixes.txt @@ -55,6 +55,7 @@ cortina Cortina Systems, Inc. cosmic Cosmic Circuits crystalfontz Crystalfontz America, Inc. cubietech Cubietech, Ltd. +cypress Cypress Semiconductor Corporation dallas Maxim Integrated Products (formerly Dallas Semiconductor) davicom DAVICOM Semiconductor, Inc. delta Delta Electronics, Inc. @@ -111,8 +112,10 @@ ingenic Ingenic Semiconductor innolux Innolux Corporation intel Intel Corporation intercontrol Inter Control Group +invensense InvenSense Inc. isee ISEE 2007 S.L. isil Intersil +jedec JEDEC Solid State Technology Association karo Ka-Ro electronics GmbH keymile Keymile GmbH kinetic Kinetic Technologies @@ -142,6 +145,7 @@ mundoreader Mundo Reader S.L. murata Murata Manufacturing Co., Ltd. mxicy Macronix International Co., Ltd. national National Semiconductor +nec NEC LCD Technologies, Ltd. neonode Neonode Inc. netgear NETGEAR netlogic Broadcom Corporation (formerly NetLogic Microsystems) @@ -149,10 +153,13 @@ netxeon Shenzhen Netxeon Technology CO., LTD newhaven Newhaven Display International nintendo Nintendo nokia Nokia +nuvoton Nuvoton Technology Corporation nvidia NVIDIA nxp NXP Semiconductors +okaya Okaya Electric America, Inc. onnn ON Semiconductor Corp. opencores OpenCores.org +option Option NV ortustech Ortus Technology Co., Ltd. ovti OmniVision Technologies panasonic Panasonic Corporation @@ -184,6 +191,7 @@ sbs Smart Battery System schindler Schindler seagate Seagate Technology PLC semtech Semtech Corporation +sharp Sharp Corporation sil Silicon Image silabs Silicon Laboratories siliconmitus Silicon Mitus, Inc. diff --git a/Documentation/devicetree/bindings/leds/leds-pm8941-wled.txt b/Documentation/devicetree/bindings/video/backlight/pm8941-wled.txt similarity index 84% rename from Documentation/devicetree/bindings/leds/leds-pm8941-wled.txt rename to Documentation/devicetree/bindings/video/backlight/pm8941-wled.txt index a85a964d61f5..424f8444a6cd 100644 --- a/Documentation/devicetree/bindings/leds/leds-pm8941-wled.txt +++ b/Documentation/devicetree/bindings/video/backlight/pm8941-wled.txt @@ -5,10 +5,7 @@ Required properties: - reg: slave address Optional properties: -- label: The label for this led - See Documentation/devicetree/bindings/leds/common.txt -- linux,default-trigger: Default trigger assigned to the LED - See Documentation/devicetree/bindings/leds/common.txt +- label: The name of the backlight device - qcom,cs-out: bool; enable current sink output - qcom,cabc: bool; enable content adaptive backlight control - qcom,ext-gen: bool; use externally generated modulator signal to dim diff --git a/Documentation/devicetree/bindings/video/fsl,dcu.txt b/Documentation/devicetree/bindings/video/fsl,dcu.txt new file mode 100644 index 000000000000..ebf1be9ae393 --- /dev/null +++ b/Documentation/devicetree/bindings/video/fsl,dcu.txt @@ -0,0 +1,22 @@ +Device Tree bindings for Freescale DCU DRM Driver + +Required properties: +- compatible: Should be one of + * "fsl,ls1021a-dcu". + * "fsl,vf610-dcu". + +- reg: Address and length of the register set for dcu. +- clocks: From common clock binding: handle to dcu clock. +- clock-names: From common clock binding: Shall be "dcu". +- big-endian Boolean property, LS1021A DCU registers are big-endian. +- fsl,panel: The phandle to panel node. + +Examples: +dcu: dcu@2ce0000 { + compatible = "fsl,ls1021a-dcu"; + reg = <0x0 0x2ce0000 0x0 0x10000>; + clocks = <&platform_clk 0>; + clock-names = "dcu"; + big-endian; + fsl,panel = <&panel>; +}; diff --git a/Documentation/devicetree/bindings/watchdog/atmel-sama5d4-wdt.txt b/Documentation/devicetree/bindings/watchdog/atmel-sama5d4-wdt.txt new file mode 100644 index 000000000000..f7cc7c060910 --- /dev/null +++ b/Documentation/devicetree/bindings/watchdog/atmel-sama5d4-wdt.txt @@ -0,0 +1,35 @@ +* Atmel SAMA5D4 Watchdog Timer (WDT) Controller + +Required properties: +- compatible: "atmel,sama5d4-wdt" +- reg: base physical address and length of memory mapped region. + +Optional properties: +- timeout-sec: watchdog timeout value (in seconds). +- interrupts: interrupt number to the CPU. +- atmel,watchdog-type: should be "hardware" or "software". + "hardware": enable watchdog fault reset. A watchdog fault triggers + watchdog reset. + "software": enable watchdog fault interrupt. A watchdog fault asserts + watchdog interrupt. +- atmel,idle-halt: present if you want to stop the watchdog when the CPU is + in idle state. + CAUTION: This property should be used with care, it actually makes the + watchdog not counting when the CPU is in idle state, therefore the + watchdog reset time depends on mean CPU usage and will not reset at all + if the CPU stop working while it is in idle state, which is probably + not what you want. +- atmel,dbg-halt: present if you want to stop the watchdog when the CPU is + in debug state. + +Example: + watchdog@fc068640 { + compatible = "atmel,sama5d4-wdt"; + reg = <0xfc068640 0x10>; + interrupts = <4 IRQ_TYPE_LEVEL_HIGH 5>; + timeout-sec = <10>; + atmel,watchdog-type = "hardware"; + atmel,dbg-halt; + atmel,idle-halt; + status = "okay"; + }; diff --git a/Documentation/devicetree/bindings/watchdog/lpc18xx-wdt.txt b/Documentation/devicetree/bindings/watchdog/lpc18xx-wdt.txt new file mode 100644 index 000000000000..09f6b24969e0 --- /dev/null +++ b/Documentation/devicetree/bindings/watchdog/lpc18xx-wdt.txt @@ -0,0 +1,19 @@ +* NXP LPC18xx Watchdog Timer (WDT) + +Required properties: +- compatible: Should be "nxp,lpc1850-wwdt" +- reg: Should contain WDT registers location and length +- clocks: Must contain an entry for each entry in clock-names. +- clock-names: Should contain "wdtclk" and "reg"; the watchdog counter + clock and register interface clock respectively. +- interrupts: Should contain WDT interrupt + +Examples: + +watchdog@40080000 { + compatible = "nxp,lpc1850-wwdt"; + reg = <0x40080000 0x24>; + clocks = <&cgu BASE_SAFE_CLK>, <&ccu1 CLK_CPU_WWDT>; + clock-names = "wdtclk", "reg"; + interrupts = <49>; +}; diff --git a/Documentation/devicetree/bindings/watchdog/st_lpc_wdt.txt b/Documentation/devicetree/bindings/watchdog/st_lpc_wdt.txt index 388c88a01222..039c5ca45577 100644 --- a/Documentation/devicetree/bindings/watchdog/st_lpc_wdt.txt +++ b/Documentation/devicetree/bindings/watchdog/st_lpc_wdt.txt @@ -1,9 +1,11 @@ STMicroelectronics Low Power Controller (LPC) - Watchdog ======================================================== -LPC currently supports Watchdog OR Real Time Clock functionality. +LPC currently supports Watchdog OR Real Time Clock OR Clocksource +functionality. [See: ../rtc/rtc-st-lpc.txt for RTC options] +[See: ../timer/st,stih407-lpc for Clocksource options] Required properties @@ -12,9 +14,11 @@ Required properties - reg : LPC registers base address + size - interrupts : LPC interrupt line number and associated flags - clocks : Clock used by LPC device (See: ../clock/clock-bindings.txt) -- st,lpc-mode : The LPC can run either one of two modes ST_LPC_MODE_RTC [0] or - ST_LPC_MODE_WDT [1]. One (and only one) mode must be - selected. +- st,lpc-mode : The LPC can run either one of three modes: + ST_LPC_MODE_RTC [0] + ST_LPC_MODE_WDT [1] + ST_LPC_MODE_CLKSRC [2] + One (and only one) mode must be selected. Required properties [watchdog mode] diff --git a/Documentation/dmaengine/provider.txt b/Documentation/dmaengine/provider.txt index ca67b0f04c6e..67d4ce4df109 100644 --- a/Documentation/dmaengine/provider.txt +++ b/Documentation/dmaengine/provider.txt @@ -345,12 +345,29 @@ where to put them) that abstracts it away. * DMA_CTRL_ACK - - If set, the transfer can be reused after being completed. - - There is a guarantee the transfer won't be freed until it is acked - by async_tx_ack(). + - If clear, the descriptor cannot be reused by provider until the + client acknowledges receipt, i.e. has has a chance to establish any + dependency chains + - This can be acked by invoking async_tx_ack() + - If set, does not mean descriptor can be reused + + * DMA_CTRL_REUSE + - If set, the descriptor can be reused after being completed. It should + not be freed by provider if this flag is set. + - The descriptor should be prepared for reuse by invoking + dmaengine_desc_set_reuse() which will set DMA_CTRL_REUSE. + - dmaengine_desc_set_reuse() will succeed only when channel support + reusable descriptor as exhibited by capablities - As a consequence, if a device driver wants to skip the dma_map_sg() and dma_unmap_sg() in between 2 transfers, because the DMA'd data wasn't used, it can resubmit the transfer right after its completion. + - Descriptor can be freed in few ways + - Clearing DMA_CTRL_REUSE by invoking dmaengine_desc_clear_reuse() + and submitting for last txn + - Explicitly invoking dmaengine_desc_free(), this can succeed only + when DMA_CTRL_REUSE is already set + - Terminating the channel + General Design Notes -------------------- diff --git a/Documentation/fault-injection/fault-injection.txt b/Documentation/fault-injection/fault-injection.txt index 4cf1a2a6bd72..415484f3d59a 100644 --- a/Documentation/fault-injection/fault-injection.txt +++ b/Documentation/fault-injection/fault-injection.txt @@ -15,6 +15,10 @@ o fail_page_alloc injects page allocation failures. (alloc_pages(), get_free_pages(), ...) +o fail_futex + + injects futex deadlock and uaddr fault errors. + o fail_make_request injects disk IO errors on devices permitted by setting @@ -113,6 +117,12 @@ configuration of fault-injection capabilities. specifies the minimum page allocation order to be injected failures. +- /sys/kernel/debug/fail_futex/ignore-private: + + Format: { 'Y' | 'N' } + default is 'N', setting it to 'Y' will disable failure injections + when dealing with private (address space) futexes. + o Boot option In order to inject faults while debugfs is not available (early boot time), @@ -121,6 +131,7 @@ use the boot option: failslab= fail_page_alloc= fail_make_request= + fail_futex= mmc_core.fail_request=,,, How to add new fault injection capability diff --git a/Documentation/features/debug/uprobes/arch-support.txt b/Documentation/features/debug/uprobes/arch-support.txt index 4efe36c3ace9..d605c3fc38fd 100644 --- a/Documentation/features/debug/uprobes/arch-support.txt +++ b/Documentation/features/debug/uprobes/arch-support.txt @@ -22,7 +22,7 @@ | m68k: | TODO | | metag: | TODO | | microblaze: | TODO | - | mips: | TODO | + | mips: | ok | | mn10300: | TODO | | nios2: | TODO | | openrisc: | TODO | diff --git a/Documentation/features/seccomp/seccomp-filter/arch-support.txt b/Documentation/features/seccomp/seccomp-filter/arch-support.txt index bea800910342..76d39d66a5d7 100644 --- a/Documentation/features/seccomp/seccomp-filter/arch-support.txt +++ b/Documentation/features/seccomp/seccomp-filter/arch-support.txt @@ -32,7 +32,7 @@ | score: | TODO | | sh: | TODO | | sparc: | TODO | - | tile: | TODO | + | tile: | ok | | um: | TODO | | unicore32: | TODO | | x86: | ok | diff --git a/Documentation/features/vm/THP/arch-support.txt b/Documentation/features/vm/THP/arch-support.txt index 972d02c2a74c..df384e3e845f 100644 --- a/Documentation/features/vm/THP/arch-support.txt +++ b/Documentation/features/vm/THP/arch-support.txt @@ -20,7 +20,7 @@ | ia64: | TODO | | m32r: | .. | | m68k: | .. | - | metag: | .. | + | metag: | TODO | | microblaze: | .. | | mips: | ok | | mn10300: | .. | diff --git a/Documentation/features/vm/TLB/arch-support.txt b/Documentation/features/vm/TLB/arch-support.txt new file mode 100644 index 000000000000..261b92e2fb1a --- /dev/null +++ b/Documentation/features/vm/TLB/arch-support.txt @@ -0,0 +1,40 @@ +# +# Feature name: batch-unmap-tlb-flush +# Kconfig: ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH +# description: arch supports deferral of TLB flush until multiple pages are unmapped +# + ----------------------- + | arch |status| + ----------------------- + | alpha: | TODO | + | arc: | TODO | + | arm: | TODO | + | arm64: | TODO | + | avr32: | .. | + | blackfin: | TODO | + | c6x: | .. | + | cris: | .. | + | frv: | .. | + | h8300: | .. | + | hexagon: | TODO | + | ia64: | TODO | + | m32r: | TODO | + | m68k: | .. | + | metag: | TODO | + | microblaze: | .. | + | mips: | TODO | + | mn10300: | TODO | + | nios2: | .. | + | openrisc: | .. | + | parisc: | TODO | + | powerpc: | TODO | + | s390: | TODO | + | score: | .. | + | sh: | TODO | + | sparc: | TODO | + | tile: | TODO | + | um: | .. | + | unicore32: | .. | + | x86: | ok | + | xtensa: | TODO | + ----------------------- diff --git a/Documentation/filesystems/Locking b/Documentation/filesystems/Locking index 6a34a0f4d37c..06d443450f21 100644 --- a/Documentation/filesystems/Locking +++ b/Documentation/filesystems/Locking @@ -397,7 +397,8 @@ prototypes: int (*release) (struct gendisk *, fmode_t); int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); - int (*direct_access) (struct block_device *, sector_t, void **, unsigned long *); + int (*direct_access) (struct block_device *, sector_t, void __pmem **, + unsigned long *); int (*media_changed) (struct gendisk *); void (*unlock_native_capacity) (struct gendisk *); int (*revalidate_disk) (struct gendisk *); diff --git a/Documentation/filesystems/dax.txt b/Documentation/filesystems/dax.txt index 7af2851d667c..7bde64014a89 100644 --- a/Documentation/filesystems/dax.txt +++ b/Documentation/filesystems/dax.txt @@ -60,9 +60,10 @@ Filesystem support consists of - implementing the direct_IO address space operation, and calling dax_do_io() instead of blockdev_direct_IO() if S_DAX is set - implementing an mmap file operation for DAX files which sets the - VM_MIXEDMAP flag on the VMA, and setting the vm_ops to include handlers - for fault and page_mkwrite (which should probably call dax_fault() and - dax_mkwrite(), passing the appropriate get_block() callback) + VM_MIXEDMAP and VM_HUGEPAGE flags on the VMA, and setting the vm_ops to + include handlers for fault, pmd_fault and page_mkwrite (which should + probably call dax_fault(), dax_pmd_fault() and dax_mkwrite(), passing the + appropriate get_block() callback) - calling dax_truncate_page() instead of block_truncate_page() for DAX files - calling dax_zero_page_range() instead of zero_user() for DAX files - ensuring that there is sufficient locking between reads, writes, diff --git a/Documentation/filesystems/ext2.txt b/Documentation/filesystems/ext2.txt index b9714569e472..55755395d3dc 100644 --- a/Documentation/filesystems/ext2.txt +++ b/Documentation/filesystems/ext2.txt @@ -360,8 +360,8 @@ and are copied into the filesystem. If a transaction is incomplete at the time of the crash, then there is no guarantee of consistency for the blocks in that transaction so they are discarded (which means any filesystem changes they represent are also lost). -Check Documentation/filesystems/ext3.txt if you want to read more about -ext3 and journaling. +Check Documentation/filesystems/ext4.txt if you want to read more about +ext4 and journaling. References ========== diff --git a/Documentation/filesystems/ext3.txt b/Documentation/filesystems/ext3.txt index 7ed0d17d6721..58758fbef9e0 100644 --- a/Documentation/filesystems/ext3.txt +++ b/Documentation/filesystems/ext3.txt @@ -6,210 +6,7 @@ Ext3 was originally released in September 1999. Written by Stephen Tweedie for the 2.2 branch, and ported to 2.4 kernels by Peter Braam, Andreas Dilger, Andrew Morton, Alexander Viro, Ted Ts'o and Stephen Tweedie. -Ext3 is the ext2 filesystem enhanced with journalling capabilities. +Ext3 is the ext2 filesystem enhanced with journalling capabilities. The +filesystem is a subset of ext4 filesystem so use ext4 driver for accessing +ext3 filesystems. -Options -======= - -When mounting an ext3 filesystem, the following option are accepted: -(*) == default - -ro Mount filesystem read only. Note that ext3 will replay - the journal (and thus write to the partition) even when - mounted "read only". Mount options "ro,noload" can be - used to prevent writes to the filesystem. - -journal=update Update the ext3 file system's journal to the current - format. - -journal=inum When a journal already exists, this option is ignored. - Otherwise, it specifies the number of the inode which - will represent the ext3 file system's journal file. - -journal_path=path -journal_dev=devnum When the external journal device's major/minor numbers - have changed, these options allow the user to specify - the new journal location. The journal device is - identified through either its new major/minor numbers - encoded in devnum, or via a path to the device. - -norecovery Don't load the journal on mounting. Note that this forces -noload mount of inconsistent filesystem, which can lead to - various problems. - -data=journal All data are committed into the journal prior to being - written into the main file system. - -data=ordered (*) All data are forced directly out to the main file - system prior to its metadata being committed to the - journal. - -data=writeback Data ordering is not preserved, data may be written - into the main file system after its metadata has been - committed to the journal. - -commit=nrsec (*) Ext3 can be told to sync all its data and metadata - every 'nrsec' seconds. The default value is 5 seconds. - This means that if you lose your power, you will lose - as much as the latest 5 seconds of work (your - filesystem will not be damaged though, thanks to the - journaling). This default value (or any low value) - will hurt performance, but it's good for data-safety. - Setting it to 0 will have the same effect as leaving - it at the default (5 seconds). - Setting it to very large values will improve - performance. - -barrier=<0|1(*)> This enables/disables the use of write barriers in -barrier (*) the jbd code. barrier=0 disables, barrier=1 enables. -nobarrier This also requires an IO stack which can support - barriers, and if jbd gets an error on a barrier - write, it will disable again with a warning. - Write barriers enforce proper on-disk ordering - of journal commits, making volatile disk write caches - safe to use, at some performance penalty. If - your disks are battery-backed in one way or another, - disabling barriers may safely improve performance. - The mount options "barrier" and "nobarrier" can - also be used to enable or disable barriers, for - consistency with other ext3 mount options. - -user_xattr Enables Extended User Attributes. Additionally, you - need to have extended attribute support enabled in the - kernel configuration (CONFIG_EXT3_FS_XATTR). See the - attr(5) manual page and http://acl.bestbits.at/ to - learn more about extended attributes. - -nouser_xattr Disables Extended User Attributes. - -acl Enables POSIX Access Control Lists support. - Additionally, you need to have ACL support enabled in - the kernel configuration (CONFIG_EXT3_FS_POSIX_ACL). - See the acl(5) manual page and http://acl.bestbits.at/ - for more information. - -noacl This option disables POSIX Access Control List - support. - -reservation - -noreservation - -bsddf (*) Make 'df' act like BSD. -minixdf Make 'df' act like Minix. - -check=none Don't do extra checking of bitmaps on mount. -nocheck - -debug Extra debugging information is sent to syslog. - -errors=remount-ro Remount the filesystem read-only on an error. -errors=continue Keep going on a filesystem error. -errors=panic Panic and halt the machine if an error occurs. - (These mount options override the errors behavior - specified in the superblock, which can be - configured using tune2fs.) - -data_err=ignore(*) Just print an error message if an error occurs - in a file data buffer in ordered mode. -data_err=abort Abort the journal if an error occurs in a file - data buffer in ordered mode. - -grpid Give objects the same group ID as their creator. -bsdgroups - -nogrpid (*) New objects have the group ID of their creator. -sysvgroups - -resgid=n The group ID which may use the reserved blocks. - -resuid=n The user ID which may use the reserved blocks. - -sb=n Use alternate superblock at this location. - -quota These options are ignored by the filesystem. They -noquota are used only by quota tools to recognize volumes -grpquota where quota should be turned on. See documentation -usrquota in the quota-tools package for more details - (http://sourceforge.net/projects/linuxquota). - -jqfmt= These options tell filesystem details about quota -usrjquota= so that quota information can be properly updated -grpjquota= during journal replay. They replace the above - quota options. See documentation in the quota-tools - package for more details - (http://sourceforge.net/projects/linuxquota). - -Specification -============= -Ext3 shares all disk implementation with the ext2 filesystem, and adds -transactions capabilities to ext2. Journaling is done by the Journaling Block -Device layer. - -Journaling Block Device layer ------------------------------ -The Journaling Block Device layer (JBD) isn't ext3 specific. It was designed -to add journaling capabilities to a block device. The ext3 filesystem code -will inform the JBD of modifications it is performing (called a transaction). -The journal supports the transactions start and stop, and in case of a crash, -the journal can replay the transactions to quickly put the partition back into -a consistent state. - -Handles represent a single atomic update to a filesystem. JBD can handle an -external journal on a block device. - -Data Mode ---------- -There are 3 different data modes: - -* writeback mode -In data=writeback mode, ext3 does not journal data at all. This mode provides -a similar level of journaling as that of XFS, JFS, and ReiserFS in its default -mode - metadata journaling. A crash+recovery can cause incorrect data to -appear in files which were written shortly before the crash. This mode will -typically provide the best ext3 performance. - -* ordered mode -In data=ordered mode, ext3 only officially journals metadata, but it logically -groups metadata and data blocks into a single unit called a transaction. When -it's time to write the new metadata out to disk, the associated data blocks -are written first. In general, this mode performs slightly slower than -writeback but significantly faster than journal mode. - -* journal mode -data=journal mode provides full data and metadata journaling. All new data is -written to the journal first, and then to its final location. -In the event of a crash, the journal can be replayed, bringing both data and -metadata into a consistent state. This mode is the slowest except when data -needs to be read from and written to disk at the same time where it -outperforms all other modes. - -Compatibility -------------- - -Ext2 partitions can be easily convert to ext3, with `tune2fs -j `. -Ext3 is fully compatible with Ext2. Ext3 partitions can easily be mounted as -Ext2. - - -External Tools -============== -See manual pages to learn more. - -tune2fs: create a ext3 journal on a ext2 partition with the -j flag. -mke2fs: create a ext3 partition with the -j flag. -debugfs: ext2 and ext3 file system debugger. -ext2online: online (mounted) ext2 and ext3 filesystem resizer - - -References -========== - -kernel source: - - -programs: http://e2fsprogs.sourceforge.net/ - http://ext2resize.sourceforge.net - -useful links: http://www.ibm.com/developerworks/library/l-fs7/index.html - http://www.ibm.com/developerworks/library/l-fs8/index.html diff --git a/Documentation/filesystems/f2fs.txt b/Documentation/filesystems/f2fs.txt index e9e750e59efc..e2d5105b7214 100644 --- a/Documentation/filesystems/f2fs.txt +++ b/Documentation/filesystems/f2fs.txt @@ -143,7 +143,9 @@ fastboot This option is used when a system wants to reduce mount extent_cache Enable an extent cache based on rb-tree, it can cache as many as extent which map between contiguous logical address and physical address per inode, resulting in - increasing the cache hit ratio. + increasing the cache hit ratio. Set by default. +noextent_cache Diable an extent cache based on rb-tree explicitly, see + the above extent_cache mount option. noinline_data Disable the inline data feature, inline data feature is enabled by default. diff --git a/Documentation/filesystems/nfs/nfs-rdma.txt b/Documentation/filesystems/nfs/nfs-rdma.txt index 95c13aa575ff..906b6c233f62 100644 --- a/Documentation/filesystems/nfs/nfs-rdma.txt +++ b/Documentation/filesystems/nfs/nfs-rdma.txt @@ -138,9 +138,9 @@ Installation - Build, install, reboot The NFS/RDMA code will be enabled automatically if NFS and RDMA - are turned on. The NFS/RDMA client and server are configured via the - SUNRPC_XPRT_RDMA_CLIENT and SUNRPC_XPRT_RDMA_SERVER config options that both - depend on SUNRPC and INFINIBAND. The default value of both options will be: + are turned on. The NFS/RDMA client and server are configured via the hidden + SUNRPC_XPRT_RDMA config option that depends on SUNRPC and INFINIBAND. The + value of SUNRPC_XPRT_RDMA will be: - N if either SUNRPC or INFINIBAND are N, in this case the NFS/RDMA client and server will not be built @@ -238,9 +238,8 @@ NFS/RDMA Setup - Start the NFS server - If the NFS/RDMA server was built as a module - (CONFIG_SUNRPC_XPRT_RDMA_SERVER=m in kernel config), load the RDMA - transport module: + If the NFS/RDMA server was built as a module (CONFIG_SUNRPC_XPRT_RDMA=m in + kernel config), load the RDMA transport module: $ modprobe svcrdma @@ -259,9 +258,8 @@ NFS/RDMA Setup - On the client system - If the NFS/RDMA client was built as a module - (CONFIG_SUNRPC_XPRT_RDMA_CLIENT=m in kernel config), load the RDMA client - module: + If the NFS/RDMA client was built as a module (CONFIG_SUNRPC_XPRT_RDMA=m in + kernel config), load the RDMA client module: $ modprobe xprtrdma.ko diff --git a/Documentation/filesystems/proc.txt b/Documentation/filesystems/proc.txt index 6f7fafde0884..d411ca63c8b6 100644 --- a/Documentation/filesystems/proc.txt +++ b/Documentation/filesystems/proc.txt @@ -424,6 +424,7 @@ Private_Dirty: 0 kB Referenced: 892 kB Anonymous: 0 kB Swap: 0 kB +SwapPss: 0 kB KernelPageSize: 4 kB MMUPageSize: 4 kB Locked: 374 kB @@ -433,16 +434,23 @@ the first of these lines shows the same information as is displayed for the mapping in /proc/PID/maps. The remaining lines show the size of the mapping (size), the amount of the mapping that is currently resident in RAM (RSS), the process' proportional share of this mapping (PSS), the number of clean and -dirty private pages in the mapping. Note that even a page which is part of a -MAP_SHARED mapping, but has only a single pte mapped, i.e. is currently used -by only one process, is accounted as private and not as shared. "Referenced" -indicates the amount of memory currently marked as referenced or accessed. +dirty private pages in the mapping. + +The "proportional set size" (PSS) of a process is the count of pages it has +in memory, where each page is divided by the number of processes sharing it. +So if a process has 1000 pages all to itself, and 1000 shared with one other +process, its PSS will be 1500. +Note that even a page which is part of a MAP_SHARED mapping, but has only +a single pte mapped, i.e. is currently used by only one process, is accounted +as private and not as shared. +"Referenced" indicates the amount of memory currently marked as referenced or +accessed. "Anonymous" shows the amount of memory that does not belong to any file. Even a mapping associated with a file may contain anonymous pages: when MAP_PRIVATE and a page is modified, the file page is replaced by a private anonymous copy. "Swap" shows how much would-be-anonymous memory is also used, but out on swap. - +"SwapPss" shows proportional swap share of this mapping. "VmFlags" field deserves a separate description. This member represents the kernel flags associated with the particular virtual memory area in two letter encoded manner. The codes are the following: diff --git a/Documentation/filesystems/vfs.txt b/Documentation/filesystems/vfs.txt index 5eb8456fc41e..8c6f07ad373a 100644 --- a/Documentation/filesystems/vfs.txt +++ b/Documentation/filesystems/vfs.txt @@ -769,7 +769,7 @@ struct address_space_operations { to stall to allow flushers a chance to complete some IO. Ordinarily it can use PageDirty and PageWriteback but some filesystems have more complex state (unstable pages in NFS prevent reclaim) or - do not set those flags due to locking problems (jbd). This callback + do not set those flags due to locking problems. This callback allows a filesystem to indicate to the VM if a page should be treated as dirty or writeback for the purposes of stalling. diff --git a/Documentation/gpio/00-INDEX b/Documentation/gpio/00-INDEX index 1de43ae46ae6..179beb234f98 100644 --- a/Documentation/gpio/00-INDEX +++ b/Documentation/gpio/00-INDEX @@ -6,6 +6,9 @@ consumer.txt - How to obtain and use GPIOs in a driver driver.txt - How to write a GPIO driver +drivers-on-gpio.txt: + - Drivers in other subsystems that can use GPIO to provide more + complex functionality. board.txt - How to assign GPIOs to a consumer device and a function sysfs.txt diff --git a/Documentation/gpio/consumer.txt b/Documentation/gpio/consumer.txt index 75542b91b766..a206639454ab 100644 --- a/Documentation/gpio/consumer.txt +++ b/Documentation/gpio/consumer.txt @@ -237,6 +237,39 @@ Note that these functions should only be used with great moderation ; a driver should not have to care about the physical line level. +The active-low property +----------------------- + +As a driver should not have to care about the physical line level, all of the +gpiod_set_value_xxx() or gpiod_set_array_value_xxx() functions operate with +the *logical* value. With this they take the active-low property into account. +This means that they check whether the GPIO is configured to be active-low, +and if so, they manipulate the passed value before the physical line level is +driven. + +With this, all the gpiod_set_(array)_value_xxx() functions interpret the +parameter "value" as "active" ("1") or "inactive" ("0"). The physical line +level will be driven accordingly. + +As an example, if the active-low property for a dedicated GPIO is set, and the +gpiod_set_(array)_value_xxx() passes "active" ("1"), the physical line level +will be driven low. + +To summarize: + +Function (example) active-low proporty physical line +gpiod_set_raw_value(desc, 0); don't care low +gpiod_set_raw_value(desc, 1); don't care high +gpiod_set_value(desc, 0); default (active-high) low +gpiod_set_value(desc, 1); default (active-high) high +gpiod_set_value(desc, 0); active-low high +gpiod_set_value(desc, 1); active-low low + +Please note again that the set_raw/get_raw functions should be avoided as much +as possible, especially by drivers which should not care about the actual +physical line level and worry about the logical value instead. + + Set multiple GPIO outputs with a single function call ----------------------------------------------------- The following functions set the output values of an array of GPIOs: diff --git a/Documentation/gpio/drivers-on-gpio.txt b/Documentation/gpio/drivers-on-gpio.txt new file mode 100644 index 000000000000..f6121328630f --- /dev/null +++ b/Documentation/gpio/drivers-on-gpio.txt @@ -0,0 +1,95 @@ +Subsystem drivers using GPIO +============================ + +Note that standard kernel drivers exist for common GPIO tasks and will provide +the right in-kernel and userspace APIs/ABIs for the job, and that these +drivers can quite easily interconnect with other kernel subsystems using +hardware descriptions such as device tree or ACPI: + +- leds-gpio: drivers/leds/leds-gpio.c will handle LEDs connected to GPIO + lines, giving you the LED sysfs interface + +- ledtrig-gpio: drivers/leds/trigger/ledtrig-gpio.c will provide a LED trigger, + i.e. a LED will turn on/off in response to a GPIO line going high or low + (and that LED may in turn use the leds-gpio as per above). + +- gpio-keys: drivers/input/keyboard/gpio_keys.c is used when your GPIO line + can generate interrupts in response to a key press. Also supports debounce. + +- gpio-keys-polled: drivers/input/keyboard/gpio_keys_polled.c is used when your + GPIO line cannot generate interrupts, so it needs to be periodically polled + by a timer. + +- gpio_mouse: drivers/input/mouse/gpio_mouse.c is used to provide a mouse with + up to three buttons by simply using GPIOs and no mouse port. You can cut the + mouse cable and connect the wires to GPIO lines or solder a mouse connector + to the lines for a more permanent solution of this type. + +- gpio-beeper: drivers/input/misc/gpio-beeper.c is used to provide a beep from + an external speaker connected to a GPIO line. + +- gpio-tilt-polled: drivers/input/misc/gpio_tilt_polled.c provides tilt + detection switches using GPIO, which is useful for your homebrewn pinball + machine if for nothing else. It can detect different tilt angles of the + monitored object. + +- extcon-gpio: drivers/extcon/extcon-gpio.c is used when you need to read an + external connector status, such as a headset line for an audio driver or an + HDMI connector. It will provide a better userspace sysfs interface than GPIO. + +- restart-gpio: drivers/power/gpio-restart.c is used to restart/reboot the + system by pulling a GPIO line and will register a restart handler so + userspace can issue the right system call to restart the system. + +- poweroff-gpio: drivers/power/gpio-poweroff.c is used to power the system down + by pulling a GPIO line and will register a pm_power_off() callback so that + userspace can issue the right system call to power down the system. + +- gpio-gate-clock: drivers/clk/clk-gpio-gate.c is used to control a gated clock + (off/on) that uses a GPIO, and integrated with the clock subsystem. + +- i2c-gpio: drivers/i2c/busses/i2c-gpio.c is used to drive an I2C bus + (two wires, SDA and SCL lines) by hammering (bitbang) two GPIO lines. It will + appear as any other I2C bus to the system and makes it possible to connect + drivers for the I2C devices on the bus like any other I2C bus driver. + +- spi_gpio: drivers/spi/spi-gpio.c is used to drive an SPI bus (variable number + of wires, atleast SCK and optionally MISO, MOSI and chip select lines) using + GPIO hammering (bitbang). It will appear as any other SPI bus on the system + and makes it possible to connect drivers for SPI devices on the bus like + any other SPI bus driver. For example any MMC/SD card can then be connected + to this SPI by using the mmc_spi host from the MMC/SD card subsystem. + +- w1-gpio: drivers/w1/masters/w1-gpio.c is used to drive a one-wire bus using + a GPIO line, integrating with the W1 subsystem and handling devices on + the bus like any other W1 device. + +- gpio-fan: drivers/hwmon/gpio-fan.c is used to control a fan for cooling the + system, connected to a GPIO line (and optionally a GPIO alarm line), + presenting all the right in-kernel and sysfs interfaces to make your system + not overheat. + +- gpio-regulator: drivers/regulator/gpio-regulator.c is used to control a + regulator providing a certain voltage by pulling a GPIO line, integrating + with the regulator subsystem and giving you all the right interfaces. + +- gpio-wdt: drivers/watchdog/gpio_wdt.c is used to provide a watchdog timer + that will periodically "ping" a hardware connected to a GPIO line by toggling + it from 1-to-0-to-1. If that hardware does not recieve its "ping" + periodically, it will reset the system. + +- gpio-nand: drivers/mtd/nand/gpio.c is used to connect a NAND flash chip to + a set of simple GPIO lines: RDY, NCE, ALE, CLE, NWP. It interacts with the + NAND flash MTD subsystem and provides chip access and partition parsing like + any other NAND driving hardware. + +Apart from this there are special GPIO drivers in subsystems like MMC/SD to +read card detect and write protect GPIO lines, and in the TTY serial subsystem +to emulate MCTRL (modem control) signals CTS/RTS by using two GPIO lines. The +MTD NOR flash has add-ons for extra GPIO lines too, though the address bus is +usually connected directly to the flash. + +Use those instead of talking directly to the GPIOs using sysfs; they integrate +with kernel frameworks better than your userspace code could. Needless to say, +just using the apropriate kernel drivers will simplify and speed up your +embedded hacking in particular by providing ready-made components. diff --git a/Documentation/gpio/sysfs.txt b/Documentation/gpio/sysfs.txt index 535b6a8a7a7c..0700b55637f5 100644 --- a/Documentation/gpio/sysfs.txt +++ b/Documentation/gpio/sysfs.txt @@ -20,11 +20,10 @@ userspace GPIO can be used to determine system configuration data that standard kernels won't know about. And for some tasks, simple userspace GPIO drivers could be all that the system really needs. -Note that standard kernel drivers exist for common "LEDs and Buttons" -GPIO tasks: "leds-gpio" and "gpio_keys", respectively. Use those -instead of talking directly to the GPIOs; they integrate with kernel -frameworks better than your userspace code could. - +DO NOT ABUSE SYFS TO CONTROL HARDWARE THAT HAS PROPER KERNEL DRIVERS. +PLEASE READ THE DOCUMENT NAMED "drivers-on-gpio.txt" IN THIS DOCUMENTATION +DIRECTORY TO AVOID REINVENTING KERNEL WHEELS IN USERSPACE. I MEAN IT. +REALLY. Paths in Sysfs -------------- diff --git a/Documentation/i2c/busses/i2c-parport b/Documentation/i2c/busses/i2c-parport index 0e2d17b460fd..c3dbb3bfd814 100644 --- a/Documentation/i2c/busses/i2c-parport +++ b/Documentation/i2c/busses/i2c-parport @@ -20,6 +20,7 @@ It currently supports the following devices: * (type=5) Analog Devices evaluation boards: ADM1025, ADM1030, ADM1031 * (type=6) Barco LPT->DVI (K5800236) adapter * (type=7) One For All JP1 parallel port adapter + * (type=8) VCT-jig These devices use different pinout configurations, so you have to tell the driver what you have, using the type module parameter. There is no diff --git a/Documentation/i2c/slave-interface b/Documentation/i2c/slave-interface index 2dee4e2d62df..61ed05cd9531 100644 --- a/Documentation/i2c/slave-interface +++ b/Documentation/i2c/slave-interface @@ -31,10 +31,13 @@ User manual =========== I2C slave backends behave like standard I2C clients. So, you can instantiate -them as described in the document 'instantiating-devices'. A quick example for -instantiating the slave-eeprom driver from userspace at address 0x64 on bus 1: +them as described in the document 'instantiating-devices'. The only difference +is that i2c slave backends have their own address space. So, you have to add +0x1000 to the address you would originally request. An example for +instantiating the slave-eeprom driver from userspace at the 7 bit address 0x64 +on bus 1: - # echo slave-24c02 0x64 > /sys/bus/i2c/devices/i2c-1/new_device + # echo slave-24c02 0x1064 > /sys/bus/i2c/devices/i2c-1/new_device Each backend should come with separate documentation to describe its specific behaviour and setup. diff --git a/Documentation/i2c/ten-bit-addresses b/Documentation/i2c/ten-bit-addresses index cdfe13901b99..7b2d11e53a49 100644 --- a/Documentation/i2c/ten-bit-addresses +++ b/Documentation/i2c/ten-bit-addresses @@ -2,6 +2,10 @@ The I2C protocol knows about two kinds of device addresses: normal 7 bit addresses, and an extended set of 10 bit addresses. The sets of addresses do not intersect: the 7 bit address 0x10 is not the same as the 10 bit address 0x10 (though a single device could respond to both of them). +To avoid ambiguity, the user sees 10 bit addresses mapped to a different +address space, namely 0xa000-0xa3ff. The leading 0xa (= 10) represents the +10 bit mode. This is used for creating device names in sysfs. It is also +needed when instantiating 10 bit devices via the new_device file in sysfs. I2C messages to and from 10-bit address devices have a different format. See the I2C specification for the details. diff --git a/Documentation/infiniband/sysfs.txt b/Documentation/infiniband/sysfs.txt index ddd519b72ee1..9028b025501a 100644 --- a/Documentation/infiniband/sysfs.txt +++ b/Documentation/infiniband/sysfs.txt @@ -64,3 +64,23 @@ MTHCA fw_ver - Firmware version hca_type - HCA type: "MT23108", "MT25208 (MT23108 compat mode)", or "MT25208" + +HFI1 + + The hfi1 driver also creates these additional files: + + hw_rev - hardware revision + board_id - manufacturing board id + tempsense - thermal sense information + serial - board serial number + nfreectxts - number of free user contexts + nctxts - number of allowed contexts (PSM2) + chip_reset - diagnostic (root only) + boardversion - board version + ports/1/ + CMgtA/ + cc_settings_bin - CCA tables used by PSM2 + cc_table_bin + sc2v/ - 32 files (0 - 31) used to translate sl->vl + sl2sc/ - 32 files (0 - 31) used to translate sl->sc + vl2mtu/ - 16 (0 - 15) files used to determine MTU for vl diff --git a/Documentation/ioctl/ioctl-number.txt b/Documentation/ioctl/ioctl-number.txt index 141f847c7648..df1b25eb8382 100644 --- a/Documentation/ioctl/ioctl-number.txt +++ b/Documentation/ioctl/ioctl-number.txt @@ -265,7 +265,7 @@ Code Seq#(hex) Include File Comments 's' all linux/cdk.h 't' 00-7F linux/ppp-ioctl.h 't' 80-8F linux/isdn_ppp.h -'t' 90 linux/toshiba.h +'t' 90-91 linux/toshiba.h toshiba and toshiba_acpi SMM 'u' 00-1F linux/smb_fs.h gone 'u' 20-3F linux/uvcvideo.h USB video class host driver 'v' 00-1F linux/ext2_fs.h conflict! @@ -303,6 +303,7 @@ Code Seq#(hex) Include File Comments 0xA3 80-8F Port ACL in development: 0xA3 90-9F linux/dtlk.h +0xAA 00-3F linux/uapi/linux/userfaultfd.h 0xAB 00-1F linux/nbd.h 0xAC 00-1F linux/raw.h 0xAD 00 Netfilter device in development: @@ -316,6 +317,7 @@ Code Seq#(hex) Include File Comments 0xB3 00 linux/mmc/ioctl.h 0xC0 00-0F linux/usb/iowarrior.h 0xCA 00-0F uapi/misc/cxl.h +0xCA 80-8F uapi/scsi/cxlflash_ioctl.h 0xCB 00-1F CBM serial IEC bus in development: 0xCD 01 linux/reiserfs_fs.h diff --git a/Documentation/kbuild/kbuild.txt b/Documentation/kbuild/kbuild.txt index 6466704d47b5..0ff6a466a05b 100644 --- a/Documentation/kbuild/kbuild.txt +++ b/Documentation/kbuild/kbuild.txt @@ -174,6 +174,11 @@ The output directory is often set using "O=..." on the commandline. The value can be overridden in which case the default value is ignored. +KBUILD_SIGN_PIN +-------------------------------------------------- +This variable allows a passphrase or PIN to be passed to the sign-file +utility when signing kernel modules, if the private key requires such. + KBUILD_MODPOST_WARN -------------------------------------------------- KBUILD_MODPOST_WARN can be set to avoid errors in case of undefined diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt index b09dc2c05fdf..22a4b687ea5b 100644 --- a/Documentation/kernel-parameters.txt +++ b/Documentation/kernel-parameters.txt @@ -1317,6 +1317,10 @@ bytes respectively. Such letter suffixes can also be entirely omitted. , i8042.debug [HW] Toggle i8042 debug mode + i8042.unmask_kbd_data + [HW] Enable printing of interrupt data from the KBD port + (disabled by default, and as a pre-condition + requires that i8042.debug=1 be enabled) i8042.direct [HW] Put keyboard port into non-translated mode i8042.dumbkbd [HW] Pretend that controller can only read data from keyboard and cannot control its state @@ -2281,6 +2285,15 @@ bytes respectively. Such letter suffixes can also be entirely omitted. The default parameter value of '0' causes the kernel not to attempt recovery of lost locks. + nfs4.layoutstats_timer = + [NFSv4.2] Change the rate at which the kernel sends + layoutstats to the pNFS metadata server. + + Setting this to value to 0 causes the kernel to use + whatever value is the default set by the layout + driver. A non-zero value sets the minimum interval + in seconds between layoutstats transmissions. + nfsd.nfs4_disable_idmapping= [NFSv4] When set to the default of '1', the NFSv4 server will return only numeric uids and gids to @@ -4093,6 +4106,13 @@ bytes respectively. Such letter suffixes can also be entirely omitted. plus one apbt timer for broadcast timer. x86_intel_mid_timer=apbt_only | lapic_and_apbt + xen_512gb_limit [KNL,X86-64,XEN] + Restricts the kernel running paravirtualized under Xen + to use only up to 512 GB of RAM. The reason to do so is + crash analysis tools and Xen tools for doing domain + save/restore/migration must be enabled to handle larger + domains. + xen_emul_unplug= [HW,X86,XEN] Unplug Xen emulated devices Format: [unplug0,][unplug1] diff --git a/Documentation/md-cluster.txt b/Documentation/md-cluster.txt index de1af7db3355..1b794369e03a 100644 --- a/Documentation/md-cluster.txt +++ b/Documentation/md-cluster.txt @@ -91,7 +91,7 @@ The algorithm is: this message inappropriate or redundant. 3. sender write LVB. - sender down-convert MESSAGE from EX to CR + sender down-convert MESSAGE from EX to CW sender try to get EX of ACK [ wait until all receiver has *processed* the MESSAGE ] @@ -112,7 +112,7 @@ The algorithm is: sender down-convert ACK from EX to CR sender release MESSAGE sender release TOKEN - receiver upconvert to EX of MESSAGE + receiver upconvert to PR of MESSAGE receiver get CR of ACK receiver release MESSAGE diff --git a/Documentation/memory-barriers.txt b/Documentation/memory-barriers.txt index eafa6a53f72c..2ba8461b0631 100644 --- a/Documentation/memory-barriers.txt +++ b/Documentation/memory-barriers.txt @@ -2327,9 +2327,7 @@ about the state (old or new) implies an SMP-conditional general memory barrier explicit lock operations, described later). These include: xchg(); - cmpxchg(); atomic_xchg(); atomic_long_xchg(); - atomic_cmpxchg(); atomic_long_cmpxchg(); atomic_inc_return(); atomic_long_inc_return(); atomic_dec_return(); atomic_long_dec_return(); atomic_add_return(); atomic_long_add_return(); @@ -2342,7 +2340,9 @@ explicit lock operations, described later). These include: test_and_clear_bit(); test_and_change_bit(); - /* when succeeds (returns 1) */ + /* when succeeds */ + cmpxchg(); + atomic_cmpxchg(); atomic_long_cmpxchg(); atomic_add_unless(); atomic_long_add_unless(); These are used for such things as implementing ACQUIRE-class and RELEASE-class diff --git a/Documentation/module-signing.txt b/Documentation/module-signing.txt index c72702ec1ded..a78bf1ffa68c 100644 --- a/Documentation/module-signing.txt +++ b/Documentation/module-signing.txt @@ -89,6 +89,32 @@ This has a number of options available: their signatures checked without causing a dependency loop. + (4) "File name or PKCS#11 URI of module signing key" (CONFIG_MODULE_SIG_KEY) + + Setting this option to something other than its default of + "certs/signing_key.pem" will disable the autogeneration of signing keys + and allow the kernel modules to be signed with a key of your choosing. + The string provided should identify a file containing both a private key + and its corresponding X.509 certificate in PEM form, or — on systems where + the OpenSSL ENGINE_pkcs11 is functional — a PKCS#11 URI as defined by + RFC7512. In the latter case, the PKCS#11 URI should reference both a + certificate and a private key. + + If the PEM file containing the private key is encrypted, or if the + PKCS#11 token requries a PIN, this can be provided at build time by + means of the KBUILD_SIGN_PIN variable. + + + (5) "Additional X.509 keys for default system keyring" (CONFIG_SYSTEM_TRUSTED_KEYS) + + This option can be set to the filename of a PEM-encoded file containing + additional certificates which will be included in the system keyring by + default. + +Note that enabling module signing adds a dependency on the OpenSSL devel +packages to the kernel build processes for the tool that does the signing. + + ======================= GENERATING SIGNING KEYS ======================= @@ -100,16 +126,16 @@ it can be deleted or stored securely. The public key gets built into the kernel so that it can be used to check the signatures as the modules are loaded. -Under normal conditions, the kernel build will automatically generate a new -keypair using openssl if one does not exist in the files: +Under normal conditions, when CONFIG_MODULE_SIG_KEY is unchanged from its +default, the kernel build will automatically generate a new keypair using +openssl if one does not exist in the file: - signing_key.priv - signing_key.x509 + certs/signing_key.pem during the building of vmlinux (the public part of the key needs to be built into vmlinux) using parameters in the: - x509.genkey + certs/x509.genkey file (which is also generated if it does not already exist). @@ -135,8 +161,12 @@ kernel sources tree and the openssl command. The following is an example to generate the public/private key files: openssl req -new -nodes -utf8 -sha256 -days 36500 -batch -x509 \ - -config x509.genkey -outform DER -out signing_key.x509 \ - -keyout signing_key.priv + -config x509.genkey -outform PEM -out kernel_key.pem \ + -keyout kernel_key.pem + +The full pathname for the resulting kernel_key.pem file can then be specified +in the CONFIG_MODULE_SIG_KEY option, and the certificate and key therein will +be used instead of an autogenerated keypair. ========================= @@ -152,10 +182,9 @@ in a keyring called ".system_keyring" that can be seen by: 302d2d52 I------ 1 perm 1f010000 0 0 asymmetri Fedora kernel signing key: d69a84e6bce3d216b979e9505b3e3ef9a7118079: X509.RSA a7118079 [] ... -Beyond the public key generated specifically for module signing, any file -placed in the kernel source root directory or the kernel build root directory -whose name is suffixed with ".x509" will be assumed to be an X.509 public key -and will be added to the keyring. +Beyond the public key generated specifically for module signing, additional +trusted certificates can be provided in a PEM-encoded file referenced by the +CONFIG_SYSTEM_TRUSTED_KEYS configuration option. Further, the architecture code may take public keys from a hardware store and add those in also (e.g. from the UEFI key database). @@ -181,7 +210,7 @@ To manually sign a module, use the scripts/sign-file tool available in the Linux kernel source tree. The script requires 4 arguments: 1. The hash algorithm (e.g., sha256) - 2. The private key filename + 2. The private key filename or PKCS#11 URI 3. The public key filename 4. The kernel module to be signed @@ -194,6 +223,9 @@ The hash algorithm used does not have to match the one configured, but if it doesn't, you should make sure that hash algorithm is either built into the kernel or can be loaded without requiring itself. +If the private key requires a passphrase or PIN, it can be provided in the +$KBUILD_SIGN_PIN environment variable. + ============================ SIGNED MODULES AND STRIPPING diff --git a/Documentation/networking/6lowpan.txt b/Documentation/networking/6lowpan.txt new file mode 100644 index 000000000000..a7dc7e939c7a --- /dev/null +++ b/Documentation/networking/6lowpan.txt @@ -0,0 +1,50 @@ + +Netdev private dataroom for 6lowpan interfaces: + +All 6lowpan able net devices, means all interfaces with ARPHRD_6LOWPAN, +must have "struct lowpan_priv" placed at beginning of netdev_priv. + +The priv_size of each interface should be calculate by: + + dev->priv_size = LOWPAN_PRIV_SIZE(LL_6LOWPAN_PRIV_DATA); + +Where LL_PRIV_6LOWPAN_DATA is sizeof linklayer 6lowpan private data struct. +To access the LL_PRIV_6LOWPAN_DATA structure you can cast: + + lowpan_priv(dev)-priv; + +to your LL_6LOWPAN_PRIV_DATA structure. + +Before registering the lowpan netdev interface you must run: + + lowpan_netdev_setup(dev, LOWPAN_LLTYPE_FOOBAR); + +wheres LOWPAN_LLTYPE_FOOBAR is a define for your 6LoWPAN linklayer type of +enum lowpan_lltypes. + +Example to evaluate the private usually you can do: + +static inline sturct lowpan_priv_foobar * +lowpan_foobar_priv(struct net_device *dev) +{ + return (sturct lowpan_priv_foobar *)lowpan_priv(dev)->priv; +} + +switch (dev->type) { +case ARPHRD_6LOWPAN: + lowpan_priv = lowpan_priv(dev); + /* do great stuff which is ARPHRD_6LOWPAN related */ + switch (lowpan_priv->lltype) { + case LOWPAN_LLTYPE_FOOBAR: + /* do 802.15.4 6LoWPAN handling here */ + lowpan_foobar_priv(dev)->bar = foo; + break; + ... + } + break; +... +} + +In case of generic 6lowpan branch ("net/6lowpan") you can remove the check +on ARPHRD_6LOWPAN, because you can be sure that these function are called +by ARPHRD_6LOWPAN interfaces. diff --git a/Documentation/networking/dsa/bcm_sf2.txt b/Documentation/networking/dsa/bcm_sf2.txt new file mode 100644 index 000000000000..d999d0c1c5b8 --- /dev/null +++ b/Documentation/networking/dsa/bcm_sf2.txt @@ -0,0 +1,114 @@ +Broadcom Starfighter 2 Ethernet switch driver +============================================= + +Broadcom's Starfighter 2 Ethernet switch hardware block is commonly found and +deployed in the following products: + +- xDSL gateways such as BCM63138 +- streaming/multimedia Set Top Box such as BCM7445 +- Cable Modem/residential gateways such as BCM7145/BCM3390 + +The switch is typically deployed in a configuration involving between 5 to 13 +ports, offering a range of built-in and customizable interfaces: + +- single integrated Gigabit PHY +- quad integrated Gigabit PHY +- quad external Gigabit PHY w/ MDIO multiplexer +- integrated MoCA PHY +- several external MII/RevMII/GMII/RGMII interfaces + +The switch also supports specific congestion control features which allow MoCA +fail-over not to lose packets during a MoCA role re-election, as well as out of +band back-pressure to the host CPU network interface when downstream interfaces +are connected at a lower speed. + +The switch hardware block is typically interfaced using MMIO accesses and +contains a bunch of sub-blocks/registers: + +* SWITCH_CORE: common switch registers +* SWITCH_REG: external interfaces switch register +* SWITCH_MDIO: external MDIO bus controller (there is another one in SWITCH_CORE, + which is used for indirect PHY accesses) +* SWITCH_INDIR_RW: 64-bits wide register helper block +* SWITCH_INTRL2_0/1: Level-2 interrupt controllers +* SWITCH_ACB: Admission control block +* SWITCH_FCB: Fail-over control block + +Implementation details +====================== + +The driver is located in drivers/net/dsa/bcm_sf2.c and is implemented as a DSA +driver; see Documentation/networking/dsa/dsa.txt for details on the subsytem +and what it provides. + +The SF2 switch is configured to enable a Broadcom specific 4-bytes switch tag +which gets inserted by the switch for every packet forwarded to the CPU +interface, conversely, the CPU network interface should insert a similar tag for +packets entering the CPU port. The tag format is described in +net/dsa/tag_brcm.c. + +Overall, the SF2 driver is a fairly regular DSA driver; there are a few +specifics covered below. + +Device Tree probing +------------------- + +The DSA platform device driver is probed using a specific compatible string +provided in net/dsa/dsa.c. The reason for that is because the DSA subsystem gets +registered as a platform device driver currently. DSA will provide the needed +device_node pointers which are then accessible by the switch driver setup +function to setup resources such as register ranges and interrupts. This +currently works very well because none of the of_* functions utilized by the +driver require a struct device to be bound to a struct device_node, but things +may change in the future. + +MDIO indirect accesses +---------------------- + +Due to a limitation in how Broadcom switches have been designed, external +Broadcom switches connected to a SF2 require the use of the DSA slave MDIO bus +in order to properly configure them. By default, the SF2 pseudo-PHY address, and +an external switch pseudo-PHY address will both be snooping for incoming MDIO +transactions, since they are at the same address (30), resulting in some kind of +"double" programming. Using DSA, and setting ds->phys_mii_mask accordingly, we +selectively divert reads and writes towards external Broadcom switches +pseudo-PHY addresses. Newer revisions of the SF2 hardware have introduced a +configurable pseudo-PHY address which circumvents the initial design limitation. + +Multimedia over CoAxial (MoCA) interfaces +----------------------------------------- + +MoCA interfaces are fairly specific and require the use of a firmware blob which +gets loaded onto the MoCA processor(s) for packet processing. The switch +hardware contains logic which will assert/de-assert link states accordingly for +the MoCA interface whenever the MoCA coaxial cable gets disconnected or the +firmware gets reloaded. The SF2 driver relies on such events to properly set its +MoCA interface carrier state and properly report this to the networking stack. + +The MoCA interfaces are supported using the PHY library's fixed PHY/emulated PHY +device and the switch driver registers a fixed_link_update callback for such +PHYs which reflects the link state obtained from the interrupt handler. + + +Power Management +---------------- + +Whenever possible, the SF2 driver tries to minimize the overall switch power +consumption by applying a combination of: + +- turning off internal buffers/memories +- disabling packet processing logic +- putting integrated PHYs in IDDQ/low-power +- reducing the switch core clock based on the active port count +- enabling and advertising EEE +- turning off RGMII data processing logic when the link goes down + +Wake-on-LAN +----------- + +Wake-on-LAN is currently implemented by utilizing the host processor Ethernet +MAC controller wake-on logic. Whenever Wake-on-LAN is requested, an intersection +between the user request and the supported host Ethernet interface WoL +capabilities is done and the intersection result gets configured. During +system-wide suspend/resume, only ports not participating in Wake-on-LAN are +disabled. diff --git a/Documentation/networking/dsa/dsa.txt b/Documentation/networking/dsa/dsa.txt new file mode 100644 index 000000000000..aa9c1f9313cd --- /dev/null +++ b/Documentation/networking/dsa/dsa.txt @@ -0,0 +1,615 @@ +Distributed Switch Architecture +=============================== + +Introduction +============ + +This document describes the Distributed Switch Architecture (DSA) subsystem +design principles, limitations, interactions with other subsystems, and how to +develop drivers for this subsystem as well as a TODO for developers interested +in joining the effort. + +Design principles +================= + +The Distributed Switch Architecture is a subsystem which was primarily designed +to support Marvell Ethernet switches (MV88E6xxx, a.k.a Linkstreet product line) +using Linux, but has since evolved to support other vendors as well. + +The original philosophy behind this design was to be able to use unmodified +Linux tools such as bridge, iproute2, ifconfig to work transparently whether +they configured/queried a switch port network device or a regular network +device. + +An Ethernet switch is typically comprised of multiple front-panel ports, and one +or more CPU or management port. The DSA subsystem currently relies on the +presence of a management port connected to an Ethernet controller capable of +receiving Ethernet frames from the switch. This is a very common setup for all +kinds of Ethernet switches found in Small Home and Office products: routers, +gateways, or even top-of-the rack switches. This host Ethernet controller will +be later referred to as "master" and "cpu" in DSA terminology and code. + +The D in DSA stands for Distributed, because the subsystem has been designed +with the ability to configure and manage cascaded switches on top of each other +using upstream and downstream Ethernet links between switches. These specific +ports are referred to as "dsa" ports in DSA terminology and code. A collection +of multiple switches connected to each other is called a "switch tree". + +For each front-panel port, DSA will create specialized network devices which are +used as controlling and data-flowing endpoints for use by the Linux networking +stack. These specialized network interfaces are referred to as "slave" network +interfaces in DSA terminology and code. + +The ideal case for using DSA is when an Ethernet switch supports a "switch tag" +which is a hardware feature making the switch insert a specific tag for each +Ethernet frames it received to/from specific ports to help the management +interface figure out: + +- what port is this frame coming from +- what was the reason why this frame got forwarded +- how to send CPU originated traffic to specific ports + +The subsystem does support switches not capable of inserting/stripping tags, but +the features might be slightly limited in that case (traffic separation relies +on Port-based VLAN IDs). + +Note that DSA does not currently create network interfaces for the "cpu" and +"dsa" ports because: + +- the "cpu" port is the Ethernet switch facing side of the management + controller, and as such, would create a duplication of feature, since you + would get two interfaces for the same conduit: master netdev, and "cpu" netdev + +- the "dsa" port(s) are just conduits between two or more switches, and as such + cannot really be used as proper network interfaces either, only the + downstream, or the top-most upstream interface makes sense with that model + +Switch tagging protocols +------------------------ + +DSA currently supports 4 different tagging protocols, and a tag-less mode as +well. The different protocols are implemented in: + +net/dsa/tag_trailer.c: Marvell's 4 trailer tag mode (legacy) +net/dsa/tag_dsa.c: Marvell's original DSA tag +net/dsa/tag_edsa.c: Marvell's enhanced DSA tag +net/dsa/tag_brcm.c: Broadcom's 4 bytes tag + +The exact format of the tag protocol is vendor specific, but in general, they +all contain something which: + +- identifies which port the Ethernet frame came from/should be sent to +- provides a reason why this frame was forwarded to the management interface + +Master network devices +---------------------- + +Master network devices are regular, unmodified Linux network device drivers for +the CPU/management Ethernet interface. Such a driver might occasionally need to +know whether DSA is enabled (e.g.: to enable/disable specific offload features), +but the DSA subsystem has been proven to work with industry standard drivers: +e1000e, mv643xx_eth etc. without having to introduce modifications to these +drivers. Such network devices are also often referred to as conduit network +devices since they act as a pipe between the host processor and the hardware +Ethernet switch. + +Networking stack hooks +---------------------- + +When a master netdev is used with DSA, a small hook is placed in in the +networking stack is in order to have the DSA subsystem process the Ethernet +switch specific tagging protocol. DSA accomplishes this by registering a +specific (and fake) Ethernet type (later becoming skb->protocol) with the +networking stack, this is also known as a ptype or packet_type. A typical +Ethernet Frame receive sequence looks like this: + +Master network device (e.g.: e1000e): + +Receive interrupt fires: +- receive function is invoked +- basic packet processing is done: getting length, status etc. +- packet is prepared to be processed by the Ethernet layer by calling + eth_type_trans + +net/ethernet/eth.c: + +eth_type_trans(skb, dev) + if (dev->dsa_ptr != NULL) + -> skb->protocol = ETH_P_XDSA + +drivers/net/ethernet/*: + +netif_receive_skb(skb) + -> iterate over registered packet_type + -> invoke handler for ETH_P_XDSA, calls dsa_switch_rcv() + +net/dsa/dsa.c: + -> dsa_switch_rcv() + -> invoke switch tag specific protocol handler in + net/dsa/tag_*.c + +net/dsa/tag_*.c: + -> inspect and strip switch tag protocol to determine originating port + -> locate per-port network device + -> invoke eth_type_trans() with the DSA slave network device + -> invoked netif_receive_skb() + +Past this point, the DSA slave network devices get delivered regular Ethernet +frames that can be processed by the networking stack. + +Slave network devices +--------------------- + +Slave network devices created by DSA are stacked on top of their master network +device, each of these network interfaces will be responsible for being a +controlling and data-flowing end-point for each front-panel port of the switch. +These interfaces are specialized in order to: + +- insert/remove the switch tag protocol (if it exists) when sending traffic + to/from specific switch ports +- query the switch for ethtool operations: statistics, link state, + Wake-on-LAN, register dumps... +- external/internal PHY management: link, auto-negotiation etc. + +These slave network devices have custom net_device_ops and ethtool_ops function +pointers which allow DSA to introduce a level of layering between the networking +stack/ethtool, and the switch driver implementation. + +Upon frame transmission from these slave network devices, DSA will look up which +switch tagging protocol is currently registered with these network devices, and +invoke a specific transmit routine which takes care of adding the relevant +switch tag in the Ethernet frames. + +These frames are then queued for transmission using the master network device +ndo_start_xmit() function, since they contain the appropriate switch tag, the +Ethernet switch will be able to process these incoming frames from the +management interface and delivers these frames to the physical switch port. + +Graphical representation +------------------------ + +Summarized, this is basically how DSA looks like from a network device +perspective: + + + |--------------------------- + | CPU network device (eth0)| + ---------------------------- + | | + |--------------------------------------------| + | Switch driver | + |--------------------------------------------| + || || || + |-------| |-------| |-------| + | sw0p0 | | sw0p1 | | sw0p2 | + |-------| |-------| |-------| + +Slave MDIO bus +-------------- + +In order to be able to read to/from a switch PHY built into it, DSA creates a +slave MDIO bus which allows a specific switch driver to divert and intercept +MDIO reads/writes towards specific PHY addresses. In most MDIO-connected +switches, these functions would utilize direct or indirect PHY addressing mode +to return standard MII registers from the switch builtin PHYs, allowing the PHY +library and/or to return link status, link partner pages, auto-negotiation +results etc.. + +For Ethernet switches which have both external and internal MDIO busses, the +slave MII bus can be utilized to mux/demux MDIO reads and writes towards either +internal or external MDIO devices this switch might be connected to: internal +PHYs, external PHYs, or even external switches. + +Data structures +--------------- + +DSA data structures are defined in include/net/dsa.h as well as +net/dsa/dsa_priv.h. + +dsa_chip_data: platform data configuration for a given switch device, this +structure describes a switch device's parent device, its address, as well as +various properties of its ports: names/labels, and finally a routing table +indication (when cascading switches) + +dsa_platform_data: platform device configuration data which can reference a +collection of dsa_chip_data structure if multiples switches are cascaded, the +master network device this switch tree is attached to needs to be referenced + +dsa_switch_tree: structure assigned to the master network device under +"dsa_ptr", this structure references a dsa_platform_data structure as well as +the tagging protocol supported by the switch tree, and which receive/transmit +function hooks should be invoked, information about the directly attached switch +is also provided: CPU port. Finally, a collection of dsa_switch are referenced +to address individual switches in the tree. + +dsa_switch: structure describing a switch device in the tree, referencing a +dsa_switch_tree as a backpointer, slave network devices, master network device, +and a reference to the backing dsa_switch_driver + +dsa_switch_driver: structure referencing function pointers, see below for a full +description. + +Design limitations +================== + +DSA is a platform device driver +------------------------------- + +DSA is implemented as a DSA platform device driver which is convenient because +it will register the entire DSA switch tree attached to a master network device +in one-shot, facilitating the device creation and simplifying the device driver +model a bit, this comes however with a number of limitations: + +- building DSA and its switch drivers as modules is currently not working +- the device driver parenting does not necessarily reflect the original + bus/device the switch can be created from +- supporting non-MDIO and non-MMIO (platform) switches is not possible + +Limits on the number of devices and ports +----------------------------------------- + +DSA currently limits the number of maximum switches within a tree to 4 +(DSA_MAX_SWITCHES), and the number of ports per switch to 12 (DSA_MAX_PORTS). +These limits could be extended to support larger configurations would this need +arise. + +Lack of CPU/DSA network devices +------------------------------- + +DSA does not currently create slave network devices for the CPU or DSA ports, as +described before. This might be an issue in the following cases: + +- inability to fetch switch CPU port statistics counters using ethtool, which + can make it harder to debug MDIO switch connected using xMII interfaces + +- inability to configure the CPU port link parameters based on the Ethernet + controller capabilities attached to it: http://patchwork.ozlabs.org/patch/509806/ + +- inability to configure specific VLAN IDs / trunking VLANs between switches + when using a cascaded setup + +Common pitfalls using DSA setups +-------------------------------- + +Once a master network device is configured to use DSA (dev->dsa_ptr becomes +non-NULL), and the switch behind it expects a tagging protocol, this network +interface can only exclusively be used as a conduit interface. Sending packets +directly through this interface (e.g.: opening a socket using this interface) +will not make us go through the switch tagging protocol transmit function, so +the Ethernet switch on the other end, expecting a tag will typically drop this +frame. + +Slave network devices check that the master network device is UP before allowing +you to administratively bring UP these slave network devices. A common +configuration mistake is forgetting to bring UP the master network device first. + +Interactions with other subsystems +================================== + +DSA currently leverages the following subsystems: + +- MDIO/PHY library: drivers/net/phy/phy.c, mdio_bus.c +- Switchdev: net/switchdev/* +- Device Tree for various of_* functions +- HWMON: drivers/hwmon/* + +MDIO/PHY library +---------------- + +Slave network devices exposed by DSA may or may not be interfacing with PHY +devices (struct phy_device as defined in include/linux/phy.h), but the DSA +subsystem deals with all possible combinations: + +- internal PHY devices, built into the Ethernet switch hardware +- external PHY devices, connected via an internal or external MDIO bus +- internal PHY devices, connected via an internal MDIO bus +- special, non-autonegotiated or non MDIO-managed PHY devices: SFPs, MoCA; a.k.a + fixed PHYs + +The PHY configuration is done by the dsa_slave_phy_setup() function and the +logic basically looks like this: + +- if Device Tree is used, the PHY device is looked up using the standard + "phy-handle" property, if found, this PHY device is created and registered + using of_phy_connect() + +- if Device Tree is used, and the PHY device is "fixed", that is, conforms to + the definition of a non-MDIO managed PHY as defined in + Documentation/devicetree/bindings/net/fixed-link.txt, the PHY is registered + and connected transparently using the special fixed MDIO bus driver + +- finally, if the PHY is built into the switch, as is very common with + standalone switch packages, the PHY is probed using the slave MII bus created + by DSA + + +SWITCHDEV +--------- + +DSA directly utilizes SWITCHDEV when interfacing with the bridge layer, and +more specifically with its VLAN filtering portion when configuring VLANs on top +of per-port slave network devices. Since DSA primarily deals with +MDIO-connected switches, although not exclusively, SWITCHDEV's +prepare/abort/commit phases are often simplified into a prepare phase which +checks whether the operation is supporte by the DSA switch driver, and a commit +phase which applies the changes. + +As of today, the only SWITCHDEV objects supported by DSA are the FDB and VLAN +objects. + +Device Tree +----------- + +DSA features a standardized binding which is documented in +Documentation/devicetree/bindings/net/dsa/dsa.txt. PHY/MDIO library helper +functions such as of_get_phy_mode(), of_phy_connect() are also used to query +per-port PHY specific details: interface connection, MDIO bus location etc.. + +HWMON +----- + +Some switch drivers feature internal temperature sensors which are exposed as +regular HWMON devices in /sys/class/hwmon/. + +Driver development +================== + +DSA switch drivers need to implement a dsa_switch_driver structure which will +contain the various members described below. + +register_switch_driver() registers this dsa_switch_driver in its internal list +of drivers to probe for. unregister_switch_driver() does the exact opposite. + +Unless requested differently by setting the priv_size member accordingly, DSA +does not allocate any driver private context space. + +Switch configuration +-------------------- + +- priv_size: additional size needed by the switch driver for its private context + +- tag_protocol: this is to indicate what kind of tagging protocol is supported, + should be a valid value from the dsa_tag_protocol enum + +- probe: probe routine which will be invoked by the DSA platform device upon + registration to test for the presence/absence of a switch device. For MDIO + devices, it is recommended to issue a read towards internal registers using + the switch pseudo-PHY and return whether this is a supported device. For other + buses, return a non-NULL string + +- setup: setup function for the switch, this function is responsible for setting + up the dsa_switch_driver private structure with all it needs: register maps, + interrupts, mutexes, locks etc.. This function is also expected to properly + configure the switch to separate all network interfaces from each other, that + is, they should be isolated by the switch hardware itself, typically by creating + a Port-based VLAN ID for each port and allowing only the CPU port and the + specific port to be in the forwarding vector. Ports that are unused by the + platform should be disabled. Past this function, the switch is expected to be + fully configured and ready to serve any kind of request. It is recommended + to issue a software reset of the switch during this setup function in order to + avoid relying on what a previous software agent such as a bootloader/firmware + may have previously configured. + +- set_addr: Some switches require the programming of the management interface's + Ethernet MAC address, switch drivers can also disable ageing of MAC addresses + on the management interface and "hardcode"/"force" this MAC address for the + CPU/management interface as an optimization + +PHY devices and link management +------------------------------- + +- get_phy_flags: Some switches are interfaced to various kinds of Ethernet PHYs, + if the PHY library PHY driver needs to know about information it cannot obtain + on its own (e.g.: coming from switch memory mapped registers), this function + should return a 32-bits bitmask of "flags", that is private between the switch + driver and the Ethernet PHY driver in drivers/net/phy/*. + +- phy_read: Function invoked by the DSA slave MDIO bus when attempting to read + the switch port MDIO registers. If unavailable, return 0xffff for each read. + For builtin switch Ethernet PHYs, this function should allow reading the link + status, auto-negotiation results, link partner pages etc.. + +- phy_write: Function invoked by the DSA slave MDIO bus when attempting to write + to the switch port MDIO registers. If unavailable return a negative error + code. + +- poll_link: Function invoked by DSA to query the link state of the switch + builtin Ethernet PHYs, per port. This function is responsible for calling + netif_carrier_{on,off} when appropriate, and can be used to poll all ports in a + single call. Executes from workqueue context. + +- adjust_link: Function invoked by the PHY library when a slave network device + is attached to a PHY device. This function is responsible for appropriately + configuring the switch port link parameters: speed, duplex, pause based on + what the phy_device is providing. + +- fixed_link_update: Function invoked by the PHY library, and specifically by + the fixed PHY driver asking the switch driver for link parameters that could + not be auto-negotiated, or obtained by reading the PHY registers through MDIO. + This is particularly useful for specific kinds of hardware such as QSGMII, + MoCA or other kinds of non-MDIO managed PHYs where out of band link + information is obtained + +Ethtool operations +------------------ + +- get_strings: ethtool function used to query the driver's strings, will + typically return statistics strings, private flags strings etc. + +- get_ethtool_stats: ethtool function used to query per-port statistics and + return their values. DSA overlays slave network devices general statistics: + RX/TX counters from the network device, with switch driver specific statistics + per port + +- get_sset_count: ethtool function used to query the number of statistics items + +- get_wol: ethtool function used to obtain Wake-on-LAN settings per-port, this + function may, for certain implementations also query the master network device + Wake-on-LAN settings if this interface needs to participate in Wake-on-LAN + +- set_wol: ethtool function used to configure Wake-on-LAN settings per-port, + direct counterpart to set_wol with similar restrictions + +- set_eee: ethtool function which is used to configure a switch port EEE (Green + Ethernet) settings, can optionally invoke the PHY library to enable EEE at the + PHY level if relevant. This function should enable EEE at the switch port MAC + controller and data-processing logic + +- get_eee: ethtool function which is used to query a switch port EEE settings, + this function should return the EEE state of the switch port MAC controller + and data-processing logic as well as query the PHY for its currently configured + EEE settings + +- get_eeprom_len: ethtool function returning for a given switch the EEPROM + length/size in bytes + +- get_eeprom: ethtool function returning for a given switch the EEPROM contents + +- set_eeprom: ethtool function writing specified data to a given switch EEPROM + +- get_regs_len: ethtool function returning the register length for a given + switch + +- get_regs: ethtool function returning the Ethernet switch internal register + contents. This function might require user-land code in ethtool to + pretty-print register values and registers + +Power management +---------------- + +- suspend: function invoked by the DSA platform device when the system goes to + suspend, should quiesce all Ethernet switch activities, but keep ports + participating in Wake-on-LAN active as well as additional wake-up logic if + supported + +- resume: function invoked by the DSA platform device when the system resumes, + should resume all Ethernet switch activities and re-configure the switch to be + in a fully active state + +- port_enable: function invoked by the DSA slave network device ndo_open + function when a port is administratively brought up, this function should be + fully enabling a given switch port. DSA takes care of marking the port with + BR_STATE_BLOCKING if the port is a bridge member, or BR_STATE_FORWARDING if it + was not, and propagating these changes down to the hardware + +- port_disable: function invoked by the DSA slave network device ndo_close + function when a port is administratively brought down, this function should be + fully disabling a given switch port. DSA takes care of marking the port with + BR_STATE_DISABLED and propagating changes to the hardware if this port is + disabled while being a bridge member + +Hardware monitoring +------------------- + +These callbacks are only available if CONFIG_NET_DSA_HWMON is enabled: + +- get_temp: this function queries the given switch for its temperature + +- get_temp_limit: this function returns the switch current maximum temperature + limit + +- set_temp_limit: this function configures the maximum temperature limit allowed + +- get_temp_alarm: this function returns the critical temperature threshold + returning an alarm notification + +See Documentation/hwmon/sysfs-interface for details. + +Bridge layer +------------ + +- port_join_bridge: bridge layer function invoked when a given switch port is + added to a bridge, this function should be doing the necessary at the switch + level to permit the joining port from being added to the relevant logical + domain for it to ingress/egress traffic with other members of the bridge. DSA + does nothing but calculate a bitmask of switch ports currently members of the + specified bridge being requested the join + +- port_leave_bridge: bridge layer function invoked when a given switch port is + removed from a bridge, this function should be doing the necessary at the + switch level to deny the leaving port from ingress/egress traffic from the + remaining bridge members. When the port leaves the bridge, it should be aged + out at the switch hardware for the switch to (re) learn MAC addresses behind + this port. DSA calculates the bitmask of ports still members of the bridge + being left + +- port_stp_update: bridge layer function invoked when a given switch port STP + state is computed by the bridge layer and should be propagated to switch + hardware to forward/block/learn traffic. The switch driver is responsible for + computing a STP state change based on current and asked parameters and perform + the relevant ageing based on the intersection results + +Bridge VLAN filtering +--------------------- + +- port_pvid_get: bridge layer function invoked when a Port-based VLAN ID is + queried for the given switch port + +- port_pvid_set: bridge layer function invoked when a Port-based VLAN ID needs + to be configured on the given switch port + +- port_vlan_add: bridge layer function invoked when a VLAN is configured + (tagged or untagged) for the given switch port + +- port_vlan_del: bridge layer function invoked when a VLAN is removed from the + given switch port + +- vlan_getnext: bridge layer function invoked to query the next configured VLAN + in the switch, i.e. returns the bitmaps of members and untagged ports + +- port_fdb_add: bridge layer function invoked when the bridge wants to install a + Forwarding Database entry, the switch hardware should be programmed with the + specified address in the specified VLAN Id in the forwarding database + associated with this VLAN ID + +Note: VLAN ID 0 corresponds to the port private database, which, in the context +of DSA, would be the its port-based VLAN, used by the associated bridge device. + +- port_fdb_del: bridge layer function invoked when the bridge wants to remove a + Forwarding Database entry, the switch hardware should be programmed to delete + the specified MAC address from the specified VLAN ID if it was mapped into + this port forwarding database + +TODO +==== + +The platform device problem +--------------------------- +DSA is currently implemented as a platform device driver which is far from ideal +as was discussed in this thread: + +http://permalink.gmane.org/gmane.linux.network/329848 + +This basically prevents the device driver model to be properly used and applied, +and support non-MDIO, non-MMIO Ethernet connected switches. + +Another problem with the platform device driver approach is that it prevents the +use of a modular switch drivers build due to a circular dependency, illustrated +here: + +http://comments.gmane.org/gmane.linux.network/345803 + +Attempts of reworking this has been done here: + +https://lwn.net/Articles/643149/ + +Making SWITCHDEV and DSA converge towards an unified codebase +------------------------------------------------------------- + +SWITCHDEV properly takes care of abstracting the networking stack with offload +capable hardware, but does not enforce a strict switch device driver model. On +the other DSA enforces a fairly strict device driver model, and deals with most +of the switch specific. At some point we should envision a merger between these +two subsystems and get the best of both worlds. + +Other hanging fruits +-------------------- + +- making the number of ports fully dynamic and not dependent on DSA_MAX_PORTS +- allowing more than one CPU/management interface: + http://comments.gmane.org/gmane.linux.network/365657 +- porting more drivers from other vendors: + http://comments.gmane.org/gmane.linux.network/365510 diff --git a/Documentation/networking/ip-sysctl.txt b/Documentation/networking/ip-sysctl.txt index 5fae7704daab..ebe94f2cab98 100644 --- a/Documentation/networking/ip-sysctl.txt +++ b/Documentation/networking/ip-sysctl.txt @@ -586,6 +586,21 @@ tcp_min_tso_segs - INTEGER if available window is too small. Default: 2 +tcp_pacing_ss_ratio - INTEGER + sk->sk_pacing_rate is set by TCP stack using a ratio applied + to current rate. (current_rate = cwnd * mss / srtt) + If TCP is in slow start, tcp_pacing_ss_ratio is applied + to let TCP probe for bigger speeds, assuming cwnd can be + doubled every other RTT. + Default: 200 + +tcp_pacing_ca_ratio - INTEGER + sk->sk_pacing_rate is set by TCP stack using a ratio applied + to current rate. (current_rate = cwnd * mss / srtt) + If TCP is in congestion avoidance phase, tcp_pacing_ca_ratio + is applied to conservatively probe for bigger throughput. + Default: 120 + tcp_tso_win_divisor - INTEGER This allows control over what percentage of the congestion window can be consumed by a single TSO frame. @@ -1181,6 +1196,16 @@ tag - INTEGER Allows you to write a number, which can be used as required. Default value is 0. +xfrm4_gc_thresh - INTEGER + The threshold at which we will start garbage collecting for IPv4 + destination cache entries. At twice this value the system will + refuse new allocations. + +igmp_link_local_mcast_reports - BOOLEAN + Enable IGMP reports for link local multicast groups in the + 224.0.0.X range. + Default TRUE + Alexey Kuznetsov. kuznet@ms2.inr.ac.ru @@ -1215,14 +1240,20 @@ flowlabel_consistency - BOOLEAN FALSE: disabled Default: TRUE -auto_flowlabels - BOOLEAN - Automatically generate flow labels based based on a flow hash - of the packet. This allows intermediate devices, such as routers, - to idenfify packet flows for mechanisms like Equal Cost Multipath +auto_flowlabels - INTEGER + Automatically generate flow labels based on a flow hash of the + packet. This allows intermediate devices, such as routers, to + identify packet flows for mechanisms like Equal Cost Multipath Routing (see RFC 6438). - TRUE: enabled - FALSE: disabled - Default: false + 0: automatic flow labels are completely disabled + 1: automatic flow labels are enabled by default, they can be + disabled on a per socket basis using the IPV6_AUTOFLOWLABEL + socket option + 2: automatic flow labels are allowed, they may be enabled on a + per socket basis using the IPV6_AUTOFLOWLABEL socket option + 3: automatic flow labels are enabled and enforced, they cannot + be disabled by the socket option + Default: 1 flowlabel_state_ranges - BOOLEAN Split the flow label number space into two ranges. 0-0x7FFFF is @@ -1340,6 +1371,14 @@ accept_ra_from_local - BOOLEAN disabled if accept_ra_from_local is disabled on a specific interface. +accept_ra_min_hop_limit - INTEGER + Minimum hop limit Information in Router Advertisement. + + Hop limit Information in Router Advertisement less than this + variable shall be ignored. + + Default: 1 + accept_ra_pinfo - BOOLEAN Learn Prefix Information in Router Advertisement. @@ -1435,6 +1474,11 @@ mtu - INTEGER Default Maximum Transfer Unit Default: 1280 (IPv6 required minimum) +ip_nonlocal_bind - BOOLEAN + If set, allows processes to bind() to non-local IPv6 addresses, + which can be quite useful - but may break some applications. + Default: 0 + router_probe_interval - INTEGER Minimum interval (in seconds) between Router Probing described in RFC4191. @@ -1455,6 +1499,13 @@ router_solicitations - INTEGER routers are present. Default: 3 +use_oif_addrs_only - BOOLEAN + When enabled, the candidate source addresses for destinations + routed via this interface are restricted to the set of addresses + configured on this interface (vis. RFC 6724, section 4). + + Default: false + use_tempaddr - INTEGER Preference for Privacy Extensions (RFC3041). <= 0 : disable Privacy Extensions @@ -1591,6 +1642,11 @@ ratelimit - INTEGER otherwise the minimal space between responses in milliseconds. Default: 1000 +xfrm6_gc_thresh - INTEGER + The threshold at which we will start garbage collecting for IPv6 + destination cache entries. At twice this value the system will + refuse new allocations. + IPv6 Update by: Pekka Savola diff --git a/Documentation/networking/stmmac.txt b/Documentation/networking/stmmac.txt index e655e2453c98..d64a14714236 100644 --- a/Documentation/networking/stmmac.txt +++ b/Documentation/networking/stmmac.txt @@ -135,12 +135,8 @@ struct plat_stmmacenet_data { int maxmtu; void (*fix_mac_speed)(void *priv, unsigned int speed); void (*bus_setup)(void __iomem *ioaddr); - void *(*setup)(struct platform_device *pdev); - void (*free)(struct platform_device *pdev, void *priv); int (*init)(struct platform_device *pdev, void *priv); void (*exit)(struct platform_device *pdev, void *priv); - void *custom_cfg; - void *custom_data; void *bsp_priv; }; @@ -179,15 +175,11 @@ Where: o bus_setup: perform HW setup of the bus. For example, on some ST platforms this field is used to configure the AMBA bridge to generate more efficient STBus traffic. - o setup/init/exit: callbacks used for calling a custom initialization; + o init/exit: callbacks used for calling a custom initialization; this is sometime necessary on some platforms (e.g. ST boxes) where the HW needs to have set some PIO lines or system cfg - registers. setup should return a pointer to private data, - which will be stored in bsp_priv, and then passed to init and - exit callbacks. init/exit callbacks should not use or modify + registers. init/exit callbacks should not use or modify platform data. - o custom_cfg/custom_data: this is a custom configuration that can be passed - while initializing the resources. o bsp_priv: another private pointer. For MDIO bus The we have: @@ -262,7 +254,7 @@ static struct fixed_phy_status stmmac0_fixed_phy_status = { During the board's device_init we can configure the first MAC for fixed_link by calling: - fixed_phy_add(PHY_POLL, 1, &stmmac0_fixed_phy_status));) + fixed_phy_add(PHY_POLL, 1, &stmmac0_fixed_phy_status, -1); and the second one, with a real PHY device attached to the bus, by using the stmmac_mdio_bus_data structure (to provide the id, the reset procedure etc). @@ -278,8 +270,6 @@ capability register can replace what has been passed from the platform. Please see the following document: Documentation/devicetree/bindings/net/stmmac.txt -and the stmmac_of_data structure inside the include/linux/stmmac.h header file. - 4.11) This is a summary of the content of some relevant files: o stmmac_main.c: to implement the main network device driver; o stmmac_mdio.c: to provide mdio functions; diff --git a/Documentation/networking/switchdev.txt b/Documentation/networking/switchdev.txt index c5d7ade10ff2..476df0496686 100644 --- a/Documentation/networking/switchdev.txt +++ b/Documentation/networking/switchdev.txt @@ -279,8 +279,18 @@ and unknown unicast packets to all ports in domain, if allowed by port's current STP state. The switch driver, knowing which ports are within which vlan L2 domain, can program the switch device for flooding. The packet should also be sent to the port netdev for processing by the bridge driver. The -bridge should not reflood the packet to the same ports the device flooded. -XXX: the mechanism to avoid duplicate flood packets is being discuseed. +bridge should not reflood the packet to the same ports the device flooded, +otherwise there will be duplicate packets on the wire. + +To avoid duplicate packets, the device/driver should mark a packet as already +forwarded using skb->offload_fwd_mark. The same mark is set on the device +ports in the domain using dev->offload_fwd_mark. If the skb->offload_fwd_mark +is non-zero and matches the forwarding egress port's dev->skb_mark, the kernel +will drop the skb right before transmit on the egress port, with the +understanding that the device already forwarded the packet on same egress port. +The driver can use switchdev_port_fwd_mark_set() to set a globally unique mark +for port's dev->offload_fwd_mark, based on the port's parent ID (switch ID) and +a group ifindex. It is possible for the switch device to not handle flooding and push the packets up to the bridge driver for flooding. This is not ideal as the number @@ -357,4 +367,5 @@ driver's rocker_port_ipv4_resolve() for an example. The driver can monitor for updates to arp_tbl using the netevent notifier NETEVENT_NEIGH_UPDATE. The device can be programmed with resolved nexthops -for the routes as arp_tbl updates. +for the routes as arp_tbl updates. The driver implements ndo_neigh_destroy +to know when arp_tbl neighbor entries are purged from the port. diff --git a/Documentation/networking/timestamping.txt b/Documentation/networking/timestamping.txt index 5f0922613f1a..a977339fbe0a 100644 --- a/Documentation/networking/timestamping.txt +++ b/Documentation/networking/timestamping.txt @@ -359,6 +359,13 @@ the requested fine-grained filtering for incoming packets is not supported, the driver may time stamp more than just the requested types of packets. +Drivers are free to use a more permissive configuration than the requested +configuration. It is expected that drivers should only implement directly the +most generic mode that can be supported. For example if the hardware can +support HWTSTAMP_FILTER_V2_EVENT, then it should generally always upscale +HWTSTAMP_FILTER_V2_L2_SYNC_MESSAGE, and so forth, as HWTSTAMP_FILTER_V2_EVENT +is more generic (and more useful to applications). + A driver which supports hardware time stamping shall update the struct with the actual, possibly more permissive configuration. If the requested packets cannot be time stamped, then nothing should be diff --git a/Documentation/networking/vxlan.txt b/Documentation/networking/vxlan.txt index 6d993510f091..c28f4989c3f0 100644 --- a/Documentation/networking/vxlan.txt +++ b/Documentation/networking/vxlan.txt @@ -1,32 +1,36 @@ Virtual eXtensible Local Area Networking documentation ====================================================== -The VXLAN protocol is a tunnelling protocol that is designed to -solve the problem of limited number of available VLAN's (4096). -With VXLAN identifier is expanded to 24 bits. +The VXLAN protocol is a tunnelling protocol designed to solve the +problem of limited VLAN IDs (4096) in IEEE 802.1q. With VXLAN the +size of the identifier is expanded to 24 bits (16777216). -It is a draft RFC standard, that is implemented by Cisco Nexus, -Vmware and Brocade. The protocol runs over UDP using a single -destination port (still not standardized by IANA). -This document describes the Linux kernel tunnel device, -there is also an implantation of VXLAN for Openvswitch. +VXLAN is described by IETF RFC 7348, and has been implemented by a +number of vendors. The protocol runs over UDP using a single +destination port. This document describes the Linux kernel tunnel +device, there is also a separate implementation of VXLAN for +Openvswitch. -Unlike most tunnels, a VXLAN is a 1 to N network, not just point -to point. A VXLAN device can either dynamically learn the IP address -of the other end, in a manner similar to a learning bridge, or the -forwarding entries can be configured statically. +Unlike most tunnels, a VXLAN is a 1 to N network, not just point to +point. A VXLAN device can learn the IP address of the other endpoint +either dynamically in a manner similar to a learning bridge, or make +use of statically-configured forwarding entries. -The management of vxlan is done in a similar fashion to it's -too closest neighbors GRE and VLAN. Configuring VXLAN requires -the version of iproute2 that matches the kernel release -where VXLAN was first merged upstream. +The management of vxlan is done in a manner similar to its two closest +neighbors GRE and VLAN. Configuring VXLAN requires the version of +iproute2 that matches the kernel release where VXLAN was first merged +upstream. 1. Create vxlan device - # ip li add vxlan0 type vxlan id 42 group 239.1.1.1 dev eth1 + # ip link add vxlan0 type vxlan id 42 group 239.1.1.1 dev eth1 dstport 4789 -This creates a new device (vxlan0). The device uses the -the multicast group 239.1.1.1 over eth1 to handle packets where -no entry is in the forwarding table. +This creates a new device named vxlan0. The device uses the multicast +group 239.1.1.1 over eth1 to handle traffic for which there is no +entry in the forwarding table. The destination port number is set to +the IANA-assigned value of 4789. The Linux implementation of VXLAN +pre-dates the IANA's selection of a standard destination port number +and uses the Linux-selected value by default to maintain backwards +compatibility. 2. Delete vxlan device # ip link delete vxlan0 diff --git a/Documentation/pcmcia/locking.txt b/Documentation/pcmcia/locking.txt index 68f622bc4064..b2c9b478906b 100644 --- a/Documentation/pcmcia/locking.txt +++ b/Documentation/pcmcia/locking.txt @@ -96,7 +96,7 @@ or single-use fields not mentioned): 3. Per PCMCIA-device Data: -------------------------- -The "main" struct pcmcia_devie is protected as follows (read-only fields +The "main" struct pcmcia_device is protected as follows (read-only fields or single-use fields not mentioned): diff --git a/Documentation/power/devices.txt b/Documentation/power/devices.txt index d172bce0fd49..8ba6625fdd63 100644 --- a/Documentation/power/devices.txt +++ b/Documentation/power/devices.txt @@ -341,6 +341,13 @@ the phases are: and is entirely responsible for bringing the device back to the functional state as appropriate. + Note that this direct-complete procedure applies even if the device is + disabled for runtime PM; only the runtime-PM status matters. It follows + that if a device has system-sleep callbacks but does not support runtime + PM, then its prepare callback must never return a positive value. This + is because all devices are initially set to runtime-suspended with + runtime PM disabled. + 2. The suspend methods should quiesce the device to stop it from performing I/O. They also may save the device registers and put it into the appropriate low-power state, depending on the bus type the device is on, diff --git a/Documentation/power/runtime_pm.txt b/Documentation/power/runtime_pm.txt index e76dc0ad4d2b..0784bc3a2ab5 100644 --- a/Documentation/power/runtime_pm.txt +++ b/Documentation/power/runtime_pm.txt @@ -445,10 +445,6 @@ drivers/base/power/runtime.c and include/linux/pm_runtime.h: bool pm_runtime_status_suspended(struct device *dev); - return true if the device's runtime PM status is 'suspended' - bool pm_runtime_suspended_if_enabled(struct device *dev); - - return true if the device's runtime PM status is 'suspended' and its - 'power.disable_depth' field is equal to 1 - void pm_runtime_allow(struct device *dev); - set the power.runtime_auto flag for the device and decrease its usage counter (used by the /sys/devices/.../power/control interface to diff --git a/Documentation/powerpc/cxlflash.txt b/Documentation/powerpc/cxlflash.txt new file mode 100644 index 000000000000..4202d1bc583c --- /dev/null +++ b/Documentation/powerpc/cxlflash.txt @@ -0,0 +1,318 @@ +Introduction +============ + + The IBM Power architecture provides support for CAPI (Coherent + Accelerator Power Interface), which is available to certain PCIe slots + on Power 8 systems. CAPI can be thought of as a special tunneling + protocol through PCIe that allow PCIe adapters to look like special + purpose co-processors which can read or write an application's + memory and generate page faults. As a result, the host interface to + an adapter running in CAPI mode does not require the data buffers to + be mapped to the device's memory (IOMMU bypass) nor does it require + memory to be pinned. + + On Linux, Coherent Accelerator (CXL) kernel services present CAPI + devices as a PCI device by implementing a virtual PCI host bridge. + This abstraction simplifies the infrastructure and programming + model, allowing for drivers to look similar to other native PCI + device drivers. + + CXL provides a mechanism by which user space applications can + directly talk to a device (network or storage) bypassing the typical + kernel/device driver stack. The CXL Flash Adapter Driver enables a + user space application direct access to Flash storage. + + The CXL Flash Adapter Driver is a kernel module that sits in the + SCSI stack as a low level device driver (below the SCSI disk and + protocol drivers) for the IBM CXL Flash Adapter. This driver is + responsible for the initialization of the adapter, setting up the + special path for user space access, and performing error recovery. It + communicates directly the Flash Accelerator Functional Unit (AFU) + as described in Documentation/powerpc/cxl.txt. + + The cxlflash driver supports two, mutually exclusive, modes of + operation at the device (LUN) level: + + - Any flash device (LUN) can be configured to be accessed as a + regular disk device (i.e.: /dev/sdc). This is the default mode. + + - Any flash device (LUN) can be configured to be accessed from + user space with a special block library. This mode further + specifies the means of accessing the device and provides for + either raw access to the entire LUN (referred to as direct + or physical LUN access) or access to a kernel/AFU-mediated + partition of the LUN (referred to as virtual LUN access). The + segmentation of a disk device into virtual LUNs is assisted + by special translation services provided by the Flash AFU. + +Overview +======== + + The Coherent Accelerator Interface Architecture (CAIA) introduces a + concept of a master context. A master typically has special privileges + granted to it by the kernel or hypervisor allowing it to perform AFU + wide management and control. The master may or may not be involved + directly in each user I/O, but at the minimum is involved in the + initial setup before the user application is allowed to send requests + directly to the AFU. + + The CXL Flash Adapter Driver establishes a master context with the + AFU. It uses memory mapped I/O (MMIO) for this control and setup. The + Adapter Problem Space Memory Map looks like this: + + +-------------------------------+ + | 512 * 64 KB User MMIO | + | (per context) | + | User Accessible | + +-------------------------------+ + | 512 * 128 B per context | + | Provisioning and Control | + | Trusted Process accessible | + +-------------------------------+ + | 64 KB Global | + | Trusted Process accessible | + +-------------------------------+ + + This driver configures itself into the SCSI software stack as an + adapter driver. The driver is the only entity that is considered a + Trusted Process to program the Provisioning and Control and Global + areas in the MMIO Space shown above. The master context driver + discovers all LUNs attached to the CXL Flash adapter and instantiates + scsi block devices (/dev/sdb, /dev/sdc etc.) for each unique LUN + seen from each path. + + Once these scsi block devices are instantiated, an application + written to a specification provided by the block library may get + access to the Flash from user space (without requiring a system call). + + This master context driver also provides a series of ioctls for this + block library to enable this user space access. The driver supports + two modes for accessing the block device. + + The first mode is called a virtual mode. In this mode a single scsi + block device (/dev/sdb) may be carved up into any number of distinct + virtual LUNs. The virtual LUNs may be resized as long as the sum of + the sizes of all the virtual LUNs, along with the meta-data associated + with it does not exceed the physical capacity. + + The second mode is called the physical mode. In this mode a single + block device (/dev/sdb) may be opened directly by the block library + and the entire space for the LUN is available to the application. + + Only the physical mode provides persistence of the data. i.e. The + data written to the block device will survive application exit and + restart and also reboot. The virtual LUNs do not persist (i.e. do + not survive after the application terminates or the system reboots). + + +Block library API +================= + + Applications intending to get access to the CXL Flash from user + space should use the block library, as it abstracts the details of + interfacing directly with the cxlflash driver that are necessary for + performing administrative actions (i.e.: setup, tear down, resize). + The block library can be thought of as a 'user' of services, + implemented as IOCTLs, that are provided by the cxlflash driver + specifically for devices (LUNs) operating in user space access + mode. While it is not a requirement that applications understand + the interface between the block library and the cxlflash driver, + a high-level overview of each supported service (IOCTL) is provided + below. + + The block library can be found on GitHub: + http://www.github.com/mikehollinger/ibmcapikv + + +CXL Flash Driver IOCTLs +======================= + + Users, such as the block library, that wish to interface with a flash + device (LUN) via user space access need to use the services provided + by the cxlflash driver. As these services are implemented as ioctls, + a file descriptor handle must first be obtained in order to establish + the communication channel between a user and the kernel. This file + descriptor is obtained by opening the device special file associated + with the scsi disk device (/dev/sdb) that was created during LUN + discovery. As per the location of the cxlflash driver within the + SCSI protocol stack, this open is actually not seen by the cxlflash + driver. Upon successful open, the user receives a file descriptor + (herein referred to as fd1) that should be used for issuing the + subsequent ioctls listed below. + + The structure definitions for these IOCTLs are available in: + uapi/scsi/cxlflash_ioctl.h + +DK_CXLFLASH_ATTACH +------------------ + + This ioctl obtains, initializes, and starts a context using the CXL + kernel services. These services specify a context id (u16) by which + to uniquely identify the context and its allocated resources. The + services additionally provide a second file descriptor (herein + referred to as fd2) that is used by the block library to initiate + memory mapped I/O (via mmap()) to the CXL flash device and poll for + completion events. This file descriptor is intentionally installed by + this driver and not the CXL kernel services to allow for intermediary + notification and access in the event of a non-user-initiated close(), + such as a killed process. This design point is described in further + detail in the description for the DK_CXLFLASH_DETACH ioctl. + + There are a few important aspects regarding the "tokens" (context id + and fd2) that are provided back to the user: + + - These tokens are only valid for the process under which they + were created. The child of a forked process cannot continue + to use the context id or file descriptor created by its parent + (see DK_CXLFLASH_VLUN_CLONE for further details). + + - These tokens are only valid for the lifetime of the context and + the process under which they were created. Once either is + destroyed, the tokens are to be considered stale and subsequent + usage will result in errors. + + - When a context is no longer needed, the user shall detach from + the context via the DK_CXLFLASH_DETACH ioctl. + + - A close on fd2 will invalidate the tokens. This operation is not + required by the user. + +DK_CXLFLASH_USER_DIRECT +----------------------- + This ioctl is responsible for transitioning the LUN to direct + (physical) mode access and configuring the AFU for direct access from + user space on a per-context basis. Additionally, the block size and + last logical block address (LBA) are returned to the user. + + As mentioned previously, when operating in user space access mode, + LUNs may be accessed in whole or in part. Only one mode is allowed + at a time and if one mode is active (outstanding references exist), + requests to use the LUN in a different mode are denied. + + The AFU is configured for direct access from user space by adding an + entry to the AFU's resource handle table. The index of the entry is + treated as a resource handle that is returned to the user. The user + is then able to use the handle to reference the LUN during I/O. + +DK_CXLFLASH_USER_VIRTUAL +------------------------ + This ioctl is responsible for transitioning the LUN to virtual mode + of access and configuring the AFU for virtual access from user space + on a per-context basis. Additionally, the block size and last logical + block address (LBA) are returned to the user. + + As mentioned previously, when operating in user space access mode, + LUNs may be accessed in whole or in part. Only one mode is allowed + at a time and if one mode is active (outstanding references exist), + requests to use the LUN in a different mode are denied. + + The AFU is configured for virtual access from user space by adding + an entry to the AFU's resource handle table. The index of the entry + is treated as a resource handle that is returned to the user. The + user is then able to use the handle to reference the LUN during I/O. + + By default, the virtual LUN is created with a size of 0. The user + would need to use the DK_CXLFLASH_VLUN_RESIZE ioctl to adjust the grow + the virtual LUN to a desired size. To avoid having to perform this + resize for the initial creation of the virtual LUN, the user has the + option of specifying a size as part of the DK_CXLFLASH_USER_VIRTUAL + ioctl, such that when success is returned to the user, the + resource handle that is provided is already referencing provisioned + storage. This is reflected by the last LBA being a non-zero value. + +DK_CXLFLASH_VLUN_RESIZE +----------------------- + This ioctl is responsible for resizing a previously created virtual + LUN and will fail if invoked upon a LUN that is not in virtual + mode. Upon success, an updated last LBA is returned to the user + indicating the new size of the virtual LUN associated with the + resource handle. + + The partitioning of virtual LUNs is jointly mediated by the cxlflash + driver and the AFU. An allocation table is kept for each LUN that is + operating in the virtual mode and used to program a LUN translation + table that the AFU references when provided with a resource handle. + +DK_CXLFLASH_RELEASE +------------------- + This ioctl is responsible for releasing a previously obtained + reference to either a physical or virtual LUN. This can be + thought of as the inverse of the DK_CXLFLASH_USER_DIRECT or + DK_CXLFLASH_USER_VIRTUAL ioctls. Upon success, the resource handle + is no longer valid and the entry in the resource handle table is + made available to be used again. + + As part of the release process for virtual LUNs, the virtual LUN + is first resized to 0 to clear out and free the translation tables + associated with the virtual LUN reference. + +DK_CXLFLASH_DETACH +------------------ + This ioctl is responsible for unregistering a context with the + cxlflash driver and release outstanding resources that were + not explicitly released via the DK_CXLFLASH_RELEASE ioctl. Upon + success, all "tokens" which had been provided to the user from the + DK_CXLFLASH_ATTACH onward are no longer valid. + +DK_CXLFLASH_VLUN_CLONE +---------------------- + This ioctl is responsible for cloning a previously created + context to a more recently created context. It exists solely to + support maintaining user space access to storage after a process + forks. Upon success, the child process (which invoked the ioctl) + will have access to the same LUNs via the same resource handle(s) + and fd2 as the parent, but under a different context. + + Context sharing across processes is not supported with CXL and + therefore each fork must be met with establishing a new context + for the child process. This ioctl simplifies the state management + and playback required by a user in such a scenario. When a process + forks, child process can clone the parents context by first creating + a context (via DK_CXLFLASH_ATTACH) and then using this ioctl to + perform the clone from the parent to the child. + + The clone itself is fairly simple. The resource handle and lun + translation tables are copied from the parent context to the child's + and then synced with the AFU. + +DK_CXLFLASH_VERIFY +------------------ + This ioctl is used to detect various changes such as the capacity of + the disk changing, the number of LUNs visible changing, etc. In cases + where the changes affect the application (such as a LUN resize), the + cxlflash driver will report the changed state to the application. + + The user calls in when they want to validate that a LUN hasn't been + changed in response to a check condition. As the user is operating out + of band from the kernel, they will see these types of events without + the kernel's knowledge. When encountered, the user's architected + behavior is to call in to this ioctl, indicating what they want to + verify and passing along any appropriate information. For now, only + verifying a LUN change (ie: size different) with sense data is + supported. + +DK_CXLFLASH_RECOVER_AFU +----------------------- + This ioctl is used to drive recovery (if such an action is warranted) + of a specified user context. Any state associated with the user context + is re-established upon successful recovery. + + User contexts are put into an error condition when the device needs to + be reset or is terminating. Users are notified of this error condition + by seeing all 0xF's on an MMIO read. Upon encountering this, the + architected behavior for a user is to call into this ioctl to recover + their context. A user may also call into this ioctl at any time to + check if the device is operating normally. If a failure is returned + from this ioctl, the user is expected to gracefully clean up their + context via release/detach ioctls. Until they do, the context they + hold is not relinquished. The user may also optionally exit the process + at which time the context/resources they held will be freed as part of + the release fop. + +DK_CXLFLASH_MANAGE_LUN +---------------------- + This ioctl is used to switch a LUN from a mode where it is available + for file-system access (legacy), to a mode where it is set aside for + exclusive user space access (superpipe). In case a LUN is visible + across multiple ports and adapters, this ioctl is used to uniquely + identify each LUN by its World Wide Node Name (WWNN). diff --git a/Documentation/security/Smack.txt b/Documentation/security/Smack.txt index de5e1aeca7fb..5e6d07fbed07 100644 --- a/Documentation/security/Smack.txt +++ b/Documentation/security/Smack.txt @@ -28,6 +28,10 @@ Smack kernels use the CIPSO IP option. Some network configurations are intolerant of IP options and can impede access to systems that use them as Smack does. +Smack is used in the Tizen operating system. Please +go to http://wiki.tizen.org for information about how +Smack is used in Tizen. + The current git repository for Smack user space is: git://github.com/smack-team/smack.git @@ -108,6 +112,8 @@ in the smackfs filesystem. This pseudo-filesystem is mounted on /sys/fs/smackfs. access + Provided for backward compatibility. The access2 interface + is preferred and should be used instead. This interface reports whether a subject with the specified Smack label has a particular access to an object with a specified Smack label. Write a fixed format access rule to @@ -136,6 +142,8 @@ change-rule those in the fourth string. If there is no such rule it will be created using the access specified in the third and the fourth strings. cipso + Provided for backward compatibility. The cipso2 interface + is preferred and should be used instead. This interface allows a specific CIPSO header to be assigned to a Smack label. The format accepted on write is: "%24s%4d%4d"["%4d"]... @@ -157,7 +165,19 @@ direct doi This contains the CIPSO domain of interpretation used in network packets. +ipv6host + This interface allows specific IPv6 internet addresses to be + treated as single label hosts. Packets are sent to single + label hosts only from processes that have Smack write access + to the host label. All packets received from single label hosts + are given the specified label. The format accepted on write is: + "%h:%h:%h:%h:%h:%h:%h:%h label" or + "%h:%h:%h:%h:%h:%h:%h:%h/%d label". + The "::" address shortcut is not supported. + If label is "-DELETE" a matched entry will be deleted. load + Provided for backward compatibility. The load2 interface + is preferred and should be used instead. This interface allows access control rules in addition to the system defined rules to be specified. The format accepted on write is: @@ -181,6 +201,8 @@ load2 permissions that are not allowed. The string "r-x--" would specify read and execute access. load-self + Provided for backward compatibility. The load-self2 interface + is preferred and should be used instead. This interface allows process specific access rules to be defined. These rules are only consulted if access would otherwise be permitted, and are intended to provide additional @@ -205,6 +227,8 @@ netlabel received from single label hosts are given the specified label. The format accepted on write is: "%d.%d.%d.%d label" or "%d.%d.%d.%d/%d label". + If the label specified is "-CIPSO" the address is treated + as a host that supports CIPSO headers. onlycap This contains labels processes must have for CAP_MAC_ADMIN and CAP_MAC_OVERRIDE to be effective. If this file is empty @@ -232,7 +256,8 @@ unconfined is dangerous and can ruin the proper labeling of your system. It should never be used in production. -You can add access rules in /etc/smack/accesses. They take the form: +If you are using the smackload utility +you can add access rules in /etc/smack/accesses. They take the form: subjectlabel objectlabel access diff --git a/Documentation/security/Yama.txt b/Documentation/security/Yama.txt index 227a63f018a2..d9ee7d7a6c7f 100644 --- a/Documentation/security/Yama.txt +++ b/Documentation/security/Yama.txt @@ -1,9 +1,7 @@ -Yama is a Linux Security Module that collects a number of system-wide DAC -security protections that are not handled by the core kernel itself. To -select it at boot time, specify "security=yama" (though this will disable -any other LSM). - -Yama is controlled through sysctl in /proc/sys/kernel/yama: +Yama is a Linux Security Module that collects system-wide DAC security +protections that are not handled by the core kernel itself. This is +selectable at build-time with CONFIG_SECURITY_YAMA, and can be controlled +at run-time through sysctls in /proc/sys/kernel/yama: - ptrace_scope diff --git a/Documentation/static-keys.txt b/Documentation/static-keys.txt index c4407a41b0fc..f4cb0b2d5cd7 100644 --- a/Documentation/static-keys.txt +++ b/Documentation/static-keys.txt @@ -1,7 +1,22 @@ Static Keys ----------- -By: Jason Baron +DEPRECATED API: + +The use of 'struct static_key' directly, is now DEPRECATED. In addition +static_key_{true,false}() is also DEPRECATED. IE DO NOT use the following: + +struct static_key false = STATIC_KEY_INIT_FALSE; +struct static_key true = STATIC_KEY_INIT_TRUE; +static_key_true() +static_key_false() + +The updated API replacements are: + +DEFINE_STATIC_KEY_TRUE(key); +DEFINE_STATIC_KEY_FALSE(key); +static_key_likely() +statick_key_unlikely() 0) Abstract @@ -9,22 +24,22 @@ Static keys allows the inclusion of seldom used features in performance-sensitive fast-path kernel code, via a GCC feature and a code patching technique. A quick example: - struct static_key key = STATIC_KEY_INIT_FALSE; + DEFINE_STATIC_KEY_FALSE(key); ... - if (static_key_false(&key)) + if (static_branch_unlikely(&key)) do unlikely code else do likely code ... - static_key_slow_inc(); + static_branch_enable(&key); ... - static_key_slow_inc(); + static_branch_disable(&key); ... -The static_key_false() branch will be generated into the code with as little +The static_branch_unlikely() branch will be generated into the code with as little impact to the likely code path as possible. @@ -56,7 +71,7 @@ the branch site to change the branch direction. For example, if we have a simple branch that is disabled by default: - if (static_key_false(&key)) + if (static_branch_unlikely(&key)) printk("I am the true branch\n"); Thus, by default the 'printk' will not be emitted. And the code generated will @@ -75,68 +90,55 @@ the basis for the static keys facility. In order to make use of this optimization you must first define a key: - struct static_key key; - -Which is initialized as: - - struct static_key key = STATIC_KEY_INIT_TRUE; + DEFINE_STATIC_KEY_TRUE(key); or: - struct static_key key = STATIC_KEY_INIT_FALSE; + DEFINE_STATIC_KEY_FALSE(key); -If the key is not initialized, it is default false. The 'struct static_key', -must be a 'global'. That is, it can't be allocated on the stack or dynamically + +The key must be global, that is, it can't be allocated on the stack or dynamically allocated at run-time. The key is then used in code as: - if (static_key_false(&key)) + if (static_branch_unlikely(&key)) do unlikely code else do likely code Or: - if (static_key_true(&key)) + if (static_branch_likely(&key)) do likely code else do unlikely code -A key that is initialized via 'STATIC_KEY_INIT_FALSE', must be used in a -'static_key_false()' construct. Likewise, a key initialized via -'STATIC_KEY_INIT_TRUE' must be used in a 'static_key_true()' construct. A -single key can be used in many branches, but all the branches must match the -way that the key has been initialized. +Keys defined via DEFINE_STATIC_KEY_TRUE(), or DEFINE_STATIC_KEY_FALSE, may +be used in either static_branch_likely() or static_branch_unlikely() +statemnts. -The branch(es) can then be switched via: +Branch(es) can be set true via: - static_key_slow_inc(&key); +static_branch_enable(&key); + +or false via: + +static_branch_disable(&key); + +The branch(es) can then be switched via reference counts: + + static_branch_inc(&key); ... - static_key_slow_dec(&key); + static_branch_dec(&key); -Thus, 'static_key_slow_inc()' means 'make the branch true', and -'static_key_slow_dec()' means 'make the branch false' with appropriate +Thus, 'static_branch_inc()' means 'make the branch true', and +'static_branch_dec()' means 'make the branch false' with appropriate reference counting. For example, if the key is initialized true, a -static_key_slow_dec(), will switch the branch to false. And a subsequent -static_key_slow_inc(), will change the branch back to true. Likewise, if the -key is initialized false, a 'static_key_slow_inc()', will change the branch to -true. And then a 'static_key_slow_dec()', will again make the branch false. - -An example usage in the kernel is the implementation of tracepoints: - - static inline void trace_##name(proto) \ - { \ - if (static_key_false(&__tracepoint_##name.key)) \ - __DO_TRACE(&__tracepoint_##name, \ - TP_PROTO(data_proto), \ - TP_ARGS(data_args), \ - TP_CONDITION(cond)); \ - } - -Tracepoints are disabled by default, and can be placed in performance critical -pieces of the kernel. Thus, by using a static key, the tracepoints can have -absolutely minimal impact when not in use. +static_branch_dec(), will switch the branch to false. And a subsequent +static_branch_inc(), will change the branch back to true. Likewise, if the +key is initialized false, a 'static_branch_inc()', will change the branch to +true. And then a 'static_branch_dec()', will again make the branch false. 4) Architecture level code patching interface, 'jump labels' @@ -150,9 +152,12 @@ simply fall back to a traditional, load, test, and jump sequence. * #define JUMP_LABEL_NOP_SIZE, see: arch/x86/include/asm/jump_label.h -* __always_inline bool arch_static_branch(struct static_key *key), see: +* __always_inline bool arch_static_branch(struct static_key *key, bool branch), see: arch/x86/include/asm/jump_label.h +* __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch), + see: arch/x86/include/asm/jump_label.h + * void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type), see: arch/x86/kernel/jump_label.c @@ -173,7 +178,7 @@ SYSCALL_DEFINE0(getppid) { int pid; -+ if (static_key_false(&key)) ++ if (static_branch_unlikely(&key)) + printk("I am the true branch\n"); rcu_read_lock(); diff --git a/Documentation/sysctl/vm.txt b/Documentation/sysctl/vm.txt index 9c3f2f8054b5..a4482fceacec 100644 --- a/Documentation/sysctl/vm.txt +++ b/Documentation/sysctl/vm.txt @@ -349,7 +349,7 @@ zone[i]'s protection[j] is calculated by following expression. (i < j): zone[i]->protection[j] - = (total sums of present_pages from zone[i+1] to zone[j] on the node) + = (total sums of managed_pages from zone[i+1] to zone[j] on the node) / lowmem_reserve_ratio[i]; (i = j): (should not be protected. = 0; @@ -360,7 +360,7 @@ The default values of lowmem_reserve_ratio[i] are 256 (if zone[i] means DMA or DMA32 zone) 32 (others). As above expression, they are reciprocal number of ratio. -256 means 1/256. # of protection pages becomes about "0.39%" of total present +256 means 1/256. # of protection pages becomes about "0.39%" of total managed pages of higher zones on the node. If you would like to protect more pages, smaller values are effective. diff --git a/Documentation/sysrq.txt b/Documentation/sysrq.txt index 0e307c94809a..13f5619b2203 100644 --- a/Documentation/sysrq.txt +++ b/Documentation/sysrq.txt @@ -75,7 +75,8 @@ On all - write a character to /proc/sysrq-trigger. e.g.: 'e' - Send a SIGTERM to all processes, except for init. -'f' - Will call oom_kill to kill a memory hog process. +'f' - Will call the oom killer to kill a memory hog process, but do not + panic if nothing can be killed. 'g' - Used by kgdb (kernel debugger) @@ -119,6 +120,7 @@ On all - write a character to /proc/sysrq-trigger. e.g.: 'x' - Used by xmon interface on ppc/powerpc platforms. Show global PMU Registers on sparc64. + Dump all TLB entries on MIPS. 'y' - Show global CPU Registers [SPARC-64 specific] diff --git a/Documentation/thermal/sysfs-api.txt b/Documentation/thermal/sysfs-api.txt index c1f6864a8c5d..10f062ea6bc2 100644 --- a/Documentation/thermal/sysfs-api.txt +++ b/Documentation/thermal/sysfs-api.txt @@ -180,6 +180,7 @@ Thermal zone device sys I/F, created once it's registered: |---temp: Current temperature |---mode: Working mode of the thermal zone |---policy: Thermal governor used for this zone + |---available_policies: Available thermal governors for this zone |---trip_point_[0-*]_temp: Trip point temperature |---trip_point_[0-*]_type: Trip point type |---trip_point_[0-*]_hyst: Hysteresis value for this trip point @@ -256,6 +257,10 @@ policy One of the various thermal governors used for a particular zone. RW, Required +available_policies + Available thermal governors which can be used for a particular zone. + RO, Required + trip_point_[0-*]_temp The temperature above which trip point will be fired. Unit: millidegree Celsius @@ -417,6 +422,7 @@ method, the sys I/F structure will be built like this: |---temp: 37000 |---mode: enabled |---policy: step_wise + |---available_policies: step_wise fair_share |---trip_point_0_temp: 100000 |---trip_point_0_type: critical |---trip_point_1_temp: 80000 diff --git a/Documentation/trace/ftrace.txt b/Documentation/trace/ftrace.txt index 7ddb1e319f84..ef621d34ba5b 100644 --- a/Documentation/trace/ftrace.txt +++ b/Documentation/trace/ftrace.txt @@ -346,6 +346,11 @@ of ftrace. Here is a list of some of the key files: x86-tsc: Architectures may define their own clocks. For example, x86 uses its own TSC cycle clock here. + ppc-tb: This uses the powerpc timebase register value. + This is in sync across CPUs and can also be used + to correlate events across hypervisor/guest if + tb_offset is known. + To set a clock, simply echo the clock name into this file. echo global > trace_clock @@ -686,6 +691,8 @@ The above is mostly meaningful for kernel developers. The marks are determined by the difference between this current trace and the next trace. '$' - greater than 1 second + '@' - greater than 100 milisecond + '*' - greater than 10 milisecond '#' - greater than 1000 microsecond '!' - greater than 100 microsecond '+' - greater than 10 microsecond @@ -1939,26 +1946,49 @@ want, depending on your needs. ie: - 0) | up_write() { - 0) 0.646 us | _spin_lock_irqsave(); - 0) 0.684 us | _spin_unlock_irqrestore(); - 0) 3.123 us | } - 0) 0.548 us | fput(); - 0) + 58.628 us | } + 3) # 1837.709 us | } /* __switch_to */ + 3) | finish_task_switch() { + 3) 0.313 us | _raw_spin_unlock_irq(); + 3) 3.177 us | } + 3) # 1889.063 us | } /* __schedule */ + 3) ! 140.417 us | } /* __schedule */ + 3) # 2034.948 us | } /* schedule */ + 3) * 33998.59 us | } /* schedule_preempt_disabled */ [...] - 0) | putname() { - 0) | kmem_cache_free() { - 0) 0.518 us | __phys_addr(); - 0) 1.757 us | } - 0) 2.861 us | } - 0) ! 115.305 us | } - 0) ! 116.402 us | } + 1) 0.260 us | msecs_to_jiffies(); + 1) 0.313 us | __rcu_read_unlock(); + 1) + 61.770 us | } + 1) + 64.479 us | } + 1) 0.313 us | rcu_bh_qs(); + 1) 0.313 us | __local_bh_enable(); + 1) ! 217.240 us | } + 1) 0.365 us | idle_cpu(); + 1) | rcu_irq_exit() { + 1) 0.417 us | rcu_eqs_enter_common.isra.47(); + 1) 3.125 us | } + 1) ! 227.812 us | } + 1) ! 457.395 us | } + 1) @ 119760.2 us | } + + [...] + + 2) | handle_IPI() { + 1) 6.979 us | } + 2) 0.417 us | scheduler_ipi(); + 1) 9.791 us | } + 1) + 12.917 us | } + 2) 3.490 us | } + 1) + 15.729 us | } + 1) + 18.542 us | } + 2) $ 3594274 us | } + means that the function exceeded 10 usecs. ! means that the function exceeded 100 usecs. # means that the function exceeded 1000 usecs. + * means that the function exceeded 10 msecs. + @ means that the function exceeded 100 msecs. $ means that the function exceeded 1 sec. diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt index a4ebcb712375..d9ecceea5a02 100644 --- a/Documentation/virtual/kvm/api.txt +++ b/Documentation/virtual/kvm/api.txt @@ -2671,7 +2671,7 @@ handled. 4.87 KVM_SET_GUEST_DEBUG Capability: KVM_CAP_SET_GUEST_DEBUG -Architectures: x86, s390, ppc +Architectures: x86, s390, ppc, arm64 Type: vcpu ioctl Parameters: struct kvm_guest_debug (in) Returns: 0 on success; -1 on error @@ -2693,8 +2693,8 @@ when running. Common control bits are: The top 16 bits of the control field are architecture specific control flags which can include the following: - - KVM_GUESTDBG_USE_SW_BP: using software breakpoints [x86] - - KVM_GUESTDBG_USE_HW_BP: using hardware breakpoints [x86, s390] + - KVM_GUESTDBG_USE_SW_BP: using software breakpoints [x86, arm64] + - KVM_GUESTDBG_USE_HW_BP: using hardware breakpoints [x86, s390, arm64] - KVM_GUESTDBG_INJECT_DB: inject DB type exception [x86] - KVM_GUESTDBG_INJECT_BP: inject BP type exception [x86] - KVM_GUESTDBG_EXIT_PENDING: trigger an immediate guest exit [s390] @@ -2709,6 +2709,11 @@ updated to the correct (supplied) values. The second part of the structure is architecture specific and typically contains a set of debug registers. +For arm64 the number of debug registers is implementation defined and +can be determined by querying the KVM_CAP_GUEST_DEBUG_HW_BPS and +KVM_CAP_GUEST_DEBUG_HW_WPS capabilities which return a positive number +indicating the number of supported registers. + When debug events exit the main run loop with the reason KVM_EXIT_DEBUG with the kvm_debug_exit_arch part of the kvm_run structure containing architecture specific debug information. @@ -3111,11 +3116,13 @@ data_offset describes where the data is located (KVM_EXIT_IO_OUT) or where kvm expects application code to place the data for the next KVM_RUN invocation (KVM_EXIT_IO_IN). Data format is a packed array. + /* KVM_EXIT_DEBUG */ struct { struct kvm_debug_exit_arch arch; } debug; -Unused. +If the exit_reason is KVM_EXIT_DEBUG, then a vcpu is processing a debug event +for which architecture specific information is returned. /* KVM_EXIT_MMIO */ struct { diff --git a/Documentation/vm/00-INDEX b/Documentation/vm/00-INDEX index 081c49777abb..6a5e2a102a45 100644 --- a/Documentation/vm/00-INDEX +++ b/Documentation/vm/00-INDEX @@ -14,6 +14,8 @@ hugetlbpage.txt - a brief summary of hugetlbpage support in the Linux kernel. hwpoison.txt - explains what hwpoison is +idle_page_tracking.txt + - description of the idle page tracking feature. ksm.txt - how to use the Kernel Samepage Merging feature. numa diff --git a/Documentation/vm/hugetlbpage.txt b/Documentation/vm/hugetlbpage.txt index 030977fb8d2d..54dd9b9c6c31 100644 --- a/Documentation/vm/hugetlbpage.txt +++ b/Documentation/vm/hugetlbpage.txt @@ -329,7 +329,14 @@ Examples 3) hugepage-mmap: see tools/testing/selftests/vm/hugepage-mmap.c -4) The libhugetlbfs (http://libhugetlbfs.sourceforge.net) library provides a - wide range of userspace tools to help with huge page usability, environment - setup, and control. Furthermore it provides useful test cases that should be - used when modifying code to ensure no regressions are introduced. +4) The libhugetlbfs (https://github.com/libhugetlbfs/libhugetlbfs) library + provides a wide range of userspace tools to help with huge page usability, + environment setup, and control. + +Kernel development regression testing +===================================== + +The most complete set of hugetlb tests are in the libhugetlbfs repository. +If you modify any hugetlb related code, use the libhugetlbfs test suite +to check for regressions. In addition, if you add any new hugetlb +functionality, please add appropriate tests to libhugetlbfs. diff --git a/Documentation/vm/idle_page_tracking.txt b/Documentation/vm/idle_page_tracking.txt new file mode 100644 index 000000000000..85dcc3bb85dc --- /dev/null +++ b/Documentation/vm/idle_page_tracking.txt @@ -0,0 +1,98 @@ +MOTIVATION + +The idle page tracking feature allows to track which memory pages are being +accessed by a workload and which are idle. This information can be useful for +estimating the workload's working set size, which, in turn, can be taken into +account when configuring the workload parameters, setting memory cgroup limits, +or deciding where to place the workload within a compute cluster. + +It is enabled by CONFIG_IDLE_PAGE_TRACKING=y. + +USER API + +The idle page tracking API is located at /sys/kernel/mm/page_idle. Currently, +it consists of the only read-write file, /sys/kernel/mm/page_idle/bitmap. + +The file implements a bitmap where each bit corresponds to a memory page. The +bitmap is represented by an array of 8-byte integers, and the page at PFN #i is +mapped to bit #i%64 of array element #i/64, byte order is native. When a bit is +set, the corresponding page is idle. + +A page is considered idle if it has not been accessed since it was marked idle +(for more details on what "accessed" actually means see the IMPLEMENTATION +DETAILS section). To mark a page idle one has to set the bit corresponding to +the page by writing to the file. A value written to the file is OR-ed with the +current bitmap value. + +Only accesses to user memory pages are tracked. These are pages mapped to a +process address space, page cache and buffer pages, swap cache pages. For other +page types (e.g. SLAB pages) an attempt to mark a page idle is silently ignored, +and hence such pages are never reported idle. + +For huge pages the idle flag is set only on the head page, so one has to read +/proc/kpageflags in order to correctly count idle huge pages. + +Reading from or writing to /sys/kernel/mm/page_idle/bitmap will return +-EINVAL if you are not starting the read/write on an 8-byte boundary, or +if the size of the read/write is not a multiple of 8 bytes. Writing to +this file beyond max PFN will return -ENXIO. + +That said, in order to estimate the amount of pages that are not used by a +workload one should: + + 1. Mark all the workload's pages as idle by setting corresponding bits in + /sys/kernel/mm/page_idle/bitmap. The pages can be found by reading + /proc/pid/pagemap if the workload is represented by a process, or by + filtering out alien pages using /proc/kpagecgroup in case the workload is + placed in a memory cgroup. + + 2. Wait until the workload accesses its working set. + + 3. Read /sys/kernel/mm/page_idle/bitmap and count the number of bits set. If + one wants to ignore certain types of pages, e.g. mlocked pages since they + are not reclaimable, he or she can filter them out using /proc/kpageflags. + +See Documentation/vm/pagemap.txt for more information about /proc/pid/pagemap, +/proc/kpageflags, and /proc/kpagecgroup. + +IMPLEMENTATION DETAILS + +The kernel internally keeps track of accesses to user memory pages in order to +reclaim unreferenced pages first on memory shortage conditions. A page is +considered referenced if it has been recently accessed via a process address +space, in which case one or more PTEs it is mapped to will have the Accessed bit +set, or marked accessed explicitly by the kernel (see mark_page_accessed()). The +latter happens when: + + - a userspace process reads or writes a page using a system call (e.g. read(2) + or write(2)) + + - a page that is used for storing filesystem buffers is read or written, + because a process needs filesystem metadata stored in it (e.g. lists a + directory tree) + + - a page is accessed by a device driver using get_user_pages() + +When a dirty page is written to swap or disk as a result of memory reclaim or +exceeding the dirty memory limit, it is not marked referenced. + +The idle memory tracking feature adds a new page flag, the Idle flag. This flag +is set manually, by writing to /sys/kernel/mm/page_idle/bitmap (see the USER API +section), and cleared automatically whenever a page is referenced as defined +above. + +When a page is marked idle, the Accessed bit must be cleared in all PTEs it is +mapped to, otherwise we will not be able to detect accesses to the page coming +from a process address space. To avoid interference with the reclaimer, which, +as noted above, uses the Accessed bit to promote actively referenced pages, one +more page flag is introduced, the Young flag. When the PTE Accessed bit is +cleared as a result of setting or updating a page's Idle flag, the Young flag +is set on the page. The reclaimer treats the Young flag as an extra PTE +Accessed bit and therefore will consider such a page as referenced. + +Since the idle memory tracking feature is based on the memory reclaimer logic, +it only works with pages that are on an LRU list, other pages are silently +ignored. That means it will ignore a user memory page if it is isolated, but +since there are usually not many of them, it should not affect the overall +result noticeably. In order not to stall scanning of the idle page bitmap, +locked pages may be skipped too. diff --git a/Documentation/vm/pagemap.txt b/Documentation/vm/pagemap.txt index 6bfbc172cdb9..0e1e55588b59 100644 --- a/Documentation/vm/pagemap.txt +++ b/Documentation/vm/pagemap.txt @@ -5,7 +5,7 @@ pagemap is a new (as of 2.6.25) set of interfaces in the kernel that allow userspace programs to examine the page tables and related information by reading files in /proc. -There are three components to pagemap: +There are four components to pagemap: * /proc/pid/pagemap. This file lets a userspace process find out which physical frame each virtual page is mapped to. It contains one 64-bit @@ -16,11 +16,17 @@ There are three components to pagemap: * Bits 0-4 swap type if swapped * Bits 5-54 swap offset if swapped * Bit 55 pte is soft-dirty (see Documentation/vm/soft-dirty.txt) - * Bits 56-60 zero - * Bit 61 page is file-page or shared-anon + * Bit 56 page exclusively mapped (since 4.2) + * Bits 57-60 zero + * Bit 61 page is file-page or shared-anon (since 3.5) * Bit 62 page swapped * Bit 63 page present + Since Linux 4.0 only users with the CAP_SYS_ADMIN capability can get PFNs. + In 4.0 and 4.1 opens by unprivileged fail with -EPERM. Starting from + 4.2 the PFN field is zeroed if the user does not have CAP_SYS_ADMIN. + Reason: information about PFNs helps in exploiting Rowhammer vulnerability. + If the page is not present but in swap, then the PFN contains an encoding of the swap file number and the page's offset into the swap. Unmapped pages return a null PFN. This allows determining @@ -64,6 +70,11 @@ There are three components to pagemap: 22. THP 23. BALLOON 24. ZERO_PAGE + 25. IDLE + + * /proc/kpagecgroup. This file contains a 64-bit inode number of the + memory cgroup each page is charged to, indexed by PFN. Only available when + CONFIG_MEMCG is set. Short descriptions to the page flags: @@ -110,6 +121,12 @@ Short descriptions to the page flags: 24. ZERO_PAGE zero page for pfn_zero or huge_zero page +25. IDLE + page has not been accessed since it was marked idle (see + Documentation/vm/idle_page_tracking.txt). Note that this flag may be + stale in case the page was accessed via a PTE. To make sure the flag + is up-to-date one has to read /sys/kernel/mm/page_idle/bitmap first. + [IO related page flags] 1. ERROR IO error occurred 3. UPTODATE page has up-to-date data @@ -159,3 +176,8 @@ Other notes: Reading from any of the files will return -EINVAL if you are not starting the read on an 8-byte boundary (e.g., if you sought an odd number of bytes into the file), or if the size of the read is not a multiple of 8 bytes. + +Before Linux 3.11 pagemap bits 55-60 were used for "page-shift" (which is +always 12 at most architectures). Since Linux 3.11 their meaning changes +after first clear of soft-dirty bits. Since Linux 4.2 they are used for +flags unconditionally. diff --git a/Documentation/vm/userfaultfd.txt b/Documentation/vm/userfaultfd.txt new file mode 100644 index 000000000000..70a3c94d1941 --- /dev/null +++ b/Documentation/vm/userfaultfd.txt @@ -0,0 +1,144 @@ += Userfaultfd = + +== Objective == + +Userfaults allow the implementation of on-demand paging from userland +and more generally they allow userland to take control of various +memory page faults, something otherwise only the kernel code could do. + +For example userfaults allows a proper and more optimal implementation +of the PROT_NONE+SIGSEGV trick. + +== Design == + +Userfaults are delivered and resolved through the userfaultfd syscall. + +The userfaultfd (aside from registering and unregistering virtual +memory ranges) provides two primary functionalities: + +1) read/POLLIN protocol to notify a userland thread of the faults + happening + +2) various UFFDIO_* ioctls that can manage the virtual memory regions + registered in the userfaultfd that allows userland to efficiently + resolve the userfaults it receives via 1) or to manage the virtual + memory in the background + +The real advantage of userfaults if compared to regular virtual memory +management of mremap/mprotect is that the userfaults in all their +operations never involve heavyweight structures like vmas (in fact the +userfaultfd runtime load never takes the mmap_sem for writing). + +Vmas are not suitable for page- (or hugepage) granular fault tracking +when dealing with virtual address spaces that could span +Terabytes. Too many vmas would be needed for that. + +The userfaultfd once opened by invoking the syscall, can also be +passed using unix domain sockets to a manager process, so the same +manager process could handle the userfaults of a multitude of +different processes without them being aware about what is going on +(well of course unless they later try to use the userfaultfd +themselves on the same region the manager is already tracking, which +is a corner case that would currently return -EBUSY). + +== API == + +When first opened the userfaultfd must be enabled invoking the +UFFDIO_API ioctl specifying a uffdio_api.api value set to UFFD_API (or +a later API version) which will specify the read/POLLIN protocol +userland intends to speak on the UFFD and the uffdio_api.features +userland requires. The UFFDIO_API ioctl if successful (i.e. if the +requested uffdio_api.api is spoken also by the running kernel and the +requested features are going to be enabled) will return into +uffdio_api.features and uffdio_api.ioctls two 64bit bitmasks of +respectively all the available features of the read(2) protocol and +the generic ioctl available. + +Once the userfaultfd has been enabled the UFFDIO_REGISTER ioctl should +be invoked (if present in the returned uffdio_api.ioctls bitmask) to +register a memory range in the userfaultfd by setting the +uffdio_register structure accordingly. The uffdio_register.mode +bitmask will specify to the kernel which kind of faults to track for +the range (UFFDIO_REGISTER_MODE_MISSING would track missing +pages). The UFFDIO_REGISTER ioctl will return the +uffdio_register.ioctls bitmask of ioctls that are suitable to resolve +userfaults on the range registered. Not all ioctls will necessarily be +supported for all memory types depending on the underlying virtual +memory backend (anonymous memory vs tmpfs vs real filebacked +mappings). + +Userland can use the uffdio_register.ioctls to manage the virtual +address space in the background (to add or potentially also remove +memory from the userfaultfd registered range). This means a userfault +could be triggering just before userland maps in the background the +user-faulted page. + +The primary ioctl to resolve userfaults is UFFDIO_COPY. That +atomically copies a page into the userfault registered range and wakes +up the blocked userfaults (unless uffdio_copy.mode & +UFFDIO_COPY_MODE_DONTWAKE is set). Other ioctl works similarly to +UFFDIO_COPY. They're atomic as in guaranteeing that nothing can see an +half copied page since it'll keep userfaulting until the copy has +finished. + +== QEMU/KVM == + +QEMU/KVM is using the userfaultfd syscall to implement postcopy live +migration. Postcopy live migration is one form of memory +externalization consisting of a virtual machine running with part or +all of its memory residing on a different node in the cloud. The +userfaultfd abstraction is generic enough that not a single line of +KVM kernel code had to be modified in order to add postcopy live +migration to QEMU. + +Guest async page faults, FOLL_NOWAIT and all other GUP features work +just fine in combination with userfaults. Userfaults trigger async +page faults in the guest scheduler so those guest processes that +aren't waiting for userfaults (i.e. network bound) can keep running in +the guest vcpus. + +It is generally beneficial to run one pass of precopy live migration +just before starting postcopy live migration, in order to avoid +generating userfaults for readonly guest regions. + +The implementation of postcopy live migration currently uses one +single bidirectional socket but in the future two different sockets +will be used (to reduce the latency of the userfaults to the minimum +possible without having to decrease /proc/sys/net/ipv4/tcp_wmem). + +The QEMU in the source node writes all pages that it knows are missing +in the destination node, into the socket, and the migration thread of +the QEMU running in the destination node runs UFFDIO_COPY|ZEROPAGE +ioctls on the userfaultfd in order to map the received pages into the +guest (UFFDIO_ZEROCOPY is used if the source page was a zero page). + +A different postcopy thread in the destination node listens with +poll() to the userfaultfd in parallel. When a POLLIN event is +generated after a userfault triggers, the postcopy thread read() from +the userfaultfd and receives the fault address (or -EAGAIN in case the +userfault was already resolved and waken by a UFFDIO_COPY|ZEROPAGE run +by the parallel QEMU migration thread). + +After the QEMU postcopy thread (running in the destination node) gets +the userfault address it writes the information about the missing page +into the socket. The QEMU source node receives the information and +roughly "seeks" to that page address and continues sending all +remaining missing pages from that new page offset. Soon after that +(just the time to flush the tcp_wmem queue through the network) the +migration thread in the QEMU running in the destination node will +receive the page that triggered the userfault and it'll map it as +usual with the UFFDIO_COPY|ZEROPAGE (without actually knowing if it +was spontaneously sent by the source or if it was an urgent page +requested through an userfault). + +By the time the userfaults start, the QEMU in the destination node +doesn't need to keep any per-page state bitmap relative to the live +migration around and a single per-page bitmap has to be maintained in +the QEMU running in the source node to know which pages are still +missing in the destination node. The bitmap in the source node is +checked to find which missing pages to send in round robin and we seek +over it when receiving incoming userfaults. After sending each page of +course the bitmap is updated accordingly. It's also useful to avoid +sending the same page twice (in case the userfault is read by the +postcopy thread just before UFFDIO_COPY|ZEROPAGE runs in the migration +thread). diff --git a/Documentation/vm/zswap.txt b/Documentation/vm/zswap.txt index 8458c0861e4e..89fff7d611cc 100644 --- a/Documentation/vm/zswap.txt +++ b/Documentation/vm/zswap.txt @@ -32,7 +32,7 @@ can also be enabled and disabled at runtime using the sysfs interface. An example command to enable zswap at runtime, assuming sysfs is mounted at /sys, is: -echo 1 > /sys/modules/zswap/parameters/enabled +echo 1 > /sys/module/zswap/parameters/enabled When zswap is disabled at runtime it will stop storing pages that are being swapped out. However, it will _not_ immediately write out or fault @@ -49,14 +49,26 @@ Zswap receives pages for compression through the Frontswap API and is able to evict pages from its own compressed pool on an LRU basis and write them back to the backing swap device in the case that the compressed pool is full. -Zswap makes use of zbud for the managing the compressed memory pool. Each -allocation in zbud is not directly accessible by address. Rather, a handle is +Zswap makes use of zpool for the managing the compressed memory pool. Each +allocation in zpool is not directly accessible by address. Rather, a handle is returned by the allocation routine and that handle must be mapped before being accessed. The compressed memory pool grows on demand and shrinks as compressed -pages are freed. The pool is not preallocated. +pages are freed. The pool is not preallocated. By default, a zpool of type +zbud is created, but it can be selected at boot time by setting the "zpool" +attribute, e.g. zswap.zpool=zbud. It can also be changed at runtime using the +sysfs "zpool" attribute, e.g. + +echo zbud > /sys/module/zswap/parameters/zpool + +The zbud type zpool allocates exactly 1 page to store 2 compressed pages, which +means the compression ratio will always be 2:1 or worse (because of half-full +zbud pages). The zsmalloc type zpool has a more complex compressed page +storage method, and it can achieve greater storage densities. However, +zsmalloc does not implement compressed page eviction, so once zswap fills it +cannot evict the oldest page, it can only reject new pages. When a swap page is passed from frontswap to zswap, zswap maintains a mapping -of the swap entry, a combination of the swap type and swap offset, to the zbud +of the swap entry, a combination of the swap type and swap offset, to the zpool handle that references that compressed swap page. This mapping is achieved with a red-black tree per swap type. The swap offset is the search key for the tree nodes. @@ -74,9 +86,17 @@ controlled policy: * max_pool_percent - The maximum percentage of memory that the compressed pool can occupy. -Zswap allows the compressor to be selected at kernel boot time by setting the -“compressor” attribute. The default compressor is lzo. e.g. -zswap.compressor=deflate +The default compressor is lzo, but it can be selected at boot time by setting +the “compressor” attribute, e.g. zswap.compressor=lzo. It can also be changed +at runtime using the sysfs "compressor" attribute, e.g. + +echo lzo > /sys/module/zswap/parameters/compressor + +When the zpool and/or compressor parameter is changed at runtime, any existing +compressed pages are not modified; they are left in their own zpool. When a +request is made for a page in an old zpool, it is uncompressed using its +original compressor. Once all pages are removed from an old zpool, the zpool +and its compressor are freed. A debugfs interface is provided for various statistic about pool size, number of pages stored, and various counters for the reasons pages are rejected. diff --git a/Documentation/watchdog/src/watchdog-test.c b/Documentation/watchdog/src/watchdog-test.c index 3da822967ee0..fcdde8fc98be 100644 --- a/Documentation/watchdog/src/watchdog-test.c +++ b/Documentation/watchdog/src/watchdog-test.c @@ -41,6 +41,7 @@ static void term(int sig) int main(int argc, char *argv[]) { int flags; + unsigned int ping_rate = 1; fd = open("/dev/watchdog", O_WRONLY); @@ -63,22 +64,33 @@ int main(int argc, char *argv[]) fprintf(stderr, "Watchdog card enabled.\n"); fflush(stderr); goto end; + } else if (!strncasecmp(argv[1], "-t", 2) && argv[2]) { + flags = atoi(argv[2]); + ioctl(fd, WDIOC_SETTIMEOUT, &flags); + fprintf(stderr, "Watchdog timeout set to %u seconds.\n", flags); + fflush(stderr); + goto end; + } else if (!strncasecmp(argv[1], "-p", 2) && argv[2]) { + ping_rate = strtoul(argv[2], NULL, 0); + fprintf(stderr, "Watchdog ping rate set to %u seconds.\n", ping_rate); + fflush(stderr); } else { - fprintf(stderr, "-d to disable, -e to enable.\n"); + fprintf(stderr, "-d to disable, -e to enable, -t to set " \ + "the timeout,\n-p to set the ping rate, and \n"); fprintf(stderr, "run by itself to tick the card.\n"); fflush(stderr); goto end; } - } else { - fprintf(stderr, "Watchdog Ticking Away!\n"); - fflush(stderr); } + fprintf(stderr, "Watchdog Ticking Away!\n"); + fflush(stderr); + signal(SIGINT, term); while(1) { keep_alive(); - sleep(1); + sleep(ping_rate); } end: close(fd); diff --git a/MAINTAINERS b/MAINTAINERS index 248b46b223b6..7ba7ab749c85 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -158,6 +158,7 @@ L: linux-wpan@vger.kernel.org S: Maintained F: net/6lowpan/ F: include/net/6lowpan.h +F: Documentation/networking/6lowpan.txt 6PACK NETWORK DRIVER FOR AX.25 M: Andreas Koensgen @@ -361,7 +362,7 @@ S: Supported F: drivers/input/touchscreen/ad7879.c ADDRESS SPACE LAYOUT RANDOMIZATION (ASLR) -M: Jiri Kosina +M: Jiri Kosina S: Maintained ADM1025 HARDWARE MONITOR DRIVER @@ -642,9 +643,14 @@ M: Oded Gabbay L: dri-devel@lists.freedesktop.org T: git git://people.freedesktop.org/~gabbayo/linux.git S: Supported +F: drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +F: drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +F: drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c +F: drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c F: drivers/gpu/drm/amd/amdkfd/ F: drivers/gpu/drm/amd/include/cik_structs.h F: drivers/gpu/drm/amd/include/kgd_kfd_interface.h +F: drivers/gpu/drm/amd/include/vi_structs.h F: drivers/gpu/drm/radeon/radeon_kfd.c F: drivers/gpu/drm/radeon/radeon_kfd.h F: include/uapi/linux/kfd_ioctl.h @@ -734,6 +740,12 @@ X: drivers/iio/*/adjd* F: drivers/staging/iio/*/ad* F: staging/iio/trigger/iio-trig-bfin-timer.c +ANALOG DEVICES INC DMA DRIVERS +M: Lars-Peter Clausen +W: http://ez.analog.com/community/linux-device-drivers +S: Supported +F: drivers/dma/dma-axi-dmac.c + ANDROID DRIVERS M: Greg Kroah-Hartman M: Arve Hjønnevåg @@ -752,7 +764,7 @@ S: Maintained F: sound/aoa/ APM DRIVER -M: Jiri Kosina +M: Jiri Kosina S: Odd fixes F: arch/x86/kernel/apm_32.c F: include/linux/apm_bios.h @@ -805,11 +817,13 @@ F: arch/arm/include/asm/floppy.h ARM PMU PROFILING AND DEBUGGING M: Will Deacon S: Maintained -F: arch/arm/kernel/perf_event* +F: arch/arm/kernel/perf_* F: arch/arm/oprofile/common.c -F: arch/arm/include/asm/pmu.h F: arch/arm/kernel/hw_breakpoint.c F: arch/arm/include/asm/hw_breakpoint.h +F: arch/arm/include/asm/perf_event.h +F: drivers/perf/arm_pmu.c +F: include/linux/perf/arm_pmu.h ARM PORT M: Russell King @@ -933,7 +947,7 @@ M: Sunil Goutham M: Robert Richter L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Supported -F: drivers/net/ethernet/cavium/ +F: drivers/net/ethernet/cavium/thunder/ ARM/CIRRUS LOGIC CLPS711X ARM ARCHITECTURE M: Alexander Shiyan @@ -1508,8 +1522,10 @@ S: Maintained F: arch/arm/mach-sti/ F: arch/arm/boot/dts/sti* F: drivers/clocksource/arm_global_timer.c +F: drivers/clocksource/clksrc_st_lpc.c F: drivers/i2c/busses/i2c-st.c F: drivers/media/rc/st_rc.c +F: drivers/media/platform/sti/c8sectpfe/ F: drivers/mmc/host/sdhci-st.c F: drivers/phy/phy-miphy28lp.c F: drivers/phy/phy-miphy365x.c @@ -1889,6 +1905,12 @@ L: linux-mtd@lists.infradead.org S: Supported F: drivers/mtd/nand/atmel_nand* +ATMEL SDMMC DRIVER +M: Ludovic Desroches +L: linux-mmc@vger.kernel.org +S: Supported +F: drivers/mmc/host/sdhci-of-at91.c + ATMEL SPI DRIVER M: Nicolas Ferre S: Supported @@ -1922,6 +1944,14 @@ W: http://atmelwlandriver.sourceforge.net/ S: Maintained F: drivers/net/wireless/atmel* +ATMEL MAXTOUCH DRIVER +M: Nick Dyer +T: git git://github.com/atmel-maxtouch/linux.git +S: Supported +F: Documentation/devicetree/bindings/input/atmel,maxtouch.txt +F: drivers/input/touchscreen/atmel_mxt_ts.c +F: include/linux/platform_data/atmel_mxt_ts.h + ATTO EXPRESSSAS SAS/SATA RAID SCSI DRIVER M: Bradley Grove L: linux-scsi@vger.kernel.org @@ -2551,7 +2581,6 @@ M: Raghu Vatsavayi L: netdev@vger.kernel.org W: http://www.cavium.com S: Supported -F: drivers/net/ethernet/cavium/ F: drivers/net/ethernet/cavium/liquidio/ CC2520 IEEE-802.15.4 RADIO DRIVER @@ -2598,6 +2627,15 @@ S: Supported F: Documentation/filesystems/ceph.txt F: fs/ceph/ +CERTIFICATE HANDLING: +M: David Howells +M: David Woodhouse +L: keyrings@linux-nfs.org +S: Maintained +F: Documentation/module-signing.txt +F: certs/ +F: scripts/extract-cert.c + CERTIFIED WIRELESS USB (WUSB) SUBSYSTEM: L: linux-usb@vger.kernel.org S: Orphan @@ -3560,6 +3598,15 @@ F: drivers/gpu/drm/exynos/ F: include/drm/exynos* F: include/uapi/drm/exynos* +DRM DRIVERS FOR FREESCALE DCU +M: Jianwei Wang +M: Alison Wang +L: dri-devel@lists.freedesktop.org +S: Supported +F: drivers/gpu/drm/fsl-dcu/ +F: Documentation/devicetree/bindings/video/fsl,dcu.txt +F: Documentation/devicetree/bindings/panel/nec,nl4827hc19_05b.txt + DRM DRIVERS FOR FREESCALE IMX M: Philipp Zabel L: dri-devel@lists.freedesktop.org @@ -4078,15 +4125,6 @@ F: Documentation/filesystems/ext2.txt F: fs/ext2/ F: include/linux/ext2* -EXT3 FILE SYSTEM -M: Jan Kara -M: Andrew Morton -M: Andreas Dilger -L: linux-ext4@vger.kernel.org -S: Maintained -F: Documentation/filesystems/ext3.txt -F: fs/ext3/ - EXT4 FILE SYSTEM M: "Theodore Ts'o" M: Andreas Dilger @@ -4264,7 +4302,7 @@ S: Maintained F: drivers/block/rsxx/ FLOPPY DRIVER -M: Jiri Kosina +M: Jiri Kosina T: git git://git.kernel.org/pub/scm/linux/kernel/git/jikos/floppy.git S: Odd fixes F: drivers/block/floppy.c @@ -4425,6 +4463,7 @@ F: include/linux/fscache*.h F2FS FILE SYSTEM M: Jaegeuk Kim M: Changman Lee +R: Chao Yu L: linux-f2fs-devel@lists.sourceforge.net W: http://en.wikipedia.org/wiki/F2FS T: git git://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git @@ -4433,6 +4472,7 @@ F: Documentation/filesystems/f2fs.txt F: Documentation/ABI/testing/sysfs-fs-f2fs F: fs/f2fs/ F: include/linux/f2fs_fs.h +F: include/trace/events/f2fs.h FUJITSU FR-V (FRV) PORT M: David Howells @@ -4835,7 +4875,7 @@ F: include/linux/pm.h F: arch/*/include/asm/suspend*.h HID CORE LAYER -M: Jiri Kosina +M: Jiri Kosina L: linux-input@vger.kernel.org T: git git://git.kernel.org/pub/scm/linux/kernel/git/jikos/hid.git S: Maintained @@ -4844,7 +4884,7 @@ F: include/linux/hid* F: include/uapi/linux/hid* HID SENSOR HUB DRIVERS -M: Jiri Kosina +M: Jiri Kosina M: Jonathan Cameron M: Srinivas Pandruvada L: linux-input@vger.kernel.org @@ -5301,6 +5341,7 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/dledford/rdma.git S: Supported F: Documentation/infiniband/ F: drivers/infiniband/ +F: drivers/staging/rdma/ F: include/uapi/linux/if_infiniband.h F: include/uapi/rdma/ F: include/rdma/ @@ -5558,7 +5599,7 @@ IPATH DRIVER M: Mike Marciniszyn L: linux-rdma@vger.kernel.org S: Maintained -F: drivers/infiniband/hw/ipath/ +F: drivers/staging/rdma/ipath/ IPMI SUBSYSTEM M: Corey Minyard @@ -5597,7 +5638,7 @@ F: include/uapi/linux/ip_vs.h F: net/netfilter/ipvs/ IPWIRELESS DRIVER -M: Jiri Kosina +M: Jiri Kosina M: David Sterba S: Odd Fixes F: drivers/tty/ipwireless/ @@ -5787,21 +5828,20 @@ S: Maintained F: fs/jffs2/ F: include/uapi/linux/jffs2.h -JOURNALLING LAYER FOR BLOCK DEVICES (JBD) -M: Andrew Morton -M: Jan Kara -L: linux-ext4@vger.kernel.org -S: Maintained -F: fs/jbd/ -F: include/linux/jbd.h - JOURNALLING LAYER FOR BLOCK DEVICES (JBD2) M: "Theodore Ts'o" +M: Jan Kara L: linux-ext4@vger.kernel.org S: Maintained F: fs/jbd2/ F: include/linux/jbd2.h +JPU V4L2 MEM2MEM DRIVER FOR RENESAS +M: Mikhail Ulyanov +L: linux-media@vger.kernel.org +S: Maintained +F: drivers/media/platform/rcar_jpu.c + JSM Neo PCI based serial card M: Thadeu Lima de Souza Cascardo L: linux-serial@vger.kernel.org @@ -5970,7 +6010,7 @@ F: kernel/kexec.c KEYS/KEYRINGS: M: David Howells -L: keyrings@linux-nfs.org +L: keyrings@vger.kernel.org S: Maintained F: Documentation/security/keys.txt F: include/linux/key.h @@ -5982,7 +6022,7 @@ KEYS-TRUSTED M: David Safford M: Mimi Zohar L: linux-security-module@vger.kernel.org -L: keyrings@linux-nfs.org +L: keyrings@vger.kernel.org S: Supported F: Documentation/security/keys-trusted-encrypted.txt F: include/keys/trusted-type.h @@ -5993,7 +6033,7 @@ KEYS-ENCRYPTED M: Mimi Zohar M: David Safford L: linux-security-module@vger.kernel.org -L: keyrings@linux-nfs.org +L: keyrings@vger.kernel.org S: Supported F: Documentation/security/keys-trusted-encrypted.txt F: include/keys/encrypted-type.h @@ -6196,6 +6236,7 @@ Q: https://patchwork.kernel.org/project/linux-nvdimm/list/ S: Supported F: drivers/nvdimm/pmem.c F: include/linux/pmem.h +F: arch/*/include/asm/pmem.h LINUX FOR IBM pSERIES (RS/6000) M: Paul Mackerras @@ -6286,7 +6327,7 @@ F: drivers/platform/x86/hp_accel.c LIVE PATCHING M: Josh Poimboeuf M: Seth Jennings -M: Jiri Kosina +M: Jiri Kosina M: Vojtech Pavlik S: Maintained F: kernel/livepatch/ @@ -6541,7 +6582,7 @@ F: drivers/net/ethernet/marvell/mvneta.* MARVELL MWIFIEX WIRELESS DRIVER M: Amitkumar Karwar -M: Avinash Patil +M: Nishant Sarmukadam L: linux-wireless@vger.kernel.org S: Maintained F: drivers/net/wireless/mwifiex/ @@ -6600,6 +6641,14 @@ S: Supported F: drivers/power/max14577_charger.c F: drivers/power/max77693_charger.c +MAXIM MAX77802 MULTIFUNCTION PMIC DEVICE DRIVERS +M: Javier Martinez Canillas +L: linux-kernel@vger.kernel.org +S: Supported +F: drivers/*/*max77802.c +F: Documentation/devicetree/bindings/*/*max77802.txt +F: include/dt-bindings/*/*max77802.h + MAXIM PMIC AND MUIC DRIVERS FOR EXYNOS BASED BOARDS M: Chanwoo Choi M: Krzysztof Kozlowski @@ -6613,7 +6662,7 @@ F: drivers/extcon/extcon-max77693.c F: drivers/rtc/rtc-max77686.c F: drivers/clk/clk-max77686.c F: Documentation/devicetree/bindings/mfd/max14577.txt -F: Documentation/devicetree/bindings/mfd/max77686.txt +F: Documentation/devicetree/bindings/*/max77686.txt F: Documentation/devicetree/bindings/mfd/max77693.txt F: Documentation/devicetree/bindings/clock/maxim,max77686.txt F: include/linux/mfd/max14577*.h @@ -6637,6 +6686,51 @@ S: Supported F: Documentation/devicetree/bindings/media/renesas,vsp1.txt F: drivers/media/platform/vsp1/ +MEDIA DRIVERS FOR ASCOT2E +M: Sergey Kozlov +L: linux-media@vger.kernel.org +W: http://linuxtv.org +W: http://netup.tv/ +T: git git://linuxtv.org/media_tree.git +S: Supported +F: drivers/media/dvb-frontends/ascot2e* + +MEDIA DRIVERS FOR CXD2841ER +M: Sergey Kozlov +L: linux-media@vger.kernel.org +W: http://linuxtv.org/ +W: http://netup.tv/ +T: git git://linuxtv.org/media_tree.git +S: Supported +F: drivers/media/dvb-frontends/cxd2841er* + +MEDIA DRIVERS FOR HORUS3A +M: Sergey Kozlov +L: linux-media@vger.kernel.org +W: http://linuxtv.org/ +W: http://netup.tv/ +T: git git://linuxtv.org/media_tree.git +S: Supported +F: drivers/media/dvb-frontends/horus3a* + +MEDIA DRIVERS FOR LNBH25 +M: Sergey Kozlov +L: linux-media@vger.kernel.org +W: http://linuxtv.org/ +W: http://netup.tv/ +T: git git://linuxtv.org/media_tree.git +S: Supported +F: drivers/media/dvb-frontends/lnbh25* + +MEDIA DRIVERS FOR NETUP PCI UNIVERSAL DVB devices +M: Sergey Kozlov +L: linux-media@vger.kernel.org +W: http://linuxtv.org/ +W: http://netup.tv/ +T: git git://linuxtv.org/media_tree.git +S: Supported +F: drivers/media/pci/netup_unidvb/* + MEDIA INPUT INFRASTRUCTURE (V4L/DVB) M: Mauro Carvalho Chehab P: LinuxTV.org Project @@ -6686,6 +6780,23 @@ W: http://www.mellanox.com Q: http://patchwork.ozlabs.org/project/netdev/list/ F: drivers/net/ethernet/mellanox/mlx4/en_* +MELLANOX ETHERNET SWITCH DRIVERS +M: Jiri Pirko +M: Ido Schimmel +L: netdev@vger.kernel.org +S: Supported +W: http://www.mellanox.com +Q: http://patchwork.ozlabs.org/project/netdev/list/ +F: drivers/net/ethernet/mellanox/mlxsw/ + +MEMBARRIER SUPPORT +M: Mathieu Desnoyers +M: "Paul E. McKenney" +L: linux-kernel@vger.kernel.org +S: Supported +F: kernel/membarrier.c +F: include/uapi/linux/membarrier.h + MEMORY MANAGEMENT L: linux-mm@kvack.org W: http://www.linux-mm.org @@ -6751,6 +6862,12 @@ T: git git://git.monstr.eu/linux-2.6-microblaze.git S: Supported F: arch/microblaze/ +MICROSOFT SURFACE PRO 3 BUTTON DRIVER +M: Chen Yu +L: platform-driver-x86@vger.kernel.org +S: Supported +F: drivers/platform/x86/surfacepro3_button.c + MICROTEK X6 SCANNER M: Oliver Neukum S: Maintained @@ -7268,7 +7385,7 @@ F: drivers/scsi/nsp32* NIOS2 ARCHITECTURE M: Ley Foon Tan L: nios2-dev@lists.rocketboards.org (moderated for non-subscribers) -T: git git://git.rocketboards.org/linux-socfpga-next.git +T: git git://git.kernel.org/pub/scm/linux/kernel/git/lftan/nios2.git S: Maintained F: arch/nios2/ @@ -7287,6 +7404,7 @@ NTB DRIVER CORE M: Jon Mason M: Dave Jiang M: Allen Hubbe +L: linux-ntb@googlegroups.com S: Supported W: https://github.com/jonmason/ntb/wiki T: git git://github.com/jonmason/ntb.git @@ -7298,6 +7416,7 @@ F: include/linux/ntb_transport.h NTB INTEL DRIVER M: Jon Mason M: Dave Jiang +L: linux-ntb@googlegroups.com S: Supported W: https://github.com/jonmason/ntb/wiki T: git git://github.com/jonmason/ntb.git @@ -7573,8 +7692,9 @@ F: Documentation/i2c/busses/i2c-ocores F: drivers/i2c/busses/i2c-ocores.c OPEN FIRMWARE AND FLATTENED DEVICE TREE -M: Grant Likely M: Rob Herring +M: Frank Rowand +M: Grant Likely L: devicetree@vger.kernel.org W: http://www.devicetree.org/ T: git git://git.kernel.org/pub/scm/linux/kernel/git/glikely/linux.git @@ -8047,7 +8167,6 @@ F: drivers/pinctrl/sh-pfc/ PIN CONTROLLER - SAMSUNG M: Tomasz Figa -M: Thomas Abraham L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) L: linux-samsung-soc@vger.kernel.org (moderated for non-subscribers) S: Maintained @@ -8062,7 +8181,7 @@ S: Maintained F: drivers/pinctrl/spear/ PKTCDVD DRIVER -M: Jiri Kosina +M: Jiri Kosina S: Maintained F: drivers/block/pktcdvd.c F: include/linux/pktcdvd.h @@ -8090,14 +8209,13 @@ F: drivers/hwmon/pmbus/ F: include/linux/i2c/pmbus.h PMC SIERRA MaxRAID DRIVER -M: Anil Ravindranath L: linux-scsi@vger.kernel.org W: http://www.pmc-sierra.com/ -S: Supported +S: Orphan F: drivers/scsi/pmcraid.* PMC SIERRA PM8001 DRIVER -M: xjtuwjp@gmail.com +M: Jack Wang M: lindar_liu@usish.com L: pmchba@pmcs.com L: linux-scsi@vger.kernel.org @@ -8124,6 +8242,15 @@ F: include/linux/power_supply.h F: drivers/power/ X: drivers/power/avs/ +POWER STATE COORDINATION INTERFACE (PSCI) +M: Mark Rutland +M: Lorenzo Pieralisi +L: linux-arm-kernel@lists.infradead.org +S: Maintained +F: drivers/firmware/psci.c +F: include/linux/psci.h +F: include/uapi/linux/psci.h + PNP SUPPORT M: "Rafael J. Wysocki" S: Maintained @@ -8907,6 +9034,12 @@ L: linux-media@vger.kernel.org S: Supported F: drivers/media/i2c/s5k5baf.c +SAMSUNG S3FWRN5 NFC DRIVER +M: Robert Baldyga +L: linux-nfc@lists.01.org (moderated for non-subscribers) +S: Supported +F: drivers/nfc/s3fwrn5 + SAMSUNG SOC CLOCK DRIVERS M: Sylwester Nawrocki M: Tomasz Figa @@ -8957,6 +9090,13 @@ F: include/linux/dma/dw.h F: include/linux/platform_data/dma-dw.h F: drivers/dma/dw/ +SYNOPSYS DESIGNWARE ETHERNET QOS 4.10a driver +M: Lars Persson +L: netdev@vger.kernel.org +S: Supported +F: Documentation/devicetree/bindings/net/snps,dwc-qos-ethernet.txt +F: drivers/net/ethernet/synopsys/dwc_eth_qos.c + SYNOPSYS DESIGNWARE MMC/SD/SDIO DRIVER M: Seungwon Jeon M: Jaehoon Chung @@ -9156,6 +9296,12 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/jj/apparmor-dev.git S: Supported F: security/apparmor/ +YAMA SECURITY MODULE +M: Kees Cook +T: git git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git yama/tip +S: Supported +F: security/yama/ + SENSABLE PHANTOM M: Jiri Slaby S: Maintained @@ -9840,6 +9986,12 @@ M: Arnaud Patard S: Odd Fixes F: drivers/staging/xgifb/ +HFI1 DRIVER +M: Mike Marciniszyn +L: linux-rdma@vger.kernel.org +S: Supported +F: drivers/staging/rdma/hfi1 + STARFIRE/DURALAN NETWORK DRIVER M: Ion Badulescu S: Odd Fixes @@ -10365,13 +10517,19 @@ F: drivers/platform/x86/toshiba_haps.c TOSHIBA SMM DRIVER M: Jonathan Buzzard -L: tlinux-users@tce.toshiba-dme.co.jp W: http://www.buzzard.org.uk/toshiba/ S: Maintained F: drivers/char/toshiba.c F: include/linux/toshiba.h F: include/uapi/linux/toshiba.h +TOSHIBA TC358743 DRIVER +M: Mats Randgaard +L: linux-media@vger.kernel.org +S: Maintained +F: drivers/media/i2c/tc358743* +F: include/media/tc358743.h + TMIO MMC DRIVER M: Ian Molton L: linux-mmc@vger.kernel.org @@ -10668,7 +10826,7 @@ F: drivers/usb/gadget/ F: include/linux/usb/gadget* USB HID/HIDBP DRIVERS (USB KEYBOARDS, MICE, REMOTE CONTROLS, ...) -M: Jiri Kosina +M: Jiri Kosina L: linux-usb@vger.kernel.org T: git git://git.kernel.org/pub/scm/linux/kernel/git/jikos/hid.git S: Maintained @@ -11063,7 +11221,7 @@ F: drivers/input/mouse/vmmouse.c F: drivers/input/mouse/vmmouse.h VMWARE VMXNET3 ETHERNET DRIVER -M: Shreyas Bhatewara +M: Shrikrishna Khare M: "VMware, Inc." L: netdev@vger.kernel.org S: Maintained @@ -11088,6 +11246,14 @@ S: Supported F: drivers/regulator/ F: include/linux/regulator/ +VRF +M: David Ahern +M: Shrijeet Mukherjee +L: netdev@vger.kernel.org +S: Maintained +F: drivers/net/vrf.c +F: include/net/vrf.h + VT1211 HARDWARE MONITOR DRIVER M: Juerg Haefliger L: lm-sensors@lm-sensors.org @@ -11243,6 +11409,7 @@ F: sound/soc/codecs/wm* WORKQUEUE M: Tejun Heo +R: Lai Jiangshan T: git git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq.git S: Maintained F: include/linux/workqueue.h diff --git a/Makefile b/Makefile index c3615937df38..1a132ea43ca5 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ VERSION = 4 -PATCHLEVEL = 2 +PATCHLEVEL = 3 SUBLEVEL = 0 -EXTRAVERSION = +EXTRAVERSION = -rc1 NAME = Hurr durr I'ma sheep # *DOCUMENTATION* @@ -666,14 +666,7 @@ endif endif KBUILD_CFLAGS += $(stackp-flag) -ifeq ($(shell $(CC) -v 2>&1 | grep -c "clang version"), 1) -COMPILER := clang -else -COMPILER := gcc -endif -export COMPILER - -ifeq ($(COMPILER),clang) +ifeq ($(cc-name),clang) KBUILD_CPPFLAGS += $(call cc-option,-Qunused-arguments,) KBUILD_CPPFLAGS += $(call cc-option,-Wno-unknown-warning-option,) KBUILD_CFLAGS += $(call cc-disable-warning, unused-variable) @@ -875,10 +868,9 @@ INITRD_COMPRESS-$(CONFIG_RD_LZ4) := lz4 # export INITRD_COMPRESS := $(INITRD_COMPRESS-y) ifdef CONFIG_MODULE_SIG_ALL -MODSECKEY = ./signing_key.priv -MODPUBKEY = ./signing_key.x509 -export MODPUBKEY -mod_sign_cmd = perl $(srctree)/scripts/sign-file $(CONFIG_MODULE_SIG_HASH) $(MODSECKEY) $(MODPUBKEY) +$(eval $(call config_filename,MODULE_SIG_KEY)) + +mod_sign_cmd = scripts/sign-file $(CONFIG_MODULE_SIG_HASH) $(MODULE_SIG_KEY_SRCPREFIX)$(CONFIG_MODULE_SIG_KEY) certs/signing_key.x509 else mod_sign_cmd = true endif @@ -886,7 +878,7 @@ export mod_sign_cmd ifeq ($(KBUILD_EXTMOD),) -core-y += kernel/ mm/ fs/ ipc/ security/ crypto/ block/ +core-y += kernel/ certs/ mm/ fs/ ipc/ security/ crypto/ block/ vmlinux-dirs := $(patsubst %/,%,$(filter %/, $(init-y) $(init-m) \ $(core-y) $(core-m) $(drivers-y) $(drivers-m) \ @@ -1178,8 +1170,8 @@ MRPROPER_DIRS += include/config usr/include include/generated \ arch/*/include/generated .tmp_objdiff MRPROPER_FILES += .config .config.old .version .old_version \ Module.symvers tags TAGS cscope* GPATH GTAGS GRTAGS GSYMS \ - signing_key.priv signing_key.x509 x509.genkey \ - extra_certificates signing_key.x509.keyid \ + signing_key.pem signing_key.priv signing_key.x509 \ + x509.genkey extra_certificates signing_key.x509.keyid \ signing_key.x509.signer vmlinux-gdb.py # clean - Delete most, but leave enough to build external modules @@ -1433,6 +1425,7 @@ clean: $(clean-dirs) \( -name '*.[oas]' -o -name '*.ko' -o -name '.*.cmd' \ -o -name '*.ko.*' \ -o -name '*.dwo' \ + -o -name '*.su' \ -o -name '.*.d' -o -name '.*.tmp' -o -name '*.mod.c' \ -o -name '*.symtypes' -o -name 'modules.order' \ -o -name modules.builtin -o -name '.tmp_*.o.*' \ diff --git a/arch/Kconfig b/arch/Kconfig index 8a8ea7110de8..4e949e58b192 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -2,6 +2,9 @@ # General architecture dependent options # +config KEXEC_CORE + bool + config OPROFILE tristate "OProfile system profiling" depends on PROFILING @@ -71,6 +74,12 @@ config JUMP_LABEL ( On 32-bit x86, the necessary options added to the compiler flags may increase the size of the kernel slightly. ) +config STATIC_KEYS_SELFTEST + bool "Static key selftest" + depends on JUMP_LABEL + help + Boot time self-test of the branch patching code. + config OPTPROBES def_bool y depends on KPROBES && HAVE_OPTPROBES @@ -87,7 +96,6 @@ config KPROBES_ON_FTRACE config UPROBES def_bool n - select PERCPU_RWSEM help Uprobes is the user-space counterpart to kprobes: they enable instrumentation applications (such as 'perf probe') diff --git a/arch/alpha/include/asm/atomic.h b/arch/alpha/include/asm/atomic.h index 8f8eafbedd7c..e8c956098424 100644 --- a/arch/alpha/include/asm/atomic.h +++ b/arch/alpha/include/asm/atomic.h @@ -29,13 +29,13 @@ * branch back to restart the operation. */ -#define ATOMIC_OP(op) \ +#define ATOMIC_OP(op, asm_op) \ static __inline__ void atomic_##op(int i, atomic_t * v) \ { \ unsigned long temp; \ __asm__ __volatile__( \ "1: ldl_l %0,%1\n" \ - " " #op "l %0,%2,%0\n" \ + " " #asm_op " %0,%2,%0\n" \ " stl_c %0,%1\n" \ " beq %0,2f\n" \ ".subsection 2\n" \ @@ -45,15 +45,15 @@ static __inline__ void atomic_##op(int i, atomic_t * v) \ :"Ir" (i), "m" (v->counter)); \ } \ -#define ATOMIC_OP_RETURN(op) \ +#define ATOMIC_OP_RETURN(op, asm_op) \ static inline int atomic_##op##_return(int i, atomic_t *v) \ { \ long temp, result; \ smp_mb(); \ __asm__ __volatile__( \ "1: ldl_l %0,%1\n" \ - " " #op "l %0,%3,%2\n" \ - " " #op "l %0,%3,%0\n" \ + " " #asm_op " %0,%3,%2\n" \ + " " #asm_op " %0,%3,%0\n" \ " stl_c %0,%1\n" \ " beq %0,2f\n" \ ".subsection 2\n" \ @@ -65,13 +65,13 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ return result; \ } -#define ATOMIC64_OP(op) \ +#define ATOMIC64_OP(op, asm_op) \ static __inline__ void atomic64_##op(long i, atomic64_t * v) \ { \ unsigned long temp; \ __asm__ __volatile__( \ "1: ldq_l %0,%1\n" \ - " " #op "q %0,%2,%0\n" \ + " " #asm_op " %0,%2,%0\n" \ " stq_c %0,%1\n" \ " beq %0,2f\n" \ ".subsection 2\n" \ @@ -81,15 +81,15 @@ static __inline__ void atomic64_##op(long i, atomic64_t * v) \ :"Ir" (i), "m" (v->counter)); \ } \ -#define ATOMIC64_OP_RETURN(op) \ +#define ATOMIC64_OP_RETURN(op, asm_op) \ static __inline__ long atomic64_##op##_return(long i, atomic64_t * v) \ { \ long temp, result; \ smp_mb(); \ __asm__ __volatile__( \ "1: ldq_l %0,%1\n" \ - " " #op "q %0,%3,%2\n" \ - " " #op "q %0,%3,%0\n" \ + " " #asm_op " %0,%3,%2\n" \ + " " #asm_op " %0,%3,%0\n" \ " stq_c %0,%1\n" \ " beq %0,2f\n" \ ".subsection 2\n" \ @@ -101,15 +101,27 @@ static __inline__ long atomic64_##op##_return(long i, atomic64_t * v) \ return result; \ } -#define ATOMIC_OPS(opg) \ - ATOMIC_OP(opg) \ - ATOMIC_OP_RETURN(opg) \ - ATOMIC64_OP(opg) \ - ATOMIC64_OP_RETURN(opg) +#define ATOMIC_OPS(op) \ + ATOMIC_OP(op, op##l) \ + ATOMIC_OP_RETURN(op, op##l) \ + ATOMIC64_OP(op, op##q) \ + ATOMIC64_OP_RETURN(op, op##q) ATOMIC_OPS(add) ATOMIC_OPS(sub) +#define atomic_andnot atomic_andnot +#define atomic64_andnot atomic64_andnot + +ATOMIC_OP(and, and) +ATOMIC_OP(andnot, bic) +ATOMIC_OP(or, bis) +ATOMIC_OP(xor, xor) +ATOMIC64_OP(and, and) +ATOMIC64_OP(andnot, bic) +ATOMIC64_OP(or, bis) +ATOMIC64_OP(xor, xor) + #undef ATOMIC_OPS #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP diff --git a/arch/alpha/include/asm/dma-mapping.h b/arch/alpha/include/asm/dma-mapping.h index dfa32f061320..72a8ca7796d9 100644 --- a/arch/alpha/include/asm/dma-mapping.h +++ b/arch/alpha/include/asm/dma-mapping.h @@ -12,42 +12,6 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev) #include -#define dma_alloc_coherent(d,s,h,f) dma_alloc_attrs(d,s,h,f,NULL) - -static inline void *dma_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, - struct dma_attrs *attrs) -{ - return get_dma_ops(dev)->alloc(dev, size, dma_handle, gfp, attrs); -} - -#define dma_free_coherent(d,s,c,h) dma_free_attrs(d,s,c,h,NULL) - -static inline void dma_free_attrs(struct device *dev, size_t size, - void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) -{ - get_dma_ops(dev)->free(dev, size, vaddr, dma_handle, attrs); -} - -static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - return get_dma_ops(dev)->mapping_error(dev, dma_addr); -} - -static inline int dma_supported(struct device *dev, u64 mask) -{ - return get_dma_ops(dev)->dma_supported(dev, mask); -} - -static inline int dma_set_mask(struct device *dev, u64 mask) -{ - return get_dma_ops(dev)->set_dma_mask(dev, mask); -} - -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) -#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) - #define dma_cache_sync(dev, va, size, dir) ((void)0) #endif /* _ALPHA_DMA_MAPPING_H */ diff --git a/arch/alpha/kernel/irq.c b/arch/alpha/kernel/irq.c index 51f2c8654253..2804648c8ff4 100644 --- a/arch/alpha/kernel/irq.c +++ b/arch/alpha/kernel/irq.c @@ -59,7 +59,7 @@ int irq_select_affinity(unsigned int irq) cpu = (cpu < (NR_CPUS-1) ? cpu + 1 : 0); last_cpu = cpu; - cpumask_copy(data->affinity, cpumask_of(cpu)); + cpumask_copy(irq_data_get_affinity_mask(data), cpumask_of(cpu)); chip->irq_set_affinity(data, cpumask_of(cpu), false); return 0; } diff --git a/arch/alpha/kernel/pci-noop.c b/arch/alpha/kernel/pci-noop.c index df24b76f9246..2b1f4a1e9272 100644 --- a/arch/alpha/kernel/pci-noop.c +++ b/arch/alpha/kernel/pci-noop.c @@ -166,15 +166,6 @@ static int alpha_noop_supported(struct device *dev, u64 mask) return mask < 0x00ffffffUL ? 0 : 1; } -static int alpha_noop_set_mask(struct device *dev, u64 mask) -{ - if (!dev->dma_mask || !dma_supported(dev, mask)) - return -EIO; - - *dev->dma_mask = mask; - return 0; -} - struct dma_map_ops alpha_noop_ops = { .alloc = alpha_noop_alloc_coherent, .free = alpha_noop_free_coherent, @@ -182,7 +173,6 @@ struct dma_map_ops alpha_noop_ops = { .map_sg = alpha_noop_map_sg, .mapping_error = alpha_noop_mapping_error, .dma_supported = alpha_noop_supported, - .set_dma_mask = alpha_noop_set_mask, }; struct dma_map_ops *dma_ops = &alpha_noop_ops; diff --git a/arch/alpha/kernel/pci_iommu.c b/arch/alpha/kernel/pci_iommu.c index eddee7720343..8969bf2dfe3a 100644 --- a/arch/alpha/kernel/pci_iommu.c +++ b/arch/alpha/kernel/pci_iommu.c @@ -939,16 +939,6 @@ static int alpha_pci_mapping_error(struct device *dev, dma_addr_t dma_addr) return dma_addr == 0; } -static int alpha_pci_set_mask(struct device *dev, u64 mask) -{ - if (!dev->dma_mask || - !pci_dma_supported(alpha_gendev_to_pci(dev), mask)) - return -EIO; - - *dev->dma_mask = mask; - return 0; -} - struct dma_map_ops alpha_pci_ops = { .alloc = alpha_pci_alloc_coherent, .free = alpha_pci_free_coherent, @@ -958,7 +948,6 @@ struct dma_map_ops alpha_pci_ops = { .unmap_sg = alpha_pci_unmap_sg, .mapping_error = alpha_pci_mapping_error, .dma_supported = alpha_pci_supported, - .set_dma_mask = alpha_pci_set_mask, }; struct dma_map_ops *dma_ops = &alpha_pci_ops; diff --git a/arch/alpha/kernel/time.c b/arch/alpha/kernel/time.c index 643a9dcdf093..5b6202a825ff 100644 --- a/arch/alpha/kernel/time.c +++ b/arch/alpha/kernel/time.c @@ -93,7 +93,7 @@ rtc_timer_interrupt(int irq, void *dev) struct clock_event_device *ce = &per_cpu(cpu_ce, cpu); /* Don't run the hook for UNUSED or SHUTDOWN. */ - if (likely(ce->mode == CLOCK_EVT_MODE_PERIODIC)) + if (likely(clockevent_state_periodic(ce))) ce->event_handler(ce); if (test_irq_work_pending()) { @@ -104,13 +104,6 @@ rtc_timer_interrupt(int irq, void *dev) return IRQ_HANDLED; } -static void -rtc_ce_set_mode(enum clock_event_mode mode, struct clock_event_device *ce) -{ - /* The mode member of CE is updated in generic code. - Since we only support periodic events, nothing to do. */ -} - static int rtc_ce_set_next_event(unsigned long evt, struct clock_event_device *ce) { @@ -129,7 +122,6 @@ init_rtc_clockevent(void) .features = CLOCK_EVT_FEAT_PERIODIC, .rating = 100, .cpumask = cpumask_of(cpu), - .set_mode = rtc_ce_set_mode, .set_next_event = rtc_ce_set_next_event, }; @@ -161,12 +153,12 @@ static struct clocksource qemu_cs = { * The QEMU alarm as a clock_event_device primitive. */ -static void -qemu_ce_set_mode(enum clock_event_mode mode, struct clock_event_device *ce) +static int qemu_ce_shutdown(struct clock_event_device *ce) { /* The mode member of CE is updated for us in generic code. Just make sure that the event is disabled. */ qemu_set_alarm_abs(0); + return 0; } static int @@ -197,7 +189,9 @@ init_qemu_clockevent(void) .features = CLOCK_EVT_FEAT_ONESHOT, .rating = 400, .cpumask = cpumask_of(cpu), - .set_mode = qemu_ce_set_mode, + .set_state_shutdown = qemu_ce_shutdown, + .set_state_oneshot = qemu_ce_shutdown, + .tick_resume = qemu_ce_shutdown, .set_next_event = qemu_ce_set_next_event, }; diff --git a/arch/arc/include/asm/atomic.h b/arch/arc/include/asm/atomic.h index 87d18ae53115..c3ecda023e3a 100644 --- a/arch/arc/include/asm/atomic.h +++ b/arch/arc/include/asm/atomic.h @@ -172,9 +172,13 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add, +=, add) ATOMIC_OPS(sub, -=, sub) -ATOMIC_OP(and, &=, and) -#define atomic_clear_mask(mask, v) atomic_and(~(mask), (v)) +#define atomic_andnot atomic_andnot + +ATOMIC_OP(and, &=, and) +ATOMIC_OP(andnot, &= ~, bic) +ATOMIC_OP(or, |=, or) +ATOMIC_OP(xor, ^=, xor) #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN diff --git a/arch/arc/kernel/mcip.c b/arch/arc/kernel/mcip.c index 2fb86589054d..d9e44b62df05 100644 --- a/arch/arc/kernel/mcip.c +++ b/arch/arc/kernel/mcip.c @@ -252,9 +252,10 @@ static struct irq_chip idu_irq_chip = { static int idu_first_irq; -static void idu_cascade_isr(unsigned int core_irq, struct irq_desc *desc) +static void idu_cascade_isr(unsigned int __core_irq, struct irq_desc *desc) { struct irq_domain *domain = irq_desc_get_handler_data(desc); + unsigned int core_irq = irq_desc_get_irq(desc); unsigned int idu_irq; idu_irq = core_irq - idu_first_irq; diff --git a/arch/arc/plat-axs10x/axs10x.c b/arch/arc/plat-axs10x/axs10x.c index ad9825d4026a..0a77b19e1df8 100644 --- a/arch/arc/plat-axs10x/axs10x.c +++ b/arch/arc/plat-axs10x/axs10x.c @@ -402,6 +402,8 @@ static void __init axs103_early_init(void) unsigned int num_cores = (read_aux_reg(ARC_REG_MCIP_BCR) >> 16) & 0x3F; if (num_cores > 2) arc_set_core_freq(50 * 1000000); + else if (num_cores == 2) + arc_set_core_freq(75 * 1000000); #endif switch (arc_get_core_freq()/1000000) { diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 41cbb4a53066..72ad724c67ae 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -188,6 +188,9 @@ config ARCH_HAS_ILOG2_U64 config ARCH_HAS_BANDGAP bool +config FIX_EARLYCON_MEM + def_bool y if MMU + config GENERIC_HWEIGHT bool default y @@ -1496,6 +1499,7 @@ config HOTPLUG_CPU config ARM_PSCI bool "Support for the ARM Power State Coordination Interface (PSCI)" depends on CPU_V7 + select ARM_PSCI_FW help Say Y here if you want Linux to communicate with system firmware implementing the PSCI specification for CPU-centric power @@ -1700,13 +1704,24 @@ config HIGHPTE consumed by page tables. Setting this option will allow user-space 2nd level page tables to reside in high memory. -config HW_PERF_EVENTS - bool "Enable hardware performance counter support for perf events" - depends on PERF_EVENTS +config CPU_SW_DOMAIN_PAN + bool "Enable use of CPU domains to implement privileged no-access" + depends on MMU && !ARM_LPAE default y help - Enable hardware performance counter support for perf events. If - disabled, perf events will use software events only. + Increase kernel security by ensuring that normal kernel accesses + are unable to access userspace addresses. This can help prevent + use-after-free bugs becoming an exploitable privilege escalation + by ensuring that magic values (such as LIST_POISON) will always + fault when dereferenced. + + CPUs with low-vector mappings use a best-efforts implementation. + Their lower 1MB needs to remain accessible for the vectors, but + the remainder of userspace will become appropriately inaccessible. + +config HW_PERF_EVENTS + def_bool y + depends on ARM_PMU config SYS_SUPPORTS_HUGETLBFS def_bool y @@ -2005,6 +2020,7 @@ config KEXEC bool "Kexec system call (EXPERIMENTAL)" depends on (!SMP || PM_SLEEP_SMP) depends on !CPU_V7M + select KEXEC_CORE help kexec is a system call that implements the ability to shutdown your current kernel, and to start another kernel. It is like a reboot diff --git a/arch/arm/boot/compressed/decompress.c b/arch/arm/boot/compressed/decompress.c index bd245d34952d..a0765e7ed6c7 100644 --- a/arch/arm/boot/compressed/decompress.c +++ b/arch/arm/boot/compressed/decompress.c @@ -57,5 +57,5 @@ extern char * strstr(const char * s1, const char *s2); int do_decompress(u8 *input, int len, u8 *output, void (*error)(char *x)) { - return decompress(input, len, NULL, NULL, output, NULL, error); + return __decompress(input, len, NULL, NULL, output, 0, NULL, error); } diff --git a/arch/arm/boot/dts/am335x-phycore-som.dtsi b/arch/arm/boot/dts/am335x-phycore-som.dtsi index 4d28fc3aac69..5dd084f3c81c 100644 --- a/arch/arm/boot/dts/am335x-phycore-som.dtsi +++ b/arch/arm/boot/dts/am335x-phycore-som.dtsi @@ -252,10 +252,10 @@ vio_reg: regulator@1 { }; vdd1_reg: regulator@2 { - /* VDD_MPU voltage limits 0.95V - 1.26V with +/-4% tolerance */ + /* VDD_MPU voltage limits 0.95V - 1.325V with +/-4% tolerance */ regulator-name = "vdd_mpu"; regulator-min-microvolt = <912500>; - regulator-max-microvolt = <1312500>; + regulator-max-microvolt = <1378000>; regulator-boot-on; regulator-always-on; }; diff --git a/arch/arm/boot/dts/am33xx.dtsi b/arch/arm/boot/dts/am33xx.dtsi index 9117c1a1a4e2..d23e2524d694 100644 --- a/arch/arm/boot/dts/am33xx.dtsi +++ b/arch/arm/boot/dts/am33xx.dtsi @@ -717,7 +717,7 @@ ehrpwm2: ehrpwm@48304200 { }; mac: ethernet@4a100000 { - compatible = "ti,cpsw"; + compatible = "ti,am335x-cpsw","ti,cpsw"; ti,hwmods = "cpgmac0"; clocks = <&cpsw_125mhz_gclk>, <&cpsw_cpts_rft_clk>; clock-names = "fck", "cpts"; diff --git a/arch/arm/boot/dts/am4372.dtsi b/arch/arm/boot/dts/am4372.dtsi index 564900b9fcce..0447c04a40cc 100644 --- a/arch/arm/boot/dts/am4372.dtsi +++ b/arch/arm/boot/dts/am4372.dtsi @@ -358,6 +358,8 @@ rtc: rtc@44e3e000 { interrupts = ; ti,hwmods = "rtc"; + clocks = <&clk_32768_ck>; + clock-names = "int-clk"; status = "disabled"; }; diff --git a/arch/arm/boot/dts/am437x-gp-evm.dts b/arch/arm/boot/dts/am437x-gp-evm.dts index 215775dc6948..22038f21f228 100644 --- a/arch/arm/boot/dts/am437x-gp-evm.dts +++ b/arch/arm/boot/dts/am437x-gp-evm.dts @@ -112,6 +112,13 @@ refclk: oscillator { clock-frequency = <12000000>; }; + /* fixed 32k external oscillator clock */ + clk_32k_rtc: clk_32k_rtc { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <32768>; + }; + sound0: sound@0 { compatible = "simple-audio-card"; simple-audio-card,name = "AM437x-GP-EVM"; @@ -941,3 +948,9 @@ &mcasp1 { tx-num-evt = <32>; rx-num-evt = <32>; }; + +&rtc { + clocks = <&clk_32k_rtc>, <&clk_32768_ck>; + clock-names = "ext-clk", "int-clk"; + status = "okay"; +}; diff --git a/arch/arm/boot/dts/am437x-idk-evm.dts b/arch/arm/boot/dts/am437x-idk-evm.dts index 378344271746..af25801418b4 100644 --- a/arch/arm/boot/dts/am437x-idk-evm.dts +++ b/arch/arm/boot/dts/am437x-idk-evm.dts @@ -110,6 +110,13 @@ switch@0 { gpios = <&gpio4 2 GPIO_ACTIVE_LOW>; }; }; + + /* fixed 32k external oscillator clock */ + clk_32k_rtc: clk_32k_rtc { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <32768>; + }; }; &am43xx_pinmux { @@ -394,6 +401,8 @@ &cpsw_emac0 { }; &rtc { + clocks = <&clk_32k_rtc>, <&clk_32768_ck>; + clock-names = "ext-clk", "int-clk"; status = "okay"; }; diff --git a/arch/arm/boot/dts/am437x-sk-evm.dts b/arch/arm/boot/dts/am437x-sk-evm.dts index 22af44894c66..7da7c2da4af1 100644 --- a/arch/arm/boot/dts/am437x-sk-evm.dts +++ b/arch/arm/boot/dts/am437x-sk-evm.dts @@ -24,6 +24,13 @@ aliases { display0 = &lcd0; }; + /* fixed 32k external oscillator clock */ + clk_32k_rtc: clk_32k_rtc { + #clock-cells = <0>; + compatible = "fixed-clock"; + clock-frequency = <32768>; + }; + backlight { compatible = "pwm-backlight"; pwms = <&ecap0 0 50000 PWM_POLARITY_INVERTED>; @@ -697,6 +704,8 @@ dpi_out: endpoint@0 { }; &rtc { + clocks = <&clk_32k_rtc>, <&clk_32768_ck>; + clock-names = "ext-clk", "int-clk"; status = "okay"; }; diff --git a/arch/arm/boot/dts/am57xx-beagle-x15.dts b/arch/arm/boot/dts/am57xx-beagle-x15.dts index 3a05b94f59ed..a4274abf1437 100644 --- a/arch/arm/boot/dts/am57xx-beagle-x15.dts +++ b/arch/arm/boot/dts/am57xx-beagle-x15.dts @@ -517,7 +517,8 @@ &i2c3 { mcp_rtc: rtc@6f { compatible = "microchip,mcp7941x"; reg = <0x6f>; - interrupts = ; /* IRQ_SYS_1N */ + interrupts-extended = <&crossbar_mpu GIC_SPI 2 IRQ_TYPE_EDGE_RISING>, + <&dra7_pmx_core 0x424>; pinctrl-names = "default"; pinctrl-0 = <&mcp79410_pins_default>; @@ -579,7 +580,6 @@ &mmc1 { pinctrl-0 = <&mmc1_pins_default>; vmmc-supply = <&ldo1_reg>; - vmmc_aux-supply = <&vdd_3v3>; bus-width = <4>; cd-gpios = <&gpio6 27 0>; /* gpio 219 */ }; diff --git a/arch/arm/boot/dts/dm8148-evm.dts b/arch/arm/boot/dts/dm8148-evm.dts index 92bacd3c8fab..109fd4711647 100644 --- a/arch/arm/boot/dts/dm8148-evm.dts +++ b/arch/arm/boot/dts/dm8148-evm.dts @@ -19,10 +19,10 @@ memory { &cpsw_emac0 { phy_id = <&davinci_mdio>, <0>; - phy-mode = "mii"; + phy-mode = "rgmii"; }; &cpsw_emac1 { phy_id = <&davinci_mdio>, <1>; - phy-mode = "mii"; + phy-mode = "rgmii"; }; diff --git a/arch/arm/boot/dts/dm8148-t410.dts b/arch/arm/boot/dts/dm8148-t410.dts index 8c4bbc7573df..79838dd8dee7 100644 --- a/arch/arm/boot/dts/dm8148-t410.dts +++ b/arch/arm/boot/dts/dm8148-t410.dts @@ -8,7 +8,7 @@ #include "dm814x.dtsi" / { - model = "DM8148 EVM"; + model = "HP t410 Smart Zero Client"; compatible = "hp,t410", "ti,dm8148"; memory { @@ -19,10 +19,10 @@ memory { &cpsw_emac0 { phy_id = <&davinci_mdio>, <0>; - phy-mode = "mii"; + phy-mode = "rgmii"; }; &cpsw_emac1 { phy_id = <&davinci_mdio>, <1>; - phy-mode = "mii"; + phy-mode = "rgmii"; }; diff --git a/arch/arm/boot/dts/dm814x.dtsi b/arch/arm/boot/dts/dm814x.dtsi index 972c9c9e885b..7988b42e5764 100644 --- a/arch/arm/boot/dts/dm814x.dtsi +++ b/arch/arm/boot/dts/dm814x.dtsi @@ -181,9 +181,9 @@ timer3: timer@42000 { ti,hwmods = "timer3"; }; - control: control@160000 { + control: control@140000 { compatible = "ti,dm814-scm", "simple-bus"; - reg = <0x160000 0x16d000>; + reg = <0x140000 0x16d000>; #address-cells = <1>; #size-cells = <1>; ranges = <0 0x160000 0x16d000>; @@ -321,9 +321,9 @@ cpsw_emac1: slave@4a100300 { mac-address = [ 00 00 00 00 00 00 ]; }; - phy_sel: cpsw-phy-sel@0x48160650 { + phy_sel: cpsw-phy-sel@48140650 { compatible = "ti,am3352-cpsw-phy-sel"; - reg= <0x48160650 0x4>; + reg= <0x48140650 0x4>; reg-names = "gmii-sel"; }; }; diff --git a/arch/arm/boot/dts/dra7.dtsi b/arch/arm/boot/dts/dra7.dtsi index a0d3d4bfe9cb..fed95a4547ee 100644 --- a/arch/arm/boot/dts/dra7.dtsi +++ b/arch/arm/boot/dts/dra7.dtsi @@ -122,7 +122,7 @@ scm_conf: scm_conf@0 { #size-cells = <1>; pbias_regulator: pbias_regulator { - compatible = "ti,pbias-omap"; + compatible = "ti,pbias-dra7", "ti,pbias-omap"; reg = <0xe00 0x4>; syscon = <&scm_conf>; pbias_mmc_reg: pbias_mmc_omap5 { @@ -1417,8 +1417,8 @@ crossbar_mpu: crossbar@4a002a48 { ti,irqs-safe-map = <0>; }; - mac: ethernet@4a100000 { - compatible = "ti,cpsw"; + mac: ethernet@48484000 { + compatible = "ti,dra7-cpsw","ti,cpsw"; ti,hwmods = "gmac"; clocks = <&dpll_gmac_ck>, <&gmac_gmii_ref_clk_div>; clock-names = "fck", "cpts"; diff --git a/arch/arm/boot/dts/exynos3250-monk.dts b/arch/arm/boot/dts/exynos3250-monk.dts index a5863acc5fff..540a0adf2be6 100644 --- a/arch/arm/boot/dts/exynos3250-monk.dts +++ b/arch/arm/boot/dts/exynos3250-monk.dts @@ -116,6 +116,21 @@ haptics { min-microvolt = <1100000>; max-microvolt = <2700000>; }; + + thermal-zones { + cpu_thermal: cpu-thermal { + cooling-maps { + map0 { + /* Correspond to 500MHz at freq_table */ + cooling-device = <&cpu0 5 5>; + }; + map1 { + /* Correspond to 200MHz at freq_table */ + cooling-device = <&cpu0 8 8>; + }; + }; + }; + }; }; &adc { @@ -141,6 +156,10 @@ thermistor-battery { }; }; +&cpu0 { + cpu0-supply = <&buck2_reg>; +}; + &exynos_usbphy { status = "okay"; }; diff --git a/arch/arm/boot/dts/exynos3250-rinato.dts b/arch/arm/boot/dts/exynos3250-rinato.dts index baa9b2f52009..41a5fafb9aa9 100644 --- a/arch/arm/boot/dts/exynos3250-rinato.dts +++ b/arch/arm/boot/dts/exynos3250-rinato.dts @@ -107,6 +107,21 @@ haptics { min-microvolt = <1100000>; max-microvolt = <2700000>; }; + + thermal-zones { + cpu_thermal: cpu-thermal { + cooling-maps { + map0 { + /* Corresponds to 500MHz */ + cooling-device = <&cpu0 5 5>; + }; + map1 { + /* Corresponds to 200MHz */ + cooling-device = <&cpu0 8 8>; + }; + }; + }; + }; }; &adc { @@ -132,6 +147,10 @@ thermistor-battery { }; }; +&cpu0 { + cpu0-supply = <&buck2_reg>; +}; + &exynos_usbphy { status = "okay"; }; diff --git a/arch/arm/boot/dts/exynos3250.dtsi b/arch/arm/boot/dts/exynos3250.dtsi index 2db99433e17f..033def482fc3 100644 --- a/arch/arm/boot/dts/exynos3250.dtsi +++ b/arch/arm/boot/dts/exynos3250.dtsi @@ -53,6 +53,22 @@ cpu0: cpu@0 { compatible = "arm,cortex-a7"; reg = <0>; clock-frequency = <1000000000>; + clocks = <&cmu CLK_ARM_CLK>; + clock-names = "cpu"; + #cooling-cells = <2>; + + operating-points = < + 1000000 1150000 + 900000 1112500 + 800000 1075000 + 700000 1037500 + 600000 1000000 + 500000 962500 + 400000 925000 + 300000 887500 + 200000 850000 + 100000 850000 + >; }; cpu1: cpu@1 { diff --git a/arch/arm/boot/dts/exynos4.dtsi b/arch/arm/boot/dts/exynos4.dtsi index b0d52b1a646a..98c0a368b777 100644 --- a/arch/arm/boot/dts/exynos4.dtsi +++ b/arch/arm/boot/dts/exynos4.dtsi @@ -702,6 +702,7 @@ jpeg_codec: jpeg-codec@11840000 { clocks = <&clock CLK_JPEG>; clock-names = "jpeg"; power-domains = <&pd_cam>; + iommus = <&sysmmu_jpeg>; }; hdmi: hdmi@12D00000 { diff --git a/arch/arm/boot/dts/exynos4212.dtsi b/arch/arm/boot/dts/exynos4212.dtsi index d9c8efeef208..538901123d37 100644 --- a/arch/arm/boot/dts/exynos4212.dtsi +++ b/arch/arm/boot/dts/exynos4212.dtsi @@ -30,6 +30,9 @@ cpu0: cpu@A00 { device_type = "cpu"; compatible = "arm,cortex-a9"; reg = <0xA00>; + clocks = <&clock CLK_ARM_CLK>; + clock-names = "cpu"; + operating-points-v2 = <&cpu0_opp_table>; cooling-min-level = <13>; cooling-max-level = <7>; #cooling-cells = <2>; /* min followed by max */ @@ -39,6 +42,84 @@ cpu@A01 { device_type = "cpu"; compatible = "arm,cortex-a9"; reg = <0xA01>; + operating-points-v2 = <&cpu0_opp_table>; + }; + }; + + cpu0_opp_table: opp_table0 { + compatible = "operating-points-v2"; + opp-shared; + + opp00 { + opp-hz = /bits/ 64 <200000000>; + opp-microvolt = <900000>; + clock-latency-ns = <200000>; + }; + opp01 { + opp-hz = /bits/ 64 <300000000>; + opp-microvolt = <900000>; + clock-latency-ns = <200000>; + }; + opp02 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <925000>; + clock-latency-ns = <200000>; + }; + opp03 { + opp-hz = /bits/ 64 <500000000>; + opp-microvolt = <950000>; + clock-latency-ns = <200000>; + }; + opp04 { + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <975000>; + clock-latency-ns = <200000>; + }; + opp05 { + opp-hz = /bits/ 64 <700000000>; + opp-microvolt = <987500>; + clock-latency-ns = <200000>; + }; + opp06 { + opp-hz = /bits/ 64 <800000000>; + opp-microvolt = <1000000>; + clock-latency-ns = <200000>; + }; + opp07 { + opp-hz = /bits/ 64 <900000000>; + opp-microvolt = <1037500>; + clock-latency-ns = <200000>; + }; + opp08 { + opp-hz = /bits/ 64 <1000000000>; + opp-microvolt = <1087500>; + clock-latency-ns = <200000>; + }; + opp09 { + opp-hz = /bits/ 64 <1100000000>; + opp-microvolt = <1137500>; + clock-latency-ns = <200000>; + }; + opp10 { + opp-hz = /bits/ 64 <1200000000>; + opp-microvolt = <1187500>; + clock-latency-ns = <200000>; + }; + opp11 { + opp-hz = /bits/ 64 <1300000000>; + opp-microvolt = <1250000>; + clock-latency-ns = <200000>; + }; + opp12 { + opp-hz = /bits/ 64 <1400000000>; + opp-microvolt = <1287500>; + clock-latency-ns = <200000>; + }; + opp13 { + opp-hz = /bits/ 64 <1500000000>; + opp-microvolt = <1350000>; + clock-latency-ns = <200000>; + turbo-mode; }; }; }; diff --git a/arch/arm/boot/dts/exynos4412-odroid-common.dtsi b/arch/arm/boot/dts/exynos4412-odroid-common.dtsi index ca7d168d1dd6..db52841297a5 100644 --- a/arch/arm/boot/dts/exynos4412-odroid-common.dtsi +++ b/arch/arm/boot/dts/exynos4412-odroid-common.dtsi @@ -107,6 +107,10 @@ map1 { }; }; +&cpu0 { + cpu0-supply = <&buck2_reg>; +}; + /* RSTN signal for eMMC */ &sd1_cd { samsung,pin-pud = <0>; diff --git a/arch/arm/boot/dts/exynos4412-odroidu3.dts b/arch/arm/boot/dts/exynos4412-odroidu3.dts index 44684e57ead1..8632f35c6c26 100644 --- a/arch/arm/boot/dts/exynos4412-odroidu3.dts +++ b/arch/arm/boot/dts/exynos4412-odroidu3.dts @@ -13,6 +13,7 @@ /dts-v1/; #include "exynos4412-odroid-common.dtsi" +#include / { model = "Hardkernel ODROID-U3 board based on Exynos4412"; @@ -61,3 +62,10 @@ &sound { "Speakers", "SPKL", "Speakers", "SPKR"; }; + +&spi_1 { + pinctrl-names = "default"; + pinctrl-0 = <&spi1_bus>; + cs-gpios = <&gpb 5 GPIO_ACTIVE_HIGH>; + status = "okay"; +}; diff --git a/arch/arm/boot/dts/exynos4412-origen.dts b/arch/arm/boot/dts/exynos4412-origen.dts index 84c76310b312..9d528af68c1a 100644 --- a/arch/arm/boot/dts/exynos4412-origen.dts +++ b/arch/arm/boot/dts/exynos4412-origen.dts @@ -78,6 +78,10 @@ xusbxti { }; }; +&cpu0 { + cpu0-supply = <&buck2_reg>; +}; + &fimd { pinctrl-0 = <&lcd_clk &lcd_data24 &pwm1_out>; pinctrl-names = "default"; diff --git a/arch/arm/boot/dts/exynos4412-trats2.dts b/arch/arm/boot/dts/exynos4412-trats2.dts index 884840059018..2a1ebb76ebe0 100644 --- a/arch/arm/boot/dts/exynos4412-trats2.dts +++ b/arch/arm/boot/dts/exynos4412-trats2.dts @@ -288,6 +288,10 @@ &adc { status = "okay"; }; +&cpu0 { + cpu0-supply = <&buck2_reg>; +}; + &csis_0 { status = "okay"; vddcore-supply = <&ldo8_reg>; diff --git a/arch/arm/boot/dts/exynos4412.dtsi b/arch/arm/boot/dts/exynos4412.dtsi index b78ada70bd05..ca0e3c15977f 100644 --- a/arch/arm/boot/dts/exynos4412.dtsi +++ b/arch/arm/boot/dts/exynos4412.dtsi @@ -30,6 +30,9 @@ cpu0: cpu@A00 { device_type = "cpu"; compatible = "arm,cortex-a9"; reg = <0xA00>; + clocks = <&clock CLK_ARM_CLK>; + clock-names = "cpu"; + operating-points-v2 = <&cpu0_opp_table>; cooling-min-level = <13>; cooling-max-level = <7>; #cooling-cells = <2>; /* min followed by max */ @@ -39,18 +42,98 @@ cpu@A01 { device_type = "cpu"; compatible = "arm,cortex-a9"; reg = <0xA01>; + operating-points-v2 = <&cpu0_opp_table>; }; cpu@A02 { device_type = "cpu"; compatible = "arm,cortex-a9"; reg = <0xA02>; + operating-points-v2 = <&cpu0_opp_table>; }; cpu@A03 { device_type = "cpu"; compatible = "arm,cortex-a9"; reg = <0xA03>; + operating-points-v2 = <&cpu0_opp_table>; + }; + }; + + cpu0_opp_table: opp_table0 { + compatible = "operating-points-v2"; + opp-shared; + + opp00 { + opp-hz = /bits/ 64 <200000000>; + opp-microvolt = <900000>; + clock-latency-ns = <200000>; + }; + opp01 { + opp-hz = /bits/ 64 <300000000>; + opp-microvolt = <900000>; + clock-latency-ns = <200000>; + }; + opp02 { + opp-hz = /bits/ 64 <400000000>; + opp-microvolt = <925000>; + clock-latency-ns = <200000>; + }; + opp03 { + opp-hz = /bits/ 64 <500000000>; + opp-microvolt = <950000>; + clock-latency-ns = <200000>; + }; + opp04 { + opp-hz = /bits/ 64 <600000000>; + opp-microvolt = <975000>; + clock-latency-ns = <200000>; + }; + opp05 { + opp-hz = /bits/ 64 <700000000>; + opp-microvolt = <987500>; + clock-latency-ns = <200000>; + }; + opp06 { + opp-hz = /bits/ 64 <800000000>; + opp-microvolt = <1000000>; + clock-latency-ns = <200000>; + }; + opp07 { + opp-hz = /bits/ 64 <900000000>; + opp-microvolt = <1037500>; + clock-latency-ns = <200000>; + }; + opp08 { + opp-hz = /bits/ 64 <1000000000>; + opp-microvolt = <1087500>; + clock-latency-ns = <200000>; + }; + opp09 { + opp-hz = /bits/ 64 <1100000000>; + opp-microvolt = <1137500>; + clock-latency-ns = <200000>; + }; + opp10 { + opp-hz = /bits/ 64 <1200000000>; + opp-microvolt = <1187500>; + clock-latency-ns = <200000>; + }; + opp11 { + opp-hz = /bits/ 64 <1300000000>; + opp-microvolt = <1250000>; + clock-latency-ns = <200000>; + }; + opp12 { + opp-hz = /bits/ 64 <1400000000>; + opp-microvolt = <1287500>; + clock-latency-ns = <200000>; + }; + opp13 { + opp-hz = /bits/ 64 <1500000000>; + opp-microvolt = <1350000>; + clock-latency-ns = <200000>; + turbo-mode; }; }; diff --git a/arch/arm/boot/dts/exynos5250-arndale.dts b/arch/arm/boot/dts/exynos5250-arndale.dts index 7e728a1b5559..db3f65f3eb45 100644 --- a/arch/arm/boot/dts/exynos5250-arndale.dts +++ b/arch/arm/boot/dts/exynos5250-arndale.dts @@ -117,6 +117,10 @@ usb_hub: usb-hub { }; }; +&cpu0 { + cpu0-supply = <&buck2_reg>; +}; + &dp { status = "okay"; samsung,color-space = <0>; diff --git a/arch/arm/boot/dts/exynos5250-smdk5250.dts b/arch/arm/boot/dts/exynos5250-smdk5250.dts index 4fe186d01f8a..15aea760c1da 100644 --- a/arch/arm/boot/dts/exynos5250-smdk5250.dts +++ b/arch/arm/boot/dts/exynos5250-smdk5250.dts @@ -74,6 +74,10 @@ codec_mclk: codec-mclk { }; }; +&cpu0 { + cpu0-supply = <&buck2_reg>; +}; + &dp { samsung,color-space = <0>; samsung,dynamic-range = <0>; diff --git a/arch/arm/boot/dts/exynos5250-snow.dts b/arch/arm/boot/dts/exynos5250-snow.dts index b7f4122df456..0720caab5511 100644 --- a/arch/arm/boot/dts/exynos5250-snow.dts +++ b/arch/arm/boot/dts/exynos5250-snow.dts @@ -235,6 +235,10 @@ mmc3_pwrseq: mmc3_pwrseq { }; }; +&cpu0 { + cpu0-supply = <&buck2_reg>; +}; + &dp { status = "okay"; pinctrl-names = "default"; @@ -688,6 +692,7 @@ &spi_1 { status = "okay"; samsung,spi-src-clk = <0>; num-cs = <1>; + cs-gpios = <&gpa2 5 GPIO_ACTIVE_HIGH>; }; &usbdrd_dwc3 { diff --git a/arch/arm/boot/dts/exynos5250-spring.dts b/arch/arm/boot/dts/exynos5250-spring.dts index d03f9b8d376d..c1edd6d038a9 100644 --- a/arch/arm/boot/dts/exynos5250-spring.dts +++ b/arch/arm/boot/dts/exynos5250-spring.dts @@ -65,6 +65,10 @@ xxti { }; }; +&cpu0 { + cpu0-supply = <&buck2_reg>; +}; + &dp { status = "okay"; pinctrl-names = "default"; diff --git a/arch/arm/boot/dts/exynos5250.dtsi b/arch/arm/boot/dts/exynos5250.dtsi index 4a1f88300a28..b24610ea8c2a 100644 --- a/arch/arm/boot/dts/exynos5250.dtsi +++ b/arch/arm/boot/dts/exynos5250.dtsi @@ -62,6 +62,28 @@ cpu0: cpu@0 { compatible = "arm,cortex-a15"; reg = <0>; clock-frequency = <1700000000>; + clocks = <&clock CLK_ARM_CLK>; + clock-names = "cpu"; + clock-latency = <140000>; + + operating-points = < + 1700000 1300000 + 1600000 1250000 + 1500000 1225000 + 1400000 1200000 + 1300000 1150000 + 1200000 1125000 + 1100000 1100000 + 1000000 1075000 + 900000 1050000 + 800000 1025000 + 700000 1012500 + 600000 1000000 + 500000 975000 + 400000 950000 + 300000 937500 + 200000 925000 + >; cooling-min-level = <15>; cooling-max-level = <9>; #cooling-cells = <2>; /* min followed by max */ diff --git a/arch/arm/boot/dts/exynos5422-cpus.dtsi b/arch/arm/boot/dts/exynos5422-cpus.dtsi new file mode 100644 index 000000000000..b7f60c855459 --- /dev/null +++ b/arch/arm/boot/dts/exynos5422-cpus.dtsi @@ -0,0 +1,81 @@ +/* + * SAMSUNG EXYNOS5422 SoC cpu device tree source + * + * Copyright (c) 2015 Samsung Electronics Co., Ltd. + * http://www.samsung.com + * + * The only difference between EXYNOS5422 and EXYNOS5800 is cpu ordering. The + * EXYNOS5422 is booting from Cortex-A7 core while the EXYNOS5800 is booting + * from Cortex-A15 core. + * + * EXYNOS5422 based board files can include this file to provide cpu ordering + * which could boot a cortex-a7 from cpu0. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +&cpu0 { + device_type = "cpu"; + compatible = "arm,cortex-a7"; + reg = <0x100>; + clock-frequency = <1000000000>; + cci-control-port = <&cci_control0>; +}; + +&cpu1 { + device_type = "cpu"; + compatible = "arm,cortex-a7"; + reg = <0x101>; + clock-frequency = <1000000000>; + cci-control-port = <&cci_control0>; +}; + +&cpu2 { + device_type = "cpu"; + compatible = "arm,cortex-a7"; + reg = <0x102>; + clock-frequency = <1000000000>; + cci-control-port = <&cci_control0>; +}; + +&cpu3 { + device_type = "cpu"; + compatible = "arm,cortex-a7"; + reg = <0x103>; + clock-frequency = <1000000000>; + cci-control-port = <&cci_control0>; +}; + +&cpu4 { + device_type = "cpu"; + compatible = "arm,cortex-a15"; + reg = <0x0>; + clock-frequency = <1800000000>; + cci-control-port = <&cci_control1>; +}; + +&cpu5 { + device_type = "cpu"; + compatible = "arm,cortex-a15"; + reg = <0x1>; + clock-frequency = <1800000000>; + cci-control-port = <&cci_control1>; +}; + +&cpu6 { + device_type = "cpu"; + compatible = "arm,cortex-a15"; + reg = <0x2>; + clock-frequency = <1800000000>; + cci-control-port = <&cci_control1>; +}; + +&cpu7 { + device_type = "cpu"; + compatible = "arm,cortex-a15"; + reg = <0x3>; + clock-frequency = <1800000000>; + cci-control-port = <&cci_control1>; +}; diff --git a/arch/arm/boot/dts/exynos5422-odroidxu3-common.dtsi b/arch/arm/boot/dts/exynos5422-odroidxu3-common.dtsi index 1565667e6f69..79ffdfe712aa 100644 --- a/arch/arm/boot/dts/exynos5422-odroidxu3-common.dtsi +++ b/arch/arm/boot/dts/exynos5422-odroidxu3-common.dtsi @@ -15,6 +15,7 @@ #include #include #include "exynos5800.dtsi" +#include "exynos5422-cpus.dtsi" #include "exynos5422-cpu-thermal.dtsi" / { diff --git a/arch/arm/boot/dts/omap2430.dtsi b/arch/arm/boot/dts/omap2430.dtsi index 2390f387c271..3961a6f170e1 100644 --- a/arch/arm/boot/dts/omap2430.dtsi +++ b/arch/arm/boot/dts/omap2430.dtsi @@ -63,7 +63,7 @@ scm_clocks: clocks { }; pbias_regulator: pbias_regulator { - compatible = "ti,pbias-omap"; + compatible = "ti,pbias-omap2", "ti,pbias-omap"; reg = <0x230 0x4>; syscon = <&scm_conf>; pbias_mmc_reg: pbias_mmc_omap2430 { diff --git a/arch/arm/boot/dts/omap3-beagle.dts b/arch/arm/boot/dts/omap3-beagle.dts index a5474113cd50..67659a0ed13e 100644 --- a/arch/arm/boot/dts/omap3-beagle.dts +++ b/arch/arm/boot/dts/omap3-beagle.dts @@ -202,7 +202,7 @@ uart3_pins: pinmux_uart3_pins { tfp410_pins: pinmux_tfp410_pins { pinctrl-single,pins = < - 0x194 (PIN_OUTPUT | MUX_MODE4) /* hdq_sio.gpio_170 */ + 0x196 (PIN_OUTPUT | MUX_MODE4) /* hdq_sio.gpio_170 */ >; }; diff --git a/arch/arm/boot/dts/omap3-igep.dtsi b/arch/arm/boot/dts/omap3-igep.dtsi index d5e5cd449b16..2230e1c03320 100644 --- a/arch/arm/boot/dts/omap3-igep.dtsi +++ b/arch/arm/boot/dts/omap3-igep.dtsi @@ -78,12 +78,6 @@ mmc2_pins: pinmux_mmc2_pins { >; }; - smsc9221_pins: pinmux_smsc9221_pins { - pinctrl-single,pins = < - 0x1a2 (PIN_INPUT | MUX_MODE4) /* mcspi1_cs2.gpio_176 */ - >; - }; - i2c1_pins: pinmux_i2c1_pins { pinctrl-single,pins = < 0x18a (PIN_INPUT | MUX_MODE0) /* i2c1_scl.i2c1_scl */ diff --git a/arch/arm/boot/dts/omap3-igep0020-common.dtsi b/arch/arm/boot/dts/omap3-igep0020-common.dtsi index e458c2185e3c..5ad688c57a00 100644 --- a/arch/arm/boot/dts/omap3-igep0020-common.dtsi +++ b/arch/arm/boot/dts/omap3-igep0020-common.dtsi @@ -156,6 +156,12 @@ OMAP3_CORE1_IOPAD(0x2178, PIN_OUTPUT | MUX_MODE0) /* uart2_tx.uart2_tx */ OMAP3_CORE1_IOPAD(0x217a, PIN_INPUT | MUX_MODE0) /* uart2_rx.uart2_rx */ >; }; + + smsc9221_pins: pinmux_smsc9221_pins { + pinctrl-single,pins = < + OMAP3_CORE1_IOPAD(0x21d2, PIN_INPUT | MUX_MODE4) /* mcspi1_cs2.gpio_176 */ + >; + }; }; &omap3_pmx_core2 { diff --git a/arch/arm/boot/dts/omap3.dtsi b/arch/arm/boot/dts/omap3.dtsi index 69a40cfc1f29..9af9ae190299 100644 --- a/arch/arm/boot/dts/omap3.dtsi +++ b/arch/arm/boot/dts/omap3.dtsi @@ -203,7 +203,7 @@ sdma: dma-controller@48056000 { }; pbias_regulator: pbias_regulator { - compatible = "ti,pbias-omap"; + compatible = "ti,pbias-omap3", "ti,pbias-omap"; reg = <0x2b0 0x4>; syscon = <&scm_conf>; pbias_mmc_reg: pbias_mmc_omap2430 { diff --git a/arch/arm/boot/dts/omap4.dtsi b/arch/arm/boot/dts/omap4.dtsi index abc4473e6f8a..5aad7f37df8b 100644 --- a/arch/arm/boot/dts/omap4.dtsi +++ b/arch/arm/boot/dts/omap4.dtsi @@ -198,7 +198,7 @@ omap4_padconf_global: omap4_padconf_global@5a0 { #size-cells = <1>; pbias_regulator: pbias_regulator { - compatible = "ti,pbias-omap"; + compatible = "ti,pbias-omap4", "ti,pbias-omap"; reg = <0x60 0x4>; syscon = <&omap4_padconf_global>; pbias_mmc_reg: pbias_mmc_omap4 { diff --git a/arch/arm/boot/dts/omap5.dtsi b/arch/arm/boot/dts/omap5.dtsi index 4205a8ac9ddb..8d5f9d26110e 100644 --- a/arch/arm/boot/dts/omap5.dtsi +++ b/arch/arm/boot/dts/omap5.dtsi @@ -187,7 +187,7 @@ omap5_padconf_global: omap5_padconf_global@5a0 { #size-cells = <1>; pbias_regulator: pbias_regulator { - compatible = "ti,pbias-omap"; + compatible = "ti,pbias-omap5", "ti,pbias-omap"; reg = <0x60 0x4>; syscon = <&omap5_padconf_global>; pbias_mmc_reg: pbias_mmc_omap5 { diff --git a/arch/arm/boot/dts/qcom-apq8064-cm-qs600.dts b/arch/arm/boot/dts/qcom-apq8064-cm-qs600.dts index 34ccb260f12a..47c0282bdfca 100644 --- a/arch/arm/boot/dts/qcom-apq8064-cm-qs600.dts +++ b/arch/arm/boot/dts/qcom-apq8064-cm-qs600.dts @@ -4,6 +4,14 @@ / { model = "CompuLab CM-QS600"; compatible = "qcom,apq8064-cm-qs600", "qcom,apq8064"; + aliases { + serial0 = &gsbi7_serial; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + soc { pinctrl@800000 { i2c1_pins: i2c1 { diff --git a/arch/arm/boot/dts/qcom-apq8064-ifc6410.dts b/arch/arm/boot/dts/qcom-apq8064-ifc6410.dts index 88d6655ddaf6..f3100da082b2 100644 --- a/arch/arm/boot/dts/qcom-apq8064-ifc6410.dts +++ b/arch/arm/boot/dts/qcom-apq8064-ifc6410.dts @@ -10,6 +10,10 @@ aliases { serial1 = &gsbi6_serial; }; + chosen { + stdout-path = "serial0:115200n8"; + }; + soc { pinctrl@800000 { card_detect: card_detect { diff --git a/arch/arm/boot/dts/qcom-apq8074-dragonboard.dts b/arch/arm/boot/dts/qcom-apq8074-dragonboard.dts index d484d08163e9..835bdc71c5ba 100644 --- a/arch/arm/boot/dts/qcom-apq8074-dragonboard.dts +++ b/arch/arm/boot/dts/qcom-apq8074-dragonboard.dts @@ -6,6 +6,14 @@ / { model = "Qualcomm APQ8074 Dragonboard"; compatible = "qcom,apq8074-dragonboard", "qcom,apq8074"; + aliases { + serial0 = &blsp1_uart2; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + soc { serial@f991e000 { status = "ok"; diff --git a/arch/arm/boot/dts/qcom-apq8084-ifc6540.dts b/arch/arm/boot/dts/qcom-apq8084-ifc6540.dts index f7725b96612c..c9c2b769554f 100644 --- a/arch/arm/boot/dts/qcom-apq8084-ifc6540.dts +++ b/arch/arm/boot/dts/qcom-apq8084-ifc6540.dts @@ -5,6 +5,14 @@ / { model = "Qualcomm APQ8084/IFC6540"; compatible = "qcom,apq8084-ifc6540", "qcom,apq8084"; + aliases { + serial0 = &blsp2_uart2; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + soc { serial@f995e000 { status = "okay"; diff --git a/arch/arm/boot/dts/qcom-apq8084-mtp.dts b/arch/arm/boot/dts/qcom-apq8084-mtp.dts index cb43acfc5d1d..3016c7048d44 100644 --- a/arch/arm/boot/dts/qcom-apq8084-mtp.dts +++ b/arch/arm/boot/dts/qcom-apq8084-mtp.dts @@ -5,6 +5,14 @@ / { model = "Qualcomm APQ 8084-MTP"; compatible = "qcom,apq8084-mtp", "qcom,apq8084"; + aliases { + serial0 = &blsp2_uart2; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + soc { serial@f995e000 { status = "okay"; diff --git a/arch/arm/boot/dts/qcom-apq8084.dtsi b/arch/arm/boot/dts/qcom-apq8084.dtsi index 7084010ee61b..0554fbd72c40 100644 --- a/arch/arm/boot/dts/qcom-apq8084.dtsi +++ b/arch/arm/boot/dts/qcom-apq8084.dtsi @@ -234,7 +234,7 @@ tlmm: pinctrl@fd510000 { interrupts = <0 208 0>; }; - serial@f995e000 { + blsp2_uart2: serial@f995e000 { compatible = "qcom,msm-uartdm-v1.4", "qcom,msm-uartdm"; reg = <0xf995e000 0x1000>; interrupts = <0 114 0x0>; diff --git a/arch/arm/boot/dts/qcom-ipq8064-ap148.dts b/arch/arm/boot/dts/qcom-ipq8064-ap148.dts index 55b2910efd87..d501382493e3 100644 --- a/arch/arm/boot/dts/qcom-ipq8064-ap148.dts +++ b/arch/arm/boot/dts/qcom-ipq8064-ap148.dts @@ -4,6 +4,14 @@ / { model = "Qualcomm IPQ8064/AP148"; compatible = "qcom,ipq8064-ap148", "qcom,ipq8064"; + aliases { + serial0 = &gsbi4_serial; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + reserved-memory { #address-cells = <1>; #size-cells = <1>; diff --git a/arch/arm/boot/dts/qcom-ipq8064.dtsi b/arch/arm/boot/dts/qcom-ipq8064.dtsi index 9f727d8eadf6..fa698635eea0 100644 --- a/arch/arm/boot/dts/qcom-ipq8064.dtsi +++ b/arch/arm/boot/dts/qcom-ipq8064.dtsi @@ -197,7 +197,7 @@ gsbi4: gsbi@16300000 { syscon-tcsr = <&tcsr>; - serial@16340000 { + gsbi4_serial: serial@16340000 { compatible = "qcom,msm-uartdm-v1.3", "qcom,msm-uartdm"; reg = <0x16340000 0x1000>, <0x16300000 0x1000>; diff --git a/arch/arm/boot/dts/qcom-msm8660-surf.dts b/arch/arm/boot/dts/qcom-msm8660-surf.dts index e0883c376248..b17f379e8c2a 100644 --- a/arch/arm/boot/dts/qcom-msm8660-surf.dts +++ b/arch/arm/boot/dts/qcom-msm8660-surf.dts @@ -6,6 +6,14 @@ / { model = "Qualcomm MSM8660 SURF"; compatible = "qcom,msm8660-surf", "qcom,msm8660"; + aliases { + serial0 = &gsbi12_serial; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + soc { gsbi@19c00000 { status = "ok"; diff --git a/arch/arm/boot/dts/qcom-msm8660.dtsi b/arch/arm/boot/dts/qcom-msm8660.dtsi index ef2fe72b54c9..e5f7f33aa467 100644 --- a/arch/arm/boot/dts/qcom-msm8660.dtsi +++ b/arch/arm/boot/dts/qcom-msm8660.dtsi @@ -98,7 +98,7 @@ gsbi12: gsbi@19c00000 { syscon-tcsr = <&tcsr>; - serial@19c40000 { + gsbi12_serial: serial@19c40000 { compatible = "qcom,msm-uartdm-v1.3", "qcom,msm-uartdm"; reg = <0x19c40000 0x1000>, <0x19c00000 0x1000>; diff --git a/arch/arm/boot/dts/qcom-msm8960-cdp.dts b/arch/arm/boot/dts/qcom-msm8960-cdp.dts index fad71d5527b0..b72a55462caf 100644 --- a/arch/arm/boot/dts/qcom-msm8960-cdp.dts +++ b/arch/arm/boot/dts/qcom-msm8960-cdp.dts @@ -6,6 +6,14 @@ / { model = "Qualcomm MSM8960 CDP"; compatible = "qcom,msm8960-cdp", "qcom,msm8960"; + aliases { + serial0 = &gsbi5_serial; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + soc { gsbi@16400000 { status = "ok"; diff --git a/arch/arm/boot/dts/qcom-msm8960.dtsi b/arch/arm/boot/dts/qcom-msm8960.dtsi index 2096a94c9b52..134cd91d68ec 100644 --- a/arch/arm/boot/dts/qcom-msm8960.dtsi +++ b/arch/arm/boot/dts/qcom-msm8960.dtsi @@ -157,7 +157,7 @@ gsbi5: gsbi@16400000 { syscon-tcsr = <&tcsr>; - serial@16440000 { + gsbi5_serial: serial@16440000 { compatible = "qcom,msm-uartdm-v1.3", "qcom,msm-uartdm"; reg = <0x16440000 0x1000>, <0x16400000 0x1000>; diff --git a/arch/arm/boot/dts/qcom-msm8974-sony-xperia-honami.dts b/arch/arm/boot/dts/qcom-msm8974-sony-xperia-honami.dts index 9bc72a3356e4..016f9ad9392a 100644 --- a/arch/arm/boot/dts/qcom-msm8974-sony-xperia-honami.dts +++ b/arch/arm/boot/dts/qcom-msm8974-sony-xperia-honami.dts @@ -6,6 +6,14 @@ / { model = "Sony Xperia Z1"; compatible = "sony,xperia-honami", "qcom,msm8974"; + aliases { + serial0 = &blsp1_uart2; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + memory@0 { reg = <0 0x40000000>, <0x40000000 0x40000000>; device_type = "memory"; diff --git a/arch/arm/boot/dts/qcom-msm8974.dtsi b/arch/arm/boot/dts/qcom-msm8974.dtsi index d7c99b894a49..ab8e57250468 100644 --- a/arch/arm/boot/dts/qcom-msm8974.dtsi +++ b/arch/arm/boot/dts/qcom-msm8974.dtsi @@ -259,7 +259,7 @@ smem@fa00000 { hwlocks = <&tcsr_mutex 3>; }; - serial@f991e000 { + blsp1_uart2: serial@f991e000 { compatible = "qcom,msm-uartdm-v1.4", "qcom,msm-uartdm"; reg = <0xf991e000 0x1000>; interrupts = <0 108 0x0>; diff --git a/arch/arm/boot/dts/ste-dbx5x0.dtsi b/arch/arm/boot/dts/ste-dbx5x0.dtsi index c5fbde3afcf6..50f5e9d09203 100644 --- a/arch/arm/boot/dts/ste-dbx5x0.dtsi +++ b/arch/arm/boot/dts/ste-dbx5x0.dtsi @@ -294,7 +294,7 @@ gpio0: gpio@8012e000 { gpio-controller; #gpio-cells = <2>; gpio-bank = <0>; - + gpio-ranges = <&pinctrl 0 0 32>; clocks = <&prcc_pclk 1 9>; }; @@ -309,7 +309,7 @@ gpio1: gpio@8012e080 { gpio-controller; #gpio-cells = <2>; gpio-bank = <1>; - + gpio-ranges = <&pinctrl 0 32 5>; clocks = <&prcc_pclk 1 9>; }; @@ -324,7 +324,7 @@ gpio2: gpio@8000e000 { gpio-controller; #gpio-cells = <2>; gpio-bank = <2>; - + gpio-ranges = <&pinctrl 0 64 32>; clocks = <&prcc_pclk 3 8>; }; @@ -339,7 +339,7 @@ gpio3: gpio@8000e080 { gpio-controller; #gpio-cells = <2>; gpio-bank = <3>; - + gpio-ranges = <&pinctrl 0 96 2>; clocks = <&prcc_pclk 3 8>; }; @@ -354,7 +354,7 @@ gpio4: gpio@8000e100 { gpio-controller; #gpio-cells = <2>; gpio-bank = <4>; - + gpio-ranges = <&pinctrl 0 128 32>; clocks = <&prcc_pclk 3 8>; }; @@ -369,7 +369,7 @@ gpio5: gpio@8000e180 { gpio-controller; #gpio-cells = <2>; gpio-bank = <5>; - + gpio-ranges = <&pinctrl 0 160 12>; clocks = <&prcc_pclk 3 8>; }; @@ -384,7 +384,7 @@ gpio6: gpio@8011e000 { gpio-controller; #gpio-cells = <2>; gpio-bank = <6>; - + gpio-ranges = <&pinctrl 0 192 32>; clocks = <&prcc_pclk 2 11>; }; @@ -399,7 +399,7 @@ gpio7: gpio@8011e080 { gpio-controller; #gpio-cells = <2>; gpio-bank = <7>; - + gpio-ranges = <&pinctrl 0 224 7>; clocks = <&prcc_pclk 2 11>; }; @@ -414,12 +414,15 @@ gpio8: gpio@a03fe000 { gpio-controller; #gpio-cells = <2>; gpio-bank = <8>; - + gpio-ranges = <&pinctrl 0 256 12>; clocks = <&prcc_pclk 5 1>; }; - pinctrl { + pinctrl: pinctrl { compatible = "stericsson,db8500-pinctrl"; + nomadik-gpio-chips = <&gpio0>, <&gpio1>, <&gpio2>, <&gpio3>, + <&gpio4>, <&gpio5>, <&gpio6>, <&gpio7>, + <&gpio8>; prcm = <&prcmu>; }; diff --git a/arch/arm/boot/dts/ste-nomadik-stn8815.dtsi b/arch/arm/boot/dts/ste-nomadik-stn8815.dtsi index 176e332fc0bd..314f59c12162 100644 --- a/arch/arm/boot/dts/ste-nomadik-stn8815.dtsi +++ b/arch/arm/boot/dts/ste-nomadik-stn8815.dtsi @@ -59,6 +59,7 @@ gpio0: gpio@101e4000 { gpio-controller; #gpio-cells = <2>; gpio-bank = <0>; + gpio-ranges = <&pinctrl 0 0 32>; clocks = <&pclk>; }; @@ -72,6 +73,7 @@ gpio1: gpio@101e5000 { gpio-controller; #gpio-cells = <2>; gpio-bank = <1>; + gpio-ranges = <&pinctrl 0 32 32>; clocks = <&pclk>; }; @@ -85,12 +87,14 @@ gpio2: gpio@101e6000 { gpio-controller; #gpio-cells = <2>; gpio-bank = <2>; + gpio-ranges = <&pinctrl 0 64 32>; clocks = <&pclk>; }; gpio3: gpio@101e7000 { compatible = "st,nomadik-gpio"; reg = <0x101e7000 0x80>; + ngpio = <28>; interrupt-parent = <&vica>; interrupts = <9>; interrupt-controller; @@ -98,11 +102,13 @@ gpio3: gpio@101e7000 { gpio-controller; #gpio-cells = <2>; gpio-bank = <3>; + gpio-ranges = <&pinctrl 0 96 28>; clocks = <&pclk>; }; - pinctrl { + pinctrl: pinctrl { compatible = "stericsson,stn8815-pinctrl"; + nomadik-gpio-chips = <&gpio0>, <&gpio1>, <&gpio2>, <&gpio3>; /* Pin configurations */ uart1 { uart1_default_mux: uart1_mux { diff --git a/arch/arm/common/mcpm_platsmp.c b/arch/arm/common/mcpm_platsmp.c index 92e54d7c6f46..2b25b6038f66 100644 --- a/arch/arm/common/mcpm_platsmp.c +++ b/arch/arm/common/mcpm_platsmp.c @@ -65,14 +65,10 @@ static int mcpm_cpu_kill(unsigned int cpu) return !mcpm_wait_for_cpu_powerdown(pcpu, pcluster); } -static int mcpm_cpu_disable(unsigned int cpu) +static bool mcpm_cpu_can_disable(unsigned int cpu) { - /* - * We assume all CPUs may be shut down. - * This would be the hook to use for eventual Secure - * OS migration requests as described in the PSCI spec. - */ - return 0; + /* We assume all CPUs may be shut down. */ + return true; } static void mcpm_cpu_die(unsigned int cpu) @@ -92,7 +88,7 @@ static struct smp_operations __initdata mcpm_smp_ops = { .smp_secondary_init = mcpm_secondary_init, #ifdef CONFIG_HOTPLUG_CPU .cpu_kill = mcpm_cpu_kill, - .cpu_disable = mcpm_cpu_disable, + .cpu_can_disable = mcpm_cpu_can_disable, .cpu_die = mcpm_cpu_die, #endif }; diff --git a/arch/arm/configs/cm_x2xx_defconfig b/arch/arm/configs/cm_x2xx_defconfig index dc01c049a520..3b32d5fd9326 100644 --- a/arch/arm/configs/cm_x2xx_defconfig +++ b/arch/arm/configs/cm_x2xx_defconfig @@ -157,7 +157,7 @@ CONFIG_LEDS_TRIGGERS=y CONFIG_LEDS_TRIGGER_HEARTBEAT=y CONFIG_RTC_CLASS=y CONFIG_RTC_DRV_V3020=y -CONFIG_RTC_DRV_SA1100=y +CONFIG_RTC_DRV_PXA=y CONFIG_EXT2_FS=y CONFIG_EXT3_FS=y CONFIG_INOTIFY=y diff --git a/arch/arm/configs/em_x270_defconfig b/arch/arm/configs/em_x270_defconfig index 4560c9ca6636..8e10df7ba1b4 100644 --- a/arch/arm/configs/em_x270_defconfig +++ b/arch/arm/configs/em_x270_defconfig @@ -157,7 +157,7 @@ CONFIG_LEDS_TRIGGERS=y CONFIG_LEDS_TRIGGER_HEARTBEAT=y CONFIG_RTC_CLASS=y CONFIG_RTC_DRV_V3020=y -CONFIG_RTC_DRV_SA1100=y +CONFIG_RTC_DRV_PXA=y CONFIG_EXT2_FS=y CONFIG_EXT3_FS=y CONFIG_INOTIFY=y diff --git a/arch/arm/configs/exynos_defconfig b/arch/arm/configs/exynos_defconfig index 9504e7790288..1ff2bfa2e183 100644 --- a/arch/arm/configs/exynos_defconfig +++ b/arch/arm/configs/exynos_defconfig @@ -27,6 +27,8 @@ CONFIG_ARM_APPENDED_DTB=y CONFIG_ARM_ATAG_DTB_COMPAT=y CONFIG_CMDLINE="root=/dev/ram0 rw ramdisk=8192 initrd=0x41000000,8M console=ttySAC1,115200 init=/linuxrc mem=256M" CONFIG_CPU_FREQ=y +CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y +CONFIG_CPUFREQ_DT=y CONFIG_CPU_IDLE=y CONFIG_ARM_EXYNOS_CPUIDLE=y CONFIG_VFP=y @@ -94,6 +96,7 @@ CONFIG_CHARGER_MAX14577=y CONFIG_CHARGER_MAX77693=y CONFIG_CHARGER_TPS65090=y CONFIG_SENSORS_LM90=y +CONFIG_SENSORS_NTC_THERMISTOR=y CONFIG_SENSORS_PWM_FAN=y CONFIG_SENSORS_INA2XX=y CONFIG_THERMAL=y @@ -124,14 +127,14 @@ CONFIG_REGULATOR_S2MPS11=y CONFIG_REGULATOR_S5M8767=y CONFIG_REGULATOR_TPS65090=y CONFIG_DRM=y -CONFIG_DRM_PTN3460=y -CONFIG_DRM_PS8622=y +CONFIG_DRM_NXP_PTN3460=y +CONFIG_DRM_PARADE_PS8622=y CONFIG_DRM_EXYNOS=y CONFIG_DRM_EXYNOS_FIMD=y CONFIG_DRM_EXYNOS_DSI=y CONFIG_DRM_EXYNOS_HDMI=y CONFIG_DRM_PANEL_SIMPLE=y -CONFIG_DRM_PANEL_S6E8AA0=y +CONFIG_DRM_PANEL_SAMSUNG_S6E8AA0=y CONFIG_FB_SIMPLE=y CONFIG_EXYNOS_VIDEO=y CONFIG_EXYNOS_MIPI_DSI=y @@ -144,6 +147,8 @@ CONFIG_SND=y CONFIG_SND_SOC=y CONFIG_SND_SOC_SAMSUNG=y CONFIG_SND_SOC_SNOW=y +CONFIG_SND_SOC_ODROIDX2=y +CONFIG_SND_SIMPLE_CARD=y CONFIG_USB=y CONFIG_USB_ANNOUNCE_NEW_DEVICES=y CONFIG_USB_XHCI_HCD=y diff --git a/arch/arm/configs/magician_defconfig b/arch/arm/configs/magician_defconfig index 557dd291288b..a5b4920cd6d4 100644 --- a/arch/arm/configs/magician_defconfig +++ b/arch/arm/configs/magician_defconfig @@ -150,7 +150,7 @@ CONFIG_LEDS_TRIGGERS=y CONFIG_LEDS_TRIGGER_BACKLIGHT=y CONFIG_RTC_CLASS=y CONFIG_RTC_DEBUG=y -CONFIG_RTC_DRV_SA1100=y +CONFIG_RTC_DRV_PXA=y CONFIG_EXT2_FS=y CONFIG_INOTIFY=y CONFIG_MSDOS_FS=m diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v7_defconfig index 824a0cff3998..03deb7fb35e8 100644 --- a/arch/arm/configs/multi_v7_defconfig +++ b/arch/arm/configs/multi_v7_defconfig @@ -362,6 +362,7 @@ CONFIG_POWER_RESET_KEYSTONE=y CONFIG_POWER_RESET_RMOBILE=y CONFIG_SENSORS_LM90=y CONFIG_SENSORS_LM95245=y +CONFIG_SENSORS_NTC_THERMISTOR=m CONFIG_THERMAL=y CONFIG_CPU_THERMAL=y CONFIG_RCAR_THERMAL=y @@ -410,7 +411,9 @@ CONFIG_REGULATOR_MAX8907=y CONFIG_REGULATOR_MAX8973=y CONFIG_REGULATOR_MAX77686=y CONFIG_REGULATOR_MAX77693=m +CONFIG_REGULATOR_MAX77802=m CONFIG_REGULATOR_PALMAS=y +CONFIG_REGULATOR_PBIAS=y CONFIG_REGULATOR_PWM=m CONFIG_REGULATOR_S2MPS11=y CONFIG_REGULATOR_S5M8767=y @@ -440,8 +443,8 @@ CONFIG_VIDEO_ML86V7667=m CONFIG_DRM=y # CONFIG_DRM_I2C_CH7006 is not set # CONFIG_DRM_I2C_SIL164 is not set -CONFIG_DRM_PTN3460=m -CONFIG_DRM_PS8622=m +CONFIG_DRM_NXP_PTN3460=m +CONFIG_DRM_PARADE_PS8622=m CONFIG_DRM_NOUVEAU=m CONFIG_DRM_EXYNOS=m CONFIG_DRM_EXYNOS_DSI=y @@ -449,7 +452,7 @@ CONFIG_DRM_EXYNOS_FIMD=y CONFIG_DRM_EXYNOS_HDMI=y CONFIG_DRM_RCAR_DU=m CONFIG_DRM_TEGRA=y -CONFIG_DRM_PANEL_S6E8AA0=m +CONFIG_DRM_PANEL_SAMSUNG_S6E8AA0=m CONFIG_DRM_PANEL_SIMPLE=y CONFIG_FB_ARMCLCD=y CONFIG_FB_WM8505=y @@ -509,8 +512,6 @@ CONFIG_USB_CHIPIDEA_HOST=y CONFIG_AB8500_USB=y CONFIG_KEYSTONE_USB_PHY=y CONFIG_OMAP_USB3=y -CONFIG_SAMSUNG_USB2PHY=y -CONFIG_SAMSUNG_USB3PHY=y CONFIG_USB_GPIO_VBUS=y CONFIG_USB_ISP1301=y CONFIG_USB_MXS_PHY=y @@ -635,6 +636,7 @@ CONFIG_EXTCON=y CONFIG_TI_AEMIF=y CONFIG_IIO=y CONFIG_AT91_ADC=m +CONFIG_EXYNOS_ADC=m CONFIG_XILINX_XADC=y CONFIG_AK8975=y CONFIG_PWM=y diff --git a/arch/arm/configs/omap2plus_defconfig b/arch/arm/configs/omap2plus_defconfig index 7b35e70d4bb3..ffc8fa2565f9 100644 --- a/arch/arm/configs/omap2plus_defconfig +++ b/arch/arm/configs/omap2plus_defconfig @@ -240,7 +240,7 @@ CONFIG_SSI_PROTOCOL=m CONFIG_PINCTRL_SINGLE=y CONFIG_DEBUG_GPIO=y CONFIG_GPIO_SYSFS=y -CONFIG_GPIO_PCF857X=m +CONFIG_GPIO_PCF857X=y CONFIG_GPIO_TWL4030=y CONFIG_GPIO_PALMAS=y CONFIG_W1=m diff --git a/arch/arm/configs/palmz72_defconfig b/arch/arm/configs/palmz72_defconfig index 4baa83c1c577..83c135e19aba 100644 --- a/arch/arm/configs/palmz72_defconfig +++ b/arch/arm/configs/palmz72_defconfig @@ -67,7 +67,7 @@ CONFIG_MMC=y CONFIG_MMC_DEBUG=y CONFIG_MMC_PXA=y CONFIG_RTC_CLASS=y -CONFIG_RTC_DRV_SA1100=y +CONFIG_RTC_DRV_PXA=y CONFIG_EXT2_FS=y CONFIG_EXT3_FS=y # CONFIG_DNOTIFY is not set diff --git a/arch/arm/configs/pcm027_defconfig b/arch/arm/configs/pcm027_defconfig index 0a847d04ddc1..b5624e325817 100644 --- a/arch/arm/configs/pcm027_defconfig +++ b/arch/arm/configs/pcm027_defconfig @@ -82,7 +82,7 @@ CONFIG_MMC=y CONFIG_MMC_PXA=y CONFIG_RTC_CLASS=y CONFIG_RTC_DRV_PCF8563=m -CONFIG_RTC_DRV_SA1100=m +CONFIG_RTC_DRV_PXA=m CONFIG_EXT2_FS=m CONFIG_EXT3_FS=m # CONFIG_DNOTIFY is not set diff --git a/arch/arm/configs/trizeps4_defconfig b/arch/arm/configs/trizeps4_defconfig index 932ee4e4a13a..4bc870028035 100644 --- a/arch/arm/configs/trizeps4_defconfig +++ b/arch/arm/configs/trizeps4_defconfig @@ -177,7 +177,7 @@ CONFIG_NEW_LEDS=y CONFIG_RTC_CLASS=y # CONFIG_RTC_HCTOSYS is not set CONFIG_RTC_DRV_PCF8583=m -CONFIG_RTC_DRV_SA1100=y +CONFIG_RTC_DRV_PXA=y CONFIG_EXT2_FS=y CONFIG_EXT2_FS_XATTR=y CONFIG_EXT2_FS_POSIX_ACL=y diff --git a/arch/arm/include/asm/Kbuild b/arch/arm/include/asm/Kbuild index 30b3bc1666d2..be648eb47cd9 100644 --- a/arch/arm/include/asm/Kbuild +++ b/arch/arm/include/asm/Kbuild @@ -12,7 +12,6 @@ generic-y += irq_regs.h generic-y += kdebug.h generic-y += local.h generic-y += local64.h -generic-y += mcs_spinlock.h generic-y += mm-arch-hooks.h generic-y += msgbuf.h generic-y += param.h diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 4abe57279c66..7bbf325a4f31 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -108,33 +108,37 @@ .endm #endif - .macro asm_trace_hardirqs_off + .macro asm_trace_hardirqs_off, save=1 #if defined(CONFIG_TRACE_IRQFLAGS) + .if \save stmdb sp!, {r0-r3, ip, lr} + .endif bl trace_hardirqs_off + .if \save ldmia sp!, {r0-r3, ip, lr} + .endif #endif .endm - .macro asm_trace_hardirqs_on_cond, cond + .macro asm_trace_hardirqs_on, cond=al, save=1 #if defined(CONFIG_TRACE_IRQFLAGS) /* * actually the registers should be pushed and pop'd conditionally, but * after bl the flags are certainly clobbered */ + .if \save stmdb sp!, {r0-r3, ip, lr} + .endif bl\cond trace_hardirqs_on + .if \save ldmia sp!, {r0-r3, ip, lr} + .endif #endif .endm - .macro asm_trace_hardirqs_on - asm_trace_hardirqs_on_cond al - .endm - - .macro disable_irq + .macro disable_irq, save=1 disable_irq_notrace - asm_trace_hardirqs_off + asm_trace_hardirqs_off \save .endm .macro enable_irq @@ -173,7 +177,7 @@ .macro restore_irqs, oldcpsr tst \oldcpsr, #PSR_I_BIT - asm_trace_hardirqs_on_cond eq + asm_trace_hardirqs_on cond=eq restore_irqs_notrace \oldcpsr .endm @@ -445,6 +449,53 @@ THUMB( orr \reg , \reg , #PSR_T_BIT ) #endif .endm + .macro uaccess_disable, tmp, isb=1 +#ifdef CONFIG_CPU_SW_DOMAIN_PAN + /* + * Whenever we re-enter userspace, the domains should always be + * set appropriately. + */ + mov \tmp, #DACR_UACCESS_DISABLE + mcr p15, 0, \tmp, c3, c0, 0 @ Set domain register + .if \isb + instr_sync + .endif +#endif + .endm + + .macro uaccess_enable, tmp, isb=1 +#ifdef CONFIG_CPU_SW_DOMAIN_PAN + /* + * Whenever we re-enter userspace, the domains should always be + * set appropriately. + */ + mov \tmp, #DACR_UACCESS_ENABLE + mcr p15, 0, \tmp, c3, c0, 0 + .if \isb + instr_sync + .endif +#endif + .endm + + .macro uaccess_save, tmp +#ifdef CONFIG_CPU_SW_DOMAIN_PAN + mrc p15, 0, \tmp, c3, c0, 0 + str \tmp, [sp, #S_FRAME_SIZE] +#endif + .endm + + .macro uaccess_restore +#ifdef CONFIG_CPU_SW_DOMAIN_PAN + ldr r0, [sp, #S_FRAME_SIZE] + mcr p15, 0, r0, c3, c0, 0 +#endif + .endm + + .macro uaccess_save_and_disable, tmp + uaccess_save \tmp + uaccess_disable \tmp + .endm + .irp c,,eq,ne,cs,cc,mi,pl,vs,vc,hi,ls,ge,lt,gt,le,hs,lo .macro ret\c, reg #if __LINUX_ARM_ARCH__ < 6 diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h index e22c11970b7b..fe3ef397f5a4 100644 --- a/arch/arm/include/asm/atomic.h +++ b/arch/arm/include/asm/atomic.h @@ -57,12 +57,11 @@ static inline void atomic_##op(int i, atomic_t *v) \ } \ #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ +static inline int atomic_##op##_return_relaxed(int i, atomic_t *v) \ { \ unsigned long tmp; \ int result; \ \ - smp_mb(); \ prefetchw(&v->counter); \ \ __asm__ __volatile__("@ atomic_" #op "_return\n" \ @@ -75,17 +74,17 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ : "r" (&v->counter), "Ir" (i) \ : "cc"); \ \ - smp_mb(); \ - \ return result; \ } -static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new) +#define atomic_add_return_relaxed atomic_add_return_relaxed +#define atomic_sub_return_relaxed atomic_sub_return_relaxed + +static inline int atomic_cmpxchg_relaxed(atomic_t *ptr, int old, int new) { int oldval; unsigned long res; - smp_mb(); prefetchw(&ptr->counter); do { @@ -99,10 +98,9 @@ static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new) : "cc"); } while (res); - smp_mb(); - return oldval; } +#define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed static inline int __atomic_add_unless(atomic_t *v, int a, int u) { @@ -194,6 +192,13 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) ATOMIC_OPS(add, +=, add) ATOMIC_OPS(sub, -=, sub) +#define atomic_andnot atomic_andnot + +ATOMIC_OP(and, &=, and) +ATOMIC_OP(andnot, &= ~, bic) +ATOMIC_OP(or, |=, orr) +ATOMIC_OP(xor, ^=, eor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -290,12 +295,12 @@ static inline void atomic64_##op(long long i, atomic64_t *v) \ } \ #define ATOMIC64_OP_RETURN(op, op1, op2) \ -static inline long long atomic64_##op##_return(long long i, atomic64_t *v) \ +static inline long long \ +atomic64_##op##_return_relaxed(long long i, atomic64_t *v) \ { \ long long result; \ unsigned long tmp; \ \ - smp_mb(); \ prefetchw(&v->counter); \ \ __asm__ __volatile__("@ atomic64_" #op "_return\n" \ @@ -309,8 +314,6 @@ static inline long long atomic64_##op##_return(long long i, atomic64_t *v) \ : "r" (&v->counter), "r" (i) \ : "cc"); \ \ - smp_mb(); \ - \ return result; \ } @@ -321,17 +324,26 @@ static inline long long atomic64_##op##_return(long long i, atomic64_t *v) \ ATOMIC64_OPS(add, adds, adc) ATOMIC64_OPS(sub, subs, sbc) +#define atomic64_add_return_relaxed atomic64_add_return_relaxed +#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed + +#define atomic64_andnot atomic64_andnot + +ATOMIC64_OP(and, and, and) +ATOMIC64_OP(andnot, bic, bic) +ATOMIC64_OP(or, orr, orr) +ATOMIC64_OP(xor, eor, eor) + #undef ATOMIC64_OPS #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP -static inline long long atomic64_cmpxchg(atomic64_t *ptr, long long old, - long long new) +static inline long long +atomic64_cmpxchg_relaxed(atomic64_t *ptr, long long old, long long new) { long long oldval; unsigned long res; - smp_mb(); prefetchw(&ptr->counter); do { @@ -346,17 +358,15 @@ static inline long long atomic64_cmpxchg(atomic64_t *ptr, long long old, : "cc"); } while (res); - smp_mb(); - return oldval; } +#define atomic64_cmpxchg_relaxed atomic64_cmpxchg_relaxed -static inline long long atomic64_xchg(atomic64_t *ptr, long long new) +static inline long long atomic64_xchg_relaxed(atomic64_t *ptr, long long new) { long long result; unsigned long tmp; - smp_mb(); prefetchw(&ptr->counter); __asm__ __volatile__("@ atomic64_xchg\n" @@ -368,10 +378,9 @@ static inline long long atomic64_xchg(atomic64_t *ptr, long long new) : "r" (&ptr->counter), "r" (new) : "cc"); - smp_mb(); - return result; } +#define atomic64_xchg_relaxed atomic64_xchg_relaxed static inline long long atomic64_dec_if_positive(atomic64_t *v) { diff --git a/arch/arm/include/asm/barrier.h b/arch/arm/include/asm/barrier.h index 6c2327e1c732..3ff5642d9788 100644 --- a/arch/arm/include/asm/barrier.h +++ b/arch/arm/include/asm/barrier.h @@ -2,7 +2,6 @@ #define __ASM_BARRIER_H #ifndef __ASSEMBLY__ -#include #define nop() __asm__ __volatile__("mov\tr0,r0\t@ nop\n\t"); @@ -37,12 +36,20 @@ #define dmb(x) __asm__ __volatile__ ("" : : : "memory") #endif +#ifdef CONFIG_ARM_HEAVY_MB +extern void (*soc_mb)(void); +extern void arm_heavy_mb(void); +#define __arm_heavy_mb(x...) do { dsb(x); arm_heavy_mb(); } while (0) +#else +#define __arm_heavy_mb(x...) dsb(x) +#endif + #ifdef CONFIG_ARCH_HAS_BARRIERS #include #elif defined(CONFIG_ARM_DMA_MEM_BUFFERABLE) || defined(CONFIG_SMP) -#define mb() do { dsb(); outer_sync(); } while (0) +#define mb() __arm_heavy_mb() #define rmb() dsb() -#define wmb() do { dsb(st); outer_sync(); } while (0) +#define wmb() __arm_heavy_mb(st) #define dma_rmb() dmb(osh) #define dma_wmb() dmb(oshst) #else @@ -67,12 +74,12 @@ do { \ compiletime_assert_atomic_type(*p); \ smp_mb(); \ - ACCESS_ONCE(*p) = (v); \ + WRITE_ONCE(*p, v); \ } while (0) #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ smp_mb(); \ ___p1; \ diff --git a/arch/arm/include/asm/bitops.h b/arch/arm/include/asm/bitops.h index 56380995f4c3..e943e6cee254 100644 --- a/arch/arm/include/asm/bitops.h +++ b/arch/arm/include/asm/bitops.h @@ -35,9 +35,9 @@ static inline void ____atomic_set_bit(unsigned int bit, volatile unsigned long *p) { unsigned long flags; - unsigned long mask = 1UL << (bit & 31); + unsigned long mask = BIT_MASK(bit); - p += bit >> 5; + p += BIT_WORD(bit); raw_local_irq_save(flags); *p |= mask; @@ -47,9 +47,9 @@ static inline void ____atomic_set_bit(unsigned int bit, volatile unsigned long * static inline void ____atomic_clear_bit(unsigned int bit, volatile unsigned long *p) { unsigned long flags; - unsigned long mask = 1UL << (bit & 31); + unsigned long mask = BIT_MASK(bit); - p += bit >> 5; + p += BIT_WORD(bit); raw_local_irq_save(flags); *p &= ~mask; @@ -59,9 +59,9 @@ static inline void ____atomic_clear_bit(unsigned int bit, volatile unsigned long static inline void ____atomic_change_bit(unsigned int bit, volatile unsigned long *p) { unsigned long flags; - unsigned long mask = 1UL << (bit & 31); + unsigned long mask = BIT_MASK(bit); - p += bit >> 5; + p += BIT_WORD(bit); raw_local_irq_save(flags); *p ^= mask; @@ -73,9 +73,9 @@ ____atomic_test_and_set_bit(unsigned int bit, volatile unsigned long *p) { unsigned long flags; unsigned int res; - unsigned long mask = 1UL << (bit & 31); + unsigned long mask = BIT_MASK(bit); - p += bit >> 5; + p += BIT_WORD(bit); raw_local_irq_save(flags); res = *p; @@ -90,9 +90,9 @@ ____atomic_test_and_clear_bit(unsigned int bit, volatile unsigned long *p) { unsigned long flags; unsigned int res; - unsigned long mask = 1UL << (bit & 31); + unsigned long mask = BIT_MASK(bit); - p += bit >> 5; + p += BIT_WORD(bit); raw_local_irq_save(flags); res = *p; @@ -107,9 +107,9 @@ ____atomic_test_and_change_bit(unsigned int bit, volatile unsigned long *p) { unsigned long flags; unsigned int res; - unsigned long mask = 1UL << (bit & 31); + unsigned long mask = BIT_MASK(bit); - p += bit >> 5; + p += BIT_WORD(bit); raw_local_irq_save(flags); res = *p; diff --git a/arch/arm/include/asm/cacheflush.h b/arch/arm/include/asm/cacheflush.h index 4812cda8fd17..d5525bfc7e3e 100644 --- a/arch/arm/include/asm/cacheflush.h +++ b/arch/arm/include/asm/cacheflush.h @@ -140,8 +140,6 @@ extern struct cpu_cache_fns cpu_cache; * is visible to DMA, or data written by DMA to system memory is * visible to the CPU. */ -#define dmac_map_area cpu_cache.dma_map_area -#define dmac_unmap_area cpu_cache.dma_unmap_area #define dmac_flush_range cpu_cache.dma_flush_range #else @@ -161,8 +159,6 @@ extern void __cpuc_flush_dcache_area(void *, size_t); * is visible to DMA, or data written by DMA to system memory is * visible to the CPU. */ -extern void dmac_map_area(const void *, size_t, int); -extern void dmac_unmap_area(const void *, size_t, int); extern void dmac_flush_range(const void *, const void *); #endif @@ -506,4 +502,21 @@ static inline void set_kernel_text_ro(void) { } void flush_uprobe_xol_access(struct page *page, unsigned long uaddr, void *kaddr, unsigned long len); +/** + * secure_flush_area - ensure coherency across the secure boundary + * @addr: virtual address + * @size: size of region + * + * Ensure that the specified area of memory is coherent across the secure + * boundary from the non-secure side. This is used when calling secure + * firmware where the secure firmware does not ensure coherency. + */ +static inline void secure_flush_area(const void *addr, size_t size) +{ + phys_addr_t phys = __pa(addr); + + __cpuc_flush_dcache_area((void *)addr, size); + outer_flush_range(phys, phys + size); +} + #endif diff --git a/arch/arm/include/asm/cmpxchg.h b/arch/arm/include/asm/cmpxchg.h index 1692a05d3207..916a2744d5c6 100644 --- a/arch/arm/include/asm/cmpxchg.h +++ b/arch/arm/include/asm/cmpxchg.h @@ -35,7 +35,6 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size unsigned int tmp; #endif - smp_mb(); prefetchw((const void *)ptr); switch (size) { @@ -98,12 +97,11 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size __bad_xchg(ptr, size), ret = 0; break; } - smp_mb(); return ret; } -#define xchg(ptr, x) ({ \ +#define xchg_relaxed(ptr, x) ({ \ (__typeof__(*(ptr)))__xchg((unsigned long)(x), (ptr), \ sizeof(*(ptr))); \ }) @@ -117,6 +115,8 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size #error "SMP is not supported on this platform" #endif +#define xchg xchg_relaxed + /* * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make * them available. @@ -194,23 +194,11 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, return oldval; } -static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, - unsigned long new, int size) -{ - unsigned long ret; - - smp_mb(); - ret = __cmpxchg(ptr, old, new, size); - smp_mb(); - - return ret; -} - -#define cmpxchg(ptr,o,n) ({ \ - (__typeof__(*(ptr)))__cmpxchg_mb((ptr), \ - (unsigned long)(o), \ - (unsigned long)(n), \ - sizeof(*(ptr))); \ +#define cmpxchg_relaxed(ptr,o,n) ({ \ + (__typeof__(*(ptr)))__cmpxchg((ptr), \ + (unsigned long)(o), \ + (unsigned long)(n), \ + sizeof(*(ptr))); \ }) static inline unsigned long __cmpxchg_local(volatile void *ptr, @@ -273,25 +261,6 @@ static inline unsigned long long __cmpxchg64(unsigned long long *ptr, #define cmpxchg64_local(ptr, o, n) cmpxchg64_relaxed((ptr), (o), (n)) -static inline unsigned long long __cmpxchg64_mb(unsigned long long *ptr, - unsigned long long old, - unsigned long long new) -{ - unsigned long long ret; - - smp_mb(); - ret = __cmpxchg64(ptr, old, new); - smp_mb(); - - return ret; -} - -#define cmpxchg64(ptr, o, n) ({ \ - (__typeof__(*(ptr)))__cmpxchg64_mb((ptr), \ - (unsigned long long)(o), \ - (unsigned long long)(n)); \ -}) - #endif /* __LINUX_ARM_ARCH__ >= 6 */ #endif /* __ASM_ARM_CMPXCHG_H */ diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h index b52101d37ec7..ccb3aa64640d 100644 --- a/arch/arm/include/asm/dma-mapping.h +++ b/arch/arm/include/asm/dma-mapping.h @@ -8,13 +8,12 @@ #include #include -#include #include #include #include -#define DMA_ERROR_CODE (~0) +#define DMA_ERROR_CODE (~(dma_addr_t)0x0) extern struct dma_map_ops arm_dma_ops; extern struct dma_map_ops arm_coherent_dma_ops; @@ -39,12 +38,15 @@ static inline void set_dma_ops(struct device *dev, struct dma_map_ops *ops) dev->archdata.dma_ops = ops; } -#include +#define HAVE_ARCH_DMA_SUPPORTED 1 +extern int dma_supported(struct device *dev, u64 mask); -static inline int dma_set_mask(struct device *dev, u64 mask) -{ - return get_dma_ops(dev)->set_dma_mask(dev, mask); -} +/* + * Note that while the generic code provides dummy dma_{alloc,free}_noncoherent + * implementations, we don't provide a dma_cache_sync function so drivers using + * this API are highlighted with build warnings. + */ +#include #ifdef __arch_page_to_dma #error Please update to __arch_pfn_to_dma @@ -167,32 +169,6 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) static inline void dma_mark_clean(void *addr, size_t size) { } -/* - * DMA errors are defined by all-bits-set in the DMA address. - */ -static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - debug_dma_mapping_error(dev, dma_addr); - return dma_addr == DMA_ERROR_CODE; -} - -/* - * Dummy noncoherent implementation. We don't provide a dma_cache_sync - * function so drivers using this API are highlighted with build warnings. - */ -static inline void *dma_alloc_noncoherent(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp) -{ - return NULL; -} - -static inline void dma_free_noncoherent(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t handle) -{ -} - -extern int dma_supported(struct device *dev, u64 mask); - extern int arm_dma_set_mask(struct device *dev, u64 dma_mask); /** @@ -209,21 +185,6 @@ extern int arm_dma_set_mask(struct device *dev, u64 dma_mask); extern void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs); -#define dma_alloc_coherent(d, s, h, f) dma_alloc_attrs(d, s, h, f, NULL) - -static inline void *dma_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - void *cpu_addr; - BUG_ON(!ops); - - cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs); - debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr); - return cpu_addr; -} - /** * arm_dma_free - free memory allocated by arm_dma_alloc * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices @@ -241,19 +202,6 @@ static inline void *dma_alloc_attrs(struct device *dev, size_t size, extern void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle, struct dma_attrs *attrs); -#define dma_free_coherent(d, s, c, h) dma_free_attrs(d, s, c, h, NULL) - -static inline void dma_free_attrs(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t dma_handle, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - BUG_ON(!ops); - - debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); - ops->free(dev, size, cpu_addr, dma_handle, attrs); -} - /** * arm_dma_mmap - map a coherent DMA allocation into user space * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices diff --git a/arch/arm/include/asm/domain.h b/arch/arm/include/asm/domain.h index 6ddbe446425e..e878129f2fee 100644 --- a/arch/arm/include/asm/domain.h +++ b/arch/arm/include/asm/domain.h @@ -34,15 +34,14 @@ */ #ifndef CONFIG_IO_36 #define DOMAIN_KERNEL 0 -#define DOMAIN_TABLE 0 #define DOMAIN_USER 1 #define DOMAIN_IO 2 #else #define DOMAIN_KERNEL 2 -#define DOMAIN_TABLE 2 #define DOMAIN_USER 1 #define DOMAIN_IO 0 #endif +#define DOMAIN_VECTORS 3 /* * Domain types @@ -55,11 +54,46 @@ #define DOMAIN_MANAGER 1 #endif -#define domain_val(dom,type) ((type) << (2*(dom))) +#define domain_mask(dom) ((3) << (2 * (dom))) +#define domain_val(dom,type) ((type) << (2 * (dom))) + +#ifdef CONFIG_CPU_SW_DOMAIN_PAN +#define DACR_INIT \ + (domain_val(DOMAIN_USER, DOMAIN_NOACCESS) | \ + domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ + domain_val(DOMAIN_IO, DOMAIN_CLIENT) | \ + domain_val(DOMAIN_VECTORS, DOMAIN_CLIENT)) +#else +#define DACR_INIT \ + (domain_val(DOMAIN_USER, DOMAIN_CLIENT) | \ + domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ + domain_val(DOMAIN_IO, DOMAIN_CLIENT) | \ + domain_val(DOMAIN_VECTORS, DOMAIN_CLIENT)) +#endif + +#define __DACR_DEFAULT \ + domain_val(DOMAIN_KERNEL, DOMAIN_CLIENT) | \ + domain_val(DOMAIN_IO, DOMAIN_CLIENT) | \ + domain_val(DOMAIN_VECTORS, DOMAIN_CLIENT) + +#define DACR_UACCESS_DISABLE \ + (__DACR_DEFAULT | domain_val(DOMAIN_USER, DOMAIN_NOACCESS)) +#define DACR_UACCESS_ENABLE \ + (__DACR_DEFAULT | domain_val(DOMAIN_USER, DOMAIN_CLIENT)) #ifndef __ASSEMBLY__ -#ifdef CONFIG_CPU_USE_DOMAINS +static inline unsigned int get_domain(void) +{ + unsigned int domain; + + asm( + "mrc p15, 0, %0, c3, c0 @ get domain" + : "=r" (domain)); + + return domain; +} + static inline void set_domain(unsigned val) { asm volatile( @@ -68,17 +102,16 @@ static inline void set_domain(unsigned val) isb(); } +#ifdef CONFIG_CPU_USE_DOMAINS #define modify_domain(dom,type) \ do { \ - struct thread_info *thread = current_thread_info(); \ - unsigned int domain = thread->cpu_domain; \ - domain &= ~domain_val(dom, DOMAIN_MANAGER); \ - thread->cpu_domain = domain | domain_val(dom, type); \ - set_domain(thread->cpu_domain); \ + unsigned int domain = get_domain(); \ + domain &= ~domain_mask(dom); \ + domain = domain | domain_val(dom, type); \ + set_domain(domain); \ } while (0) #else -static inline void set_domain(unsigned val) { } static inline void modify_domain(unsigned dom, unsigned type) { } #endif diff --git a/arch/arm/include/asm/fixmap.h b/arch/arm/include/asm/fixmap.h index 0415eae1df27..58cfe9f1a687 100644 --- a/arch/arm/include/asm/fixmap.h +++ b/arch/arm/include/asm/fixmap.h @@ -6,9 +6,13 @@ #define FIXADDR_TOP (FIXADDR_END - PAGE_SIZE) #include +#include enum fixed_addresses { - FIX_KMAP_BEGIN, + FIX_EARLYCON_MEM_BASE, + __end_of_permanent_fixed_addresses, + + FIX_KMAP_BEGIN = __end_of_permanent_fixed_addresses, FIX_KMAP_END = FIX_KMAP_BEGIN + (KM_TYPE_NR * NR_CPUS) - 1, /* Support writing RO kernel text via kprobes, jump labels, etc. */ @@ -18,7 +22,16 @@ enum fixed_addresses { __end_of_fixed_addresses }; +#define FIXMAP_PAGE_COMMON (L_PTE_YOUNG | L_PTE_PRESENT | L_PTE_XN | L_PTE_DIRTY) + +#define FIXMAP_PAGE_NORMAL (FIXMAP_PAGE_COMMON | L_PTE_MT_WRITEBACK) + +/* Used by set_fixmap_(io|nocache), both meant for mapping a device */ +#define FIXMAP_PAGE_IO (FIXMAP_PAGE_COMMON | L_PTE_MT_DEV_SHARED | L_PTE_SHARED) +#define FIXMAP_PAGE_NOCACHE FIXMAP_PAGE_IO + void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot); +void __init early_fixmap_init(void); #include diff --git a/arch/arm/include/asm/futex.h b/arch/arm/include/asm/futex.h index 5eed82809d82..6795368ad023 100644 --- a/arch/arm/include/asm/futex.h +++ b/arch/arm/include/asm/futex.h @@ -22,8 +22,11 @@ #ifdef CONFIG_SMP #define __futex_atomic_op(insn, ret, oldval, tmp, uaddr, oparg) \ +({ \ + unsigned int __ua_flags; \ smp_mb(); \ prefetchw(uaddr); \ + __ua_flags = uaccess_save_and_enable(); \ __asm__ __volatile__( \ "1: ldrex %1, [%3]\n" \ " " insn "\n" \ @@ -34,12 +37,15 @@ __futex_atomic_ex_table("%5") \ : "=&r" (ret), "=&r" (oldval), "=&r" (tmp) \ : "r" (uaddr), "r" (oparg), "Ir" (-EFAULT) \ - : "cc", "memory") + : "cc", "memory"); \ + uaccess_restore(__ua_flags); \ +}) static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, u32 newval) { + unsigned int __ua_flags; int ret; u32 val; @@ -49,6 +55,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, smp_mb(); /* Prefetching cannot fault */ prefetchw(uaddr); + __ua_flags = uaccess_save_and_enable(); __asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n" "1: ldrex %1, [%4]\n" " teq %1, %2\n" @@ -61,6 +68,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, : "=&r" (ret), "=&r" (val) : "r" (oldval), "r" (newval), "r" (uaddr), "Ir" (-EFAULT) : "cc", "memory"); + uaccess_restore(__ua_flags); smp_mb(); *uval = val; @@ -73,6 +81,8 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, #include #define __futex_atomic_op(insn, ret, oldval, tmp, uaddr, oparg) \ +({ \ + unsigned int __ua_flags = uaccess_save_and_enable(); \ __asm__ __volatile__( \ "1: " TUSER(ldr) " %1, [%3]\n" \ " " insn "\n" \ @@ -81,12 +91,15 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, __futex_atomic_ex_table("%5") \ : "=&r" (ret), "=&r" (oldval), "=&r" (tmp) \ : "r" (uaddr), "r" (oparg), "Ir" (-EFAULT) \ - : "cc", "memory") + : "cc", "memory"); \ + uaccess_restore(__ua_flags); \ +}) static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, u32 newval) { + unsigned int __ua_flags; int ret = 0; u32 val; @@ -94,6 +107,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, return -EFAULT; preempt_disable(); + __ua_flags = uaccess_save_and_enable(); __asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n" "1: " TUSER(ldr) " %1, [%4]\n" " teq %1, %2\n" @@ -103,6 +117,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, : "+r" (ret), "=&r" (val) : "r" (oldval), "r" (newval), "r" (uaddr), "Ir" (-EFAULT) : "cc", "memory"); + uaccess_restore(__ua_flags); *uval = val; preempt_enable(); diff --git a/arch/arm/include/asm/glue-cache.h b/arch/arm/include/asm/glue-cache.h index a3c24cd5b7c8..cab07f69382d 100644 --- a/arch/arm/include/asm/glue-cache.h +++ b/arch/arm/include/asm/glue-cache.h @@ -158,8 +158,6 @@ static inline void nop_dma_unmap_area(const void *s, size_t l, int f) { } #define __cpuc_coherent_user_range __glue(_CACHE,_coherent_user_range) #define __cpuc_flush_dcache_area __glue(_CACHE,_flush_kern_dcache_area) -#define dmac_map_area __glue(_CACHE,_dma_map_area) -#define dmac_unmap_area __glue(_CACHE,_dma_unmap_area) #define dmac_flush_range __glue(_CACHE,_dma_flush_range) #endif diff --git a/arch/arm/include/asm/irq.h b/arch/arm/include/asm/irq.h index 53c15dec7af6..be1d07d59ee9 100644 --- a/arch/arm/include/asm/irq.h +++ b/arch/arm/include/asm/irq.h @@ -35,6 +35,11 @@ extern void (*handle_arch_irq)(struct pt_regs *); extern void set_handle_irq(void (*handle_irq)(struct pt_regs *)); #endif +#ifdef CONFIG_SMP +extern void arch_trigger_all_cpu_backtrace(bool); +#define arch_trigger_all_cpu_backtrace(x) arch_trigger_all_cpu_backtrace(x) +#endif + #endif #endif diff --git a/arch/arm/include/asm/jump_label.h b/arch/arm/include/asm/jump_label.h index 5f337dc5c108..34f7b6980d21 100644 --- a/arch/arm/include/asm/jump_label.h +++ b/arch/arm/include/asm/jump_label.h @@ -4,23 +4,32 @@ #ifndef __ASSEMBLY__ #include +#include #define JUMP_LABEL_NOP_SIZE 4 -#ifdef CONFIG_THUMB2_KERNEL -#define JUMP_LABEL_NOP "nop.w" -#else -#define JUMP_LABEL_NOP "nop" -#endif - -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("1:\n\t" - JUMP_LABEL_NOP "\n\t" + WASM(nop) "\n\t" ".pushsection __jump_table, \"aw\"\n\t" ".word 1b, %l[l_yes], %c0\n\t" ".popsection\n\t" - : : "i" (key) : : l_yes); + : : "i" (&((char *)key)[branch]) : : l_yes); + + return false; +l_yes: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("1:\n\t" + WASM(b) " %l[l_yes]\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".word 1b, %l[l_yes], %c0\n\t" + ".popsection\n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); return false; l_yes: diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h index e896d2c196e6..dcba0fa5176e 100644 --- a/arch/arm/include/asm/kvm_host.h +++ b/arch/arm/include/asm/kvm_host.h @@ -231,4 +231,9 @@ static inline void kvm_arch_sync_events(struct kvm *kvm) {} static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} +static inline void kvm_arm_init_debug(void) {} +static inline void kvm_arm_setup_debug(struct kvm_vcpu *vcpu) {} +static inline void kvm_arm_clear_debug(struct kvm_vcpu *vcpu) {} +static inline void kvm_arm_reset_debug_ptr(struct kvm_vcpu *vcpu) {} + #endif /* __ARM_KVM_HOST_H__ */ diff --git a/arch/arm/include/asm/memory.h b/arch/arm/include/asm/memory.h index b7f6fb462ea0..98d58bb04ac5 100644 --- a/arch/arm/include/asm/memory.h +++ b/arch/arm/include/asm/memory.h @@ -118,12 +118,6 @@ #define DTCM_OFFSET UL(0xfffe8000) #endif -/* - * Convert a physical address to a Page Frame Number and back - */ -#define __phys_to_pfn(paddr) ((unsigned long)((paddr) >> PAGE_SHIFT)) -#define __pfn_to_phys(pfn) ((phys_addr_t)(pfn) << PAGE_SHIFT) - /* * Convert a page to/from a physical address */ diff --git a/arch/arm/include/asm/outercache.h b/arch/arm/include/asm/outercache.h index 563b92fc2f41..c2bf24f40177 100644 --- a/arch/arm/include/asm/outercache.h +++ b/arch/arm/include/asm/outercache.h @@ -129,21 +129,4 @@ static inline void outer_resume(void) { } #endif -#ifdef CONFIG_OUTER_CACHE_SYNC -/** - * outer_sync - perform a sync point for outer cache - * - * Ensure that all outer cache operations are complete and any store - * buffers are drained. - */ -static inline void outer_sync(void) -{ - if (outer_cache.sync) - outer_cache.sync(); -} -#else -static inline void outer_sync(void) -{ } -#endif - #endif /* __ASM_OUTERCACHE_H */ diff --git a/arch/arm/include/asm/pgtable-2level-hwdef.h b/arch/arm/include/asm/pgtable-2level-hwdef.h index 5e68278e953e..d0131ee6f6af 100644 --- a/arch/arm/include/asm/pgtable-2level-hwdef.h +++ b/arch/arm/include/asm/pgtable-2level-hwdef.h @@ -23,6 +23,7 @@ #define PMD_PXNTABLE (_AT(pmdval_t, 1) << 2) /* v7 */ #define PMD_BIT4 (_AT(pmdval_t, 1) << 4) #define PMD_DOMAIN(x) (_AT(pmdval_t, (x)) << 5) +#define PMD_DOMAIN_MASK PMD_DOMAIN(0x0f) #define PMD_PROTECTION (_AT(pmdval_t, 1) << 9) /* v5 */ /* * - section diff --git a/arch/arm/include/asm/psci.h b/arch/arm/include/asm/psci.h index c25ef3ec6d1f..68ee3ce17b82 100644 --- a/arch/arm/include/asm/psci.h +++ b/arch/arm/include/asm/psci.h @@ -14,34 +14,11 @@ #ifndef __ASM_ARM_PSCI_H #define __ASM_ARM_PSCI_H -#define PSCI_POWER_STATE_TYPE_STANDBY 0 -#define PSCI_POWER_STATE_TYPE_POWER_DOWN 1 - -struct psci_power_state { - u16 id; - u8 type; - u8 affinity_level; -}; - -struct psci_operations { - int (*cpu_suspend)(struct psci_power_state state, - unsigned long entry_point); - int (*cpu_off)(struct psci_power_state state); - int (*cpu_on)(unsigned long cpuid, unsigned long entry_point); - int (*migrate)(unsigned long cpuid); - int (*affinity_info)(unsigned long target_affinity, - unsigned long lowest_affinity_level); - int (*migrate_info_type)(void); -}; - -extern struct psci_operations psci_ops; extern struct smp_operations psci_smp_ops; #ifdef CONFIG_ARM_PSCI -int psci_init(void); bool psci_smp_available(void); #else -static inline int psci_init(void) { return 0; } static inline bool psci_smp_available(void) { return false; } #endif diff --git a/arch/arm/include/asm/smp.h b/arch/arm/include/asm/smp.h index 2f3ac1ba6fb4..ef356659b4f4 100644 --- a/arch/arm/include/asm/smp.h +++ b/arch/arm/include/asm/smp.h @@ -74,7 +74,6 @@ extern void secondary_startup_arm(void); extern int __cpu_disable(void); extern void __cpu_die(unsigned int cpu); -extern void cpu_die(void); extern void arch_send_call_function_single_ipi(int cpu); extern void arch_send_call_function_ipi_mask(const struct cpumask *mask); @@ -105,6 +104,7 @@ struct smp_operations { #ifdef CONFIG_HOTPLUG_CPU int (*cpu_kill)(unsigned int cpu); void (*cpu_die)(unsigned int cpu); + bool (*cpu_can_disable)(unsigned int cpu); int (*cpu_disable)(unsigned int cpu); #endif #endif diff --git a/arch/arm/include/asm/smp_plat.h b/arch/arm/include/asm/smp_plat.h index 993e5224d8f7..f9080717fc88 100644 --- a/arch/arm/include/asm/smp_plat.h +++ b/arch/arm/include/asm/smp_plat.h @@ -107,4 +107,13 @@ static inline u32 mpidr_hash_size(void) extern int platform_can_secondary_boot(void); extern int platform_can_cpu_hotplug(void); +#ifdef CONFIG_HOTPLUG_CPU +extern int platform_can_hotplug_cpu(unsigned int cpu); +#else +static inline int platform_can_hotplug_cpu(unsigned int cpu) +{ + return 0; +} +#endif + #endif diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index bd32eded3e50..d0a1119dcaf3 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -74,9 +74,6 @@ struct thread_info { .flags = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ .addr_limit = KERNEL_DS, \ - .cpu_domain = domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \ - domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ - domain_val(DOMAIN_IO, DOMAIN_CLIENT), \ } #define init_thread_info (init_thread_union.thread_info) @@ -136,22 +133,18 @@ extern int vfp_restore_user_hwstate(struct user_vfp __user *, /* * thread information flags: - * TIF_SYSCALL_TRACE - syscall trace active - * TIF_SYSCAL_AUDIT - syscall auditing active - * TIF_SIGPENDING - signal pending - * TIF_NEED_RESCHED - rescheduling necessary - * TIF_NOTIFY_RESUME - callback before returning to user * TIF_USEDFPU - FPU was used by this task this quantum (SMP) * TIF_POLLING_NRFLAG - true if poll_idle() is polling TIF_NEED_RESCHED */ -#define TIF_SIGPENDING 0 -#define TIF_NEED_RESCHED 1 +#define TIF_SIGPENDING 0 /* signal pending */ +#define TIF_NEED_RESCHED 1 /* rescheduling necessary */ #define TIF_NOTIFY_RESUME 2 /* callback before returning to user */ -#define TIF_UPROBE 7 -#define TIF_SYSCALL_TRACE 8 -#define TIF_SYSCALL_AUDIT 9 -#define TIF_SYSCALL_TRACEPOINT 10 -#define TIF_SECCOMP 11 /* seccomp syscall filtering active */ +#define TIF_UPROBE 3 /* breakpointed or singlestepping */ +#define TIF_SYSCALL_TRACE 4 /* syscall trace active */ +#define TIF_SYSCALL_AUDIT 5 /* syscall auditing active */ +#define TIF_SYSCALL_TRACEPOINT 6 /* syscall tracepoint instrumentation */ +#define TIF_SECCOMP 7 /* seccomp syscall filtering active */ + #define TIF_NOHZ 12 /* in adaptive nohz mode */ #define TIF_USING_IWMMXT 17 #define TIF_MEMDIE 18 /* is terminating due to OOM killer */ diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index 74b17d09ef7a..8cc85a4ebec2 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -49,6 +49,35 @@ struct exception_table_entry extern int fixup_exception(struct pt_regs *regs); +/* + * These two functions allow hooking accesses to userspace to increase + * system integrity by ensuring that the kernel can not inadvertantly + * perform such accesses (eg, via list poison values) which could then + * be exploited for priviledge escalation. + */ +static inline unsigned int uaccess_save_and_enable(void) +{ +#ifdef CONFIG_CPU_SW_DOMAIN_PAN + unsigned int old_domain = get_domain(); + + /* Set the current domain access to permit user accesses */ + set_domain((old_domain & ~domain_mask(DOMAIN_USER)) | + domain_val(DOMAIN_USER, DOMAIN_CLIENT)); + + return old_domain; +#else + return 0; +#endif +} + +static inline void uaccess_restore(unsigned int flags) +{ +#ifdef CONFIG_CPU_SW_DOMAIN_PAN + /* Restore the user access mask */ + set_domain(flags); +#endif +} + /* * These two are intentionally not defined anywhere - if the kernel * code generates any references to them, that's a bug. @@ -165,6 +194,7 @@ extern int __get_user_64t_4(void *); register typeof(x) __r2 asm("r2"); \ register unsigned long __l asm("r1") = __limit; \ register int __e asm("r0"); \ + unsigned int __ua_flags = uaccess_save_and_enable(); \ switch (sizeof(*(__p))) { \ case 1: \ if (sizeof((x)) >= 8) \ @@ -192,6 +222,7 @@ extern int __get_user_64t_4(void *); break; \ default: __e = __get_user_bad(); break; \ } \ + uaccess_restore(__ua_flags); \ x = (typeof(*(p))) __r2; \ __e; \ }) @@ -224,6 +255,7 @@ extern int __put_user_8(void *, unsigned long long); register const typeof(*(p)) __user *__p asm("r0") = __tmp_p; \ register unsigned long __l asm("r1") = __limit; \ register int __e asm("r0"); \ + unsigned int __ua_flags = uaccess_save_and_enable(); \ switch (sizeof(*(__p))) { \ case 1: \ __put_user_x(__r2, __p, __e, __l, 1); \ @@ -239,6 +271,7 @@ extern int __put_user_8(void *, unsigned long long); break; \ default: __e = __put_user_bad(); break; \ } \ + uaccess_restore(__ua_flags); \ __e; \ }) @@ -300,20 +333,23 @@ static inline void set_fs(mm_segment_t fs) do { \ unsigned long __gu_addr = (unsigned long)(ptr); \ unsigned long __gu_val; \ + unsigned int __ua_flags; \ __chk_user_ptr(ptr); \ might_fault(); \ + __ua_flags = uaccess_save_and_enable(); \ switch (sizeof(*(ptr))) { \ case 1: __get_user_asm_byte(__gu_val, __gu_addr, err); break; \ case 2: __get_user_asm_half(__gu_val, __gu_addr, err); break; \ case 4: __get_user_asm_word(__gu_val, __gu_addr, err); break; \ default: (__gu_val) = __get_user_bad(); \ } \ + uaccess_restore(__ua_flags); \ (x) = (__typeof__(*(ptr)))__gu_val; \ } while (0) -#define __get_user_asm_byte(x, addr, err) \ +#define __get_user_asm(x, addr, err, instr) \ __asm__ __volatile__( \ - "1: " TUSER(ldrb) " %1,[%2],#0\n" \ + "1: " TUSER(instr) " %1, [%2], #0\n" \ "2:\n" \ " .pushsection .text.fixup,\"ax\"\n" \ " .align 2\n" \ @@ -329,6 +365,9 @@ do { \ : "r" (addr), "i" (-EFAULT) \ : "cc") +#define __get_user_asm_byte(x, addr, err) \ + __get_user_asm(x, addr, err, ldrb) + #ifndef __ARMEB__ #define __get_user_asm_half(x, __gu_addr, err) \ ({ \ @@ -348,22 +387,7 @@ do { \ #endif #define __get_user_asm_word(x, addr, err) \ - __asm__ __volatile__( \ - "1: " TUSER(ldr) " %1,[%2],#0\n" \ - "2:\n" \ - " .pushsection .text.fixup,\"ax\"\n" \ - " .align 2\n" \ - "3: mov %0, %3\n" \ - " mov %1, #0\n" \ - " b 2b\n" \ - " .popsection\n" \ - " .pushsection __ex_table,\"a\"\n" \ - " .align 3\n" \ - " .long 1b, 3b\n" \ - " .popsection" \ - : "+r" (err), "=&r" (x) \ - : "r" (addr), "i" (-EFAULT) \ - : "cc") + __get_user_asm(x, addr, err, ldr) #define __put_user(x, ptr) \ ({ \ @@ -381,9 +405,11 @@ do { \ #define __put_user_err(x, ptr, err) \ do { \ unsigned long __pu_addr = (unsigned long)(ptr); \ + unsigned int __ua_flags; \ __typeof__(*(ptr)) __pu_val = (x); \ __chk_user_ptr(ptr); \ might_fault(); \ + __ua_flags = uaccess_save_and_enable(); \ switch (sizeof(*(ptr))) { \ case 1: __put_user_asm_byte(__pu_val, __pu_addr, err); break; \ case 2: __put_user_asm_half(__pu_val, __pu_addr, err); break; \ @@ -391,11 +417,12 @@ do { \ case 8: __put_user_asm_dword(__pu_val, __pu_addr, err); break; \ default: __put_user_bad(); \ } \ + uaccess_restore(__ua_flags); \ } while (0) -#define __put_user_asm_byte(x, __pu_addr, err) \ +#define __put_user_asm(x, __pu_addr, err, instr) \ __asm__ __volatile__( \ - "1: " TUSER(strb) " %1,[%2],#0\n" \ + "1: " TUSER(instr) " %1, [%2], #0\n" \ "2:\n" \ " .pushsection .text.fixup,\"ax\"\n" \ " .align 2\n" \ @@ -410,6 +437,9 @@ do { \ : "r" (x), "r" (__pu_addr), "i" (-EFAULT) \ : "cc") +#define __put_user_asm_byte(x, __pu_addr, err) \ + __put_user_asm(x, __pu_addr, err, strb) + #ifndef __ARMEB__ #define __put_user_asm_half(x, __pu_addr, err) \ ({ \ @@ -427,21 +457,7 @@ do { \ #endif #define __put_user_asm_word(x, __pu_addr, err) \ - __asm__ __volatile__( \ - "1: " TUSER(str) " %1,[%2],#0\n" \ - "2:\n" \ - " .pushsection .text.fixup,\"ax\"\n" \ - " .align 2\n" \ - "3: mov %0, %3\n" \ - " b 2b\n" \ - " .popsection\n" \ - " .pushsection __ex_table,\"a\"\n" \ - " .align 3\n" \ - " .long 1b, 3b\n" \ - " .popsection" \ - : "+r" (err) \ - : "r" (x), "r" (__pu_addr), "i" (-EFAULT) \ - : "cc") + __put_user_asm(x, __pu_addr, err, str) #ifndef __ARMEB__ #define __reg_oper0 "%R2" @@ -474,11 +490,46 @@ do { \ #ifdef CONFIG_MMU -extern unsigned long __must_check __copy_from_user(void *to, const void __user *from, unsigned long n); -extern unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n); -extern unsigned long __must_check __copy_to_user_std(void __user *to, const void *from, unsigned long n); -extern unsigned long __must_check __clear_user(void __user *addr, unsigned long n); -extern unsigned long __must_check __clear_user_std(void __user *addr, unsigned long n); +extern unsigned long __must_check +arm_copy_from_user(void *to, const void __user *from, unsigned long n); + +static inline unsigned long __must_check +__copy_from_user(void *to, const void __user *from, unsigned long n) +{ + unsigned int __ua_flags = uaccess_save_and_enable(); + n = arm_copy_from_user(to, from, n); + uaccess_restore(__ua_flags); + return n; +} + +extern unsigned long __must_check +arm_copy_to_user(void __user *to, const void *from, unsigned long n); +extern unsigned long __must_check +__copy_to_user_std(void __user *to, const void *from, unsigned long n); + +static inline unsigned long __must_check +__copy_to_user(void __user *to, const void *from, unsigned long n) +{ + unsigned int __ua_flags = uaccess_save_and_enable(); + n = arm_copy_to_user(to, from, n); + uaccess_restore(__ua_flags); + return n; +} + +extern unsigned long __must_check +arm_clear_user(void __user *addr, unsigned long n); +extern unsigned long __must_check +__clear_user_std(void __user *addr, unsigned long n); + +static inline unsigned long __must_check +__clear_user(void __user *addr, unsigned long n) +{ + unsigned int __ua_flags = uaccess_save_and_enable(); + n = arm_clear_user(addr, n); + uaccess_restore(__ua_flags); + return n; +} + #else #define __copy_from_user(to, from, n) (memcpy(to, (void __force *)from, n), 0) #define __copy_to_user(to, from, n) (memcpy((void __force *)to, from, n), 0) @@ -511,6 +562,7 @@ static inline unsigned long __must_check clear_user(void __user *to, unsigned lo return n; } +/* These are from lib/ code, and use __get_user() and friends */ extern long strncpy_from_user(char *dest, const char __user *src, long count); extern __must_check long strlen_user(const char __user *str); diff --git a/arch/arm/include/asm/xen/events.h b/arch/arm/include/asm/xen/events.h index 8b1f37bfeeec..71e473d05fcc 100644 --- a/arch/arm/include/asm/xen/events.h +++ b/arch/arm/include/asm/xen/events.h @@ -20,4 +20,10 @@ static inline int xen_irqs_disabled(struct pt_regs *regs) atomic64_t, \ counter), (val)) +/* Rebind event channel is supported by default */ +static inline bool xen_support_evtchn_rebind(void) +{ + return true; +} + #endif /* _ASM_ARM_XEN_EVENTS_H */ diff --git a/arch/arm/include/asm/xen/page.h b/arch/arm/include/asm/xen/page.h index 1bee8ca12494..127956353b00 100644 --- a/arch/arm/include/asm/xen/page.h +++ b/arch/arm/include/asm/xen/page.h @@ -34,7 +34,19 @@ typedef struct xpaddr { unsigned long __pfn_to_mfn(unsigned long pfn); extern struct rb_root phys_to_mach; -static inline unsigned long pfn_to_mfn(unsigned long pfn) +/* Pseudo-physical <-> Guest conversion */ +static inline unsigned long pfn_to_gfn(unsigned long pfn) +{ + return pfn; +} + +static inline unsigned long gfn_to_pfn(unsigned long gfn) +{ + return gfn; +} + +/* Pseudo-physical <-> BUS conversion */ +static inline unsigned long pfn_to_bfn(unsigned long pfn) { unsigned long mfn; @@ -47,33 +59,21 @@ static inline unsigned long pfn_to_mfn(unsigned long pfn) return pfn; } -static inline unsigned long mfn_to_pfn(unsigned long mfn) +static inline unsigned long bfn_to_pfn(unsigned long bfn) { - return mfn; + return bfn; } -#define mfn_to_local_pfn(mfn) mfn_to_pfn(mfn) +#define bfn_to_local_pfn(bfn) bfn_to_pfn(bfn) -static inline xmaddr_t phys_to_machine(xpaddr_t phys) -{ - unsigned offset = phys.paddr & ~PAGE_MASK; - return XMADDR(PFN_PHYS(pfn_to_mfn(PFN_DOWN(phys.paddr))) | offset); -} - -static inline xpaddr_t machine_to_phys(xmaddr_t machine) -{ - unsigned offset = machine.maddr & ~PAGE_MASK; - return XPADDR(PFN_PHYS(mfn_to_pfn(PFN_DOWN(machine.maddr))) | offset); -} -/* VIRT <-> MACHINE conversion */ -#define virt_to_machine(v) (phys_to_machine(XPADDR(__pa(v)))) -#define virt_to_mfn(v) (pfn_to_mfn(virt_to_pfn(v))) -#define mfn_to_virt(m) (__va(mfn_to_pfn(m) << PAGE_SHIFT)) +/* VIRT <-> GUEST conversion */ +#define virt_to_gfn(v) (pfn_to_gfn(virt_to_pfn(v))) +#define gfn_to_virt(m) (__va(gfn_to_pfn(m) << PAGE_SHIFT)) +/* Only used in PV code. But ARM guests are always HVM. */ static inline xmaddr_t arbitrary_virt_to_machine(void *vaddr) { - /* TODO: assuming it is mapped in the kernel 1:1 */ - return virt_to_machine(vaddr); + BUG(); } /* TODO: this shouldn't be here but it is because the frontend drivers @@ -108,7 +108,7 @@ static inline bool set_phys_to_machine(unsigned long pfn, unsigned long mfn) bool xen_arch_need_swiotlb(struct device *dev, unsigned long pfn, - unsigned long mfn); + unsigned long bfn); unsigned long xen_get_swiotlb_free_pages(unsigned int order); #endif /* _ASM_ARM_XEN_PAGE_H */ diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index e69f7a19735d..af9e59bf3831 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -71,8 +71,7 @@ obj-$(CONFIG_CPU_PJ4) += pj4-cp0.o obj-$(CONFIG_CPU_PJ4B) += pj4-cp0.o obj-$(CONFIG_IWMMXT) += iwmmxt.o obj-$(CONFIG_PERF_EVENTS) += perf_regs.o perf_callchain.o -obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o \ - perf_event_xscale.o perf_event_v6.o \ +obj-$(CONFIG_HW_PERF_EVENTS) += perf_event_xscale.o perf_event_v6.o \ perf_event_v7.o CFLAGS_pj4-cp0.o := -marm AFLAGS_iwmmxt.o := -Wa,-mcpu=iwmmxt @@ -89,7 +88,7 @@ obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-$(CONFIG_ARM_VIRT_EXT) += hyp-stub.o ifeq ($(CONFIG_ARM_PSCI),y) -obj-y += psci.o psci-call.o +obj-y += psci-call.o obj-$(CONFIG_SMP) += psci_smp.o endif diff --git a/arch/arm/kernel/armksyms.c b/arch/arm/kernel/armksyms.c index 5e5a51a99e68..f89811fb9a55 100644 --- a/arch/arm/kernel/armksyms.c +++ b/arch/arm/kernel/armksyms.c @@ -97,9 +97,9 @@ EXPORT_SYMBOL(mmiocpy); #ifdef CONFIG_MMU EXPORT_SYMBOL(copy_page); -EXPORT_SYMBOL(__copy_from_user); -EXPORT_SYMBOL(__copy_to_user); -EXPORT_SYMBOL(__clear_user); +EXPORT_SYMBOL(arm_copy_from_user); +EXPORT_SYMBOL(arm_copy_to_user); +EXPORT_SYMBOL(arm_clear_user); EXPORT_SYMBOL(__get_user_1); EXPORT_SYMBOL(__get_user_2); diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index cb4fb1e69778..3e1c26eb32b4 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -149,10 +149,10 @@ ENDPROC(__und_invalid) #define SPFIX(code...) #endif - .macro svc_entry, stack_hole=0, trace=1 + .macro svc_entry, stack_hole=0, trace=1, uaccess=1 UNWIND(.fnstart ) UNWIND(.save {r0 - pc} ) - sub sp, sp, #(S_FRAME_SIZE + \stack_hole - 4) + sub sp, sp, #(S_FRAME_SIZE + 8 + \stack_hole - 4) #ifdef CONFIG_THUMB2_KERNEL SPFIX( str r0, [sp] ) @ temporarily saved SPFIX( mov r0, sp ) @@ -167,7 +167,7 @@ ENDPROC(__und_invalid) ldmia r0, {r3 - r5} add r7, sp, #S_SP - 4 @ here for interlock avoidance mov r6, #-1 @ "" "" "" "" - add r2, sp, #(S_FRAME_SIZE + \stack_hole - 4) + add r2, sp, #(S_FRAME_SIZE + 8 + \stack_hole - 4) SPFIX( addeq r2, r2, #4 ) str r3, [sp, #-4]! @ save the "real" r0 copied @ from the exception stack @@ -185,6 +185,11 @@ ENDPROC(__und_invalid) @ stmia r7, {r2 - r6} + uaccess_save r0 + .if \uaccess + uaccess_disable r0 + .endif + .if \trace #ifdef CONFIG_TRACE_IRQFLAGS bl trace_hardirqs_off @@ -194,7 +199,7 @@ ENDPROC(__und_invalid) .align 5 __dabt_svc: - svc_entry + svc_entry uaccess=0 mov r2, sp dabt_helper THUMB( ldr r5, [sp, #S_PSR] ) @ potentially updated CPSR @@ -368,7 +373,7 @@ ENDPROC(__fiq_abt) #error "sizeof(struct pt_regs) must be a multiple of 8" #endif - .macro usr_entry, trace=1 + .macro usr_entry, trace=1, uaccess=1 UNWIND(.fnstart ) UNWIND(.cantunwind ) @ don't unwind the user space sub sp, sp, #S_FRAME_SIZE @@ -400,6 +405,10 @@ ENDPROC(__fiq_abt) ARM( stmdb r0, {sp, lr}^ ) THUMB( store_user_sp_lr r0, r1, S_SP - S_PC ) + .if \uaccess + uaccess_disable ip + .endif + @ Enable the alignment trap while in kernel mode ATRAP( teq r8, r7) ATRAP( mcrne p15, 0, r8, c1, c0, 0) @@ -435,7 +444,7 @@ ENDPROC(__fiq_abt) .align 5 __dabt_usr: - usr_entry + usr_entry uaccess=0 kuser_cmpxchg_check mov r2, sp dabt_helper @@ -458,7 +467,7 @@ ENDPROC(__irq_usr) .align 5 __und_usr: - usr_entry + usr_entry uaccess=0 mov r2, r4 mov r3, r5 @@ -484,6 +493,8 @@ __und_usr: 1: ldrt r0, [r4] ARM_BE8(rev r0, r0) @ little endian instruction + uaccess_disable ip + @ r0 = 32-bit ARM instruction which caused the exception @ r2 = PC value for the following instruction (:= regs->ARM_pc) @ r4 = PC value for the faulting instruction @@ -518,9 +529,10 @@ __und_usr_thumb: 2: ldrht r5, [r4] ARM_BE8(rev16 r5, r5) @ little endian instruction cmp r5, #0xe800 @ 32bit instruction if xx != 0 - blo __und_usr_fault_16 @ 16bit undefined instruction + blo __und_usr_fault_16_pan @ 16bit undefined instruction 3: ldrht r0, [r2] ARM_BE8(rev16 r0, r0) @ little endian instruction + uaccess_disable ip add r2, r2, #2 @ r2 is PC + 2, make it PC + 4 str r2, [sp, #S_PC] @ it's a 2x16bit instr, update orr r0, r0, r5, lsl #16 @@ -715,6 +727,8 @@ ENDPROC(no_fp) __und_usr_fault_32: mov r1, #4 b 1f +__und_usr_fault_16_pan: + uaccess_disable ip __und_usr_fault_16: mov r1, #2 1: mov r0, sp @@ -770,6 +784,8 @@ ENTRY(__switch_to) ldr r4, [r2, #TI_TP_VALUE] ldr r5, [r2, #TI_TP_VALUE + 4] #ifdef CONFIG_CPU_USE_DOMAINS + mrc p15, 0, r6, c3, c0, 0 @ Get domain register + str r6, [r1, #TI_CPU_DOMAIN] @ Save old domain register ldr r6, [r2, #TI_CPU_DOMAIN] #endif switch_tls r1, r4, r5, r3, r7 diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index b48dd4f37f80..30a7228eaceb 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -24,35 +24,55 @@ .align 5 +#if !(IS_ENABLED(CONFIG_TRACE_IRQFLAGS) || IS_ENABLED(CONFIG_CONTEXT_TRACKING)) /* - * This is the fast syscall return path. We do as little as - * possible here, and this includes saving r0 back into the SVC - * stack. + * This is the fast syscall return path. We do as little as possible here, + * such as avoiding writing r0 to the stack. We only use this path if we + * have tracing and context tracking disabled - the overheads from those + * features make this path too inefficient. */ ret_fast_syscall: UNWIND(.fnstart ) UNWIND(.cantunwind ) - disable_irq @ disable interrupts + disable_irq_notrace @ disable interrupts ldr r1, [tsk, #TI_FLAGS] @ re-check for syscall tracing - tst r1, #_TIF_SYSCALL_WORK - bne __sys_trace_return - tst r1, #_TIF_WORK_MASK + tst r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK bne fast_work_pending - asm_trace_hardirqs_on /* perform architecture specific actions before user return */ arch_ret_to_user r1, lr - ct_user_enter restore_user_regs fast = 1, offset = S_OFF UNWIND(.fnend ) +ENDPROC(ret_fast_syscall) -/* - * Ok, we need to do extra processing, enter the slow path. - */ + /* Ok, we need to do extra processing, enter the slow path. */ fast_work_pending: str r0, [sp, #S_R0+S_OFF]! @ returned r0 -work_pending: + /* fall through to work_pending */ +#else +/* + * The "replacement" ret_fast_syscall for when tracing or context tracking + * is enabled. As we will need to call out to some C functions, we save + * r0 first to avoid needing to save registers around each C function call. + */ +ret_fast_syscall: + UNWIND(.fnstart ) + UNWIND(.cantunwind ) + str r0, [sp, #S_R0 + S_OFF]! @ save returned r0 + disable_irq_notrace @ disable interrupts + ldr r1, [tsk, #TI_FLAGS] @ re-check for syscall tracing + tst r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK + beq no_work_pending + UNWIND(.fnend ) +ENDPROC(ret_fast_syscall) + + /* Slower path - fall through to work_pending */ +#endif + + tst r1, #_TIF_SYSCALL_WORK + bne __sys_trace_return_nosave +slow_work_pending: mov r0, sp @ 'regs' mov r2, why @ 'syscall' bl do_work_pending @@ -65,16 +85,19 @@ ENDPROC(ret_fast_syscall) /* * "slow" syscall return path. "why" tells us if this was a real syscall. + * IRQs may be enabled here, so always disable them. Note that we use the + * "notrace" version to avoid calling into the tracing code unnecessarily. + * do_work_pending() will update this state if necessary. */ ENTRY(ret_to_user) ret_slow_syscall: - disable_irq @ disable interrupts + disable_irq_notrace @ disable interrupts ENTRY(ret_to_user_from_irq) ldr r1, [tsk, #TI_FLAGS] tst r1, #_TIF_WORK_MASK - bne work_pending + bne slow_work_pending no_work_pending: - asm_trace_hardirqs_on + asm_trace_hardirqs_on save = 0 /* perform architecture specific actions before user return */ arch_ret_to_user r1, lr @@ -174,6 +197,8 @@ ENTRY(vector_swi) USER( ldr scno, [lr, #-4] ) @ get SWI instruction #endif + uaccess_disable tbl + adr tbl, sys_call_table @ load syscall table pointer #if defined(CONFIG_OABI_COMPAT) @@ -252,6 +277,12 @@ __sys_trace_return: bl syscall_trace_exit b ret_slow_syscall +__sys_trace_return_nosave: + enable_irq_notrace + mov r0, sp + bl syscall_trace_exit + b ret_slow_syscall + .align 5 #ifdef CONFIG_ALIGNMENT_TRAP .type __cr_alignment, #object diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S index 1a0045abead7..0d22ad206d52 100644 --- a/arch/arm/kernel/entry-header.S +++ b/arch/arm/kernel/entry-header.S @@ -196,7 +196,7 @@ msr cpsr_c, \rtemp @ switch back to the SVC mode .endm -#ifndef CONFIG_THUMB2_KERNEL + .macro svc_exit, rpsr, irq = 0 .if \irq != 0 @ IRQs already off @@ -215,6 +215,10 @@ blne trace_hardirqs_off #endif .endif + uaccess_restore + +#ifndef CONFIG_THUMB2_KERNEL + @ ARM mode SVC restore msr spsr_cxsf, \rpsr #if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_32v6K) @ We must avoid clrex due to Cortex-A15 erratum #830321 @@ -222,6 +226,20 @@ strex r1, r2, [r0] @ clear the exclusive monitor #endif ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr +#else + @ Thumb mode SVC restore + ldr lr, [sp, #S_SP] @ top of the stack + ldrd r0, r1, [sp, #S_LR] @ calling lr and pc + + @ We must avoid clrex due to Cortex-A15 erratum #830321 + strex r2, r1, [sp, #S_LR] @ clear the exclusive monitor + + stmdb lr!, {r0, r1, \rpsr} @ calling lr and rfe context + ldmia sp, {r0 - r12} + mov sp, lr + ldr lr, [sp], #4 + rfeia sp! +#endif .endm @ @@ -241,6 +259,9 @@ @ on the stack remains correct). @ .macro svc_exit_via_fiq + uaccess_restore +#ifndef CONFIG_THUMB2_KERNEL + @ ARM mode restore mov r0, sp ldmib r0, {r1 - r14} @ abort is deadly from here onward (it will @ clobber state restored below) @@ -250,9 +271,27 @@ msr spsr_cxsf, r9 ldr r0, [r0, #S_R0] ldmia r8, {pc}^ +#else + @ Thumb mode restore + add r0, sp, #S_R2 + ldr lr, [sp, #S_LR] + ldr sp, [sp, #S_SP] @ abort is deadly from here onward (it will + @ clobber state restored below) + ldmia r0, {r2 - r12} + mov r1, #FIQ_MODE | PSR_I_BIT | PSR_F_BIT + msr cpsr_c, r1 + sub r0, #S_R2 + add r8, r0, #S_PC + ldmia r0, {r0 - r1} + rfeia r8 +#endif .endm + .macro restore_user_regs, fast = 0, offset = 0 + uaccess_enable r1, isb=0 +#ifndef CONFIG_THUMB2_KERNEL + @ ARM mode restore mov r2, sp ldr r1, [r2, #\offset + S_PSR] @ get calling cpsr ldr lr, [r2, #\offset + S_PC]! @ get pc @@ -270,72 +309,16 @@ @ after ldm {}^ add sp, sp, #\offset + S_FRAME_SIZE movs pc, lr @ return & move spsr_svc into cpsr - .endm - -#else /* CONFIG_THUMB2_KERNEL */ - .macro svc_exit, rpsr, irq = 0 - .if \irq != 0 - @ IRQs already off -#ifdef CONFIG_TRACE_IRQFLAGS - @ The parent context IRQs must have been enabled to get here in - @ the first place, so there's no point checking the PSR I bit. - bl trace_hardirqs_on -#endif - .else - @ IRQs off again before pulling preserved data off the stack - disable_irq_notrace -#ifdef CONFIG_TRACE_IRQFLAGS - tst \rpsr, #PSR_I_BIT - bleq trace_hardirqs_on - tst \rpsr, #PSR_I_BIT - blne trace_hardirqs_off -#endif - .endif - ldr lr, [sp, #S_SP] @ top of the stack - ldrd r0, r1, [sp, #S_LR] @ calling lr and pc - - @ We must avoid clrex due to Cortex-A15 erratum #830321 - strex r2, r1, [sp, #S_LR] @ clear the exclusive monitor - - stmdb lr!, {r0, r1, \rpsr} @ calling lr and rfe context - ldmia sp, {r0 - r12} - mov sp, lr - ldr lr, [sp], #4 - rfeia sp! - .endm - - @ - @ svc_exit_via_fiq - like svc_exit but switches to FIQ mode before exit - @ - @ For full details see non-Thumb implementation above. - @ - .macro svc_exit_via_fiq - add r0, sp, #S_R2 - ldr lr, [sp, #S_LR] - ldr sp, [sp, #S_SP] @ abort is deadly from here onward (it will - @ clobber state restored below) - ldmia r0, {r2 - r12} - mov r1, #FIQ_MODE | PSR_I_BIT | PSR_F_BIT - msr cpsr_c, r1 - sub r0, #S_R2 - add r8, r0, #S_PC - ldmia r0, {r0 - r1} - rfeia r8 - .endm - -#ifdef CONFIG_CPU_V7M - /* - * Note we don't need to do clrex here as clearing the local monitor is - * part of each exception entry and exit sequence. - */ - .macro restore_user_regs, fast = 0, offset = 0 +#elif defined(CONFIG_CPU_V7M) + @ V7M restore. + @ Note that we don't need to do clrex here as clearing the local + @ monitor is part of the exception entry and exit sequence. .if \offset add sp, #\offset .endif v7m_exception_slow_exit ret_r0 = \fast - .endm -#else /* ifdef CONFIG_CPU_V7M */ - .macro restore_user_regs, fast = 0, offset = 0 +#else + @ Thumb mode restore mov r2, sp load_user_sp_lr r2, r3, \offset + S_SP @ calling sp, lr ldr r1, [sp, #\offset + S_PSR] @ get calling cpsr @@ -353,9 +336,8 @@ .endif add sp, sp, #S_FRAME_SIZE - S_SP movs pc, lr @ return & move spsr_svc into cpsr - .endm -#endif /* ifdef CONFIG_CPU_V7M / else */ #endif /* !CONFIG_THUMB2_KERNEL */ + .endm /* * Context tracking subsystem. Used to instrument transitions diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S index 29e2991465cb..04286fd9e09c 100644 --- a/arch/arm/kernel/head.S +++ b/arch/arm/kernel/head.S @@ -464,10 +464,7 @@ __enable_mmu: #ifdef CONFIG_ARM_LPAE mcrr p15, 0, r4, r5, c2 @ load TTBR0 #else - mov r5, #(domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \ - domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ - domain_val(DOMAIN_TABLE, DOMAIN_MANAGER) | \ - domain_val(DOMAIN_IO, DOMAIN_CLIENT)) + mov r5, #DACR_INIT mcr p15, 0, r5, c3, c0, 0 @ load domain access register mcr p15, 0, r4, c2, c0, 0 @ load page table pointer #endif diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index baf8edebe26f..5ff4826cb154 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -39,6 +39,7 @@ #include #include +#include #include #include #include diff --git a/arch/arm/kernel/jump_label.c b/arch/arm/kernel/jump_label.c index e39cbf488cfe..845a5dd9c42b 100644 --- a/arch/arm/kernel/jump_label.c +++ b/arch/arm/kernel/jump_label.c @@ -12,7 +12,7 @@ static void __arch_jump_label_transform(struct jump_entry *entry, void *addr = (void *)entry->code; unsigned int insn; - if (type == JUMP_LABEL_ENABLE) + if (type == JUMP_LABEL_JMP) insn = arm_gen_branch(entry->code, entry->target); else insn = arm_gen_nop(); diff --git a/arch/arm/kernel/perf_event_v6.c b/arch/arm/kernel/perf_event_v6.c index 09f83e414a72..09413e7b49aa 100644 --- a/arch/arm/kernel/perf_event_v6.c +++ b/arch/arm/kernel/perf_event_v6.c @@ -34,9 +34,9 @@ #include #include -#include #include +#include #include enum armv6_perf_types { diff --git a/arch/arm/kernel/perf_event_v7.c b/arch/arm/kernel/perf_event_v7.c index f9b37f876e20..126dc679b230 100644 --- a/arch/arm/kernel/perf_event_v7.c +++ b/arch/arm/kernel/perf_event_v7.c @@ -21,11 +21,11 @@ #include #include #include -#include #include #include "../vfp/vfpinstr.h" #include +#include #include /* diff --git a/arch/arm/kernel/perf_event_xscale.c b/arch/arm/kernel/perf_event_xscale.c index 304d056d5b25..aa0499e2eef7 100644 --- a/arch/arm/kernel/perf_event_xscale.c +++ b/arch/arm/kernel/perf_event_xscale.c @@ -16,9 +16,9 @@ #include #include -#include #include +#include #include enum xscale_perf_types { diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index f192a2a41719..a3089bacb8d8 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -91,13 +91,6 @@ void arch_cpu_idle_exit(void) ledtrig_cpu(CPU_LED_IDLE_END); } -#ifdef CONFIG_HOTPLUG_CPU -void arch_cpu_idle_dead(void) -{ - cpu_die(); -} -#endif - void __show_regs(struct pt_regs *regs) { unsigned long flags; @@ -129,12 +122,36 @@ void __show_regs(struct pt_regs *regs) buf[4] = '\0'; #ifndef CONFIG_CPU_V7M - printk("Flags: %s IRQs o%s FIQs o%s Mode %s ISA %s Segment %s\n", - buf, interrupts_enabled(regs) ? "n" : "ff", - fast_interrupts_enabled(regs) ? "n" : "ff", - processor_modes[processor_mode(regs)], - isa_modes[isa_mode(regs)], - get_fs() == get_ds() ? "kernel" : "user"); + { + unsigned int domain = get_domain(); + const char *segment; + +#ifdef CONFIG_CPU_SW_DOMAIN_PAN + /* + * Get the domain register for the parent context. In user + * mode, we don't save the DACR, so lets use what it should + * be. For other modes, we place it after the pt_regs struct. + */ + if (user_mode(regs)) + domain = DACR_UACCESS_ENABLE; + else + domain = *(unsigned int *)(regs + 1); +#endif + + if ((domain & domain_mask(DOMAIN_USER)) == + domain_val(DOMAIN_USER, DOMAIN_NOACCESS)) + segment = "none"; + else if (get_fs() == get_ds()) + segment = "kernel"; + else + segment = "user"; + + printk("Flags: %s IRQs o%s FIQs o%s Mode %s ISA %s Segment %s\n", + buf, interrupts_enabled(regs) ? "n" : "ff", + fast_interrupts_enabled(regs) ? "n" : "ff", + processor_modes[processor_mode(regs)], + isa_modes[isa_mode(regs)], segment); + } #else printk("xPSR: %08lx\n", regs->ARM_cpsr); #endif @@ -146,10 +163,9 @@ void __show_regs(struct pt_regs *regs) buf[0] = '\0'; #ifdef CONFIG_CPU_CP15_MMU { - unsigned int transbase, dac; + unsigned int transbase, dac = get_domain(); asm("mrc p15, 0, %0, c2, c0\n\t" - "mrc p15, 0, %1, c3, c0\n" - : "=r" (transbase), "=r" (dac)); + : "=r" (transbase)); snprintf(buf, sizeof(buf), " Table: %08x DAC: %08x", transbase, dac); } @@ -210,6 +226,14 @@ copy_thread(unsigned long clone_flags, unsigned long stack_start, memset(&thread->cpu_context, 0, sizeof(struct cpu_context_save)); + /* + * Copy the initial value of the domain access control register + * from the current thread: thread->addr_limit will have been + * copied from the current thread via setup_thread_stack() in + * kernel/fork.c + */ + thread->cpu_domain = get_domain(); + if (likely(!(p->flags & PF_KTHREAD))) { *childregs = *current_pt_regs(); childregs->ARM_r0 = 0; diff --git a/arch/arm/kernel/psci.c b/arch/arm/kernel/psci.c deleted file mode 100644 index 2e6024334790..000000000000 --- a/arch/arm/kernel/psci.c +++ /dev/null @@ -1,299 +0,0 @@ -/* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * Copyright (C) 2012 ARM Limited - * - * Author: Will Deacon - */ - -#define pr_fmt(fmt) "psci: " fmt - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -struct psci_operations psci_ops; - -static int (*invoke_psci_fn)(u32, u32, u32, u32); -typedef int (*psci_initcall_t)(const struct device_node *); - -asmlinkage int __invoke_psci_fn_hvc(u32, u32, u32, u32); -asmlinkage int __invoke_psci_fn_smc(u32, u32, u32, u32); - -enum psci_function { - PSCI_FN_CPU_SUSPEND, - PSCI_FN_CPU_ON, - PSCI_FN_CPU_OFF, - PSCI_FN_MIGRATE, - PSCI_FN_AFFINITY_INFO, - PSCI_FN_MIGRATE_INFO_TYPE, - PSCI_FN_MAX, -}; - -static u32 psci_function_id[PSCI_FN_MAX]; - -static int psci_to_linux_errno(int errno) -{ - switch (errno) { - case PSCI_RET_SUCCESS: - return 0; - case PSCI_RET_NOT_SUPPORTED: - return -EOPNOTSUPP; - case PSCI_RET_INVALID_PARAMS: - return -EINVAL; - case PSCI_RET_DENIED: - return -EPERM; - }; - - return -EINVAL; -} - -static u32 psci_power_state_pack(struct psci_power_state state) -{ - return ((state.id << PSCI_0_2_POWER_STATE_ID_SHIFT) - & PSCI_0_2_POWER_STATE_ID_MASK) | - ((state.type << PSCI_0_2_POWER_STATE_TYPE_SHIFT) - & PSCI_0_2_POWER_STATE_TYPE_MASK) | - ((state.affinity_level << PSCI_0_2_POWER_STATE_AFFL_SHIFT) - & PSCI_0_2_POWER_STATE_AFFL_MASK); -} - -static int psci_get_version(void) -{ - int err; - - err = invoke_psci_fn(PSCI_0_2_FN_PSCI_VERSION, 0, 0, 0); - return err; -} - -static int psci_cpu_suspend(struct psci_power_state state, - unsigned long entry_point) -{ - int err; - u32 fn, power_state; - - fn = psci_function_id[PSCI_FN_CPU_SUSPEND]; - power_state = psci_power_state_pack(state); - err = invoke_psci_fn(fn, power_state, entry_point, 0); - return psci_to_linux_errno(err); -} - -static int psci_cpu_off(struct psci_power_state state) -{ - int err; - u32 fn, power_state; - - fn = psci_function_id[PSCI_FN_CPU_OFF]; - power_state = psci_power_state_pack(state); - err = invoke_psci_fn(fn, power_state, 0, 0); - return psci_to_linux_errno(err); -} - -static int psci_cpu_on(unsigned long cpuid, unsigned long entry_point) -{ - int err; - u32 fn; - - fn = psci_function_id[PSCI_FN_CPU_ON]; - err = invoke_psci_fn(fn, cpuid, entry_point, 0); - return psci_to_linux_errno(err); -} - -static int psci_migrate(unsigned long cpuid) -{ - int err; - u32 fn; - - fn = psci_function_id[PSCI_FN_MIGRATE]; - err = invoke_psci_fn(fn, cpuid, 0, 0); - return psci_to_linux_errno(err); -} - -static int psci_affinity_info(unsigned long target_affinity, - unsigned long lowest_affinity_level) -{ - int err; - u32 fn; - - fn = psci_function_id[PSCI_FN_AFFINITY_INFO]; - err = invoke_psci_fn(fn, target_affinity, lowest_affinity_level, 0); - return err; -} - -static int psci_migrate_info_type(void) -{ - int err; - u32 fn; - - fn = psci_function_id[PSCI_FN_MIGRATE_INFO_TYPE]; - err = invoke_psci_fn(fn, 0, 0, 0); - return err; -} - -static int get_set_conduit_method(struct device_node *np) -{ - const char *method; - - pr_info("probing for conduit method from DT.\n"); - - if (of_property_read_string(np, "method", &method)) { - pr_warn("missing \"method\" property\n"); - return -ENXIO; - } - - if (!strcmp("hvc", method)) { - invoke_psci_fn = __invoke_psci_fn_hvc; - } else if (!strcmp("smc", method)) { - invoke_psci_fn = __invoke_psci_fn_smc; - } else { - pr_warn("invalid \"method\" property: %s\n", method); - return -EINVAL; - } - return 0; -} - -static void psci_sys_reset(enum reboot_mode reboot_mode, const char *cmd) -{ - invoke_psci_fn(PSCI_0_2_FN_SYSTEM_RESET, 0, 0, 0); -} - -static void psci_sys_poweroff(void) -{ - invoke_psci_fn(PSCI_0_2_FN_SYSTEM_OFF, 0, 0, 0); -} - -/* - * PSCI Function IDs for v0.2+ are well defined so use - * standard values. - */ -static int psci_0_2_init(struct device_node *np) -{ - int err, ver; - - err = get_set_conduit_method(np); - - if (err) - goto out_put_node; - - ver = psci_get_version(); - - if (ver == PSCI_RET_NOT_SUPPORTED) { - /* PSCI v0.2 mandates implementation of PSCI_ID_VERSION. */ - pr_err("PSCI firmware does not comply with the v0.2 spec.\n"); - err = -EOPNOTSUPP; - goto out_put_node; - } else { - pr_info("PSCIv%d.%d detected in firmware.\n", - PSCI_VERSION_MAJOR(ver), - PSCI_VERSION_MINOR(ver)); - - if (PSCI_VERSION_MAJOR(ver) == 0 && - PSCI_VERSION_MINOR(ver) < 2) { - err = -EINVAL; - pr_err("Conflicting PSCI version detected.\n"); - goto out_put_node; - } - } - - pr_info("Using standard PSCI v0.2 function IDs\n"); - psci_function_id[PSCI_FN_CPU_SUSPEND] = PSCI_0_2_FN_CPU_SUSPEND; - psci_ops.cpu_suspend = psci_cpu_suspend; - - psci_function_id[PSCI_FN_CPU_OFF] = PSCI_0_2_FN_CPU_OFF; - psci_ops.cpu_off = psci_cpu_off; - - psci_function_id[PSCI_FN_CPU_ON] = PSCI_0_2_FN_CPU_ON; - psci_ops.cpu_on = psci_cpu_on; - - psci_function_id[PSCI_FN_MIGRATE] = PSCI_0_2_FN_MIGRATE; - psci_ops.migrate = psci_migrate; - - psci_function_id[PSCI_FN_AFFINITY_INFO] = PSCI_0_2_FN_AFFINITY_INFO; - psci_ops.affinity_info = psci_affinity_info; - - psci_function_id[PSCI_FN_MIGRATE_INFO_TYPE] = - PSCI_0_2_FN_MIGRATE_INFO_TYPE; - psci_ops.migrate_info_type = psci_migrate_info_type; - - arm_pm_restart = psci_sys_reset; - - pm_power_off = psci_sys_poweroff; - -out_put_node: - of_node_put(np); - return err; -} - -/* - * PSCI < v0.2 get PSCI Function IDs via DT. - */ -static int psci_0_1_init(struct device_node *np) -{ - u32 id; - int err; - - err = get_set_conduit_method(np); - - if (err) - goto out_put_node; - - pr_info("Using PSCI v0.1 Function IDs from DT\n"); - - if (!of_property_read_u32(np, "cpu_suspend", &id)) { - psci_function_id[PSCI_FN_CPU_SUSPEND] = id; - psci_ops.cpu_suspend = psci_cpu_suspend; - } - - if (!of_property_read_u32(np, "cpu_off", &id)) { - psci_function_id[PSCI_FN_CPU_OFF] = id; - psci_ops.cpu_off = psci_cpu_off; - } - - if (!of_property_read_u32(np, "cpu_on", &id)) { - psci_function_id[PSCI_FN_CPU_ON] = id; - psci_ops.cpu_on = psci_cpu_on; - } - - if (!of_property_read_u32(np, "migrate", &id)) { - psci_function_id[PSCI_FN_MIGRATE] = id; - psci_ops.migrate = psci_migrate; - } - -out_put_node: - of_node_put(np); - return err; -} - -static const struct of_device_id const psci_of_match[] __initconst = { - { .compatible = "arm,psci", .data = psci_0_1_init}, - { .compatible = "arm,psci-0.2", .data = psci_0_2_init}, - {}, -}; - -int __init psci_init(void) -{ - struct device_node *np; - const struct of_device_id *matched_np; - psci_initcall_t init_fn; - - np = of_find_matching_node_and_match(NULL, psci_of_match, &matched_np); - if (!np) - return -ENODEV; - - init_fn = (psci_initcall_t)matched_np->data; - return init_fn(np); -} diff --git a/arch/arm/kernel/psci_smp.c b/arch/arm/kernel/psci_smp.c index 28a1db4da704..61c04b02faeb 100644 --- a/arch/arm/kernel/psci_smp.c +++ b/arch/arm/kernel/psci_smp.c @@ -17,6 +17,8 @@ #include #include #include +#include + #include #include @@ -51,22 +53,34 @@ static int psci_boot_secondary(unsigned int cpu, struct task_struct *idle) { if (psci_ops.cpu_on) return psci_ops.cpu_on(cpu_logical_map(cpu), - __pa(secondary_startup)); + virt_to_idmap(&secondary_startup)); return -ENODEV; } #ifdef CONFIG_HOTPLUG_CPU +int psci_cpu_disable(unsigned int cpu) +{ + /* Fail early if we don't have CPU_OFF support */ + if (!psci_ops.cpu_off) + return -EOPNOTSUPP; + + /* Trusted OS will deny CPU_OFF */ + if (psci_tos_resident_on(cpu)) + return -EPERM; + + return 0; +} + void __ref psci_cpu_die(unsigned int cpu) { - const struct psci_power_state ps = { - .type = PSCI_POWER_STATE_TYPE_POWER_DOWN, - }; + u32 state = PSCI_POWER_STATE_TYPE_POWER_DOWN << + PSCI_0_2_POWER_STATE_TYPE_SHIFT; - if (psci_ops.cpu_off) - psci_ops.cpu_off(ps); + if (psci_ops.cpu_off) + psci_ops.cpu_off(state); - /* We should never return */ - panic("psci: cpu %d failed to shutdown\n", cpu); + /* We should never return */ + panic("psci: cpu %d failed to shutdown\n", cpu); } int __ref psci_cpu_kill(unsigned int cpu) @@ -109,6 +123,7 @@ bool __init psci_smp_available(void) struct smp_operations __initdata psci_smp_ops = { .smp_boot_secondary = psci_boot_secondary, #ifdef CONFIG_HOTPLUG_CPU + .cpu_disable = psci_cpu_disable, .cpu_die = psci_cpu_die, .cpu_kill = psci_cpu_kill, #endif diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 36c18b73c1f4..20edd349d379 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -31,12 +31,14 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -954,6 +956,9 @@ void __init setup_arch(char **cmdline_p) strlcpy(cmd_line, boot_command_line, COMMAND_LINE_SIZE); *cmdline_p = cmd_line; + if (IS_ENABLED(CONFIG_FIX_EARLYCON_MEM)) + early_fixmap_init(); + parse_early_param(); #ifdef CONFIG_MMU @@ -972,7 +977,7 @@ void __init setup_arch(char **cmdline_p) unflatten_device_tree(); arm_dt_init_cpu_maps(); - psci_init(); + psci_dt_init(); xen_early_init(); #ifdef CONFIG_SMP if (is_smp()) { @@ -1015,7 +1020,7 @@ static int __init topology_init(void) for_each_possible_cpu(cpu) { struct cpuinfo_arm *cpuinfo = &per_cpu(cpu_data, cpu); - cpuinfo->cpu.hotpluggable = 1; + cpuinfo->cpu.hotpluggable = platform_can_hotplug_cpu(cpu); register_cpu(&cpuinfo->cpu, cpu); } diff --git a/arch/arm/kernel/signal.c b/arch/arm/kernel/signal.c index 423663e23791..b6cda06b455f 100644 --- a/arch/arm/kernel/signal.c +++ b/arch/arm/kernel/signal.c @@ -562,6 +562,12 @@ static int do_signal(struct pt_regs *regs, int syscall) asmlinkage int do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall) { + /* + * The assembly code enters us with IRQs off, but it hasn't + * informed the tracing code of that for efficiency reasons. + * Update the trace code with the current status. + */ + trace_hardirqs_off(); do { if (likely(thread_flags & _TIF_NEED_RESCHED)) { schedule(); diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 3d6b7821cff8..48185a773852 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -72,6 +73,7 @@ enum ipi_msg_type { IPI_CPU_STOP, IPI_IRQ_WORK, IPI_COMPLETION, + IPI_CPU_BACKTRACE = 15, }; static DECLARE_COMPLETION(cpu_running); @@ -175,13 +177,26 @@ static int platform_cpu_disable(unsigned int cpu) if (smp_ops.cpu_disable) return smp_ops.cpu_disable(cpu); + return 0; +} + +int platform_can_hotplug_cpu(unsigned int cpu) +{ + /* cpu_die must be specified to support hotplug */ + if (!smp_ops.cpu_die) + return 0; + + if (smp_ops.cpu_can_disable) + return smp_ops.cpu_can_disable(cpu); + /* * By default, allow disabling all CPUs except the first one, * since this is special on a lot of platforms, e.g. because * of clock tick interrupts. */ - return cpu == 0 ? -EPERM : 0; + return cpu != 0; } + /* * __cpu_disable runs on the processor to be shutdown. */ @@ -253,7 +268,7 @@ void __cpu_die(unsigned int cpu) * of the other hotplug-cpu capable cores, so presumably coming * out of idle fixes this. */ -void __ref cpu_die(void) +void arch_cpu_idle_dead(void) { unsigned int cpu = smp_processor_id(); @@ -630,6 +645,12 @@ void handle_IPI(int ipinr, struct pt_regs *regs) irq_exit(); break; + case IPI_CPU_BACKTRACE: + irq_enter(); + nmi_cpu_backtrace(regs); + irq_exit(); + break; + default: pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr); @@ -724,3 +745,13 @@ static int __init register_cpufreq_notifier(void) core_initcall(register_cpufreq_notifier); #endif + +static void raise_nmi(cpumask_t *mask) +{ + smp_cross_call(mask, IPI_CPU_BACKTRACE); +} + +void arch_trigger_all_cpu_backtrace(bool include_self) +{ + nmi_trigger_all_cpu_backtrace(include_self, raise_nmi); +} diff --git a/arch/arm/kernel/swp_emulate.c b/arch/arm/kernel/swp_emulate.c index 1361756782c7..5b26e7efa9ea 100644 --- a/arch/arm/kernel/swp_emulate.c +++ b/arch/arm/kernel/swp_emulate.c @@ -141,11 +141,14 @@ static int emulate_swpX(unsigned int address, unsigned int *data, while (1) { unsigned long temp; + unsigned int __ua_flags; + __ua_flags = uaccess_save_and_enable(); if (type == TYPE_SWPB) __user_swpb_asm(*data, address, res, temp); else __user_swp_asm(*data, address, res, temp); + uaccess_restore(__ua_flags); if (likely(res != -EAGAIN) || signal_pending(current)) break; diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index d358226236f2..969f9d9e665f 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -870,7 +870,6 @@ void __init early_trap_init(void *vectors_base) kuser_init(vectors_base); flush_icache_range(vectors, vectors + PAGE_SIZE * 2); - modify_domain(DOMAIN_USER, DOMAIN_CLIENT); #else /* ifndef CONFIG_CPU_V7M */ /* * on V7-M there is no need to copy the vector table to a dedicated diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c index bc738d2b8392..ce404a5c3062 100644 --- a/arch/arm/kvm/arm.c +++ b/arch/arm/kvm/arm.c @@ -125,6 +125,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) if (ret) goto out_free_stage2_pgd; + kvm_vgic_early_init(kvm); kvm_timer_init(kvm); /* Mark the initial VMID generation invalid */ @@ -249,6 +250,7 @@ struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, unsigned int id) void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) { + kvm_vgic_vcpu_early_init(vcpu); } void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) @@ -278,6 +280,8 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) /* Set up the timer */ kvm_timer_vcpu_init(vcpu); + kvm_arm_reset_debug_ptr(vcpu); + return 0; } @@ -301,13 +305,6 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) kvm_arm_set_running_vcpu(NULL); } -int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, - struct kvm_guest_debug *dbg) -{ - return -EINVAL; -} - - int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, struct kvm_mp_state *mp_state) { @@ -528,10 +525,20 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) if (vcpu->arch.pause) vcpu_pause(vcpu); - kvm_vgic_flush_hwstate(vcpu); + /* + * Disarming the background timer must be done in a + * preemptible context, as this call may sleep. + */ kvm_timer_flush_hwstate(vcpu); + /* + * Preparing the interrupts to be injected also + * involves poking the GIC, which must be done in a + * non-preemptible context. + */ preempt_disable(); + kvm_vgic_flush_hwstate(vcpu); + local_irq_disable(); /* @@ -544,12 +551,14 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) if (ret <= 0 || need_new_vmid_gen(vcpu->kvm)) { local_irq_enable(); + kvm_vgic_sync_hwstate(vcpu); preempt_enable(); kvm_timer_sync_hwstate(vcpu); - kvm_vgic_sync_hwstate(vcpu); continue; } + kvm_arm_setup_debug(vcpu); + /************************************************************** * Enter the guest */ @@ -564,6 +573,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) * Back from guest *************************************************************/ + kvm_arm_clear_debug(vcpu); + /* * We may have taken a host interrupt in HYP mode (ie * while executing the guest). This interrupt is still @@ -586,11 +597,12 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) */ kvm_guest_exit(); trace_kvm_exit(kvm_vcpu_trap_get_class(vcpu), *vcpu_pc(vcpu)); + + kvm_vgic_sync_hwstate(vcpu); + preempt_enable(); - kvm_timer_sync_hwstate(vcpu); - kvm_vgic_sync_hwstate(vcpu); ret = handle_exit(vcpu, run, ret); } @@ -921,6 +933,8 @@ static void cpu_init_hyp_mode(void *dummy) vector_ptr = (unsigned long)__kvm_hyp_vector; __cpu_init_hyp_mode(boot_pgd_ptr, pgd_ptr, hyp_stack_ptr, vector_ptr); + + kvm_arm_init_debug(); } static int hyp_init_cpu_notify(struct notifier_block *self, diff --git a/arch/arm/kvm/guest.c b/arch/arm/kvm/guest.c index d503fbb787d3..96e935bbc38c 100644 --- a/arch/arm/kvm/guest.c +++ b/arch/arm/kvm/guest.c @@ -290,3 +290,9 @@ int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, { return -EINVAL; } + +int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, + struct kvm_guest_debug *dbg) +{ + return -EINVAL; +} diff --git a/arch/arm/kvm/interrupts.S b/arch/arm/kvm/interrupts.S index 568494dbbbb5..900ef6dd8f72 100644 --- a/arch/arm/kvm/interrupts.S +++ b/arch/arm/kvm/interrupts.S @@ -361,10 +361,6 @@ hyp_hvc: @ Check syndrome register mrc p15, 4, r1, c5, c2, 0 @ HSR lsr r0, r1, #HSR_EC_SHIFT -#ifdef CONFIG_VFPv3 - cmp r0, #HSR_EC_CP_0_13 - beq switch_to_guest_vfp -#endif cmp r0, #HSR_EC_HVC bne guest_trap @ Not HVC instr. @@ -378,7 +374,10 @@ hyp_hvc: cmp r2, #0 bne guest_trap @ Guest called HVC -host_switch_to_hyp: + /* + * Getting here means host called HVC, we shift parameters and branch + * to Hyp function. + */ pop {r0, r1, r2} /* Check for __hyp_get_vectors */ @@ -409,6 +408,10 @@ guest_trap: @ Check if we need the fault information lsr r1, r1, #HSR_EC_SHIFT +#ifdef CONFIG_VFPv3 + cmp r1, #HSR_EC_CP_0_13 + beq switch_to_guest_vfp +#endif cmp r1, #HSR_EC_IABT mrceq p15, 4, r2, c6, c0, 2 @ HIFAR beq 2f @@ -477,7 +480,6 @@ guest_trap: */ #ifdef CONFIG_VFPv3 switch_to_guest_vfp: - load_vcpu @ Load VCPU pointer to r0 push {r3-r7} @ NEON/VFP used. Turn on VFP access. diff --git a/arch/arm/kvm/reset.c b/arch/arm/kvm/reset.c index f558c073c023..eeb85858d6bb 100644 --- a/arch/arm/kvm/reset.c +++ b/arch/arm/kvm/reset.c @@ -77,7 +77,5 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu) kvm_reset_coprocs(vcpu); /* Reset arch_timer context */ - kvm_timer_vcpu_reset(vcpu, cpu_vtimer_irq); - - return 0; + return kvm_timer_vcpu_reset(vcpu, cpu_vtimer_irq); } diff --git a/arch/arm/lib/clear_user.S b/arch/arm/lib/clear_user.S index 1710fd7db2d5..970d6c043774 100644 --- a/arch/arm/lib/clear_user.S +++ b/arch/arm/lib/clear_user.S @@ -12,14 +12,14 @@ .text -/* Prototype: int __clear_user(void *addr, size_t sz) +/* Prototype: unsigned long arm_clear_user(void *addr, size_t sz) * Purpose : clear some user memory * Params : addr - user memory address to clear * : sz - number of bytes to clear * Returns : number of bytes NOT cleared */ ENTRY(__clear_user_std) -WEAK(__clear_user) +WEAK(arm_clear_user) stmfd sp!, {r1, lr} mov r2, #0 cmp r1, #4 @@ -44,7 +44,7 @@ WEAK(__clear_user) USER( strnebt r2, [r0]) mov r0, #0 ldmfd sp!, {r1, pc} -ENDPROC(__clear_user) +ENDPROC(arm_clear_user) ENDPROC(__clear_user_std) .pushsection .text.fixup,"ax" diff --git a/arch/arm/lib/copy_from_user.S b/arch/arm/lib/copy_from_user.S index 7a235b9952be..1512bebfbf1b 100644 --- a/arch/arm/lib/copy_from_user.S +++ b/arch/arm/lib/copy_from_user.S @@ -17,7 +17,7 @@ /* * Prototype: * - * size_t __copy_from_user(void *to, const void *from, size_t n) + * size_t arm_copy_from_user(void *to, const void *from, size_t n) * * Purpose: * @@ -89,11 +89,11 @@ .text -ENTRY(__copy_from_user) +ENTRY(arm_copy_from_user) #include "copy_template.S" -ENDPROC(__copy_from_user) +ENDPROC(arm_copy_from_user) .pushsection .fixup,"ax" .align 0 diff --git a/arch/arm/lib/copy_to_user.S b/arch/arm/lib/copy_to_user.S index 9648b0675a3e..caf5019d8161 100644 --- a/arch/arm/lib/copy_to_user.S +++ b/arch/arm/lib/copy_to_user.S @@ -17,7 +17,7 @@ /* * Prototype: * - * size_t __copy_to_user(void *to, const void *from, size_t n) + * size_t arm_copy_to_user(void *to, const void *from, size_t n) * * Purpose: * @@ -93,11 +93,11 @@ .text ENTRY(__copy_to_user_std) -WEAK(__copy_to_user) +WEAK(arm_copy_to_user) #include "copy_template.S" -ENDPROC(__copy_to_user) +ENDPROC(arm_copy_to_user) ENDPROC(__copy_to_user_std) .pushsection .text.fixup,"ax" diff --git a/arch/arm/lib/csumpartialcopyuser.S b/arch/arm/lib/csumpartialcopyuser.S index 1d0957e61f89..1712f132b80d 100644 --- a/arch/arm/lib/csumpartialcopyuser.S +++ b/arch/arm/lib/csumpartialcopyuser.S @@ -17,6 +17,19 @@ .text +#ifdef CONFIG_CPU_SW_DOMAIN_PAN + .macro save_regs + mrc p15, 0, ip, c3, c0, 0 + stmfd sp!, {r1, r2, r4 - r8, ip, lr} + uaccess_enable ip + .endm + + .macro load_regs + ldmfd sp!, {r1, r2, r4 - r8, ip, lr} + mcr p15, 0, ip, c3, c0, 0 + ret lr + .endm +#else .macro save_regs stmfd sp!, {r1, r2, r4 - r8, lr} .endm @@ -24,6 +37,7 @@ .macro load_regs ldmfd sp!, {r1, r2, r4 - r8, pc} .endm +#endif .macro load1b, reg1 ldrusr \reg1, r0, 1 diff --git a/arch/arm/lib/uaccess_with_memcpy.c b/arch/arm/lib/uaccess_with_memcpy.c index 4b39af2dfda9..d72b90905132 100644 --- a/arch/arm/lib/uaccess_with_memcpy.c +++ b/arch/arm/lib/uaccess_with_memcpy.c @@ -136,7 +136,7 @@ __copy_to_user_memcpy(void __user *to, const void *from, unsigned long n) } unsigned long -__copy_to_user(void __user *to, const void *from, unsigned long n) +arm_copy_to_user(void __user *to, const void *from, unsigned long n) { /* * This test is stubbed out of the main function above to keep @@ -190,7 +190,7 @@ __clear_user_memset(void __user *addr, unsigned long n) return n; } -unsigned long __clear_user(void __user *addr, unsigned long n) +unsigned long arm_clear_user(void __user *addr, unsigned long n) { /* See rational for this in __copy_to_user() above. */ if (n < 64) diff --git a/arch/arm/mach-at91/pm.c b/arch/arm/mach-at91/pm.c index 265ffeb2037e..80e277cfcc8b 100644 --- a/arch/arm/mach-at91/pm.c +++ b/arch/arm/mach-at91/pm.c @@ -369,7 +369,7 @@ static void __init at91_pm_sram_init(void) return; } - sram_pool = gen_pool_get(&pdev->dev); + sram_pool = gen_pool_get(&pdev->dev, NULL); if (!sram_pool) { pr_warn("%s: sram pool unavailable!\n", __func__); return; diff --git a/arch/arm/mach-clps711x/board-cdb89712.c b/arch/arm/mach-clps711x/board-cdb89712.c index 1ec378c334e5..972abdb10028 100644 --- a/arch/arm/mach-clps711x/board-cdb89712.c +++ b/arch/arm/mach-clps711x/board-cdb89712.c @@ -95,7 +95,7 @@ static struct physmap_flash_data cdb89712_bootrom_pdata __initdata = { static struct resource cdb89712_bootrom_resources[] __initdata = { DEFINE_RES_NAMED(CS7_PHYS_BASE, SZ_128, "BOOTROM", IORESOURCE_MEM | - IORESOURCE_CACHEABLE | IORESOURCE_READONLY), + IORESOURCE_READONLY), }; static struct platform_device cdb89712_bootrom_pdev __initdata = { diff --git a/arch/arm/mach-exynos/Kconfig b/arch/arm/mach-exynos/Kconfig index 4c4858c566d8..3a10f1a8317a 100644 --- a/arch/arm/mach-exynos/Kconfig +++ b/arch/arm/mach-exynos/Kconfig @@ -15,6 +15,7 @@ menuconfig ARCH_EXYNOS select ARM_AMBA select ARM_GIC select COMMON_CLK_SAMSUNG + select EXYNOS_THERMAL select HAVE_ARM_SCU if SMP select HAVE_S3C2410_I2C if I2C select HAVE_S3C2410_WATCHDOG if WATCHDOG @@ -24,6 +25,7 @@ menuconfig ARCH_EXYNOS select PM_GENERIC_DOMAINS if PM select S5P_DEV_MFC select SRAM + select THERMAL select MFD_SYSCON help Support for SAMSUNG EXYNOS SoCs (EXYNOS4/5) diff --git a/arch/arm/mach-exynos/exynos.c b/arch/arm/mach-exynos/exynos.c index 5f8ddcdeeacf..1c47aee31e9c 100644 --- a/arch/arm/mach-exynos/exynos.c +++ b/arch/arm/mach-exynos/exynos.c @@ -225,7 +225,11 @@ static void __init exynos_init_irq(void) } static const struct of_device_id exynos_cpufreq_matches[] = { + { .compatible = "samsung,exynos3250", .data = "cpufreq-dt" }, { .compatible = "samsung,exynos4210", .data = "cpufreq-dt" }, + { .compatible = "samsung,exynos4212", .data = "cpufreq-dt" }, + { .compatible = "samsung,exynos4412", .data = "cpufreq-dt" }, + { .compatible = "samsung,exynos5250", .data = "cpufreq-dt" }, { /* sentinel */ } }; diff --git a/arch/arm/mach-highbank/highbank.c b/arch/arm/mach-highbank/highbank.c index 231fba0d03e5..6050a14faee6 100644 --- a/arch/arm/mach-highbank/highbank.c +++ b/arch/arm/mach-highbank/highbank.c @@ -28,8 +28,8 @@ #include #include #include +#include -#include #include #include #include diff --git a/arch/arm/mach-highbank/pm.c b/arch/arm/mach-highbank/pm.c index 7f2bd85eb935..400311695548 100644 --- a/arch/arm/mach-highbank/pm.c +++ b/arch/arm/mach-highbank/pm.c @@ -16,19 +16,21 @@ #include #include +#include #include #include -#include + +#include + +#define HIGHBANK_SUSPEND_PARAM \ + ((0 << PSCI_0_2_POWER_STATE_ID_SHIFT) | \ + (1 << PSCI_0_2_POWER_STATE_AFFL_SHIFT) | \ + (PSCI_POWER_STATE_TYPE_POWER_DOWN << PSCI_0_2_POWER_STATE_TYPE_SHIFT)) static int highbank_suspend_finish(unsigned long val) { - const struct psci_power_state ps = { - .type = PSCI_POWER_STATE_TYPE_POWER_DOWN, - .affinity_level = 1, - }; - - return psci_ops.cpu_suspend(ps, __pa(cpu_resume)); + return psci_ops.cpu_suspend(HIGHBANK_SUSPEND_PARAM, __pa(cpu_resume)); } static int highbank_pm_enter(suspend_state_t state) diff --git a/arch/arm/mach-imx/pm-imx5.c b/arch/arm/mach-imx/pm-imx5.c index 1885676c23c0..532d4b08276d 100644 --- a/arch/arm/mach-imx/pm-imx5.c +++ b/arch/arm/mach-imx/pm-imx5.c @@ -297,7 +297,7 @@ static int __init imx_suspend_alloc_ocram( goto put_node; } - ocram_pool = gen_pool_get(&pdev->dev); + ocram_pool = gen_pool_get(&pdev->dev, NULL); if (!ocram_pool) { pr_warn("%s: ocram pool unavailable!\n", __func__); ret = -ENODEV; diff --git a/arch/arm/mach-imx/pm-imx6.c b/arch/arm/mach-imx/pm-imx6.c index 93ecf559d06d..8ff8fc0b261c 100644 --- a/arch/arm/mach-imx/pm-imx6.c +++ b/arch/arm/mach-imx/pm-imx6.c @@ -451,7 +451,7 @@ static int __init imx6q_suspend_init(const struct imx6_pm_socdata *socdata) goto put_node; } - ocram_pool = gen_pool_get(&pdev->dev); + ocram_pool = gen_pool_get(&pdev->dev, NULL); if (!ocram_pool) { pr_warn("%s: ocram pool unavailable!\n", __func__); ret = -ENODEV; diff --git a/arch/arm/mach-mmp/include/mach/regs-rtc.h b/arch/arm/mach-mmp/include/mach/regs-rtc.h deleted file mode 100644 index 5bff886a3941..000000000000 --- a/arch/arm/mach-mmp/include/mach/regs-rtc.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef __ASM_MACH_REGS_RTC_H -#define __ASM_MACH_REGS_RTC_H - -#include - -#define RTC_VIRT_BASE (APB_VIRT_BASE + 0x10000) -#define RTC_REG(x) (*((volatile u32 __iomem *)(RTC_VIRT_BASE + (x)))) - -/* - * Real Time Clock - */ - -#define RCNR RTC_REG(0x00) /* RTC Count Register */ -#define RTAR RTC_REG(0x04) /* RTC Alarm Register */ -#define RTSR RTC_REG(0x08) /* RTC Status Register */ -#define RTTR RTC_REG(0x0C) /* RTC Timer Trim Register */ - -#define RTSR_HZE (1 << 3) /* HZ interrupt enable */ -#define RTSR_ALE (1 << 2) /* RTC alarm interrupt enable */ -#define RTSR_HZ (1 << 1) /* HZ rising-edge detected */ -#define RTSR_AL (1 << 0) /* RTC alarm detected */ - -#endif /* __ASM_MACH_REGS_RTC_H */ diff --git a/arch/arm/mach-mmp/pm-pxa910.c b/arch/arm/mach-mmp/pm-pxa910.c index 04c9daf9f8d7..7db5870d127f 100644 --- a/arch/arm/mach-mmp/pm-pxa910.c +++ b/arch/arm/mach-mmp/pm-pxa910.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/arm/mach-omap2/Kconfig b/arch/arm/mach-omap2/Kconfig index 9e2a68456b81..b3a0dff67e3f 100644 --- a/arch/arm/mach-omap2/Kconfig +++ b/arch/arm/mach-omap2/Kconfig @@ -29,6 +29,7 @@ config ARCH_OMAP4 select HAVE_ARM_SCU if SMP select HAVE_ARM_TWD if SMP select OMAP_INTERCONNECT + select OMAP_INTERCONNECT_BARRIER select PL310_ERRATA_588369 if CACHE_L2X0 select PL310_ERRATA_727915 if CACHE_L2X0 select PM_OPP if PM @@ -43,9 +44,11 @@ config SOC_OMAP5 select ARM_CPU_SUSPEND if PM select ARM_GIC select HAVE_ARM_SCU if SMP - select HAVE_ARM_TWD if SMP select HAVE_ARM_ARCH_TIMER select ARM_ERRATA_798181 if SMP + select OMAP_INTERCONNECT + select OMAP_INTERCONNECT_BARRIER + select PM_OPP if PM config SOC_AM33XX bool "TI AM33XX" @@ -68,9 +71,13 @@ config SOC_DRA7XX select ARCH_OMAP2PLUS select ARM_CPU_SUSPEND if PM select ARM_GIC + select HAVE_ARM_SCU if SMP select HAVE_ARM_ARCH_TIMER select IRQ_CROSSBAR select ARM_ERRATA_798181 if SMP + select OMAP_INTERCONNECT + select OMAP_INTERCONNECT_BARRIER + select PM_OPP if PM config ARCH_OMAP2PLUS bool @@ -92,6 +99,10 @@ config ARCH_OMAP2PLUS help Systems based on OMAP2, OMAP3, OMAP4 or OMAP5 +config OMAP_INTERCONNECT_BARRIER + bool + select ARM_HEAVY_MB + if ARCH_OMAP2PLUS diff --git a/arch/arm/mach-omap2/board-generic.c b/arch/arm/mach-omap2/board-generic.c index 24c9afc9e8a7..6133eaac685d 100644 --- a/arch/arm/mach-omap2/board-generic.c +++ b/arch/arm/mach-omap2/board-generic.c @@ -20,13 +20,6 @@ #include "common.h" -#if !(defined(CONFIG_ARCH_OMAP2) || defined(CONFIG_ARCH_OMAP3)) -#define intc_of_init NULL -#endif -#ifndef CONFIG_ARCH_OMAP4 -#define gic_of_init NULL -#endif - static const struct of_device_id omap_dt_match_table[] __initconst = { { .compatible = "simple-bus", }, { .compatible = "ti,omap-infra", }, diff --git a/arch/arm/mach-omap2/common.c b/arch/arm/mach-omap2/common.c index eae6a0e87c90..484cdadfb187 100644 --- a/arch/arm/mach-omap2/common.c +++ b/arch/arm/mach-omap2/common.c @@ -30,4 +30,5 @@ int __weak omap_secure_ram_reserve_memblock(void) void __init omap_reserve(void) { omap_secure_ram_reserve_memblock(); + omap_barrier_reserve_memblock(); } diff --git a/arch/arm/mach-omap2/common.h b/arch/arm/mach-omap2/common.h index 749d50bb4ca5..92e92cfc2775 100644 --- a/arch/arm/mach-omap2/common.h +++ b/arch/arm/mach-omap2/common.h @@ -189,6 +189,15 @@ static inline void omap44xx_restart(enum reboot_mode mode, const char *cmd) } #endif +#ifdef CONFIG_OMAP_INTERCONNECT_BARRIER +void omap_barrier_reserve_memblock(void); +void omap_barriers_init(void); +#else +static inline void omap_barrier_reserve_memblock(void) +{ +} +#endif + /* This gets called from mach-omap2/io.c, do not call this */ void __init omap2_set_globals_tap(u32 class, void __iomem *tap); diff --git a/arch/arm/mach-omap2/include/mach/barriers.h b/arch/arm/mach-omap2/include/mach/barriers.h deleted file mode 100644 index 1c582a8592b9..000000000000 --- a/arch/arm/mach-omap2/include/mach/barriers.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * OMAP memory barrier header. - * - * Copyright (C) 2011 Texas Instruments, Inc. - * Santosh Shilimkar - * Richard Woodruff - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#ifndef __MACH_BARRIERS_H -#define __MACH_BARRIERS_H - -#include - -extern void omap_bus_sync(void); - -#define rmb() dsb() -#define wmb() do { dsb(); outer_sync(); omap_bus_sync(); } while (0) -#define mb() wmb() - -#endif /* __MACH_BARRIERS_H */ diff --git a/arch/arm/mach-omap2/io.c b/arch/arm/mach-omap2/io.c index 6a4822dbb4ea..980c9372e6fd 100644 --- a/arch/arm/mach-omap2/io.c +++ b/arch/arm/mach-omap2/io.c @@ -352,6 +352,7 @@ void __init am33xx_map_io(void) void __init omap4_map_io(void) { iotable_init(omap44xx_io_desc, ARRAY_SIZE(omap44xx_io_desc)); + omap_barriers_init(); } #endif @@ -359,6 +360,7 @@ void __init omap4_map_io(void) void __init omap5_map_io(void) { iotable_init(omap54xx_io_desc, ARRAY_SIZE(omap54xx_io_desc)); + omap_barriers_init(); } #endif diff --git a/arch/arm/mach-omap2/omap4-common.c b/arch/arm/mach-omap2/omap4-common.c index 16350eefa66c..949696b6f17b 100644 --- a/arch/arm/mach-omap2/omap4-common.c +++ b/arch/arm/mach-omap2/omap4-common.c @@ -51,6 +51,127 @@ static void __iomem *twd_base; #define IRQ_LOCALTIMER 29 +#ifdef CONFIG_OMAP_INTERCONNECT_BARRIER + +/* Used to implement memory barrier on DRAM path */ +#define OMAP4_DRAM_BARRIER_VA 0xfe600000 + +static void __iomem *dram_sync, *sram_sync; +static phys_addr_t dram_sync_paddr; +static u32 dram_sync_size; + +/* + * The OMAP4 bus structure contains asynchrnous bridges which can buffer + * data writes from the MPU. These asynchronous bridges can be found on + * paths between the MPU to EMIF, and the MPU to L3 interconnects. + * + * We need to be careful about re-ordering which can happen as a result + * of different accesses being performed via different paths, and + * therefore different asynchronous bridges. + */ + +/* + * OMAP4 interconnect barrier which is called for each mb() and wmb(). + * This is to ensure that normal paths to DRAM (normal memory, cacheable + * accesses) are properly synchronised with writes to DMA coherent memory + * (normal memory, uncacheable) and device writes. + * + * The mb() and wmb() barriers only operate only on the MPU->MA->EMIF + * path, as we need to ensure that data is visible to other system + * masters prior to writes to those system masters being seen. + * + * Note: the SRAM path is not synchronised via mb() and wmb(). + */ +static void omap4_mb(void) +{ + if (dram_sync) + writel_relaxed(0, dram_sync); +} + +/* + * OMAP4 Errata i688 - asynchronous bridge corruption when entering WFI. + * + * If a data is stalled inside asynchronous bridge because of back + * pressure, it may be accepted multiple times, creating pointer + * misalignment that will corrupt next transfers on that data path until + * next reset of the system. No recovery procedure once the issue is hit, + * the path remains consistently broken. + * + * Async bridges can be found on paths between MPU to EMIF and MPU to L3 + * interconnects. + * + * This situation can happen only when the idle is initiated by a Master + * Request Disconnection (which is trigged by software when executing WFI + * on the CPU). + * + * The work-around for this errata needs all the initiators connected + * through an async bridge to ensure that data path is properly drained + * before issuing WFI. This condition will be met if one Strongly ordered + * access is performed to the target right before executing the WFI. + * + * In MPU case, L3 T2ASYNC FIFO and DDR T2ASYNC FIFO needs to be drained. + * IO barrier ensure that there is no synchronisation loss on initiators + * operating on both interconnect port simultaneously. + * + * This is a stronger version of the OMAP4 memory barrier below, and + * operates on both the MPU->MA->EMIF path but also the MPU->OCP path + * as well, and is necessary prior to executing a WFI. + */ +void omap_interconnect_sync(void) +{ + if (dram_sync && sram_sync) { + writel_relaxed(readl_relaxed(dram_sync), dram_sync); + writel_relaxed(readl_relaxed(sram_sync), sram_sync); + isb(); + } +} + +static int __init omap4_sram_init(void) +{ + struct device_node *np; + struct gen_pool *sram_pool; + + np = of_find_compatible_node(NULL, NULL, "ti,omap4-mpu"); + if (!np) + pr_warn("%s:Unable to allocate sram needed to handle errata I688\n", + __func__); + sram_pool = of_gen_pool_get(np, "sram", 0); + if (!sram_pool) + pr_warn("%s:Unable to get sram pool needed to handle errata I688\n", + __func__); + else + sram_sync = (void *)gen_pool_alloc(sram_pool, PAGE_SIZE); + + return 0; +} +omap_arch_initcall(omap4_sram_init); + +/* Steal one page physical memory for barrier implementation */ +void __init omap_barrier_reserve_memblock(void) +{ + dram_sync_size = ALIGN(PAGE_SIZE, SZ_1M); + dram_sync_paddr = arm_memblock_steal(dram_sync_size, SZ_1M); +} + +void __init omap_barriers_init(void) +{ + struct map_desc dram_io_desc[1]; + + dram_io_desc[0].virtual = OMAP4_DRAM_BARRIER_VA; + dram_io_desc[0].pfn = __phys_to_pfn(dram_sync_paddr); + dram_io_desc[0].length = dram_sync_size; + dram_io_desc[0].type = MT_MEMORY_RW_SO; + iotable_init(dram_io_desc, ARRAY_SIZE(dram_io_desc)); + dram_sync = (void __iomem *) dram_io_desc[0].virtual; + + pr_info("OMAP4: Map %pa to %p for dram barrier\n", + &dram_sync_paddr, dram_sync); + + soc_mb = omap4_mb; +} + +#endif + void gic_dist_disable(void) { if (gic_dist_base_addr) diff --git a/arch/arm/mach-omap2/pm.h b/arch/arm/mach-omap2/pm.h index 425bfcd67db6..b668719b9b25 100644 --- a/arch/arm/mach-omap2/pm.h +++ b/arch/arm/mach-omap2/pm.h @@ -103,7 +103,8 @@ static inline void enable_omap3630_toggle_l2_on_restore(void) { } #define PM_OMAP4_ROM_SMP_BOOT_ERRATUM_GICD (1 << 0) #define PM_OMAP4_CPU_OSWR_DISABLE (1 << 1) -#if defined(CONFIG_PM) && defined(CONFIG_ARCH_OMAP4) +#if defined(CONFIG_PM) && (defined(CONFIG_ARCH_OMAP4) ||\ + defined(CONFIG_SOC_OMAP5) || defined(CONFIG_SOC_DRA7XX)) extern u16 pm44xx_errata; #define IS_PM44XX_ERRATUM(id) (pm44xx_errata & (id)) #else diff --git a/arch/arm/mach-omap2/sleep44xx.S b/arch/arm/mach-omap2/sleep44xx.S index ad1bb9431e94..9b09d85d811a 100644 --- a/arch/arm/mach-omap2/sleep44xx.S +++ b/arch/arm/mach-omap2/sleep44xx.S @@ -333,14 +333,12 @@ ENDPROC(omap4_cpu_resume) #endif /* defined(CONFIG_SMP) && defined(CONFIG_PM) */ -ENTRY(omap_bus_sync) - ret lr -ENDPROC(omap_bus_sync) - ENTRY(omap_do_wfi) stmfd sp!, {lr} +#ifdef CONFIG_OMAP_INTERCONNECT_BARRIER /* Drain interconnect write buffers. */ - bl omap_bus_sync + bl omap_interconnect_sync +#endif /* * Execute an ISB instruction to ensure that all of the diff --git a/arch/arm/mach-prima2/pm.c b/arch/arm/mach-prima2/pm.c index d99d08eeb966..83e94c95e314 100644 --- a/arch/arm/mach-prima2/pm.c +++ b/arch/arm/mach-prima2/pm.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include diff --git a/arch/arm/mach-pxa/devices.c b/arch/arm/mach-pxa/devices.c index e6ce669b54af..c62473235a13 100644 --- a/arch/arm/mach-pxa/devices.c +++ b/arch/arm/mach-pxa/devices.c @@ -440,25 +440,11 @@ struct platform_device pxa_device_rtc = { .resource = pxa_rtc_resources, }; -static struct resource sa1100_rtc_resources[] = { - { - .start = IRQ_RTC1Hz, - .end = IRQ_RTC1Hz, - .name = "rtc 1Hz", - .flags = IORESOURCE_IRQ, - }, { - .start = IRQ_RTCAlrm, - .end = IRQ_RTCAlrm, - .name = "rtc alarm", - .flags = IORESOURCE_IRQ, - }, -}; - struct platform_device sa1100_device_rtc = { .name = "sa1100-rtc", .id = -1, - .num_resources = ARRAY_SIZE(sa1100_rtc_resources), - .resource = sa1100_rtc_resources, + .num_resources = ARRAY_SIZE(pxa_rtc_resources), + .resource = pxa_rtc_resources, }; static struct resource pxa_ac97_resources[] = { diff --git a/arch/arm/mach-pxa/pxa27x.c b/arch/arm/mach-pxa/pxa27x.c index e6aae9e8adfb..221260d5d109 100644 --- a/arch/arm/mach-pxa/pxa27x.c +++ b/arch/arm/mach-pxa/pxa27x.c @@ -282,7 +282,6 @@ static struct platform_device *devices[] __initdata = { &pxa_device_asoc_ssp2, &pxa_device_asoc_ssp3, &pxa_device_asoc_platform, - &sa1100_device_rtc, &pxa_device_rtc, &pxa27x_device_ssp1, &pxa27x_device_ssp2, diff --git a/arch/arm/mach-pxa/pxa3xx.c b/arch/arm/mach-pxa/pxa3xx.c index 165638462a2f..ce0f8d6242e2 100644 --- a/arch/arm/mach-pxa/pxa3xx.c +++ b/arch/arm/mach-pxa/pxa3xx.c @@ -394,7 +394,6 @@ static struct platform_device *devices[] __initdata = { &pxa_device_asoc_ssp3, &pxa_device_asoc_ssp4, &pxa_device_asoc_platform, - &sa1100_device_rtc, &pxa_device_rtc, &pxa3xx_device_ssp1, &pxa3xx_device_ssp2, diff --git a/arch/arm/mach-sa1100/include/mach/SA-1100.h b/arch/arm/mach-sa1100/include/mach/SA-1100.h index 0ac6cc08a19c..7972617cca64 100644 --- a/arch/arm/mach-sa1100/include/mach/SA-1100.h +++ b/arch/arm/mach-sa1100/include/mach/SA-1100.h @@ -857,40 +857,6 @@ #define OIER_E3 OIER_E (3) /* match interrupt Enable 3 */ -/* - * Real-Time Clock (RTC) control registers - * - * Registers - * RTAR Real-Time Clock (RTC) Alarm Register (read/write). - * RCNR Real-Time Clock (RTC) CouNt Register (read/write). - * RTTR Real-Time Clock (RTC) Trim Register (read/write). - * RTSR Real-Time Clock (RTC) Status Register (read/write). - * - * Clocks - * frtx, Trtx Frequency, period of the real-time clock crystal - * (32.768 kHz nominal). - * frtc, Trtc Frequency, period of the real-time clock counter - * (1 Hz nominal). - */ - -#define RTAR __REG(0x90010000) /* RTC Alarm Reg. */ -#define RCNR __REG(0x90010004) /* RTC CouNt Reg. */ -#define RTTR __REG(0x90010008) /* RTC Trim Reg. */ -#define RTSR __REG(0x90010010) /* RTC Status Reg. */ - -#define RTTR_C Fld (16, 0) /* clock divider Count - 1 */ -#define RTTR_D Fld (10, 16) /* trim Delete count */ - /* frtc = (1023*(C + 1) - D)*frtx/ */ - /* (1023*(C + 1)^2) */ - /* Trtc = (1023*(C + 1)^2)*Trtx/ */ - /* (1023*(C + 1) - D) */ - -#define RTSR_AL 0x00000001 /* ALarm detected */ -#define RTSR_HZ 0x00000002 /* 1 Hz clock detected */ -#define RTSR_ALE 0x00000004 /* ALarm interrupt Enable */ -#define RTSR_HZE 0x00000008 /* 1 Hz clock interrupt Enable */ - - /* * Power Manager (PM) control registers * diff --git a/arch/arm/mach-shmobile/common.h b/arch/arm/mach-shmobile/common.h index 476092b86c6e..8d27ec546a35 100644 --- a/arch/arm/mach-shmobile/common.h +++ b/arch/arm/mach-shmobile/common.h @@ -13,7 +13,7 @@ extern void shmobile_smp_boot(void); extern void shmobile_smp_sleep(void); extern void shmobile_smp_hook(unsigned int cpu, unsigned long fn, unsigned long arg); -extern int shmobile_smp_cpu_disable(unsigned int cpu); +extern bool shmobile_smp_cpu_can_disable(unsigned int cpu); extern void shmobile_boot_scu(void); extern void shmobile_smp_scu_prepare_cpus(unsigned int max_cpus); extern void shmobile_smp_scu_cpu_die(unsigned int cpu); diff --git a/arch/arm/mach-shmobile/platsmp.c b/arch/arm/mach-shmobile/platsmp.c index 3923e09e966d..b23378f3d7e1 100644 --- a/arch/arm/mach-shmobile/platsmp.c +++ b/arch/arm/mach-shmobile/platsmp.c @@ -31,8 +31,8 @@ void shmobile_smp_hook(unsigned int cpu, unsigned long fn, unsigned long arg) } #ifdef CONFIG_HOTPLUG_CPU -int shmobile_smp_cpu_disable(unsigned int cpu) +bool shmobile_smp_cpu_can_disable(unsigned int cpu) { - return 0; /* Hotplug of any CPU is supported */ + return true; /* Hotplug of any CPU is supported */ } #endif diff --git a/arch/arm/mach-shmobile/pm-rcar.c b/arch/arm/mach-shmobile/pm-rcar.c index 4092ad16e0a4..0af05d288b09 100644 --- a/arch/arm/mach-shmobile/pm-rcar.c +++ b/arch/arm/mach-shmobile/pm-rcar.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include "pm-rcar.h" /* SYSC Common */ diff --git a/arch/arm/mach-shmobile/setup-r8a7779.c b/arch/arm/mach-shmobile/setup-r8a7779.c index 7ca891999547..6bfa6407a27c 100644 --- a/arch/arm/mach-shmobile/setup-r8a7779.c +++ b/arch/arm/mach-shmobile/setup-r8a7779.c @@ -62,8 +62,6 @@ static void __init r8a7779_map_io(void) static void __init r8a7779_init_irq_dt(void) { - gic_set_irqchip_flags(IRQCHIP_SKIP_SET_WAKE); - irqchip_init(); /* route all interrupts to ARM */ diff --git a/arch/arm/mach-shmobile/smp-r8a7790.c b/arch/arm/mach-shmobile/smp-r8a7790.c index 2ef0054ce934..4b33d432a364 100644 --- a/arch/arm/mach-shmobile/smp-r8a7790.c +++ b/arch/arm/mach-shmobile/smp-r8a7790.c @@ -64,7 +64,7 @@ struct smp_operations r8a7790_smp_ops __initdata = { .smp_prepare_cpus = r8a7790_smp_prepare_cpus, .smp_boot_secondary = shmobile_smp_apmu_boot_secondary, #ifdef CONFIG_HOTPLUG_CPU - .cpu_disable = shmobile_smp_cpu_disable, + .cpu_can_disable = shmobile_smp_cpu_can_disable, .cpu_die = shmobile_smp_apmu_cpu_die, .cpu_kill = shmobile_smp_apmu_cpu_kill, #endif diff --git a/arch/arm/mach-shmobile/smp-r8a7791.c b/arch/arm/mach-shmobile/smp-r8a7791.c index 5e2d1db79afa..b2508c0d276b 100644 --- a/arch/arm/mach-shmobile/smp-r8a7791.c +++ b/arch/arm/mach-shmobile/smp-r8a7791.c @@ -58,7 +58,7 @@ struct smp_operations r8a7791_smp_ops __initdata = { .smp_prepare_cpus = r8a7791_smp_prepare_cpus, .smp_boot_secondary = r8a7791_smp_boot_secondary, #ifdef CONFIG_HOTPLUG_CPU - .cpu_disable = shmobile_smp_cpu_disable, + .cpu_can_disable = shmobile_smp_cpu_can_disable, .cpu_die = shmobile_smp_apmu_cpu_die, .cpu_kill = shmobile_smp_apmu_cpu_kill, #endif diff --git a/arch/arm/mach-shmobile/smp-sh73a0.c b/arch/arm/mach-shmobile/smp-sh73a0.c index d03aa11fb46d..bc2824a036e1 100644 --- a/arch/arm/mach-shmobile/smp-sh73a0.c +++ b/arch/arm/mach-shmobile/smp-sh73a0.c @@ -60,7 +60,7 @@ struct smp_operations sh73a0_smp_ops __initdata = { .smp_prepare_cpus = sh73a0_smp_prepare_cpus, .smp_boot_secondary = sh73a0_boot_secondary, #ifdef CONFIG_HOTPLUG_CPU - .cpu_disable = shmobile_smp_cpu_disable, + .cpu_can_disable = shmobile_smp_cpu_can_disable, .cpu_die = shmobile_smp_scu_cpu_die, .cpu_kill = shmobile_smp_scu_cpu_kill, #endif diff --git a/arch/arm/mach-socfpga/pm.c b/arch/arm/mach-socfpga/pm.c index 6a4199f2bffb..c378ab0c2431 100644 --- a/arch/arm/mach-socfpga/pm.c +++ b/arch/arm/mach-socfpga/pm.c @@ -56,7 +56,7 @@ static int socfpga_setup_ocram_self_refresh(void) goto put_node; } - ocram_pool = gen_pool_get(&pdev->dev); + ocram_pool = gen_pool_get(&pdev->dev, NULL); if (!ocram_pool) { pr_warn("%s: ocram pool unavailable!\n", __func__); ret = -ENODEV; diff --git a/arch/arm/mach-ux500/cache-l2x0.c b/arch/arm/mach-ux500/cache-l2x0.c index 7557bede7ae6..780bd13cd7e3 100644 --- a/arch/arm/mach-ux500/cache-l2x0.c +++ b/arch/arm/mach-ux500/cache-l2x0.c @@ -8,6 +8,7 @@ #include #include +#include #include #include "db8500-regs.h" diff --git a/arch/arm/mach-ux500/cpu-db8500.c b/arch/arm/mach-ux500/cpu-db8500.c index ba708ce08616..f80560318c58 100644 --- a/arch/arm/mach-ux500/cpu-db8500.c +++ b/arch/arm/mach-ux500/cpu-db8500.c @@ -20,10 +20,10 @@ #include #include #include +#include #include #include -#include #include #include "setup.h" diff --git a/arch/arm/mach-ux500/cpu.c b/arch/arm/mach-ux500/cpu.c index b316e18a76aa..41b81c4fbe63 100644 --- a/arch/arm/mach-ux500/cpu.c +++ b/arch/arm/mach-ux500/cpu.c @@ -56,7 +56,6 @@ void __init ux500_init_irq(void) struct device_node *np; struct resource r; - gic_set_irqchip_flags(IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND); irqchip_init(); np = of_find_compatible_node(NULL, NULL, "stericsson,db8500-prcmu"); of_address_to_resource(np, 0, &r); diff --git a/arch/arm/mach-vexpress/tc2_pm.c b/arch/arm/mach-vexpress/tc2_pm.c index b3328cd46c33..1aa4ccece69f 100644 --- a/arch/arm/mach-vexpress/tc2_pm.c +++ b/arch/arm/mach-vexpress/tc2_pm.c @@ -80,7 +80,7 @@ static void tc2_pm_cpu_powerdown_prepare(unsigned int cpu, unsigned int cluster) * to the CPU by disabling the GIC CPU IF to prevent wfi * from completing execution behind power controller back */ - gic_cpu_if_down(); + gic_cpu_if_down(0); } static void tc2_pm_cluster_powerdown_prepare(unsigned int cluster) diff --git a/arch/arm/mach-zynq/common.c b/arch/arm/mach-zynq/common.c index 6bd4a43e1a78..5a6e4e20ca0a 100644 --- a/arch/arm/mach-zynq/common.c +++ b/arch/arm/mach-zynq/common.c @@ -186,7 +186,6 @@ static void __init zynq_map_io(void) static void __init zynq_irq_init(void) { - gic_set_irqchip_flags(IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND); irqchip_init(); } diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 7c6b976ab8d3..df7537f12469 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -883,6 +883,7 @@ config OUTER_CACHE config OUTER_CACHE_SYNC bool + select ARM_HEAVY_MB help The outer cache has a outer_cache_fns.sync function pointer that can be used to drain the write buffer of the outer cache. @@ -1031,6 +1032,9 @@ config ARCH_HAS_BARRIERS This option allows the use of custom mandatory barriers included via the mach/barriers.h file. +config ARM_HEAVY_MB + bool + config ARCH_SUPPORTS_BIG_ENDIAN bool help diff --git a/arch/arm/mm/abort-ev4.S b/arch/arm/mm/abort-ev4.S index 54473cd4aba9..b3b31e30cadd 100644 --- a/arch/arm/mm/abort-ev4.S +++ b/arch/arm/mm/abort-ev4.S @@ -19,6 +19,7 @@ ENTRY(v4_early_abort) mrc p15, 0, r1, c5, c0, 0 @ get FSR mrc p15, 0, r0, c6, c0, 0 @ get FAR ldr r3, [r4] @ read aborted ARM instruction + uaccess_disable ip @ disable userspace access bic r1, r1, #1 << 11 | 1 << 10 @ clear bits 11 and 10 of FSR tst r3, #1 << 20 @ L = 1 -> write? orreq r1, r1, #1 << 11 @ yes. diff --git a/arch/arm/mm/abort-ev5t.S b/arch/arm/mm/abort-ev5t.S index a0908d4653a3..a6a381a6caa5 100644 --- a/arch/arm/mm/abort-ev5t.S +++ b/arch/arm/mm/abort-ev5t.S @@ -21,8 +21,10 @@ ENTRY(v5t_early_abort) mrc p15, 0, r0, c6, c0, 0 @ get FAR do_thumb_abort fsr=r1, pc=r4, psr=r5, tmp=r3 ldreq r3, [r4] @ read aborted ARM instruction + uaccess_disable ip @ disable user access bic r1, r1, #1 << 11 @ clear bits 11 of FSR - do_ldrd_abort tmp=ip, insn=r3 + teq_ldrd tmp=ip, insn=r3 @ insn was LDRD? + beq do_DataAbort @ yes tst r3, #1 << 20 @ check write orreq r1, r1, #1 << 11 b do_DataAbort diff --git a/arch/arm/mm/abort-ev5tj.S b/arch/arm/mm/abort-ev5tj.S index 4006b7a61264..00ab011bef58 100644 --- a/arch/arm/mm/abort-ev5tj.S +++ b/arch/arm/mm/abort-ev5tj.S @@ -24,7 +24,9 @@ ENTRY(v5tj_early_abort) bne do_DataAbort do_thumb_abort fsr=r1, pc=r4, psr=r5, tmp=r3 ldreq r3, [r4] @ read aborted ARM instruction - do_ldrd_abort tmp=ip, insn=r3 + uaccess_disable ip @ disable userspace access + teq_ldrd tmp=ip, insn=r3 @ insn was LDRD? + beq do_DataAbort @ yes tst r3, #1 << 20 @ L = 0 -> write orreq r1, r1, #1 << 11 @ yes. b do_DataAbort diff --git a/arch/arm/mm/abort-ev6.S b/arch/arm/mm/abort-ev6.S index 8c48c5c22a33..8801a15aa105 100644 --- a/arch/arm/mm/abort-ev6.S +++ b/arch/arm/mm/abort-ev6.S @@ -26,16 +26,18 @@ ENTRY(v6_early_abort) ldr ip, =0x4107b36 mrc p15, 0, r3, c0, c0, 0 @ get processor id teq ip, r3, lsr #4 @ r0 ARM1136? - bne do_DataAbort + bne 1f tst r5, #PSR_J_BIT @ Java? tsteq r5, #PSR_T_BIT @ Thumb? - bne do_DataAbort + bne 1f bic r1, r1, #1 << 11 @ clear bit 11 of FSR ldr r3, [r4] @ read aborted ARM instruction ARM_BE8(rev r3, r3) - do_ldrd_abort tmp=ip, insn=r3 + teq_ldrd tmp=ip, insn=r3 @ insn was LDRD? + beq 1f @ yes tst r3, #1 << 20 @ L = 0 -> write orreq r1, r1, #1 << 11 @ yes. #endif +1: uaccess_disable ip @ disable userspace access b do_DataAbort diff --git a/arch/arm/mm/abort-ev7.S b/arch/arm/mm/abort-ev7.S index 4812ad054214..e8d0e08c227f 100644 --- a/arch/arm/mm/abort-ev7.S +++ b/arch/arm/mm/abort-ev7.S @@ -15,6 +15,7 @@ ENTRY(v7_early_abort) mrc p15, 0, r1, c5, c0, 0 @ get FSR mrc p15, 0, r0, c6, c0, 0 @ get FAR + uaccess_disable ip @ disable userspace access /* * V6 code adjusts the returned DFSR. diff --git a/arch/arm/mm/abort-lv4t.S b/arch/arm/mm/abort-lv4t.S index f3982580c273..6d8e8e3365d1 100644 --- a/arch/arm/mm/abort-lv4t.S +++ b/arch/arm/mm/abort-lv4t.S @@ -26,6 +26,7 @@ ENTRY(v4t_late_abort) #endif bne .data_thumb_abort ldr r8, [r4] @ read arm instruction + uaccess_disable ip @ disable userspace access tst r8, #1 << 20 @ L = 1 -> write? orreq r1, r1, #1 << 11 @ yes. and r7, r8, #15 << 24 @@ -155,6 +156,7 @@ ENTRY(v4t_late_abort) .data_thumb_abort: ldrh r8, [r4] @ read instruction + uaccess_disable ip @ disable userspace access tst r8, #1 << 11 @ L = 1 -> write? orreq r1, r1, #1 << 8 @ yes and r7, r8, #15 << 12 diff --git a/arch/arm/mm/abort-macro.S b/arch/arm/mm/abort-macro.S index 2cbf68ef0e83..4509bee4e081 100644 --- a/arch/arm/mm/abort-macro.S +++ b/arch/arm/mm/abort-macro.S @@ -13,6 +13,7 @@ tst \psr, #PSR_T_BIT beq not_thumb ldrh \tmp, [\pc] @ Read aborted Thumb instruction + uaccess_disable ip @ disable userspace access and \tmp, \tmp, # 0xfe00 @ Mask opcode field cmp \tmp, # 0x5600 @ Is it ldrsb? orreq \tmp, \tmp, #1 << 11 @ Set L-bit if yes @@ -29,12 +30,9 @@ not_thumb: * [7:4] == 1101 * [20] == 0 */ - .macro do_ldrd_abort, tmp, insn - tst \insn, #0x0e100000 @ [27:25,20] == 0 - bne not_ldrd - and \tmp, \insn, #0x000000f0 @ [7:4] == 1101 - cmp \tmp, #0x000000d0 - beq do_DataAbort -not_ldrd: + .macro teq_ldrd, tmp, insn + mov \tmp, #0x0e100000 + orr \tmp, #0x000000f0 + and \tmp, \insn, \tmp + teq \tmp, #0x000000d0 .endm - diff --git a/arch/arm/mm/cache-feroceon-l2.c b/arch/arm/mm/cache-feroceon-l2.c index 097181e08c25..5c1b7a7b9af6 100644 --- a/arch/arm/mm/cache-feroceon-l2.c +++ b/arch/arm/mm/cache-feroceon-l2.c @@ -368,7 +368,6 @@ int __init feroceon_of_init(void) struct device_node *node; void __iomem *base; bool l2_wt_override = false; - struct resource res; #if defined(CONFIG_CACHE_FEROCEON_L2_WRITETHROUGH) l2_wt_override = true; @@ -376,10 +375,7 @@ int __init feroceon_of_init(void) node = of_find_matching_node(NULL, feroceon_ids); if (node && of_device_is_compatible(node, "marvell,kirkwood-cache")) { - if (of_address_to_resource(node, 0, &res)) - return -ENODEV; - - base = ioremap(res.start, resource_size(&res)); + base = of_iomap(node, 0); if (!base) return -ENOMEM; diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c index 71b3d3309024..493692d838c6 100644 --- a/arch/arm/mm/cache-l2x0.c +++ b/arch/arm/mm/cache-l2x0.c @@ -1171,6 +1171,11 @@ static void __init l2c310_of_parse(const struct device_node *np, } } + if (of_property_read_bool(np, "arm,shared-override")) { + *aux_val |= L2C_AUX_CTRL_SHARED_OVERRIDE; + *aux_mask &= ~L2C_AUX_CTRL_SHARED_OVERRIDE; + } + prefetch = l2x0_saved_regs.prefetch_ctrl; ret = of_property_read_u32(np, "arm,double-linefill", &val); diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index cba12f34ff77..e62604384945 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -39,6 +39,7 @@ #include #include +#include "dma.h" #include "mm.h" /* @@ -648,14 +649,18 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, size = PAGE_ALIGN(size); want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs); - if (is_coherent || nommu()) + if (nommu()) + addr = __alloc_simple_buffer(dev, size, gfp, &page); + else if (dev_get_cma_area(dev) && (gfp & __GFP_WAIT)) + addr = __alloc_from_contiguous(dev, size, prot, &page, + caller, want_vaddr); + else if (is_coherent) addr = __alloc_simple_buffer(dev, size, gfp, &page); else if (!(gfp & __GFP_WAIT)) addr = __alloc_from_pool(size, &page); - else if (!dev_get_cma_area(dev)) - addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller, want_vaddr); else - addr = __alloc_from_contiguous(dev, size, prot, &page, caller, want_vaddr); + addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, + caller, want_vaddr); if (page) *handle = pfn_to_dma(dev, page_to_pfn(page)); @@ -671,10 +676,6 @@ void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) { pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); - void *memory; - - if (dma_alloc_from_coherent(dev, size, handle, &memory)) - return memory; return __dma_alloc(dev, size, handle, gfp, prot, false, attrs, __builtin_return_address(0)); @@ -683,13 +684,7 @@ void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, static void *arm_coherent_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs) { - pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); - void *memory; - - if (dma_alloc_from_coherent(dev, size, handle, &memory)) - return memory; - - return __dma_alloc(dev, size, handle, gfp, prot, true, + return __dma_alloc(dev, size, handle, gfp, PAGE_KERNEL, true, attrs, __builtin_return_address(0)); } @@ -748,17 +743,14 @@ static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); bool want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs); - if (dma_release_from_coherent(dev, get_order(size), cpu_addr)) - return; - size = PAGE_ALIGN(size); - if (is_coherent || nommu()) { + if (nommu()) { __dma_free_buffer(page, size); - } else if (__free_from_pool(cpu_addr, size)) { + } else if (!is_coherent && __free_from_pool(cpu_addr, size)) { return; } else if (!dev_get_cma_area(dev)) { - if (want_vaddr) + if (want_vaddr && !is_coherent) __dma_free_remap(cpu_addr, size); __dma_free_buffer(page, size); } else { @@ -1520,7 +1512,7 @@ static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, return -ENOMEM; for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) { - phys_addr_t phys = page_to_phys(sg_page(s)); + phys_addr_t phys = sg_phys(s) & PAGE_MASK; unsigned int len = PAGE_ALIGN(s->offset + s->length); if (!is_coherent && diff --git a/arch/arm/mm/dma.h b/arch/arm/mm/dma.h new file mode 100644 index 000000000000..70ea6852f94e --- /dev/null +++ b/arch/arm/mm/dma.h @@ -0,0 +1,32 @@ +#ifndef DMA_H +#define DMA_H + +#include + +#ifndef MULTI_CACHE +#define dmac_map_area __glue(_CACHE,_dma_map_area) +#define dmac_unmap_area __glue(_CACHE,_dma_unmap_area) + +/* + * These are private to the dma-mapping API. Do not use directly. + * Their sole purpose is to ensure that data held in the cache + * is visible to DMA, or data written by DMA to system memory is + * visible to the CPU. + */ +extern void dmac_map_area(const void *, size_t, int); +extern void dmac_unmap_area(const void *, size_t, int); + +#else + +/* + * These are private to the dma-mapping API. Do not use directly. + * Their sole purpose is to ensure that data held in the cache + * is visible to DMA, or data written by DMA to system memory is + * visible to the CPU. + */ +#define dmac_map_area cpu_cache.dma_map_area +#define dmac_unmap_area cpu_cache.dma_unmap_area + +#endif + +#endif diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index 34b66af516ea..1ec8e7590fc6 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -21,6 +21,21 @@ #include "mm.h" +#ifdef CONFIG_ARM_HEAVY_MB +void (*soc_mb)(void); + +void arm_heavy_mb(void) +{ +#ifdef CONFIG_OUTER_CACHE_SYNC + if (outer_cache.sync) + outer_cache.sync(); +#endif + if (soc_mb) + soc_mb(); +} +EXPORT_SYMBOL(arm_heavy_mb); +#endif + #ifdef CONFIG_CPU_CACHE_VIPT static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr) diff --git a/arch/arm/mm/highmem.c b/arch/arm/mm/highmem.c index ee8dfa793989..9df5f09585ca 100644 --- a/arch/arm/mm/highmem.c +++ b/arch/arm/mm/highmem.c @@ -79,7 +79,7 @@ void *kmap_atomic(struct page *page) type = kmap_atomic_idx_push(); - idx = type + KM_TYPE_NR * smp_processor_id(); + idx = FIX_KMAP_BEGIN + type + KM_TYPE_NR * smp_processor_id(); vaddr = __fix_to_virt(idx); #ifdef CONFIG_DEBUG_HIGHMEM /* @@ -106,7 +106,7 @@ void __kunmap_atomic(void *kvaddr) if (kvaddr >= (void *)FIXADDR_START) { type = kmap_atomic_idx(); - idx = type + KM_TYPE_NR * smp_processor_id(); + idx = FIX_KMAP_BEGIN + type + KM_TYPE_NR * smp_processor_id(); if (cache_is_vivt()) __cpuc_flush_dcache_area((void *)vaddr, PAGE_SIZE); @@ -138,7 +138,7 @@ void *kmap_atomic_pfn(unsigned long pfn) return page_address(page); type = kmap_atomic_idx_push(); - idx = type + KM_TYPE_NR * smp_processor_id(); + idx = FIX_KMAP_BEGIN + type + KM_TYPE_NR * smp_processor_id(); vaddr = __fix_to_virt(idx); #ifdef CONFIG_DEBUG_HIGHMEM BUG_ON(!pte_none(get_fixmap_pte(vaddr))); diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 870838a46d52..7cd15143a507 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -291,13 +291,13 @@ static struct mem_type mem_types[] = { .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | L_PTE_RDONLY, .prot_l1 = PMD_TYPE_TABLE, - .domain = DOMAIN_USER, + .domain = DOMAIN_VECTORS, }, [MT_HIGH_VECTORS] = { .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | L_PTE_USER | L_PTE_RDONLY, .prot_l1 = PMD_TYPE_TABLE, - .domain = DOMAIN_USER, + .domain = DOMAIN_VECTORS, }, [MT_MEMORY_RWX] = { .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY, @@ -357,6 +357,47 @@ const struct mem_type *get_mem_type(unsigned int type) } EXPORT_SYMBOL(get_mem_type); +static pte_t *(*pte_offset_fixmap)(pmd_t *dir, unsigned long addr); + +static pte_t bm_pte[PTRS_PER_PTE + PTE_HWTABLE_PTRS] + __aligned(PTE_HWTABLE_OFF + PTE_HWTABLE_SIZE) __initdata; + +static pte_t * __init pte_offset_early_fixmap(pmd_t *dir, unsigned long addr) +{ + return &bm_pte[pte_index(addr)]; +} + +static pte_t *pte_offset_late_fixmap(pmd_t *dir, unsigned long addr) +{ + return pte_offset_kernel(dir, addr); +} + +static inline pmd_t * __init fixmap_pmd(unsigned long addr) +{ + pgd_t *pgd = pgd_offset_k(addr); + pud_t *pud = pud_offset(pgd, addr); + pmd_t *pmd = pmd_offset(pud, addr); + + return pmd; +} + +void __init early_fixmap_init(void) +{ + pmd_t *pmd; + + /* + * The early fixmap range spans multiple pmds, for which + * we are not prepared: + */ + BUILD_BUG_ON((__fix_to_virt(__end_of_permanent_fixed_addresses) >> PMD_SHIFT) + != FIXADDR_TOP >> PMD_SHIFT); + + pmd = fixmap_pmd(FIXADDR_TOP); + pmd_populate_kernel(&init_mm, pmd, bm_pte); + + pte_offset_fixmap = pte_offset_early_fixmap; +} + /* * To avoid TLB flush broadcasts, this uses local_flush_tlb_kernel_range(). * As a result, this can only be called with preemption disabled, as under @@ -365,7 +406,7 @@ EXPORT_SYMBOL(get_mem_type); void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot) { unsigned long vaddr = __fix_to_virt(idx); - pte_t *pte = pte_offset_kernel(pmd_off_k(vaddr), vaddr); + pte_t *pte = pte_offset_fixmap(pmd_off_k(vaddr), vaddr); /* Make sure fixmap region does not exceed available allocation. */ BUILD_BUG_ON(FIXADDR_START + (__end_of_fixed_addresses * PAGE_SIZE) > @@ -855,7 +896,7 @@ static void __init create_mapping(struct map_desc *md) } if ((md->type == MT_DEVICE || md->type == MT_ROM) && - md->virtual >= PAGE_OFFSET && + md->virtual >= PAGE_OFFSET && md->virtual < FIXADDR_START && (md->virtual < VMALLOC_START || md->virtual >= VMALLOC_END)) { pr_warn("BUG: mapping for 0x%08llx at 0x%08lx out of vmalloc space\n", (long long)__pfn_to_phys((u64)md->pfn), md->virtual); @@ -1219,10 +1260,10 @@ void __init arm_mm_memblock_reserve(void) /* * Set up the device mappings. Since we clear out the page tables for all - * mappings above VMALLOC_START, we will remove any debug device mappings. - * This means you have to be careful how you debug this function, or any - * called function. This means you can't use any function or debugging - * method which may touch any device, otherwise the kernel _will_ crash. + * mappings above VMALLOC_START, except early fixmap, we might remove debug + * device mappings. This means earlycon can be used to debug this function + * Any other function or debugging method which may touch any device _will_ + * crash the kernel. */ static void __init devicemaps_init(const struct machine_desc *mdesc) { @@ -1237,7 +1278,10 @@ static void __init devicemaps_init(const struct machine_desc *mdesc) early_trap_init(vectors); - for (addr = VMALLOC_START; addr; addr += PMD_SIZE) + /* + * Clear page table except top pmd used by early fixmaps + */ + for (addr = VMALLOC_START; addr < (FIXADDR_TOP & PMD_MASK); addr += PMD_SIZE) pmd_clear(pmd_off_k(addr)); /* @@ -1489,6 +1533,35 @@ void __init early_paging_init(const struct machine_desc *mdesc) #endif +static void __init early_fixmap_shutdown(void) +{ + int i; + unsigned long va = fix_to_virt(__end_of_permanent_fixed_addresses - 1); + + pte_offset_fixmap = pte_offset_late_fixmap; + pmd_clear(fixmap_pmd(va)); + local_flush_tlb_kernel_page(va); + + for (i = 0; i < __end_of_permanent_fixed_addresses; i++) { + pte_t *pte; + struct map_desc map; + + map.virtual = fix_to_virt(i); + pte = pte_offset_early_fixmap(pmd_off_k(map.virtual), map.virtual); + + /* Only i/o device mappings are supported ATM */ + if (pte_none(*pte) || + (pte_val(*pte) & L_PTE_MT_MASK) != L_PTE_MT_DEV_SHARED) + continue; + + map.pfn = pte_pfn(*pte); + map.type = MT_DEVICE; + map.length = PAGE_SIZE; + + create_mapping(&map); + } +} + /* * paging_init() sets up the page tables, initialises the zone memory * maps, and sets up the zero page, bad page and bad page tables. @@ -1502,6 +1575,7 @@ void __init paging_init(const struct machine_desc *mdesc) map_lowmem(); memblock_set_current_limit(arm_lowmem_limit); dma_contiguous_remap(); + early_fixmap_shutdown(); devicemaps_init(mdesc); kmap_init(); tcm_init(); diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c index a3681f11dd9f..e683db1b90a3 100644 --- a/arch/arm/mm/pgd.c +++ b/arch/arm/mm/pgd.c @@ -84,6 +84,16 @@ pgd_t *pgd_alloc(struct mm_struct *mm) if (!new_pte) goto no_pte; +#ifndef CONFIG_ARM_LPAE + /* + * Modify the PTE pointer to have the correct domain. This + * needs to be the vectors domain to avoid the low vectors + * being unmapped. + */ + pmd_val(*new_pmd) &= ~PMD_DOMAIN_MASK; + pmd_val(*new_pmd) |= PMD_DOMAIN(DOMAIN_VECTORS); +#endif + init_pud = pud_offset(init_pgd, 0); init_pmd = pmd_offset(init_pud, 0); init_pte = pte_offset_map(init_pmd, 0); diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index c011e2296cb1..876060bcceeb 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -857,7 +857,9 @@ static int build_body(struct jit_ctx *ctx) emit(ARM_LDR_I(r_A, r_scratch, off), ctx); break; case BPF_ANC | SKF_AD_IFINDEX: + case BPF_ANC | SKF_AD_HATYPE: /* A = skb->dev->ifindex */ + /* A = skb->dev->type */ ctx->seen |= SEEN_SKB; off = offsetof(struct sk_buff, dev); emit(ARM_LDR_I(r_scratch, r_skb, off), ctx); @@ -867,8 +869,24 @@ static int build_body(struct jit_ctx *ctx) BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4); - off = offsetof(struct net_device, ifindex); - emit(ARM_LDR_I(r_A, r_scratch, off), ctx); + BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, + type) != 2); + + if (code == (BPF_ANC | SKF_AD_IFINDEX)) { + off = offsetof(struct net_device, ifindex); + emit(ARM_LDR_I(r_A, r_scratch, off), ctx); + } else { + /* + * offset of field "type" in "struct + * net_device" is above what can be + * used in the ldrh rd, [rn, #imm] + * instruction, so load the offset in + * a register and use ldrh rd, [rn, rm] + */ + off = offsetof(struct net_device, type); + emit_mov_i(ARM_R3, off, ctx); + emit(ARM_LDRH_R(r_A, r_scratch, ARM_R3), ctx); + } break; case BPF_ANC | SKF_AD_MARK: ctx->seen |= SEEN_SKB; @@ -895,6 +913,17 @@ static int build_body(struct jit_ctx *ctx) OP_IMM3(ARM_AND, r_A, r_A, 0x1, ctx); } break; + case BPF_ANC | SKF_AD_PKTTYPE: + ctx->seen |= SEEN_SKB; + BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, + __pkt_type_offset[0]) != 1); + off = PKT_TYPE_OFFSET(); + emit(ARM_LDRB_I(r_A, r_skb, off), ctx); + emit(ARM_AND_I(r_A, r_A, PKT_TYPE_MAX), ctx); +#ifdef __BIG_ENDIAN_BITFIELD + emit(ARM_LSR_I(r_A, r_A, 5), ctx); +#endif + break; case BPF_ANC | SKF_AD_QUEUE: ctx->seen |= SEEN_SKB; BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, @@ -904,6 +933,14 @@ static int build_body(struct jit_ctx *ctx) off = offsetof(struct sk_buff, queue_mapping); emit(ARM_LDRH_I(r_A, r_skb, off), ctx); break; + case BPF_ANC | SKF_AD_PAY_OFFSET: + ctx->seen |= SEEN_SKB | SEEN_CALL; + + emit(ARM_MOV_R(ARM_R0, r_skb), ctx); + emit_mov_i(ARM_R3, (unsigned int)skb_get_poff, ctx); + emit_blx_r(ARM_R3, ctx); + emit(ARM_MOV_R(r_A, ARM_R0), ctx); + break; case BPF_LDX | BPF_W | BPF_ABS: /* * load a 32bit word from struct seccomp_data. diff --git a/arch/arm/net/bpf_jit_32.h b/arch/arm/net/bpf_jit_32.h index b2d7d92859d3..4b17d5ab652a 100644 --- a/arch/arm/net/bpf_jit_32.h +++ b/arch/arm/net/bpf_jit_32.h @@ -74,6 +74,7 @@ #define ARM_INST_LDRB_I 0x05d00000 #define ARM_INST_LDRB_R 0x07d00000 #define ARM_INST_LDRH_I 0x01d000b0 +#define ARM_INST_LDRH_R 0x019000b0 #define ARM_INST_LDR_I 0x05900000 #define ARM_INST_LDM 0x08900000 @@ -160,6 +161,8 @@ | (rm)) #define ARM_LDRH_I(rt, rn, off) (ARM_INST_LDRH_I | (rt) << 12 | (rn) << 16 \ | (((off) & 0xf0) << 4) | ((off) & 0xf)) +#define ARM_LDRH_R(rt, rn, rm) (ARM_INST_LDRH_R | (rt) << 12 | (rn) << 16 \ + | (rm)) #define ARM_LDM(rn, regs) (ARM_INST_LDM | (rn) << 16 | (regs)) diff --git a/arch/arm/xen/enlighten.c b/arch/arm/xen/enlighten.c index 6c09cc440a2b..eeeab074e154 100644 --- a/arch/arm/xen/enlighten.c +++ b/arch/arm/xen/enlighten.c @@ -45,46 +45,39 @@ static struct vcpu_info __percpu *xen_vcpu_info; unsigned long xen_released_pages; struct xen_memory_region xen_extra_mem[XEN_EXTRA_MEM_MAX_REGIONS] __initdata; -/* TODO: to be removed */ -__read_mostly int xen_have_vector_callback; -EXPORT_SYMBOL_GPL(xen_have_vector_callback); - -int xen_platform_pci_unplug = XEN_UNPLUG_ALL; -EXPORT_SYMBOL_GPL(xen_platform_pci_unplug); - static __read_mostly unsigned int xen_events_irq; static __initdata struct device_node *xen_node; -int xen_remap_domain_mfn_array(struct vm_area_struct *vma, +int xen_remap_domain_gfn_array(struct vm_area_struct *vma, unsigned long addr, - xen_pfn_t *mfn, int nr, + xen_pfn_t *gfn, int nr, int *err_ptr, pgprot_t prot, unsigned domid, struct page **pages) { - return xen_xlate_remap_gfn_array(vma, addr, mfn, nr, err_ptr, + return xen_xlate_remap_gfn_array(vma, addr, gfn, nr, err_ptr, prot, domid, pages); } -EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_array); +EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_array); /* Not used by XENFEAT_auto_translated guests. */ -int xen_remap_domain_mfn_range(struct vm_area_struct *vma, +int xen_remap_domain_gfn_range(struct vm_area_struct *vma, unsigned long addr, - xen_pfn_t mfn, int nr, + xen_pfn_t gfn, int nr, pgprot_t prot, unsigned domid, struct page **pages) { return -ENOSYS; } -EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_range); +EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_range); -int xen_unmap_domain_mfn_range(struct vm_area_struct *vma, +int xen_unmap_domain_gfn_range(struct vm_area_struct *vma, int nr, struct page **pages) { return xen_xlate_unmap_gfn_range(vma, nr, pages); } -EXPORT_SYMBOL_GPL(xen_unmap_domain_mfn_range); +EXPORT_SYMBOL_GPL(xen_unmap_domain_gfn_range); static void xen_percpu_init(void) { diff --git a/arch/arm/xen/mm.c b/arch/arm/xen/mm.c index 03e75fef15b8..6dd911d1f0ac 100644 --- a/arch/arm/xen/mm.c +++ b/arch/arm/xen/mm.c @@ -139,9 +139,9 @@ void __xen_dma_sync_single_for_device(struct device *hwdev, bool xen_arch_need_swiotlb(struct device *dev, unsigned long pfn, - unsigned long mfn) + unsigned long bfn) { - return (!hypercall_cflush && (pfn != mfn) && !is_device_dma_coherent(dev)); + return (!hypercall_cflush && (pfn != bfn) && !is_device_dma_coherent(dev)); } int xen_create_contiguous_region(phys_addr_t pstart, unsigned int order, diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 318175f62c24..7d95663c0160 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -20,6 +20,7 @@ config ARM64 select ARM_GIC_V2M if PCI_MSI select ARM_GIC_V3 select ARM_GIC_V3_ITS if PCI_MSI + select ARM_PSCI_FW select BUILDTIME_EXTABLE_SORT select CLONE_BACKWARDS select COMMON_CLK @@ -28,7 +29,7 @@ config ARM64 select EDAC_SUPPORT select GENERIC_ALLOCATOR select GENERIC_CLOCKEVENTS - select GENERIC_CLOCKEVENTS_BROADCAST if SMP + select GENERIC_CLOCKEVENTS_BROADCAST select GENERIC_CPU_AUTOPROBE select GENERIC_EARLY_IOREMAP select GENERIC_IRQ_PROBE @@ -53,6 +54,7 @@ config ARM64 select HAVE_C_RECORDMCOUNT select HAVE_CC_STACKPROTECTOR select HAVE_CMPXCHG_DOUBLE + select HAVE_CMPXCHG_LOCAL select HAVE_DEBUG_BUGVERBOSE select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_API_DEBUG @@ -104,6 +106,10 @@ config NO_IOPORT_MAP config STACKTRACE_SUPPORT def_bool y +config ILLEGAL_POINTER_VALUE + hex + default 0xdead000000000000 + config LOCKDEP_SUPPORT def_bool y @@ -113,6 +119,14 @@ config TRACE_IRQFLAGS_SUPPORT config RWSEM_XCHGADD_ALGORITHM def_bool y +config GENERIC_BUG + def_bool y + depends on BUG + +config GENERIC_BUG_RELATIVE_POINTERS + def_bool y + depends on GENERIC_BUG + config GENERIC_HWEIGHT def_bool y @@ -137,6 +151,9 @@ config NEED_DMA_MAP_STATE config NEED_SG_DMA_LENGTH def_bool y +config SMP + def_bool y + config SWIOTLB def_bool y @@ -160,110 +177,7 @@ source "init/Kconfig" source "kernel/Kconfig.freezer" -menu "Platform selection" - -config ARCH_EXYNOS - bool - help - This enables support for Samsung Exynos SoC family - -config ARCH_EXYNOS7 - bool "ARMv8 based Samsung Exynos7" - select ARCH_EXYNOS - select COMMON_CLK_SAMSUNG - select HAVE_S3C2410_WATCHDOG if WATCHDOG - select HAVE_S3C_RTC if RTC_CLASS - select PINCTRL - select PINCTRL_EXYNOS - - help - This enables support for Samsung Exynos7 SoC family - -config ARCH_FSL_LS2085A - bool "Freescale LS2085A SOC" - help - This enables support for Freescale LS2085A SOC. - -config ARCH_HISI - bool "Hisilicon SoC Family" - help - This enables support for Hisilicon ARMv8 SoC family - -config ARCH_MEDIATEK - bool "Mediatek MT65xx & MT81xx ARMv8 SoC" - select ARM_GIC - select PINCTRL - help - Support for Mediatek MT65xx & MT81xx ARMv8 SoCs - -config ARCH_QCOM - bool "Qualcomm Platforms" - select PINCTRL - help - This enables support for the ARMv8 based Qualcomm chipsets. - -config ARCH_SEATTLE - bool "AMD Seattle SoC Family" - help - This enables support for AMD Seattle SOC Family - -config ARCH_TEGRA - bool "NVIDIA Tegra SoC Family" - select ARCH_HAS_RESET_CONTROLLER - select ARCH_REQUIRE_GPIOLIB - select CLKDEV_LOOKUP - select CLKSRC_MMIO - select CLKSRC_OF - select GENERIC_CLOCKEVENTS - select HAVE_CLK - select PINCTRL - select RESET_CONTROLLER - help - This enables support for the NVIDIA Tegra SoC family. - -config ARCH_TEGRA_132_SOC - bool "NVIDIA Tegra132 SoC" - depends on ARCH_TEGRA - select PINCTRL_TEGRA124 - select USB_ULPI if USB_PHY - select USB_ULPI_VIEWPORT if USB_PHY - help - Enable support for NVIDIA Tegra132 SoC, based on the Denver - ARMv8 CPU. The Tegra132 SoC is similar to the Tegra124 SoC, - but contains an NVIDIA Denver CPU complex in place of - Tegra124's "4+1" Cortex-A15 CPU complex. - -config ARCH_SPRD - bool "Spreadtrum SoC platform" - help - Support for Spreadtrum ARM based SoCs - -config ARCH_THUNDER - bool "Cavium Inc. Thunder SoC Family" - help - This enables support for Cavium's Thunder Family of SoCs. - -config ARCH_VEXPRESS - bool "ARMv8 software model (Versatile Express)" - select ARCH_REQUIRE_GPIOLIB - select COMMON_CLK_VERSATILE - select POWER_RESET_VEXPRESS - select VEXPRESS_CONFIG - help - This enables support for the ARMv8 software model (Versatile - Express). - -config ARCH_XGENE - bool "AppliedMicro X-Gene SOC Family" - help - This enables support for AppliedMicro X-Gene SOC Family - -config ARCH_ZYNQMP - bool "Xilinx ZynqMP Family" - help - This enables support for Xilinx ZynqMP Family - -endmenu +source "arch/arm64/Kconfig.platforms" menu "Bus support" @@ -474,22 +388,8 @@ config CPU_BIG_ENDIAN help Say Y if you plan on running a kernel in big-endian mode. -config SMP - bool "Symmetric Multi-Processing" - help - This enables support for systems with more than one CPU. If - you say N here, the kernel will run on single and - multiprocessor machines, but will use only one CPU of a - multiprocessor machine. If you say Y here, the kernel will run - on many, but not all, single processor machines. On a single - processor machine, the kernel will run faster if you say N - here. - - If you don't know what to do here, say N. - config SCHED_MC bool "Multi-core scheduler support" - depends on SMP help Multi-core scheduler support improves the CPU scheduler's decision making when dealing with multi-core CPU chips at a cost of slightly @@ -497,7 +397,6 @@ config SCHED_MC config SCHED_SMT bool "SMT scheduler support" - depends on SMP help Improves the CPU scheduler's decision making when dealing with MultiThreading at a cost of slightly increased overhead in some @@ -506,23 +405,17 @@ config SCHED_SMT config NR_CPUS int "Maximum number of CPUs (2-4096)" range 2 4096 - depends on SMP # These have to remain sorted largest to smallest default "64" config HOTPLUG_CPU bool "Support for hot-pluggable CPUs" - depends on SMP help Say Y here to experiment with turning CPUs off and on. CPUs can be controlled through /sys/devices/system/cpu. source kernel/Kconfig.preempt -config UP_LATE_INIT - def_bool y - depends on !SMP - config HZ int default 100 @@ -664,6 +557,53 @@ config SETEND_EMULATION If unsure, say Y endif +menu "ARMv8.1 architectural features" + +config ARM64_HW_AFDBM + bool "Support for hardware updates of the Access and Dirty page flags" + default y + help + The ARMv8.1 architecture extensions introduce support for + hardware updates of the access and dirty information in page + table entries. When enabled in TCR_EL1 (HA and HD bits) on + capable processors, accesses to pages with PTE_AF cleared will + set this bit instead of raising an access flag fault. + Similarly, writes to read-only pages with the DBM bit set will + clear the read-only bit (AP[2]) instead of raising a + permission fault. + + Kernels built with this configuration option enabled continue + to work on pre-ARMv8.1 hardware and the performance impact is + minimal. If unsure, say Y. + +config ARM64_PAN + bool "Enable support for Privileged Access Never (PAN)" + default y + help + Privileged Access Never (PAN; part of the ARMv8.1 Extensions) + prevents the kernel or hypervisor from accessing user-space (EL0) + memory directly. + + Choosing this option will cause any unprotected (not using + copy_to_user et al) memory access to fail with a permission fault. + + The feature is detected at runtime, and will remain as a 'nop' + instruction if the cpu does not implement the feature. + +config ARM64_LSE_ATOMICS + bool "Atomic instructions" + help + As part of the Large System Extensions, ARMv8.1 introduces new + atomic instructions that are designed specifically to scale in + very large systems. + + Say Y here to make use of these instructions for the in-kernel + atomic routines. This incurs a small overhead on CPUs that do + not support these instructions and requires the kernel to be + built with binutils >= 2.25. + +endmenu + endmenu menu "Boot options" diff --git a/arch/arm64/Kconfig.platforms b/arch/arm64/Kconfig.platforms new file mode 100644 index 000000000000..23800a19a7bc --- /dev/null +++ b/arch/arm64/Kconfig.platforms @@ -0,0 +1,125 @@ +menu "Platform selection" + +config ARCH_BCM_IPROC + bool "Broadcom iProc SoC Family" + help + This enables support for Broadcom iProc based SoCs + +config ARCH_BERLIN + bool "Marvell Berlin SoC Family" + select DW_APB_ICTL + help + This enables support for Marvell Berlin SoC Family + +config ARCH_EXYNOS + bool + help + This enables support for Samsung Exynos SoC family + +config ARCH_EXYNOS7 + bool "ARMv8 based Samsung Exynos7" + select ARCH_EXYNOS + select COMMON_CLK_SAMSUNG + select HAVE_S3C2410_WATCHDOG if WATCHDOG + select HAVE_S3C_RTC if RTC_CLASS + select PINCTRL + select PINCTRL_EXYNOS + + help + This enables support for Samsung Exynos7 SoC family + +config ARCH_FSL_LS2085A + bool "Freescale LS2085A SOC" + help + This enables support for Freescale LS2085A SOC. + +config ARCH_HISI + bool "Hisilicon SoC Family" + help + This enables support for Hisilicon ARMv8 SoC family + +config ARCH_MEDIATEK + bool "Mediatek MT65xx & MT81xx ARMv8 SoC" + select ARM_GIC + select PINCTRL + help + Support for Mediatek MT65xx & MT81xx ARMv8 SoCs + +config ARCH_QCOM + bool "Qualcomm Platforms" + select PINCTRL + help + This enables support for the ARMv8 based Qualcomm chipsets. + +config ARCH_ROCKCHIP + bool "Rockchip Platforms" + select ARCH_HAS_RESET_CONTROLLER + select ARCH_REQUIRE_GPIOLIB + select PINCTRL + select PINCTRL_ROCKCHIP + help + This enables support for the ARMv8 based Rockchip chipsets, + like the RK3368. + +config ARCH_SEATTLE + bool "AMD Seattle SoC Family" + help + This enables support for AMD Seattle SOC Family + +config ARCH_TEGRA + bool "NVIDIA Tegra SoC Family" + select ARCH_HAS_RESET_CONTROLLER + select ARCH_REQUIRE_GPIOLIB + select CLKDEV_LOOKUP + select CLKSRC_MMIO + select CLKSRC_OF + select GENERIC_CLOCKEVENTS + select HAVE_CLK + select PINCTRL + select RESET_CONTROLLER + help + This enables support for the NVIDIA Tegra SoC family. + +config ARCH_TEGRA_132_SOC + bool "NVIDIA Tegra132 SoC" + depends on ARCH_TEGRA + select PINCTRL_TEGRA124 + select USB_ULPI if USB_PHY + select USB_ULPI_VIEWPORT if USB_PHY + help + Enable support for NVIDIA Tegra132 SoC, based on the Denver + ARMv8 CPU. The Tegra132 SoC is similar to the Tegra124 SoC, + but contains an NVIDIA Denver CPU complex in place of + Tegra124's "4+1" Cortex-A15 CPU complex. + +config ARCH_SPRD + bool "Spreadtrum SoC platform" + help + Support for Spreadtrum ARM based SoCs + +config ARCH_THUNDER + bool "Cavium Inc. Thunder SoC Family" + help + This enables support for Cavium's Thunder Family of SoCs. + +config ARCH_VEXPRESS + bool "ARMv8 software model (Versatile Express)" + select ARCH_REQUIRE_GPIOLIB + select COMMON_CLK_VERSATILE + select POWER_RESET_VEXPRESS + select VEXPRESS_CONFIG + help + This enables support for the ARMv8 software model (Versatile + Express). + +config ARCH_XGENE + bool "AppliedMicro X-Gene SOC Family" + help + This enables support for AppliedMicro X-Gene SOC Family + +config ARCH_ZYNQMP + bool "Xilinx ZynqMP Family" + help + This enables support for Xilinx ZynqMP Family + +endmenu diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 4d2a925998f9..15ff5b4156fd 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -17,7 +17,18 @@ GZFLAGS :=-9 KBUILD_DEFCONFIG := defconfig -KBUILD_CFLAGS += -mgeneral-regs-only +# Check for binutils support for specific extensions +lseinstr := $(call as-instr,.arch_extension lse,-DCONFIG_AS_LSE=1) + +ifeq ($(CONFIG_ARM64_LSE_ATOMICS), y) + ifeq ($(lseinstr),) +$(warning LSE atomics not supported by binutils) + endif +endif + +KBUILD_CFLAGS += -mgeneral-regs-only $(lseinstr) +KBUILD_AFLAGS += $(lseinstr) + ifeq ($(CONFIG_CPU_BIG_ENDIAN), y) KBUILD_CPPFLAGS += -mbig-endian AS += -EB @@ -58,7 +69,10 @@ all: $(KBUILD_IMAGE) $(KBUILD_DTBS) boot := arch/arm64/boot -Image Image.gz: vmlinux +Image: vmlinux + $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ + +Image.%: vmlinux $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ zinstall install: vmlinux diff --git a/arch/arm64/boot/Makefile b/arch/arm64/boot/Makefile index 5a0e3ab854a5..abcbba2f01ba 100644 --- a/arch/arm64/boot/Makefile +++ b/arch/arm64/boot/Makefile @@ -19,9 +19,21 @@ targets := Image Image.gz $(obj)/Image: vmlinux FORCE $(call if_changed,objcopy) +$(obj)/Image.bz2: $(obj)/Image FORCE + $(call if_changed,bzip2) + $(obj)/Image.gz: $(obj)/Image FORCE $(call if_changed,gzip) +$(obj)/Image.lz4: $(obj)/Image FORCE + $(call if_changed,lz4) + +$(obj)/Image.lzma: $(obj)/Image FORCE + $(call if_changed,lzma) + +$(obj)/Image.lzo: $(obj)/Image FORCE + $(call if_changed,lzo) + install: $(obj)/Image $(CONFIG_SHELL) $(srctree)/$(src)/install.sh $(KERNELRELEASE) \ $(obj)/Image System.map "$(INSTALL_PATH)" diff --git a/arch/arm64/boot/dts/Makefile b/arch/arm64/boot/dts/Makefile index 38913be23695..d9f88330e7b0 100644 --- a/arch/arm64/boot/dts/Makefile +++ b/arch/arm64/boot/dts/Makefile @@ -1,12 +1,15 @@ dts-dirs += amd dts-dirs += apm dts-dirs += arm +dts-dirs += broadcom dts-dirs += cavium dts-dirs += exynos dts-dirs += freescale dts-dirs += hisilicon +dts-dirs += marvell dts-dirs += mediatek dts-dirs += qcom +dts-dirs += rockchip dts-dirs += sprd dts-dirs += xilinx diff --git a/arch/arm64/boot/dts/arm/juno-motherboard.dtsi b/arch/arm64/boot/dts/arm/juno-motherboard.dtsi index 021e0f40f419..637e046f0e36 100644 --- a/arch/arm64/boot/dts/arm/juno-motherboard.dtsi +++ b/arch/arm64/boot/dts/arm/juno-motherboard.dtsi @@ -136,6 +136,8 @@ v2m_sysctl: sysctl@020000 { clock-names = "refclk", "timclk", "apb_pclk"; #clock-cells = <1>; clock-output-names = "timerclken0", "timerclken1", "timerclken2", "timerclken3"; + assigned-clocks = <&v2m_sysctl 0>, <&v2m_sysctl 1>, <&v2m_sysctl 3>, <&v2m_sysctl 3>; + assigned-clock-parents = <&v2m_refclk1mhz>, <&v2m_refclk1mhz>, <&v2m_refclk1mhz>, <&v2m_refclk1mhz>; }; apbregs@010000 { diff --git a/arch/arm64/boot/dts/arm/rtsm_ve-motherboard.dtsi b/arch/arm64/boot/dts/arm/rtsm_ve-motherboard.dtsi index c46cbb29f3c6..88a7583ed7a7 100644 --- a/arch/arm64/boot/dts/arm/rtsm_ve-motherboard.dtsi +++ b/arch/arm64/boot/dts/arm/rtsm_ve-motherboard.dtsi @@ -74,6 +74,8 @@ v2m_sysctl: sysctl@020000 { clock-names = "refclk", "timclk", "apb_pclk"; #clock-cells = <1>; clock-output-names = "timerclken0", "timerclken1", "timerclken2", "timerclken3"; + assigned-clocks = <&v2m_sysctl 0>, <&v2m_sysctl 1>, <&v2m_sysctl 3>, <&v2m_sysctl 3>; + assigned-clock-parents = <&v2m_refclk1mhz>, <&v2m_refclk1mhz>, <&v2m_refclk1mhz>, <&v2m_refclk1mhz>; }; aaci@040000 { diff --git a/arch/arm64/boot/dts/broadcom/Makefile b/arch/arm64/boot/dts/broadcom/Makefile new file mode 100644 index 000000000000..e21fe66f1837 --- /dev/null +++ b/arch/arm64/boot/dts/broadcom/Makefile @@ -0,0 +1,5 @@ +dtb-$(CONFIG_ARCH_BCM_IPROC) += ns2-svk.dtb + +always := $(dtb-y) +subdir-y := $(dts-dirs) +clean-files := *.dtb diff --git a/arch/arm64/boot/dts/broadcom/ns2-svk.dts b/arch/arm64/boot/dts/broadcom/ns2-svk.dts new file mode 100644 index 000000000000..244baf879dc9 --- /dev/null +++ b/arch/arm64/boot/dts/broadcom/ns2-svk.dts @@ -0,0 +1,59 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2015 Broadcom Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Broadcom Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/dts-v1/; + +#include "ns2.dtsi" + +/ { + model = "Broadcom NS2 SVK"; + compatible = "brcm,ns2-svk", "brcm,ns2"; + + aliases { + serial0 = &uart3; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + memory { + device_type = "memory"; + reg = <0x000000000 0x80000000 0x00000000 0x40000000>; + }; + + soc: soc { + uart3: serial@66130000 { + status = "ok"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/broadcom/ns2.dtsi b/arch/arm64/boot/dts/broadcom/ns2.dtsi new file mode 100644 index 000000000000..3c92d92278e5 --- /dev/null +++ b/arch/arm64/boot/dts/broadcom/ns2.dtsi @@ -0,0 +1,118 @@ +/* + * BSD LICENSE + * + * Copyright(c) 2015 Broadcom Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Broadcom Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include + +/memreserve/ 0x84b00000 0x00000008; + +/ { + compatible = "brcm,ns2"; + interrupt-parent = <&gic>; + #address-cells = <2>; + #size-cells = <2>; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a57", "arm,armv8"; + reg = <0 0>; + enable-method = "spin-table"; + cpu-release-addr = <0 0x84b00000>; + }; + + cpu@1 { + device_type = "cpu"; + compatible = "arm,cortex-a57", "arm,armv8"; + reg = <0 1>; + enable-method = "spin-table"; + cpu-release-addr = <0 0x84b00000>; + }; + + cpu@2 { + device_type = "cpu"; + compatible = "arm,cortex-a57", "arm,armv8"; + reg = <0 2>; + enable-method = "spin-table"; + cpu-release-addr = <0 0x84b00000>; + }; + + cpu@3 { + device_type = "cpu"; + compatible = "arm,cortex-a57", "arm,armv8"; + reg = <0 3>; + enable-method = "spin-table"; + cpu-release-addr = <0 0x84b00000>; + }; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupts = , + , + , + ; + }; + + soc: soc { + compatible = "simple-bus"; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 0 0 0xffffffff>; + + gic: interrupt-controller@65210000 { + compatible = "arm,gic-400"; + #interrupt-cells = <3>; + interrupt-controller; + reg = <0x65210000 0x1000>, + <0x65220000 0x1000>, + <0x65240000 0x2000>, + <0x65260000 0x1000>; + }; + + uart3: serial@66130000 { + compatible = "snps,dw-apb-uart"; + reg = <0x66130000 0x100>; + interrupts = ; + reg-shift = <2>; + reg-io-width = <4>; + clock-frequency = <23961600>; + status = "disabled"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/marvell/Makefile b/arch/arm64/boot/dts/marvell/Makefile new file mode 100644 index 000000000000..e2f6afa7f849 --- /dev/null +++ b/arch/arm64/boot/dts/marvell/Makefile @@ -0,0 +1,5 @@ +dtb-$(CONFIG_ARCH_BERLIN) += berlin4ct-dmp.dtb + +always := $(dtb-y) +subdir-y := $(dts-dirs) +clean-files := *.dtb diff --git a/arch/arm64/boot/dts/marvell/berlin4ct-dmp.dts b/arch/arm64/boot/dts/marvell/berlin4ct-dmp.dts new file mode 100644 index 000000000000..0d70d39fa8d2 --- /dev/null +++ b/arch/arm64/boot/dts/marvell/berlin4ct-dmp.dts @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2015 Marvell Technology Group Ltd. + * + * Author: Jisheng Zhang + * + * This file is dual-licensed: you can use it either under the terms + * of the GPLv2 or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; + +#include "berlin4ct.dtsi" + +/ { + model = "Marvell BG4CT DMP board"; + compatible = "marvell,berlin4ct-dmp", "marvell,berlin4ct", "marvell,berlin"; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + memory { + device_type = "memory"; + /* the first 16MB is for firmwares' usage */ + reg = <0 0x01000000 0 0x7f000000>; + }; +}; + +&uart0 { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/marvell/berlin4ct.dtsi b/arch/arm64/boot/dts/marvell/berlin4ct.dtsi new file mode 100644 index 000000000000..dd4a10d605d9 --- /dev/null +++ b/arch/arm64/boot/dts/marvell/berlin4ct.dtsi @@ -0,0 +1,164 @@ +/* + * Copyright (C) 2015 Marvell Technology Group Ltd. + * + * Author: Jisheng Zhang + * + * This file is dual-licensed: you can use it either under the terms + * of the GPLv2 or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include + +/ { + compatible = "marvell,berlin4ct", "marvell,berlin"; + interrupt-parent = <&gic>; + #address-cells = <2>; + #size-cells = <2>; + + aliases { + serial0 = &uart0; + }; + + psci { + compatible = "arm,psci-0.2"; + method = "smc"; + }; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu0: cpu@0 { + compatible = "arm,cortex-a53", "arm,armv8"; + device_type = "cpu"; + reg = <0x0>; + enable-method = "psci"; + }; + + cpu1: cpu@1 { + compatible = "arm,cortex-a53", "arm,armv8"; + device_type = "cpu"; + reg = <0x1>; + enable-method = "psci"; + }; + + cpu2: cpu@2 { + compatible = "arm,cortex-a53", "arm,armv8"; + device_type = "cpu"; + reg = <0x2>; + enable-method = "psci"; + }; + + cpu3: cpu@3 { + compatible = "arm,cortex-a53", "arm,armv8"; + device_type = "cpu"; + reg = <0x3>; + enable-method = "psci"; + }; + }; + + osc: osc { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <25000000>; + }; + + pmu { + compatible = "arm,armv8-pmuv3"; + interrupts = , + , + , + ; + interrupt-affinity = <&cpu0>, + <&cpu1>, + <&cpu2>, + <&cpu3>; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupts = , + , + , + ; + }; + + soc { + compatible = "simple-bus"; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 0 0xf7000000 0x1000000>; + + gic: interrupt-controller@901000 { + compatible = "arm,gic-400"; + #interrupt-cells = <3>; + interrupt-controller; + reg = <0x901000 0x1000>, + <0x902000 0x2000>, + <0x904000 0x2000>, + <0x906000 0x2000>; + interrupts = ; + }; + + apb@fc0000 { + compatible = "simple-bus"; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 0xfc0000 0x10000>; + interrupt-parent = <&sic>; + + sic: interrupt-controller@1000 { + compatible = "snps,dw-apb-ictl"; + reg = <0x1000 0x30>; + interrupt-controller; + #interrupt-cells = <1>; + interrupt-parent = <&gic>; + interrupts = ; + }; + + uart0: uart@d000 { + compatible = "snps,dw-apb-uart"; + reg = <0xd000 0x100>; + interrupts = <8>; + clocks = <&osc>; + reg-shift = <2>; + status = "disabled"; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/mediatek/Makefile b/arch/arm64/boot/dts/mediatek/Makefile index 3ce24622b231..e0a4bff2fc17 100644 --- a/arch/arm64/boot/dts/mediatek/Makefile +++ b/arch/arm64/boot/dts/mediatek/Makefile @@ -1,3 +1,4 @@ +dtb-$(CONFIG_ARCH_MEDIATEK) += mt6795-evb.dtb dtb-$(CONFIG_ARCH_MEDIATEK) += mt8173-evb.dtb always := $(dtb-y) diff --git a/arch/arm64/boot/dts/mediatek/mt6795-evb.dts b/arch/arm64/boot/dts/mediatek/mt6795-evb.dts new file mode 100644 index 000000000000..ad665f5835f0 --- /dev/null +++ b/arch/arm64/boot/dts/mediatek/mt6795-evb.dts @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2015 MediaTek Inc. + * Author: Mars.C + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +/dts-v1/; +#include "mt6795.dtsi" + +/ { + model = "MediaTek MT6795 Evaluation Board"; + compatible = "mediatek,mt6795-evb", "mediatek,mt6795"; + + aliases { + serial0 = &uart0; + serial1 = &uart1; + serial2 = &uart2; + serial3 = &uart3; + }; + + memory@40000000 { + device_type = "memory"; + reg = <0 0x40000000 0 0x1e800000>; + }; + + chosen { + stdout-path = "serial0:921600n8"; + }; +}; + +&uart0 { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/mediatek/mt6795.dtsi b/arch/arm64/boot/dts/mediatek/mt6795.dtsi new file mode 100644 index 000000000000..c85659d0ff5d --- /dev/null +++ b/arch/arm64/boot/dts/mediatek/mt6795.dtsi @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2015 MediaTek Inc. + * Author: Mars.C + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include + +/ { + compatible = "mediatek,mt6795"; + interrupt-parent = <&sysirq>; + #address-cells = <2>; + #size-cells = <2>; + + psci { + compatible = "arm,psci-0.2"; + method = "smc"; + }; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu0: cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a53"; + enable-method = "psci"; + reg = <0x000>; + }; + + cpu1: cpu@1 { + device_type = "cpu"; + compatible = "arm,cortex-a53"; + enable-method = "psci"; + reg = <0x001>; + }; + + cpu2: cpu@2 { + device_type = "cpu"; + compatible = "arm,cortex-a53"; + enable-method = "psci"; + reg = <0x002>; + }; + + cpu3: cpu@3 { + device_type = "cpu"; + compatible = "arm,cortex-a53"; + enable-method = "psci"; + reg = <0x003>; + }; + + cpu4: cpu@100 { + device_type = "cpu"; + compatible = "arm,cortex-a53"; + enable-method = "psci"; + reg = <0x100>; + }; + + cpu5: cpu@101 { + device_type = "cpu"; + compatible = "arm,cortex-a53"; + enable-method = "psci"; + reg = <0x101>; + }; + + cpu6: cpu@102 { + device_type = "cpu"; + compatible = "arm,cortex-a53"; + enable-method = "psci"; + reg = <0x102>; + }; + + cpu7: cpu@103 { + device_type = "cpu"; + compatible = "arm,cortex-a53"; + enable-method = "psci"; + reg = <0x103>; + }; + }; + + system_clk: dummy13m { + compatible = "fixed-clock"; + clock-frequency = <13000000>; + #clock-cells = <0>; + }; + + rtc_clk: dummy32k { + compatible = "fixed-clock"; + clock-frequency = <32000>; + #clock-cells = <0>; + }; + + uart_clk: dummy26m { + compatible = "fixed-clock"; + clock-frequency = <26000000>; + #clock-cells = <0>; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupt-parent = <&gic>; + interrupts = , + , + , + ; + }; + + sysirq: intpol-controller@10200620 { + compatible = "mediatek,mt6795-sysirq", + "mediatek,mt6577-sysirq"; + interrupt-controller; + #interrupt-cells = <3>; + interrupt-parent = <&gic>; + reg = <0 0x10200620 0 0x20>; + }; + + gic: interrupt-controller@10221000 { + compatible = "arm,gic-400"; + #interrupt-cells = <3>; + interrupt-parent = <&gic>; + interrupt-controller; + reg = <0 0x10221000 0 0x1000>, + <0 0x10222000 0 0x2000>, + <0 0x10224000 0 0x2000>, + <0 0x10226000 0 0x2000>; + }; + + uart0: serial@11002000 { + compatible = "mediatek,mt6795-uart", + "mediatek,mt6577-uart"; + reg = <0 0x11002000 0 0x400>; + interrupts = ; + clocks = <&uart_clk>; + status = "disabled"; + }; + + uart1: serial@11003000 { + compatible = "mediatek,mt6795-uart", + "mediatek,mt6577-uart"; + reg = <0 0x11003000 0 0x400>; + interrupts = ; + clocks = <&uart_clk>; + status = "disabled"; + }; + + uart2: serial@11004000 { + compatible = "mediatek,mt6795-uart", + "mediatek,mt6577-uart"; + reg = <0 0x11004000 0 0x400>; + interrupts = ; + clocks = <&uart_clk>; + status = "disabled"; + }; + + uart3: serial@11005000 { + compatible = "mediatek,mt6795-uart", + "mediatek,mt6577-uart"; + reg = <0 0x11005000 0 0x400>; + interrupts = ; + clocks = <&uart_clk>; + status = "disabled"; + }; +}; diff --git a/arch/arm64/boot/dts/mediatek/mt8173-evb.dts b/arch/arm64/boot/dts/mediatek/mt8173-evb.dts index d0ab012fa379..4be66cadbc7c 100644 --- a/arch/arm64/boot/dts/mediatek/mt8173-evb.dts +++ b/arch/arm64/boot/dts/mediatek/mt8173-evb.dts @@ -34,6 +34,359 @@ memory@40000000 { chosen { }; }; +&i2c1 { + status = "okay"; + + buck: da9211@68 { + compatible = "dlg,da9211"; + reg = <0x68>; + + regulators { + da9211_vcpu_reg: BUCKA { + regulator-name = "VBUCKA"; + regulator-min-microvolt = < 700000>; + regulator-max-microvolt = <1310000>; + regulator-min-microamp = <2000000>; + regulator-max-microamp = <4400000>; + regulator-ramp-delay = <10000>; + regulator-always-on; + }; + + da9211_vgpu_reg: BUCKB { + regulator-name = "VBUCKB"; + regulator-min-microvolt = < 700000>; + regulator-max-microvolt = <1310000>; + regulator-min-microamp = <2000000>; + regulator-max-microamp = <3000000>; + regulator-ramp-delay = <10000>; + }; + }; + }; +}; + +&mmc0 { + status = "okay"; + pinctrl-names = "default", "state_uhs"; + pinctrl-0 = <&mmc0_pins_default>; + pinctrl-1 = <&mmc0_pins_uhs>; + bus-width = <8>; + max-frequency = <50000000>; + cap-mmc-highspeed; + vmmc-supply = <&mt6397_vemc_3v3_reg>; + vqmmc-supply = <&mt6397_vio18_reg>; + non-removable; +}; + +&mmc1 { + status = "okay"; + pinctrl-names = "default", "state_uhs"; + pinctrl-0 = <&mmc1_pins_default>; + pinctrl-1 = <&mmc1_pins_uhs>; + bus-width = <4>; + max-frequency = <50000000>; + cap-sd-highspeed; + sd-uhs-sdr25; + cd-gpios = <&pio 132 0>; + vmmc-supply = <&mt6397_vmch_reg>; + vqmmc-supply = <&mt6397_vmc_reg>; +}; + +&pio { + mmc0_pins_default: mmc0default { + pins_cmd_dat { + pinmux = , + , + , + , + , + , + , + , + ; + input-enable; + bias-pull-up; + }; + + pins_clk { + pinmux = ; + bias-pull-down; + }; + + pins_rst { + pinmux = ; + bias-pull-up; + }; + }; + + mmc1_pins_default: mmc1default { + pins_cmd_dat { + pinmux = , + , + , + , + ; + input-enable; + drive-strength = ; + bias-pull-up = ; + }; + + pins_clk { + pinmux = ; + bias-pull-down; + drive-strength = ; + }; + + pins_insert { + pinmux = ; + bias-pull-up; + }; + }; + + mmc0_pins_uhs: mmc0 { + pins_cmd_dat { + pinmux = , + , + , + , + , + , + , + , + ; + input-enable; + drive-strength = ; + bias-pull-up = ; + }; + + pins_clk { + pinmux = ; + drive-strength = ; + bias-pull-down = ; + }; + + pins_rst { + pinmux = ; + bias-pull-up; + }; + }; + + mmc1_pins_uhs: mmc1 { + pins_cmd_dat { + pinmux = , + , + , + , + ; + input-enable; + drive-strength = ; + bias-pull-up = ; + }; + + pins_clk { + pinmux = ; + drive-strength = ; + bias-pull-down = ; + }; + }; +}; + +&pwrap { + pmic: mt6397 { + compatible = "mediatek,mt6397"; + interrupt-parent = <&pio>; + interrupts = <11 IRQ_TYPE_LEVEL_HIGH>; + interrupt-controller; + #interrupt-cells = <2>; + + mt6397regulator: mt6397regulator { + compatible = "mediatek,mt6397-regulator"; + + mt6397_vpca15_reg: buck_vpca15 { + regulator-compatible = "buck_vpca15"; + regulator-name = "vpca15"; + regulator-min-microvolt = < 700000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <12500>; + regulator-always-on; + }; + + mt6397_vpca7_reg: buck_vpca7 { + regulator-compatible = "buck_vpca7"; + regulator-name = "vpca7"; + regulator-min-microvolt = < 700000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <12500>; + regulator-enable-ramp-delay = <115>; + }; + + mt6397_vsramca15_reg: buck_vsramca15 { + regulator-compatible = "buck_vsramca15"; + regulator-name = "vsramca15"; + regulator-min-microvolt = < 700000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <12500>; + regulator-always-on; + }; + + mt6397_vsramca7_reg: buck_vsramca7 { + regulator-compatible = "buck_vsramca7"; + regulator-name = "vsramca7"; + regulator-min-microvolt = < 700000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <12500>; + regulator-always-on; + }; + + mt6397_vcore_reg: buck_vcore { + regulator-compatible = "buck_vcore"; + regulator-name = "vcore"; + regulator-min-microvolt = < 700000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <12500>; + regulator-always-on; + }; + + mt6397_vgpu_reg: buck_vgpu { + regulator-compatible = "buck_vgpu"; + regulator-name = "vgpu"; + regulator-min-microvolt = < 700000>; + regulator-max-microvolt = <1350000>; + regulator-ramp-delay = <12500>; + regulator-enable-ramp-delay = <115>; + }; + + mt6397_vdrm_reg: buck_vdrm { + regulator-compatible = "buck_vdrm"; + regulator-name = "vdrm"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <1400000>; + regulator-ramp-delay = <12500>; + regulator-always-on; + }; + + mt6397_vio18_reg: buck_vio18 { + regulator-compatible = "buck_vio18"; + regulator-name = "vio18"; + regulator-min-microvolt = <1620000>; + regulator-max-microvolt = <1980000>; + regulator-ramp-delay = <12500>; + regulator-always-on; + }; + + mt6397_vtcxo_reg: ldo_vtcxo { + regulator-compatible = "ldo_vtcxo"; + regulator-name = "vtcxo"; + regulator-always-on; + }; + + mt6397_va28_reg: ldo_va28 { + regulator-compatible = "ldo_va28"; + regulator-name = "va28"; + regulator-always-on; + }; + + mt6397_vcama_reg: ldo_vcama { + regulator-compatible = "ldo_vcama"; + regulator-name = "vcama"; + regulator-min-microvolt = <1500000>; + regulator-max-microvolt = <2800000>; + regulator-enable-ramp-delay = <218>; + }; + + mt6397_vio28_reg: ldo_vio28 { + regulator-compatible = "ldo_vio28"; + regulator-name = "vio28"; + regulator-always-on; + }; + + mt6397_vusb_reg: ldo_vusb { + regulator-compatible = "ldo_vusb"; + regulator-name = "vusb"; + }; + + mt6397_vmc_reg: ldo_vmc { + regulator-compatible = "ldo_vmc"; + regulator-name = "vmc"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <3300000>; + regulator-enable-ramp-delay = <218>; + }; + + mt6397_vmch_reg: ldo_vmch { + regulator-compatible = "ldo_vmch"; + regulator-name = "vmch"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3300000>; + regulator-enable-ramp-delay = <218>; + }; + + mt6397_vemc_3v3_reg: ldo_vemc3v3 { + regulator-compatible = "ldo_vemc3v3"; + regulator-name = "vemc_3v3"; + regulator-min-microvolt = <3000000>; + regulator-max-microvolt = <3300000>; + regulator-enable-ramp-delay = <218>; + }; + + mt6397_vgp1_reg: ldo_vgp1 { + regulator-compatible = "ldo_vgp1"; + regulator-name = "vcamd"; + regulator-min-microvolt = <1220000>; + regulator-max-microvolt = <3300000>; + regulator-enable-ramp-delay = <240>; + }; + + mt6397_vgp2_reg: ldo_vgp2 { + regulator-compatible = "ldo_vgp2"; + regulator-name = "vcamio"; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <3300000>; + regulator-enable-ramp-delay = <218>; + }; + + mt6397_vgp3_reg: ldo_vgp3 { + regulator-compatible = "ldo_vgp3"; + regulator-name = "vcamaf"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <3300000>; + regulator-enable-ramp-delay = <218>; + }; + + mt6397_vgp4_reg: ldo_vgp4 { + regulator-compatible = "ldo_vgp4"; + regulator-name = "vgp4"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <3300000>; + regulator-enable-ramp-delay = <218>; + }; + + mt6397_vgp5_reg: ldo_vgp5 { + regulator-compatible = "ldo_vgp5"; + regulator-name = "vgp5"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <3000000>; + regulator-enable-ramp-delay = <218>; + }; + + mt6397_vgp6_reg: ldo_vgp6 { + regulator-compatible = "ldo_vgp6"; + regulator-name = "vgp6"; + regulator-min-microvolt = <1200000>; + regulator-max-microvolt = <3300000>; + regulator-enable-ramp-delay = <218>; + }; + + mt6397_vibr_reg: ldo_vibr { + regulator-compatible = "ldo_vibr"; + regulator-name = "vibr"; + regulator-min-microvolt = <1300000>; + regulator-max-microvolt = <3300000>; + regulator-enable-ramp-delay = <218>; + }; + }; + }; +}; + &uart0 { status = "okay"; }; diff --git a/arch/arm64/boot/dts/mediatek/mt8173.dtsi b/arch/arm64/boot/dts/mediatek/mt8173.dtsi index 27237a1c1a87..d18ee4259ee5 100644 --- a/arch/arm64/boot/dts/mediatek/mt8173.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8173.dtsi @@ -11,8 +11,11 @@ * GNU General Public License for more details. */ +#include #include #include +#include +#include #include "mt8173-pinfunc.h" / { @@ -49,6 +52,8 @@ cpu0: cpu@0 { device_type = "cpu"; compatible = "arm,cortex-a53"; reg = <0x000>; + enable-method = "psci"; + cpu-idle-states = <&CPU_SLEEP_0>; }; cpu1: cpu@1 { @@ -56,6 +61,7 @@ cpu1: cpu@1 { compatible = "arm,cortex-a53"; reg = <0x001>; enable-method = "psci"; + cpu-idle-states = <&CPU_SLEEP_0>; }; cpu2: cpu@100 { @@ -63,6 +69,7 @@ cpu2: cpu@100 { compatible = "arm,cortex-a57"; reg = <0x100>; enable-method = "psci"; + cpu-idle-states = <&CPU_SLEEP_0>; }; cpu3: cpu@101 { @@ -70,6 +77,20 @@ cpu3: cpu@101 { compatible = "arm,cortex-a57"; reg = <0x101>; enable-method = "psci"; + cpu-idle-states = <&CPU_SLEEP_0>; + }; + + idle-states { + entry-method = "arm,psci"; + + CPU_SLEEP_0: cpu-sleep-0 { + compatible = "arm,idle-state"; + local-timer-stop; + entry-latency-us = <639>; + exit-latency-us = <680>; + min-residency-us = <1088>; + arm,psci-suspend-param = <0x0010000>; + }; }; }; @@ -81,10 +102,18 @@ psci { cpu_on = <0x84000003>; }; - uart_clk: dummy26m { + clk26m: oscillator@0 { compatible = "fixed-clock"; - clock-frequency = <26000000>; #clock-cells = <0>; + clock-frequency = <26000000>; + clock-output-names = "clk26m"; + }; + + clk32k: oscillator@1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <32000>; + clock-output-names = "clk32k"; }; timer { @@ -106,11 +135,32 @@ soc { compatible = "simple-bus"; ranges; - /* - * Pinctrl access register at 0x10005000 through regmap. - * Register 0x1000b000 is used by EINT. - */ - pio: pinctrl@10005000 { + topckgen: clock-controller@10000000 { + compatible = "mediatek,mt8173-topckgen"; + reg = <0 0x10000000 0 0x1000>; + #clock-cells = <1>; + }; + + infracfg: power-controller@10001000 { + compatible = "mediatek,mt8173-infracfg", "syscon"; + reg = <0 0x10001000 0 0x1000>; + #clock-cells = <1>; + #reset-cells = <1>; + }; + + pericfg: power-controller@10003000 { + compatible = "mediatek,mt8173-pericfg", "syscon"; + reg = <0 0x10003000 0 0x1000>; + #clock-cells = <1>; + #reset-cells = <1>; + }; + + syscfg_pctl_a: syscfg_pctl_a@10005000 { + compatible = "mediatek,mt8173-pctl-a-syscfg", "syscon"; + reg = <0 0x10005000 0 0x1000>; + }; + + pio: pinctrl@0x10005000 { compatible = "mediatek,mt8173-pinctrl"; reg = <0 0x1000b000 0 0x1000>; mediatek,pctl-regmap = <&syscfg_pctl_a>; @@ -122,11 +172,81 @@ pio: pinctrl@10005000 { interrupts = , , ; + + i2c0_pins_a: i2c0 { + pins1 { + pinmux = , + ; + bias-disable; + }; + }; + + i2c1_pins_a: i2c1 { + pins1 { + pinmux = , + ; + bias-disable; + }; + }; + + i2c2_pins_a: i2c2 { + pins1 { + pinmux = , + ; + bias-disable; + }; + }; + + i2c3_pins_a: i2c3 { + pins1 { + pinmux = , + ; + bias-disable; + }; + }; + + i2c4_pins_a: i2c4 { + pins1 { + pinmux = , + ; + bias-disable; + }; + }; + + i2c6_pins_a: i2c6 { + pins1 { + pinmux = , + ; + bias-disable; + }; + }; }; - syscfg_pctl_a: syscfg_pctl_a@10005000 { - compatible = "mediatek,mt8173-pctl-a-syscfg", "syscon"; - reg = <0 0x10005000 0 0x1000>; + scpsys: scpsys@10006000 { + compatible = "mediatek,mt8173-scpsys"; + #power-domain-cells = <1>; + reg = <0 0x10006000 0 0x1000>; + clocks = <&clk26m>, + <&topckgen CLK_TOP_MM_SEL>; + clock-names = "mfg", "mm"; + infracfg = <&infracfg>; + }; + + watchdog: watchdog@10007000 { + compatible = "mediatek,mt8173-wdt", + "mediatek,mt6589-wdt"; + reg = <0 0x10007000 0 0x100>; + }; + + pwrap: pwrap@1000d000 { + compatible = "mediatek,mt8173-pwrap"; + reg = <0 0x1000d000 0 0x1000>; + reg-names = "pwrap"; + interrupts = ; + resets = <&infracfg MT8173_INFRA_PMIC_WRAP_RST>; + reset-names = "pwrap"; + clocks = <&infracfg CLK_INFRA_PMICSPI>, <&infracfg CLK_INFRA_PMICWRAP>; + clock-names = "spi", "wrap"; }; sysirq: intpol-controller@10200620 { @@ -138,6 +258,12 @@ sysirq: intpol-controller@10200620 { reg = <0 0x10200620 0 0x20>; }; + apmixedsys: clock-controller@10209000 { + compatible = "mediatek,mt8173-apmixedsys"; + reg = <0 0x10209000 0 0x1000>; + #clock-cells = <1>; + }; + gic: interrupt-controller@10220000 { compatible = "arm,gic-400"; #interrupt-cells = <3>; @@ -156,7 +282,8 @@ uart0: serial@11002000 { "mediatek,mt6577-uart"; reg = <0 0x11002000 0 0x400>; interrupts = ; - clocks = <&uart_clk>; + clocks = <&pericfg CLK_PERI_UART0_SEL>, <&pericfg CLK_PERI_UART0>; + clock-names = "baud", "bus"; status = "disabled"; }; @@ -165,7 +292,8 @@ uart1: serial@11003000 { "mediatek,mt6577-uart"; reg = <0 0x11003000 0 0x400>; interrupts = ; - clocks = <&uart_clk>; + clocks = <&pericfg CLK_PERI_UART1_SEL>, <&pericfg CLK_PERI_UART1>; + clock-names = "baud", "bus"; status = "disabled"; }; @@ -174,7 +302,8 @@ uart2: serial@11004000 { "mediatek,mt6577-uart"; reg = <0 0x11004000 0 0x400>; interrupts = ; - clocks = <&uart_clk>; + clocks = <&pericfg CLK_PERI_UART2_SEL>, <&pericfg CLK_PERI_UART2>; + clock-names = "baud", "bus"; status = "disabled"; }; @@ -183,7 +312,179 @@ uart3: serial@11005000 { "mediatek,mt6577-uart"; reg = <0 0x11005000 0 0x400>; interrupts = ; - clocks = <&uart_clk>; + clocks = <&pericfg CLK_PERI_UART3_SEL>, <&pericfg CLK_PERI_UART3>; + clock-names = "baud", "bus"; + status = "disabled"; + }; + + i2c0: i2c@11007000 { + compatible = "mediatek,mt8173-i2c"; + reg = <0 0x11007000 0 0x70>, + <0 0x11000100 0 0x80>; + interrupts = ; + clock-div = <16>; + clocks = <&pericfg CLK_PERI_I2C0>, + <&pericfg CLK_PERI_AP_DMA>; + clock-names = "main", "dma"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_pins_a>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c1: i2c@11008000 { + compatible = "mediatek,mt8173-i2c"; + reg = <0 0x11008000 0 0x70>, + <0 0x11000180 0 0x80>; + interrupts = ; + clock-div = <16>; + clocks = <&pericfg CLK_PERI_I2C1>, + <&pericfg CLK_PERI_AP_DMA>; + clock-names = "main", "dma"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_pins_a>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c2: i2c@11009000 { + compatible = "mediatek,mt8173-i2c"; + reg = <0 0x11009000 0 0x70>, + <0 0x11000200 0 0x80>; + interrupts = ; + clock-div = <16>; + clocks = <&pericfg CLK_PERI_I2C2>, + <&pericfg CLK_PERI_AP_DMA>; + clock-names = "main", "dma"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c2_pins_a>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c3: i2c3@11010000 { + compatible = "mediatek,mt8173-i2c"; + reg = <0 0x11010000 0 0x70>, + <0 0x11000280 0 0x80>; + interrupts = ; + clock-div = <16>; + clocks = <&pericfg CLK_PERI_I2C3>, + <&pericfg CLK_PERI_AP_DMA>; + clock-names = "main", "dma"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c3_pins_a>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c4: i2c4@11011000 { + compatible = "mediatek,mt8173-i2c"; + reg = <0 0x11011000 0 0x70>, + <0 0x11000300 0 0x80>; + interrupts = ; + clock-div = <16>; + clocks = <&pericfg CLK_PERI_I2C4>, + <&pericfg CLK_PERI_AP_DMA>; + clock-names = "main", "dma"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c4_pins_a>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c6: i2c6@11013000 { + compatible = "mediatek,mt8173-i2c"; + reg = <0 0x11013000 0 0x70>, + <0 0x11000080 0 0x80>; + interrupts = ; + clock-div = <16>; + clocks = <&pericfg CLK_PERI_I2C6>, + <&pericfg CLK_PERI_AP_DMA>; + clock-names = "main", "dma"; + pinctrl-names = "default"; + pinctrl-0 = <&i2c6_pins_a>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + afe: audio-controller@11220000 { + compatible = "mediatek,mt8173-afe-pcm"; + reg = <0 0x11220000 0 0x1000>; + interrupts = ; + power-domains = <&scpsys MT8173_POWER_DOMAIN_AUDIO>; + clocks = <&infracfg CLK_INFRA_AUDIO>, + <&topckgen CLK_TOP_AUDIO_SEL>, + <&topckgen CLK_TOP_AUD_INTBUS_SEL>, + <&topckgen CLK_TOP_APLL1_DIV0>, + <&topckgen CLK_TOP_APLL2_DIV0>, + <&topckgen CLK_TOP_I2S0_M_SEL>, + <&topckgen CLK_TOP_I2S1_M_SEL>, + <&topckgen CLK_TOP_I2S2_M_SEL>, + <&topckgen CLK_TOP_I2S3_M_SEL>, + <&topckgen CLK_TOP_I2S3_B_SEL>; + clock-names = "infra_sys_audio_clk", + "top_pdn_audio", + "top_pdn_aud_intbus", + "bck0", + "bck1", + "i2s0_m", + "i2s1_m", + "i2s2_m", + "i2s3_m", + "i2s3_b"; + assigned-clocks = <&topckgen CLK_TOP_AUD_1_SEL>, + <&topckgen CLK_TOP_AUD_2_SEL>; + assigned-clock-parents = <&topckgen CLK_TOP_APLL1>, + <&topckgen CLK_TOP_APLL2>; + }; + + mmc0: mmc@11230000 { + compatible = "mediatek,mt8173-mmc", + "mediatek,mt8135-mmc"; + reg = <0 0x11230000 0 0x1000>; + interrupts = ; + clocks = <&pericfg CLK_PERI_MSDC30_0>, + <&topckgen CLK_TOP_MSDC50_0_H_SEL>; + clock-names = "source", "hclk"; + status = "disabled"; + }; + + mmc1: mmc@11240000 { + compatible = "mediatek,mt8173-mmc", + "mediatek,mt8135-mmc"; + reg = <0 0x11240000 0 0x1000>; + interrupts = ; + clocks = <&pericfg CLK_PERI_MSDC30_1>, + <&topckgen CLK_TOP_AXI_SEL>; + clock-names = "source", "hclk"; + status = "disabled"; + }; + + mmc2: mmc@11250000 { + compatible = "mediatek,mt8173-mmc", + "mediatek,mt8135-mmc"; + reg = <0 0x11250000 0 0x1000>; + interrupts = ; + clocks = <&pericfg CLK_PERI_MSDC30_2>, + <&topckgen CLK_TOP_AXI_SEL>; + clock-names = "source", "hclk"; + status = "disabled"; + }; + + mmc3: mmc@11260000 { + compatible = "mediatek,mt8173-mmc", + "mediatek,mt8135-mmc"; + reg = <0 0x11260000 0 0x1000>; + interrupts = ; + clocks = <&pericfg CLK_PERI_MSDC30_3>, + <&topckgen CLK_TOP_MSDC50_2_H_SEL>; + clock-names = "source", "hclk"; status = "disabled"; }; }; diff --git a/arch/arm64/boot/dts/qcom/apq8016-sbc-pmic-pins.dtsi b/arch/arm64/boot/dts/qcom/apq8016-sbc-pmic-pins.dtsi index 535532b9287f..e03c11d9d834 100644 --- a/arch/arm64/boot/dts/qcom/apq8016-sbc-pmic-pins.dtsi +++ b/arch/arm64/boot/dts/qcom/apq8016-sbc-pmic-pins.dtsi @@ -2,28 +2,38 @@ &pm8916_gpios { - pinctrl-names = "default"; - pinctrl-0 = <&pm8916_gpios_default>; - - pm8916_gpios_default: default { - usb_hub_reset_pm { - pins = "gpio1"; - function = PMIC_GPIO_FUNC_NORMAL; - output-low; - }; - usb_sw_sel_pm { - pins = "gpio2"; - function = PMIC_GPIO_FUNC_NORMAL; - input-disable; - }; - usr_led_3_ctrl { + usb_hub_reset_pm: usb_hub_reset_pm { + pinconf { pins = "gpio3"; function = PMIC_GPIO_FUNC_NORMAL; output-low; }; - usr_led_4_ctrl { + }; + + usb_sw_sel_pm: usb_sw_sel_pm { + pinconf { pins = "gpio4"; function = PMIC_GPIO_FUNC_NORMAL; + power-source = ; + input-disable; + }; + }; + + pm8916_gpios_leds: pm8916_gpios_leds { + pinconf { + pins = "gpio1", "gpio2"; + function = PMIC_GPIO_FUNC_NORMAL; + output-low; + }; + }; +}; + +&pm8916_mpps { + + pm8916_mpps_leds: pm8916_mpps_leds { + pinconf { + pins = "mpp2", "mpp3"; + function = PMIC_GPIO_FUNC_NORMAL; output-low; }; }; diff --git a/arch/arm64/boot/dts/qcom/apq8016-sbc-soc-pins.dtsi b/arch/arm64/boot/dts/qcom/apq8016-sbc-soc-pins.dtsi index 5f7023f90df7..cbeee0bcdf52 100644 --- a/arch/arm64/boot/dts/qcom/apq8016-sbc-soc-pins.dtsi +++ b/arch/arm64/boot/dts/qcom/apq8016-sbc-soc-pins.dtsi @@ -3,17 +3,9 @@ &msmgpio { - pinctrl-names = "default"; - pinctrl-0 = <&soc_gpios_default>; - - soc_gpios_default: default { - usr_led_1_ctrl_default: usr_led_1_ctrl_default { - pins = "gpio21"; - function = "gpio"; - output-low; - }; - usr_led_2_ctrl_default: usr_led_2_ctrl_default { - pins = "gpio120"; + msmgpio_leds: msmgpio_leds { + pinconf { + pins = "gpio21", "gpio120"; function = "gpio"; output-low; }; diff --git a/arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi b/arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi index 98abece6b233..66804ffbc6d2 100644 --- a/arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi +++ b/arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi @@ -32,5 +32,56 @@ serial@78b0000 { pinctrl-0 = <&blsp1_uart2_default>; pinctrl-1 = <&blsp1_uart2_sleep>; }; + + leds { + pinctrl-names = "default"; + pinctrl-0 = <&msmgpio_leds>, + <&pm8916_gpios_leds>, + <&pm8916_mpps_leds>; + + compatible = "gpio-leds"; + + led@1 { + label = "apq8016-sbc:green:user1"; + gpios = <&msmgpio 21 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + default-state = "off"; + }; + + led@2 { + label = "apq8016-sbc:green:user2"; + gpios = <&msmgpio 120 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "mmc0"; + default-state = "off"; + }; + + led@3 { + label = "apq8016-sbc:green:user3"; + gpios = <&pm8916_gpios 1 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "mmc1"; + default-state = "off"; + }; + + led@4 { + label = "apq8016-sbc:green:user4"; + gpios = <&pm8916_gpios 2 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "none"; + default-state = "off"; + }; + + led@5 { + label = "apq8016-sbc:yellow:wlan"; + gpios = <&pm8916_mpps 2 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "wlan"; + default-state = "off"; + }; + + led@6 { + label = "apq8016-sbc:blue:bt"; + gpios = <&pm8916_mpps 3 GPIO_ACTIVE_HIGH>; + linux,default-trigger = "bt"; + default-state = "off"; + }; + }; }; }; diff --git a/arch/arm64/boot/dts/qcom/msm8916-pins.dtsi b/arch/arm64/boot/dts/qcom/msm8916-pins.dtsi new file mode 100644 index 000000000000..568956859088 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/msm8916-pins.dtsi @@ -0,0 +1,430 @@ +/* + * Copyright (c) 2013-2015, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +&msmgpio { + + blsp1_uart2_default: blsp1_uart2_default { + pinmux { + function = "blsp_uart2"; + pins = "gpio4", "gpio5"; + }; + pinconf { + pins = "gpio4", "gpio5"; + drive-strength = <16>; + bias-disable; + }; + }; + + blsp1_uart2_sleep: blsp1_uart2_sleep { + pinmux { + function = "blsp_uart2"; + pins = "gpio4", "gpio5"; + }; + pinconf { + pins = "gpio4", "gpio5"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + spi1_default: spi1_default { + pinmux { + function = "blsp_spi1"; + pins = "gpio0", "gpio1", "gpio3"; + }; + pinmux_cs { + function = "gpio"; + pins = "gpio2"; + }; + pinconf { + pins = "gpio0", "gpio1", "gpio3"; + drive-strength = <12>; + bias-disable; + }; + pinconf_cs { + pins = "gpio2"; + drive-strength = <2>; + bias-disable; + output-high; + }; + }; + + spi1_sleep: spi1_sleep { + pinmux { + function = "gpio"; + pins = "gpio0", "gpio1", "gpio2", "gpio3"; + }; + pinconf { + pins = "gpio0", "gpio1", "gpio2", "gpio3"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + spi2_default: spi2_default { + pinmux { + function = "blsp_spi2"; + pins = "gpio4", "gpio5", "gpio7"; + }; + pinmux_cs { + function = "gpio"; + pins = "gpio6"; + }; + pinconf { + pins = "gpio4", "gpio5", "gpio6", "gpio7"; + drive-strength = <12>; + bias-disable; + }; + pinconf_cs { + pins = "gpio6"; + drive-strength = <2>; + bias-disable; + output-high; + }; + }; + + spi2_sleep: spi2_sleep { + pinmux { + function = "gpio"; + pins = "gpio4", "gpio5", "gpio6", "gpio7"; + }; + pinconf { + pins = "gpio4", "gpio5", "gpio6", "gpio7"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + spi3_default: spi3_default { + pinmux { + function = "blsp_spi3"; + pins = "gpio8", "gpio9", "gpio11"; + }; + pinmux_cs { + function = "gpio"; + pins = "gpio10"; + }; + pinconf { + pins = "gpio8", "gpio9", "gpio10", "gpio11"; + drive-strength = <12>; + bias-disable; + }; + pinconf_cs { + pins = "gpio10"; + drive-strength = <2>; + bias-disable; + output-high; + }; + }; + + spi3_sleep: spi3_sleep { + pinmux { + function = "gpio"; + pins = "gpio8", "gpio9", "gpio10", "gpio11"; + }; + pinconf { + pins = "gpio8", "gpio9", "gpio10", "gpio11"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + spi4_default: spi4_default { + pinmux { + function = "blsp_spi4"; + pins = "gpio12", "gpio13", "gpio15"; + }; + pinmux_cs { + function = "gpio"; + pins = "gpio14"; + }; + pinconf { + pins = "gpio12", "gpio13", "gpio14", "gpio15"; + drive-strength = <12>; + bias-disable; + }; + pinconf_cs { + pins = "gpio14"; + drive-strength = <2>; + bias-disable; + output-high; + }; + }; + + spi4_sleep: spi4_sleep { + pinmux { + function = "gpio"; + pins = "gpio12", "gpio13", "gpio14", "gpio15"; + }; + pinconf { + pins = "gpio12", "gpio13", "gpio14", "gpio15"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + spi5_default: spi5_default { + pinmux { + function = "blsp_spi5"; + pins = "gpio16", "gpio17", "gpio19"; + }; + pinmux_cs { + function = "gpio"; + pins = "gpio18"; + }; + pinconf { + pins = "gpio16", "gpio17", "gpio18", "gpio19"; + drive-strength = <12>; + bias-disable; + }; + pinconf_cs { + pins = "gpio18"; + drive-strength = <2>; + bias-disable; + output-high; + }; + }; + + spi5_sleep: spi5_sleep { + pinmux { + function = "gpio"; + pins = "gpio16", "gpio17", "gpio18", "gpio19"; + }; + pinconf { + pins = "gpio16", "gpio17", "gpio18", "gpio19"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + spi6_default: spi6_default { + pinmux { + function = "blsp_spi6"; + pins = "gpio20", "gpio21", "gpio23"; + }; + pinmux_cs { + function = "gpio"; + pins = "gpio22"; + }; + pinconf { + pins = "gpio20", "gpio21", "gpio22", "gpio23"; + drive-strength = <12>; + bias-disable; + }; + pinconf_cs { + pins = "gpio22"; + drive-strength = <2>; + bias-disable; + output-high; + }; + }; + + spi6_sleep: spi6_sleep { + pinmux { + function = "gpio"; + pins = "gpio20", "gpio21", "gpio22", "gpio23"; + }; + pinconf { + pins = "gpio20", "gpio21", "gpio22", "gpio23"; + drive-strength = <2>; + bias-pull-down; + }; + }; + + i2c4_default: i2c4_default { + pinmux { + function = "blsp_i2c4"; + pins = "gpio14", "gpio15"; + }; + pinconf { + pins = "gpio14", "gpio15"; + drive-strength = <2>; + bias-disable = <0>; + }; + }; + + i2c4_sleep: i2c4_sleep { + pinmux { + function = "blsp_i2c4"; + pins = "gpio14", "gpio15"; + }; + pinconf { + pins = "gpio14", "gpio15"; + drive-strength = <2>; + bias-disable = <0>; + }; + }; + + sdhc2_cd_pin { + sdc2_cd_on: cd_on { + pinmux { + function = "gpio"; + pins = "gpio38"; + }; + pinconf { + pins = "gpio38"; + drive-strength = <2>; + bias-pull-up; + }; + }; + sdc2_cd_off: cd_off { + pinmux { + function = "gpio"; + pins = "gpio38"; + }; + pinconf { + pins = "gpio38"; + drive-strength = <2>; + bias-disable; + }; + }; + }; + + pmx_sdc1_clk { + sdc1_clk_on: clk_on { + pinmux { + pins = "sdc1_clk"; + }; + pinconf { + pins = "sdc1_clk"; + bias-disable; + drive-strength = <16>; + }; + }; + sdc1_clk_off: clk_off { + pinmux { + pins = "sdc1_clk"; + }; + pinconf { + pins = "sdc1_clk"; + bias-disable; + drive-strength = <2>; + }; + }; + }; + + pmx_sdc1_cmd { + sdc1_cmd_on: cmd_on { + pinmux { + pins = "sdc1_cmd"; + }; + pinconf { + pins = "sdc1_cmd"; + bias-pull-up; + drive-strength = <10>; + }; + }; + sdc1_cmd_off: cmd_off { + pinmux { + pins = "sdc1_cmd"; + }; + pinconf { + pins = "sdc1_cmd"; + bias-pull-up; + drive-strength = <2>; + }; + }; + }; + + pmx_sdc1_data { + sdc1_data_on: data_on { + pinmux { + pins = "sdc1_data"; + }; + pinconf { + pins = "sdc1_data"; + bias-pull-up; + drive-strength = <10>; + }; + }; + sdc1_data_off: data_off { + pinmux { + pins = "sdc1_data"; + }; + pinconf { + pins = "sdc1_data"; + bias-pull-up; + drive-strength = <2>; + }; + }; + }; + + pmx_sdc2_clk { + sdc2_clk_on: clk_on { + pinmux { + pins = "sdc2_clk"; + }; + pinconf { + pins = "sdc2_clk"; + bias-disable; + drive-strength = <16>; + }; + }; + sdc2_clk_off: clk_off { + pinmux { + pins = "sdc2_clk"; + }; + pinconf { + pins = "sdc2_clk"; + bias-disable; + drive-strength = <2>; + }; + }; + }; + + pmx_sdc2_cmd { + sdc2_cmd_on: cmd_on { + pinmux { + pins = "sdc2_cmd"; + }; + pinconf { + pins = "sdc2_cmd"; + bias-pull-up; + drive-strength = <10>; + }; + }; + sdc2_cmd_off: cmd_off { + pinmux { + pins = "sdc2_cmd"; + }; + pinconf { + pins = "sdc2_cmd"; + bias-pull-up; + drive-strength = <2>; + }; + }; + }; + + pmx_sdc2_data { + sdc2_data_on: data_on { + pinmux { + pins = "sdc2_data"; + }; + pinconf { + pins = "sdc2_data"; + bias-pull-up; + drive-strength = <10>; + }; + }; + sdc2_data_off: data_off { + pinmux { + pins = "sdc2_data"; + }; + pinconf { + pins = "sdc2_data"; + bias-pull-up; + drive-strength = <2>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/msm8916.dtsi b/arch/arm64/boot/dts/qcom/msm8916.dtsi index 0f49ebd0aa8b..5911de008dd5 100644 --- a/arch/arm64/boot/dts/qcom/msm8916.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8916.dtsi @@ -24,7 +24,10 @@ / { #address-cells = <2>; #size-cells = <2>; - aliases { }; + aliases { + sdhc1 = &sdhc_1; /* SDC1 eMMC slot */ + sdhc2 = &sdhc_2; /* SDC2 SD card slot */ + }; chosen { }; @@ -90,30 +93,6 @@ msmgpio: pinctrl@1000000 { #gpio-cells = <2>; interrupt-controller; #interrupt-cells = <2>; - - blsp1_uart2_default: blsp1_uart2_default { - pinmux { - function = "blsp_uart2"; - pins = "gpio4", "gpio5"; - }; - pinconf { - pins = "gpio4", "gpio5"; - drive-strength = <16>; - bias-disable; - }; - }; - - blsp1_uart2_sleep: blsp1_uart2_sleep { - pinmux { - function = "blsp_uart2"; - pins = "gpio4", "gpio5"; - }; - pinconf { - pins = "gpio4", "gpio5"; - drive-strength = <2>; - bias-pull-down; - }; - }; }; gcc: qcom,gcc@1800000 { @@ -132,6 +111,202 @@ blsp1_uart2: serial@78b0000 { status = "disabled"; }; + blsp_dma: dma@7884000 { + compatible = "qcom,bam-v1.7.0"; + reg = <0x07884000 0x23000>; + interrupts = ; + clocks = <&gcc GCC_BLSP1_AHB_CLK>; + clock-names = "bam_clk"; + #dma-cells = <1>; + qcom,ee = <0>; + status = "disabled"; + }; + + blsp_spi1: spi@78b5000 { + compatible = "qcom,spi-qup-v2.2.1"; + reg = <0x078b5000 0x600>; + interrupts = ; + clocks = <&gcc GCC_BLSP1_QUP1_SPI_APPS_CLK>, + <&gcc GCC_BLSP1_AHB_CLK>; + clock-names = "core", "iface"; + dmas = <&blsp_dma 5>, <&blsp_dma 4>; + dma-names = "rx", "tx"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&spi1_default>; + pinctrl-1 = <&spi1_sleep>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + blsp_spi2: spi@78b6000 { + compatible = "qcom,spi-qup-v2.2.1"; + reg = <0x078b6000 0x600>; + interrupts = ; + clocks = <&gcc GCC_BLSP1_QUP2_SPI_APPS_CLK>, + <&gcc GCC_BLSP1_AHB_CLK>; + clock-names = "core", "iface"; + dmas = <&blsp_dma 7>, <&blsp_dma 6>; + dma-names = "rx", "tx"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&spi2_default>; + pinctrl-1 = <&spi2_sleep>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + blsp_spi3: spi@78b7000 { + compatible = "qcom,spi-qup-v2.2.1"; + reg = <0x078b7000 0x600>; + interrupts = ; + clocks = <&gcc GCC_BLSP1_QUP3_SPI_APPS_CLK>, + <&gcc GCC_BLSP1_AHB_CLK>; + clock-names = "core", "iface"; + dmas = <&blsp_dma 9>, <&blsp_dma 8>; + dma-names = "rx", "tx"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&spi3_default>; + pinctrl-1 = <&spi3_sleep>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + blsp_spi4: spi@78b8000 { + compatible = "qcom,spi-qup-v2.2.1"; + reg = <0x078b8000 0x600>; + interrupts = ; + clocks = <&gcc GCC_BLSP1_QUP4_SPI_APPS_CLK>, + <&gcc GCC_BLSP1_AHB_CLK>; + clock-names = "core", "iface"; + dmas = <&blsp_dma 11>, <&blsp_dma 10>; + dma-names = "rx", "tx"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&spi4_default>; + pinctrl-1 = <&spi4_sleep>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + blsp_spi5: spi@78b9000 { + compatible = "qcom,spi-qup-v2.2.1"; + reg = <0x078b9000 0x600>; + interrupts = ; + clocks = <&gcc GCC_BLSP1_QUP5_SPI_APPS_CLK>, + <&gcc GCC_BLSP1_AHB_CLK>; + clock-names = "core", "iface"; + dmas = <&blsp_dma 13>, <&blsp_dma 12>; + dma-names = "rx", "tx"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&spi5_default>; + pinctrl-1 = <&spi5_sleep>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + blsp_spi6: spi@78ba000 { + compatible = "qcom,spi-qup-v2.2.1"; + reg = <0x078ba000 0x600>; + interrupts = ; + clocks = <&gcc GCC_BLSP1_QUP6_SPI_APPS_CLK>, + <&gcc GCC_BLSP1_AHB_CLK>; + clock-names = "core", "iface"; + dmas = <&blsp_dma 15>, <&blsp_dma 14>; + dma-names = "rx", "tx"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&spi6_default>; + pinctrl-1 = <&spi6_sleep>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + blsp_i2c4: i2c@78b8000 { + compatible = "qcom,i2c-qup-v2.2.1"; + reg = <0x78b8000 0x1000>; + interrupts = ; + clocks = <&gcc GCC_BLSP1_AHB_CLK>, + <&gcc GCC_BLSP1_QUP4_I2C_APPS_CLK>; + clock-names = "iface", "core"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&i2c4_default>; + pinctrl-1 = <&i2c4_sleep>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + sdhc_1: sdhci@07824000 { + compatible = "qcom,sdhci-msm-v4"; + reg = <0x07824900 0x11c>, <0x07824000 0x800>; + reg-names = "hc_mem", "core_mem"; + + interrupts = <0 123 0>, <0 138 0>; + interrupt-names = "hc_irq", "pwr_irq"; + clocks = <&gcc GCC_SDCC1_APPS_CLK>, + <&gcc GCC_SDCC1_AHB_CLK>; + clock-names = "core", "iface"; + bus-width = <8>; + non-removable; + status = "disabled"; + }; + + sdhc_2: sdhci@07864000 { + compatible = "qcom,sdhci-msm-v4"; + reg = <0x07864900 0x11c>, <0x07864000 0x800>; + reg-names = "hc_mem", "core_mem"; + + interrupts = <0 125 0>, <0 221 0>; + interrupt-names = "hc_irq", "pwr_irq"; + clocks = <&gcc GCC_SDCC2_APPS_CLK>, + <&gcc GCC_SDCC2_AHB_CLK>; + clock-names = "core", "iface"; + bus-width = <4>; + status = "disabled"; + }; + + usb_dev: usb@78d9000 { + compatible = "qcom,ci-hdrc"; + reg = <0x78d9000 0x400>; + dr_mode = "peripheral"; + interrupts = ; + usb-phy = <&usb_otg>; + status = "disabled"; + }; + + usb_host: ehci@78d9000 { + compatible = "qcom,ehci-host"; + reg = <0x78d9000 0x400>; + interrupts = ; + usb-phy = <&usb_otg>; + status = "disabled"; + }; + + usb_otg: phy@78d9000 { + compatible = "qcom,usb-otg-snps"; + reg = <0x78d9000 0x400>; + interrupts = , + ; + + qcom,vdd-levels = <1 5 7>; + qcom,phy-init-sequence = <0x44 0x6B 0x24 0x13>; + dr_mode = "peripheral"; + qcom,otg-control = <2>; // PMIC + + clocks = <&gcc GCC_USB_HS_AHB_CLK>, + <&gcc GCC_USB_HS_SYSTEM_CLK>, + <&gcc GCC_USB2A_PHY_SLEEP_CLK>; + clock-names = "iface", "core", "sleep"; + + resets = <&gcc GCC_USB2A_PHY_BCR>, + <&gcc GCC_USB_HS_BCR>; + reset-names = "phy", "link"; + status = "disabled"; + }; + intc: interrupt-controller@b000000 { compatible = "qcom,msm-qgic2"; interrupt-controller; @@ -217,3 +392,5 @@ spmi_bus: spmi@200f000 { }; }; }; + +#include "msm8916-pins.dtsi" diff --git a/arch/arm64/boot/dts/rockchip/Makefile b/arch/arm64/boot/dts/rockchip/Makefile new file mode 100644 index 000000000000..601e6a236c1d --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/Makefile @@ -0,0 +1,5 @@ +dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3368-r88.dtb + +always := $(dtb-y) +subdir-y := $(dts-dirs) +clean-files := *.dtb diff --git a/arch/arm64/boot/dts/rockchip/rk3368-r88.dts b/arch/arm64/boot/dts/rockchip/rk3368-r88.dts new file mode 100644 index 000000000000..401a81231eb9 --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368-r88.dts @@ -0,0 +1,354 @@ +/* + * Copyright (c) 2015 Heiko Stuebner + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This file is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This file is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; +#include "rk3368.dtsi" + +/ { + model = "Rockchip R88"; + compatible = "rockchip,r88", "rockchip,rk3368"; + + chosen { + stdout-path = "serial2:115200n8"; + }; + + memory { + device_type = "memory"; + reg = <0x0 0x0 0x0 0x40000000>; + }; + + emmc_pwrseq: emmc-pwrseq { + compatible = "mmc-pwrseq-emmc"; + pinctrl-0 = <&emmc_reset>; + pinctrl-names = "default"; + reset-gpios = <&gpio2 3 GPIO_ACTIVE_HIGH>; + }; + + keys: gpio-keys { + compatible = "gpio-keys"; + #address-cells = <1>; + #size-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&pwr_key>; + + button@0 { + gpio-key,wakeup = <1>; + gpios = <&gpio0 2 GPIO_ACTIVE_LOW>; + label = "GPIO Power"; + linux,code = <116>; + }; + }; + + leds: gpio-leds { + compatible = "gpio-leds"; + + work { + gpios = <&gpio3 29 GPIO_ACTIVE_HIGH>; + label = "r88:green:led"; + pinctrl-names = "default"; + pinctrl-0 = <&led_ctl>; + }; + }; + + ir: ir-receiver { + compatible = "gpio-ir-receiver"; + gpios = <&gpio3 30 GPIO_ACTIVE_LOW>; + pinctrl-names = "default"; + pinctrl-0 = <&ir_int>; + }; + + sdio_pwrseq: sdio-pwrseq { + compatible = "mmc-pwrseq-simple"; + clocks = <&hym8563>; + clock-names = "ext_clock"; + pinctrl-names = "default"; + pinctrl-0 = <&bt_rst>, <&wifi_reg_on>; + + reset-gpios = + /* BT_RST_N */ + <&gpio3 5 GPIO_ACTIVE_LOW>, + + /* WL_REG_ON */ + <&gpio3 4 GPIO_ACTIVE_LOW>; + }; + + vcc_18: vcc18-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_18"; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc_sys>; + }; + + /* supplies both host and otg */ + vcc_host: vcc-host-regulator { + compatible = "regulator-fixed"; + enable-active-high; + gpio = <&gpio0 4 GPIO_ACTIVE_HIGH>; + pinctrl-names = "default"; + pinctrl-0 = <&host_vbus_drv>; + regulator-name = "vcc_host"; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc_sys>; + }; + + vcc_io: vcc-io-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_io"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc_sys>; + }; + + vcc_lan: vcc-lan-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_lan"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc_io>; + }; + + vcc_sys: vcc-sys-regulator { + compatible = "regulator-fixed"; + regulator-name = "vcc_sys"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + regulator-always-on; + regulator-boot-on; + }; + + vccio_wl: vccio-wl-regulator { + compatible = "regulator-fixed"; + regulator-name = "vccio_wl"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc_io>; + }; + + vdd_10: vdd-10-regulator { + compatible = "regulator-fixed"; + regulator-name = "vdd_10"; + regulator-min-microvolt = <1000000>; + regulator-max-microvolt = <1000000>; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc_sys>; + }; +}; + +&emmc { + broken-cd; + bus-width = <8>; + cap-mmc-highspeed; + disable-wp; + mmc-pwrseq = <&emmc_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&emmc_clk &emmc_cmd &emmc_bus8>; + status = "okay"; +}; + +&gmac { + phy-supply = <&vcc_lan>; + phy-mode = "rmii"; + clock_in_out = "output"; + snps,reset-gpio = <&gpio3 12 0>; + snps,reset-active-low; + snps,reset-delays-us = <0 10000 1000000>; + pinctrl-names = "default"; + pinctrl-0 = <&rmii_pins>; + tx_delay = <0x30>; + rx_delay = <0x10>; + status = "ok"; +}; + +&i2c0 { + status = "okay"; + + vdd_cpu: syr827@40 { + compatible = "silergy,syr827"; + reg = <0x40>; + fcs,suspend-voltage-selector = <1>; + regulator-name = "vdd_cpu"; + regulator-enable-ramp-delay = <300>; + regulator-min-microvolt = <712500>; + regulator-max-microvolt = <1500000>; + regulator-ramp-delay = <8000>; + regulator-always-on; + regulator-boot-on; + vin-supply = <&vcc_sys>; + }; + + hym8563: hym8563@51 { + compatible = "haoyu,hym8563"; + reg = <0x51>; + #clock-cells = <0>; + clock-frequency = <32768>; + clock-output-names = "xin32k"; + /* rtc_int is not connected */ + }; +}; + +&sdio0 { + assigned-clocks = <&cru SCLK_SDIO0>; + assigned-clock-parents = <&cru PLL_CPLL>; + broken-cd; + bus-width = <4>; + cap-sd-highspeed; + cap-sdio-irq; + keep-power-in-suspend; + mmc-pwrseq = <&sdio_pwrseq>; + non-removable; + num-slots = <1>; + pinctrl-names = "default"; + pinctrl-0 = <&sdio0_clk &sdio0_cmd &sdio0_bus4>; + vmmc-supply = <&vcc_io>; + vqmmc-supply = <&vccio_wl>; + status = "okay"; +}; + +&pinctrl { + pcfg_pull_none_drv_8ma: pcfg-pull-none-drv-8ma { + bias-disable; + drive-strength = <8>; + }; + + pcfg_pull_up_drv_8ma: pcfg-pull-up-drv-8ma { + bias-pull-up; + drive-strength = <8>; + }; + + emmc { + emmc_bus8: emmc-bus8 { + rockchip,pins = <1 18 RK_FUNC_2 &pcfg_pull_up_drv_8ma>, + <1 19 RK_FUNC_2 &pcfg_pull_up_drv_8ma>, + <1 20 RK_FUNC_2 &pcfg_pull_up_drv_8ma>, + <1 21 RK_FUNC_2 &pcfg_pull_up_drv_8ma>, + <1 22 RK_FUNC_2 &pcfg_pull_up_drv_8ma>, + <1 23 RK_FUNC_2 &pcfg_pull_up_drv_8ma>, + <1 24 RK_FUNC_2 &pcfg_pull_up_drv_8ma>, + <1 25 RK_FUNC_2 &pcfg_pull_up_drv_8ma>; + }; + + emmc-clk { + rockchip,pins = <2 4 RK_FUNC_2 &pcfg_pull_none_drv_8ma>; + }; + + emmc-cmd { + rockchip,pins = <1 26 RK_FUNC_2 &pcfg_pull_up_drv_8ma>; + }; + + emmc_reset: emmc-reset { + rockchip,pins = <2 3 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + ir { + ir_int: ir-int { + rockchip,pins = <3 30 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + keys { + pwr_key: pwr-key { + rockchip,pins = <0 2 RK_FUNC_GPIO &pcfg_pull_up>; + }; + }; + + leds { + stby_pwren: stby-pwren { + rockchip,pins = <0 12 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + led_ctl: led-ctl { + rockchip,pins = <3 29 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + sdio { + wifi_reg_on: wifi-reg-on { + rockchip,pins = <3 4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + + bt_rst: bt-rst { + rockchip,pins = <3 5 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; + + usb { + host_vbus_drv: host-vbus-drv { + rockchip,pins = <0 4 RK_FUNC_GPIO &pcfg_pull_none>; + }; + }; +}; + +&saradc { + vref-supply = <&vcc_18>; + status = "okay"; +}; + +&uart2 { + status = "okay"; +}; + +&usb_host0_ehci { + status = "okay"; +}; + +&usb_otg { + dr_mode = "host"; + status = "okay"; +}; + +&wdt { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/rockchip/rk3368.dtsi b/arch/arm64/boot/dts/rockchip/rk3368.dtsi new file mode 100644 index 000000000000..a712bea3bf2c --- /dev/null +++ b/arch/arm64/boot/dts/rockchip/rk3368.dtsi @@ -0,0 +1,900 @@ +/* + * Copyright (c) 2015 Heiko Stuebner + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include + +/ { + compatible = "rockchip,rk3368"; + interrupt-parent = <&gic>; + #address-cells = <2>; + #size-cells = <2>; + + aliases { + i2c0 = &i2c0; + i2c1 = &i2c1; + i2c2 = &i2c2; + i2c3 = &i2c3; + i2c4 = &i2c4; + i2c5 = &i2c5; + serial0 = &uart0; + serial1 = &uart1; + serial2 = &uart2; + serial3 = &uart3; + serial4 = &uart4; + spi0 = &spi0; + spi1 = &spi1; + spi2 = &spi2; + }; + + cpus { + #address-cells = <0x2>; + #size-cells = <0x0>; + + cpu-map { + cluster0 { + core0 { + cpu = <&cpu_b0>; + }; + core1 { + cpu = <&cpu_b1>; + }; + core2 { + cpu = <&cpu_b2>; + }; + core3 { + cpu = <&cpu_b3>; + }; + }; + + cluster1 { + core0 { + cpu = <&cpu_l0>; + }; + core1 { + cpu = <&cpu_l1>; + }; + core2 { + cpu = <&cpu_l2>; + }; + core3 { + cpu = <&cpu_l3>; + }; + }; + }; + + idle-states { + entry-method = "arm,psci"; + + cpu_sleep: cpu-sleep-0 { + compatible = "arm,idle-state"; + arm,psci-suspend-param = <0x1010000>; + entry-latency-us = <0x3fffffff>; + exit-latency-us = <0x40000000>; + min-residency-us = <0xffffffff>; + }; + }; + + cpu_l0: cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a53", "arm,armv8"; + reg = <0x0 0x0>; + cpu-idle-states = <&cpu_sleep>; + enable-method = "psci"; + }; + + cpu_l1: cpu@1 { + device_type = "cpu"; + compatible = "arm,cortex-a53", "arm,armv8"; + reg = <0x0 0x1>; + cpu-idle-states = <&cpu_sleep>; + enable-method = "psci"; + }; + + cpu_l2: cpu@2 { + device_type = "cpu"; + compatible = "arm,cortex-a53", "arm,armv8"; + reg = <0x0 0x2>; + cpu-idle-states = <&cpu_sleep>; + enable-method = "psci"; + }; + + cpu_l3: cpu@3 { + device_type = "cpu"; + compatible = "arm,cortex-a53", "arm,armv8"; + reg = <0x0 0x3>; + cpu-idle-states = <&cpu_sleep>; + enable-method = "psci"; + }; + + cpu_b0: cpu@100 { + device_type = "cpu"; + compatible = "arm,cortex-a53", "arm,armv8"; + reg = <0x0 0x100>; + cpu-idle-states = <&cpu_sleep>; + enable-method = "psci"; + }; + + cpu_b1: cpu@101 { + device_type = "cpu"; + compatible = "arm,cortex-a53", "arm,armv8"; + reg = <0x0 0x101>; + cpu-idle-states = <&cpu_sleep>; + enable-method = "psci"; + }; + + cpu_b2: cpu@102 { + device_type = "cpu"; + compatible = "arm,cortex-a53", "arm,armv8"; + reg = <0x0 0x102>; + cpu-idle-states = <&cpu_sleep>; + enable-method = "psci"; + }; + + cpu_b3: cpu@103 { + device_type = "cpu"; + compatible = "arm,cortex-a53", "arm,armv8"; + reg = <0x0 0x103>; + cpu-idle-states = <&cpu_sleep>; + enable-method = "psci"; + }; + }; + + arm-pmu { + compatible = "arm,armv8-pmuv3"; + interrupts = , + , + , + , + , + , + , + ; + interrupt-affinity = <&cpu_l0>, <&cpu_l1>, <&cpu_l2>, + <&cpu_l3>, <&cpu_b0>, <&cpu_b1>, + <&cpu_b2>, <&cpu_b3>; + }; + + psci { + compatible = "arm,psci-0.2"; + method = "smc"; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupts = , + , + , + ; + }; + + xin24m: oscillator { + compatible = "fixed-clock"; + clock-frequency = <24000000>; + clock-output-names = "xin24m"; + #clock-cells = <0>; + }; + + sdmmc: dwmmc@ff0c0000 { + compatible = "rockchip,rk3368-dw-mshc", "rockchip,rk3288-dw-mshc"; + reg = <0x0 0xff0c0000 0x0 0x4000>; + clock-freq-min-max = <400000 150000000>; + clocks = <&cru HCLK_SDMMC>, <&cru SCLK_SDMMC>; + clock-names = "biu", "ciu"; + fifo-depth = <0x100>; + interrupts = ; + status = "disabled"; + }; + + sdio0: dwmmc@ff0d0000 { + compatible = "rockchip,rk3368-dw-mshc", "rockchip,rk3288-dw-mshc"; + reg = <0x0 0xff0d0000 0x0 0x4000>; + clock-freq-min-max = <400000 150000000>; + clocks = <&cru HCLK_SDIO0>, <&cru SCLK_SDIO0>, + <&cru SCLK_SDIO0_DRV>, <&cru SCLK_SDIO0_SAMPLE>; + clock-names = "biu", "ciu", "ciu_drv", "ciu_sample"; + fifo-depth = <0x100>; + interrupts = ; + status = "disabled"; + }; + + emmc: dwmmc@ff0f0000 { + compatible = "rockchip,rk3368-dw-mshc", "rockchip,rk3288-dw-mshc"; + reg = <0x0 0xff0f0000 0x0 0x4000>; + clock-freq-min-max = <400000 150000000>; + clocks = <&cru HCLK_EMMC>, <&cru SCLK_EMMC>; + clock-names = "biu", "ciu"; + fifo-depth = <0x100>; + interrupts = ; + status = "disabled"; + }; + + saradc: saradc@ff100000 { + compatible = "rockchip,saradc"; + reg = <0x0 0xff100000 0x0 0x100>; + interrupts = ; + #io-channel-cells = <1>; + clocks = <&cru SCLK_SARADC>, <&cru PCLK_SARADC>; + clock-names = "saradc", "apb_pclk"; + status = "disabled"; + }; + + spi0: spi@ff110000 { + compatible = "rockchip,rk3368-spi", "rockchip,rk3066-spi"; + reg = <0x0 0xff110000 0x0 0x1000>; + clocks = <&cru SCLK_SPI0>, <&cru PCLK_SPI0>; + clock-names = "spiclk", "apb_pclk"; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&spi0_clk &spi0_tx &spi0_rx &spi0_cs0>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi1: spi@ff120000 { + compatible = "rockchip,rk3368-spi", "rockchip,rk3066-spi"; + reg = <0x0 0xff120000 0x0 0x1000>; + clocks = <&cru SCLK_SPI1>, <&cru PCLK_SPI1>; + clock-names = "spiclk", "apb_pclk"; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&spi1_clk &spi1_tx &spi1_rx &spi1_cs0>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + spi2: spi@ff130000 { + compatible = "rockchip,rk3368-spi", "rockchip,rk3066-spi"; + reg = <0x0 0xff130000 0x0 0x1000>; + clocks = <&cru SCLK_SPI2>, <&cru PCLK_SPI2>; + clock-names = "spiclk", "apb_pclk"; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&spi2_clk &spi2_tx &spi2_rx &spi2_cs0>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c1: i2c@ff140000 { + compatible = "rockchip,rk3368-i2c", "rockchip,rk3288-i2c"; + reg = <0x0 0xff140000 0x0 0x1000>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clock-names = "i2c"; + clocks = <&cru PCLK_I2C1>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_xfer>; + status = "disabled"; + }; + + i2c3: i2c@ff150000 { + compatible = "rockchip,rk3368-i2c", "rockchip,rk3288-i2c"; + reg = <0x0 0xff150000 0x0 0x1000>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clock-names = "i2c"; + clocks = <&cru PCLK_I2C3>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c3_xfer>; + status = "disabled"; + }; + + i2c4: i2c@ff160000 { + compatible = "rockchip,rk3368-i2c", "rockchip,rk3288-i2c"; + reg = <0x0 0xff160000 0x0 0x1000>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clock-names = "i2c"; + clocks = <&cru PCLK_I2C4>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c4_xfer>; + status = "disabled"; + }; + + i2c5: i2c@ff170000 { + compatible = "rockchip,rk3368-i2c", "rockchip,rk3288-i2c"; + reg = <0x0 0xff170000 0x0 0x1000>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clock-names = "i2c"; + clocks = <&cru PCLK_I2C5>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c5_xfer>; + status = "disabled"; + }; + + uart0: serial@ff180000 { + compatible = "rockchip,rk3368-uart", "snps,dw-apb-uart"; + reg = <0x0 0xff180000 0x0 0x100>; + clock-frequency = <24000000>; + clocks = <&cru SCLK_UART0>, <&cru PCLK_UART0>; + clock-names = "baudclk", "apb_pclk"; + interrupts = ; + reg-shift = <2>; + reg-io-width = <4>; + status = "disabled"; + }; + + uart1: serial@ff190000 { + compatible = "rockchip,rk3368-uart", "snps,dw-apb-uart"; + reg = <0x0 0xff190000 0x0 0x100>; + clock-frequency = <24000000>; + clocks = <&cru SCLK_UART1>, <&cru PCLK_UART1>; + clock-names = "baudclk", "apb_pclk"; + interrupts = ; + reg-shift = <2>; + reg-io-width = <4>; + status = "disabled"; + }; + + uart3: serial@ff1b0000 { + compatible = "rockchip,rk3368-uart", "snps,dw-apb-uart"; + reg = <0x0 0xff1b0000 0x0 0x100>; + clock-frequency = <24000000>; + clocks = <&cru SCLK_UART3>, <&cru PCLK_UART3>; + clock-names = "baudclk", "apb_pclk"; + interrupts = ; + reg-shift = <2>; + reg-io-width = <4>; + status = "disabled"; + }; + + uart4: serial@ff1c0000 { + compatible = "rockchip,rk3368-uart", "snps,dw-apb-uart"; + reg = <0x0 0xff1c0000 0x0 0x100>; + clock-frequency = <24000000>; + clocks = <&cru SCLK_UART4>, <&cru PCLK_UART4>; + clock-names = "baudclk", "apb_pclk"; + interrupts = ; + reg-shift = <2>; + reg-io-width = <4>; + status = "disabled"; + }; + + gmac: ethernet@ff290000 { + compatible = "rockchip,rk3368-gmac"; + reg = <0x0 0xff290000 0x0 0x10000>; + interrupts = ; + interrupt-names = "macirq"; + rockchip,grf = <&grf>; + clocks = <&cru SCLK_MAC>, + <&cru SCLK_MAC_RX>, <&cru SCLK_MAC_TX>, + <&cru SCLK_MACREF>, <&cru SCLK_MACREF_OUT>, + <&cru ACLK_GMAC>, <&cru PCLK_GMAC>; + clock-names = "stmmaceth", + "mac_clk_rx", "mac_clk_tx", + "clk_mac_ref", "clk_mac_refout", + "aclk_mac", "pclk_mac"; + status = "disabled"; + }; + + usb_host0_ehci: usb@ff500000 { + compatible = "generic-ehci"; + reg = <0x0 0xff500000 0x0 0x100>; + interrupts = ; + clocks = <&cru HCLK_HOST0>; + clock-names = "usbhost"; + status = "disabled"; + }; + + usb_otg: usb@ff580000 { + compatible = "rockchip,rk3368-usb", "rockchip,rk3066-usb", + "snps,dwc2"; + reg = <0x0 0xff580000 0x0 0x40000>; + interrupts = ; + clocks = <&cru HCLK_OTG0>; + clock-names = "otg"; + dr_mode = "otg"; + g-np-tx-fifo-size = <16>; + g-rx-fifo-size = <275>; + g-tx-fifo-size = <256 128 128 64 64 32>; + g-use-dma; + status = "disabled"; + }; + + i2c0: i2c@ff650000 { + compatible = "rockchip,rk3368-i2c", "rockchip,rk3288-i2c"; + reg = <0x0 0xff650000 0x0 0x1000>; + clocks = <&cru PCLK_I2C0>; + clock-names = "i2c"; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_xfer>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c2: i2c@ff660000 { + compatible = "rockchip,rk3368-i2c", "rockchip,rk3288-i2c"; + reg = <0x0 0xff660000 0x0 0x1000>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + clock-names = "i2c"; + clocks = <&cru PCLK_I2C2>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c2_xfer>; + status = "disabled"; + }; + + uart2: serial@ff690000 { + compatible = "rockchip,rk3368-uart", "snps,dw-apb-uart"; + reg = <0x0 0xff690000 0x0 0x100>; + clocks = <&cru SCLK_UART2>, <&cru PCLK_UART2>; + clock-names = "baudclk", "apb_pclk"; + interrupts = ; + pinctrl-names = "default"; + pinctrl-0 = <&uart2_xfer>; + reg-shift = <2>; + reg-io-width = <4>; + status = "disabled"; + }; + + pmugrf: syscon@ff738000 { + compatible = "rockchip,rk3368-pmugrf", "syscon"; + reg = <0x0 0xff738000 0x0 0x1000>; + }; + + cru: clock-controller@ff760000 { + compatible = "rockchip,rk3368-cru"; + reg = <0x0 0xff760000 0x0 0x1000>; + rockchip,grf = <&grf>; + #clock-cells = <1>; + #reset-cells = <1>; + }; + + grf: syscon@ff770000 { + compatible = "rockchip,rk3368-grf", "syscon"; + reg = <0x0 0xff770000 0x0 0x1000>; + }; + + wdt: watchdog@ff800000 { + compatible = "rockchip,rk3368-wdt", "snps,dw-wdt"; + reg = <0x0 0xff800000 0x0 0x100>; + clocks = <&cru PCLK_WDT>; + interrupts = ; + status = "disabled"; + }; + + gic: interrupt-controller@ffb71000 { + compatible = "arm,gic-400"; + interrupt-controller; + #interrupt-cells = <3>; + #address-cells = <0>; + + reg = <0x0 0xffb71000 0x0 0x1000>, + <0x0 0xffb72000 0x0 0x1000>, + <0x0 0xffb74000 0x0 0x2000>, + <0x0 0xffb76000 0x0 0x2000>; + interrupts = ; + }; + + pinctrl: pinctrl { + compatible = "rockchip,rk3368-pinctrl"; + rockchip,grf = <&grf>; + rockchip,pmu = <&pmugrf>; + #address-cells = <0x2>; + #size-cells = <0x2>; + ranges; + + gpio0: gpio0@ff750000 { + compatible = "rockchip,gpio-bank"; + reg = <0x0 0xff750000 0x0 0x100>; + clocks = <&cru PCLK_GPIO0>; + interrupts = ; + + gpio-controller; + #gpio-cells = <0x2>; + + interrupt-controller; + #interrupt-cells = <0x2>; + }; + + gpio1: gpio1@ff780000 { + compatible = "rockchip,gpio-bank"; + reg = <0x0 0xff780000 0x0 0x100>; + clocks = <&cru PCLK_GPIO1>; + interrupts = ; + + gpio-controller; + #gpio-cells = <0x2>; + + interrupt-controller; + #interrupt-cells = <0x2>; + }; + + gpio2: gpio2@ff790000 { + compatible = "rockchip,gpio-bank"; + reg = <0x0 0xff790000 0x0 0x100>; + clocks = <&cru PCLK_GPIO2>; + interrupts = ; + + gpio-controller; + #gpio-cells = <0x2>; + + interrupt-controller; + #interrupt-cells = <0x2>; + }; + + gpio3: gpio3@ff7a0000 { + compatible = "rockchip,gpio-bank"; + reg = <0x0 0xff7a0000 0x0 0x100>; + clocks = <&cru PCLK_GPIO3>; + interrupts = ; + + gpio-controller; + #gpio-cells = <0x2>; + + interrupt-controller; + #interrupt-cells = <0x2>; + }; + + pcfg_pull_up: pcfg-pull-up { + bias-pull-up; + }; + + pcfg_pull_down: pcfg-pull-down { + bias-pull-down; + }; + + pcfg_pull_none: pcfg-pull-none { + bias-disable; + }; + + pcfg_pull_none_12ma: pcfg-pull-none-12ma { + bias-disable; + drive-strength = <12>; + }; + + emmc { + emmc_clk: emmc-clk { + rockchip,pins = <2 4 RK_FUNC_2 &pcfg_pull_none>; + }; + + emmc_cmd: emmc-cmd { + rockchip,pins = <1 26 RK_FUNC_2 &pcfg_pull_up>; + }; + + emmc_pwr: emmc-pwr { + rockchip,pins = <1 27 RK_FUNC_2 &pcfg_pull_up>; + }; + + emmc_bus1: emmc-bus1 { + rockchip,pins = <1 18 RK_FUNC_2 &pcfg_pull_up>; + }; + + emmc_bus4: emmc-bus4 { + rockchip,pins = <1 18 RK_FUNC_2 &pcfg_pull_up>, + <1 19 RK_FUNC_2 &pcfg_pull_up>, + <1 20 RK_FUNC_2 &pcfg_pull_up>, + <1 21 RK_FUNC_2 &pcfg_pull_up>; + }; + + emmc_bus8: emmc-bus8 { + rockchip,pins = <1 18 RK_FUNC_2 &pcfg_pull_up>, + <1 19 RK_FUNC_2 &pcfg_pull_up>, + <1 20 RK_FUNC_2 &pcfg_pull_up>, + <1 21 RK_FUNC_2 &pcfg_pull_up>, + <1 22 RK_FUNC_2 &pcfg_pull_up>, + <1 23 RK_FUNC_2 &pcfg_pull_up>, + <1 24 RK_FUNC_2 &pcfg_pull_up>, + <1 25 RK_FUNC_2 &pcfg_pull_up>; + }; + }; + + gmac { + rgmii_pins: rgmii-pins { + rockchip,pins = <3 22 RK_FUNC_1 &pcfg_pull_none>, + <3 24 RK_FUNC_1 &pcfg_pull_none>, + <3 19 RK_FUNC_1 &pcfg_pull_none>, + <3 8 RK_FUNC_1 &pcfg_pull_none_12ma>, + <3 9 RK_FUNC_1 &pcfg_pull_none_12ma>, + <3 10 RK_FUNC_1 &pcfg_pull_none_12ma>, + <3 14 RK_FUNC_1 &pcfg_pull_none_12ma>, + <3 28 RK_FUNC_1 &pcfg_pull_none_12ma>, + <3 13 RK_FUNC_1 &pcfg_pull_none_12ma>, + <3 15 RK_FUNC_1 &pcfg_pull_none>, + <3 16 RK_FUNC_1 &pcfg_pull_none>, + <3 17 RK_FUNC_1 &pcfg_pull_none>, + <3 18 RK_FUNC_1 &pcfg_pull_none>, + <3 25 RK_FUNC_1 &pcfg_pull_none>, + <3 20 RK_FUNC_1 &pcfg_pull_none>; + }; + + rmii_pins: rmii-pins { + rockchip,pins = <3 22 RK_FUNC_1 &pcfg_pull_none>, + <3 24 RK_FUNC_1 &pcfg_pull_none>, + <3 19 RK_FUNC_1 &pcfg_pull_none>, + <3 8 RK_FUNC_1 &pcfg_pull_none_12ma>, + <3 9 RK_FUNC_1 &pcfg_pull_none_12ma>, + <3 13 RK_FUNC_1 &pcfg_pull_none_12ma>, + <3 15 RK_FUNC_1 &pcfg_pull_none>, + <3 16 RK_FUNC_1 &pcfg_pull_none>, + <3 20 RK_FUNC_1 &pcfg_pull_none>, + <3 21 RK_FUNC_1 &pcfg_pull_none>; + }; + }; + + i2c0 { + i2c0_xfer: i2c0-xfer { + rockchip,pins = <0 6 RK_FUNC_1 &pcfg_pull_none>, + <0 7 RK_FUNC_1 &pcfg_pull_none>; + }; + }; + + i2c1 { + i2c1_xfer: i2c1-xfer { + rockchip,pins = <2 21 RK_FUNC_1 &pcfg_pull_none>, + <2 22 RK_FUNC_1 &pcfg_pull_none>; + }; + }; + + i2c2 { + i2c2_xfer: i2c2-xfer { + rockchip,pins = <0 9 RK_FUNC_2 &pcfg_pull_none>, + <3 31 RK_FUNC_2 &pcfg_pull_none>; + }; + }; + + i2c3 { + i2c3_xfer: i2c3-xfer { + rockchip,pins = <1 16 RK_FUNC_1 &pcfg_pull_none>, + <1 17 RK_FUNC_1 &pcfg_pull_none>; + }; + }; + + i2c4 { + i2c4_xfer: i2c4-xfer { + rockchip,pins = <3 24 RK_FUNC_2 &pcfg_pull_none>, + <3 25 RK_FUNC_2 &pcfg_pull_none>; + }; + }; + + i2c5 { + i2c5_xfer: i2c5-xfer { + rockchip,pins = <3 26 RK_FUNC_2 &pcfg_pull_none>, + <3 27 RK_FUNC_2 &pcfg_pull_none>; + }; + }; + + sdio0 { + sdio0_bus1: sdio0-bus1 { + rockchip,pins = <2 28 RK_FUNC_1 &pcfg_pull_up>; + }; + + sdio0_bus4: sdio0-bus4 { + rockchip,pins = <2 28 RK_FUNC_1 &pcfg_pull_up>, + <2 29 RK_FUNC_1 &pcfg_pull_up>, + <2 30 RK_FUNC_1 &pcfg_pull_up>, + <2 31 RK_FUNC_1 &pcfg_pull_up>; + }; + + sdio0_cmd: sdio0-cmd { + rockchip,pins = <3 0 RK_FUNC_1 &pcfg_pull_up>; + }; + + sdio0_clk: sdio0-clk { + rockchip,pins = <3 1 RK_FUNC_1 &pcfg_pull_none>; + }; + + sdio0_cd: sdio0-cd { + rockchip,pins = <3 2 RK_FUNC_1 &pcfg_pull_up>; + }; + + sdio0_wp: sdio0-wp { + rockchip,pins = <3 3 RK_FUNC_1 &pcfg_pull_up>; + }; + + sdio0_pwr: sdio0-pwr { + rockchip,pins = <3 4 RK_FUNC_1 &pcfg_pull_up>; + }; + + sdio0_bkpwr: sdio0-bkpwr { + rockchip,pins = <3 5 RK_FUNC_1 &pcfg_pull_up>; + }; + + sdio0_int: sdio0-int { + rockchip,pins = <3 6 RK_FUNC_1 &pcfg_pull_up>; + }; + }; + + sdmmc { + sdmmc_clk: sdmmc-clk { + rockchip,pins = <2 9 RK_FUNC_1 &pcfg_pull_none>; + }; + + sdmmc_cmd: sdmmc-cmd { + rockchip,pins = <2 10 RK_FUNC_1 &pcfg_pull_up>; + }; + + sdmmc_cd: sdmcc-cd { + rockchip,pins = <2 11 RK_FUNC_1 &pcfg_pull_up>; + }; + + sdmmc_bus1: sdmmc-bus1 { + rockchip,pins = <2 5 RK_FUNC_1 &pcfg_pull_up>; + }; + + sdmmc_bus4: sdmmc-bus4 { + rockchip,pins = <2 5 RK_FUNC_1 &pcfg_pull_up>, + <2 6 RK_FUNC_1 &pcfg_pull_up>, + <2 7 RK_FUNC_1 &pcfg_pull_up>, + <2 8 RK_FUNC_1 &pcfg_pull_up>; + }; + }; + + spi0 { + spi0_clk: spi0-clk { + rockchip,pins = <1 29 RK_FUNC_2 &pcfg_pull_up>; + }; + spi0_cs0: spi0-cs0 { + rockchip,pins = <1 24 RK_FUNC_3 &pcfg_pull_up>; + }; + spi0_cs1: spi0-cs1 { + rockchip,pins = <1 25 RK_FUNC_3 &pcfg_pull_up>; + }; + spi0_tx: spi0-tx { + rockchip,pins = <1 23 RK_FUNC_3 &pcfg_pull_up>; + }; + spi0_rx: spi0-rx { + rockchip,pins = <1 22 RK_FUNC_3 &pcfg_pull_up>; + }; + }; + + spi1 { + spi1_clk: spi1-clk { + rockchip,pins = <1 14 RK_FUNC_2 &pcfg_pull_up>; + }; + spi1_cs0: spi1-cs0 { + rockchip,pins = <1 15 RK_FUNC_2 &pcfg_pull_up>; + }; + spi1_cs1: spi1-cs1 { + rockchip,pins = <3 28 RK_FUNC_2 &pcfg_pull_up>; + }; + spi1_rx: spi1-rx { + rockchip,pins = <1 16 RK_FUNC_2 &pcfg_pull_up>; + }; + spi1_tx: spi1-tx { + rockchip,pins = <1 17 RK_FUNC_2 &pcfg_pull_up>; + }; + }; + + spi2 { + spi2_clk: spi2-clk { + rockchip,pins = <0 12 RK_FUNC_2 &pcfg_pull_up>; + }; + spi2_cs0: spi2-cs0 { + rockchip,pins = <0 13 RK_FUNC_2 &pcfg_pull_up>; + }; + spi2_rx: spi2-rx { + rockchip,pins = <0 10 RK_FUNC_2 &pcfg_pull_up>; + }; + spi2_tx: spi2-tx { + rockchip,pins = <0 11 RK_FUNC_2 &pcfg_pull_up>; + }; + }; + + uart0 { + uart0_xfer: uart0-xfer { + rockchip,pins = <2 24 RK_FUNC_1 &pcfg_pull_up>, + <2 25 RK_FUNC_1 &pcfg_pull_none>; + }; + + uart0_cts: uart0-cts { + rockchip,pins = <2 26 RK_FUNC_1 &pcfg_pull_none>; + }; + + uart0_rts: uart0-rts { + rockchip,pins = <2 27 RK_FUNC_1 &pcfg_pull_none>; + }; + }; + + uart1 { + uart1_xfer: uart1-xfer { + rockchip,pins = <0 20 RK_FUNC_3 &pcfg_pull_up>, + <0 21 RK_FUNC_3 &pcfg_pull_none>; + }; + + uart1_cts: uart1-cts { + rockchip,pins = <0 22 RK_FUNC_3 &pcfg_pull_none>; + }; + + uart1_rts: uart1-rts { + rockchip,pins = <0 23 RK_FUNC_3 &pcfg_pull_none>; + }; + }; + + uart2 { + uart2_xfer: uart2-xfer { + rockchip,pins = <2 6 RK_FUNC_2 &pcfg_pull_up>, + <2 5 RK_FUNC_2 &pcfg_pull_none>; + }; + /* no rts / cts for uart2 */ + }; + + uart3 { + uart3_xfer: uart3-xfer { + rockchip,pins = <3 29 RK_FUNC_2 &pcfg_pull_up>, + <3 30 RK_FUNC_3 &pcfg_pull_none>; + }; + + uart3_cts: uart3-cts { + rockchip,pins = <3 16 RK_FUNC_2 &pcfg_pull_none>; + }; + + uart3_rts: uart3-rts { + rockchip,pins = <3 17 RK_FUNC_2 &pcfg_pull_none>; + }; + }; + + uart4 { + uart4_xfer: uart4-xfer { + rockchip,pins = <0 27 RK_FUNC_3 &pcfg_pull_up>, + <0 26 RK_FUNC_3 &pcfg_pull_none>; + }; + + uart4_cts: uart4-cts { + rockchip,pins = <0 24 RK_FUNC_3 &pcfg_pull_none>; + }; + + uart4_rts: uart4-rts { + rockchip,pins = <0 25 RK_FUNC_3 &pcfg_pull_none>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/sprd/sc9836.dtsi b/arch/arm64/boot/dts/sprd/sc9836.dtsi index ee34e1a36e03..63894c456969 100644 --- a/arch/arm64/boot/dts/sprd/sc9836.dtsi +++ b/arch/arm64/boot/dts/sprd/sc9836.dtsi @@ -16,28 +16,28 @@ cpus { #address-cells = <2>; #size-cells = <0>; - cpu@0 { + cpu0: cpu@0 { device_type = "cpu"; compatible = "arm,cortex-a53", "arm,armv8"; reg = <0x0 0x0>; enable-method = "psci"; }; - cpu@1 { + cpu1: cpu@1 { device_type = "cpu"; compatible = "arm,cortex-a53", "arm,armv8"; reg = <0x0 0x1>; enable-method = "psci"; }; - cpu@2 { + cpu2: cpu@2 { device_type = "cpu"; compatible = "arm,cortex-a53", "arm,armv8"; reg = <0x0 0x2>; enable-method = "psci"; }; - cpu@3 { + cpu3: cpu@3 { device_type = "cpu"; compatible = "arm,cortex-a53", "arm,armv8"; reg = <0x0 0x3>; @@ -75,14 +75,103 @@ funnel_out_port0: endpoint { }; }; - /* funnel input port 0~3 is reserved for ETMs */ + /* funnel input port 0-4 */ port@1 { + reg = <0>; + funnel_in_port0: endpoint { + slave-mode; + remote-endpoint = <&etm0_out>; + }; + }; + + port@2 { + reg = <1>; + funnel_in_port1: endpoint { + slave-mode; + remote-endpoint = <&etm1_out>; + }; + }; + + port@3 { + reg = <2>; + funnel_in_port2: endpoint { + slave-mode; + remote-endpoint = <&etm2_out>; + }; + }; + + port@4 { + reg = <3>; + funnel_in_port3: endpoint { + slave-mode; + remote-endpoint = <&etm3_out>; + }; + }; + + port@5 { reg = <4>; funnel_in_port4: endpoint { slave-mode; remote-endpoint = <&stm_out>; }; }; + /* Other input ports aren't connected to anyone */ + }; + }; + + etm@10440000 { + compatible = "arm,coresight-etm4x", "arm,primecell"; + reg = <0 0x10440000 0 0x1000>; + + cpu = <&cpu0>; + clocks = <&clk26mhz>; + clock-names = "apb_pclk"; + port { + etm0_out: endpoint { + remote-endpoint = <&funnel_in_port0>; + }; + }; + }; + + etm@10540000 { + compatible = "arm,coresight-etm4x", "arm,primecell"; + reg = <0 0x10540000 0 0x1000>; + + cpu = <&cpu1>; + clocks = <&clk26mhz>; + clock-names = "apb_pclk"; + port { + etm1_out: endpoint { + remote-endpoint = <&funnel_in_port1>; + }; + }; + }; + + etm@10640000 { + compatible = "arm,coresight-etm4x", "arm,primecell"; + reg = <0 0x10640000 0 0x1000>; + + cpu = <&cpu2>; + clocks = <&clk26mhz>; + clock-names = "apb_pclk"; + port { + etm2_out: endpoint { + remote-endpoint = <&funnel_in_port2>; + }; + }; + }; + + etm@10740000 { + compatible = "arm,coresight-etm4x", "arm,primecell"; + reg = <0 0x10740000 0 0x1000>; + + cpu = <&cpu3>; + clocks = <&clk26mhz>; + clock-names = "apb_pclk"; + port { + etm3_out: endpoint { + remote-endpoint = <&funnel_in_port3>; + }; }; }; diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-ep108.dts b/arch/arm64/boot/dts/xilinx/zynqmp-ep108.dts index 0a3f40ecd06d..ce5d848251fa 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp-ep108.dts +++ b/arch/arm64/boot/dts/xilinx/zynqmp-ep108.dts @@ -32,6 +32,10 @@ memory { }; }; +&can0 { + status = "okay"; +}; + &gem0 { status = "okay"; phy-handle = <&phy0>; @@ -42,6 +46,91 @@ phy0: phy@0{ }; }; +&gpio { + status = "okay"; +}; + +&i2c0 { + status = "okay"; + clock-frequency = <400000>; + eeprom@54 { + compatible = "at,24c64"; + reg = <0x54>; + }; +}; + +&i2c1 { + status = "okay"; + clock-frequency = <400000>; + eeprom@55 { + compatible = "at,24c64"; + reg = <0x55>; + }; +}; + +&sata { + status = "okay"; + ceva,broken-gen2; +}; + +&sdhci0 { + status = "okay"; +}; + +&sdhci1 { + status = "okay"; +}; + +&spi0 { + status = "okay"; + num-cs = <1>; + spi0_flash0: spi0_flash0@0 { + compatible = "m25p80"; + #address-cells = <1>; + #size-cells = <1>; + spi-max-frequency = <50000000>; + reg = <0>; + + spi0_flash0@00000000 { + label = "spi0_flash0"; + reg = <0x0 0x100000>; + }; + }; +}; + +&spi1 { + status = "okay"; + num-cs = <1>; + spi1_flash0: spi1_flash0@0 { + compatible = "m25p80"; + #address-cells = <1>; + #size-cells = <1>; + spi-max-frequency = <50000000>; + reg = <0>; + + spi1_flash0@00000000 { + label = "spi1_flash0"; + reg = <0x0 0x100000>; + }; + }; +}; + &uart0 { status = "okay"; }; + +&usb0 { + status = "okay"; + dr_mode = "peripheral"; + maximum-speed = "high-speed"; +}; + +&usb1 { + status = "okay"; + dr_mode = "host"; + maximum-speed = "high-speed"; +}; + +&watchdog0 { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/xilinx/zynqmp.dtsi b/arch/arm64/boot/dts/xilinx/zynqmp.dtsi index 11e0b00045cf..857eda5c7217 100644 --- a/arch/arm64/boot/dts/xilinx/zynqmp.dtsi +++ b/arch/arm64/boot/dts/xilinx/zynqmp.dtsi @@ -96,12 +96,193 @@ amba { #size-cells = <1>; ranges; + can0: can@ff060000 { + compatible = "xlnx,zynq-can-1.0"; + status = "disabled"; + clocks = <&misc_clk &misc_clk>; + clock-names = "can_clk", "pclk"; + reg = <0x0 0xff060000 0x1000>; + interrupts = <0 23 4>; + interrupt-parent = <&gic>; + tx-fifo-depth = <0x40>; + rx-fifo-depth = <0x40>; + }; + + can1: can@ff070000 { + compatible = "xlnx,zynq-can-1.0"; + status = "disabled"; + clocks = <&misc_clk &misc_clk>; + clock-names = "can_clk", "pclk"; + reg = <0x0 0xff070000 0x1000>; + interrupts = <0 24 4>; + interrupt-parent = <&gic>; + tx-fifo-depth = <0x40>; + rx-fifo-depth = <0x40>; + }; + misc_clk: misc_clk { compatible = "fixed-clock"; #clock-cells = <0>; clock-frequency = <25000000>; }; + gpio: gpio@ff0a0000 { + compatible = "xlnx,zynqmp-gpio-1.0"; + status = "disabled"; + #gpio-cells = <0x2>; + clocks = <&misc_clk>; + interrupt-parent = <&gic>; + interrupts = <0 16 4>; + reg = <0x0 0xff0a0000 0x1000>; + }; + + gem0: ethernet@ff0b0000 { + compatible = "cdns,gem"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 57 4>, <0 57 4>; + reg = <0x0 0xff0b0000 0x1000>; + clock-names = "pclk", "hclk", "tx_clk"; + clocks = <&misc_clk>, <&misc_clk>, <&misc_clk>; + #address-cells = <1>; + #size-cells = <0>; + }; + + gem1: ethernet@ff0c0000 { + compatible = "cdns,gem"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 59 4>, <0 59 4>; + reg = <0x0 0xff0c0000 0x1000>; + clock-names = "pclk", "hclk", "tx_clk"; + clocks = <&misc_clk>, <&misc_clk>, <&misc_clk>; + #address-cells = <1>; + #size-cells = <0>; + }; + + gem2: ethernet@ff0d0000 { + compatible = "cdns,gem"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 61 4>, <0 61 4>; + reg = <0x0 0xff0d0000 0x1000>; + clock-names = "pclk", "hclk", "tx_clk"; + clocks = <&misc_clk>, <&misc_clk>, <&misc_clk>; + #address-cells = <1>; + #size-cells = <0>; + }; + + gem3: ethernet@ff0e0000 { + compatible = "cdns,gem"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 63 4>, <0 63 4>; + reg = <0x0 0xff0e0000 0x1000>; + clock-names = "pclk", "hclk", "tx_clk"; + clocks = <&misc_clk>, <&misc_clk>, <&misc_clk>; + #address-cells = <1>; + #size-cells = <0>; + }; + + i2c_clk: i2c_clk { + compatible = "fixed-clock"; + #clock-cells = <0x0>; + clock-frequency = <111111111>; + }; + + i2c0: i2c@ff020000 { + compatible = "cdns,i2c-r1p10"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 17 4>; + reg = <0x0 0xff020000 0x1000>; + clocks = <&i2c_clk>; + #address-cells = <1>; + #size-cells = <0>; + }; + + i2c1: i2c@ff030000 { + compatible = "cdns,i2c-r1p10"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 18 4>; + reg = <0x0 0xff030000 0x1000>; + clocks = <&i2c_clk>; + #address-cells = <1>; + #size-cells = <0>; + }; + + sata_clk: sata_clk { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <75000000>; + }; + + sata: ahci@fd0c0000 { + compatible = "ceva,ahci-1v84"; + status = "disabled"; + reg = <0x0 0xfd0c0000 0x2000>; + interrupt-parent = <&gic>; + interrupts = <0 133 4>; + clocks = <&sata_clk>; + }; + + sdhci0: sdhci@ff160000 { + compatible = "arasan,sdhci-8.9a"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 48 4>; + reg = <0x0 0xff160000 0x1000>; + clock-names = "clk_xin", "clk_ahb"; + clocks = <&misc_clk>, <&misc_clk>; + }; + + sdhci1: sdhci@ff170000 { + compatible = "arasan,sdhci-8.9a"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 49 4>; + reg = <0x0 0xff170000 0x1000>; + clock-names = "clk_xin", "clk_ahb"; + clocks = <&misc_clk>, <&misc_clk>; + }; + + smmu: smmu@fd800000 { + compatible = "arm,mmu-500"; + reg = <0x0 0xfd800000 0x20000>; + #global-interrupts = <1>; + interrupt-parent = <&gic>; + interrupts = <0 157 4>, + <0 157 4>, <0 157 4>, <0 157 4>, <0 157 4>, + <0 157 4>, <0 157 4>, <0 157 4>, <0 157 4>, + <0 157 4>, <0 157 4>, <0 157 4>, <0 157 4>, + <0 157 4>, <0 157 4>, <0 157 4>, <0 157 4>; + }; + + spi0: spi@ff040000 { + compatible = "cdns,spi-r1p6"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 19 4>; + reg = <0x0 0xff040000 0x1000>; + clock-names = "ref_clk", "pclk"; + clocks = <&misc_clk &misc_clk>; + #address-cells = <1>; + #size-cells = <0>; + }; + + spi1: spi@ff050000 { + compatible = "cdns,spi-r1p6"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 20 4>; + reg = <0x0 0xff050000 0x1000>; + clock-names = "ref_clk", "pclk"; + clocks = <&misc_clk &misc_clk>; + #address-cells = <1>; + #size-cells = <0>; + }; + ttc0: timer@ff110000 { compatible = "cdns,ttc"; status = "disabled"; @@ -162,132 +343,22 @@ uart1: serial@ff010000 { clocks = <&misc_clk &misc_clk>; }; - gpio: gpio@ff0a0000 { - compatible = "xlnx,zynq-gpio-1.0"; - status = "disabled"; - #gpio-cells = <0x2>; - clocks = <&misc_clk>; - interrupt-parent = <&gic>; - interrupts = <0 16 4>; - reg = <0x0 0xff0a0000 0x1000>; - }; - - gem0: ethernet@ff0b0000 { - compatible = "cdns,gem"; + usb0: usb@fe200000 { + compatible = "snps,dwc3"; status = "disabled"; interrupt-parent = <&gic>; - interrupts = <0 57 4>, <0 57 4>; - reg = <0x0 0xff0b0000 0x1000>; - clock-names = "pclk", "hclk", "tx_clk"; - clocks = <&misc_clk>, <&misc_clk>, <&misc_clk>; - #address-cells = <1>; - #size-cells = <0>; - }; - - gem1: ethernet@ff0c0000 { - compatible = "cdns,gem"; - status = "disabled"; - interrupt-parent = <&gic>; - interrupts = <0 59 4>, <0 59 4>; - reg = <0x0 0xff0c0000 0x1000>; - clock-names = "pclk", "hclk", "tx_clk"; - clocks = <&misc_clk>, <&misc_clk>, <&misc_clk>; - #address-cells = <1>; - #size-cells = <0>; - }; - - gem2: ethernet@ff0d0000 { - compatible = "cdns,gem"; - status = "disabled"; - interrupt-parent = <&gic>; - interrupts = <0 61 4>, <0 61 4>; - reg = <0x0 0xff0d0000 0x1000>; - clock-names = "pclk", "hclk", "tx_clk"; - clocks = <&misc_clk>, <&misc_clk>, <&misc_clk>; - #address-cells = <1>; - #size-cells = <0>; - }; - - gem3: ethernet@ff0e0000 { - compatible = "cdns,gem"; - status = "disabled"; - interrupt-parent = <&gic>; - interrupts = <0 63 4>, <0 63 4>; - reg = <0x0 0xff0e0000 0x1000>; - clock-names = "pclk", "hclk", "tx_clk"; - clocks = <&misc_clk>, <&misc_clk>, <&misc_clk>; - #address-cells = <1>; - #size-cells = <0>; - }; - - spi0: spi@ff040000 { - compatible = "cdns,spi-r1p6"; - status = "disabled"; - interrupt-parent = <&gic>; - interrupts = <0 19 4>; - reg = <0x0 0xff040000 0x1000>; - clock-names = "ref_clk", "pclk"; - clocks = <&misc_clk &misc_clk>; - #address-cells = <1>; - #size-cells = <0>; - }; - - spi1: spi@ff050000 { - compatible = "cdns,spi-r1p6"; - status = "disabled"; - interrupt-parent = <&gic>; - interrupts = <0 20 4>; - reg = <0x0 0xff050000 0x1000>; - clock-names = "ref_clk", "pclk"; - clocks = <&misc_clk &misc_clk>; - #address-cells = <1>; - #size-cells = <0>; - }; - - i2c_clk: i2c_clk { - compatible = "fixed-clock"; - #clock-cells = <0x0>; - clock-frequency = <111111111>; - }; - - i2c0: i2c@ff020000 { - compatible = "cdns,i2c-r1p10"; - status = "disabled"; - interrupt-parent = <&gic>; - interrupts = <0 17 4>; - reg = <0x0 0xff020000 0x1000>; - clocks = <&i2c_clk>; - #address-cells = <1>; - #size-cells = <0>; - }; - - i2c1: i2c@ff030000 { - compatible = "cdns,i2c-r1p10"; - status = "disabled"; - interrupt-parent = <&gic>; - interrupts = <0 18 4>; - reg = <0x0 0xff030000 0x1000>; - clocks = <&i2c_clk>; - #address-cells = <1>; - #size-cells = <0>; - }; - - sdhci0: sdhci@ff160000 { - compatible = "arasan,sdhci-8.9a"; - status = "disabled"; - interrupt-parent = <&gic>; - interrupts = <0 48 4>; - reg = <0x0 0xff160000 0x1000>; + interrupts = <0 65 4>; + reg = <0x0 0xfe200000 0x40000>; clock-names = "clk_xin", "clk_ahb"; clocks = <&misc_clk>, <&misc_clk>; }; - sdhci1: sdhci@ff170000 { - compatible = "arasan,sdhci-8.9a"; + usb1: usb@fe300000 { + compatible = "snps,dwc3"; status = "disabled"; interrupt-parent = <&gic>; - interrupts = <0 49 4>; - reg = <0x0 0xff170000 0x1000>; + interrupts = <0 70 4>; + reg = <0x0 0xfe300000 0x40000>; clock-names = "clk_xin", "clk_ahb"; clocks = <&misc_clk>, <&misc_clk>; }; diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 4e17e7ede33d..34d71dd86781 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -31,10 +31,13 @@ CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set # CONFIG_IOSCHED_DEADLINE is not set +CONFIG_ARCH_BCM_IPROC=y +CONFIG_ARCH_BERLIN=y CONFIG_ARCH_EXYNOS7=y CONFIG_ARCH_FSL_LS2085A=y CONFIG_ARCH_HISI=y CONFIG_ARCH_MEDIATEK=y +CONFIG_ARCH_ROCKCHIP=y CONFIG_ARCH_SEATTLE=y CONFIG_ARCH_TEGRA=y CONFIG_ARCH_TEGRA_132_SOC=y @@ -102,6 +105,7 @@ CONFIG_SERIO_AMBAKMI=y CONFIG_LEGACY_PTY_COUNT=16 CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_SERIAL_8250_DW=y CONFIG_SERIAL_8250_MT6577=y CONFIG_SERIAL_AMBA_PL011=y CONFIG_SERIAL_AMBA_PL011_CONSOLE=y diff --git a/arch/arm64/include/asm/acpi.h b/arch/arm64/include/asm/acpi.h index 406485ed110a..208cec08a74f 100644 --- a/arch/arm64/include/asm/acpi.h +++ b/arch/arm64/include/asm/acpi.h @@ -12,11 +12,11 @@ #ifndef _ASM_ACPI_H #define _ASM_ACPI_H -#include #include +#include +#include #include -#include #include /* Macros for consistency checks of the GICC subtable of MADT */ diff --git a/arch/arm64/include/asm/alternative.h b/arch/arm64/include/asm/alternative.h index c385a0c4057f..d56ec0715157 100644 --- a/arch/arm64/include/asm/alternative.h +++ b/arch/arm64/include/asm/alternative.h @@ -3,6 +3,8 @@ #ifndef __ASSEMBLY__ +#include +#include #include #include #include @@ -15,7 +17,7 @@ struct alt_instr { u8 alt_len; /* size of new instruction(s), <= orig_len */ }; -void apply_alternatives_all(void); +void __init apply_alternatives_all(void); void apply_alternatives(void *start, size_t length); void free_alternatives_memory(void); @@ -40,7 +42,8 @@ void free_alternatives_memory(void); * be fixed in a binutils release posterior to 2.25.51.0.2 (anything * containing commit 4e4d08cf7399b606 or c1baaddf8861). */ -#define ALTERNATIVE(oldinstr, newinstr, feature) \ +#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled) \ + ".if "__stringify(cfg_enabled)" == 1\n" \ "661:\n\t" \ oldinstr "\n" \ "662:\n" \ @@ -53,7 +56,11 @@ void free_alternatives_memory(void); "664:\n\t" \ ".popsection\n\t" \ ".org . - (664b-663b) + (662b-661b)\n\t" \ - ".org . - (662b-661b) + (664b-663b)\n" + ".org . - (662b-661b) + (664b-663b)\n" \ + ".endif\n" + +#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \ + __ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg)) #else @@ -65,7 +72,8 @@ void free_alternatives_memory(void); .byte \alt_len .endm -.macro alternative_insn insn1 insn2 cap +.macro alternative_insn insn1, insn2, cap, enable = 1 + .if \enable 661: \insn1 662: .pushsection .altinstructions, "a" altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f @@ -75,8 +83,70 @@ void free_alternatives_memory(void); 664: .popsection .org . - (664b-663b) + (662b-661b) .org . - (662b-661b) + (664b-663b) + .endif .endm +/* + * Begin an alternative code sequence. + * + * The code that follows this macro will be assembled and linked as + * normal. There are no restrictions on this code. + */ +.macro alternative_if_not cap, enable = 1 + .if \enable + .pushsection .altinstructions, "a" + altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f + .popsection +661: + .endif +.endm + +/* + * Provide the alternative code sequence. + * + * The code that follows this macro is assembled into a special + * section to be used for dynamic patching. Code that follows this + * macro must: + * + * 1. Be exactly the same length (in bytes) as the default code + * sequence. + * + * 2. Not contain a branch target that is used outside of the + * alternative sequence it is defined in (branches into an + * alternative sequence are not fixed up). + */ +.macro alternative_else, enable = 1 + .if \enable +662: .pushsection .altinstr_replacement, "ax" +663: + .endif +.endm + +/* + * Complete an alternative code sequence. + */ +.macro alternative_endif, enable = 1 + .if \enable +664: .popsection + .org . - (664b-663b) + (662b-661b) + .org . - (662b-661b) + (664b-663b) + .endif +.endm + +#define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...) \ + alternative_insn insn1, insn2, cap, IS_ENABLED(cfg) + + #endif /* __ASSEMBLY__ */ +/* + * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature)); + * + * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature, CONFIG_FOO)); + * N.B. If CONFIG_FOO is specified, but not selected, the whole block + * will be omitted, including oldinstr. + */ +#define ALTERNATIVE(oldinstr, newinstr, ...) \ + _ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1) + #endif /* __ASM_ALTERNATIVE_H */ diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 144b64ad96c3..b51f2cc22ca9 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -49,18 +49,6 @@ msr daifclr, #2 .endm -/* - * Save/disable and restore interrupts. - */ - .macro save_and_disable_irqs, olddaif - mrs \olddaif, daif - disable_irq - .endm - - .macro restore_irqs, olddaif - msr daif, \olddaif - .endm - /* * Enable and disable debug exceptions. */ @@ -103,9 +91,7 @@ * SMP data memory barrier */ .macro smp_dmb, opt -#ifdef CONFIG_SMP dmb \opt -#endif .endm #define USER(l, x...) \ diff --git a/arch/arm64/include/asm/atomic.h b/arch/arm64/include/asm/atomic.h index 7047051ded40..35a67783cfa0 100644 --- a/arch/arm64/include/asm/atomic.h +++ b/arch/arm64/include/asm/atomic.h @@ -24,233 +24,72 @@ #include #include -#include - -#define ATOMIC_INIT(i) { (i) } +#include #ifdef __KERNEL__ -/* - * On ARM, ordinary assignment (str instruction) doesn't clear the local - * strex/ldrex monitor on some implementations. The reason we can use it for - * atomic_set() is the clrex or dummy strex done on every exception return. - */ -#define atomic_read(v) ACCESS_ONCE((v)->counter) -#define atomic_set(v,i) (((v)->counter) = (i)) +#define __ARM64_IN_ATOMIC_IMPL -/* - * AArch64 UP and SMP safe atomic ops. We use load exclusive and - * store exclusive to ensure that these are atomic. We may loop - * to ensure that the update happens. - */ +#if defined(CONFIG_ARM64_LSE_ATOMICS) && defined(CONFIG_AS_LSE) +#include +#else +#include +#endif -#define ATOMIC_OP(op, asm_op) \ -static inline void atomic_##op(int i, atomic_t *v) \ -{ \ - unsigned long tmp; \ - int result; \ +#undef __ARM64_IN_ATOMIC_IMPL + +#include + +#define ___atomic_add_unless(v, a, u, sfx) \ +({ \ + typeof((v)->counter) c, old; \ \ - asm volatile("// atomic_" #op "\n" \ -"1: ldxr %w0, %2\n" \ -" " #asm_op " %w0, %w0, %w3\n" \ -" stxr %w1, %w0, %2\n" \ -" cbnz %w1, 1b" \ - : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ - : "Ir" (i)); \ -} \ + c = atomic##sfx##_read(v); \ + while (c != (u) && \ + (old = atomic##sfx##_cmpxchg((v), c, c + (a))) != c) \ + c = old; \ + c; \ + }) -#define ATOMIC_OP_RETURN(op, asm_op) \ -static inline int atomic_##op##_return(int i, atomic_t *v) \ -{ \ - unsigned long tmp; \ - int result; \ - \ - asm volatile("// atomic_" #op "_return\n" \ -"1: ldxr %w0, %2\n" \ -" " #asm_op " %w0, %w0, %w3\n" \ -" stlxr %w1, %w0, %2\n" \ -" cbnz %w1, 1b" \ - : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ - : "Ir" (i) \ - : "memory"); \ - \ - smp_mb(); \ - return result; \ -} +#define ATOMIC_INIT(i) { (i) } -#define ATOMIC_OPS(op, asm_op) \ - ATOMIC_OP(op, asm_op) \ - ATOMIC_OP_RETURN(op, asm_op) +#define atomic_read(v) READ_ONCE((v)->counter) +#define atomic_set(v, i) (((v)->counter) = (i)) +#define atomic_xchg(v, new) xchg(&((v)->counter), (new)) +#define atomic_cmpxchg(v, old, new) cmpxchg(&((v)->counter), (old), (new)) -ATOMIC_OPS(add, add) -ATOMIC_OPS(sub, sub) - -#undef ATOMIC_OPS -#undef ATOMIC_OP_RETURN -#undef ATOMIC_OP - -static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new) -{ - unsigned long tmp; - int oldval; - - smp_mb(); - - asm volatile("// atomic_cmpxchg\n" -"1: ldxr %w1, %2\n" -" cmp %w1, %w3\n" -" b.ne 2f\n" -" stxr %w0, %w4, %2\n" -" cbnz %w0, 1b\n" -"2:" - : "=&r" (tmp), "=&r" (oldval), "+Q" (ptr->counter) - : "Ir" (old), "r" (new) - : "cc"); - - smp_mb(); - return oldval; -} - -#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) - -static inline int __atomic_add_unless(atomic_t *v, int a, int u) -{ - int c, old; - - c = atomic_read(v); - while (c != u && (old = atomic_cmpxchg((v), c, c + a)) != c) - c = old; - return c; -} - -#define atomic_inc(v) atomic_add(1, v) -#define atomic_dec(v) atomic_sub(1, v) - -#define atomic_inc_and_test(v) (atomic_add_return(1, v) == 0) -#define atomic_dec_and_test(v) (atomic_sub_return(1, v) == 0) -#define atomic_inc_return(v) (atomic_add_return(1, v)) -#define atomic_dec_return(v) (atomic_sub_return(1, v)) -#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0) - -#define atomic_add_negative(i,v) (atomic_add_return(i, v) < 0) +#define atomic_inc(v) atomic_add(1, (v)) +#define atomic_dec(v) atomic_sub(1, (v)) +#define atomic_inc_return(v) atomic_add_return(1, (v)) +#define atomic_dec_return(v) atomic_sub_return(1, (v)) +#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) +#define atomic_dec_and_test(v) (atomic_dec_return(v) == 0) +#define atomic_sub_and_test(i, v) (atomic_sub_return((i), (v)) == 0) +#define atomic_add_negative(i, v) (atomic_add_return((i), (v)) < 0) +#define __atomic_add_unless(v, a, u) ___atomic_add_unless(v, a, u,) +#define atomic_andnot atomic_andnot /* * 64-bit atomic operations. */ -#define ATOMIC64_INIT(i) { (i) } +#define ATOMIC64_INIT ATOMIC_INIT +#define atomic64_read atomic_read +#define atomic64_set atomic_set +#define atomic64_xchg atomic_xchg +#define atomic64_cmpxchg atomic_cmpxchg -#define atomic64_read(v) ACCESS_ONCE((v)->counter) -#define atomic64_set(v,i) (((v)->counter) = (i)) - -#define ATOMIC64_OP(op, asm_op) \ -static inline void atomic64_##op(long i, atomic64_t *v) \ -{ \ - long result; \ - unsigned long tmp; \ - \ - asm volatile("// atomic64_" #op "\n" \ -"1: ldxr %0, %2\n" \ -" " #asm_op " %0, %0, %3\n" \ -" stxr %w1, %0, %2\n" \ -" cbnz %w1, 1b" \ - : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ - : "Ir" (i)); \ -} \ - -#define ATOMIC64_OP_RETURN(op, asm_op) \ -static inline long atomic64_##op##_return(long i, atomic64_t *v) \ -{ \ - long result; \ - unsigned long tmp; \ - \ - asm volatile("// atomic64_" #op "_return\n" \ -"1: ldxr %0, %2\n" \ -" " #asm_op " %0, %0, %3\n" \ -" stlxr %w1, %0, %2\n" \ -" cbnz %w1, 1b" \ - : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ - : "Ir" (i) \ - : "memory"); \ - \ - smp_mb(); \ - return result; \ -} - -#define ATOMIC64_OPS(op, asm_op) \ - ATOMIC64_OP(op, asm_op) \ - ATOMIC64_OP_RETURN(op, asm_op) - -ATOMIC64_OPS(add, add) -ATOMIC64_OPS(sub, sub) - -#undef ATOMIC64_OPS -#undef ATOMIC64_OP_RETURN -#undef ATOMIC64_OP - -static inline long atomic64_cmpxchg(atomic64_t *ptr, long old, long new) -{ - long oldval; - unsigned long res; - - smp_mb(); - - asm volatile("// atomic64_cmpxchg\n" -"1: ldxr %1, %2\n" -" cmp %1, %3\n" -" b.ne 2f\n" -" stxr %w0, %4, %2\n" -" cbnz %w0, 1b\n" -"2:" - : "=&r" (res), "=&r" (oldval), "+Q" (ptr->counter) - : "Ir" (old), "r" (new) - : "cc"); - - smp_mb(); - return oldval; -} - -#define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) - -static inline long atomic64_dec_if_positive(atomic64_t *v) -{ - long result; - unsigned long tmp; - - asm volatile("// atomic64_dec_if_positive\n" -"1: ldxr %0, %2\n" -" subs %0, %0, #1\n" -" b.mi 2f\n" -" stlxr %w1, %0, %2\n" -" cbnz %w1, 1b\n" -" dmb ish\n" -"2:" - : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) - : - : "cc", "memory"); - - return result; -} - -static inline int atomic64_add_unless(atomic64_t *v, long a, long u) -{ - long c, old; - - c = atomic64_read(v); - while (c != u && (old = atomic64_cmpxchg((v), c, c + a)) != c) - c = old; - - return c != u; -} - -#define atomic64_add_negative(a, v) (atomic64_add_return((a), (v)) < 0) -#define atomic64_inc(v) atomic64_add(1LL, (v)) -#define atomic64_inc_return(v) atomic64_add_return(1LL, (v)) +#define atomic64_inc(v) atomic64_add(1, (v)) +#define atomic64_dec(v) atomic64_sub(1, (v)) +#define atomic64_inc_return(v) atomic64_add_return(1, (v)) +#define atomic64_dec_return(v) atomic64_sub_return(1, (v)) #define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0) -#define atomic64_sub_and_test(a, v) (atomic64_sub_return((a), (v)) == 0) -#define atomic64_dec(v) atomic64_sub(1LL, (v)) -#define atomic64_dec_return(v) atomic64_sub_return(1LL, (v)) -#define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0) -#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1LL, 0LL) +#define atomic64_dec_and_test(v) (atomic64_dec_return(v) == 0) +#define atomic64_sub_and_test(i, v) (atomic64_sub_return((i), (v)) == 0) +#define atomic64_add_negative(i, v) (atomic64_add_return((i), (v)) < 0) +#define atomic64_add_unless(v, a, u) (___atomic_add_unless(v, a, u, 64) != u) +#define atomic64_andnot atomic64_andnot + +#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) #endif #endif diff --git a/arch/arm64/include/asm/atomic_ll_sc.h b/arch/arm64/include/asm/atomic_ll_sc.h new file mode 100644 index 000000000000..b3b5c4ae3800 --- /dev/null +++ b/arch/arm64/include/asm/atomic_ll_sc.h @@ -0,0 +1,247 @@ +/* + * Based on arch/arm/include/asm/atomic.h + * + * Copyright (C) 1996 Russell King. + * Copyright (C) 2002 Deep Blue Solutions Ltd. + * Copyright (C) 2012 ARM Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef __ASM_ATOMIC_LL_SC_H +#define __ASM_ATOMIC_LL_SC_H + +#ifndef __ARM64_IN_ATOMIC_IMPL +#error "please don't include this file directly" +#endif + +/* + * AArch64 UP and SMP safe atomic ops. We use load exclusive and + * store exclusive to ensure that these are atomic. We may loop + * to ensure that the update happens. + * + * NOTE: these functions do *not* follow the PCS and must explicitly + * save any clobbered registers other than x0 (regardless of return + * value). This is achieved through -fcall-saved-* compiler flags for + * this file, which unfortunately don't work on a per-function basis + * (the optimize attribute silently ignores these options). + */ + +#define ATOMIC_OP(op, asm_op) \ +__LL_SC_INLINE void \ +__LL_SC_PREFIX(atomic_##op(int i, atomic_t *v)) \ +{ \ + unsigned long tmp; \ + int result; \ + \ + asm volatile("// atomic_" #op "\n" \ +" prfm pstl1strm, %2\n" \ +"1: ldxr %w0, %2\n" \ +" " #asm_op " %w0, %w0, %w3\n" \ +" stxr %w1, %w0, %2\n" \ +" cbnz %w1, 1b" \ + : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ + : "Ir" (i)); \ +} \ +__LL_SC_EXPORT(atomic_##op); + +#define ATOMIC_OP_RETURN(op, asm_op) \ +__LL_SC_INLINE int \ +__LL_SC_PREFIX(atomic_##op##_return(int i, atomic_t *v)) \ +{ \ + unsigned long tmp; \ + int result; \ + \ + asm volatile("// atomic_" #op "_return\n" \ +" prfm pstl1strm, %2\n" \ +"1: ldxr %w0, %2\n" \ +" " #asm_op " %w0, %w0, %w3\n" \ +" stlxr %w1, %w0, %2\n" \ +" cbnz %w1, 1b" \ + : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ + : "Ir" (i) \ + : "memory"); \ + \ + smp_mb(); \ + return result; \ +} \ +__LL_SC_EXPORT(atomic_##op##_return); + +#define ATOMIC_OPS(op, asm_op) \ + ATOMIC_OP(op, asm_op) \ + ATOMIC_OP_RETURN(op, asm_op) + +ATOMIC_OPS(add, add) +ATOMIC_OPS(sub, sub) + +ATOMIC_OP(and, and) +ATOMIC_OP(andnot, bic) +ATOMIC_OP(or, orr) +ATOMIC_OP(xor, eor) + +#undef ATOMIC_OPS +#undef ATOMIC_OP_RETURN +#undef ATOMIC_OP + +#define ATOMIC64_OP(op, asm_op) \ +__LL_SC_INLINE void \ +__LL_SC_PREFIX(atomic64_##op(long i, atomic64_t *v)) \ +{ \ + long result; \ + unsigned long tmp; \ + \ + asm volatile("// atomic64_" #op "\n" \ +" prfm pstl1strm, %2\n" \ +"1: ldxr %0, %2\n" \ +" " #asm_op " %0, %0, %3\n" \ +" stxr %w1, %0, %2\n" \ +" cbnz %w1, 1b" \ + : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ + : "Ir" (i)); \ +} \ +__LL_SC_EXPORT(atomic64_##op); + +#define ATOMIC64_OP_RETURN(op, asm_op) \ +__LL_SC_INLINE long \ +__LL_SC_PREFIX(atomic64_##op##_return(long i, atomic64_t *v)) \ +{ \ + long result; \ + unsigned long tmp; \ + \ + asm volatile("// atomic64_" #op "_return\n" \ +" prfm pstl1strm, %2\n" \ +"1: ldxr %0, %2\n" \ +" " #asm_op " %0, %0, %3\n" \ +" stlxr %w1, %0, %2\n" \ +" cbnz %w1, 1b" \ + : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \ + : "Ir" (i) \ + : "memory"); \ + \ + smp_mb(); \ + return result; \ +} \ +__LL_SC_EXPORT(atomic64_##op##_return); + +#define ATOMIC64_OPS(op, asm_op) \ + ATOMIC64_OP(op, asm_op) \ + ATOMIC64_OP_RETURN(op, asm_op) + +ATOMIC64_OPS(add, add) +ATOMIC64_OPS(sub, sub) + +ATOMIC64_OP(and, and) +ATOMIC64_OP(andnot, bic) +ATOMIC64_OP(or, orr) +ATOMIC64_OP(xor, eor) + +#undef ATOMIC64_OPS +#undef ATOMIC64_OP_RETURN +#undef ATOMIC64_OP + +__LL_SC_INLINE long +__LL_SC_PREFIX(atomic64_dec_if_positive(atomic64_t *v)) +{ + long result; + unsigned long tmp; + + asm volatile("// atomic64_dec_if_positive\n" +" prfm pstl1strm, %2\n" +"1: ldxr %0, %2\n" +" subs %0, %0, #1\n" +" b.lt 2f\n" +" stlxr %w1, %0, %2\n" +" cbnz %w1, 1b\n" +" dmb ish\n" +"2:" + : "=&r" (result), "=&r" (tmp), "+Q" (v->counter) + : + : "cc", "memory"); + + return result; +} +__LL_SC_EXPORT(atomic64_dec_if_positive); + +#define __CMPXCHG_CASE(w, sz, name, mb, rel, cl) \ +__LL_SC_INLINE unsigned long \ +__LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr, \ + unsigned long old, \ + unsigned long new)) \ +{ \ + unsigned long tmp, oldval; \ + \ + asm volatile( \ + " prfm pstl1strm, %[v]\n" \ + "1: ldxr" #sz "\t%" #w "[oldval], %[v]\n" \ + " eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \ + " cbnz %" #w "[tmp], 2f\n" \ + " st" #rel "xr" #sz "\t%w[tmp], %" #w "[new], %[v]\n" \ + " cbnz %w[tmp], 1b\n" \ + " " #mb "\n" \ + " mov %" #w "[oldval], %" #w "[old]\n" \ + "2:" \ + : [tmp] "=&r" (tmp), [oldval] "=&r" (oldval), \ + [v] "+Q" (*(unsigned long *)ptr) \ + : [old] "Lr" (old), [new] "r" (new) \ + : cl); \ + \ + return oldval; \ +} \ +__LL_SC_EXPORT(__cmpxchg_case_##name); + +__CMPXCHG_CASE(w, b, 1, , , ) +__CMPXCHG_CASE(w, h, 2, , , ) +__CMPXCHG_CASE(w, , 4, , , ) +__CMPXCHG_CASE( , , 8, , , ) +__CMPXCHG_CASE(w, b, mb_1, dmb ish, l, "memory") +__CMPXCHG_CASE(w, h, mb_2, dmb ish, l, "memory") +__CMPXCHG_CASE(w, , mb_4, dmb ish, l, "memory") +__CMPXCHG_CASE( , , mb_8, dmb ish, l, "memory") + +#undef __CMPXCHG_CASE + +#define __CMPXCHG_DBL(name, mb, rel, cl) \ +__LL_SC_INLINE int \ +__LL_SC_PREFIX(__cmpxchg_double##name(unsigned long old1, \ + unsigned long old2, \ + unsigned long new1, \ + unsigned long new2, \ + volatile void *ptr)) \ +{ \ + unsigned long tmp, ret; \ + \ + asm volatile("// __cmpxchg_double" #name "\n" \ + " prfm pstl1strm, %2\n" \ + "1: ldxp %0, %1, %2\n" \ + " eor %0, %0, %3\n" \ + " eor %1, %1, %4\n" \ + " orr %1, %0, %1\n" \ + " cbnz %1, 2f\n" \ + " st" #rel "xp %w0, %5, %6, %2\n" \ + " cbnz %w0, 1b\n" \ + " " #mb "\n" \ + "2:" \ + : "=&r" (tmp), "=&r" (ret), "+Q" (*(unsigned long *)ptr) \ + : "r" (old1), "r" (old2), "r" (new1), "r" (new2) \ + : cl); \ + \ + return ret; \ +} \ +__LL_SC_EXPORT(__cmpxchg_double##name); + +__CMPXCHG_DBL( , , , ) +__CMPXCHG_DBL(_mb, dmb ish, l, "memory") + +#undef __CMPXCHG_DBL + +#endif /* __ASM_ATOMIC_LL_SC_H */ diff --git a/arch/arm64/include/asm/atomic_lse.h b/arch/arm64/include/asm/atomic_lse.h new file mode 100644 index 000000000000..55d740e63459 --- /dev/null +++ b/arch/arm64/include/asm/atomic_lse.h @@ -0,0 +1,391 @@ +/* + * Based on arch/arm/include/asm/atomic.h + * + * Copyright (C) 1996 Russell King. + * Copyright (C) 2002 Deep Blue Solutions Ltd. + * Copyright (C) 2012 ARM Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef __ASM_ATOMIC_LSE_H +#define __ASM_ATOMIC_LSE_H + +#ifndef __ARM64_IN_ATOMIC_IMPL +#error "please don't include this file directly" +#endif + +#define __LL_SC_ATOMIC(op) __LL_SC_CALL(atomic_##op) + +static inline void atomic_andnot(int i, atomic_t *v) +{ + register int w0 asm ("w0") = i; + register atomic_t *x1 asm ("x1") = v; + + asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(andnot), + " stclr %w[i], %[v]\n") + : [i] "+r" (w0), [v] "+Q" (v->counter) + : "r" (x1) + : "x30"); +} + +static inline void atomic_or(int i, atomic_t *v) +{ + register int w0 asm ("w0") = i; + register atomic_t *x1 asm ("x1") = v; + + asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(or), + " stset %w[i], %[v]\n") + : [i] "+r" (w0), [v] "+Q" (v->counter) + : "r" (x1) + : "x30"); +} + +static inline void atomic_xor(int i, atomic_t *v) +{ + register int w0 asm ("w0") = i; + register atomic_t *x1 asm ("x1") = v; + + asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(xor), + " steor %w[i], %[v]\n") + : [i] "+r" (w0), [v] "+Q" (v->counter) + : "r" (x1) + : "x30"); +} + +static inline void atomic_add(int i, atomic_t *v) +{ + register int w0 asm ("w0") = i; + register atomic_t *x1 asm ("x1") = v; + + asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(add), + " stadd %w[i], %[v]\n") + : [i] "+r" (w0), [v] "+Q" (v->counter) + : "r" (x1) + : "x30"); +} + +static inline int atomic_add_return(int i, atomic_t *v) +{ + register int w0 asm ("w0") = i; + register atomic_t *x1 asm ("x1") = v; + + asm volatile(ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ + " nop\n" + __LL_SC_ATOMIC(add_return), + /* LSE atomics */ + " ldaddal %w[i], w30, %[v]\n" + " add %w[i], %w[i], w30") + : [i] "+r" (w0), [v] "+Q" (v->counter) + : "r" (x1) + : "x30", "memory"); + + return w0; +} + +static inline void atomic_and(int i, atomic_t *v) +{ + register int w0 asm ("w0") = i; + register atomic_t *x1 asm ("x1") = v; + + asm volatile(ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ + " nop\n" + __LL_SC_ATOMIC(and), + /* LSE atomics */ + " mvn %w[i], %w[i]\n" + " stclr %w[i], %[v]") + : [i] "+r" (w0), [v] "+Q" (v->counter) + : "r" (x1) + : "x30"); +} + +static inline void atomic_sub(int i, atomic_t *v) +{ + register int w0 asm ("w0") = i; + register atomic_t *x1 asm ("x1") = v; + + asm volatile(ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ + " nop\n" + __LL_SC_ATOMIC(sub), + /* LSE atomics */ + " neg %w[i], %w[i]\n" + " stadd %w[i], %[v]") + : [i] "+r" (w0), [v] "+Q" (v->counter) + : "r" (x1) + : "x30"); +} + +static inline int atomic_sub_return(int i, atomic_t *v) +{ + register int w0 asm ("w0") = i; + register atomic_t *x1 asm ("x1") = v; + + asm volatile(ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ + " nop\n" + __LL_SC_ATOMIC(sub_return) + " nop", + /* LSE atomics */ + " neg %w[i], %w[i]\n" + " ldaddal %w[i], w30, %[v]\n" + " add %w[i], %w[i], w30") + : [i] "+r" (w0), [v] "+Q" (v->counter) + : "r" (x1) + : "x30", "memory"); + + return w0; +} + +#undef __LL_SC_ATOMIC + +#define __LL_SC_ATOMIC64(op) __LL_SC_CALL(atomic64_##op) + +static inline void atomic64_andnot(long i, atomic64_t *v) +{ + register long x0 asm ("x0") = i; + register atomic64_t *x1 asm ("x1") = v; + + asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(andnot), + " stclr %[i], %[v]\n") + : [i] "+r" (x0), [v] "+Q" (v->counter) + : "r" (x1) + : "x30"); +} + +static inline void atomic64_or(long i, atomic64_t *v) +{ + register long x0 asm ("x0") = i; + register atomic64_t *x1 asm ("x1") = v; + + asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(or), + " stset %[i], %[v]\n") + : [i] "+r" (x0), [v] "+Q" (v->counter) + : "r" (x1) + : "x30"); +} + +static inline void atomic64_xor(long i, atomic64_t *v) +{ + register long x0 asm ("x0") = i; + register atomic64_t *x1 asm ("x1") = v; + + asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(xor), + " steor %[i], %[v]\n") + : [i] "+r" (x0), [v] "+Q" (v->counter) + : "r" (x1) + : "x30"); +} + +static inline void atomic64_add(long i, atomic64_t *v) +{ + register long x0 asm ("x0") = i; + register atomic64_t *x1 asm ("x1") = v; + + asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(add), + " stadd %[i], %[v]\n") + : [i] "+r" (x0), [v] "+Q" (v->counter) + : "r" (x1) + : "x30"); +} + +static inline long atomic64_add_return(long i, atomic64_t *v) +{ + register long x0 asm ("x0") = i; + register atomic64_t *x1 asm ("x1") = v; + + asm volatile(ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ + " nop\n" + __LL_SC_ATOMIC64(add_return), + /* LSE atomics */ + " ldaddal %[i], x30, %[v]\n" + " add %[i], %[i], x30") + : [i] "+r" (x0), [v] "+Q" (v->counter) + : "r" (x1) + : "x30", "memory"); + + return x0; +} + +static inline void atomic64_and(long i, atomic64_t *v) +{ + register long x0 asm ("x0") = i; + register atomic64_t *x1 asm ("x1") = v; + + asm volatile(ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ + " nop\n" + __LL_SC_ATOMIC64(and), + /* LSE atomics */ + " mvn %[i], %[i]\n" + " stclr %[i], %[v]") + : [i] "+r" (x0), [v] "+Q" (v->counter) + : "r" (x1) + : "x30"); +} + +static inline void atomic64_sub(long i, atomic64_t *v) +{ + register long x0 asm ("x0") = i; + register atomic64_t *x1 asm ("x1") = v; + + asm volatile(ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ + " nop\n" + __LL_SC_ATOMIC64(sub), + /* LSE atomics */ + " neg %[i], %[i]\n" + " stadd %[i], %[v]") + : [i] "+r" (x0), [v] "+Q" (v->counter) + : "r" (x1) + : "x30"); +} + +static inline long atomic64_sub_return(long i, atomic64_t *v) +{ + register long x0 asm ("x0") = i; + register atomic64_t *x1 asm ("x1") = v; + + asm volatile(ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ + " nop\n" + __LL_SC_ATOMIC64(sub_return) + " nop", + /* LSE atomics */ + " neg %[i], %[i]\n" + " ldaddal %[i], x30, %[v]\n" + " add %[i], %[i], x30") + : [i] "+r" (x0), [v] "+Q" (v->counter) + : "r" (x1) + : "x30", "memory"); + + return x0; +} + +static inline long atomic64_dec_if_positive(atomic64_t *v) +{ + register long x0 asm ("x0") = (long)v; + + asm volatile(ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ + " nop\n" + __LL_SC_ATOMIC64(dec_if_positive) + " nop\n" + " nop\n" + " nop\n" + " nop\n" + " nop", + /* LSE atomics */ + "1: ldr x30, %[v]\n" + " subs %[ret], x30, #1\n" + " b.lt 2f\n" + " casal x30, %[ret], %[v]\n" + " sub x30, x30, #1\n" + " sub x30, x30, %[ret]\n" + " cbnz x30, 1b\n" + "2:") + : [ret] "+&r" (x0), [v] "+Q" (v->counter) + : + : "x30", "cc", "memory"); + + return x0; +} + +#undef __LL_SC_ATOMIC64 + +#define __LL_SC_CMPXCHG(op) __LL_SC_CALL(__cmpxchg_case_##op) + +#define __CMPXCHG_CASE(w, sz, name, mb, cl...) \ +static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \ + unsigned long old, \ + unsigned long new) \ +{ \ + register unsigned long x0 asm ("x0") = (unsigned long)ptr; \ + register unsigned long x1 asm ("x1") = old; \ + register unsigned long x2 asm ("x2") = new; \ + \ + asm volatile(ARM64_LSE_ATOMIC_INSN( \ + /* LL/SC */ \ + " nop\n" \ + __LL_SC_CMPXCHG(name) \ + " nop", \ + /* LSE atomics */ \ + " mov " #w "30, %" #w "[old]\n" \ + " cas" #mb #sz "\t" #w "30, %" #w "[new], %[v]\n" \ + " mov %" #w "[ret], " #w "30") \ + : [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr) \ + : [old] "r" (x1), [new] "r" (x2) \ + : "x30" , ##cl); \ + \ + return x0; \ +} + +__CMPXCHG_CASE(w, b, 1, ) +__CMPXCHG_CASE(w, h, 2, ) +__CMPXCHG_CASE(w, , 4, ) +__CMPXCHG_CASE(x, , 8, ) +__CMPXCHG_CASE(w, b, mb_1, al, "memory") +__CMPXCHG_CASE(w, h, mb_2, al, "memory") +__CMPXCHG_CASE(w, , mb_4, al, "memory") +__CMPXCHG_CASE(x, , mb_8, al, "memory") + +#undef __LL_SC_CMPXCHG +#undef __CMPXCHG_CASE + +#define __LL_SC_CMPXCHG_DBL(op) __LL_SC_CALL(__cmpxchg_double##op) + +#define __CMPXCHG_DBL(name, mb, cl...) \ +static inline int __cmpxchg_double##name(unsigned long old1, \ + unsigned long old2, \ + unsigned long new1, \ + unsigned long new2, \ + volatile void *ptr) \ +{ \ + unsigned long oldval1 = old1; \ + unsigned long oldval2 = old2; \ + register unsigned long x0 asm ("x0") = old1; \ + register unsigned long x1 asm ("x1") = old2; \ + register unsigned long x2 asm ("x2") = new1; \ + register unsigned long x3 asm ("x3") = new2; \ + register unsigned long x4 asm ("x4") = (unsigned long)ptr; \ + \ + asm volatile(ARM64_LSE_ATOMIC_INSN( \ + /* LL/SC */ \ + " nop\n" \ + " nop\n" \ + " nop\n" \ + __LL_SC_CMPXCHG_DBL(name), \ + /* LSE atomics */ \ + " casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\ + " eor %[old1], %[old1], %[oldval1]\n" \ + " eor %[old2], %[old2], %[oldval2]\n" \ + " orr %[old1], %[old1], %[old2]") \ + : [old1] "+r" (x0), [old2] "+r" (x1), \ + [v] "+Q" (*(unsigned long *)ptr) \ + : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \ + [oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \ + : "x30" , ##cl); \ + \ + return x0; \ +} + +__CMPXCHG_DBL( , ) +__CMPXCHG_DBL(_mb, al, "memory") + +#undef __LL_SC_CMPXCHG_DBL +#undef __CMPXCHG_DBL + +#endif /* __ASM_ATOMIC_LSE_H */ diff --git a/arch/arm64/include/asm/barrier.h b/arch/arm64/include/asm/barrier.h index 0fa47c4275cb..624f9679f4b0 100644 --- a/arch/arm64/include/asm/barrier.h +++ b/arch/arm64/include/asm/barrier.h @@ -35,28 +35,6 @@ #define dma_rmb() dmb(oshld) #define dma_wmb() dmb(oshst) -#ifndef CONFIG_SMP -#define smp_mb() barrier() -#define smp_rmb() barrier() -#define smp_wmb() barrier() - -#define smp_store_release(p, v) \ -do { \ - compiletime_assert_atomic_type(*p); \ - barrier(); \ - ACCESS_ONCE(*p) = (v); \ -} while (0) - -#define smp_load_acquire(p) \ -({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ - compiletime_assert_atomic_type(*p); \ - barrier(); \ - ___p1; \ -}) - -#else - #define smp_mb() dmb(ish) #define smp_rmb() dmb(ishld) #define smp_wmb() dmb(ishst) @@ -109,8 +87,6 @@ do { \ ___p1; \ }) -#endif - #define read_barrier_depends() do { } while(0) #define smp_read_barrier_depends() do { } while(0) diff --git a/arch/arm64/include/asm/bug.h b/arch/arm64/include/asm/bug.h new file mode 100644 index 000000000000..4a748ce9ba1a --- /dev/null +++ b/arch/arm64/include/asm/bug.h @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2015 ARM Limited + * Author: Dave Martin + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef _ARCH_ARM64_ASM_BUG_H +#define _ARCH_ARM64_ASM_BUG_H + +#include + +#ifdef CONFIG_GENERIC_BUG +#define HAVE_ARCH_BUG + +#ifdef CONFIG_DEBUG_BUGVERBOSE +#define _BUGVERBOSE_LOCATION(file, line) __BUGVERBOSE_LOCATION(file, line) +#define __BUGVERBOSE_LOCATION(file, line) \ + ".pushsection .rodata.str,\"aMS\",@progbits,1\n" \ + "2: .string \"" file "\"\n\t" \ + ".popsection\n\t" \ + \ + ".long 2b - 0b\n\t" \ + ".short " #line "\n\t" +#else +#define _BUGVERBOSE_LOCATION(file, line) +#endif + +#define _BUG_FLAGS(flags) __BUG_FLAGS(flags) + +#define __BUG_FLAGS(flags) asm volatile ( \ + ".pushsection __bug_table,\"a\"\n\t" \ + ".align 2\n\t" \ + "0: .long 1f - 0b\n\t" \ +_BUGVERBOSE_LOCATION(__FILE__, __LINE__) \ + ".short " #flags "\n\t" \ + ".popsection\n" \ + \ + "1: brk %[imm]" \ + :: [imm] "i" (BUG_BRK_IMM) \ +) + +#define BUG() do { \ + _BUG_FLAGS(0); \ + unreachable(); \ +} while (0) + +#define __WARN_TAINT(taint) _BUG_FLAGS(BUGFLAG_TAINT(taint)) + +#endif /* ! CONFIG_GENERIC_BUG */ + +#include + +#endif /* ! _ARCH_ARM64_ASM_BUG_H */ diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h index d8c25b7b18fb..899e9f1d19e4 100644 --- a/arch/arm64/include/asm/cmpxchg.h +++ b/arch/arm64/include/asm/cmpxchg.h @@ -21,7 +21,9 @@ #include #include +#include #include +#include static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size) { @@ -29,37 +31,73 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size switch (size) { case 1: - asm volatile("// __xchg1\n" + asm volatile(ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ + " prfm pstl1strm, %2\n" "1: ldxrb %w0, %2\n" " stlxrb %w1, %w3, %2\n" " cbnz %w1, 1b\n" + " dmb ish", + /* LSE atomics */ + " nop\n" + " nop\n" + " swpalb %w3, %w0, %2\n" + " nop\n" + " nop") : "=&r" (ret), "=&r" (tmp), "+Q" (*(u8 *)ptr) : "r" (x) : "memory"); break; case 2: - asm volatile("// __xchg2\n" + asm volatile(ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ + " prfm pstl1strm, %2\n" "1: ldxrh %w0, %2\n" " stlxrh %w1, %w3, %2\n" " cbnz %w1, 1b\n" + " dmb ish", + /* LSE atomics */ + " nop\n" + " nop\n" + " swpalh %w3, %w0, %2\n" + " nop\n" + " nop") : "=&r" (ret), "=&r" (tmp), "+Q" (*(u16 *)ptr) : "r" (x) : "memory"); break; case 4: - asm volatile("// __xchg4\n" + asm volatile(ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ + " prfm pstl1strm, %2\n" "1: ldxr %w0, %2\n" " stlxr %w1, %w3, %2\n" " cbnz %w1, 1b\n" + " dmb ish", + /* LSE atomics */ + " nop\n" + " nop\n" + " swpal %w3, %w0, %2\n" + " nop\n" + " nop") : "=&r" (ret), "=&r" (tmp), "+Q" (*(u32 *)ptr) : "r" (x) : "memory"); break; case 8: - asm volatile("// __xchg8\n" + asm volatile(ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ + " prfm pstl1strm, %2\n" "1: ldxr %0, %2\n" " stlxr %w1, %3, %2\n" " cbnz %w1, 1b\n" + " dmb ish", + /* LSE atomics */ + " nop\n" + " nop\n" + " swpal %3, %0, %2\n" + " nop\n" + " nop") : "=&r" (ret), "=&r" (tmp), "+Q" (*(u64 *)ptr) : "r" (x) : "memory"); @@ -68,7 +106,6 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size BUILD_BUG(); } - smp_mb(); return ret; } @@ -83,131 +120,39 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size) { - unsigned long oldval = 0, res; - switch (size) { case 1: - do { - asm volatile("// __cmpxchg1\n" - " ldxrb %w1, %2\n" - " mov %w0, #0\n" - " cmp %w1, %w3\n" - " b.ne 1f\n" - " stxrb %w0, %w4, %2\n" - "1:\n" - : "=&r" (res), "=&r" (oldval), "+Q" (*(u8 *)ptr) - : "Ir" (old), "r" (new) - : "cc"); - } while (res); - break; - + return __cmpxchg_case_1(ptr, (u8)old, new); case 2: - do { - asm volatile("// __cmpxchg2\n" - " ldxrh %w1, %2\n" - " mov %w0, #0\n" - " cmp %w1, %w3\n" - " b.ne 1f\n" - " stxrh %w0, %w4, %2\n" - "1:\n" - : "=&r" (res), "=&r" (oldval), "+Q" (*(u16 *)ptr) - : "Ir" (old), "r" (new) - : "cc"); - } while (res); - break; - + return __cmpxchg_case_2(ptr, (u16)old, new); case 4: - do { - asm volatile("// __cmpxchg4\n" - " ldxr %w1, %2\n" - " mov %w0, #0\n" - " cmp %w1, %w3\n" - " b.ne 1f\n" - " stxr %w0, %w4, %2\n" - "1:\n" - : "=&r" (res), "=&r" (oldval), "+Q" (*(u32 *)ptr) - : "Ir" (old), "r" (new) - : "cc"); - } while (res); - break; - + return __cmpxchg_case_4(ptr, old, new); case 8: - do { - asm volatile("// __cmpxchg8\n" - " ldxr %1, %2\n" - " mov %w0, #0\n" - " cmp %1, %3\n" - " b.ne 1f\n" - " stxr %w0, %4, %2\n" - "1:\n" - : "=&r" (res), "=&r" (oldval), "+Q" (*(u64 *)ptr) - : "Ir" (old), "r" (new) - : "cc"); - } while (res); - break; - + return __cmpxchg_case_8(ptr, old, new); default: BUILD_BUG(); } - return oldval; -} - -#define system_has_cmpxchg_double() 1 - -static inline int __cmpxchg_double(volatile void *ptr1, volatile void *ptr2, - unsigned long old1, unsigned long old2, - unsigned long new1, unsigned long new2, int size) -{ - unsigned long loop, lost; - - switch (size) { - case 8: - VM_BUG_ON((unsigned long *)ptr2 - (unsigned long *)ptr1 != 1); - do { - asm volatile("// __cmpxchg_double8\n" - " ldxp %0, %1, %2\n" - " eor %0, %0, %3\n" - " eor %1, %1, %4\n" - " orr %1, %0, %1\n" - " mov %w0, #0\n" - " cbnz %1, 1f\n" - " stxp %w0, %5, %6, %2\n" - "1:\n" - : "=&r"(loop), "=&r"(lost), "+Q" (*(u64 *)ptr1) - : "r" (old1), "r"(old2), "r"(new1), "r"(new2)); - } while (loop); - break; - default: - BUILD_BUG(); - } - - return !lost; -} - -static inline int __cmpxchg_double_mb(volatile void *ptr1, volatile void *ptr2, - unsigned long old1, unsigned long old2, - unsigned long new1, unsigned long new2, int size) -{ - int ret; - - smp_mb(); - ret = __cmpxchg_double(ptr1, ptr2, old1, old2, new1, new2, size); - smp_mb(); - - return ret; + unreachable(); } static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, unsigned long new, int size) { - unsigned long ret; + switch (size) { + case 1: + return __cmpxchg_case_mb_1(ptr, (u8)old, new); + case 2: + return __cmpxchg_case_mb_2(ptr, (u16)old, new); + case 4: + return __cmpxchg_case_mb_4(ptr, old, new); + case 8: + return __cmpxchg_case_mb_8(ptr, old, new); + default: + BUILD_BUG(); + } - smp_mb(); - ret = __cmpxchg(ptr, old, new, size); - smp_mb(); - - return ret; + unreachable(); } #define cmpxchg(ptr, o, n) \ @@ -228,21 +173,32 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, __ret; \ }) +#define system_has_cmpxchg_double() 1 + +#define __cmpxchg_double_check(ptr1, ptr2) \ +({ \ + if (sizeof(*(ptr1)) != 8) \ + BUILD_BUG(); \ + VM_BUG_ON((unsigned long *)(ptr2) - (unsigned long *)(ptr1) != 1); \ +}) + #define cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2) \ ({\ int __ret;\ - __ret = __cmpxchg_double_mb((ptr1), (ptr2), (unsigned long)(o1), \ - (unsigned long)(o2), (unsigned long)(n1), \ - (unsigned long)(n2), sizeof(*(ptr1)));\ + __cmpxchg_double_check(ptr1, ptr2); \ + __ret = !__cmpxchg_double_mb((unsigned long)(o1), (unsigned long)(o2), \ + (unsigned long)(n1), (unsigned long)(n2), \ + ptr1); \ __ret; \ }) #define cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2) \ ({\ int __ret;\ - __ret = __cmpxchg_double((ptr1), (ptr2), (unsigned long)(o1), \ - (unsigned long)(o2), (unsigned long)(n1), \ - (unsigned long)(n2), sizeof(*(ptr1)));\ + __cmpxchg_double_check(ptr1, ptr2); \ + __ret = !__cmpxchg_double((unsigned long)(o1), (unsigned long)(o2), \ + (unsigned long)(n1), (unsigned long)(n2), \ + ptr1); \ __ret; \ }) diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index c1044218a63a..171570702bb8 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -25,15 +25,20 @@ #define ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE 1 #define ARM64_WORKAROUND_845719 2 #define ARM64_HAS_SYSREG_GIC_CPUIF 3 +#define ARM64_HAS_PAN 4 +#define ARM64_HAS_LSE_ATOMICS 5 -#define ARM64_NCAPS 4 +#define ARM64_NCAPS 6 #ifndef __ASSEMBLY__ +#include + struct arm64_cpu_capabilities { const char *desc; u16 capability; bool (*matches)(const struct arm64_cpu_capabilities *); + void (*enable)(void); union { struct { /* To be used for erratum handling only */ u32 midr_model; @@ -41,8 +46,8 @@ struct arm64_cpu_capabilities { }; struct { /* Feature register checking */ - u64 register_mask; - u64 register_value; + int field_pos; + int min_field_value; }; }; }; @@ -70,6 +75,13 @@ static inline void cpus_set_cap(unsigned int num) __set_bit(num, cpu_hwcaps); } +static inline int __attribute_const__ cpuid_feature_extract_field(u64 features, + int field) +{ + return (s64)(features << (64 - 4 - field)) >> (64 - 4); +} + + void check_cpu_capabilities(const struct arm64_cpu_capabilities *caps, const char *info); void check_local_cpu_errata(void); diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index a84ec605bed8..ee6403df9fe4 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -81,9 +81,6 @@ #define ID_AA64MMFR0_BIGEND(mmfr0) \ (((mmfr0) & ID_AA64MMFR0_BIGEND_MASK) >> ID_AA64MMFR0_BIGEND_SHIFT) -#define SCTLR_EL1_CP15BEN (0x1 << 5) -#define SCTLR_EL1_SED (0x1 << 8) - #ifndef __ASSEMBLY__ /* diff --git a/arch/arm64/include/asm/debug-monitors.h b/arch/arm64/include/asm/debug-monitors.h index 40ec68aa6870..279c85b5ec09 100644 --- a/arch/arm64/include/asm/debug-monitors.h +++ b/arch/arm64/include/asm/debug-monitors.h @@ -18,6 +18,12 @@ #ifdef __KERNEL__ +#include +#include +#include +#include +#include + /* Low-level stepping controls. */ #define DBG_MDSCR_SS (1 << 0) #define DBG_SPSR_SS (1 << 21) @@ -38,12 +44,7 @@ /* * Break point instruction encoding */ -#define BREAK_INSTR_SIZE 4 - -/* - * ESR values expected for dynamic and compile time BRK instruction - */ -#define DBG_ESR_VAL_BRK(x) (0xf2000000 | ((x) & 0xfffff)) +#define BREAK_INSTR_SIZE AARCH64_INSN_SIZE /* * #imm16 values used for BRK instruction generation @@ -51,10 +52,12 @@ * 0x100: for triggering a fault on purpose (reserved) * 0x400: for dynamic BRK instruction * 0x401: for compile time BRK instruction + * 0x800: kernel-mode BUG() and WARN() traps */ #define FAULT_BRK_IMM 0x100 #define KGDB_DYN_DBG_BRK_IMM 0x400 #define KGDB_COMPILED_DBG_BRK_IMM 0x401 +#define BUG_BRK_IMM 0x800 /* * BRK instruction encoding @@ -68,25 +71,10 @@ */ #define AARCH64_BREAK_FAULT (AARCH64_BREAK_MON | (FAULT_BRK_IMM << 5)) -/* - * Extract byte from BRK instruction - */ -#define KGDB_DYN_DBG_BRK_INS_BYTE(x) \ - ((((AARCH64_BREAK_MON) & 0xffe0001f) >> (x * 8)) & 0xff) - -/* - * Extract byte from BRK #imm16 - */ -#define KGBD_DYN_DBG_BRK_IMM_BYTE(x) \ - (((((KGDB_DYN_DBG_BRK_IMM) & 0xffff) << 5) >> (x * 8)) & 0xff) - -#define KGDB_DYN_DBG_BRK_BYTE(x) \ - (KGDB_DYN_DBG_BRK_INS_BYTE(x) | KGBD_DYN_DBG_BRK_IMM_BYTE(x)) - -#define KGDB_DYN_BRK_INS_BYTE0 KGDB_DYN_DBG_BRK_BYTE(0) -#define KGDB_DYN_BRK_INS_BYTE1 KGDB_DYN_DBG_BRK_BYTE(1) -#define KGDB_DYN_BRK_INS_BYTE2 KGDB_DYN_DBG_BRK_BYTE(2) -#define KGDB_DYN_BRK_INS_BYTE3 KGDB_DYN_DBG_BRK_BYTE(3) +#define AARCH64_BREAK_KGDB_DYN_DBG \ + (AARCH64_BREAK_MON | (KGDB_DYN_DBG_BRK_IMM << 5)) +#define KGDB_DYN_BRK_INS_BYTE(x) \ + ((AARCH64_BREAK_KGDB_DYN_DBG >> (8 * (x))) & 0xff) #define CACHE_FLUSH_IS_SAFE 1 @@ -127,13 +115,13 @@ void unregister_break_hook(struct break_hook *hook); u8 debug_monitors_arch(void); -enum debug_el { +enum dbg_active_el { DBG_ACTIVE_EL0 = 0, DBG_ACTIVE_EL1, }; -void enable_debug_monitors(enum debug_el el); -void disable_debug_monitors(enum debug_el el); +void enable_debug_monitors(enum dbg_active_el el); +void disable_debug_monitors(enum dbg_active_el el); void user_rewind_single_step(struct task_struct *task); void user_fastforward_single_step(struct task_struct *task); diff --git a/arch/arm64/include/asm/dma-mapping.h b/arch/arm64/include/asm/dma-mapping.h index f0d6d0bfe55c..cfdb34bedbcd 100644 --- a/arch/arm64/include/asm/dma-mapping.h +++ b/arch/arm64/include/asm/dma-mapping.h @@ -22,8 +22,6 @@ #include #include -#include - #include #include @@ -86,28 +84,6 @@ static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t dev_addr) return (phys_addr_t)dev_addr; } -static inline int dma_mapping_error(struct device *dev, dma_addr_t dev_addr) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - debug_dma_mapping_error(dev, dev_addr); - return ops->mapping_error(dev, dev_addr); -} - -static inline int dma_supported(struct device *dev, u64 mask) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - return ops->dma_supported(dev, mask); -} - -static inline int dma_set_mask(struct device *dev, u64 mask) -{ - if (!dev->dma_mask || !dma_supported(dev, mask)) - return -EIO; - *dev->dma_mask = mask; - - return 0; -} - static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) { if (!dev->dma_mask) @@ -120,50 +96,5 @@ static inline void dma_mark_clean(void *addr, size_t size) { } -#define dma_alloc_coherent(d, s, h, f) dma_alloc_attrs(d, s, h, f, NULL) -#define dma_free_coherent(d, s, h, f) dma_free_attrs(d, s, h, f, NULL) - -static inline void *dma_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flags, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - void *vaddr; - - if (dma_alloc_from_coherent(dev, size, dma_handle, &vaddr)) - return vaddr; - - vaddr = ops->alloc(dev, size, dma_handle, flags, attrs); - debug_dma_alloc_coherent(dev, size, *dma_handle, vaddr); - return vaddr; -} - -static inline void dma_free_attrs(struct device *dev, size_t size, - void *vaddr, dma_addr_t dev_addr, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - - if (dma_release_from_coherent(dev, get_order(size), vaddr)) - return; - - debug_dma_free_coherent(dev, size, vaddr, dev_addr); - ops->free(dev, size, vaddr, dev_addr, attrs); -} - -/* - * There is no dma_cache_sync() implementation, so just return NULL here. - */ -static inline void *dma_alloc_noncoherent(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t flags) -{ - return NULL; -} - -static inline void dma_free_noncoherent(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t handle) -{ -} - #endif /* __KERNEL__ */ #endif /* __ASM_DMA_MAPPING_H */ diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index 70522450ca23..77eeb2cc648f 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -18,6 +18,8 @@ #ifndef __ASM_ESR_H #define __ASM_ESR_H +#include + #define ESR_ELx_EC_UNKNOWN (0x00) #define ESR_ELx_EC_WFx (0x01) /* Unallocated EC: 0x02 */ @@ -99,6 +101,13 @@ #define ESR_ELx_WFx_ISS_WFE (UL(1) << 0) #define ESR_ELx_xVC_IMM_MASK ((1UL << 16) - 1) +/* ESR value templates for specific events */ + +/* BRK instruction trap from AArch64 state */ +#define ESR_ELx_VAL_BRK64(imm) \ + ((ESR_ELx_EC_BRK64 << ESR_ELx_EC_SHIFT) | ESR_ELx_IL | \ + ((imm) & 0xffff)) + #ifndef __ASSEMBLY__ #include diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h index 0303705fcad6..6cb7e1a6bc02 100644 --- a/arch/arm64/include/asm/exception.h +++ b/arch/arm64/include/asm/exception.h @@ -18,7 +18,13 @@ #ifndef __ASM_EXCEPTION_H #define __ASM_EXCEPTION_H +#include + #define __exception __attribute__((section(".exception.text"))) +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +#define __exception_irq_entry __irq_entry +#else #define __exception_irq_entry __exception +#endif #endif /* __ASM_EXCEPTION_H */ diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h index c0739187a920..8b9884c726ad 100644 --- a/arch/arm64/include/asm/fixmap.h +++ b/arch/arm64/include/asm/fixmap.h @@ -8,7 +8,7 @@ * Copyright (C) 1998 Ingo Molnar * Copyright (C) 2013 Mark Salter * - * Adapted from arch/x86_64 version. + * Adapted from arch/x86 version. * */ diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h index 74069b3bd919..007a69fc4f40 100644 --- a/arch/arm64/include/asm/futex.h +++ b/arch/arm64/include/asm/futex.h @@ -20,10 +20,17 @@ #include #include + +#include +#include #include +#include #define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg) \ asm volatile( \ + ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) \ +" prfm pstl1strm, %2\n" \ "1: ldxr %w1, %2\n" \ insn "\n" \ "2: stlxr %w3, %w0, %2\n" \ @@ -39,6 +46,8 @@ " .align 3\n" \ " .quad 1b, 4b, 2b, 4b\n" \ " .popsection\n" \ + ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) \ : "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp) \ : "r" (oparg), "Ir" (-EFAULT) \ : "memory") @@ -112,6 +121,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, return -EFAULT; asm volatile("// futex_atomic_cmpxchg_inatomic\n" +" prfm pstl1strm, %2\n" "1: ldxr %w1, %2\n" " sub %w3, %w1, %w4\n" " cbnz %w3, 3f\n" diff --git a/arch/arm64/include/asm/hardirq.h b/arch/arm64/include/asm/hardirq.h index 6aae421f4d73..2bb7009bdac7 100644 --- a/arch/arm64/include/asm/hardirq.h +++ b/arch/arm64/include/asm/hardirq.h @@ -24,9 +24,7 @@ typedef struct { unsigned int __softirq_pending; -#ifdef CONFIG_SMP unsigned int ipi_irqs[NR_IPI]; -#endif } ____cacheline_aligned irq_cpustat_t; #include /* Standard mappings for irq_cpustat_t above */ @@ -34,10 +32,8 @@ typedef struct { #define __inc_irq_stat(cpu, member) __IRQ_STAT(cpu, member)++ #define __get_irq_stat(cpu, member) __IRQ_STAT(cpu, member) -#ifdef CONFIG_SMP u64 smp_irq_stat_cpu(unsigned int cpu); #define arch_irq_stat_cpu smp_irq_stat_cpu -#endif #define __ARCH_IRQ_EXIT_IRQS_DISABLED 1 diff --git a/arch/arm64/include/asm/hugetlb.h b/arch/arm64/include/asm/hugetlb.h index 2fd9b14ca295..bb4052e85dba 100644 --- a/arch/arm64/include/asm/hugetlb.h +++ b/arch/arm64/include/asm/hugetlb.h @@ -13,10 +13,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #ifndef __ASM_HUGETLB_H diff --git a/arch/arm64/include/asm/hw_breakpoint.h b/arch/arm64/include/asm/hw_breakpoint.h index 52b484b6aa1a..4c47cb2fbb52 100644 --- a/arch/arm64/include/asm/hw_breakpoint.h +++ b/arch/arm64/include/asm/hw_breakpoint.h @@ -16,6 +16,8 @@ #ifndef __ASM_HW_BREAKPOINT_H #define __ASM_HW_BREAKPOINT_H +#include + #ifdef __KERNEL__ struct arch_hw_breakpoint_ctrl { @@ -132,5 +134,17 @@ static inline void ptrace_hw_copy_thread(struct task_struct *task) extern struct pmu perf_ops_bp; +/* Determine number of BRP registers available. */ +static inline int get_num_brps(void) +{ + return ((read_cpuid(ID_AA64DFR0_EL1) >> 12) & 0xf) + 1; +} + +/* Determine number of WRP registers available. */ +static inline int get_num_wrps(void) +{ + return ((read_cpuid(ID_AA64DFR0_EL1) >> 20) & 0xf) + 1; +} + #endif /* __KERNEL__ */ #endif /* __ASM_BREAKPOINT_H */ diff --git a/arch/arm64/include/asm/irq_work.h b/arch/arm64/include/asm/irq_work.h index b4f6b19a8a68..8e24ef3f7c82 100644 --- a/arch/arm64/include/asm/irq_work.h +++ b/arch/arm64/include/asm/irq_work.h @@ -1,8 +1,6 @@ #ifndef __ASM_IRQ_WORK_H #define __ASM_IRQ_WORK_H -#ifdef CONFIG_SMP - #include static inline bool arch_irq_work_has_interrupt(void) @@ -10,13 +8,4 @@ static inline bool arch_irq_work_has_interrupt(void) return !!__smp_cross_call; } -#else - -static inline bool arch_irq_work_has_interrupt(void) -{ - return false; -} - -#endif - #endif /* __ASM_IRQ_WORK_H */ diff --git a/arch/arm64/include/asm/jump_label.h b/arch/arm64/include/asm/jump_label.h index c0e5165c2f76..1b5e0e843c3a 100644 --- a/arch/arm64/include/asm/jump_label.h +++ b/arch/arm64/include/asm/jump_label.h @@ -26,14 +26,28 @@ #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm goto("1: nop\n\t" ".pushsection __jump_table, \"aw\"\n\t" ".align 3\n\t" ".quad 1b, %l[l_yes], %c0\n\t" ".popsection\n\t" - : : "i"(key) : : l_yes); + : : "i"(&((char *)key)[branch]) : : l_yes); + + return false; +l_yes: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm goto("1: b %l[l_yes]\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".align 3\n\t" + ".quad 1b, %l[l_yes], %c0\n\t" + ".popsection\n\t" + : : "i"(&((char *)key)[branch]) : : l_yes); return false; l_yes: diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index ac6fafb95fe7..7605e095217f 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -171,10 +171,13 @@ #define HSTR_EL2_TTEE (1 << 16) #define HSTR_EL2_T(x) (1 << x) +/* Hyp Coproccessor Trap Register Shifts */ +#define CPTR_EL2_TFP_SHIFT 10 + /* Hyp Coprocessor Trap Register */ #define CPTR_EL2_TCPAC (1 << 31) #define CPTR_EL2_TTA (1 << 20) -#define CPTR_EL2_TFP (1 << 10) +#define CPTR_EL2_TFP (1 << CPTR_EL2_TFP_SHIFT) /* Hyp Debug Configuration Register bits */ #define MDCR_EL2_TDRA (1 << 11) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 3c5fe685a2d6..67fa0de3d483 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -46,24 +46,16 @@ #define CNTKCTL_EL1 20 /* Timer Control Register (EL1) */ #define PAR_EL1 21 /* Physical Address Register */ #define MDSCR_EL1 22 /* Monitor Debug System Control Register */ -#define DBGBCR0_EL1 23 /* Debug Breakpoint Control Registers (0-15) */ -#define DBGBCR15_EL1 38 -#define DBGBVR0_EL1 39 /* Debug Breakpoint Value Registers (0-15) */ -#define DBGBVR15_EL1 54 -#define DBGWCR0_EL1 55 /* Debug Watchpoint Control Registers (0-15) */ -#define DBGWCR15_EL1 70 -#define DBGWVR0_EL1 71 /* Debug Watchpoint Value Registers (0-15) */ -#define DBGWVR15_EL1 86 -#define MDCCINT_EL1 87 /* Monitor Debug Comms Channel Interrupt Enable Reg */ +#define MDCCINT_EL1 23 /* Monitor Debug Comms Channel Interrupt Enable Reg */ /* 32bit specific registers. Keep them at the end of the range */ -#define DACR32_EL2 88 /* Domain Access Control Register */ -#define IFSR32_EL2 89 /* Instruction Fault Status Register */ -#define FPEXC32_EL2 90 /* Floating-Point Exception Control Register */ -#define DBGVCR32_EL2 91 /* Debug Vector Catch Register */ -#define TEECR32_EL1 92 /* ThumbEE Configuration Register */ -#define TEEHBR32_EL1 93 /* ThumbEE Handler Base Register */ -#define NR_SYS_REGS 94 +#define DACR32_EL2 24 /* Domain Access Control Register */ +#define IFSR32_EL2 25 /* Instruction Fault Status Register */ +#define FPEXC32_EL2 26 /* Floating-Point Exception Control Register */ +#define DBGVCR32_EL2 27 /* Debug Vector Catch Register */ +#define TEECR32_EL1 28 /* ThumbEE Configuration Register */ +#define TEEHBR32_EL1 29 /* ThumbEE Handler Base Register */ +#define NR_SYS_REGS 30 /* 32bit mapping */ #define c0_MPIDR (MPIDR_EL1 * 2) /* MultiProcessor ID Register */ @@ -132,6 +124,8 @@ extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu); extern u64 __vgic_v3_get_ich_vtr_el2(void); +extern u32 __kvm_get_mdcr_el2(void); + #endif #endif /* __ARM_KVM_ASM_H__ */ diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 2709db2a7eac..415938dc45cf 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -103,15 +103,34 @@ struct kvm_vcpu_arch { /* HYP configuration */ u64 hcr_el2; + u32 mdcr_el2; /* Exception Information */ struct kvm_vcpu_fault_info fault; - /* Debug state */ + /* Guest debug state */ u64 debug_flags; + /* + * We maintain more than a single set of debug registers to support + * debugging the guest from the host and to maintain separate host and + * guest state during world switches. vcpu_debug_state are the debug + * registers of the vcpu as the guest sees them. host_debug_state are + * the host registers which are saved and restored during + * world switches. external_debug_state contains the debug + * values we want to debug the guest. This is set via the + * KVM_SET_GUEST_DEBUG ioctl. + * + * debug_ptr points to the set of debug registers that should be loaded + * onto the hardware when running the guest. + */ + struct kvm_guest_debug_arch *debug_ptr; + struct kvm_guest_debug_arch vcpu_debug_state; + struct kvm_guest_debug_arch external_debug_state; + /* Pointer to host CPU context */ kvm_cpu_context_t *host_cpu_context; + struct kvm_guest_debug_arch host_debug_state; /* VGIC state */ struct vgic_cpu vgic_cpu; @@ -122,6 +141,17 @@ struct kvm_vcpu_arch { * here. */ + /* + * Guest registers we preserve during guest debugging. + * + * These shadow registers are updated by the kvm_handle_sys_reg + * trap handler if the guest accesses or updates them while we + * are using guest debug. + */ + struct { + u32 mdscr_el1; + } guest_debug_preserved; + /* Don't run the guest */ bool pause; @@ -216,15 +246,15 @@ static inline void __cpu_init_hyp_mode(phys_addr_t boot_pgd_ptr, hyp_stack_ptr, vector_ptr); } -struct vgic_sr_vectors { - void *save_vgic; - void *restore_vgic; -}; - static inline void kvm_arch_hardware_disable(void) {} static inline void kvm_arch_hardware_unsetup(void) {} static inline void kvm_arch_sync_events(struct kvm *kvm) {} static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {} static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {} +void kvm_arm_init_debug(void); +void kvm_arm_setup_debug(struct kvm_vcpu *vcpu); +void kvm_arm_clear_debug(struct kvm_vcpu *vcpu); +void kvm_arm_reset_debug_ptr(struct kvm_vcpu *vcpu); + #endif /* __ARM64_KVM_HOST_H__ */ diff --git a/arch/arm64/include/asm/lse.h b/arch/arm64/include/asm/lse.h new file mode 100644 index 000000000000..3de42d68611d --- /dev/null +++ b/arch/arm64/include/asm/lse.h @@ -0,0 +1,53 @@ +#ifndef __ASM_LSE_H +#define __ASM_LSE_H + +#if defined(CONFIG_AS_LSE) && defined(CONFIG_ARM64_LSE_ATOMICS) + +#include +#include +#include + +#ifdef __ASSEMBLER__ + +.arch_extension lse + +.macro alt_lse, llsc, lse + alternative_insn "\llsc", "\lse", ARM64_HAS_LSE_ATOMICS +.endm + +#else /* __ASSEMBLER__ */ + +__asm__(".arch_extension lse"); + +/* Move the ll/sc atomics out-of-line */ +#define __LL_SC_INLINE +#define __LL_SC_PREFIX(x) __ll_sc_##x +#define __LL_SC_EXPORT(x) EXPORT_SYMBOL(__LL_SC_PREFIX(x)) + +/* Macro for constructing calls to out-of-line ll/sc atomics */ +#define __LL_SC_CALL(op) "bl\t" __stringify(__LL_SC_PREFIX(op)) "\n" + +/* In-line patching at runtime */ +#define ARM64_LSE_ATOMIC_INSN(llsc, lse) \ + ALTERNATIVE(llsc, lse, ARM64_HAS_LSE_ATOMICS) + +#endif /* __ASSEMBLER__ */ +#else /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */ + +#ifdef __ASSEMBLER__ + +.macro alt_lse, llsc, lse + \llsc +.endm + +#else /* __ASSEMBLER__ */ + +#define __LL_SC_INLINE static inline +#define __LL_SC_PREFIX(x) x +#define __LL_SC_EXPORT(x) + +#define ARM64_LSE_ATOMIC_INSN(llsc, lse) llsc + +#endif /* __ASSEMBLER__ */ +#endif /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */ +#endif /* __ASM_LSE_H */ diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index f800d45ea226..6b4c3ad75a2a 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -80,12 +80,6 @@ #define __virt_to_phys(x) (((phys_addr_t)(x) - PAGE_OFFSET + PHYS_OFFSET)) #define __phys_to_virt(x) ((unsigned long)((x) - PHYS_OFFSET + PAGE_OFFSET)) -/* - * Convert a physical address to a Page Frame Number and back - */ -#define __phys_to_pfn(paddr) ((unsigned long)((paddr) >> PAGE_SHIFT)) -#define __pfn_to_phys(pfn) ((phys_addr_t)(pfn) << PAGE_SHIFT) - /* * Convert a page to/from a physical address */ @@ -113,6 +107,14 @@ extern phys_addr_t memstart_addr; /* PHYS_OFFSET - the physical address of the start of memory. */ #define PHYS_OFFSET ({ memstart_addr; }) +/* + * The maximum physical address that the linear direct mapping + * of system RAM can cover. (PAGE_OFFSET can be interpreted as + * a 2's complement signed quantity and negated to derive the + * maximum size of the linear mapping.) + */ +#define MAX_MEMBLOCK_ADDR ({ memstart_addr - PAGE_OFFSET - 1; }) + /* * PFNs are used to describe any physical page; this means * PFN 0 == physical address 0. diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 79fcfb048884..030208767185 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -28,7 +28,6 @@ typedef struct { #define ASID(mm) ((mm)->context.id & 0xffff) extern void paging_init(void); -extern void setup_mm_for_reboot(void); extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt); extern void init_mem_pgprot(void); extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h index 4fde8c1df97f..0a456bef8c79 100644 --- a/arch/arm64/include/asm/percpu.h +++ b/arch/arm64/include/asm/percpu.h @@ -16,8 +16,6 @@ #ifndef __ASM_PERCPU_H #define __ASM_PERCPU_H -#ifdef CONFIG_SMP - static inline void set_my_cpu_offset(unsigned long off) { asm volatile("msr tpidr_el1, %0" :: "r" (off) : "memory"); @@ -38,12 +36,6 @@ static inline unsigned long __my_cpu_offset(void) } #define __my_cpu_offset __my_cpu_offset() -#else /* !CONFIG_SMP */ - -#define set_my_cpu_offset(x) do { } while (0) - -#endif /* CONFIG_SMP */ - #define PERCPU_OP(op, asm_op) \ static inline unsigned long __percpu_##op(void *ptr, \ unsigned long val, int size) \ diff --git a/arch/arm64/include/asm/perf_event.h b/arch/arm64/include/asm/perf_event.h index 6471773db6fd..7bd3cdb533ea 100644 --- a/arch/arm64/include/asm/perf_event.h +++ b/arch/arm64/include/asm/perf_event.h @@ -17,7 +17,7 @@ #ifndef __ASM_PERF_EVENT_H #define __ASM_PERF_EVENT_H -#ifdef CONFIG_HW_PERF_EVENTS +#ifdef CONFIG_PERF_EVENTS struct pt_regs; extern unsigned long perf_instruction_pointer(struct pt_regs *regs); extern unsigned long perf_misc_flags(struct pt_regs *regs); diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h index 59bfae75dc98..24154b055835 100644 --- a/arch/arm64/include/asm/pgtable-hwdef.h +++ b/arch/arm64/include/asm/pgtable-hwdef.h @@ -104,6 +104,7 @@ #define PTE_SHARED (_AT(pteval_t, 3) << 8) /* SH[1:0], inner shareable */ #define PTE_AF (_AT(pteval_t, 1) << 10) /* Access Flag */ #define PTE_NG (_AT(pteval_t, 1) << 11) /* nG */ +#define PTE_DBM (_AT(pteval_t, 1) << 51) /* Dirty Bit Management */ #define PTE_PXN (_AT(pteval_t, 1) << 53) /* Privileged XN */ #define PTE_UXN (_AT(pteval_t, 1) << 54) /* User XN */ @@ -168,5 +169,7 @@ #define TCR_TG1_64K (UL(3) << 30) #define TCR_ASID16 (UL(1) << 36) #define TCR_TBI0 (UL(1) << 37) +#define TCR_HA (UL(1) << 39) +#define TCR_HD (UL(1) << 40) #endif diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 56283f8a675c..6900b2d95371 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -16,6 +16,7 @@ #ifndef __ASM_PGTABLE_H #define __ASM_PGTABLE_H +#include #include #include @@ -27,7 +28,11 @@ #define PTE_VALID (_AT(pteval_t, 1) << 0) #define PTE_DIRTY (_AT(pteval_t, 1) << 55) #define PTE_SPECIAL (_AT(pteval_t, 1) << 56) +#ifdef CONFIG_ARM64_HW_AFDBM +#define PTE_WRITE (PTE_DBM) /* same as DBM */ +#else #define PTE_WRITE (_AT(pteval_t, 1) << 57) +#endif #define PTE_PROT_NONE (_AT(pteval_t, 1) << 58) /* only when !PTE_VALID */ /* @@ -48,18 +53,16 @@ #define FIRST_USER_ADDRESS 0UL #ifndef __ASSEMBLY__ + +#include + extern void __pte_error(const char *file, int line, unsigned long val); extern void __pmd_error(const char *file, int line, unsigned long val); extern void __pud_error(const char *file, int line, unsigned long val); extern void __pgd_error(const char *file, int line, unsigned long val); -#ifdef CONFIG_SMP #define PROT_DEFAULT (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED) #define PROT_SECT_DEFAULT (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S) -#else -#define PROT_DEFAULT (PTE_TYPE_PAGE | PTE_AF) -#define PROT_SECT_DEFAULT (PMD_TYPE_SECT | PMD_SECT_AF) -#endif #define PROT_DEVICE_nGnRE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_DEVICE_nGnRE)) #define PROT_NORMAL_NC (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_NORMAL_NC)) @@ -137,12 +140,20 @@ extern struct page *empty_zero_page; * The following only work if pte_present(). Undefined behaviour otherwise. */ #define pte_present(pte) (!!(pte_val(pte) & (PTE_VALID | PTE_PROT_NONE))) -#define pte_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY)) #define pte_young(pte) (!!(pte_val(pte) & PTE_AF)) #define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL)) #define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE)) #define pte_exec(pte) (!(pte_val(pte) & PTE_UXN)) +#ifdef CONFIG_ARM64_HW_AFDBM +#define pte_hw_dirty(pte) (!(pte_val(pte) & PTE_RDONLY)) +#else +#define pte_hw_dirty(pte) (0) +#endif +#define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY)) +#define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte)) + +#define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID)) #define pte_valid_user(pte) \ ((pte_val(pte) & (PTE_VALID | PTE_USER)) == (PTE_VALID | PTE_USER)) #define pte_valid_not_user(pte) \ @@ -209,20 +220,49 @@ static inline void set_pte(pte_t *ptep, pte_t pte) } } +struct mm_struct; +struct vm_area_struct; + extern void __sync_icache_dcache(pte_t pteval, unsigned long addr); +/* + * PTE bits configuration in the presence of hardware Dirty Bit Management + * (PTE_WRITE == PTE_DBM): + * + * Dirty Writable | PTE_RDONLY PTE_WRITE PTE_DIRTY (sw) + * 0 0 | 1 0 0 + * 0 1 | 1 1 0 + * 1 0 | 1 0 1 + * 1 1 | 0 1 x + * + * When hardware DBM is not present, the sofware PTE_DIRTY bit is updated via + * the page fault mechanism. Checking the dirty status of a pte becomes: + * + * PTE_DIRTY || !PTE_RDONLY + */ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) { if (pte_valid_user(pte)) { if (!pte_special(pte) && pte_exec(pte)) __sync_icache_dcache(pte, addr); - if (pte_dirty(pte) && pte_write(pte)) + if (pte_sw_dirty(pte) && pte_write(pte)) pte_val(pte) &= ~PTE_RDONLY; else pte_val(pte) |= PTE_RDONLY; } + /* + * If the existing pte is valid, check for potential race with + * hardware updates of the pte (ptep_set_access_flags safely changes + * valid ptes without going through an invalid entry). + */ + if (IS_ENABLED(CONFIG_DEBUG_VM) && IS_ENABLED(CONFIG_ARM64_HW_AFDBM) && + pte_valid(*ptep)) { + BUG_ON(!pte_young(pte)); + BUG_ON(pte_write(*ptep) && !pte_dirty(pte)); + } + set_pte(ptep, pte); } @@ -461,6 +501,9 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY | PTE_PROT_NONE | PTE_WRITE | PTE_TYPE_MASK; + /* preserve the hardware dirty information */ + if (pte_hw_dirty(pte)) + newprot |= PTE_DIRTY; pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask); return pte; } @@ -470,6 +513,101 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) return pte_pmd(pte_modify(pmd_pte(pmd), newprot)); } +#ifdef CONFIG_ARM64_HW_AFDBM +/* + * Atomic pte/pmd modifications. + */ +#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG +static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, + pte_t *ptep) +{ + pteval_t pteval; + unsigned int tmp, res; + + asm volatile("// ptep_test_and_clear_young\n" + " prfm pstl1strm, %2\n" + "1: ldxr %0, %2\n" + " ubfx %w3, %w0, %5, #1 // extract PTE_AF (young)\n" + " and %0, %0, %4 // clear PTE_AF\n" + " stxr %w1, %0, %2\n" + " cbnz %w1, 1b\n" + : "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)), "=&r" (res) + : "L" (~PTE_AF), "I" (ilog2(PTE_AF))); + + return res; +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG +static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, + pmd_t *pmdp) +{ + return ptep_test_and_clear_young(vma, address, (pte_t *)pmdp); +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +#define __HAVE_ARCH_PTEP_GET_AND_CLEAR +static inline pte_t ptep_get_and_clear(struct mm_struct *mm, + unsigned long address, pte_t *ptep) +{ + pteval_t old_pteval; + unsigned int tmp; + + asm volatile("// ptep_get_and_clear\n" + " prfm pstl1strm, %2\n" + "1: ldxr %0, %2\n" + " stxr %w1, xzr, %2\n" + " cbnz %w1, 1b\n" + : "=&r" (old_pteval), "=&r" (tmp), "+Q" (pte_val(*ptep))); + + return __pte(old_pteval); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define __HAVE_ARCH_PMDP_GET_AND_CLEAR +static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, + unsigned long address, pmd_t *pmdp) +{ + return pte_pmd(ptep_get_and_clear(mm, address, (pte_t *)pmdp)); +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +/* + * ptep_set_wrprotect - mark read-only while trasferring potential hardware + * dirty status (PTE_DBM && !PTE_RDONLY) to the software PTE_DIRTY bit. + */ +#define __HAVE_ARCH_PTEP_SET_WRPROTECT +static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep) +{ + pteval_t pteval; + unsigned long tmp; + + asm volatile("// ptep_set_wrprotect\n" + " prfm pstl1strm, %2\n" + "1: ldxr %0, %2\n" + " tst %0, %4 // check for hw dirty (!PTE_RDONLY)\n" + " csel %1, %3, xzr, eq // set PTE_DIRTY|PTE_RDONLY if dirty\n" + " orr %0, %0, %1 // if !dirty, PTE_RDONLY is already set\n" + " and %0, %0, %5 // clear PTE_WRITE/PTE_DBM\n" + " stxr %w1, %0, %2\n" + " cbnz %w1, 1b\n" + : "=&r" (pteval), "=&r" (tmp), "+Q" (pte_val(*ptep)) + : "r" (PTE_DIRTY|PTE_RDONLY), "L" (PTE_RDONLY), "L" (~PTE_WRITE) + : "cc"); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define __HAVE_ARCH_PMDP_SET_WRPROTECT +static inline void pmdp_set_wrprotect(struct mm_struct *mm, + unsigned long address, pmd_t *pmdp) +{ + ptep_set_wrprotect(mm, address, (pte_t *)pmdp); +} +#endif +#endif /* CONFIG_ARM64_HW_AFDBM */ + extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; extern pgd_t idmap_pg_dir[PTRS_PER_PGD]; @@ -505,6 +643,21 @@ extern int kern_addr_valid(unsigned long addr); #define pgtable_cache_init() do { } while (0) +/* + * On AArch64, the cache coherency is handled via the set_pte_at() function. + */ +static inline void update_mmu_cache(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + /* + * set_pte() does not have a DSB for user mappings, so make sure that + * the page table write is visible. + */ + dsb(ishst); +} + +#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0) + #endif /* !__ASSEMBLY__ */ #endif /* __ASM_PGTABLE_H */ diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index e4c893e54f01..98f32355dc97 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -186,4 +186,6 @@ static inline void spin_lock_prefetch(const void *x) #endif +void cpu_enable_pan(void); + #endif /* __ASM_PROCESSOR_H */ diff --git a/arch/arm64/include/asm/psci.h b/arch/arm64/include/asm/psci.h deleted file mode 100644 index 49d7e1aaebdc..000000000000 --- a/arch/arm64/include/asm/psci.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * Copyright (C) 2013 ARM Limited - */ - -#ifndef __ASM_PSCI_H -#define __ASM_PSCI_H - -int __init psci_dt_init(void); - -#ifdef CONFIG_ACPI -int __init psci_acpi_init(void); -bool __init acpi_psci_present(void); -bool __init acpi_psci_use_hvc(void); -#else -static inline int psci_acpi_init(void) { return 0; } -static inline bool acpi_psci_present(void) { return false; } -#endif - -#endif /* __ASM_PSCI_H */ diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h index d6dd9fdbc3be..536274ed292e 100644 --- a/arch/arm64/include/asm/ptrace.h +++ b/arch/arm64/include/asm/ptrace.h @@ -183,11 +183,7 @@ static inline int valid_user_regs(struct user_pt_regs *regs) #define instruction_pointer(regs) ((unsigned long)(regs)->pc) -#ifdef CONFIG_SMP extern unsigned long profile_pc(struct pt_regs *regs); -#else -#define profile_pc(regs) instruction_pointer(regs) -#endif #endif /* __ASSEMBLY__ */ #endif diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h index db02be81b90a..d9c3d6a6100a 100644 --- a/arch/arm64/include/asm/smp.h +++ b/arch/arm64/include/asm/smp.h @@ -20,10 +20,6 @@ #include #include -#ifndef CONFIG_SMP -# error " included in non-SMP build" -#endif - #define raw_smp_processor_id() (current_thread_info()->cpu) struct seq_file; diff --git a/arch/arm64/include/asm/smp_plat.h b/arch/arm64/include/asm/smp_plat.h index 7abf7570c00f..af58dcdefb21 100644 --- a/arch/arm64/include/asm/smp_plat.h +++ b/arch/arm64/include/asm/smp_plat.h @@ -56,6 +56,4 @@ static inline int get_logical_index(u64 mpidr) return -EINVAL; } -void __init do_post_cpus_up_work(void); - #endif /* __ASM_SMP_PLAT_H */ diff --git a/arch/arm64/include/asm/spinlock.h b/arch/arm64/include/asm/spinlock.h index cee128732435..c85e96d174a5 100644 --- a/arch/arm64/include/asm/spinlock.h +++ b/arch/arm64/include/asm/spinlock.h @@ -16,6 +16,7 @@ #ifndef __ASM_SPINLOCK_H #define __ASM_SPINLOCK_H +#include #include #include @@ -38,11 +39,21 @@ static inline void arch_spin_lock(arch_spinlock_t *lock) asm volatile( /* Atomically increment the next ticket. */ + ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ " prfm pstl1strm, %3\n" "1: ldaxr %w0, %3\n" " add %w1, %w0, %w5\n" " stxr %w2, %w1, %3\n" -" cbnz %w2, 1b\n" +" cbnz %w2, 1b\n", + /* LSE atomics */ +" mov %w2, %w5\n" +" ldadda %w2, %w0, %3\n" +" nop\n" +" nop\n" +" nop\n" + ) + /* Did we get the lock? */ " eor %w1, %w0, %w0, ror #16\n" " cbz %w1, 3f\n" @@ -67,15 +78,25 @@ static inline int arch_spin_trylock(arch_spinlock_t *lock) unsigned int tmp; arch_spinlock_t lockval; - asm volatile( -" prfm pstl1strm, %2\n" -"1: ldaxr %w0, %2\n" -" eor %w1, %w0, %w0, ror #16\n" -" cbnz %w1, 2f\n" -" add %w0, %w0, %3\n" -" stxr %w1, %w0, %2\n" -" cbnz %w1, 1b\n" -"2:" + asm volatile(ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ + " prfm pstl1strm, %2\n" + "1: ldaxr %w0, %2\n" + " eor %w1, %w0, %w0, ror #16\n" + " cbnz %w1, 2f\n" + " add %w0, %w0, %3\n" + " stxr %w1, %w0, %2\n" + " cbnz %w1, 1b\n" + "2:", + /* LSE atomics */ + " ldr %w0, %2\n" + " eor %w1, %w0, %w0, ror #16\n" + " cbnz %w1, 1f\n" + " add %w1, %w0, %3\n" + " casa %w0, %w1, %2\n" + " and %w1, %w1, #0xffff\n" + " eor %w1, %w1, %w0, lsr #16\n" + "1:") : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock) : "I" (1 << TICKET_SHIFT) : "memory"); @@ -85,10 +106,19 @@ static inline int arch_spin_trylock(arch_spinlock_t *lock) static inline void arch_spin_unlock(arch_spinlock_t *lock) { - asm volatile( -" stlrh %w1, %0\n" - : "=Q" (lock->owner) - : "r" (lock->owner + 1) + unsigned long tmp; + + asm volatile(ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ + " ldrh %w1, %0\n" + " add %w1, %w1, #1\n" + " stlrh %w1, %0", + /* LSE atomics */ + " mov %w1, #1\n" + " nop\n" + " staddlh %w1, %0") + : "=Q" (lock->owner), "=&r" (tmp) + : : "memory"); } @@ -123,13 +153,24 @@ static inline void arch_write_lock(arch_rwlock_t *rw) { unsigned int tmp; - asm volatile( + asm volatile(ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ " sevl\n" "1: wfe\n" "2: ldaxr %w0, %1\n" " cbnz %w0, 1b\n" " stxr %w0, %w2, %1\n" " cbnz %w0, 2b\n" + " nop", + /* LSE atomics */ + "1: mov %w0, wzr\n" + "2: casa %w0, %w2, %1\n" + " cbz %w0, 3f\n" + " ldxr %w0, %1\n" + " cbz %w0, 2b\n" + " wfe\n" + " b 1b\n" + "3:") : "=&r" (tmp), "+Q" (rw->lock) : "r" (0x80000000) : "memory"); @@ -139,11 +180,18 @@ static inline int arch_write_trylock(arch_rwlock_t *rw) { unsigned int tmp; - asm volatile( - " ldaxr %w0, %1\n" - " cbnz %w0, 1f\n" + asm volatile(ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ + "1: ldaxr %w0, %1\n" + " cbnz %w0, 2f\n" " stxr %w0, %w2, %1\n" - "1:\n" + " cbnz %w0, 1b\n" + "2:", + /* LSE atomics */ + " mov %w0, wzr\n" + " casa %w0, %w2, %1\n" + " nop\n" + " nop") : "=&r" (tmp), "+Q" (rw->lock) : "r" (0x80000000) : "memory"); @@ -153,9 +201,10 @@ static inline int arch_write_trylock(arch_rwlock_t *rw) static inline void arch_write_unlock(arch_rwlock_t *rw) { - asm volatile( - " stlr %w1, %0\n" - : "=Q" (rw->lock) : "r" (0) : "memory"); + asm volatile(ARM64_LSE_ATOMIC_INSN( + " stlr wzr, %0", + " swpl wzr, wzr, %0") + : "=Q" (rw->lock) :: "memory"); } /* write_can_lock - would write_trylock() succeed? */ @@ -172,6 +221,10 @@ static inline void arch_write_unlock(arch_rwlock_t *rw) * * The memory barriers are implicit with the load-acquire and store-release * instructions. + * + * Note that in UNDEFINED cases, such as unlocking a lock twice, the LL/SC + * and LSE implementations may exhibit different behaviour (although this + * will have no effect on lockdep). */ static inline void arch_read_lock(arch_rwlock_t *rw) { @@ -179,26 +232,43 @@ static inline void arch_read_lock(arch_rwlock_t *rw) asm volatile( " sevl\n" + ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ "1: wfe\n" "2: ldaxr %w0, %2\n" " add %w0, %w0, #1\n" " tbnz %w0, #31, 1b\n" " stxr %w1, %w0, %2\n" - " cbnz %w1, 2b\n" + " nop\n" + " cbnz %w1, 2b", + /* LSE atomics */ + "1: wfe\n" + "2: ldxr %w0, %2\n" + " adds %w1, %w0, #1\n" + " tbnz %w1, #31, 1b\n" + " casa %w0, %w1, %2\n" + " sbc %w0, %w1, %w0\n" + " cbnz %w0, 2b") : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) : - : "memory"); + : "cc", "memory"); } static inline void arch_read_unlock(arch_rwlock_t *rw) { unsigned int tmp, tmp2; - asm volatile( + asm volatile(ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ "1: ldxr %w0, %2\n" " sub %w0, %w0, #1\n" " stlxr %w1, %w0, %2\n" - " cbnz %w1, 1b\n" + " cbnz %w1, 1b", + /* LSE atomics */ + " movn %w0, #0\n" + " nop\n" + " nop\n" + " staddl %w0, %2") : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) : : "memory"); @@ -206,17 +276,28 @@ static inline void arch_read_unlock(arch_rwlock_t *rw) static inline int arch_read_trylock(arch_rwlock_t *rw) { - unsigned int tmp, tmp2 = 1; + unsigned int tmp, tmp2; - asm volatile( - " ldaxr %w0, %2\n" + asm volatile(ARM64_LSE_ATOMIC_INSN( + /* LL/SC */ + " mov %w1, #1\n" + "1: ldaxr %w0, %2\n" " add %w0, %w0, #1\n" - " tbnz %w0, #31, 1f\n" + " tbnz %w0, #31, 2f\n" " stxr %w1, %w0, %2\n" - "1:\n" - : "=&r" (tmp), "+r" (tmp2), "+Q" (rw->lock) + " cbnz %w1, 1b\n" + "2:", + /* LSE atomics */ + " ldr %w0, %2\n" + " adds %w1, %w0, #1\n" + " tbnz %w1, #31, 1f\n" + " casa %w0, %w1, %2\n" + " sbc %w1, %w1, %w0\n" + " nop\n" + "1:") + : "=&r" (tmp), "=&r" (tmp2), "+Q" (rw->lock) : - : "memory"); + : "cc", "memory"); return !tmp2; } diff --git a/arch/arm64/include/asm/spinlock_types.h b/arch/arm64/include/asm/spinlock_types.h index b8d383665f56..55be59a35e3f 100644 --- a/arch/arm64/include/asm/spinlock_types.h +++ b/arch/arm64/include/asm/spinlock_types.h @@ -20,6 +20,8 @@ # error "please don't include this file directly" #endif +#include + #define TICKET_SHIFT 16 typedef struct { diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 5c89df0acbcb..a7f3d4b2514d 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -20,8 +20,29 @@ #ifndef __ASM_SYSREG_H #define __ASM_SYSREG_H +#include + +#define SCTLR_EL1_CP15BEN (0x1 << 5) +#define SCTLR_EL1_SED (0x1 << 8) + +/* + * ARMv8 ARM reserves the following encoding for system registers: + * (Ref: ARMv8 ARM, Section: "System instruction class encoding overview", + * C5.2, version:ARM DDI 0487A.f) + * [20-19] : Op0 + * [18-16] : Op1 + * [15-12] : CRn + * [11-8] : CRm + * [7-5] : Op2 + */ #define sys_reg(op0, op1, crn, crm, op2) \ - ((((op0)-2)<<19)|((op1)<<16)|((crn)<<12)|((crm)<<8)|((op2)<<5)) + ((((op0)&3)<<19)|((op1)<<16)|((crn)<<12)|((crm)<<8)|((op2)<<5)) + +#define REG_PSTATE_PAN_IMM sys_reg(0, 0, 4, 0, 4) +#define SCTLR_EL1_SPAN (1 << 23) + +#define SET_PSTATE_PAN(x) __inst_arm(0xd5000000 | REG_PSTATE_PAN_IMM |\ + (!!x)<<8 | 0x1f) #ifdef __ASSEMBLY__ @@ -31,11 +52,11 @@ .equ __reg_num_xzr, 31 .macro mrs_s, rt, sreg - .inst 0xd5300000|(\sreg)|(__reg_num_\rt) + .inst 0xd5200000|(\sreg)|(__reg_num_\rt) .endm .macro msr_s, sreg, rt - .inst 0xd5100000|(\sreg)|(__reg_num_\rt) + .inst 0xd5000000|(\sreg)|(__reg_num_\rt) .endm #else @@ -47,14 +68,23 @@ asm( " .equ __reg_num_xzr, 31\n" "\n" " .macro mrs_s, rt, sreg\n" -" .inst 0xd5300000|(\\sreg)|(__reg_num_\\rt)\n" +" .inst 0xd5200000|(\\sreg)|(__reg_num_\\rt)\n" " .endm\n" "\n" " .macro msr_s, sreg, rt\n" -" .inst 0xd5100000|(\\sreg)|(__reg_num_\\rt)\n" +" .inst 0xd5000000|(\\sreg)|(__reg_num_\\rt)\n" " .endm\n" ); +static inline void config_sctlr_el1(u32 clear, u32 set) +{ + u32 val; + + asm volatile("mrs %0, sctlr_el1" : "=r" (val)); + val &= ~clear; + val |= set; + asm volatile("msr sctlr_el1, %0" : : "r" (val)); +} #endif #endif /* __ASM_SYSREG_H */ diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h index 3a0242c7eb8d..d6e6b6660380 100644 --- a/arch/arm64/include/asm/tlb.h +++ b/arch/arm64/include/asm/tlb.h @@ -41,7 +41,12 @@ static inline void tlb_flush(struct mmu_gather *tlb) flush_tlb_mm(tlb->mm); } else { struct vm_area_struct vma = { .vm_mm = tlb->mm, }; - flush_tlb_range(&vma, tlb->start, tlb->end); + /* + * The intermediate page table levels are already handled by + * the __(pte|pmd|pud)_free_tlb() functions, so last level + * TLBI is sufficient here. + */ + __flush_tlb_range(&vma, tlb->start, tlb->end, true); } } diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index 934815d45eda..7bd2da021658 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -87,58 +87,64 @@ static inline void flush_tlb_page(struct vm_area_struct *vma, ((unsigned long)ASID(vma->vm_mm) << 48); dsb(ishst); - asm("tlbi vae1is, %0" : : "r" (addr)); + asm("tlbi vale1is, %0" : : "r" (addr)); dsb(ish); } -static inline void __flush_tlb_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end) -{ - unsigned long asid = (unsigned long)ASID(vma->vm_mm) << 48; - unsigned long addr; - start = asid | (start >> 12); - end = asid | (end >> 12); - - dsb(ishst); - for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12)) - asm("tlbi vae1is, %0" : : "r"(addr)); - dsb(ish); -} - -static inline void __flush_tlb_kernel_range(unsigned long start, unsigned long end) -{ - unsigned long addr; - start >>= 12; - end >>= 12; - - dsb(ishst); - for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12)) - asm("tlbi vaae1is, %0" : : "r"(addr)); - dsb(ish); - isb(); -} - /* * This is meant to avoid soft lock-ups on large TLB flushing ranges and not * necessarily a performance improvement. */ #define MAX_TLB_RANGE (1024UL << PAGE_SHIFT) +static inline void __flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end, + bool last_level) +{ + unsigned long asid = (unsigned long)ASID(vma->vm_mm) << 48; + unsigned long addr; + + if ((end - start) > MAX_TLB_RANGE) { + flush_tlb_mm(vma->vm_mm); + return; + } + + start = asid | (start >> 12); + end = asid | (end >> 12); + + dsb(ishst); + for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12)) { + if (last_level) + asm("tlbi vale1is, %0" : : "r"(addr)); + else + asm("tlbi vae1is, %0" : : "r"(addr)); + } + dsb(ish); +} + static inline void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - if ((end - start) <= MAX_TLB_RANGE) - __flush_tlb_range(vma, start, end); - else - flush_tlb_mm(vma->vm_mm); + __flush_tlb_range(vma, start, end, false); } static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end) { - if ((end - start) <= MAX_TLB_RANGE) - __flush_tlb_kernel_range(start, end); - else + unsigned long addr; + + if ((end - start) > MAX_TLB_RANGE) { flush_tlb_all(); + return; + } + + start >>= 12; + end >>= 12; + + dsb(ishst); + for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12)) + asm("tlbi vaae1is, %0" : : "r"(addr)); + dsb(ish); + isb(); } /* @@ -154,20 +160,6 @@ static inline void __flush_tlb_pgtable(struct mm_struct *mm, asm("tlbi vae1is, %0" : : "r" (addr)); dsb(ish); } -/* - * On AArch64, the cache coherency is handled via the set_pte_at() function. - */ -static inline void update_mmu_cache(struct vm_area_struct *vma, - unsigned long addr, pte_t *ptep) -{ - /* - * set_pte() does not have a DSB for user mappings, so make sure that - * the page table write is visible. - */ - dsb(ishst); -} - -#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0) #endif diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h index 225ec3524fbf..a3e9d6fdbf21 100644 --- a/arch/arm64/include/asm/topology.h +++ b/arch/arm64/include/asm/topology.h @@ -1,8 +1,6 @@ #ifndef __ASM_TOPOLOGY_H #define __ASM_TOPOLOGY_H -#ifdef CONFIG_SMP - #include struct cpu_topology { @@ -24,13 +22,6 @@ void init_cpu_topology(void); void store_cpu_topology(unsigned int cpuid); const struct cpumask *cpu_coregroup_mask(int cpu); -#else - -static inline void init_cpu_topology(void) { } -static inline void store_cpu_topology(unsigned int cpuid) { } - -#endif - #include #endif /* _ASM_ARM_TOPOLOGY_H */ diff --git a/arch/arm64/include/asm/traps.h b/arch/arm64/include/asm/traps.h index 232e4ba5d314..0cc2f29bf9da 100644 --- a/arch/arm64/include/asm/traps.h +++ b/arch/arm64/include/asm/traps.h @@ -34,13 +34,32 @@ struct undef_hook { void register_undef_hook(struct undef_hook *hook); void unregister_undef_hook(struct undef_hook *hook); +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +static inline int __in_irqentry_text(unsigned long ptr) +{ + extern char __irqentry_text_start[]; + extern char __irqentry_text_end[]; + + return ptr >= (unsigned long)&__irqentry_text_start && + ptr < (unsigned long)&__irqentry_text_end; +} +#else +static inline int __in_irqentry_text(unsigned long ptr) +{ + return 0; +} +#endif + static inline int in_exception_text(unsigned long ptr) { extern char __exception_text_start[]; extern char __exception_text_end[]; + int in; - return ptr >= (unsigned long)&__exception_text_start && - ptr < (unsigned long)&__exception_text_end; + in = ptr >= (unsigned long)&__exception_text_start && + ptr < (unsigned long)&__exception_text_end; + + return in ? : __in_irqentry_text(ptr); } #endif diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 07e1ba449bf1..b2ede967fe7d 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -24,7 +24,10 @@ #include #include +#include +#include #include +#include #include #include #include @@ -131,6 +134,8 @@ static inline void set_fs(mm_segment_t fs) do { \ unsigned long __gu_val; \ __chk_user_ptr(ptr); \ + asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN)); \ switch (sizeof(*(ptr))) { \ case 1: \ __get_user_asm("ldrb", "%w", __gu_val, (ptr), (err)); \ @@ -148,6 +153,8 @@ do { \ BUILD_BUG(); \ } \ (x) = (__force __typeof__(*(ptr)))__gu_val; \ + asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN)); \ } while (0) #define __get_user(x, ptr) \ @@ -194,6 +201,8 @@ do { \ do { \ __typeof__(*(ptr)) __pu_val = (x); \ __chk_user_ptr(ptr); \ + asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN)); \ switch (sizeof(*(ptr))) { \ case 1: \ __put_user_asm("strb", "%w", __pu_val, (ptr), (err)); \ @@ -210,6 +219,8 @@ do { \ default: \ BUILD_BUG(); \ } \ + asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN)); \ } while (0) #define __put_user(x, ptr) \ diff --git a/arch/arm64/include/asm/xen/events.h b/arch/arm64/include/asm/xen/events.h index 86553213c132..4318866d053c 100644 --- a/arch/arm64/include/asm/xen/events.h +++ b/arch/arm64/include/asm/xen/events.h @@ -18,4 +18,10 @@ static inline int xen_irqs_disabled(struct pt_regs *regs) #define xchg_xen_ulong(ptr, val) xchg((ptr), (val)) +/* Rebind event channel is supported by default */ +static inline bool xen_support_evtchn_rebind(void) +{ + return true; +} + #endif /* _ASM_ARM64_XEN_EVENTS_H */ diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index 73cf0f54d57c..361c8a8ef55f 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -27,5 +27,6 @@ #define HWCAP_SHA1 (1 << 5) #define HWCAP_SHA2 (1 << 6) #define HWCAP_CRC32 (1 << 7) +#define HWCAP_ATOMICS (1 << 8) #endif /* _UAPI__ASM_HWCAP_H */ diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index d26832022127..0cd7b5947dfc 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -53,14 +53,20 @@ struct kvm_regs { struct user_fpsimd_state fp_regs; }; -/* Supported Processor Types */ +/* + * Supported CPU Targets - Adding a new target type is not recommended, + * unless there are some special registers not supported by the + * genericv8 syreg table. + */ #define KVM_ARM_TARGET_AEM_V8 0 #define KVM_ARM_TARGET_FOUNDATION_V8 1 #define KVM_ARM_TARGET_CORTEX_A57 2 #define KVM_ARM_TARGET_XGENE_POTENZA 3 #define KVM_ARM_TARGET_CORTEX_A53 4 +/* Generic ARM v8 target */ +#define KVM_ARM_TARGET_GENERIC_V8 5 -#define KVM_ARM_NUM_TARGETS 5 +#define KVM_ARM_NUM_TARGETS 6 /* KVM_ARM_SET_DEVICE_ADDR ioctl id encoding */ #define KVM_ARM_DEVICE_TYPE_SHIFT 0 @@ -100,12 +106,39 @@ struct kvm_sregs { struct kvm_fpu { }; +/* + * See v8 ARM ARM D7.3: Debug Registers + * + * The architectural limit is 16 debug registers of each type although + * in practice there are usually less (see ID_AA64DFR0_EL1). + * + * Although the control registers are architecturally defined as 32 + * bits wide we use a 64 bit structure here to keep parity with + * KVM_GET/SET_ONE_REG behaviour which treats all system registers as + * 64 bit values. It also allows for the possibility of the + * architecture expanding the control registers without having to + * change the userspace ABI. + */ +#define KVM_ARM_MAX_DBG_REGS 16 struct kvm_guest_debug_arch { + __u64 dbg_bcr[KVM_ARM_MAX_DBG_REGS]; + __u64 dbg_bvr[KVM_ARM_MAX_DBG_REGS]; + __u64 dbg_wcr[KVM_ARM_MAX_DBG_REGS]; + __u64 dbg_wvr[KVM_ARM_MAX_DBG_REGS]; }; struct kvm_debug_exit_arch { + __u32 hsr; + __u64 far; /* used for watchpoints */ }; +/* + * Architecture specific defines for kvm_guest_debug->control + */ + +#define KVM_GUESTDBG_USE_SW_BP (1 << 16) +#define KVM_GUESTDBG_USE_HW (1 << 17) + struct kvm_sync_regs { }; diff --git a/arch/arm64/include/uapi/asm/ptrace.h b/arch/arm64/include/uapi/asm/ptrace.h index 6913643bbe54..208db3df135a 100644 --- a/arch/arm64/include/uapi/asm/ptrace.h +++ b/arch/arm64/include/uapi/asm/ptrace.h @@ -44,6 +44,7 @@ #define PSR_I_BIT 0x00000080 #define PSR_A_BIT 0x00000100 #define PSR_D_BIT 0x00000200 +#define PSR_PAN_BIT 0x00400000 #define PSR_Q_BIT 0x08000000 #define PSR_V_BIT 0x10000000 #define PSR_C_BIT 0x20000000 diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 426d0763c81b..22dc9bc781be 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -17,15 +17,15 @@ arm64-obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ sys.o stacktrace.o time.o traps.o io.o vdso.o \ hyp-stub.o psci.o psci-call.o cpu_ops.o insn.o \ return_address.o cpuinfo.o cpu_errata.o \ - cpufeature.o alternative.o cacheinfo.o + cpufeature.o alternative.o cacheinfo.o \ + smp.o smp_spin_table.o topology.o arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ sys_compat.o entry32.o \ ../../arm/kernel/opcodes.o arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o -arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o topology.o -arm64-obj-$(CONFIG_PERF_EVENTS) += perf_regs.o +arm64-obj-$(CONFIG_PERF_EVENTS) += perf_regs.o perf_callchain.o arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o arm64-obj-$(CONFIG_CPU_PM) += sleep.o suspend.o diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c index 221b98312f0c..ab9db0e9818c 100644 --- a/arch/arm64/kernel/alternative.c +++ b/arch/arm64/kernel/alternative.c @@ -85,7 +85,7 @@ static u32 get_alt_insn(struct alt_instr *alt, u32 *insnptr, u32 *altinsnptr) return insn; } -static int __apply_alternatives(void *alt_region) +static void __apply_alternatives(void *alt_region) { struct alt_instr *alt; struct alt_region *region = alt_region; @@ -114,19 +114,39 @@ static int __apply_alternatives(void *alt_region) flush_icache_range((uintptr_t)origptr, (uintptr_t)(origptr + nr_inst)); } - - return 0; } -void apply_alternatives_all(void) +/* + * We might be patching the stop_machine state machine, so implement a + * really simple polling protocol here. + */ +static int __apply_alternatives_multi_stop(void *unused) { + static int patched = 0; struct alt_region region = { .begin = __alt_instructions, .end = __alt_instructions_end, }; + /* We always have a CPU 0 at this point (__init) */ + if (smp_processor_id()) { + while (!READ_ONCE(patched)) + cpu_relax(); + isb(); + } else { + BUG_ON(patched); + __apply_alternatives(®ion); + /* Barriers provided by the cache flushing */ + WRITE_ONCE(patched, 1); + } + + return 0; +} + +void __init apply_alternatives_all(void) +{ /* better not try code patching on a live SMP system */ - stop_machine(__apply_alternatives, ®ion, NULL); + stop_machine(__apply_alternatives_multi_stop, NULL, cpu_online_mask); } void apply_alternatives(void *start, size_t length) diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c index 7922c2e710ca..bcee7abac68e 100644 --- a/arch/arm64/kernel/armv8_deprecated.c +++ b/arch/arm64/kernel/armv8_deprecated.c @@ -14,8 +14,11 @@ #include #include +#include +#include #include #include +#include #include #include #include @@ -279,6 +282,8 @@ static void register_insn_emulation_sysctl(struct ctl_table *table) */ #define __user_swpX_asm(data, addr, res, temp, B) \ __asm__ __volatile__( \ + ALTERNATIVE("nop", SET_PSTATE_PAN(0), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) \ " mov %w2, %w1\n" \ "0: ldxr"B" %w1, [%3]\n" \ "1: stxr"B" %w0, %w2, [%3]\n" \ @@ -294,7 +299,9 @@ static void register_insn_emulation_sysctl(struct ctl_table *table) " .align 3\n" \ " .quad 0b, 3b\n" \ " .quad 1b, 3b\n" \ - " .popsection" \ + " .popsection\n" \ + ALTERNATIVE("nop", SET_PSTATE_PAN(1), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) \ : "=&r" (res), "+r" (data), "=&r" (temp) \ : "r" (addr), "i" (-EAGAIN), "i" (-EFAULT) \ : "memory") @@ -504,16 +511,6 @@ static int cp15barrier_handler(struct pt_regs *regs, u32 instr) return 0; } -static inline void config_sctlr_el1(u32 clear, u32 set) -{ - u32 val; - - asm volatile("mrs %0, sctlr_el1" : "=r" (val)); - val &= ~clear; - val |= set; - asm volatile("msr sctlr_el1, %0" : : "r" (val)); -} - static int cp15_barrier_set_hw_mode(bool enable) { if (enable) diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index c99701a34d7b..8d89cf8dae55 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -116,17 +116,22 @@ int main(void) DEFINE(VCPU_FAR_EL2, offsetof(struct kvm_vcpu, arch.fault.far_el2)); DEFINE(VCPU_HPFAR_EL2, offsetof(struct kvm_vcpu, arch.fault.hpfar_el2)); DEFINE(VCPU_DEBUG_FLAGS, offsetof(struct kvm_vcpu, arch.debug_flags)); + DEFINE(VCPU_DEBUG_PTR, offsetof(struct kvm_vcpu, arch.debug_ptr)); + DEFINE(DEBUG_BCR, offsetof(struct kvm_guest_debug_arch, dbg_bcr)); + DEFINE(DEBUG_BVR, offsetof(struct kvm_guest_debug_arch, dbg_bvr)); + DEFINE(DEBUG_WCR, offsetof(struct kvm_guest_debug_arch, dbg_wcr)); + DEFINE(DEBUG_WVR, offsetof(struct kvm_guest_debug_arch, dbg_wvr)); DEFINE(VCPU_HCR_EL2, offsetof(struct kvm_vcpu, arch.hcr_el2)); + DEFINE(VCPU_MDCR_EL2, offsetof(struct kvm_vcpu, arch.mdcr_el2)); DEFINE(VCPU_IRQ_LINES, offsetof(struct kvm_vcpu, arch.irq_lines)); DEFINE(VCPU_HOST_CONTEXT, offsetof(struct kvm_vcpu, arch.host_cpu_context)); + DEFINE(VCPU_HOST_DEBUG_STATE, offsetof(struct kvm_vcpu, arch.host_debug_state)); DEFINE(VCPU_TIMER_CNTV_CTL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_ctl)); DEFINE(VCPU_TIMER_CNTV_CVAL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_cval)); DEFINE(KVM_TIMER_CNTVOFF, offsetof(struct kvm, arch.timer.cntvoff)); DEFINE(KVM_TIMER_ENABLED, offsetof(struct kvm, arch.timer.enabled)); DEFINE(VCPU_KVM, offsetof(struct kvm_vcpu, kvm)); DEFINE(VCPU_VGIC_CPU, offsetof(struct kvm_vcpu, arch.vgic_cpu)); - DEFINE(VGIC_SAVE_FN, offsetof(struct vgic_sr_vectors, save_vgic)); - DEFINE(VGIC_RESTORE_FN, offsetof(struct vgic_sr_vectors, restore_vgic)); DEFINE(VGIC_V2_CPU_HCR, offsetof(struct vgic_cpu, vgic_v2.vgic_hcr)); DEFINE(VGIC_V2_CPU_VMCR, offsetof(struct vgic_cpu, vgic_v2.vgic_vmcr)); DEFINE(VGIC_V2_CPU_MISR, offsetof(struct vgic_cpu, vgic_v2.vgic_misr)); diff --git a/arch/arm64/kernel/cpu_ops.c b/arch/arm64/kernel/cpu_ops.c index 5ea337dd2f15..b6bd7d447768 100644 --- a/arch/arm64/kernel/cpu_ops.c +++ b/arch/arm64/kernel/cpu_ops.c @@ -30,9 +30,7 @@ extern const struct cpu_operations cpu_psci_ops; const struct cpu_operations *cpu_ops[NR_CPUS]; static const struct cpu_operations *supported_cpu_ops[] __initconst = { -#ifdef CONFIG_SMP &smp_spin_table_ops, -#endif &cpu_psci_ops, NULL, }; diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 5ad86ceac010..3c9aed32f70b 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -21,24 +21,57 @@ #include #include #include +#include static bool -has_id_aa64pfr0_feature(const struct arm64_cpu_capabilities *entry) +feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry) { - u64 val; + int val = cpuid_feature_extract_field(reg, entry->field_pos); - val = read_cpuid(id_aa64pfr0_el1); - return (val & entry->register_mask) == entry->register_value; + return val >= entry->min_field_value; } +#define __ID_FEAT_CHK(reg) \ +static bool __maybe_unused \ +has_##reg##_feature(const struct arm64_cpu_capabilities *entry) \ +{ \ + u64 val; \ + \ + val = read_cpuid(reg##_el1); \ + return feature_matches(val, entry); \ +} + +__ID_FEAT_CHK(id_aa64pfr0); +__ID_FEAT_CHK(id_aa64mmfr1); +__ID_FEAT_CHK(id_aa64isar0); + static const struct arm64_cpu_capabilities arm64_features[] = { { .desc = "GIC system register CPU interface", .capability = ARM64_HAS_SYSREG_GIC_CPUIF, .matches = has_id_aa64pfr0_feature, - .register_mask = (0xf << 24), - .register_value = (1 << 24), + .field_pos = 24, + .min_field_value = 1, }, +#ifdef CONFIG_ARM64_PAN + { + .desc = "Privileged Access Never", + .capability = ARM64_HAS_PAN, + .matches = has_id_aa64mmfr1_feature, + .field_pos = 20, + .min_field_value = 1, + .enable = cpu_enable_pan, + }, +#endif /* CONFIG_ARM64_PAN */ +#if defined(CONFIG_AS_LSE) && defined(CONFIG_ARM64_LSE_ATOMICS) + { + .desc = "LSE atomic instructions", + .capability = ARM64_HAS_LSE_ATOMICS, + .matches = has_id_aa64isar0_feature, + .field_pos = 20, + .min_field_value = 2, + }, +#endif /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */ {}, }; @@ -55,9 +88,15 @@ void check_cpu_capabilities(const struct arm64_cpu_capabilities *caps, pr_info("%s %s\n", info, caps[i].desc); cpus_set_cap(caps[i].capability); } + + /* second pass allows enable() to consider interacting capabilities */ + for (i = 0; caps[i].desc; i++) { + if (cpus_have_cap(caps[i].capability) && caps[i].enable) + caps[i].enable(); + } } void check_local_cpu_features(void) { - check_cpu_capabilities(arm64_features, "detected feature"); + check_cpu_capabilities(arm64_features, "detected feature:"); } diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c index b056369fd47d..9b3b62ac9c24 100644 --- a/arch/arm64/kernel/debug-monitors.c +++ b/arch/arm64/kernel/debug-monitors.c @@ -82,7 +82,7 @@ early_param("nodebugmon", early_debug_disable); static DEFINE_PER_CPU(int, mde_ref_count); static DEFINE_PER_CPU(int, kde_ref_count); -void enable_debug_monitors(enum debug_el el) +void enable_debug_monitors(enum dbg_active_el el) { u32 mdscr, enable = 0; @@ -102,7 +102,7 @@ void enable_debug_monitors(enum debug_el el) } } -void disable_debug_monitors(enum debug_el el) +void disable_debug_monitors(enum dbg_active_el el) { u32 mdscr, disable = 0; diff --git a/arch/arm64/kernel/efi-stub.c b/arch/arm64/kernel/efi-stub.c index f5374065ad53..816120ece6bc 100644 --- a/arch/arm64/kernel/efi-stub.c +++ b/arch/arm64/kernel/efi-stub.c @@ -13,7 +13,7 @@ #include #include -efi_status_t __init handle_kernel_image(efi_system_table_t *sys_table, +efi_status_t __init handle_kernel_image(efi_system_table_t *sys_table_arg, unsigned long *image_addr, unsigned long *image_size, unsigned long *reserve_addr, @@ -23,21 +23,44 @@ efi_status_t __init handle_kernel_image(efi_system_table_t *sys_table, { efi_status_t status; unsigned long kernel_size, kernel_memsize = 0; + unsigned long nr_pages; + void *old_image_addr = (void *)*image_addr; /* Relocate the image, if required. */ kernel_size = _edata - _text; if (*image_addr != (dram_base + TEXT_OFFSET)) { kernel_memsize = kernel_size + (_end - _edata); - status = efi_low_alloc(sys_table, kernel_memsize + TEXT_OFFSET, - SZ_2M, reserve_addr); + + /* + * First, try a straight allocation at the preferred offset. + * This will work around the issue where, if dram_base == 0x0, + * efi_low_alloc() refuses to allocate at 0x0 (to prevent the + * address of the allocation to be mistaken for a FAIL return + * value or a NULL pointer). It will also ensure that, on + * platforms where the [dram_base, dram_base + TEXT_OFFSET) + * interval is partially occupied by the firmware (like on APM + * Mustang), we can still place the kernel at the address + * 'dram_base + TEXT_OFFSET'. + */ + *image_addr = *reserve_addr = dram_base + TEXT_OFFSET; + nr_pages = round_up(kernel_memsize, EFI_ALLOC_ALIGN) / + EFI_PAGE_SIZE; + status = efi_call_early(allocate_pages, EFI_ALLOCATE_ADDRESS, + EFI_LOADER_DATA, nr_pages, + (efi_physical_addr_t *)reserve_addr); if (status != EFI_SUCCESS) { - pr_efi_err(sys_table, "Failed to relocate kernel\n"); - return status; + kernel_memsize += TEXT_OFFSET; + status = efi_low_alloc(sys_table_arg, kernel_memsize, + SZ_2M, reserve_addr); + + if (status != EFI_SUCCESS) { + pr_efi_err(sys_table_arg, "Failed to relocate kernel\n"); + return status; + } + *image_addr = *reserve_addr + TEXT_OFFSET; } - memcpy((void *)*reserve_addr + TEXT_OFFSET, (void *)*image_addr, - kernel_size); - *image_addr = *reserve_addr + TEXT_OFFSET; - *reserve_size = kernel_memsize + TEXT_OFFSET; + memcpy((void *)*image_addr, old_image_addr, kernel_size); + *reserve_size = kernel_memsize; } diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index e16351819fed..4306c937b1ff 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -116,41 +116,34 @@ */ .endm - .macro kernel_exit, el, ret = 0 + .macro kernel_exit, el ldp x21, x22, [sp, #S_PC] // load ELR, SPSR .if \el == 0 ct_user_enter ldr x23, [sp, #S_SP] // load return stack pointer msr sp_el0, x23 - #ifdef CONFIG_ARM64_ERRATUM_845719 - -#undef SEQUENCE_ORG -#undef SEQUENCE_ALT - +alternative_if_not ARM64_WORKAROUND_845719 + nop + nop #ifdef CONFIG_PID_IN_CONTEXTIDR - -#define SEQUENCE_ORG "nop ; nop ; nop" -#define SEQUENCE_ALT "tbz x22, #4, 1f ; mrs x29, contextidr_el1; msr contextidr_el1, x29; 1:" - -#else - -#define SEQUENCE_ORG "nop ; nop" -#define SEQUENCE_ALT "tbz x22, #4, 1f ; msr contextidr_el1, xzr; 1:" - + nop #endif - - alternative_insn SEQUENCE_ORG, SEQUENCE_ALT, ARM64_WORKAROUND_845719 - +alternative_else + tbz x22, #4, 1f +#ifdef CONFIG_PID_IN_CONTEXTIDR + mrs x29, contextidr_el1 + msr contextidr_el1, x29 +#else + msr contextidr_el1, xzr +#endif +1: +alternative_endif #endif .endif msr elr_el1, x21 // set up the return data msr spsr_el1, x22 - .if \ret - ldr x1, [sp, #S_X1] // preserve x0 (syscall return) - .else ldp x0, x1, [sp, #16 * 0] - .endif ldp x2, x3, [sp, #16 * 1] ldp x4, x5, [sp, #16 * 2] ldp x6, x7, [sp, #16 * 3] @@ -613,22 +606,21 @@ ENDPROC(cpu_switch_to) */ ret_fast_syscall: disable_irq // disable interrupts + str x0, [sp, #S_X0] // returned x0 ldr x1, [tsk, #TI_FLAGS] // re-check for syscall tracing and x2, x1, #_TIF_SYSCALL_WORK cbnz x2, ret_fast_syscall_trace and x2, x1, #_TIF_WORK_MASK - cbnz x2, fast_work_pending + cbnz x2, work_pending enable_step_tsk x1, x2 - kernel_exit 0, ret = 1 + kernel_exit 0 ret_fast_syscall_trace: enable_irq // enable interrupts - b __sys_trace_return + b __sys_trace_return_skipped // we already saved x0 /* * Ok, we need to do extra processing, enter the slow path. */ -fast_work_pending: - str x0, [sp, #S_X0] // returned x0 work_pending: tbnz x1, #TIF_NEED_RESCHED, work_resched /* TIF_SIGPENDING, TIF_NOTIFY_RESUME or TIF_FOREIGN_FPSTATE case */ @@ -652,7 +644,7 @@ ret_to_user: cbnz x2, work_pending enable_step_tsk x1, x2 no_work_pending: - kernel_exit 0, ret = 0 + kernel_exit 0 ENDPROC(ret_to_user) /* diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 44d6f7545505..c56956a16d3f 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -158,6 +158,7 @@ void fpsimd_thread_switch(struct task_struct *next) void fpsimd_flush_thread(void) { memset(¤t->thread.fpsimd_state, 0, sizeof(struct fpsimd_state)); + fpsimd_flush_task_state(current); set_thread_flag(TIF_FOREIGN_FPSTATE); } diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index c0ff3ce4299e..a055be6125cf 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -62,13 +62,8 @@ /* * Initial memory map attributes. */ -#ifndef CONFIG_SMP -#define PTE_FLAGS PTE_TYPE_PAGE | PTE_AF -#define PMD_FLAGS PMD_TYPE_SECT | PMD_SECT_AF -#else #define PTE_FLAGS PTE_TYPE_PAGE | PTE_AF | PTE_SHARED #define PMD_FLAGS PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S -#endif #ifdef CONFIG_ARM64_64K_PAGES #define MM_MMUFLAGS PTE_ATTRINDX(MT_NORMAL) | PTE_FLAGS @@ -574,7 +569,6 @@ ENTRY(__boot_cpu_mode) .long BOOT_CPU_MODE_EL1 .popsection -#ifdef CONFIG_SMP /* * This provides a "holding pen" for platforms to hold all secondary * cores are held until we're ready for them to initialise. @@ -622,7 +616,6 @@ ENTRY(__secondary_switched) mov x29, #0 b secondary_start_kernel ENDPROC(__secondary_switched) -#endif /* CONFIG_SMP */ /* * Enable the MMU. @@ -641,5 +634,13 @@ __enable_mmu: isb msr sctlr_el1, x0 isb + /* + * Invalidate the local I-cache so that any instructions fetched + * speculatively from the PoC are discarded, since they may have + * been dynamically patched at the PoU. + */ + ic iallu + dsb nsh + isb br x27 ENDPROC(__enable_mmu) diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c index 7a1a5da6c8c1..c97040ecf838 100644 --- a/arch/arm64/kernel/hw_breakpoint.c +++ b/arch/arm64/kernel/hw_breakpoint.c @@ -48,18 +48,6 @@ static DEFINE_PER_CPU(int, stepping_kernel_bp); static int core_num_brps; static int core_num_wrps; -/* Determine number of BRP registers available. */ -static int get_num_brps(void) -{ - return ((read_cpuid(ID_AA64DFR0_EL1) >> 12) & 0xf) + 1; -} - -/* Determine number of WRP registers available. */ -static int get_num_wrps(void) -{ - return ((read_cpuid(ID_AA64DFR0_EL1) >> 20) & 0xf) + 1; -} - int hw_breakpoint_slots(int type) { /* @@ -156,7 +144,7 @@ static void write_wb_reg(int reg, int n, u64 val) * Convert a breakpoint privilege level to the corresponding exception * level. */ -static enum debug_el debug_exception_level(int privilege) +static enum dbg_active_el debug_exception_level(int privilege) { switch (privilege) { case AARCH64_BREAKPOINT_EL0: @@ -230,7 +218,7 @@ static int hw_breakpoint_control(struct perf_event *bp, struct perf_event **slots; struct debug_info *debug_info = ¤t->thread.debug; int i, max_slots, ctrl_reg, val_reg, reg_enable; - enum debug_el dbg_el = debug_exception_level(info->ctrl.privilege); + enum dbg_active_el dbg_el = debug_exception_level(info->ctrl.privilege); u32 ctrl; if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { @@ -537,7 +525,7 @@ int arch_validate_hwbkpt_settings(struct perf_event *bp) * exception level at the register level. * This is used when single-stepping after a breakpoint exception. */ -static void toggle_bp_registers(int reg, enum debug_el el, int enable) +static void toggle_bp_registers(int reg, enum dbg_active_el el, int enable) { int i, max_slots, privilege; u32 ctrl; diff --git a/arch/arm64/kernel/insn.c b/arch/arm64/kernel/insn.c index dd9671cd0bb2..f341866aa810 100644 --- a/arch/arm64/kernel/insn.c +++ b/arch/arm64/kernel/insn.c @@ -101,9 +101,8 @@ static void __kprobes *patch_map(void *addr, int fixmap) return addr; BUG_ON(!page); - set_fixmap(fixmap, page_to_phys(page)); - - return (void *) (__fix_to_virt(fixmap) + (uintaddr & ~PAGE_MASK)); + return (void *)set_fixmap_offset(fixmap, page_to_phys(page) + + (uintaddr & ~PAGE_MASK)); } static void __kprobes patch_unmap(int fixmap) diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c index 463fa2e7e34c..11dc3fd47853 100644 --- a/arch/arm64/kernel/irq.c +++ b/arch/arm64/kernel/irq.c @@ -33,9 +33,7 @@ unsigned long irq_err_count; int arch_show_interrupts(struct seq_file *p, int prec) { -#ifdef CONFIG_SMP show_ipi_list(p, prec); -#endif seq_printf(p, "%*s: %10lu\n", prec, "Err", irq_err_count); return 0; } diff --git a/arch/arm64/kernel/jump_label.c b/arch/arm64/kernel/jump_label.c index 4f1fec7a46db..c2dd1ad3e648 100644 --- a/arch/arm64/kernel/jump_label.c +++ b/arch/arm64/kernel/jump_label.c @@ -28,7 +28,7 @@ void arch_jump_label_transform(struct jump_entry *entry, void *addr = (void *)entry->code; u32 insn; - if (type == JUMP_LABEL_ENABLE) { + if (type == JUMP_LABEL_JMP) { insn = aarch64_insn_gen_branch_imm(entry->code, entry->target, AARCH64_INSN_BRANCH_NOLINK); diff --git a/arch/arm64/kernel/kgdb.c b/arch/arm64/kernel/kgdb.c index a0d10c55f307..bcac81e600b9 100644 --- a/arch/arm64/kernel/kgdb.c +++ b/arch/arm64/kernel/kgdb.c @@ -235,13 +235,13 @@ static int kgdb_step_brk_fn(struct pt_regs *regs, unsigned int esr) static struct break_hook kgdb_brkpt_hook = { .esr_mask = 0xffffffff, - .esr_val = DBG_ESR_VAL_BRK(KGDB_DYN_DBG_BRK_IMM), + .esr_val = (u32)ESR_ELx_VAL_BRK64(KGDB_DYN_DBG_BRK_IMM), .fn = kgdb_brk_fn }; static struct break_hook kgdb_compiled_brkpt_hook = { .esr_mask = 0xffffffff, - .esr_val = DBG_ESR_VAL_BRK(KGDB_COMPILED_DBG_BRK_IMM), + .esr_val = (u32)ESR_ELx_VAL_BRK64(KGDB_COMPILED_DBG_BRK_IMM), .fn = kgdb_compiled_brk_fn }; @@ -328,9 +328,9 @@ void kgdb_arch_exit(void) */ struct kgdb_arch arch_kgdb_ops = { .gdb_bpt_instr = { - KGDB_DYN_BRK_INS_BYTE0, - KGDB_DYN_BRK_INS_BYTE1, - KGDB_DYN_BRK_INS_BYTE2, - KGDB_DYN_BRK_INS_BYTE3, + KGDB_DYN_BRK_INS_BYTE(0), + KGDB_DYN_BRK_INS_BYTE(1), + KGDB_DYN_BRK_INS_BYTE(2), + KGDB_DYN_BRK_INS_BYTE(3), } }; diff --git a/arch/arm64/kernel/pci.c b/arch/arm64/kernel/pci.c index 4095379dc069..b3d098bd34aa 100644 --- a/arch/arm64/kernel/pci.c +++ b/arch/arm64/kernel/pci.c @@ -38,6 +38,19 @@ resource_size_t pcibios_align_resource(void *data, const struct resource *res, return res->start; } +/** + * pcibios_enable_device - Enable I/O and memory. + * @dev: PCI device to be enabled + * @mask: bitmask of BARs to enable + */ +int pcibios_enable_device(struct pci_dev *dev, int mask) +{ + if (pci_has_flag(PCI_PROBE_ONLY)) + return 0; + + return pci_enable_resources(dev, mask); +} + /* * Try to assign the IRQ number from DT when adding a new device */ diff --git a/arch/arm64/kernel/perf_callchain.c b/arch/arm64/kernel/perf_callchain.c new file mode 100644 index 000000000000..3aa74830cc69 --- /dev/null +++ b/arch/arm64/kernel/perf_callchain.c @@ -0,0 +1,196 @@ +/* + * arm64 callchain support + * + * Copyright (C) 2015 ARM Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include +#include + +#include + +struct frame_tail { + struct frame_tail __user *fp; + unsigned long lr; +} __attribute__((packed)); + +/* + * Get the return address for a single stackframe and return a pointer to the + * next frame tail. + */ +static struct frame_tail __user * +user_backtrace(struct frame_tail __user *tail, + struct perf_callchain_entry *entry) +{ + struct frame_tail buftail; + unsigned long err; + + /* Also check accessibility of one struct frame_tail beyond */ + if (!access_ok(VERIFY_READ, tail, sizeof(buftail))) + return NULL; + + pagefault_disable(); + err = __copy_from_user_inatomic(&buftail, tail, sizeof(buftail)); + pagefault_enable(); + + if (err) + return NULL; + + perf_callchain_store(entry, buftail.lr); + + /* + * Frame pointers should strictly progress back up the stack + * (towards higher addresses). + */ + if (tail >= buftail.fp) + return NULL; + + return buftail.fp; +} + +#ifdef CONFIG_COMPAT +/* + * The registers we're interested in are at the end of the variable + * length saved register structure. The fp points at the end of this + * structure so the address of this struct is: + * (struct compat_frame_tail *)(xxx->fp)-1 + * + * This code has been adapted from the ARM OProfile support. + */ +struct compat_frame_tail { + compat_uptr_t fp; /* a (struct compat_frame_tail *) in compat mode */ + u32 sp; + u32 lr; +} __attribute__((packed)); + +static struct compat_frame_tail __user * +compat_user_backtrace(struct compat_frame_tail __user *tail, + struct perf_callchain_entry *entry) +{ + struct compat_frame_tail buftail; + unsigned long err; + + /* Also check accessibility of one struct frame_tail beyond */ + if (!access_ok(VERIFY_READ, tail, sizeof(buftail))) + return NULL; + + pagefault_disable(); + err = __copy_from_user_inatomic(&buftail, tail, sizeof(buftail)); + pagefault_enable(); + + if (err) + return NULL; + + perf_callchain_store(entry, buftail.lr); + + /* + * Frame pointers should strictly progress back up the stack + * (towards higher addresses). + */ + if (tail + 1 >= (struct compat_frame_tail __user *) + compat_ptr(buftail.fp)) + return NULL; + + return (struct compat_frame_tail __user *)compat_ptr(buftail.fp) - 1; +} +#endif /* CONFIG_COMPAT */ + +void perf_callchain_user(struct perf_callchain_entry *entry, + struct pt_regs *regs) +{ + if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { + /* We don't support guest os callchain now */ + return; + } + + perf_callchain_store(entry, regs->pc); + + if (!compat_user_mode(regs)) { + /* AARCH64 mode */ + struct frame_tail __user *tail; + + tail = (struct frame_tail __user *)regs->regs[29]; + + while (entry->nr < PERF_MAX_STACK_DEPTH && + tail && !((unsigned long)tail & 0xf)) + tail = user_backtrace(tail, entry); + } else { +#ifdef CONFIG_COMPAT + /* AARCH32 compat mode */ + struct compat_frame_tail __user *tail; + + tail = (struct compat_frame_tail __user *)regs->compat_fp - 1; + + while ((entry->nr < PERF_MAX_STACK_DEPTH) && + tail && !((unsigned long)tail & 0x3)) + tail = compat_user_backtrace(tail, entry); +#endif + } +} + +/* + * Gets called by walk_stackframe() for every stackframe. This will be called + * whist unwinding the stackframe and is like a subroutine return so we use + * the PC. + */ +static int callchain_trace(struct stackframe *frame, void *data) +{ + struct perf_callchain_entry *entry = data; + perf_callchain_store(entry, frame->pc); + return 0; +} + +void perf_callchain_kernel(struct perf_callchain_entry *entry, + struct pt_regs *regs) +{ + struct stackframe frame; + + if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { + /* We don't support guest os callchain now */ + return; + } + + frame.fp = regs->regs[29]; + frame.sp = regs->sp; + frame.pc = regs->pc; + + walk_stackframe(&frame, callchain_trace, entry); +} + +unsigned long perf_instruction_pointer(struct pt_regs *regs) +{ + if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) + return perf_guest_cbs->get_guest_ip(); + + return instruction_pointer(regs); +} + +unsigned long perf_misc_flags(struct pt_regs *regs) +{ + int misc = 0; + + if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { + if (perf_guest_cbs->is_user_mode()) + misc |= PERF_RECORD_MISC_GUEST_USER; + else + misc |= PERF_RECORD_MISC_GUEST_KERNEL; + } else { + if (user_mode(regs)) + misc |= PERF_RECORD_MISC_USER; + else + misc |= PERF_RECORD_MISC_KERNEL; + } + + return misc; +} diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c index b31e9a4b6275..f9a74d4fff3b 100644 --- a/arch/arm64/kernel/perf_event.c +++ b/arch/arm64/kernel/perf_event.c @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include #include #include @@ -36,7 +36,6 @@ #include #include #include -#include /* * ARMv8 supports a maximum of 32 events. @@ -78,6 +77,16 @@ EXPORT_SYMBOL_GPL(perf_num_counters); #define CACHE_OP_UNSUPPORTED 0xFFFF +#define PERF_MAP_ALL_UNSUPPORTED \ + [0 ... PERF_COUNT_HW_MAX - 1] = HW_OP_UNSUPPORTED + +#define PERF_CACHE_MAP_ALL_UNSUPPORTED \ +[0 ... C(MAX) - 1] = { \ + [0 ... C(OP_MAX) - 1] = { \ + [0 ... C(RESULT_MAX) - 1] = CACHE_OP_UNSUPPORTED, \ + }, \ +} + static int armpmu_map_cache_event(const unsigned (*cache_map) [PERF_COUNT_HW_CACHE_MAX] @@ -435,10 +444,8 @@ armpmu_reserve_hardware(struct arm_pmu *armpmu) unsigned int i, irqs; struct platform_device *pmu_device = armpmu->plat_device; - if (!pmu_device) { - pr_err("no PMU device registered\n"); + if (!pmu_device) return -ENODEV; - } irqs = min(pmu_device->num_resources, num_possible_cpus()); if (!irqs) { @@ -703,118 +710,28 @@ enum armv8_pmuv3_perf_types { /* PMUv3 HW events mapping. */ static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = { + PERF_MAP_ALL_UNSUPPORTED, [PERF_COUNT_HW_CPU_CYCLES] = ARMV8_PMUV3_PERFCTR_CLOCK_CYCLES, [PERF_COUNT_HW_INSTRUCTIONS] = ARMV8_PMUV3_PERFCTR_INSTR_EXECUTED, [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS, [PERF_COUNT_HW_CACHE_MISSES] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = HW_OP_UNSUPPORTED, [PERF_COUNT_HW_BRANCH_MISSES] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED, - [PERF_COUNT_HW_BUS_CYCLES] = HW_OP_UNSUPPORTED, - [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = HW_OP_UNSUPPORTED, - [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = HW_OP_UNSUPPORTED, }; static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_RESULT_MAX] = { - [C(L1D)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS, - [C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS, - [C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - }, - [C(L1I)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - }, - [C(LL)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - }, - [C(DTLB)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - }, - [C(ITLB)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - }, - [C(BPU)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED, - [C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED, - [C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - }, - [C(NODE)] = { - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, - [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, - }, - }, + PERF_CACHE_MAP_ALL_UNSUPPORTED, + + [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS, + [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL, + [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_ACCESS, + [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1_DCACHE_REFILL, + + [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED, + [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED, + [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_PRED, + [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_PC_BRANCH_MIS_PRED, }; /* @@ -1337,7 +1254,7 @@ static int armpmu_device_probe(struct platform_device *pdev) } for_each_possible_cpu(cpu) - if (arch_find_n_match_cpu_physical_id(dn, cpu, NULL)) + if (dn == of_cpu_device_node_get(cpu)) break; if (cpu >= nr_cpu_ids) { @@ -1415,180 +1332,3 @@ static int __init init_hw_perf_events(void) } early_initcall(init_hw_perf_events); -/* - * Callchain handling code. - */ -struct frame_tail { - struct frame_tail __user *fp; - unsigned long lr; -} __attribute__((packed)); - -/* - * Get the return address for a single stackframe and return a pointer to the - * next frame tail. - */ -static struct frame_tail __user * -user_backtrace(struct frame_tail __user *tail, - struct perf_callchain_entry *entry) -{ - struct frame_tail buftail; - unsigned long err; - - /* Also check accessibility of one struct frame_tail beyond */ - if (!access_ok(VERIFY_READ, tail, sizeof(buftail))) - return NULL; - - pagefault_disable(); - err = __copy_from_user_inatomic(&buftail, tail, sizeof(buftail)); - pagefault_enable(); - - if (err) - return NULL; - - perf_callchain_store(entry, buftail.lr); - - /* - * Frame pointers should strictly progress back up the stack - * (towards higher addresses). - */ - if (tail >= buftail.fp) - return NULL; - - return buftail.fp; -} - -#ifdef CONFIG_COMPAT -/* - * The registers we're interested in are at the end of the variable - * length saved register structure. The fp points at the end of this - * structure so the address of this struct is: - * (struct compat_frame_tail *)(xxx->fp)-1 - * - * This code has been adapted from the ARM OProfile support. - */ -struct compat_frame_tail { - compat_uptr_t fp; /* a (struct compat_frame_tail *) in compat mode */ - u32 sp; - u32 lr; -} __attribute__((packed)); - -static struct compat_frame_tail __user * -compat_user_backtrace(struct compat_frame_tail __user *tail, - struct perf_callchain_entry *entry) -{ - struct compat_frame_tail buftail; - unsigned long err; - - /* Also check accessibility of one struct frame_tail beyond */ - if (!access_ok(VERIFY_READ, tail, sizeof(buftail))) - return NULL; - - pagefault_disable(); - err = __copy_from_user_inatomic(&buftail, tail, sizeof(buftail)); - pagefault_enable(); - - if (err) - return NULL; - - perf_callchain_store(entry, buftail.lr); - - /* - * Frame pointers should strictly progress back up the stack - * (towards higher addresses). - */ - if (tail + 1 >= (struct compat_frame_tail __user *) - compat_ptr(buftail.fp)) - return NULL; - - return (struct compat_frame_tail __user *)compat_ptr(buftail.fp) - 1; -} -#endif /* CONFIG_COMPAT */ - -void perf_callchain_user(struct perf_callchain_entry *entry, - struct pt_regs *regs) -{ - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { - /* We don't support guest os callchain now */ - return; - } - - perf_callchain_store(entry, regs->pc); - - if (!compat_user_mode(regs)) { - /* AARCH64 mode */ - struct frame_tail __user *tail; - - tail = (struct frame_tail __user *)regs->regs[29]; - - while (entry->nr < PERF_MAX_STACK_DEPTH && - tail && !((unsigned long)tail & 0xf)) - tail = user_backtrace(tail, entry); - } else { -#ifdef CONFIG_COMPAT - /* AARCH32 compat mode */ - struct compat_frame_tail __user *tail; - - tail = (struct compat_frame_tail __user *)regs->compat_fp - 1; - - while ((entry->nr < PERF_MAX_STACK_DEPTH) && - tail && !((unsigned long)tail & 0x3)) - tail = compat_user_backtrace(tail, entry); -#endif - } -} - -/* - * Gets called by walk_stackframe() for every stackframe. This will be called - * whist unwinding the stackframe and is like a subroutine return so we use - * the PC. - */ -static int callchain_trace(struct stackframe *frame, void *data) -{ - struct perf_callchain_entry *entry = data; - perf_callchain_store(entry, frame->pc); - return 0; -} - -void perf_callchain_kernel(struct perf_callchain_entry *entry, - struct pt_regs *regs) -{ - struct stackframe frame; - - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { - /* We don't support guest os callchain now */ - return; - } - - frame.fp = regs->regs[29]; - frame.sp = regs->sp; - frame.pc = regs->pc; - - walk_stackframe(&frame, callchain_trace, entry); -} - -unsigned long perf_instruction_pointer(struct pt_regs *regs) -{ - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) - return perf_guest_cbs->get_guest_ip(); - - return instruction_pointer(regs); -} - -unsigned long perf_misc_flags(struct pt_regs *regs) -{ - int misc = 0; - - if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { - if (perf_guest_cbs->is_user_mode()) - misc |= PERF_RECORD_MISC_GUEST_USER; - else - misc |= PERF_RECORD_MISC_GUEST_KERNEL; - } else { - if (user_mode(regs)) - misc |= PERF_RECORD_MISC_USER; - else - misc |= PERF_RECORD_MISC_KERNEL; - } - - return misc; -} diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c index 869f202748e8..aa94a88f6279 100644 --- a/arch/arm64/kernel/psci.c +++ b/arch/arm64/kernel/psci.c @@ -18,23 +18,17 @@ #include #include #include -#include -#include #include +#include #include + #include #include -#include #include #include -#include #include #include -#include - -#define PSCI_POWER_STATE_TYPE_STANDBY 0 -#define PSCI_POWER_STATE_TYPE_POWER_DOWN 1 static bool psci_power_state_loses_context(u32 state) { @@ -50,122 +44,8 @@ static bool psci_power_state_is_valid(u32 state) return !(state & ~valid_mask); } -/* - * The CPU any Trusted OS is resident on. The trusted OS may reject CPU_OFF - * calls to its resident CPU, so we must avoid issuing those. We never migrate - * a Trusted OS even if it claims to be capable of migration -- doing so will - * require cooperation with a Trusted OS driver. - */ -static int resident_cpu = -1; - -struct psci_operations { - int (*cpu_suspend)(u32 state, unsigned long entry_point); - int (*cpu_off)(u32 state); - int (*cpu_on)(unsigned long cpuid, unsigned long entry_point); - int (*migrate)(unsigned long cpuid); - int (*affinity_info)(unsigned long target_affinity, - unsigned long lowest_affinity_level); - int (*migrate_info_type)(void); -}; - -static struct psci_operations psci_ops; - -typedef unsigned long (psci_fn)(unsigned long, unsigned long, - unsigned long, unsigned long); -asmlinkage psci_fn __invoke_psci_fn_hvc; -asmlinkage psci_fn __invoke_psci_fn_smc; -static psci_fn *invoke_psci_fn; - -enum psci_function { - PSCI_FN_CPU_SUSPEND, - PSCI_FN_CPU_ON, - PSCI_FN_CPU_OFF, - PSCI_FN_MIGRATE, - PSCI_FN_MAX, -}; - static DEFINE_PER_CPU_READ_MOSTLY(u32 *, psci_power_state); -static u32 psci_function_id[PSCI_FN_MAX]; - -static int psci_to_linux_errno(int errno) -{ - switch (errno) { - case PSCI_RET_SUCCESS: - return 0; - case PSCI_RET_NOT_SUPPORTED: - return -EOPNOTSUPP; - case PSCI_RET_INVALID_PARAMS: - return -EINVAL; - case PSCI_RET_DENIED: - return -EPERM; - }; - - return -EINVAL; -} - -static u32 psci_get_version(void) -{ - return invoke_psci_fn(PSCI_0_2_FN_PSCI_VERSION, 0, 0, 0); -} - -static int psci_cpu_suspend(u32 state, unsigned long entry_point) -{ - int err; - u32 fn; - - fn = psci_function_id[PSCI_FN_CPU_SUSPEND]; - err = invoke_psci_fn(fn, state, entry_point, 0); - return psci_to_linux_errno(err); -} - -static int psci_cpu_off(u32 state) -{ - int err; - u32 fn; - - fn = psci_function_id[PSCI_FN_CPU_OFF]; - err = invoke_psci_fn(fn, state, 0, 0); - return psci_to_linux_errno(err); -} - -static int psci_cpu_on(unsigned long cpuid, unsigned long entry_point) -{ - int err; - u32 fn; - - fn = psci_function_id[PSCI_FN_CPU_ON]; - err = invoke_psci_fn(fn, cpuid, entry_point, 0); - return psci_to_linux_errno(err); -} - -static int psci_migrate(unsigned long cpuid) -{ - int err; - u32 fn; - - fn = psci_function_id[PSCI_FN_MIGRATE]; - err = invoke_psci_fn(fn, cpuid, 0, 0); - return psci_to_linux_errno(err); -} - -static int psci_affinity_info(unsigned long target_affinity, - unsigned long lowest_affinity_level) -{ - return invoke_psci_fn(PSCI_0_2_FN64_AFFINITY_INFO, target_affinity, - lowest_affinity_level, 0); -} - -static int psci_migrate_info_type(void) -{ - return invoke_psci_fn(PSCI_0_2_FN_MIGRATE_INFO_TYPE, 0, 0, 0); -} - -static unsigned long psci_migrate_info_up_cpu(void) -{ - return invoke_psci_fn(PSCI_0_2_FN64_MIGRATE_INFO_UP_CPU, 0, 0, 0); -} - static int __maybe_unused cpu_psci_cpu_init_idle(unsigned int cpu) { int i, ret, count = 0; @@ -230,240 +110,6 @@ static int __maybe_unused cpu_psci_cpu_init_idle(unsigned int cpu) return ret; } -static int get_set_conduit_method(struct device_node *np) -{ - const char *method; - - pr_info("probing for conduit method from DT.\n"); - - if (of_property_read_string(np, "method", &method)) { - pr_warn("missing \"method\" property\n"); - return -ENXIO; - } - - if (!strcmp("hvc", method)) { - invoke_psci_fn = __invoke_psci_fn_hvc; - } else if (!strcmp("smc", method)) { - invoke_psci_fn = __invoke_psci_fn_smc; - } else { - pr_warn("invalid \"method\" property: %s\n", method); - return -EINVAL; - } - return 0; -} - -static void psci_sys_reset(enum reboot_mode reboot_mode, const char *cmd) -{ - invoke_psci_fn(PSCI_0_2_FN_SYSTEM_RESET, 0, 0, 0); -} - -static void psci_sys_poweroff(void) -{ - invoke_psci_fn(PSCI_0_2_FN_SYSTEM_OFF, 0, 0, 0); -} - -/* - * Detect the presence of a resident Trusted OS which may cause CPU_OFF to - * return DENIED (which would be fatal). - */ -static void __init psci_init_migrate(void) -{ - unsigned long cpuid; - int type, cpu; - - type = psci_ops.migrate_info_type(); - - if (type == PSCI_0_2_TOS_MP) { - pr_info("Trusted OS migration not required\n"); - return; - } - - if (type == PSCI_RET_NOT_SUPPORTED) { - pr_info("MIGRATE_INFO_TYPE not supported.\n"); - return; - } - - if (type != PSCI_0_2_TOS_UP_MIGRATE && - type != PSCI_0_2_TOS_UP_NO_MIGRATE) { - pr_err("MIGRATE_INFO_TYPE returned unknown type (%d)\n", type); - return; - } - - cpuid = psci_migrate_info_up_cpu(); - if (cpuid & ~MPIDR_HWID_BITMASK) { - pr_warn("MIGRATE_INFO_UP_CPU reported invalid physical ID (0x%lx)\n", - cpuid); - return; - } - - cpu = get_logical_index(cpuid); - resident_cpu = cpu >= 0 ? cpu : -1; - - pr_info("Trusted OS resident on physical CPU 0x%lx\n", cpuid); -} - -static void __init psci_0_2_set_functions(void) -{ - pr_info("Using standard PSCI v0.2 function IDs\n"); - psci_function_id[PSCI_FN_CPU_SUSPEND] = PSCI_0_2_FN64_CPU_SUSPEND; - psci_ops.cpu_suspend = psci_cpu_suspend; - - psci_function_id[PSCI_FN_CPU_OFF] = PSCI_0_2_FN_CPU_OFF; - psci_ops.cpu_off = psci_cpu_off; - - psci_function_id[PSCI_FN_CPU_ON] = PSCI_0_2_FN64_CPU_ON; - psci_ops.cpu_on = psci_cpu_on; - - psci_function_id[PSCI_FN_MIGRATE] = PSCI_0_2_FN64_MIGRATE; - psci_ops.migrate = psci_migrate; - - psci_ops.affinity_info = psci_affinity_info; - - psci_ops.migrate_info_type = psci_migrate_info_type; - - arm_pm_restart = psci_sys_reset; - - pm_power_off = psci_sys_poweroff; -} - -/* - * Probe function for PSCI firmware versions >= 0.2 - */ -static int __init psci_probe(void) -{ - u32 ver = psci_get_version(); - - pr_info("PSCIv%d.%d detected in firmware.\n", - PSCI_VERSION_MAJOR(ver), - PSCI_VERSION_MINOR(ver)); - - if (PSCI_VERSION_MAJOR(ver) == 0 && PSCI_VERSION_MINOR(ver) < 2) { - pr_err("Conflicting PSCI version detected.\n"); - return -EINVAL; - } - - psci_0_2_set_functions(); - - psci_init_migrate(); - - return 0; -} - -typedef int (*psci_initcall_t)(const struct device_node *); - -/* - * PSCI init function for PSCI versions >=0.2 - * - * Probe based on PSCI PSCI_VERSION function - */ -static int __init psci_0_2_init(struct device_node *np) -{ - int err; - - err = get_set_conduit_method(np); - - if (err) - goto out_put_node; - /* - * Starting with v0.2, the PSCI specification introduced a call - * (PSCI_VERSION) that allows probing the firmware version, so - * that PSCI function IDs and version specific initialization - * can be carried out according to the specific version reported - * by firmware - */ - err = psci_probe(); - -out_put_node: - of_node_put(np); - return err; -} - -/* - * PSCI < v0.2 get PSCI Function IDs via DT. - */ -static int __init psci_0_1_init(struct device_node *np) -{ - u32 id; - int err; - - err = get_set_conduit_method(np); - - if (err) - goto out_put_node; - - pr_info("Using PSCI v0.1 Function IDs from DT\n"); - - if (!of_property_read_u32(np, "cpu_suspend", &id)) { - psci_function_id[PSCI_FN_CPU_SUSPEND] = id; - psci_ops.cpu_suspend = psci_cpu_suspend; - } - - if (!of_property_read_u32(np, "cpu_off", &id)) { - psci_function_id[PSCI_FN_CPU_OFF] = id; - psci_ops.cpu_off = psci_cpu_off; - } - - if (!of_property_read_u32(np, "cpu_on", &id)) { - psci_function_id[PSCI_FN_CPU_ON] = id; - psci_ops.cpu_on = psci_cpu_on; - } - - if (!of_property_read_u32(np, "migrate", &id)) { - psci_function_id[PSCI_FN_MIGRATE] = id; - psci_ops.migrate = psci_migrate; - } - -out_put_node: - of_node_put(np); - return err; -} - -static const struct of_device_id psci_of_match[] __initconst = { - { .compatible = "arm,psci", .data = psci_0_1_init}, - { .compatible = "arm,psci-0.2", .data = psci_0_2_init}, - {}, -}; - -int __init psci_dt_init(void) -{ - struct device_node *np; - const struct of_device_id *matched_np; - psci_initcall_t init_fn; - - np = of_find_matching_node_and_match(NULL, psci_of_match, &matched_np); - - if (!np) - return -ENODEV; - - init_fn = (psci_initcall_t)matched_np->data; - return init_fn(np); -} - -#ifdef CONFIG_ACPI -/* - * We use PSCI 0.2+ when ACPI is deployed on ARM64 and it's - * explicitly clarified in SBBR - */ -int __init psci_acpi_init(void) -{ - if (!acpi_psci_present()) { - pr_info("is not implemented in ACPI.\n"); - return -EOPNOTSUPP; - } - - pr_info("probing for conduit method from ACPI.\n"); - - if (acpi_psci_use_hvc()) - invoke_psci_fn = __invoke_psci_fn_hvc; - else - invoke_psci_fn = __invoke_psci_fn_smc; - - return psci_probe(); -} -#endif - -#ifdef CONFIG_SMP - static int __init cpu_psci_cpu_init(unsigned int cpu) { return 0; @@ -489,11 +135,6 @@ static int cpu_psci_cpu_boot(unsigned int cpu) } #ifdef CONFIG_HOTPLUG_CPU -static bool psci_tos_resident_on(int cpu) -{ - return cpu == resident_cpu; -} - static int cpu_psci_cpu_disable(unsigned int cpu) { /* Fail early if we don't have CPU_OFF support */ @@ -550,7 +191,6 @@ static int cpu_psci_cpu_kill(unsigned int cpu) return -ETIMEDOUT; } #endif -#endif static int psci_suspend_finisher(unsigned long index) { @@ -585,7 +225,6 @@ const struct cpu_operations cpu_psci_ops = { .cpu_init_idle = cpu_psci_cpu_init_idle, .cpu_suspend = cpu_psci_cpu_suspend, #endif -#ifdef CONFIG_SMP .cpu_init = cpu_psci_cpu_init, .cpu_prepare = cpu_psci_cpu_prepare, .cpu_boot = cpu_psci_cpu_boot, @@ -594,6 +233,5 @@ const struct cpu_operations cpu_psci_ops = { .cpu_die = cpu_psci_cpu_die, .cpu_kill = cpu_psci_cpu_kill, #endif -#endif }; diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index d882b833dbdb..1971f491bb90 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -826,6 +826,30 @@ static int compat_vfp_set(struct task_struct *target, return ret; } +static int compat_tls_get(struct task_struct *target, + const struct user_regset *regset, unsigned int pos, + unsigned int count, void *kbuf, void __user *ubuf) +{ + compat_ulong_t tls = (compat_ulong_t)target->thread.tp_value; + return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &tls, 0, -1); +} + +static int compat_tls_set(struct task_struct *target, + const struct user_regset *regset, unsigned int pos, + unsigned int count, const void *kbuf, + const void __user *ubuf) +{ + int ret; + compat_ulong_t tls; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &tls, 0, -1); + if (ret) + return ret; + + target->thread.tp_value = tls; + return ret; +} + static const struct user_regset aarch32_regsets[] = { [REGSET_COMPAT_GPR] = { .core_note_type = NT_PRSTATUS, @@ -850,6 +874,64 @@ static const struct user_regset_view user_aarch32_view = { .regsets = aarch32_regsets, .n = ARRAY_SIZE(aarch32_regsets) }; +static const struct user_regset aarch32_ptrace_regsets[] = { + [REGSET_GPR] = { + .core_note_type = NT_PRSTATUS, + .n = COMPAT_ELF_NGREG, + .size = sizeof(compat_elf_greg_t), + .align = sizeof(compat_elf_greg_t), + .get = compat_gpr_get, + .set = compat_gpr_set + }, + [REGSET_FPR] = { + .core_note_type = NT_ARM_VFP, + .n = VFP_STATE_SIZE / sizeof(compat_ulong_t), + .size = sizeof(compat_ulong_t), + .align = sizeof(compat_ulong_t), + .get = compat_vfp_get, + .set = compat_vfp_set + }, + [REGSET_TLS] = { + .core_note_type = NT_ARM_TLS, + .n = 1, + .size = sizeof(compat_ulong_t), + .align = sizeof(compat_ulong_t), + .get = compat_tls_get, + .set = compat_tls_set, + }, +#ifdef CONFIG_HAVE_HW_BREAKPOINT + [REGSET_HW_BREAK] = { + .core_note_type = NT_ARM_HW_BREAK, + .n = sizeof(struct user_hwdebug_state) / sizeof(u32), + .size = sizeof(u32), + .align = sizeof(u32), + .get = hw_break_get, + .set = hw_break_set, + }, + [REGSET_HW_WATCH] = { + .core_note_type = NT_ARM_HW_WATCH, + .n = sizeof(struct user_hwdebug_state) / sizeof(u32), + .size = sizeof(u32), + .align = sizeof(u32), + .get = hw_break_get, + .set = hw_break_set, + }, +#endif + [REGSET_SYSTEM_CALL] = { + .core_note_type = NT_ARM_SYSTEM_CALL, + .n = 1, + .size = sizeof(int), + .align = sizeof(int), + .get = system_call_get, + .set = system_call_set, + }, +}; + +static const struct user_regset_view user_aarch32_ptrace_view = { + .name = "aarch32", .e_machine = EM_ARM, + .regsets = aarch32_ptrace_regsets, .n = ARRAY_SIZE(aarch32_ptrace_regsets) +}; + static int compat_ptrace_read_user(struct task_struct *tsk, compat_ulong_t off, compat_ulong_t __user *ret) { @@ -1109,8 +1191,16 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, const struct user_regset_view *task_user_regset_view(struct task_struct *task) { #ifdef CONFIG_COMPAT - if (is_compat_thread(task_thread_info(task))) + /* + * Core dumping of 32-bit tasks or compat ptrace requests must use the + * user_aarch32_view compatible with arm32. Native ptrace requests on + * 32-bit children use an extended user_aarch32_ptrace_view to allow + * access to the TLS register. + */ + if (is_compat_task()) return &user_aarch32_view; + else if (is_compat_thread(task_thread_info(task))) + return &user_aarch32_ptrace_view; #endif return &user_aarch64_view; } diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 926ae8d9abc5..6bab21f84a9f 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -45,6 +45,7 @@ #include #include #include +#include #include #include @@ -60,9 +61,7 @@ #include #include #include -#include #include -#include #include unsigned long elf_hwcap __read_mostly; @@ -130,7 +129,6 @@ bool arch_match_cpu_phys_id(int cpu, u64 phys_id) } struct mpidr_hash mpidr_hash; -#ifdef CONFIG_SMP /** * smp_build_mpidr_hash - Pre-compute shifts required at each affinity * level in order to build a linear index from an @@ -196,35 +194,11 @@ static void __init smp_build_mpidr_hash(void) pr_warn("Large number of MPIDR hash buckets detected\n"); __flush_dcache_area(&mpidr_hash, sizeof(struct mpidr_hash)); } -#endif - -static void __init hyp_mode_check(void) -{ - if (is_hyp_mode_available()) - pr_info("CPU: All CPU(s) started at EL2\n"); - else if (is_hyp_mode_mismatched()) - WARN_TAINT(1, TAINT_CPU_OUT_OF_SPEC, - "CPU: CPUs started in inconsistent modes"); - else - pr_info("CPU: All CPU(s) started at EL1\n"); -} - -void __init do_post_cpus_up_work(void) -{ - hyp_mode_check(); - apply_alternatives_all(); -} - -#ifdef CONFIG_UP_LATE_INIT -void __init up_late_init(void) -{ - do_post_cpus_up_work(); -} -#endif /* CONFIG_UP_LATE_INIT */ static void __init setup_processor(void) { - u64 features, block; + u64 features; + s64 block; u32 cwg; int cls; @@ -254,8 +228,8 @@ static void __init setup_processor(void) * for non-negative values. Negative values are reserved. */ features = read_cpuid(ID_AA64ISAR0_EL1); - block = (features >> 4) & 0xf; - if (!(block & 0x8)) { + block = cpuid_feature_extract_field(features, 4); + if (block > 0) { switch (block) { default: case 2: @@ -267,26 +241,36 @@ static void __init setup_processor(void) } } - block = (features >> 8) & 0xf; - if (block && !(block & 0x8)) + if (cpuid_feature_extract_field(features, 8) > 0) elf_hwcap |= HWCAP_SHA1; - block = (features >> 12) & 0xf; - if (block && !(block & 0x8)) + if (cpuid_feature_extract_field(features, 12) > 0) elf_hwcap |= HWCAP_SHA2; - block = (features >> 16) & 0xf; - if (block && !(block & 0x8)) + if (cpuid_feature_extract_field(features, 16) > 0) elf_hwcap |= HWCAP_CRC32; + block = cpuid_feature_extract_field(features, 20); + if (block > 0) { + switch (block) { + default: + case 2: + elf_hwcap |= HWCAP_ATOMICS; + case 1: + /* RESERVED */ + case 0: + break; + } + } + #ifdef CONFIG_COMPAT /* * ID_ISAR5_EL1 carries similar information as above, but pertaining to - * the Aarch32 32-bit execution state. + * the AArch32 32-bit execution state. */ features = read_cpuid(ID_ISAR5_EL1); - block = (features >> 4) & 0xf; - if (!(block & 0x8)) { + block = cpuid_feature_extract_field(features, 4); + if (block > 0) { switch (block) { default: case 2: @@ -298,16 +282,13 @@ static void __init setup_processor(void) } } - block = (features >> 8) & 0xf; - if (block && !(block & 0x8)) + if (cpuid_feature_extract_field(features, 8) > 0) compat_elf_hwcap2 |= COMPAT_HWCAP2_SHA1; - block = (features >> 12) & 0xf; - if (block && !(block & 0x8)) + if (cpuid_feature_extract_field(features, 12) > 0) compat_elf_hwcap2 |= COMPAT_HWCAP2_SHA2; - block = (features >> 16) & 0xf; - if (block && !(block & 0x8)) + if (cpuid_feature_extract_field(features, 16) > 0) compat_elf_hwcap2 |= COMPAT_HWCAP2_CRC32; #endif } @@ -358,6 +339,67 @@ static void __init request_standard_resources(void) } } +#ifdef CONFIG_BLK_DEV_INITRD +/* + * Relocate initrd if it is not completely within the linear mapping. + * This would be the case if mem= cuts out all or part of it. + */ +static void __init relocate_initrd(void) +{ + phys_addr_t orig_start = __virt_to_phys(initrd_start); + phys_addr_t orig_end = __virt_to_phys(initrd_end); + phys_addr_t ram_end = memblock_end_of_DRAM(); + phys_addr_t new_start; + unsigned long size, to_free = 0; + void *dest; + + if (orig_end <= ram_end) + return; + + /* + * Any of the original initrd which overlaps the linear map should + * be freed after relocating. + */ + if (orig_start < ram_end) + to_free = ram_end - orig_start; + + size = orig_end - orig_start; + + /* initrd needs to be relocated completely inside linear mapping */ + new_start = memblock_find_in_range(0, PFN_PHYS(max_pfn), + size, PAGE_SIZE); + if (!new_start) + panic("Cannot relocate initrd of size %ld\n", size); + memblock_reserve(new_start, size); + + initrd_start = __phys_to_virt(new_start); + initrd_end = initrd_start + size; + + pr_info("Moving initrd from [%llx-%llx] to [%llx-%llx]\n", + orig_start, orig_start + size - 1, + new_start, new_start + size - 1); + + dest = (void *)initrd_start; + + if (to_free) { + memcpy(dest, (void *)__phys_to_virt(orig_start), to_free); + dest += to_free; + } + + copy_from_early_mem(dest, orig_start + to_free, size - to_free); + + if (to_free) { + pr_info("Freeing original RAMDISK from [%llx-%llx]\n", + orig_start, orig_start + to_free - 1); + memblock_free(orig_start, to_free); + } +} +#else +static inline void __init relocate_initrd(void) +{ +} +#endif + u64 __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] = INVALID_HWID }; void __init setup_arch(char **cmdline_p) @@ -391,6 +433,7 @@ void __init setup_arch(char **cmdline_p) acpi_boot_table_init(); paging_init(); + relocate_initrd(); request_standard_resources(); early_ioremap_reset(); @@ -404,10 +447,8 @@ void __init setup_arch(char **cmdline_p) xen_early_init(); cpu_read_bootcpu_ops(); -#ifdef CONFIG_SMP smp_init_cpus(); smp_build_mpidr_hash(); -#endif #ifdef CONFIG_VT #if defined(CONFIG_VGA_CONSOLE) @@ -426,8 +467,13 @@ void __init setup_arch(char **cmdline_p) static int __init arm64_device_init(void) { - of_iommu_init(); - of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL); + if (of_have_populated_dt()) { + of_iommu_init(); + of_platform_populate(NULL, of_default_bus_match_table, + NULL, NULL); + } else if (acpi_disabled) { + pr_crit("Device tree not populated\n"); + } return 0; } arch_initcall_sync(arm64_device_init); @@ -455,6 +501,7 @@ static const char *hwcap_str[] = { "sha1", "sha2", "crc32", + "atomics", NULL }; @@ -507,9 +554,7 @@ static int c_show(struct seq_file *m, void *v) * online processors, looking for lines beginning with * "processor". Give glibc what it expects. */ -#ifdef CONFIG_SMP seq_printf(m, "processor\t: %d\n", i); -#endif /* * Dump out the common processor features in a single line. diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S index 803cfea41962..f586f7c875e2 100644 --- a/arch/arm64/kernel/sleep.S +++ b/arch/arm64/kernel/sleep.S @@ -82,7 +82,6 @@ ENTRY(__cpu_suspend_enter) str x2, [x0, #CPU_CTX_SP] ldr x1, =sleep_save_sp ldr x1, [x1, #SLEEP_SAVE_SP_VIRT] -#ifdef CONFIG_SMP mrs x7, mpidr_el1 ldr x9, =mpidr_hash ldr x10, [x9, #MPIDR_HASH_MASK] @@ -94,7 +93,6 @@ ENTRY(__cpu_suspend_enter) ldp w5, w6, [x9, #(MPIDR_HASH_SHIFTS + 8)] compute_mpidr_hash x8, x3, x4, x5, x6, x7, x10 add x1, x1, x8, lsl #3 -#endif bl __cpu_suspend_save /* * Grab suspend finisher in x20 and its argument in x19 @@ -135,6 +133,14 @@ ENTRY(cpu_resume_mmu) ldr x3, =cpu_resume_after_mmu msr sctlr_el1, x0 // restore sctlr_el1 isb + /* + * Invalidate the local I-cache so that any instructions fetched + * speculatively from the PoC are discarded, since they may have + * been dynamically patched at the PoU. + */ + ic iallu + dsb nsh + isb br x3 // global jump to virtual address ENDPROC(cpu_resume_mmu) .popsection @@ -151,7 +157,6 @@ ENDPROC(cpu_resume_after_mmu) ENTRY(cpu_resume) bl el2_setup // if in EL2 drop to EL1 cleanly -#ifdef CONFIG_SMP mrs x1, mpidr_el1 adrp x8, mpidr_hash add x8, x8, #:lo12:mpidr_hash // x8 = struct mpidr_hash phys address @@ -161,9 +166,6 @@ ENTRY(cpu_resume) ldp w5, w6, [x8, #(MPIDR_HASH_SHIFTS + 8)] compute_mpidr_hash x7, x3, x4, x5, x6, x1, x2 /* x7 contains hash index, let's use it to grab context pointer */ -#else - mov x7, xzr -#endif ldr_l x0, sleep_save_sp + SLEEP_SAVE_SP_PHYS ldr x0, [x0, x7, lsl #3] /* load sp from context */ diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 50fb4696654e..dbdaacddd9a5 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -52,6 +52,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -310,10 +311,22 @@ void cpu_die(void) } #endif +static void __init hyp_mode_check(void) +{ + if (is_hyp_mode_available()) + pr_info("CPU: All CPU(s) started at EL2\n"); + else if (is_hyp_mode_mismatched()) + WARN_TAINT(1, TAINT_CPU_OUT_OF_SPEC, + "CPU: CPUs started in inconsistent modes"); + else + pr_info("CPU: All CPU(s) started at EL1\n"); +} + void __init smp_cpus_done(unsigned int max_cpus) { pr_info("SMP: Total of %d processors activated.\n", num_online_cpus()); - do_post_cpus_up_work(); + hyp_mode_check(); + apply_alternatives_all(); } void __init smp_prepare_boot_cpu(void) diff --git a/arch/arm64/kernel/time.c b/arch/arm64/kernel/time.c index 42f9195cf2f8..149151fb42bb 100644 --- a/arch/arm64/kernel/time.c +++ b/arch/arm64/kernel/time.c @@ -42,7 +42,6 @@ #include #include -#ifdef CONFIG_SMP unsigned long profile_pc(struct pt_regs *regs) { struct stackframe frame; @@ -62,7 +61,6 @@ unsigned long profile_pc(struct pt_regs *regs) return frame.pc; } EXPORT_SYMBOL(profile_pc); -#endif void __init time_init(void) { diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index fcb8f7b42271..694f6deedbab 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -300,6 +300,6 @@ void __init init_cpu_topology(void) * Discard anything that was parsed if we hit an error so we * don't use partial information. */ - if (parse_dt_topology()) + if (of_have_populated_dt() && parse_dt_topology()) reset_cpu_topology(); } diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 566bc4c35040..f93aae5e4307 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -17,6 +17,7 @@ * along with this program. If not, see . */ +#include #include #include #include @@ -32,8 +33,10 @@ #include #include +#include #include #include +#include #include #include #include @@ -52,11 +55,12 @@ int show_unhandled_signals = 1; * Dump out the contents of some memory nicely... */ static void dump_mem(const char *lvl, const char *str, unsigned long bottom, - unsigned long top) + unsigned long top, bool compat) { unsigned long first; mm_segment_t fs; int i; + unsigned int width = compat ? 4 : 8; /* * We need to switch to kernel mode so that we can use __get_user @@ -75,13 +79,22 @@ static void dump_mem(const char *lvl, const char *str, unsigned long bottom, memset(str, ' ', sizeof(str)); str[sizeof(str) - 1] = '\0'; - for (p = first, i = 0; i < 8 && p < top; i++, p += 4) { + for (p = first, i = 0; i < (32 / width) + && p < top; i++, p += width) { if (p >= bottom && p < top) { - unsigned int val; - if (__get_user(val, (unsigned int *)p) == 0) - sprintf(str + i * 9, " %08x", val); - else - sprintf(str + i * 9, " ????????"); + unsigned long val; + + if (width == 8) { + if (__get_user(val, (unsigned long *)p) == 0) + sprintf(str + i * 17, " %016lx", val); + else + sprintf(str + i * 17, " ????????????????"); + } else { + if (__get_user(val, (unsigned int *)p) == 0) + sprintf(str + i * 9, " %08lx", val); + else + sprintf(str + i * 9, " ????????"); + } } } printk("%s%04lx:%s\n", lvl, first & 0xffff, str); @@ -95,7 +108,7 @@ static void dump_backtrace_entry(unsigned long where, unsigned long stack) print_ip_sym(where); if (in_exception_text(where)) dump_mem("", "Exception stack", stack, - stack + sizeof(struct pt_regs)); + stack + sizeof(struct pt_regs), false); } static void dump_instr(const char *lvl, struct pt_regs *regs) @@ -179,11 +192,7 @@ void show_stack(struct task_struct *tsk, unsigned long *sp) #else #define S_PREEMPT "" #endif -#ifdef CONFIG_SMP #define S_SMP " SMP" -#else -#define S_SMP "" -#endif static int __die(const char *str, int err, struct thread_info *thread, struct pt_regs *regs) @@ -207,7 +216,8 @@ static int __die(const char *str, int err, struct thread_info *thread, if (!user_mode(regs) || in_interrupt()) { dump_mem(KERN_EMERG, "Stack: ", regs->sp, - THREAD_SIZE + (unsigned long)task_stack_page(tsk)); + THREAD_SIZE + (unsigned long)task_stack_page(tsk), + compat_user_mode(regs)); dump_backtrace(regs, tsk); dump_instr(KERN_EMERG, regs); } @@ -459,7 +469,63 @@ void __pgd_error(const char *file, int line, unsigned long val) pr_crit("%s:%d: bad pgd %016lx.\n", file, line, val); } +/* GENERIC_BUG traps */ + +int is_valid_bugaddr(unsigned long addr) +{ + /* + * bug_handler() only called for BRK #BUG_BRK_IMM. + * So the answer is trivial -- any spurious instances with no + * bug table entry will be rejected by report_bug() and passed + * back to the debug-monitors code and handled as a fatal + * unexpected debug exception. + */ + return 1; +} + +static int bug_handler(struct pt_regs *regs, unsigned int esr) +{ + if (user_mode(regs)) + return DBG_HOOK_ERROR; + + switch (report_bug(regs->pc, regs)) { + case BUG_TRAP_TYPE_BUG: + die("Oops - BUG", regs, 0); + break; + + case BUG_TRAP_TYPE_WARN: + /* Ideally, report_bug() should backtrace for us... but no. */ + dump_backtrace(regs, NULL); + break; + + default: + /* unknown/unrecognised bug trap type */ + return DBG_HOOK_ERROR; + } + + /* If thread survives, skip over the BUG instruction and continue: */ + regs->pc += AARCH64_INSN_SIZE; /* skip BRK and resume */ + return DBG_HOOK_HANDLED; +} + +static struct break_hook bug_break_hook = { + .esr_val = 0xf2000000 | BUG_BRK_IMM, + .esr_mask = 0xffffffff, + .fn = bug_handler, +}; + +/* + * Initial handler for AArch64 BRK exceptions + * This handler only used until debug_traps_init(). + */ +int __init early_brk64(unsigned long addr, unsigned int esr, + struct pt_regs *regs) +{ + return bug_handler(regs, esr) != DBG_HOOK_HANDLED; +} + +/* This registration must happen early, before debug_traps_init(). */ void __init trap_init(void) { - return; + register_break_hook(&bug_break_hook); } diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index f90f4aa7f88d..1949fe5f5424 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -17,7 +17,7 @@ kvm-$(CONFIG_KVM_ARM_HOST) += $(ARM)/psci.o $(ARM)/perf.o kvm-$(CONFIG_KVM_ARM_HOST) += emulate.o inject_fault.o regmap.o kvm-$(CONFIG_KVM_ARM_HOST) += hyp.o hyp-init.o handle_exit.o -kvm-$(CONFIG_KVM_ARM_HOST) += guest.o reset.o sys_regs.o sys_regs_generic_v8.o +kvm-$(CONFIG_KVM_ARM_HOST) += guest.o debug.o reset.o sys_regs.o sys_regs_generic_v8.o kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic.o kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v2.o diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c new file mode 100644 index 000000000000..47e5f0feaee8 --- /dev/null +++ b/arch/arm64/kvm/debug.c @@ -0,0 +1,217 @@ +/* + * Debug and Guest Debug support + * + * Copyright (C) 2015 - Linaro Ltd + * Author: Alex Bennée + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include + +#include +#include +#include +#include + +#include "trace.h" + +/* These are the bits of MDSCR_EL1 we may manipulate */ +#define MDSCR_EL1_DEBUG_MASK (DBG_MDSCR_SS | \ + DBG_MDSCR_KDE | \ + DBG_MDSCR_MDE) + +static DEFINE_PER_CPU(u32, mdcr_el2); + +/** + * save/restore_guest_debug_regs + * + * For some debug operations we need to tweak some guest registers. As + * a result we need to save the state of those registers before we + * make those modifications. + * + * Guest access to MDSCR_EL1 is trapped by the hypervisor and handled + * after we have restored the preserved value to the main context. + */ +static void save_guest_debug_regs(struct kvm_vcpu *vcpu) +{ + vcpu->arch.guest_debug_preserved.mdscr_el1 = vcpu_sys_reg(vcpu, MDSCR_EL1); + + trace_kvm_arm_set_dreg32("Saved MDSCR_EL1", + vcpu->arch.guest_debug_preserved.mdscr_el1); +} + +static void restore_guest_debug_regs(struct kvm_vcpu *vcpu) +{ + vcpu_sys_reg(vcpu, MDSCR_EL1) = vcpu->arch.guest_debug_preserved.mdscr_el1; + + trace_kvm_arm_set_dreg32("Restored MDSCR_EL1", + vcpu_sys_reg(vcpu, MDSCR_EL1)); +} + +/** + * kvm_arm_init_debug - grab what we need for debug + * + * Currently the sole task of this function is to retrieve the initial + * value of mdcr_el2 so we can preserve MDCR_EL2.HPMN which has + * presumably been set-up by some knowledgeable bootcode. + * + * It is called once per-cpu during CPU hyp initialisation. + */ + +void kvm_arm_init_debug(void) +{ + __this_cpu_write(mdcr_el2, kvm_call_hyp(__kvm_get_mdcr_el2)); +} + +/** + * kvm_arm_reset_debug_ptr - reset the debug ptr to point to the vcpu state + */ + +void kvm_arm_reset_debug_ptr(struct kvm_vcpu *vcpu) +{ + vcpu->arch.debug_ptr = &vcpu->arch.vcpu_debug_state; +} + +/** + * kvm_arm_setup_debug - set up debug related stuff + * + * @vcpu: the vcpu pointer + * + * This is called before each entry into the hypervisor to setup any + * debug related registers. Currently this just ensures we will trap + * access to: + * - Performance monitors (MDCR_EL2_TPM/MDCR_EL2_TPMCR) + * - Debug ROM Address (MDCR_EL2_TDRA) + * - OS related registers (MDCR_EL2_TDOSA) + * + * Additionally, KVM only traps guest accesses to the debug registers if + * the guest is not actively using them (see the KVM_ARM64_DEBUG_DIRTY + * flag on vcpu->arch.debug_flags). Since the guest must not interfere + * with the hardware state when debugging the guest, we must ensure that + * trapping is enabled whenever we are debugging the guest using the + * debug registers. + */ + +void kvm_arm_setup_debug(struct kvm_vcpu *vcpu) +{ + bool trap_debug = !(vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY); + + trace_kvm_arm_setup_debug(vcpu, vcpu->guest_debug); + + vcpu->arch.mdcr_el2 = __this_cpu_read(mdcr_el2) & MDCR_EL2_HPMN_MASK; + vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | + MDCR_EL2_TPMCR | + MDCR_EL2_TDRA | + MDCR_EL2_TDOSA); + + /* Is Guest debugging in effect? */ + if (vcpu->guest_debug) { + /* Route all software debug exceptions to EL2 */ + vcpu->arch.mdcr_el2 |= MDCR_EL2_TDE; + + /* Save guest debug state */ + save_guest_debug_regs(vcpu); + + /* + * Single Step (ARM ARM D2.12.3 The software step state + * machine) + * + * If we are doing Single Step we need to manipulate + * the guest's MDSCR_EL1.SS and PSTATE.SS. Once the + * step has occurred the hypervisor will trap the + * debug exception and we return to userspace. + * + * If the guest attempts to single step its userspace + * we would have to deal with a trapped exception + * while in the guest kernel. Because this would be + * hard to unwind we suppress the guest's ability to + * do so by masking MDSCR_EL.SS. + * + * This confuses guest debuggers which use + * single-step behind the scenes but everything + * returns to normal once the host is no longer + * debugging the system. + */ + if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) { + *vcpu_cpsr(vcpu) |= DBG_SPSR_SS; + vcpu_sys_reg(vcpu, MDSCR_EL1) |= DBG_MDSCR_SS; + } else { + vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~DBG_MDSCR_SS; + } + + trace_kvm_arm_set_dreg32("SPSR_EL2", *vcpu_cpsr(vcpu)); + + /* + * HW Breakpoints and watchpoints + * + * We simply switch the debug_ptr to point to our new + * external_debug_state which has been populated by the + * debug ioctl. The existing KVM_ARM64_DEBUG_DIRTY + * mechanism ensures the registers are updated on the + * world switch. + */ + if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW) { + /* Enable breakpoints/watchpoints */ + vcpu_sys_reg(vcpu, MDSCR_EL1) |= DBG_MDSCR_MDE; + + vcpu->arch.debug_ptr = &vcpu->arch.external_debug_state; + vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY; + trap_debug = true; + + trace_kvm_arm_set_regset("BKPTS", get_num_brps(), + &vcpu->arch.debug_ptr->dbg_bcr[0], + &vcpu->arch.debug_ptr->dbg_bvr[0]); + + trace_kvm_arm_set_regset("WAPTS", get_num_wrps(), + &vcpu->arch.debug_ptr->dbg_wcr[0], + &vcpu->arch.debug_ptr->dbg_wvr[0]); + } + } + + BUG_ON(!vcpu->guest_debug && + vcpu->arch.debug_ptr != &vcpu->arch.vcpu_debug_state); + + /* Trap debug register access */ + if (trap_debug) + vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA; + + trace_kvm_arm_set_dreg32("MDCR_EL2", vcpu->arch.mdcr_el2); + trace_kvm_arm_set_dreg32("MDSCR_EL1", vcpu_sys_reg(vcpu, MDSCR_EL1)); +} + +void kvm_arm_clear_debug(struct kvm_vcpu *vcpu) +{ + trace_kvm_arm_clear_debug(vcpu->guest_debug); + + if (vcpu->guest_debug) { + restore_guest_debug_regs(vcpu); + + /* + * If we were using HW debug we need to restore the + * debug_ptr to the guest debug state. + */ + if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW) { + kvm_arm_reset_debug_ptr(vcpu); + + trace_kvm_arm_set_regset("BKPTS", get_num_brps(), + &vcpu->arch.debug_ptr->dbg_bcr[0], + &vcpu->arch.debug_ptr->dbg_bvr[0]); + + trace_kvm_arm_set_regset("WAPTS", get_num_wrps(), + &vcpu->arch.debug_ptr->dbg_wcr[0], + &vcpu->arch.debug_ptr->dbg_wvr[0]); + } + } +} diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 9535bd555d1d..d250160d32bc 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -32,6 +32,8 @@ #include #include +#include "trace.h" + struct kvm_stats_debugfs_item debugfs_entries[] = { { NULL } }; @@ -293,7 +295,8 @@ int __attribute_const__ kvm_target_cpu(void) break; }; - return -EINVAL; + /* Return a default generic target */ + return KVM_ARM_TARGET_GENERIC_V8; } int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init) @@ -331,3 +334,41 @@ int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, { return -EINVAL; } + +#define KVM_GUESTDBG_VALID_MASK (KVM_GUESTDBG_ENABLE | \ + KVM_GUESTDBG_USE_SW_BP | \ + KVM_GUESTDBG_USE_HW | \ + KVM_GUESTDBG_SINGLESTEP) + +/** + * kvm_arch_vcpu_ioctl_set_guest_debug - set up guest debugging + * @kvm: pointer to the KVM struct + * @kvm_guest_debug: the ioctl data buffer + * + * This sets up and enables the VM for guest debugging. Userspace + * passes in a control flag to enable different debug types and + * potentially other architecture specific information in the rest of + * the structure. + */ +int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, + struct kvm_guest_debug *dbg) +{ + trace_kvm_set_guest_debug(vcpu, dbg->control); + + if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) + return -EINVAL; + + if (dbg->control & KVM_GUESTDBG_ENABLE) { + vcpu->guest_debug = dbg->control; + + /* Hardware assisted Break and Watch points */ + if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW) { + vcpu->arch.external_debug_state = dbg->arch; + } + + } else { + /* If not enabled clear all flags */ + vcpu->guest_debug = 0; + } + return 0; +} diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index 524fa25671fc..68a0759b1375 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -82,6 +82,45 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run) return 1; } +/** + * kvm_handle_guest_debug - handle a debug exception instruction + * + * @vcpu: the vcpu pointer + * @run: access to the kvm_run structure for results + * + * We route all debug exceptions through the same handler. If both the + * guest and host are using the same debug facilities it will be up to + * userspace to re-inject the correct exception for guest delivery. + * + * @return: 0 (while setting run->exit_reason), -1 for error + */ +static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run) +{ + u32 hsr = kvm_vcpu_get_hsr(vcpu); + int ret = 0; + + run->exit_reason = KVM_EXIT_DEBUG; + run->debug.arch.hsr = hsr; + + switch (hsr >> ESR_ELx_EC_SHIFT) { + case ESR_ELx_EC_WATCHPT_LOW: + run->debug.arch.far = vcpu->arch.fault.far_el2; + /* fall through */ + case ESR_ELx_EC_SOFTSTP_LOW: + case ESR_ELx_EC_BREAKPT_LOW: + case ESR_ELx_EC_BKPT32: + case ESR_ELx_EC_BRK64: + break; + default: + kvm_err("%s: un-handled case hsr: %#08x\n", + __func__, (unsigned int) hsr); + ret = -1; + break; + } + + return ret; +} + static exit_handle_fn arm_exit_handlers[] = { [ESR_ELx_EC_WFx] = kvm_handle_wfx, [ESR_ELx_EC_CP15_32] = kvm_handle_cp15_32, @@ -96,6 +135,11 @@ static exit_handle_fn arm_exit_handlers[] = { [ESR_ELx_EC_SYS64] = kvm_handle_sys_reg, [ESR_ELx_EC_IABT_LOW] = kvm_handle_guest_abort, [ESR_ELx_EC_DABT_LOW] = kvm_handle_guest_abort, + [ESR_ELx_EC_SOFTSTP_LOW]= kvm_handle_guest_debug, + [ESR_ELx_EC_WATCHPT_LOW]= kvm_handle_guest_debug, + [ESR_ELx_EC_BREAKPT_LOW]= kvm_handle_guest_debug, + [ESR_ELx_EC_BKPT32] = kvm_handle_guest_debug, + [ESR_ELx_EC_BRK64] = kvm_handle_guest_debug, }; static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu) diff --git a/arch/arm64/kvm/hyp.S b/arch/arm64/kvm/hyp.S index 17a8fb14f428..37c89ea2c572 100644 --- a/arch/arm64/kvm/hyp.S +++ b/arch/arm64/kvm/hyp.S @@ -230,199 +230,52 @@ stp x24, x25, [x3, #160] .endm -.macro save_debug - // x2: base address for cpu context - // x3: tmp register +.macro save_debug type + // x4: pointer to register set + // x5: number of registers to skip + // x6..x22 trashed - mrs x26, id_aa64dfr0_el1 - ubfx x24, x26, #12, #4 // Extract BRPs - ubfx x25, x26, #20, #4 // Extract WRPs - mov w26, #15 - sub w24, w26, w24 // How many BPs to skip - sub w25, w26, w25 // How many WPs to skip - - add x3, x2, #CPU_SYSREG_OFFSET(DBGBCR0_EL1) - - adr x26, 1f - add x26, x26, x24, lsl #2 - br x26 + adr x22, 1f + add x22, x22, x5, lsl #2 + br x22 1: - mrs x20, dbgbcr15_el1 - mrs x19, dbgbcr14_el1 - mrs x18, dbgbcr13_el1 - mrs x17, dbgbcr12_el1 - mrs x16, dbgbcr11_el1 - mrs x15, dbgbcr10_el1 - mrs x14, dbgbcr9_el1 - mrs x13, dbgbcr8_el1 - mrs x12, dbgbcr7_el1 - mrs x11, dbgbcr6_el1 - mrs x10, dbgbcr5_el1 - mrs x9, dbgbcr4_el1 - mrs x8, dbgbcr3_el1 - mrs x7, dbgbcr2_el1 - mrs x6, dbgbcr1_el1 - mrs x5, dbgbcr0_el1 - - adr x26, 1f - add x26, x26, x24, lsl #2 - br x26 + mrs x21, \type\()15_el1 + mrs x20, \type\()14_el1 + mrs x19, \type\()13_el1 + mrs x18, \type\()12_el1 + mrs x17, \type\()11_el1 + mrs x16, \type\()10_el1 + mrs x15, \type\()9_el1 + mrs x14, \type\()8_el1 + mrs x13, \type\()7_el1 + mrs x12, \type\()6_el1 + mrs x11, \type\()5_el1 + mrs x10, \type\()4_el1 + mrs x9, \type\()3_el1 + mrs x8, \type\()2_el1 + mrs x7, \type\()1_el1 + mrs x6, \type\()0_el1 + adr x22, 1f + add x22, x22, x5, lsl #2 + br x22 1: - str x20, [x3, #(15 * 8)] - str x19, [x3, #(14 * 8)] - str x18, [x3, #(13 * 8)] - str x17, [x3, #(12 * 8)] - str x16, [x3, #(11 * 8)] - str x15, [x3, #(10 * 8)] - str x14, [x3, #(9 * 8)] - str x13, [x3, #(8 * 8)] - str x12, [x3, #(7 * 8)] - str x11, [x3, #(6 * 8)] - str x10, [x3, #(5 * 8)] - str x9, [x3, #(4 * 8)] - str x8, [x3, #(3 * 8)] - str x7, [x3, #(2 * 8)] - str x6, [x3, #(1 * 8)] - str x5, [x3, #(0 * 8)] - - add x3, x2, #CPU_SYSREG_OFFSET(DBGBVR0_EL1) - - adr x26, 1f - add x26, x26, x24, lsl #2 - br x26 -1: - mrs x20, dbgbvr15_el1 - mrs x19, dbgbvr14_el1 - mrs x18, dbgbvr13_el1 - mrs x17, dbgbvr12_el1 - mrs x16, dbgbvr11_el1 - mrs x15, dbgbvr10_el1 - mrs x14, dbgbvr9_el1 - mrs x13, dbgbvr8_el1 - mrs x12, dbgbvr7_el1 - mrs x11, dbgbvr6_el1 - mrs x10, dbgbvr5_el1 - mrs x9, dbgbvr4_el1 - mrs x8, dbgbvr3_el1 - mrs x7, dbgbvr2_el1 - mrs x6, dbgbvr1_el1 - mrs x5, dbgbvr0_el1 - - adr x26, 1f - add x26, x26, x24, lsl #2 - br x26 - -1: - str x20, [x3, #(15 * 8)] - str x19, [x3, #(14 * 8)] - str x18, [x3, #(13 * 8)] - str x17, [x3, #(12 * 8)] - str x16, [x3, #(11 * 8)] - str x15, [x3, #(10 * 8)] - str x14, [x3, #(9 * 8)] - str x13, [x3, #(8 * 8)] - str x12, [x3, #(7 * 8)] - str x11, [x3, #(6 * 8)] - str x10, [x3, #(5 * 8)] - str x9, [x3, #(4 * 8)] - str x8, [x3, #(3 * 8)] - str x7, [x3, #(2 * 8)] - str x6, [x3, #(1 * 8)] - str x5, [x3, #(0 * 8)] - - add x3, x2, #CPU_SYSREG_OFFSET(DBGWCR0_EL1) - - adr x26, 1f - add x26, x26, x25, lsl #2 - br x26 -1: - mrs x20, dbgwcr15_el1 - mrs x19, dbgwcr14_el1 - mrs x18, dbgwcr13_el1 - mrs x17, dbgwcr12_el1 - mrs x16, dbgwcr11_el1 - mrs x15, dbgwcr10_el1 - mrs x14, dbgwcr9_el1 - mrs x13, dbgwcr8_el1 - mrs x12, dbgwcr7_el1 - mrs x11, dbgwcr6_el1 - mrs x10, dbgwcr5_el1 - mrs x9, dbgwcr4_el1 - mrs x8, dbgwcr3_el1 - mrs x7, dbgwcr2_el1 - mrs x6, dbgwcr1_el1 - mrs x5, dbgwcr0_el1 - - adr x26, 1f - add x26, x26, x25, lsl #2 - br x26 - -1: - str x20, [x3, #(15 * 8)] - str x19, [x3, #(14 * 8)] - str x18, [x3, #(13 * 8)] - str x17, [x3, #(12 * 8)] - str x16, [x3, #(11 * 8)] - str x15, [x3, #(10 * 8)] - str x14, [x3, #(9 * 8)] - str x13, [x3, #(8 * 8)] - str x12, [x3, #(7 * 8)] - str x11, [x3, #(6 * 8)] - str x10, [x3, #(5 * 8)] - str x9, [x3, #(4 * 8)] - str x8, [x3, #(3 * 8)] - str x7, [x3, #(2 * 8)] - str x6, [x3, #(1 * 8)] - str x5, [x3, #(0 * 8)] - - add x3, x2, #CPU_SYSREG_OFFSET(DBGWVR0_EL1) - - adr x26, 1f - add x26, x26, x25, lsl #2 - br x26 -1: - mrs x20, dbgwvr15_el1 - mrs x19, dbgwvr14_el1 - mrs x18, dbgwvr13_el1 - mrs x17, dbgwvr12_el1 - mrs x16, dbgwvr11_el1 - mrs x15, dbgwvr10_el1 - mrs x14, dbgwvr9_el1 - mrs x13, dbgwvr8_el1 - mrs x12, dbgwvr7_el1 - mrs x11, dbgwvr6_el1 - mrs x10, dbgwvr5_el1 - mrs x9, dbgwvr4_el1 - mrs x8, dbgwvr3_el1 - mrs x7, dbgwvr2_el1 - mrs x6, dbgwvr1_el1 - mrs x5, dbgwvr0_el1 - - adr x26, 1f - add x26, x26, x25, lsl #2 - br x26 - -1: - str x20, [x3, #(15 * 8)] - str x19, [x3, #(14 * 8)] - str x18, [x3, #(13 * 8)] - str x17, [x3, #(12 * 8)] - str x16, [x3, #(11 * 8)] - str x15, [x3, #(10 * 8)] - str x14, [x3, #(9 * 8)] - str x13, [x3, #(8 * 8)] - str x12, [x3, #(7 * 8)] - str x11, [x3, #(6 * 8)] - str x10, [x3, #(5 * 8)] - str x9, [x3, #(4 * 8)] - str x8, [x3, #(3 * 8)] - str x7, [x3, #(2 * 8)] - str x6, [x3, #(1 * 8)] - str x5, [x3, #(0 * 8)] - - mrs x21, mdccint_el1 - str x21, [x2, #CPU_SYSREG_OFFSET(MDCCINT_EL1)] + str x21, [x4, #(15 * 8)] + str x20, [x4, #(14 * 8)] + str x19, [x4, #(13 * 8)] + str x18, [x4, #(12 * 8)] + str x17, [x4, #(11 * 8)] + str x16, [x4, #(10 * 8)] + str x15, [x4, #(9 * 8)] + str x14, [x4, #(8 * 8)] + str x13, [x4, #(7 * 8)] + str x12, [x4, #(6 * 8)] + str x11, [x4, #(5 * 8)] + str x10, [x4, #(4 * 8)] + str x9, [x4, #(3 * 8)] + str x8, [x4, #(2 * 8)] + str x7, [x4, #(1 * 8)] + str x6, [x4, #(0 * 8)] .endm .macro restore_sysregs @@ -467,195 +320,52 @@ msr mdscr_el1, x25 .endm -.macro restore_debug - // x2: base address for cpu context - // x3: tmp register +.macro restore_debug type + // x4: pointer to register set + // x5: number of registers to skip + // x6..x22 trashed - mrs x26, id_aa64dfr0_el1 - ubfx x24, x26, #12, #4 // Extract BRPs - ubfx x25, x26, #20, #4 // Extract WRPs - mov w26, #15 - sub w24, w26, w24 // How many BPs to skip - sub w25, w26, w25 // How many WPs to skip - - add x3, x2, #CPU_SYSREG_OFFSET(DBGBCR0_EL1) - - adr x26, 1f - add x26, x26, x24, lsl #2 - br x26 + adr x22, 1f + add x22, x22, x5, lsl #2 + br x22 1: - ldr x20, [x3, #(15 * 8)] - ldr x19, [x3, #(14 * 8)] - ldr x18, [x3, #(13 * 8)] - ldr x17, [x3, #(12 * 8)] - ldr x16, [x3, #(11 * 8)] - ldr x15, [x3, #(10 * 8)] - ldr x14, [x3, #(9 * 8)] - ldr x13, [x3, #(8 * 8)] - ldr x12, [x3, #(7 * 8)] - ldr x11, [x3, #(6 * 8)] - ldr x10, [x3, #(5 * 8)] - ldr x9, [x3, #(4 * 8)] - ldr x8, [x3, #(3 * 8)] - ldr x7, [x3, #(2 * 8)] - ldr x6, [x3, #(1 * 8)] - ldr x5, [x3, #(0 * 8)] + ldr x21, [x4, #(15 * 8)] + ldr x20, [x4, #(14 * 8)] + ldr x19, [x4, #(13 * 8)] + ldr x18, [x4, #(12 * 8)] + ldr x17, [x4, #(11 * 8)] + ldr x16, [x4, #(10 * 8)] + ldr x15, [x4, #(9 * 8)] + ldr x14, [x4, #(8 * 8)] + ldr x13, [x4, #(7 * 8)] + ldr x12, [x4, #(6 * 8)] + ldr x11, [x4, #(5 * 8)] + ldr x10, [x4, #(4 * 8)] + ldr x9, [x4, #(3 * 8)] + ldr x8, [x4, #(2 * 8)] + ldr x7, [x4, #(1 * 8)] + ldr x6, [x4, #(0 * 8)] - adr x26, 1f - add x26, x26, x24, lsl #2 - br x26 + adr x22, 1f + add x22, x22, x5, lsl #2 + br x22 1: - msr dbgbcr15_el1, x20 - msr dbgbcr14_el1, x19 - msr dbgbcr13_el1, x18 - msr dbgbcr12_el1, x17 - msr dbgbcr11_el1, x16 - msr dbgbcr10_el1, x15 - msr dbgbcr9_el1, x14 - msr dbgbcr8_el1, x13 - msr dbgbcr7_el1, x12 - msr dbgbcr6_el1, x11 - msr dbgbcr5_el1, x10 - msr dbgbcr4_el1, x9 - msr dbgbcr3_el1, x8 - msr dbgbcr2_el1, x7 - msr dbgbcr1_el1, x6 - msr dbgbcr0_el1, x5 - - add x3, x2, #CPU_SYSREG_OFFSET(DBGBVR0_EL1) - - adr x26, 1f - add x26, x26, x24, lsl #2 - br x26 -1: - ldr x20, [x3, #(15 * 8)] - ldr x19, [x3, #(14 * 8)] - ldr x18, [x3, #(13 * 8)] - ldr x17, [x3, #(12 * 8)] - ldr x16, [x3, #(11 * 8)] - ldr x15, [x3, #(10 * 8)] - ldr x14, [x3, #(9 * 8)] - ldr x13, [x3, #(8 * 8)] - ldr x12, [x3, #(7 * 8)] - ldr x11, [x3, #(6 * 8)] - ldr x10, [x3, #(5 * 8)] - ldr x9, [x3, #(4 * 8)] - ldr x8, [x3, #(3 * 8)] - ldr x7, [x3, #(2 * 8)] - ldr x6, [x3, #(1 * 8)] - ldr x5, [x3, #(0 * 8)] - - adr x26, 1f - add x26, x26, x24, lsl #2 - br x26 -1: - msr dbgbvr15_el1, x20 - msr dbgbvr14_el1, x19 - msr dbgbvr13_el1, x18 - msr dbgbvr12_el1, x17 - msr dbgbvr11_el1, x16 - msr dbgbvr10_el1, x15 - msr dbgbvr9_el1, x14 - msr dbgbvr8_el1, x13 - msr dbgbvr7_el1, x12 - msr dbgbvr6_el1, x11 - msr dbgbvr5_el1, x10 - msr dbgbvr4_el1, x9 - msr dbgbvr3_el1, x8 - msr dbgbvr2_el1, x7 - msr dbgbvr1_el1, x6 - msr dbgbvr0_el1, x5 - - add x3, x2, #CPU_SYSREG_OFFSET(DBGWCR0_EL1) - - adr x26, 1f - add x26, x26, x25, lsl #2 - br x26 -1: - ldr x20, [x3, #(15 * 8)] - ldr x19, [x3, #(14 * 8)] - ldr x18, [x3, #(13 * 8)] - ldr x17, [x3, #(12 * 8)] - ldr x16, [x3, #(11 * 8)] - ldr x15, [x3, #(10 * 8)] - ldr x14, [x3, #(9 * 8)] - ldr x13, [x3, #(8 * 8)] - ldr x12, [x3, #(7 * 8)] - ldr x11, [x3, #(6 * 8)] - ldr x10, [x3, #(5 * 8)] - ldr x9, [x3, #(4 * 8)] - ldr x8, [x3, #(3 * 8)] - ldr x7, [x3, #(2 * 8)] - ldr x6, [x3, #(1 * 8)] - ldr x5, [x3, #(0 * 8)] - - adr x26, 1f - add x26, x26, x25, lsl #2 - br x26 -1: - msr dbgwcr15_el1, x20 - msr dbgwcr14_el1, x19 - msr dbgwcr13_el1, x18 - msr dbgwcr12_el1, x17 - msr dbgwcr11_el1, x16 - msr dbgwcr10_el1, x15 - msr dbgwcr9_el1, x14 - msr dbgwcr8_el1, x13 - msr dbgwcr7_el1, x12 - msr dbgwcr6_el1, x11 - msr dbgwcr5_el1, x10 - msr dbgwcr4_el1, x9 - msr dbgwcr3_el1, x8 - msr dbgwcr2_el1, x7 - msr dbgwcr1_el1, x6 - msr dbgwcr0_el1, x5 - - add x3, x2, #CPU_SYSREG_OFFSET(DBGWVR0_EL1) - - adr x26, 1f - add x26, x26, x25, lsl #2 - br x26 -1: - ldr x20, [x3, #(15 * 8)] - ldr x19, [x3, #(14 * 8)] - ldr x18, [x3, #(13 * 8)] - ldr x17, [x3, #(12 * 8)] - ldr x16, [x3, #(11 * 8)] - ldr x15, [x3, #(10 * 8)] - ldr x14, [x3, #(9 * 8)] - ldr x13, [x3, #(8 * 8)] - ldr x12, [x3, #(7 * 8)] - ldr x11, [x3, #(6 * 8)] - ldr x10, [x3, #(5 * 8)] - ldr x9, [x3, #(4 * 8)] - ldr x8, [x3, #(3 * 8)] - ldr x7, [x3, #(2 * 8)] - ldr x6, [x3, #(1 * 8)] - ldr x5, [x3, #(0 * 8)] - - adr x26, 1f - add x26, x26, x25, lsl #2 - br x26 -1: - msr dbgwvr15_el1, x20 - msr dbgwvr14_el1, x19 - msr dbgwvr13_el1, x18 - msr dbgwvr12_el1, x17 - msr dbgwvr11_el1, x16 - msr dbgwvr10_el1, x15 - msr dbgwvr9_el1, x14 - msr dbgwvr8_el1, x13 - msr dbgwvr7_el1, x12 - msr dbgwvr6_el1, x11 - msr dbgwvr5_el1, x10 - msr dbgwvr4_el1, x9 - msr dbgwvr3_el1, x8 - msr dbgwvr2_el1, x7 - msr dbgwvr1_el1, x6 - msr dbgwvr0_el1, x5 - - ldr x21, [x2, #CPU_SYSREG_OFFSET(MDCCINT_EL1)] - msr mdccint_el1, x21 + msr \type\()15_el1, x21 + msr \type\()14_el1, x20 + msr \type\()13_el1, x19 + msr \type\()12_el1, x18 + msr \type\()11_el1, x17 + msr \type\()10_el1, x16 + msr \type\()9_el1, x15 + msr \type\()8_el1, x14 + msr \type\()7_el1, x13 + msr \type\()6_el1, x12 + msr \type\()5_el1, x11 + msr \type\()4_el1, x10 + msr \type\()3_el1, x9 + msr \type\()2_el1, x8 + msr \type\()1_el1, x7 + msr \type\()0_el1, x6 .endm .macro skip_32bit_state tmp, target @@ -675,6 +385,14 @@ tbz \tmp, #KVM_ARM64_DEBUG_DIRTY_SHIFT, \target .endm +/* + * Branch to target if CPTR_EL2.TFP bit is set (VFP/SIMD trapping enabled) + */ +.macro skip_fpsimd_state tmp, target + mrs \tmp, cptr_el2 + tbnz \tmp, #CPTR_EL2_TFP_SHIFT, \target +.endm + .macro compute_debug_state target // Compute debug state: If any of KDE, MDE or KVM_ARM64_DEBUG_DIRTY // is set, we do a full save/restore cycle and disable trapping. @@ -713,10 +431,12 @@ add x3, x2, #CPU_SYSREG_OFFSET(DACR32_EL2) mrs x4, dacr32_el2 mrs x5, ifsr32_el2 - mrs x6, fpexc32_el2 stp x4, x5, [x3] - str x6, [x3, #16] + skip_fpsimd_state x8, 3f + mrs x6, fpexc32_el2 + str x6, [x3, #16] +3: skip_debug_state x8, 2f mrs x7, dbgvcr32_el2 str x7, [x3, #24] @@ -743,10 +463,8 @@ add x3, x2, #CPU_SYSREG_OFFSET(DACR32_EL2) ldp x4, x5, [x3] - ldr x6, [x3, #16] msr dacr32_el2, x4 msr ifsr32_el2, x5 - msr fpexc32_el2, x6 skip_debug_state x8, 2f ldr x7, [x3, #24] @@ -763,31 +481,35 @@ .macro activate_traps ldr x2, [x0, #VCPU_HCR_EL2] + + /* + * We are about to set CPTR_EL2.TFP to trap all floating point + * register accesses to EL2, however, the ARM ARM clearly states that + * traps are only taken to EL2 if the operation would not otherwise + * trap to EL1. Therefore, always make sure that for 32-bit guests, + * we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit. + */ + tbnz x2, #HCR_RW_SHIFT, 99f // open code skip_32bit_state + mov x3, #(1 << 30) + msr fpexc32_el2, x3 + isb +99: msr hcr_el2, x2 mov x2, #CPTR_EL2_TTA + orr x2, x2, #CPTR_EL2_TFP msr cptr_el2, x2 mov x2, #(1 << 15) // Trap CP15 Cr=15 msr hstr_el2, x2 - mrs x2, mdcr_el2 - and x2, x2, #MDCR_EL2_HPMN_MASK - orr x2, x2, #(MDCR_EL2_TPM | MDCR_EL2_TPMCR) - orr x2, x2, #(MDCR_EL2_TDRA | MDCR_EL2_TDOSA) - - // Check for KVM_ARM64_DEBUG_DIRTY, and set debug to trap - // if not dirty. - ldr x3, [x0, #VCPU_DEBUG_FLAGS] - tbnz x3, #KVM_ARM64_DEBUG_DIRTY_SHIFT, 1f - orr x2, x2, #MDCR_EL2_TDA -1: + // Monitor Debug Config - see kvm_arm_setup_debug() + ldr x2, [x0, #VCPU_MDCR_EL2] msr mdcr_el2, x2 .endm .macro deactivate_traps mov x2, #HCR_RW msr hcr_el2, x2 - msr cptr_el2, xzr msr hstr_el2, xzr mrs x2, mdcr_el2 @@ -810,7 +532,11 @@ * Call into the vgic backend for state saving */ .macro save_vgic_state - alternative_insn "bl __save_vgic_v2_state", "bl __save_vgic_v3_state", ARM64_HAS_SYSREG_GIC_CPUIF +alternative_if_not ARM64_HAS_SYSREG_GIC_CPUIF + bl __save_vgic_v2_state +alternative_else + bl __save_vgic_v3_state +alternative_endif mrs x24, hcr_el2 mov x25, #HCR_INT_OVERRIDE neg x25, x25 @@ -827,7 +553,11 @@ orr x24, x24, #HCR_INT_OVERRIDE orr x24, x24, x25 msr hcr_el2, x24 - alternative_insn "bl __restore_vgic_v2_state", "bl __restore_vgic_v3_state", ARM64_HAS_SYSREG_GIC_CPUIF +alternative_if_not ARM64_HAS_SYSREG_GIC_CPUIF + bl __restore_vgic_v2_state +alternative_else + bl __restore_vgic_v3_state +alternative_endif .endm .macro save_timer_state @@ -892,21 +622,101 @@ __restore_sysregs: restore_sysregs ret +/* Save debug state */ __save_debug: - save_debug + // x2: ptr to CPU context + // x3: ptr to debug reg struct + // x4/x5/x6-22/x24-26: trashed + + mrs x26, id_aa64dfr0_el1 + ubfx x24, x26, #12, #4 // Extract BRPs + ubfx x25, x26, #20, #4 // Extract WRPs + mov w26, #15 + sub w24, w26, w24 // How many BPs to skip + sub w25, w26, w25 // How many WPs to skip + + mov x5, x24 + add x4, x3, #DEBUG_BCR + save_debug dbgbcr + add x4, x3, #DEBUG_BVR + save_debug dbgbvr + + mov x5, x25 + add x4, x3, #DEBUG_WCR + save_debug dbgwcr + add x4, x3, #DEBUG_WVR + save_debug dbgwvr + + mrs x21, mdccint_el1 + str x21, [x2, #CPU_SYSREG_OFFSET(MDCCINT_EL1)] ret +/* Restore debug state */ __restore_debug: - restore_debug + // x2: ptr to CPU context + // x3: ptr to debug reg struct + // x4/x5/x6-22/x24-26: trashed + + mrs x26, id_aa64dfr0_el1 + ubfx x24, x26, #12, #4 // Extract BRPs + ubfx x25, x26, #20, #4 // Extract WRPs + mov w26, #15 + sub w24, w26, w24 // How many BPs to skip + sub w25, w26, w25 // How many WPs to skip + + mov x5, x24 + add x4, x3, #DEBUG_BCR + restore_debug dbgbcr + add x4, x3, #DEBUG_BVR + restore_debug dbgbvr + + mov x5, x25 + add x4, x3, #DEBUG_WCR + restore_debug dbgwcr + add x4, x3, #DEBUG_WVR + restore_debug dbgwvr + + ldr x21, [x2, #CPU_SYSREG_OFFSET(MDCCINT_EL1)] + msr mdccint_el1, x21 + ret __save_fpsimd: + skip_fpsimd_state x3, 1f save_fpsimd - ret +1: ret __restore_fpsimd: + skip_fpsimd_state x3, 1f restore_fpsimd - ret +1: ret + +switch_to_guest_fpsimd: + push x4, lr + + mrs x2, cptr_el2 + bic x2, x2, #CPTR_EL2_TFP + msr cptr_el2, x2 + isb + + mrs x0, tpidr_el2 + + ldr x2, [x0, #VCPU_HOST_CONTEXT] + kern_hyp_va x2 + bl __save_fpsimd + + add x2, x0, #VCPU_CONTEXT + bl __restore_fpsimd + + skip_32bit_state x3, 1f + ldr x4, [x2, #CPU_SYSREG_OFFSET(FPEXC32_EL2)] + msr fpexc32_el2, x4 +1: + pop x4, lr + pop x2, x3 + pop x0, x1 + + eret /* * u64 __kvm_vcpu_run(struct kvm_vcpu *vcpu); @@ -928,10 +738,10 @@ ENTRY(__kvm_vcpu_run) kern_hyp_va x2 save_host_regs - bl __save_fpsimd bl __save_sysregs compute_debug_state 1f + add x3, x0, #VCPU_HOST_DEBUG_STATE bl __save_debug 1: activate_traps @@ -944,9 +754,10 @@ ENTRY(__kvm_vcpu_run) add x2, x0, #VCPU_CONTEXT bl __restore_sysregs - bl __restore_fpsimd skip_debug_state x3, 1f + ldr x3, [x0, #VCPU_DEBUG_PTR] + kern_hyp_va x3 bl __restore_debug 1: restore_guest_32bit_state @@ -967,6 +778,8 @@ __kvm_vcpu_return: bl __save_sysregs skip_debug_state x3, 1f + ldr x3, [x0, #VCPU_DEBUG_PTR] + kern_hyp_va x3 bl __save_debug 1: save_guest_32bit_state @@ -983,12 +796,15 @@ __kvm_vcpu_return: bl __restore_sysregs bl __restore_fpsimd + /* Clear FPSIMD and Trace trapping */ + msr cptr_el2, xzr skip_debug_state x3, 1f // Clear the dirty flag for the next run, as all the state has // already been saved. Note that we nuke the whole 64bit word. // If we ever add more flags, we'll have to be more careful... str xzr, [x0, #VCPU_DEBUG_FLAGS] + add x3, x0, #VCPU_HOST_DEBUG_STATE bl __restore_debug 1: restore_host_regs @@ -1191,6 +1007,11 @@ el1_trap: * x1: ESR * x2: ESR_EC */ + + /* Guest accessed VFP/SIMD registers, save host, restore Guest */ + cmp x2, #ESR_ELx_EC_FP_ASIMD + b.eq switch_to_guest_fpsimd + cmp x2, #ESR_ELx_EC_DABT_LOW mov x0, #ESR_ELx_EC_IABT_LOW ccmp x2, x0, #4, ne @@ -1285,4 +1106,10 @@ ENTRY(__kvm_hyp_vector) ventry el1_error_invalid // Error 32-bit EL1 ENDPROC(__kvm_hyp_vector) + +ENTRY(__kvm_get_mdcr_el2) + mrs x0, mdcr_el2 + ret +ENDPROC(__kvm_get_mdcr_el2) + .popsection diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c index 0b4326578985..91cf5350b328 100644 --- a/arch/arm64/kvm/reset.c +++ b/arch/arm64/kvm/reset.c @@ -22,6 +22,7 @@ #include #include #include +#include #include @@ -56,6 +57,12 @@ static bool cpu_has_32bit_el1(void) return !!(pfr0 & 0x20); } +/** + * kvm_arch_dev_ioctl_check_extension + * + * We currently assume that the number of HW registers is uniform + * across all CPUs (see cpuinfo_sanity_check). + */ int kvm_arch_dev_ioctl_check_extension(long ext) { int r; @@ -64,6 +71,15 @@ int kvm_arch_dev_ioctl_check_extension(long ext) case KVM_CAP_ARM_EL1_32BIT: r = cpu_has_32bit_el1(); break; + case KVM_CAP_GUEST_DEBUG_HW_BPS: + r = get_num_brps(); + break; + case KVM_CAP_GUEST_DEBUG_HW_WPS: + r = get_num_wrps(); + break; + case KVM_CAP_SET_GUEST_DEBUG: + r = 1; + break; default: r = 0; } @@ -105,7 +121,5 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu) kvm_reset_sys_regs(vcpu); /* Reset timer */ - kvm_timer_vcpu_reset(vcpu, cpu_vtimer_irq); - - return 0; + return kvm_timer_vcpu_reset(vcpu, cpu_vtimer_irq); } diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index c370b4014799..b41607d270ac 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -38,6 +38,8 @@ #include "sys_regs.h" +#include "trace.h" + /* * All of this file is extremly similar to the ARM coproc.c, but the * types are different. My gut feeling is that it should be pretty @@ -208,9 +210,217 @@ static bool trap_debug_regs(struct kvm_vcpu *vcpu, *vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg); } + trace_trap_reg(__func__, r->reg, p->is_write, *vcpu_reg(vcpu, p->Rt)); + return true; } +/* + * reg_to_dbg/dbg_to_reg + * + * A 32 bit write to a debug register leave top bits alone + * A 32 bit read from a debug register only returns the bottom bits + * + * All writes will set the KVM_ARM64_DEBUG_DIRTY flag to ensure the + * hyp.S code switches between host and guest values in future. + */ +static inline void reg_to_dbg(struct kvm_vcpu *vcpu, + const struct sys_reg_params *p, + u64 *dbg_reg) +{ + u64 val = *vcpu_reg(vcpu, p->Rt); + + if (p->is_32bit) { + val &= 0xffffffffUL; + val |= ((*dbg_reg >> 32) << 32); + } + + *dbg_reg = val; + vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY; +} + +static inline void dbg_to_reg(struct kvm_vcpu *vcpu, + const struct sys_reg_params *p, + u64 *dbg_reg) +{ + u64 val = *dbg_reg; + + if (p->is_32bit) + val &= 0xffffffffUL; + + *vcpu_reg(vcpu, p->Rt) = val; +} + +static inline bool trap_bvr(struct kvm_vcpu *vcpu, + const struct sys_reg_params *p, + const struct sys_reg_desc *rd) +{ + u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg]; + + if (p->is_write) + reg_to_dbg(vcpu, p, dbg_reg); + else + dbg_to_reg(vcpu, p, dbg_reg); + + trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg); + + return true; +} + +static int set_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, + const struct kvm_one_reg *reg, void __user *uaddr) +{ + __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg]; + + if (copy_from_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0) + return -EFAULT; + return 0; +} + +static int get_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, + const struct kvm_one_reg *reg, void __user *uaddr) +{ + __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg]; + + if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0) + return -EFAULT; + return 0; +} + +static inline void reset_bvr(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *rd) +{ + vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg] = rd->val; +} + +static inline bool trap_bcr(struct kvm_vcpu *vcpu, + const struct sys_reg_params *p, + const struct sys_reg_desc *rd) +{ + u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg]; + + if (p->is_write) + reg_to_dbg(vcpu, p, dbg_reg); + else + dbg_to_reg(vcpu, p, dbg_reg); + + trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg); + + return true; +} + +static int set_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, + const struct kvm_one_reg *reg, void __user *uaddr) +{ + __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg]; + + if (copy_from_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0) + return -EFAULT; + + return 0; +} + +static int get_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, + const struct kvm_one_reg *reg, void __user *uaddr) +{ + __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg]; + + if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0) + return -EFAULT; + return 0; +} + +static inline void reset_bcr(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *rd) +{ + vcpu->arch.vcpu_debug_state.dbg_bcr[rd->reg] = rd->val; +} + +static inline bool trap_wvr(struct kvm_vcpu *vcpu, + const struct sys_reg_params *p, + const struct sys_reg_desc *rd) +{ + u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg]; + + if (p->is_write) + reg_to_dbg(vcpu, p, dbg_reg); + else + dbg_to_reg(vcpu, p, dbg_reg); + + trace_trap_reg(__func__, rd->reg, p->is_write, + vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg]); + + return true; +} + +static int set_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, + const struct kvm_one_reg *reg, void __user *uaddr) +{ + __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg]; + + if (copy_from_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0) + return -EFAULT; + return 0; +} + +static int get_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, + const struct kvm_one_reg *reg, void __user *uaddr) +{ + __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg]; + + if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0) + return -EFAULT; + return 0; +} + +static inline void reset_wvr(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *rd) +{ + vcpu->arch.vcpu_debug_state.dbg_wvr[rd->reg] = rd->val; +} + +static inline bool trap_wcr(struct kvm_vcpu *vcpu, + const struct sys_reg_params *p, + const struct sys_reg_desc *rd) +{ + u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg]; + + if (p->is_write) + reg_to_dbg(vcpu, p, dbg_reg); + else + dbg_to_reg(vcpu, p, dbg_reg); + + trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg); + + return true; +} + +static int set_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, + const struct kvm_one_reg *reg, void __user *uaddr) +{ + __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg]; + + if (copy_from_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0) + return -EFAULT; + return 0; +} + +static int get_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, + const struct kvm_one_reg *reg, void __user *uaddr) +{ + __u64 *r = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg]; + + if (copy_to_user(uaddr, r, KVM_REG_SIZE(reg->id)) != 0) + return -EFAULT; + return 0; +} + +static inline void reset_wcr(struct kvm_vcpu *vcpu, + const struct sys_reg_desc *rd) +{ + vcpu->arch.vcpu_debug_state.dbg_wcr[rd->reg] = rd->val; +} + static void reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { u64 amair; @@ -240,16 +450,16 @@ static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) #define DBG_BCR_BVR_WCR_WVR_EL1(n) \ /* DBGBVRn_EL1 */ \ { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b100), \ - trap_debug_regs, reset_val, (DBGBVR0_EL1 + (n)), 0 }, \ + trap_bvr, reset_bvr, n, 0, get_bvr, set_bvr }, \ /* DBGBCRn_EL1 */ \ { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b101), \ - trap_debug_regs, reset_val, (DBGBCR0_EL1 + (n)), 0 }, \ + trap_bcr, reset_bcr, n, 0, get_bcr, set_bcr }, \ /* DBGWVRn_EL1 */ \ { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b110), \ - trap_debug_regs, reset_val, (DBGWVR0_EL1 + (n)), 0 }, \ + trap_wvr, reset_wvr, n, 0, get_wvr, set_wvr }, \ /* DBGWCRn_EL1 */ \ { Op0(0b10), Op1(0b000), CRn(0b0000), CRm((n)), Op2(0b111), \ - trap_debug_regs, reset_val, (DBGWCR0_EL1 + (n)), 0 } + trap_wcr, reset_wcr, n, 0, get_wcr, set_wcr } /* * Architected system registers. @@ -516,28 +726,57 @@ static bool trap_debug32(struct kvm_vcpu *vcpu, return true; } -#define DBG_BCR_BVR_WCR_WVR(n) \ - /* DBGBVRn */ \ - { Op1( 0), CRn( 0), CRm((n)), Op2( 4), trap_debug32, \ - NULL, (cp14_DBGBVR0 + (n) * 2) }, \ - /* DBGBCRn */ \ - { Op1( 0), CRn( 0), CRm((n)), Op2( 5), trap_debug32, \ - NULL, (cp14_DBGBCR0 + (n) * 2) }, \ - /* DBGWVRn */ \ - { Op1( 0), CRn( 0), CRm((n)), Op2( 6), trap_debug32, \ - NULL, (cp14_DBGWVR0 + (n) * 2) }, \ - /* DBGWCRn */ \ - { Op1( 0), CRn( 0), CRm((n)), Op2( 7), trap_debug32, \ - NULL, (cp14_DBGWCR0 + (n) * 2) } +/* AArch32 debug register mappings + * + * AArch32 DBGBVRn is mapped to DBGBVRn_EL1[31:0] + * AArch32 DBGBXVRn is mapped to DBGBVRn_EL1[63:32] + * + * All control registers and watchpoint value registers are mapped to + * the lower 32 bits of their AArch64 equivalents. We share the trap + * handlers with the above AArch64 code which checks what mode the + * system is in. + */ -#define DBGBXVR(n) \ - { Op1( 0), CRn( 1), CRm((n)), Op2( 1), trap_debug32, \ - NULL, cp14_DBGBXVR0 + n * 2 } +static inline bool trap_xvr(struct kvm_vcpu *vcpu, + const struct sys_reg_params *p, + const struct sys_reg_desc *rd) +{ + u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->reg]; + + if (p->is_write) { + u64 val = *dbg_reg; + + val &= 0xffffffffUL; + val |= *vcpu_reg(vcpu, p->Rt) << 32; + *dbg_reg = val; + + vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY; + } else { + *vcpu_reg(vcpu, p->Rt) = *dbg_reg >> 32; + } + + trace_trap_reg(__func__, rd->reg, p->is_write, *dbg_reg); + + return true; +} + +#define DBG_BCR_BVR_WCR_WVR(n) \ + /* DBGBVRn */ \ + { Op1( 0), CRn( 0), CRm((n)), Op2( 4), trap_bvr, NULL, n }, \ + /* DBGBCRn */ \ + { Op1( 0), CRn( 0), CRm((n)), Op2( 5), trap_bcr, NULL, n }, \ + /* DBGWVRn */ \ + { Op1( 0), CRn( 0), CRm((n)), Op2( 6), trap_wvr, NULL, n }, \ + /* DBGWCRn */ \ + { Op1( 0), CRn( 0), CRm((n)), Op2( 7), trap_wcr, NULL, n } + +#define DBGBXVR(n) \ + { Op1( 0), CRn( 1), CRm((n)), Op2( 1), trap_xvr, NULL, n } /* * Trapped cp14 registers. We generally ignore most of the external * debug, on the principle that they don't really make sense to a - * guest. Revisit this one day, whould this principle change. + * guest. Revisit this one day, would this principle change. */ static const struct sys_reg_desc cp14_regs[] = { /* DBGIDR */ @@ -999,6 +1238,8 @@ int kvm_handle_sys_reg(struct kvm_vcpu *vcpu, struct kvm_run *run) struct sys_reg_params params; unsigned long esr = kvm_vcpu_get_hsr(vcpu); + trace_kvm_handle_sys_reg(esr); + params.is_aarch32 = false; params.is_32bit = false; params.Op0 = (esr >> 20) & 3; @@ -1303,6 +1544,9 @@ int kvm_arm_sys_reg_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg if (!r) return get_invariant_sys_reg(reg->id, uaddr); + if (r->get_user) + return (r->get_user)(vcpu, r, reg, uaddr); + return reg_to_user(uaddr, &vcpu_sys_reg(vcpu, r->reg), reg->id); } @@ -1321,6 +1565,9 @@ int kvm_arm_sys_reg_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg if (!r) return set_invariant_sys_reg(reg->id, uaddr); + if (r->set_user) + return (r->set_user)(vcpu, r, reg, uaddr); + return reg_from_user(&vcpu_sys_reg(vcpu, r->reg), uaddr, reg->id); } diff --git a/arch/arm64/kvm/sys_regs.h b/arch/arm64/kvm/sys_regs.h index d411e251412c..eaa324e4db4d 100644 --- a/arch/arm64/kvm/sys_regs.h +++ b/arch/arm64/kvm/sys_regs.h @@ -55,6 +55,12 @@ struct sys_reg_desc { /* Value (usually reset value) */ u64 val; + + /* Custom get/set_user functions, fallback to generic if NULL */ + int (*get_user)(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, + const struct kvm_one_reg *reg, void __user *uaddr); + int (*set_user)(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd, + const struct kvm_one_reg *reg, void __user *uaddr); }; static inline void print_sys_reg_instr(const struct sys_reg_params *p) diff --git a/arch/arm64/kvm/sys_regs_generic_v8.c b/arch/arm64/kvm/sys_regs_generic_v8.c index 475fd2929310..1e4576824165 100644 --- a/arch/arm64/kvm/sys_regs_generic_v8.c +++ b/arch/arm64/kvm/sys_regs_generic_v8.c @@ -94,6 +94,8 @@ static int __init sys_reg_genericv8_init(void) &genericv8_target_table); kvm_register_target_sys_reg_table(KVM_ARM_TARGET_XGENE_POTENZA, &genericv8_target_table); + kvm_register_target_sys_reg_table(KVM_ARM_TARGET_GENERIC_V8, + &genericv8_target_table); return 0; } diff --git a/arch/arm64/kvm/trace.h b/arch/arm64/kvm/trace.h index 157416e963f2..7fb0008c4fa3 100644 --- a/arch/arm64/kvm/trace.h +++ b/arch/arm64/kvm/trace.h @@ -44,6 +44,129 @@ TRACE_EVENT(kvm_hvc_arm64, __entry->vcpu_pc, __entry->r0, __entry->imm) ); +TRACE_EVENT(kvm_arm_setup_debug, + TP_PROTO(struct kvm_vcpu *vcpu, __u32 guest_debug), + TP_ARGS(vcpu, guest_debug), + + TP_STRUCT__entry( + __field(struct kvm_vcpu *, vcpu) + __field(__u32, guest_debug) + ), + + TP_fast_assign( + __entry->vcpu = vcpu; + __entry->guest_debug = guest_debug; + ), + + TP_printk("vcpu: %p, flags: 0x%08x", __entry->vcpu, __entry->guest_debug) +); + +TRACE_EVENT(kvm_arm_clear_debug, + TP_PROTO(__u32 guest_debug), + TP_ARGS(guest_debug), + + TP_STRUCT__entry( + __field(__u32, guest_debug) + ), + + TP_fast_assign( + __entry->guest_debug = guest_debug; + ), + + TP_printk("flags: 0x%08x", __entry->guest_debug) +); + +TRACE_EVENT(kvm_arm_set_dreg32, + TP_PROTO(const char *name, __u32 value), + TP_ARGS(name, value), + + TP_STRUCT__entry( + __field(const char *, name) + __field(__u32, value) + ), + + TP_fast_assign( + __entry->name = name; + __entry->value = value; + ), + + TP_printk("%s: 0x%08x", __entry->name, __entry->value) +); + +TRACE_EVENT(kvm_arm_set_regset, + TP_PROTO(const char *type, int len, __u64 *control, __u64 *value), + TP_ARGS(type, len, control, value), + TP_STRUCT__entry( + __field(const char *, name) + __field(int, len) + __array(u64, ctrls, 16) + __array(u64, values, 16) + ), + TP_fast_assign( + __entry->name = type; + __entry->len = len; + memcpy(__entry->ctrls, control, len << 3); + memcpy(__entry->values, value, len << 3); + ), + TP_printk("%d %s CTRL:%s VALUE:%s", __entry->len, __entry->name, + __print_array(__entry->ctrls, __entry->len, sizeof(__u64)), + __print_array(__entry->values, __entry->len, sizeof(__u64))) +); + +TRACE_EVENT(trap_reg, + TP_PROTO(const char *fn, int reg, bool is_write, u64 write_value), + TP_ARGS(fn, reg, is_write, write_value), + + TP_STRUCT__entry( + __field(const char *, fn) + __field(int, reg) + __field(bool, is_write) + __field(u64, write_value) + ), + + TP_fast_assign( + __entry->fn = fn; + __entry->reg = reg; + __entry->is_write = is_write; + __entry->write_value = write_value; + ), + + TP_printk("%s %s reg %d (0x%08llx)", __entry->fn, __entry->is_write?"write to":"read from", __entry->reg, __entry->write_value) +); + +TRACE_EVENT(kvm_handle_sys_reg, + TP_PROTO(unsigned long hsr), + TP_ARGS(hsr), + + TP_STRUCT__entry( + __field(unsigned long, hsr) + ), + + TP_fast_assign( + __entry->hsr = hsr; + ), + + TP_printk("HSR 0x%08lx", __entry->hsr) +); + +TRACE_EVENT(kvm_set_guest_debug, + TP_PROTO(struct kvm_vcpu *vcpu, __u32 guest_debug), + TP_ARGS(vcpu, guest_debug), + + TP_STRUCT__entry( + __field(struct kvm_vcpu *, vcpu) + __field(__u32, guest_debug) + ), + + TP_fast_assign( + __entry->vcpu = vcpu; + __entry->guest_debug = guest_debug; + ), + + TP_printk("vcpu: %p, flags: 0x%08x", __entry->vcpu, __entry->guest_debug) +); + + #endif /* _TRACE_ARM64_KVM_H */ #undef TRACE_INCLUDE_PATH diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index d98d3e39879e..1a811ecf71da 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -3,3 +3,16 @@ lib-y := bitops.o clear_user.o delay.o copy_from_user.o \ clear_page.o memchr.o memcpy.o memmove.o memset.o \ memcmp.o strcmp.o strncmp.o strlen.o strnlen.o \ strchr.o strrchr.o + +# Tell the compiler to treat all general purpose registers as +# callee-saved, which allows for efficient runtime patching of the bl +# instruction in the caller with an atomic instruction when supported by +# the CPU. Result and argument registers are handled correctly, based on +# the function prototype. +lib-$(CONFIG_ARM64_LSE_ATOMICS) += atomic_ll_sc.o +CFLAGS_atomic_ll_sc.o := -fcall-used-x0 -ffixed-x1 -ffixed-x2 \ + -ffixed-x3 -ffixed-x4 -ffixed-x5 -ffixed-x6 \ + -ffixed-x7 -fcall-saved-x8 -fcall-saved-x9 \ + -fcall-saved-x10 -fcall-saved-x11 -fcall-saved-x12 \ + -fcall-saved-x13 -fcall-saved-x14 -fcall-saved-x15 \ + -fcall-saved-x16 -fcall-saved-x17 -fcall-saved-x18 diff --git a/arch/arm64/lib/atomic_ll_sc.c b/arch/arm64/lib/atomic_ll_sc.c new file mode 100644 index 000000000000..b0c538b0da28 --- /dev/null +++ b/arch/arm64/lib/atomic_ll_sc.c @@ -0,0 +1,3 @@ +#include +#define __ARM64_IN_ATOMIC_IMPL +#include diff --git a/arch/arm64/lib/bitops.S b/arch/arm64/lib/bitops.S index 7dac371cc9a2..43ac736baa5b 100644 --- a/arch/arm64/lib/bitops.S +++ b/arch/arm64/lib/bitops.S @@ -18,52 +18,59 @@ #include #include +#include /* * x0: bits 5:0 bit offset * bits 31:6 word offset * x1: address */ - .macro bitop, name, instr + .macro bitop, name, llsc, lse ENTRY( \name ) and w3, w0, #63 // Get bit offset eor w0, w0, w3 // Clear low bits mov x2, #1 add x1, x1, x0, lsr #3 // Get word offset +alt_lse " prfm pstl1strm, [x1]", "nop" lsl x3, x2, x3 // Create mask -1: ldxr x2, [x1] - \instr x2, x2, x3 - stxr w0, x2, [x1] - cbnz w0, 1b + +alt_lse "1: ldxr x2, [x1]", "\lse x3, [x1]" +alt_lse " \llsc x2, x2, x3", "nop" +alt_lse " stxr w0, x2, [x1]", "nop" +alt_lse " cbnz w0, 1b", "nop" + ret ENDPROC(\name ) .endm - .macro testop, name, instr + .macro testop, name, llsc, lse ENTRY( \name ) and w3, w0, #63 // Get bit offset eor w0, w0, w3 // Clear low bits mov x2, #1 add x1, x1, x0, lsr #3 // Get word offset +alt_lse " prfm pstl1strm, [x1]", "nop" lsl x4, x2, x3 // Create mask -1: ldxr x2, [x1] - lsr x0, x2, x3 // Save old value of bit - \instr x2, x2, x4 // toggle bit - stlxr w5, x2, [x1] - cbnz w5, 1b - dmb ish + +alt_lse "1: ldxr x2, [x1]", "\lse x4, x2, [x1]" + lsr x0, x2, x3 +alt_lse " \llsc x2, x2, x4", "nop" +alt_lse " stlxr w5, x2, [x1]", "nop" +alt_lse " cbnz w5, 1b", "nop" +alt_lse " dmb ish", "nop" + and x0, x0, #1 -3: ret + ret ENDPROC(\name ) .endm /* * Atomic bit operations. */ - bitop change_bit, eor - bitop clear_bit, bic - bitop set_bit, orr + bitop change_bit, eor, steor + bitop clear_bit, bic, stclr + bitop set_bit, orr, stset - testop test_and_change_bit, eor - testop test_and_clear_bit, bic - testop test_and_set_bit, orr + testop test_and_change_bit, eor, ldeoral + testop test_and_clear_bit, bic, ldclral + testop test_and_set_bit, orr, ldsetal diff --git a/arch/arm64/lib/clear_user.S b/arch/arm64/lib/clear_user.S index c17967fdf5f6..a9723c71c52b 100644 --- a/arch/arm64/lib/clear_user.S +++ b/arch/arm64/lib/clear_user.S @@ -16,7 +16,11 @@ * along with this program. If not, see . */ #include + +#include #include +#include +#include .text @@ -29,6 +33,8 @@ * Alignment fixed up by hardware. */ ENTRY(__clear_user) +ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) mov x2, x1 // save the size for fixup return subs x1, x1, #8 b.mi 2f @@ -48,6 +54,8 @@ USER(9f, strh wzr, [x0], #2 ) b.mi 5f USER(9f, strb wzr, [x0] ) 5: mov x0, #0 +ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) ret ENDPROC(__clear_user) diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S index 5e27add9d362..1be9ef27be97 100644 --- a/arch/arm64/lib/copy_from_user.S +++ b/arch/arm64/lib/copy_from_user.S @@ -15,7 +15,11 @@ */ #include + +#include #include +#include +#include /* * Copy from user space to a kernel buffer (alignment handled by the hardware) @@ -28,14 +32,21 @@ * x0 - bytes not copied */ ENTRY(__copy_from_user) - add x4, x1, x2 // upper user buffer boundary - subs x2, x2, #8 +ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) + add x5, x1, x2 // upper user buffer boundary + subs x2, x2, #16 + b.mi 1f +0: +USER(9f, ldp x3, x4, [x1], #16) + subs x2, x2, #16 + stp x3, x4, [x0], #16 + b.pl 0b +1: adds x2, x2, #8 b.mi 2f -1: USER(9f, ldr x3, [x1], #8 ) - subs x2, x2, #8 + sub x2, x2, #8 str x3, [x0], #8 - b.pl 1b 2: adds x2, x2, #4 b.mi 3f USER(9f, ldr w3, [x1], #4 ) @@ -51,12 +62,14 @@ USER(9f, ldrh w3, [x1], #2 ) USER(9f, ldrb w3, [x1] ) strb w3, [x0] 5: mov x0, #0 +ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) ret ENDPROC(__copy_from_user) .section .fixup,"ax" .align 2 -9: sub x2, x4, x1 +9: sub x2, x5, x1 mov x3, x2 10: strb wzr, [x0], #1 // zero remaining buffer space subs x3, x3, #1 diff --git a/arch/arm64/lib/copy_in_user.S b/arch/arm64/lib/copy_in_user.S index 84b6c9bb9b93..1b94661e22b3 100644 --- a/arch/arm64/lib/copy_in_user.S +++ b/arch/arm64/lib/copy_in_user.S @@ -17,7 +17,11 @@ */ #include + +#include #include +#include +#include /* * Copy from user space to user space (alignment handled by the hardware) @@ -30,14 +34,21 @@ * x0 - bytes not copied */ ENTRY(__copy_in_user) - add x4, x0, x2 // upper user buffer boundary - subs x2, x2, #8 +ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) + add x5, x0, x2 // upper user buffer boundary + subs x2, x2, #16 + b.mi 1f +0: +USER(9f, ldp x3, x4, [x1], #16) + subs x2, x2, #16 +USER(9f, stp x3, x4, [x0], #16) + b.pl 0b +1: adds x2, x2, #8 b.mi 2f -1: USER(9f, ldr x3, [x1], #8 ) - subs x2, x2, #8 + sub x2, x2, #8 USER(9f, str x3, [x0], #8 ) - b.pl 1b 2: adds x2, x2, #4 b.mi 3f USER(9f, ldr w3, [x1], #4 ) @@ -53,11 +64,13 @@ USER(9f, strh w3, [x0], #2 ) USER(9f, ldrb w3, [x1] ) USER(9f, strb w3, [x0] ) 5: mov x0, #0 +ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) ret ENDPROC(__copy_in_user) .section .fixup,"ax" .align 2 -9: sub x0, x4, x0 // bytes not copied +9: sub x0, x5, x0 // bytes not copied ret .previous diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S index a0aeeb9b7a28..a257b47e2dc4 100644 --- a/arch/arm64/lib/copy_to_user.S +++ b/arch/arm64/lib/copy_to_user.S @@ -15,7 +15,11 @@ */ #include + +#include #include +#include +#include /* * Copy to user space from a kernel buffer (alignment handled by the hardware) @@ -28,14 +32,21 @@ * x0 - bytes not copied */ ENTRY(__copy_to_user) - add x4, x0, x2 // upper user buffer boundary - subs x2, x2, #8 +ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) + add x5, x0, x2 // upper user buffer boundary + subs x2, x2, #16 + b.mi 1f +0: + ldp x3, x4, [x1], #16 + subs x2, x2, #16 +USER(9f, stp x3, x4, [x0], #16) + b.pl 0b +1: adds x2, x2, #8 b.mi 2f -1: ldr x3, [x1], #8 - subs x2, x2, #8 + sub x2, x2, #8 USER(9f, str x3, [x0], #8 ) - b.pl 1b 2: adds x2, x2, #4 b.mi 3f ldr w3, [x1], #4 @@ -51,11 +62,13 @@ USER(9f, strh w3, [x0], #2 ) ldrb w3, [x1] USER(9f, strb w3, [x0] ) 5: mov x0, #0 +ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ + CONFIG_ARM64_PAN) ret ENDPROC(__copy_to_user) .section .fixup,"ax" .align 2 -9: sub x0, x4, x0 // bytes not copied +9: sub x0, x5, x0 // bytes not copied ret .previous diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S index bdeb5d38c2dd..eb48d5df4a0f 100644 --- a/arch/arm64/mm/cache.S +++ b/arch/arm64/mm/cache.S @@ -143,7 +143,12 @@ __dma_clean_range: dcache_line_size x2, x3 sub x3, x2, #1 bic x0, x0, x3 -1: alternative_insn "dc cvac, x0", "dc civac, x0", ARM64_WORKAROUND_CLEAN_CACHE +1: +alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE + dc cvac, x0 +alternative_else + dc civac, x0 +alternative_endif add x0, x0, x2 cmp x0, x1 b.lo 1b diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index 76c1e6cd36fc..d70ff14dbdbd 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -53,8 +53,6 @@ static void flush_context(void) __flush_icache_all(); } -#ifdef CONFIG_SMP - static void set_mm_context(struct mm_struct *mm, unsigned int asid) { unsigned long flags; @@ -110,23 +108,12 @@ static void reset_context(void *info) cpu_switch_mm(mm->pgd, mm); } -#else - -static inline void set_mm_context(struct mm_struct *mm, unsigned int asid) -{ - mm->context.id = asid; - cpumask_copy(mm_cpumask(mm), cpumask_of(smp_processor_id())); -} - -#endif - void __new_context(struct mm_struct *mm) { unsigned int asid; unsigned int bits = asid_bits(); raw_spin_lock(&cpu_asid_lock); -#ifdef CONFIG_SMP /* * Check the ASID again, in case the change was broadcast from another * CPU before we acquired the lock. @@ -136,7 +123,6 @@ void __new_context(struct mm_struct *mm) raw_spin_unlock(&cpu_asid_lock); return; } -#endif /* * At this point, it is guaranteed that the current mm (with an old * ASID) isn't active on any other CPU since the ASIDs are changed @@ -155,10 +141,8 @@ void __new_context(struct mm_struct *mm) cpu_last_asid = ASID_FIRST_VERSION; asid = cpu_last_asid + smp_processor_id(); flush_context(); -#ifdef CONFIG_SMP smp_wmb(); smp_call_function(reset_context, NULL, 1); -#endif cpu_last_asid += NR_CPUS - 1; } diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index d16a1cead23f..0bcc4bc94b4a 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -144,6 +144,7 @@ static void *__dma_alloc(struct device *dev, size_t size, struct page *page; void *ptr, *coherent_ptr; bool coherent = is_device_dma_coherent(dev); + pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL, false); size = PAGE_ALIGN(size); @@ -171,9 +172,7 @@ static void *__dma_alloc(struct device *dev, size_t size, /* create a coherent mapping */ page = virt_to_page(ptr); coherent_ptr = dma_common_contiguous_remap(page, size, VM_USERMAP, - __get_dma_pgprot(attrs, - __pgprot(PROT_NORMAL_NC), false), - NULL); + prot, NULL); if (!coherent_ptr) goto no_map; @@ -303,9 +302,10 @@ static void __swiotlb_sync_sg_for_device(struct device *dev, sg->length, dir); } -/* vma->vm_page_prot must be set appropriately before calling this function */ -static int __dma_common_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size) +static int __swiotlb_mmap(struct device *dev, + struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + struct dma_attrs *attrs) { int ret = -ENXIO; unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> @@ -314,6 +314,9 @@ static int __dma_common_mmap(struct device *dev, struct vm_area_struct *vma, unsigned long pfn = dma_to_phys(dev, dma_addr) >> PAGE_SHIFT; unsigned long off = vma->vm_pgoff; + vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, + is_device_dma_coherent(dev)); + if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret)) return ret; @@ -327,20 +330,24 @@ static int __dma_common_mmap(struct device *dev, struct vm_area_struct *vma, return ret; } -static int __swiotlb_mmap(struct device *dev, - struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - struct dma_attrs *attrs) +static int __swiotlb_get_sgtable(struct device *dev, struct sg_table *sgt, + void *cpu_addr, dma_addr_t handle, size_t size, + struct dma_attrs *attrs) { - vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, - is_device_dma_coherent(dev)); - return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size); + int ret = sg_alloc_table(sgt, 1, GFP_KERNEL); + + if (!ret) + sg_set_page(sgt->sgl, phys_to_page(dma_to_phys(dev, handle)), + PAGE_ALIGN(size), 0); + + return ret; } static struct dma_map_ops swiotlb_dma_ops = { .alloc = __dma_alloc, .free = __dma_free, .mmap = __swiotlb_mmap, + .get_sgtable = __swiotlb_get_sgtable, .map_page = __swiotlb_map_page, .unmap_page = __swiotlb_unmap_page, .map_sg = __swiotlb_map_sg_attrs, diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 94d98cd1aad8..aba9ead1384c 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -30,9 +30,11 @@ #include #include +#include #include #include #include +#include #include #include #include @@ -223,6 +225,13 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, mm_flags |= FAULT_FLAG_WRITE; } + /* + * PAN bit set implies the fault happened in kernel space, but not + * in the arch's user access functions. + */ + if (IS_ENABLED(CONFIG_ARM64_PAN) && (regs->pstate & PSR_PAN_BIT)) + goto no_context; + /* * As per x86, we may deadlock here. However, since the kernel only * validly references user space from well defined areas of the code, @@ -492,14 +501,22 @@ asmlinkage void __exception do_sp_pc_abort(unsigned long addr, arm64_notify_die("Oops - SP/PC alignment exception", regs, &info, esr); } -static struct fault_info debug_fault_info[] = { +int __init early_brk64(unsigned long addr, unsigned int esr, + struct pt_regs *regs); + +/* + * __refdata because early_brk64 is __init, but the reference to it is + * clobbered at arch_initcall time. + * See traps.c and debug-monitors.c:debug_traps_init(). + */ +static struct fault_info __refdata debug_fault_info[] = { { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware breakpoint" }, { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware single-step" }, { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware watchpoint" }, { do_bad, SIGBUS, 0, "unknown 3" }, { do_bad, SIGTRAP, TRAP_BRKPT, "aarch32 BKPT" }, { do_bad, SIGTRAP, 0, "aarch32 vector catch" }, - { do_bad, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" }, + { early_brk64, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" }, { do_bad, SIGBUS, 0, "unknown 7" }, }; @@ -536,3 +553,10 @@ asmlinkage int __exception do_debug_exception(unsigned long addr, return 0; } + +#ifdef CONFIG_ARM64_PAN +void cpu_enable_pan(void) +{ + config_sctlr_el1(SCTLR_EL1_SPAN, 0); +} +#endif /* CONFIG_ARM64_PAN */ diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c index 4dfa3975ce5b..c26b804015e8 100644 --- a/arch/arm64/mm/flush.c +++ b/arch/arm64/mm/flush.c @@ -60,14 +60,10 @@ void copy_to_user_page(struct vm_area_struct *vma, struct page *page, unsigned long uaddr, void *dst, const void *src, unsigned long len) { -#ifdef CONFIG_SMP preempt_disable(); -#endif memcpy(dst, src, len); flush_ptrace_access(vma, page, uaddr, dst, len); -#ifdef CONFIG_SMP preempt_enable(); -#endif } void __sync_icache_dcache(pte_t pte, unsigned long addr) diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c index 831ec534d449..383b03ff38f8 100644 --- a/arch/arm64/mm/hugetlbpage.c +++ b/arch/arm64/mm/hugetlbpage.c @@ -13,10 +13,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index ad87ce826cce..f5c0680d17d9 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -358,9 +358,9 @@ void free_initmem(void) #ifdef CONFIG_BLK_DEV_INITRD -static int keep_initrd; +static int keep_initrd __initdata; -void free_initrd_mem(unsigned long start, unsigned long end) +void __init free_initrd_mem(unsigned long start, unsigned long end) { if (!keep_initrd) free_reserved_area((void *)start, (void *)end, 0, "initrd"); diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index a4ede4e2ddd1..9211b8527f25 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -267,7 +267,7 @@ static void *late_alloc(unsigned long size) return ptr; } -static void __ref create_mapping(phys_addr_t phys, unsigned long virt, +static void __init create_mapping(phys_addr_t phys, unsigned long virt, phys_addr_t size, pgprot_t prot) { if (virt < VMALLOC_START) { @@ -460,17 +460,6 @@ void __init paging_init(void) cpu_set_default_tcr_t0sz(); } -/* - * Enable the identity mapping to allow the MMU disabling. - */ -void setup_mm_for_reboot(void) -{ - cpu_set_reserved_ttbr0(); - flush_tlb_all(); - cpu_set_idmap_tcr_t0sz(); - cpu_switch_mm(idmap_pg_dir, &init_mm); -} - /* * Check whether a kernel address is valid (derived from arch/x86/). */ diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 39139a3aa16d..e4ee7bd8830a 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -34,11 +34,7 @@ #define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K #endif -#ifdef CONFIG_SMP #define TCR_SMP_FLAGS TCR_SHARED -#else -#define TCR_SMP_FLAGS 0 -#endif /* PTWs cacheable, inner/outer WBWA */ #define TCR_CACHE_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA @@ -150,13 +146,13 @@ ENDPROC(cpu_do_switch_mm) * value of the SCTLR_EL1 register. */ ENTRY(__cpu_setup) - ic iallu // I+BTB cache invalidate tlbi vmalle1is // invalidate I + D TLBs dsb ish mov x0, #3 << 20 msr cpacr_el1, x0 // Enable FP/ASIMD - msr mdscr_el1, xzr // Reset mdscr_el1 + mov x0, #1 << 12 // Reset mdscr_el1 and disable + msr mdscr_el1, x0 // access to the DCC from EL0 /* * Memory region attributes for LPAE: * @@ -196,6 +192,19 @@ ENTRY(__cpu_setup) */ mrs x9, ID_AA64MMFR0_EL1 bfi x10, x9, #32, #3 +#ifdef CONFIG_ARM64_HW_AFDBM + /* + * Hardware update of the Access and Dirty bits. + */ + mrs x9, ID_AA64MMFR1_EL1 + and x9, x9, #0xf + cbz x9, 2f + cmp x9, #2 + b.lt 1f + orr x10, x10, #TCR_HD // hardware Dirty flag update +1: orr x10, x10, #TCR_HA // hardware Access flag update +2: +#endif /* CONFIG_ARM64_HW_AFDBM */ msr tcr_el1, x10 ret // return to head.S ENDPROC(__cpu_setup) diff --git a/arch/avr32/include/asm/atomic.h b/arch/avr32/include/asm/atomic.h index 2d07ce1c5327..97c9bdf83409 100644 --- a/arch/avr32/include/asm/atomic.h +++ b/arch/avr32/include/asm/atomic.h @@ -44,6 +44,18 @@ static inline int __atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OP_RETURN(sub, sub, rKs21) ATOMIC_OP_RETURN(add, add, r) +#define ATOMIC_OP(op, asm_op) \ +ATOMIC_OP_RETURN(op, asm_op, r) \ +static inline void atomic_##op(int i, atomic_t *v) \ +{ \ + (void)__atomic_##op##_return(i, v); \ +} + +ATOMIC_OP(and, and) +ATOMIC_OP(or, or) +ATOMIC_OP(xor, eor) + +#undef ATOMIC_OP #undef ATOMIC_OP_RETURN /* diff --git a/arch/avr32/mach-at32ap/extint.c b/arch/avr32/mach-at32ap/extint.c index 2d48b6a46166..d51ff8f1c541 100644 --- a/arch/avr32/mach-at32ap/extint.c +++ b/arch/avr32/mach-at32ap/extint.c @@ -128,9 +128,9 @@ static int eic_set_irq_type(struct irq_data *d, unsigned int flow_type) irqd_set_trigger_type(d, flow_type); if (flow_type & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_LEVEL_HIGH)) - __irq_set_handler_locked(irq, handle_level_irq); + irq_set_handler_locked(d, handle_level_irq); else - __irq_set_handler_locked(irq, handle_edge_irq); + irq_set_handler_locked(d, handle_edge_irq); return IRQ_SET_MASK_OK_NOCOPY; } diff --git a/arch/avr32/mach-at32ap/pio.c b/arch/avr32/mach-at32ap/pio.c index 903c7d81d0d5..157a5e0e789f 100644 --- a/arch/avr32/mach-at32ap/pio.c +++ b/arch/avr32/mach-at32ap/pio.c @@ -286,7 +286,7 @@ static void gpio_irq_handler(unsigned irq, struct irq_desc *desc) struct pio_device *pio = irq_desc_get_chip_data(desc); unsigned gpio_irq; - gpio_irq = (unsigned) irq_get_handler_data(irq); + gpio_irq = (unsigned) irq_desc_get_handler_data(desc); for (;;) { u32 isr; @@ -312,7 +312,6 @@ gpio_irq_setup(struct pio_device *pio, int irq, int gpio_irq) unsigned i; irq_set_chip_data(irq, pio); - irq_set_handler_data(irq, (void *)gpio_irq); for (i = 0; i < 32; i++, gpio_irq++) { irq_set_chip_data(gpio_irq, pio); @@ -320,7 +319,8 @@ gpio_irq_setup(struct pio_device *pio, int irq, int gpio_irq) handle_simple_irq); } - irq_set_chained_handler(irq, gpio_irq_handler); + irq_set_chained_handler_and_data(irq, gpio_irq_handler, + (void *)gpio_irq); } /*--------------------------------------------------------------------------*/ diff --git a/arch/blackfin/include/asm/atomic.h b/arch/blackfin/include/asm/atomic.h index a107a98e9978..1c1c42330c99 100644 --- a/arch/blackfin/include/asm/atomic.h +++ b/arch/blackfin/include/asm/atomic.h @@ -16,19 +16,21 @@ #include asmlinkage int __raw_uncached_fetch_asm(const volatile int *ptr); -asmlinkage int __raw_atomic_update_asm(volatile int *ptr, int value); -asmlinkage int __raw_atomic_clear_asm(volatile int *ptr, int value); -asmlinkage int __raw_atomic_set_asm(volatile int *ptr, int value); +asmlinkage int __raw_atomic_add_asm(volatile int *ptr, int value); + +asmlinkage int __raw_atomic_and_asm(volatile int *ptr, int value); +asmlinkage int __raw_atomic_or_asm(volatile int *ptr, int value); asmlinkage int __raw_atomic_xor_asm(volatile int *ptr, int value); asmlinkage int __raw_atomic_test_asm(const volatile int *ptr, int value); #define atomic_read(v) __raw_uncached_fetch_asm(&(v)->counter) -#define atomic_add_return(i, v) __raw_atomic_update_asm(&(v)->counter, i) -#define atomic_sub_return(i, v) __raw_atomic_update_asm(&(v)->counter, -(i)) +#define atomic_add_return(i, v) __raw_atomic_add_asm(&(v)->counter, i) +#define atomic_sub_return(i, v) __raw_atomic_add_asm(&(v)->counter, -(i)) -#define atomic_clear_mask(m, v) __raw_atomic_clear_asm(&(v)->counter, m) -#define atomic_set_mask(m, v) __raw_atomic_set_asm(&(v)->counter, m) +#define atomic_or(i, v) (void)__raw_atomic_or_asm(&(v)->counter, i) +#define atomic_and(i, v) (void)__raw_atomic_and_asm(&(v)->counter, i) +#define atomic_xor(i, v) (void)__raw_atomic_xor_asm(&(v)->counter, i) #endif diff --git a/arch/blackfin/kernel/bfin_ksyms.c b/arch/blackfin/kernel/bfin_ksyms.c index c446591b961d..a401c27b69b4 100644 --- a/arch/blackfin/kernel/bfin_ksyms.c +++ b/arch/blackfin/kernel/bfin_ksyms.c @@ -83,11 +83,12 @@ EXPORT_SYMBOL(insl); EXPORT_SYMBOL(insl_16); #ifdef CONFIG_SMP -EXPORT_SYMBOL(__raw_atomic_update_asm); -EXPORT_SYMBOL(__raw_atomic_clear_asm); -EXPORT_SYMBOL(__raw_atomic_set_asm); +EXPORT_SYMBOL(__raw_atomic_add_asm); +EXPORT_SYMBOL(__raw_atomic_and_asm); +EXPORT_SYMBOL(__raw_atomic_or_asm); EXPORT_SYMBOL(__raw_atomic_xor_asm); EXPORT_SYMBOL(__raw_atomic_test_asm); + EXPORT_SYMBOL(__raw_xchg_1_asm); EXPORT_SYMBOL(__raw_xchg_2_asm); EXPORT_SYMBOL(__raw_xchg_4_asm); diff --git a/arch/blackfin/kernel/time-ts.c b/arch/blackfin/kernel/time-ts.c index cb0a4845339e..fb9e95f1b719 100644 --- a/arch/blackfin/kernel/time-ts.c +++ b/arch/blackfin/kernel/time-ts.c @@ -136,44 +136,44 @@ static int bfin_gptmr0_set_next_event(unsigned long cycles, return 0; } -static void bfin_gptmr0_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int bfin_gptmr0_set_periodic(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: { #ifndef CONFIG_BF60x - set_gptimer_config(TIMER0_id, \ - TIMER_OUT_DIS | TIMER_IRQ_ENA | \ - TIMER_PERIOD_CNT | TIMER_MODE_PWM); + set_gptimer_config(TIMER0_id, + TIMER_OUT_DIS | TIMER_IRQ_ENA | + TIMER_PERIOD_CNT | TIMER_MODE_PWM); #else - set_gptimer_config(TIMER0_id, TIMER_OUT_DIS - | TIMER_MODE_PWM_CONT | TIMER_PULSE_HI | TIMER_IRQ_PER); + set_gptimer_config(TIMER0_id, + TIMER_OUT_DIS | TIMER_MODE_PWM_CONT | + TIMER_PULSE_HI | TIMER_IRQ_PER); #endif - set_gptimer_period(TIMER0_id, get_sclk() / HZ); - set_gptimer_pwidth(TIMER0_id, get_sclk() / HZ - 1); - enable_gptimers(TIMER0bit); - break; - } - case CLOCK_EVT_MODE_ONESHOT: - disable_gptimers(TIMER0bit); + set_gptimer_period(TIMER0_id, get_sclk() / HZ); + set_gptimer_pwidth(TIMER0_id, get_sclk() / HZ - 1); + enable_gptimers(TIMER0bit); + return 0; +} + +static int bfin_gptmr0_set_oneshot(struct clock_event_device *evt) +{ + disable_gptimers(TIMER0bit); #ifndef CONFIG_BF60x - set_gptimer_config(TIMER0_id, \ - TIMER_OUT_DIS | TIMER_IRQ_ENA | TIMER_MODE_PWM); + set_gptimer_config(TIMER0_id, + TIMER_OUT_DIS | TIMER_IRQ_ENA | TIMER_MODE_PWM); #else - set_gptimer_config(TIMER0_id, TIMER_OUT_DIS | TIMER_MODE_PWM - | TIMER_PULSE_HI | TIMER_IRQ_WID_DLY); + set_gptimer_config(TIMER0_id, + TIMER_OUT_DIS | TIMER_MODE_PWM | TIMER_PULSE_HI | + TIMER_IRQ_WID_DLY); #endif - set_gptimer_period(TIMER0_id, 0); - break; - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - disable_gptimers(TIMER0bit); - break; - case CLOCK_EVT_MODE_RESUME: - break; - } + set_gptimer_period(TIMER0_id, 0); + return 0; +} + +static int bfin_gptmr0_shutdown(struct clock_event_device *evt) +{ + disable_gptimers(TIMER0bit); + return 0; } static void bfin_gptmr0_ack(void) @@ -211,13 +211,16 @@ static struct irqaction gptmr0_irq = { }; static struct clock_event_device clockevent_gptmr0 = { - .name = "bfin_gptimer0", - .rating = 300, - .irq = IRQ_TIMER0, - .shift = 32, - .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .set_next_event = bfin_gptmr0_set_next_event, - .set_mode = bfin_gptmr0_set_mode, + .name = "bfin_gptimer0", + .rating = 300, + .irq = IRQ_TIMER0, + .shift = 32, + .features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT, + .set_next_event = bfin_gptmr0_set_next_event, + .set_state_shutdown = bfin_gptmr0_shutdown, + .set_state_periodic = bfin_gptmr0_set_periodic, + .set_state_oneshot = bfin_gptmr0_set_oneshot, }; static void __init bfin_gptmr0_clockevent_init(struct clock_event_device *evt) @@ -250,36 +253,35 @@ static int bfin_coretmr_set_next_event(unsigned long cycles, return 0; } -static void bfin_coretmr_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int bfin_coretmr_set_periodic(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: { - unsigned long tcount = ((get_cclk() / (HZ * TIME_SCALE)) - 1); - bfin_write_TCNTL(TMPWR); - CSYNC(); - bfin_write_TSCALE(TIME_SCALE - 1); - bfin_write_TPERIOD(tcount); - bfin_write_TCOUNT(tcount); - CSYNC(); - bfin_write_TCNTL(TMPWR | TMREN | TAUTORLD); - break; - } - case CLOCK_EVT_MODE_ONESHOT: - bfin_write_TCNTL(TMPWR); - CSYNC(); - bfin_write_TSCALE(TIME_SCALE - 1); - bfin_write_TPERIOD(0); - bfin_write_TCOUNT(0); - break; - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - bfin_write_TCNTL(0); - CSYNC(); - break; - case CLOCK_EVT_MODE_RESUME: - break; - } + unsigned long tcount = ((get_cclk() / (HZ * TIME_SCALE)) - 1); + + bfin_write_TCNTL(TMPWR); + CSYNC(); + bfin_write_TSCALE(TIME_SCALE - 1); + bfin_write_TPERIOD(tcount); + bfin_write_TCOUNT(tcount); + CSYNC(); + bfin_write_TCNTL(TMPWR | TMREN | TAUTORLD); + return 0; +} + +static int bfin_coretmr_set_oneshot(struct clock_event_device *evt) +{ + bfin_write_TCNTL(TMPWR); + CSYNC(); + bfin_write_TSCALE(TIME_SCALE - 1); + bfin_write_TPERIOD(0); + bfin_write_TCOUNT(0); + return 0; +} + +static int bfin_coretmr_shutdown(struct clock_event_device *evt) +{ + bfin_write_TCNTL(0); + CSYNC(); + return 0; } void bfin_coretmr_init(void) @@ -335,7 +337,9 @@ void bfin_coretmr_clockevent_init(void) evt->shift = 32; evt->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; evt->set_next_event = bfin_coretmr_set_next_event; - evt->set_mode = bfin_coretmr_set_mode; + evt->set_state_shutdown = bfin_coretmr_shutdown; + evt->set_state_periodic = bfin_coretmr_set_periodic; + evt->set_state_oneshot = bfin_coretmr_set_oneshot; clock_tick = get_cclk() / TIME_SCALE; evt->mult = div_sc(clock_tick, NSEC_PER_SEC, evt->shift); diff --git a/arch/blackfin/mach-bf537/ints-priority.c b/arch/blackfin/mach-bf537/ints-priority.c index 2137a209a22b..14b2f74554dc 100644 --- a/arch/blackfin/mach-bf537/ints-priority.c +++ b/arch/blackfin/mach-bf537/ints-priority.c @@ -182,9 +182,11 @@ static struct irq_chip bf537_mac_rx_irqchip = { .irq_unmask = bf537_mac_rx_unmask_irq, }; -static void bf537_demux_mac_rx_irq(unsigned int int_irq, +static void bf537_demux_mac_rx_irq(unsigned int __int_irq, struct irq_desc *desc) { + unsigned int int_irq = irq_desc_get_irq(desc); + if (bfin_read_DMA1_IRQ_STATUS() & (DMA_DONE | DMA_ERR)) bfin_handle_irq(IRQ_MAC_RX); else diff --git a/arch/blackfin/mach-bf561/atomic.S b/arch/blackfin/mach-bf561/atomic.S index 2a08df8e8c4c..26fccb5568b9 100644 --- a/arch/blackfin/mach-bf561/atomic.S +++ b/arch/blackfin/mach-bf561/atomic.S @@ -587,10 +587,10 @@ ENDPROC(___raw_write_unlock_asm) * r0 = ptr * r1 = value * - * Add a signed value to a 32bit word and return the new value atomically. + * ADD a signed value to a 32bit word and return the new value atomically. * Clobbers: r3:0, p1:0 */ -ENTRY(___raw_atomic_update_asm) +ENTRY(___raw_atomic_add_asm) p1 = r0; r3 = r1; [--sp] = rets; @@ -603,19 +603,19 @@ ENTRY(___raw_atomic_update_asm) r0 = r3; rets = [sp++]; rts; -ENDPROC(___raw_atomic_update_asm) +ENDPROC(___raw_atomic_add_asm) /* * r0 = ptr * r1 = mask * - * Clear the mask bits from a 32bit word and return the old 32bit value + * AND the mask bits from a 32bit word and return the old 32bit value * atomically. * Clobbers: r3:0, p1:0 */ -ENTRY(___raw_atomic_clear_asm) +ENTRY(___raw_atomic_and_asm) p1 = r0; - r3 = ~r1; + r3 = r1; [--sp] = rets; call _get_core_lock; r2 = [p1]; @@ -627,17 +627,17 @@ ENTRY(___raw_atomic_clear_asm) r0 = r3; rets = [sp++]; rts; -ENDPROC(___raw_atomic_clear_asm) +ENDPROC(___raw_atomic_and_asm) /* * r0 = ptr * r1 = mask * - * Set the mask bits into a 32bit word and return the old 32bit value + * OR the mask bits into a 32bit word and return the old 32bit value * atomically. * Clobbers: r3:0, p1:0 */ -ENTRY(___raw_atomic_set_asm) +ENTRY(___raw_atomic_or_asm) p1 = r0; r3 = r1; [--sp] = rets; @@ -651,7 +651,7 @@ ENTRY(___raw_atomic_set_asm) r0 = r3; rets = [sp++]; rts; -ENDPROC(___raw_atomic_set_asm) +ENDPROC(___raw_atomic_or_asm) /* * r0 = ptr @@ -787,7 +787,7 @@ ENTRY(___raw_bit_set_asm) r2 = r1; r1 = 1; r1 <<= r2; - jump ___raw_atomic_set_asm + jump ___raw_atomic_or_asm ENDPROC(___raw_bit_set_asm) /* @@ -798,10 +798,10 @@ ENDPROC(___raw_bit_set_asm) * Clobbers: r3:0, p1:0 */ ENTRY(___raw_bit_clear_asm) - r2 = r1; - r1 = 1; - r1 <<= r2; - jump ___raw_atomic_clear_asm + r2 = 1; + r2 <<= r1; + r1 = ~r2; + jump ___raw_atomic_and_asm ENDPROC(___raw_bit_clear_asm) /* diff --git a/arch/blackfin/mach-common/ints-priority.c b/arch/blackfin/mach-common/ints-priority.c index 7236bdfc71e6..a6d1b03cdf36 100644 --- a/arch/blackfin/mach-common/ints-priority.c +++ b/arch/blackfin/mach-common/ints-priority.c @@ -194,7 +194,8 @@ void bfin_internal_unmask_irq(unsigned int irq) #ifdef CONFIG_SMP static void bfin_internal_unmask_irq_chip(struct irq_data *d) { - bfin_internal_unmask_irq_affinity(d->irq, d->affinity); + bfin_internal_unmask_irq_affinity(d->irq, + irq_data_get_affinity_mask(d)); } static int bfin_internal_set_affinity(struct irq_data *d, @@ -685,12 +686,12 @@ void bfin_demux_mac_status_irq(unsigned int int_err_irq, } #endif -static inline void bfin_set_irq_handler(unsigned irq, irq_flow_handler_t handle) +static inline void bfin_set_irq_handler(struct irq_data *d, irq_flow_handler_t handle) { #ifdef CONFIG_IPIPE handle = handle_level_irq; #endif - __irq_set_handler_locked(irq, handle); + irq_set_handler_locked(d, handle); } #ifdef CONFIG_GPIO_ADI @@ -802,9 +803,9 @@ static int bfin_gpio_irq_type(struct irq_data *d, unsigned int type) } if (type & (IRQ_TYPE_EDGE_RISING | IRQ_TYPE_EDGE_FALLING)) - bfin_set_irq_handler(irq, handle_edge_irq); + bfin_set_irq_handler(d, handle_edge_irq); else - bfin_set_irq_handler(irq, handle_level_irq); + bfin_set_irq_handler(d, handle_level_irq); return 0; } @@ -824,9 +825,9 @@ static void bfin_demux_gpio_block(unsigned int irq) } } -void bfin_demux_gpio_irq(unsigned int inta_irq, - struct irq_desc *desc) +void bfin_demux_gpio_irq(unsigned int __inta_irq, struct irq_desc *desc) { + unsigned int inta_irq = irq_desc_get_irq(desc); unsigned int irq; switch (inta_irq) { diff --git a/arch/blackfin/mach-common/smp.c b/arch/blackfin/mach-common/smp.c index 1c7259597395..0030e21cfceb 100644 --- a/arch/blackfin/mach-common/smp.c +++ b/arch/blackfin/mach-common/smp.c @@ -195,7 +195,7 @@ void send_ipi(const struct cpumask *cpumask, enum ipi_message_type msg) local_irq_save(flags); for_each_cpu(cpu, cpumask) { bfin_ipi_data = &per_cpu(bfin_ipi, cpu); - atomic_set_mask((1 << msg), &bfin_ipi_data->bits); + atomic_or((1 << msg), &bfin_ipi_data->bits); atomic_inc(&bfin_ipi_data->count); } local_irq_restore(flags); diff --git a/arch/c6x/platforms/megamod-pic.c b/arch/c6x/platforms/megamod-pic.c index 74e3371eb824..d487698e978a 100644 --- a/arch/c6x/platforms/megamod-pic.c +++ b/arch/c6x/platforms/megamod-pic.c @@ -93,10 +93,11 @@ static struct irq_chip megamod_chip = { .irq_unmask = unmask_megamod, }; -static void megamod_irq_cascade(unsigned int irq, struct irq_desc *desc) +static void megamod_irq_cascade(unsigned int __irq, struct irq_desc *desc) { struct megamod_cascade_data *cascade; struct megamod_pic *pic; + unsigned int irq; u32 events; int n, idx; @@ -282,8 +283,8 @@ static struct megamod_pic * __init init_megamod_pic(struct device_node *np) soc_writel(~0, &pic->regs->evtmask[i]); soc_writel(~0, &pic->regs->evtclr[i]); - irq_set_handler_data(irq, &cascade_data[i]); - irq_set_chained_handler(irq, megamod_irq_cascade); + irq_set_chained_handler_and_data(irq, megamod_irq_cascade, + &cascade_data[i]); } /* Finally, set up the MUX registers */ diff --git a/arch/c6x/platforms/timer64.c b/arch/c6x/platforms/timer64.c index 3c73d74a4674..c19901e5f055 100644 --- a/arch/c6x/platforms/timer64.c +++ b/arch/c6x/platforms/timer64.c @@ -126,35 +126,37 @@ static int next_event(unsigned long delta, return 0; } -static void set_clock_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int set_periodic(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - timer64_enable(); - timer64_mode = TIMER64_MODE_PERIODIC; - timer64_config(TIMER64_RATE / HZ); - break; - case CLOCK_EVT_MODE_ONESHOT: - timer64_enable(); - timer64_mode = TIMER64_MODE_ONE_SHOT; - break; - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - timer64_mode = TIMER64_MODE_DISABLED; - timer64_disable(); - break; - case CLOCK_EVT_MODE_RESUME: - break; - } + timer64_enable(); + timer64_mode = TIMER64_MODE_PERIODIC; + timer64_config(TIMER64_RATE / HZ); + return 0; +} + +static int set_oneshot(struct clock_event_device *evt) +{ + timer64_enable(); + timer64_mode = TIMER64_MODE_ONE_SHOT; + return 0; +} + +static int shutdown(struct clock_event_device *evt) +{ + timer64_mode = TIMER64_MODE_DISABLED; + timer64_disable(); + return 0; } static struct clock_event_device t64_clockevent_device = { - .name = "TIMER64_EVT32_TIMER", - .features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_PERIODIC, - .rating = 200, - .set_mode = set_clock_mode, - .set_next_event = next_event, + .name = "TIMER64_EVT32_TIMER", + .features = CLOCK_EVT_FEAT_ONESHOT | + CLOCK_EVT_FEAT_PERIODIC, + .rating = 200, + .set_state_shutdown = shutdown, + .set_state_periodic = set_periodic, + .set_state_oneshot = set_oneshot, + .set_next_event = next_event, }; static irqreturn_t timer_interrupt(int irq, void *dev_id) diff --git a/arch/cris/Kconfig b/arch/cris/Kconfig index 0314e325a669..8da5653bd895 100644 --- a/arch/cris/Kconfig +++ b/arch/cris/Kconfig @@ -36,6 +36,17 @@ config FORCE_MAX_ZONEORDER int default 6 +config TRACE_IRQFLAGS_SUPPORT + depends on ETRAX_ARCH_V32 + def_bool y + +config STACKTRACE_SUPPORT + def_bool y + +config LOCKDEP_SUPPORT + depends on ETRAX_ARCH_V32 + def_bool y + config CRIS bool default y @@ -58,6 +69,7 @@ config CRIS select CLKSRC_MMIO if ETRAX_ARCH_V32 select GENERIC_CLOCKEVENTS if ETRAX_ARCH_V32 select GENERIC_SCHED_CLOCK if ETRAX_ARCH_V32 + select HAVE_DEBUG_BUGVERBOSE if ETRAX_ARCH_V32 config HZ int diff --git a/arch/cris/arch-v10/kernel/entry.S b/arch/cris/arch-v10/kernel/entry.S index 81570fcd0412..b5622521dad5 100644 --- a/arch/cris/arch-v10/kernel/entry.S +++ b/arch/cris/arch-v10/kernel/entry.S @@ -955,6 +955,14 @@ sys_call_table: .long sys_process_vm_writev .long sys_kcmp /* 350 */ .long sys_finit_module + .long sys_sched_setattr + .long sys_sched_getattr + .long sys_renameat2 + .long sys_seccomp /* 355 */ + .long sys_getrandom + .long sys_memfd_create + .long sys_bpf + .long sys_execveat /* * NOTE!! This doesn't have to be exact - we just have diff --git a/arch/cris/arch-v10/lib/dmacopy.c b/arch/cris/arch-v10/lib/dmacopy.c deleted file mode 100644 index 49f5b8ca5b47..000000000000 --- a/arch/cris/arch-v10/lib/dmacopy.c +++ /dev/null @@ -1,42 +0,0 @@ -/* - * memcpy for large blocks, using memory-memory DMA channels 6 and 7 in Etrax - */ - -#include -#include - -#define D(x) - -void *dma_memcpy(void *pdst, - const void *psrc, - unsigned int pn) -{ - static etrax_dma_descr indma, outdma; - - D(printk(KERN_DEBUG "dma_memcpy %d bytes... ", pn)); - -#if 0 - *R_GEN_CONFIG = genconfig_shadow = - (genconfig_shadow & ~0x3c0000) | - IO_STATE(R_GEN_CONFIG, dma6, intdma7) | - IO_STATE(R_GEN_CONFIG, dma7, intdma6); -#endif - indma.sw_len = outdma.sw_len = pn; - indma.ctrl = d_eol | d_eop; - outdma.ctrl = d_eol; - indma.buf = psrc; - outdma.buf = pdst; - - *R_DMA_CH6_FIRST = &indma; - *R_DMA_CH7_FIRST = &outdma; - *R_DMA_CH6_CMD = IO_STATE(R_DMA_CH6_CMD, cmd, start); - *R_DMA_CH7_CMD = IO_STATE(R_DMA_CH7_CMD, cmd, start); - - while (*R_DMA_CH7_CMD == 1) - /* wait for completion */; - - D(printk(KERN_DEBUG "done\n")); -} - - - diff --git a/arch/cris/arch-v10/lib/old_checksum.c b/arch/cris/arch-v10/lib/old_checksum.c deleted file mode 100644 index 8f79163f1394..000000000000 --- a/arch/cris/arch-v10/lib/old_checksum.c +++ /dev/null @@ -1,86 +0,0 @@ -/* - * INET An implementation of the TCP/IP protocol suite for the LINUX - * operating system. INET is implemented using the BSD Socket - * interface as the means of communication with the user level. - * - * IP/TCP/UDP checksumming routines - * - * Authors: Jorge Cwik, - * Arnt Gulbrandsen, - * Tom May, - * Lots of code moved from tcp.c and ip.c; see those files - * for more names. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ - -#include -#include - -#undef PROFILE_CHECKSUM - -#ifdef PROFILE_CHECKSUM -/* these are just for profiling the checksum code with an oscillioscope.. uh */ -#if 0 -#define BITOFF *((unsigned char *)0xb0000030) = 0xff -#define BITON *((unsigned char *)0xb0000030) = 0x0 -#endif -#include -#define CBITON LED_ACTIVE_SET(1) -#define CBITOFF LED_ACTIVE_SET(0) -#define BITOFF -#define BITON -#else -#define BITOFF -#define BITON -#define CBITOFF -#define CBITON -#endif - -/* - * computes a partial checksum, e.g. for TCP/UDP fragments - */ - -#include - -__wsum csum_partial(const void *p, int len, __wsum __sum) -{ - u32 sum = (__force u32)__sum; - const u16 *buff = p; - /* - * Experiments with ethernet and slip connections show that buff - * is aligned on either a 2-byte or 4-byte boundary. - */ - const void *endMarker = p + len; - const void *marker = endMarker - (len % 16); -#if 0 - if((int)buff & 0x3) - printk("unaligned buff %p\n", buff); - __delay(900); /* extra delay of 90 us to test performance hit */ -#endif - BITON; - while (buff < marker) { - sum += *buff++; - sum += *buff++; - sum += *buff++; - sum += *buff++; - sum += *buff++; - sum += *buff++; - sum += *buff++; - sum += *buff++; - } - marker = endMarker - (len % 2); - while (buff < marker) - sum += *buff++; - - if (endMarker > buff) - sum += *(const u8 *)buff; /* add extra byte separately */ - - BITOFF; - return (__force __wsum)sum; -} - -EXPORT_SYMBOL(csum_partial); diff --git a/arch/cris/arch-v32/drivers/Kconfig b/arch/cris/arch-v32/drivers/Kconfig index 4fc16b44fff2..e6c523cc40bc 100644 --- a/arch/cris/arch-v32/drivers/Kconfig +++ b/arch/cris/arch-v32/drivers/Kconfig @@ -202,7 +202,7 @@ config ETRAX_PA_CHANGEABLE_DIR default "0x00" if ETRAXFS default "0x00000000" if !ETRAXFS help - This is a bitmask (8 bits) with information of what bits in PA that a + This is a bitmask with information of what bits in PA that a user can change direction on using ioctl's. Bit set = changeable. You probably want 0 here, but it depends on your hardware. @@ -213,7 +213,7 @@ config ETRAX_PA_CHANGEABLE_BITS default "0x00" if ETRAXFS default "0x00000000" if !ETRAXFS help - This is a bitmask (8 bits) with information of what bits in PA + This is a bitmask with information of what bits in PA that a user can change the value on using ioctl's. Bit set = changeable. @@ -223,7 +223,7 @@ config ETRAX_PB_CHANGEABLE_DIR default "0x00000" if ETRAXFS default "0x00000000" if !ETRAXFS help - This is a bitmask (18 bits) with information of what bits in PB + This is a bitmask with information of what bits in PB that a user can change direction on using ioctl's. Bit set = changeable. You probably want 0 here, but it depends on your hardware. @@ -234,7 +234,7 @@ config ETRAX_PB_CHANGEABLE_BITS default "0x00000" if ETRAXFS default "0x00000000" if !ETRAXFS help - This is a bitmask (18 bits) with information of what bits in PB + This is a bitmask with information of what bits in PB that a user can change the value on using ioctl's. Bit set = changeable. @@ -244,7 +244,7 @@ config ETRAX_PC_CHANGEABLE_DIR default "0x00000" if ETRAXFS default "0x00000000" if !ETRAXFS help - This is a bitmask (18 bits) with information of what bits in PC + This is a bitmask with information of what bits in PC that a user can change direction on using ioctl's. Bit set = changeable. You probably want 0 here, but it depends on your hardware. @@ -253,9 +253,9 @@ config ETRAX_PC_CHANGEABLE_BITS hex "PC user changeable bits mask" depends on ETRAX_GPIO default "0x00000" if ETRAXFS - default "0x00000000" if ETRAXFS + default "0x00000000" if !ETRAXFS help - This is a bitmask (18 bits) with information of what bits in PC + This is a bitmask with information of what bits in PC that a user can change the value on using ioctl's. Bit set = changeable. @@ -264,7 +264,7 @@ config ETRAX_PD_CHANGEABLE_DIR depends on ETRAX_GPIO && ETRAXFS default "0x00000" help - This is a bitmask (18 bits) with information of what bits in PD + This is a bitmask with information of what bits in PD that a user can change direction on using ioctl's. Bit set = changeable. You probably want 0x00000 here, but it depends on your hardware. diff --git a/arch/cris/arch-v32/drivers/axisflashmap.c b/arch/cris/arch-v32/drivers/axisflashmap.c index 28dd77144e8f..5387424683cc 100644 --- a/arch/cris/arch-v32/drivers/axisflashmap.c +++ b/arch/cris/arch-v32/drivers/axisflashmap.c @@ -313,6 +313,7 @@ static int __init init_axis_flash(void) size_t len; int ram_rootfs_partition = -1; /* -1 => no RAM rootfs partition */ int part; + struct mtd_partition *partition; /* We need a root fs. If it resides in RAM, we need to use an * MTDRAM device, so it must be enabled in the kernel config, @@ -329,7 +330,7 @@ static int __init init_axis_flash(void) main_mtd = flash_probe(); if (main_mtd) - printk(KERN_INFO "%s: 0x%08x bytes of NOR flash memory.\n", + printk(KERN_INFO "%s: 0x%08llx bytes of NOR flash memory.\n", main_mtd->name, main_mtd->size); #ifdef CONFIG_ETRAX_NANDFLASH @@ -388,10 +389,10 @@ static int __init init_axis_flash(void) #endif if (main_mtd) { + loff_t ptable_sector = CONFIG_ETRAX_PTABLE_SECTOR; main_mtd->owner = THIS_MODULE; axisflash_mtd = main_mtd; - loff_t ptable_sector = CONFIG_ETRAX_PTABLE_SECTOR; /* First partition (rescue) is always set to the default. */ pidx++; @@ -517,7 +518,7 @@ static int __init init_axis_flash(void) /* Decide whether to use default partition table. */ /* Only use default table if we actually have a device (main_mtd) */ - struct mtd_partition *partition = &axis_partitions[0]; + partition = &axis_partitions[0]; if (main_mtd && !ptable_ok) { memcpy(axis_partitions, axis_default_partitions, sizeof(axis_default_partitions)); @@ -580,7 +581,7 @@ static int __init init_axis_flash(void) printk(KERN_INFO "axisflashmap: Adding RAM partition " "for rootfs image.\n"); err = mtdram_init_device(mtd_ram, - (void *)partition[part].offset, + (void *)(u_int32_t)partition[part].offset, partition[part].size, partition[part].name); if (err) diff --git a/arch/cris/arch-v32/drivers/mach-a3/gpio.c b/arch/cris/arch-v32/drivers/mach-a3/gpio.c index 74f9fe80940c..c92e1da3684d 100644 --- a/arch/cris/arch-v32/drivers/mach-a3/gpio.c +++ b/arch/cris/arch-v32/drivers/mach-a3/gpio.c @@ -957,7 +957,7 @@ static void __init virtual_gpio_init(void) static int __init gpio_init(void) { - int res; + int res, res2; printk(KERN_INFO "ETRAX FS GPIO driver v2.7, (c) 2003-2008 " "Axis Communications AB\n"); @@ -977,7 +977,7 @@ static int __init gpio_init(void) CRIS_LED_DISK_READ(0); CRIS_LED_DISK_WRITE(0); - int res2 = request_irq(GIO_INTR_VECT, gpio_interrupt, + res2 = request_irq(GIO_INTR_VECT, gpio_interrupt, IRQF_SHARED, "gpio", &alarmlist); if (res2) { printk(KERN_ERR "err: irq for gpio\n"); diff --git a/arch/cris/arch-v32/drivers/mach-fs/gpio.c b/arch/cris/arch-v32/drivers/mach-fs/gpio.c index 009f4ee1bd09..72968fbf814b 100644 --- a/arch/cris/arch-v32/drivers/mach-fs/gpio.c +++ b/arch/cris/arch-v32/drivers/mach-fs/gpio.c @@ -425,12 +425,11 @@ gpio_open(struct inode *inode, struct file *filp) if (p > GPIO_MINOR_LAST) return -EINVAL; - priv = kmalloc(sizeof(struct gpio_private), GFP_KERNEL); + priv = kzalloc(sizeof(struct gpio_private), GFP_KERNEL); if (!priv) return -ENOMEM; mutex_lock(&gpio_mutex); - memset(priv, 0, sizeof(*priv)); priv->minor = p; diff --git a/arch/cris/arch-v32/kernel/entry.S b/arch/cris/arch-v32/kernel/entry.S index 026a0b21b8f0..b17a20999f87 100644 --- a/arch/cris/arch-v32/kernel/entry.S +++ b/arch/cris/arch-v32/kernel/entry.S @@ -240,6 +240,17 @@ ret_from_sys_call: .type _Rexit,@function _Rexit: +#if defined(CONFIG_TRACE_IRQFLAGS) + addoq +PT_ccs, $sp, $acr + move.d [$acr], $r0 + btstq 15, $r0 ; I1 + bpl 1f + nop + jsr trace_hardirqs_on + nop +1: +#endif + ;; This epilogue MUST match the prologues in multiple_interrupt, irq.h ;; and ptregs.h. addq 4, $sp ; Skip orig_r10. @@ -875,6 +886,14 @@ sys_call_table: .long sys_process_vm_writev .long sys_kcmp /* 350 */ .long sys_finit_module + .long sys_sched_setattr + .long sys_sched_getattr + .long sys_renameat2 + .long sys_seccomp /* 355 */ + .long sys_getrandom + .long sys_memfd_create + .long sys_bpf + .long sys_execveat /* * NOTE!! This doesn't have to be exact - we just have diff --git a/arch/cris/arch-v32/kernel/process.c b/arch/cris/arch-v32/kernel/process.c index cebd32e2a8fb..c7ce784a393c 100644 --- a/arch/cris/arch-v32/kernel/process.c +++ b/arch/cris/arch-v32/kernel/process.c @@ -23,9 +23,9 @@ extern void stop_watchdog(void); /* We use this if we don't have any better idle routine. */ void default_idle(void) { + local_irq_enable(); /* Halt until exception. */ - __asm__ volatile("ei \n\t" - "halt "); + __asm__ volatile("halt"); } /* diff --git a/arch/cris/arch-v32/kernel/signal.c b/arch/cris/arch-v32/kernel/signal.c index 3a36ae6b79d5..150d1d76c29d 100644 --- a/arch/cris/arch-v32/kernel/signal.c +++ b/arch/cris/arch-v32/kernel/signal.c @@ -19,7 +19,6 @@ #include #include #include -#include #include extern unsigned long cris_signal_return_page; diff --git a/arch/cris/arch-v32/kernel/time.c b/arch/cris/arch-v32/kernel/time.c index 4fce9f1f7cc0..d2a84407654a 100644 --- a/arch/cris/arch-v32/kernel/time.c +++ b/arch/cris/arch-v32/kernel/time.c @@ -172,8 +172,7 @@ void handle_watchdog_bite(struct pt_regs *regs) extern void cris_profile_sample(struct pt_regs *regs); static void __iomem *timer_base; -static void crisv32_clkevt_mode(enum clock_event_mode mode, - struct clock_event_device *dev) +static int crisv32_clkevt_switch_state(struct clock_event_device *dev) { reg_timer_rw_tmr0_ctrl ctrl = { .op = regk_timer_hold, @@ -181,6 +180,7 @@ static void crisv32_clkevt_mode(enum clock_event_mode mode, }; REG_WR(timer, timer_base, rw_tmr0_ctrl, ctrl); + return 0; } static int crisv32_clkevt_next_event(unsigned long evt, @@ -231,7 +231,9 @@ static struct clock_event_device crisv32_clockevent = { .name = "crisv32-timer", .rating = 300, .features = CLOCK_EVT_FEAT_ONESHOT, - .set_mode = crisv32_clkevt_mode, + .set_state_oneshot = crisv32_clkevt_switch_state, + .set_state_shutdown = crisv32_clkevt_switch_state, + .tick_resume = crisv32_clkevt_switch_state, .set_next_event = crisv32_clkevt_next_event, }; diff --git a/arch/cris/arch-v32/mach-fs/pinmux.c b/arch/cris/arch-v32/mach-fs/pinmux.c index 05a04708b8eb..d8a3a3c439dd 100644 --- a/arch/cris/arch-v32/mach-fs/pinmux.c +++ b/arch/cris/arch-v32/mach-fs/pinmux.c @@ -46,6 +46,8 @@ static int __crisv32_pinmux_alloc(int port, int first_pin, int last_pin, pins[port][i] = mode; crisv32_pinmux_set(port); + + return 0; } static int crisv32_pinmux_init(void) @@ -93,6 +95,7 @@ int crisv32_pinmux_alloc_fixed(enum fixed_function function) int ret = -EINVAL; char saved[sizeof pins]; unsigned long flags; + reg_pinmux_rw_hwprot hwprot; spin_lock_irqsave(&pinmux_lock, flags); @@ -101,7 +104,7 @@ int crisv32_pinmux_alloc_fixed(enum fixed_function function) crisv32_pinmux_init(); /* Must be done before we read rw_hwprot */ - reg_pinmux_rw_hwprot hwprot = REG_RD(pinmux, regi_pinmux, rw_hwprot); + hwprot = REG_RD(pinmux, regi_pinmux, rw_hwprot); switch (function) { case pinmux_ser1: @@ -227,6 +230,7 @@ int crisv32_pinmux_dealloc_fixed(enum fixed_function function) int ret = -EINVAL; char saved[sizeof pins]; unsigned long flags; + reg_pinmux_rw_hwprot hwprot; spin_lock_irqsave(&pinmux_lock, flags); @@ -235,7 +239,7 @@ int crisv32_pinmux_dealloc_fixed(enum fixed_function function) crisv32_pinmux_init(); /* Must be done before we read rw_hwprot */ - reg_pinmux_rw_hwprot hwprot = REG_RD(pinmux, regi_pinmux, rw_hwprot); + hwprot = REG_RD(pinmux, regi_pinmux, rw_hwprot); switch (function) { case pinmux_ser1: diff --git a/arch/cris/configs/artpec_3_defconfig b/arch/cris/configs/artpec_3_defconfig index 71854d41c5a0..70e497e0b03e 100644 --- a/arch/cris/configs/artpec_3_defconfig +++ b/arch/cris/configs/artpec_3_defconfig @@ -12,10 +12,6 @@ CONFIG_ETRAX_FAST_TIMER=y CONFIG_CRIS_MACH_ARTPEC3=y CONFIG_ETRAX_DRAM_SIZE=32 CONFIG_ETRAX_FLASH1_SIZE=4 -CONFIG_ETRAX_DEF_GIO_PA_OE=1c -CONFIG_ETRAX_DEF_GIO_PA_OUT=00 -CONFIG_ETRAX_DEF_GIO_PB_OE=00000 -CONFIG_ETRAX_DEF_GIO_PB_OUT=00000 CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y @@ -42,3 +38,4 @@ CONFIG_JFFS2_FS=y CONFIG_CRAMFS=y CONFIG_NFS_FS=y CONFIG_NFS_V3=y +CONFIG_ETRAX_GPIO=y diff --git a/arch/cris/configs/etraxfs_defconfig b/arch/cris/configs/etraxfs_defconfig index 87c7227fecb2..91232680d6c8 100644 --- a/arch/cris/configs/etraxfs_defconfig +++ b/arch/cris/configs/etraxfs_defconfig @@ -38,3 +38,4 @@ CONFIG_JFFS2_FS=y CONFIG_CRAMFS=y CONFIG_NFS_FS=y CONFIG_NFS_V3=y +CONFIG_ETRAX_GPIO=y diff --git a/arch/cris/include/arch-v32/arch/bug.h b/arch/cris/include/arch-v32/arch/bug.h index 0f211e135248..fb59faaaae0a 100644 --- a/arch/cris/include/arch-v32/arch/bug.h +++ b/arch/cris/include/arch-v32/arch/bug.h @@ -10,6 +10,7 @@ * All other stuff is done out-of-band with exception handlers. */ #define BUG() \ +do { \ __asm__ __volatile__ ("0: break 14\n\t" \ ".section .fixup,\"ax\"\n" \ "1:\n\t" \ @@ -21,9 +22,15 @@ ".section __ex_table,\"a\"\n\t" \ ".dword 0b, 1b\n\t" \ ".previous\n\t" \ - : : "ri" (__FILE__), "i" (__LINE__)) + : : "ri" (__FILE__), "i" (__LINE__)); \ + unreachable(); \ +} while (0) #else -#define BUG() __asm__ __volatile__ ("break 14\n\t") +#define BUG() \ +do { \ + __asm__ __volatile__ ("break 14\n\t"); \ + unreachable(); \ +} while (0) #endif #define HAVE_ARCH_BUG diff --git a/arch/cris/include/arch-v32/arch/irqflags.h b/arch/cris/include/arch-v32/arch/irqflags.h index 041851f8ec6f..5f6fddf99509 100644 --- a/arch/cris/include/arch-v32/arch/irqflags.h +++ b/arch/cris/include/arch-v32/arch/irqflags.h @@ -2,7 +2,7 @@ #define __ASM_CRIS_ARCH_IRQFLAGS_H #include -#include +#include static inline unsigned long arch_local_save_flags(void) { diff --git a/arch/cris/include/asm/Kbuild b/arch/cris/include/asm/Kbuild index ad2244f35bca..b7f68192d15b 100644 --- a/arch/cris/include/asm/Kbuild +++ b/arch/cris/include/asm/Kbuild @@ -1,14 +1,20 @@ generic-y += atomic.h +generic-y += auxvec.h generic-y += barrier.h +generic-y += bitsperlong.h generic-y += clkdev.h generic-y += cmpxchg.h generic-y += cputime.h generic-y += device.h generic-y += div64.h +generic-y += errno.h generic-y += exec.h generic-y += emergency-restart.h +generic-y += fcntl.h generic-y += futex.h generic-y += hardirq.h +generic-y += ioctl.h +generic-y += ipcbuf.h generic-y += irq_regs.h generic-y += irq_work.h generic-y += kdebug.h @@ -19,11 +25,22 @@ generic-y += local.h generic-y += local64.h generic-y += mcs_spinlock.h generic-y += mm-arch-hooks.h +generic-y += mman.h generic-y += module.h +generic-y += msgbuf.h generic-y += percpu.h +generic-y += poll.h generic-y += preempt.h +generic-y += resource.h generic-y += sections.h +generic-y += sembuf.h +generic-y += shmbuf.h +generic-y += siginfo.h +generic-y += socket.h +generic-y += sockios.h +generic-y += statfs.h generic-y += topology.h generic-y += trace_clock.h +generic-y += types.h generic-y += vga.h generic-y += xor.h diff --git a/arch/cris/include/asm/mmu_context.h b/arch/cris/include/asm/mmu_context.h index 1d45fd6365b7..349acfd25d2f 100644 --- a/arch/cris/include/asm/mmu_context.h +++ b/arch/cris/include/asm/mmu_context.h @@ -11,7 +11,14 @@ extern void switch_mm(struct mm_struct *prev, struct mm_struct *next, #define deactivate_mm(tsk,mm) do { } while (0) -#define activate_mm(prev,next) switch_mm((prev),(next),NULL) +static inline void activate_mm(struct mm_struct *prev, struct mm_struct *next) +{ + unsigned long flags; + + local_irq_save(flags); + switch_mm(prev, next, NULL); + local_irq_restore(flags); +} /* current active pgd - this is similar to other processors pgd * registers like cr3 on the i386 diff --git a/arch/cris/include/asm/stacktrace.h b/arch/cris/include/asm/stacktrace.h new file mode 100644 index 000000000000..2d90856943ad --- /dev/null +++ b/arch/cris/include/asm/stacktrace.h @@ -0,0 +1,8 @@ +#ifndef __CRIS_STACKTRACE_H +#define __CRIS_STACKTRACE_H + +void walk_stackframe(unsigned long sp, + int (*fn)(unsigned long addr, void *data), + void *data); + +#endif diff --git a/arch/cris/include/asm/types.h b/arch/cris/include/asm/types.h deleted file mode 100644 index a3cac7757c7f..000000000000 --- a/arch/cris/include/asm/types.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _ETRAX_TYPES_H -#define _ETRAX_TYPES_H - -#include - -/* - * These aren't exported outside the kernel to avoid name space clashes - */ - -#define BITS_PER_LONG 32 - -#endif diff --git a/arch/cris/include/asm/unistd.h b/arch/cris/include/asm/unistd.h index 0f40fed1ba25..9c23535821c0 100644 --- a/arch/cris/include/asm/unistd.h +++ b/arch/cris/include/asm/unistd.h @@ -4,7 +4,7 @@ #include -#define NR_syscalls 360 +#define NR_syscalls 365 #include diff --git a/arch/cris/include/uapi/asm/Kbuild b/arch/cris/include/uapi/asm/Kbuild index 01f66b8f15e5..d5564a0ae66a 100644 --- a/arch/cris/include/uapi/asm/Kbuild +++ b/arch/cris/include/uapi/asm/Kbuild @@ -6,6 +6,9 @@ header-y += ../arch-v32/arch/ header-y += auxvec.h header-y += bitsperlong.h header-y += byteorder.h +header-y += elf.h +header-y += elf_v10.h +header-y += elf_v32.h header-y += errno.h header-y += ethernet.h header-y += etraxgpio.h @@ -19,6 +22,8 @@ header-y += param.h header-y += poll.h header-y += posix_types.h header-y += ptrace.h +header-y += ptrace_v10.h +header-y += ptrace_v32.h header-y += resource.h header-y += rs485.h header-y += sembuf.h diff --git a/arch/cris/include/uapi/asm/auxvec.h b/arch/cris/include/uapi/asm/auxvec.h deleted file mode 100644 index cb30b01bf19f..000000000000 --- a/arch/cris/include/uapi/asm/auxvec.h +++ /dev/null @@ -1,4 +0,0 @@ -#ifndef __ASMCRIS_AUXVEC_H -#define __ASMCRIS_AUXVEC_H - -#endif diff --git a/arch/cris/include/uapi/asm/bitsperlong.h b/arch/cris/include/uapi/asm/bitsperlong.h deleted file mode 100644 index 6dc0bb0c13b2..000000000000 --- a/arch/cris/include/uapi/asm/bitsperlong.h +++ /dev/null @@ -1 +0,0 @@ -#include diff --git a/arch/cris/include/asm/elf.h b/arch/cris/include/uapi/asm/elf.h similarity index 95% rename from arch/cris/include/asm/elf.h rename to arch/cris/include/uapi/asm/elf.h index c2a394ff55ff..a5df05bfee66 100644 --- a/arch/cris/include/asm/elf.h +++ b/arch/cris/include/uapi/asm/elf.h @@ -5,7 +5,11 @@ * ELF register definitions.. */ -#include +#ifdef __arch_v32 +#include +#else +#include +#endif #define R_CRIS_NONE 0 #define R_CRIS_8 1 @@ -32,7 +36,6 @@ typedef unsigned long elf_greg_t; /* Note that NGREG is defined to ELF_NGREG in include/linux/elfcore.h, and is thus exposed to user-space. */ -#define ELF_NGREG (sizeof (struct user_regs_struct) / sizeof(elf_greg_t)) typedef elf_greg_t elf_gregset_t[ELF_NGREG]; /* A placeholder; CRIS does not have any fp regs. */ @@ -45,8 +48,6 @@ typedef unsigned long elf_fpregset_t; #define ELF_DATA ELFDATA2LSB #define ELF_ARCH EM_CRIS -#include - /* The master for these definitions is {binutils}/include/elf/cris.h: */ /* User symbols in this file have a leading underscore. */ #define EF_CRIS_UNDERSCORE 0x00000001 diff --git a/arch/cris/include/arch-v10/arch/elf.h b/arch/cris/include/uapi/asm/elf_v10.h similarity index 98% rename from arch/cris/include/arch-v10/arch/elf.h rename to arch/cris/include/uapi/asm/elf_v10.h index 1eb638aeddb4..3ea65cef529d 100644 --- a/arch/cris/include/arch-v10/arch/elf.h +++ b/arch/cris/include/uapi/asm/elf_v10.h @@ -1,10 +1,11 @@ #ifndef __ASMCRIS_ARCH_ELF_H #define __ASMCRIS_ARCH_ELF_H -#include - #define ELF_MACH EF_CRIS_VARIANT_ANY_V0_V10 +/* Matches struct user_regs_struct */ +#define ELF_NGREG 35 + /* * This is used to ensure we don't load something for the wrong architecture. */ diff --git a/arch/cris/include/arch-v32/arch/elf.h b/arch/cris/include/uapi/asm/elf_v32.h similarity index 98% rename from arch/cris/include/arch-v32/arch/elf.h rename to arch/cris/include/uapi/asm/elf_v32.h index c46d58291166..f09fe49005c0 100644 --- a/arch/cris/include/arch-v32/arch/elf.h +++ b/arch/cris/include/uapi/asm/elf_v32.h @@ -1,10 +1,11 @@ #ifndef _ASM_CRIS_ELF_H #define _ASM_CRIS_ELF_H -#include - #define ELF_CORE_EFLAGS EF_CRIS_VARIANT_V32 +/* Matches struct user_regs_struct */ +#define ELF_NGREG 32 + /* * This is used to ensure we don't load something for the wrong architecture. */ diff --git a/arch/cris/include/uapi/asm/errno.h b/arch/cris/include/uapi/asm/errno.h deleted file mode 100644 index 2bf5eb5fa773..000000000000 --- a/arch/cris/include/uapi/asm/errno.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef _CRIS_ERRNO_H -#define _CRIS_ERRNO_H - -#include - -#endif diff --git a/arch/cris/include/uapi/asm/fcntl.h b/arch/cris/include/uapi/asm/fcntl.h deleted file mode 100644 index 46ab12db5739..000000000000 --- a/arch/cris/include/uapi/asm/fcntl.h +++ /dev/null @@ -1 +0,0 @@ -#include diff --git a/arch/cris/include/uapi/asm/ioctl.h b/arch/cris/include/uapi/asm/ioctl.h deleted file mode 100644 index b279fe06dfe5..000000000000 --- a/arch/cris/include/uapi/asm/ioctl.h +++ /dev/null @@ -1 +0,0 @@ -#include diff --git a/arch/cris/include/uapi/asm/ipcbuf.h b/arch/cris/include/uapi/asm/ipcbuf.h deleted file mode 100644 index 84c7e51cb6d0..000000000000 --- a/arch/cris/include/uapi/asm/ipcbuf.h +++ /dev/null @@ -1 +0,0 @@ -#include diff --git a/arch/cris/include/uapi/asm/kvm_para.h b/arch/cris/include/uapi/asm/kvm_para.h deleted file mode 100644 index 14fab8f0b957..000000000000 --- a/arch/cris/include/uapi/asm/kvm_para.h +++ /dev/null @@ -1 +0,0 @@ -#include diff --git a/arch/cris/include/uapi/asm/mman.h b/arch/cris/include/uapi/asm/mman.h deleted file mode 100644 index 8eebf89f5ab1..000000000000 --- a/arch/cris/include/uapi/asm/mman.h +++ /dev/null @@ -1 +0,0 @@ -#include diff --git a/arch/cris/include/uapi/asm/msgbuf.h b/arch/cris/include/uapi/asm/msgbuf.h deleted file mode 100644 index ada63df1d574..000000000000 --- a/arch/cris/include/uapi/asm/msgbuf.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef _CRIS_MSGBUF_H -#define _CRIS_MSGBUF_H - -/* verbatim copy of asm-i386 version */ - -/* - * The msqid64_ds structure for CRIS architecture. - * Note extra padding because this structure is passed back and forth - * between kernel and user space. - * - * Pad space is left for: - * - 64-bit time_t to solve y2038 problem - * - 2 miscellaneous 32-bit values - */ - -struct msqid64_ds { - struct ipc64_perm msg_perm; - __kernel_time_t msg_stime; /* last msgsnd time */ - unsigned long __unused1; - __kernel_time_t msg_rtime; /* last msgrcv time */ - unsigned long __unused2; - __kernel_time_t msg_ctime; /* last change time */ - unsigned long __unused3; - unsigned long msg_cbytes; /* current number of bytes on queue */ - unsigned long msg_qnum; /* number of messages in queue */ - unsigned long msg_qbytes; /* max number of bytes on queue */ - __kernel_pid_t msg_lspid; /* pid of last msgsnd */ - __kernel_pid_t msg_lrpid; /* last receive pid */ - unsigned long __unused4; - unsigned long __unused5; -}; - -#endif /* _CRIS_MSGBUF_H */ diff --git a/arch/cris/include/uapi/asm/poll.h b/arch/cris/include/uapi/asm/poll.h deleted file mode 100644 index c98509d3149e..000000000000 --- a/arch/cris/include/uapi/asm/poll.h +++ /dev/null @@ -1 +0,0 @@ -#include diff --git a/arch/cris/include/uapi/asm/ptrace.h b/arch/cris/include/uapi/asm/ptrace.h index c689c9bbbe50..bd8946f83ed3 100644 --- a/arch/cris/include/uapi/asm/ptrace.h +++ b/arch/cris/include/uapi/asm/ptrace.h @@ -1 +1,5 @@ -#include +#ifdef __arch_v32 +#include +#else +#include +#endif diff --git a/arch/cris/include/arch-v10/arch/ptrace.h b/arch/cris/include/uapi/asm/ptrace_v10.h similarity index 100% rename from arch/cris/include/arch-v10/arch/ptrace.h rename to arch/cris/include/uapi/asm/ptrace_v10.h diff --git a/arch/cris/include/arch-v32/arch/ptrace.h b/arch/cris/include/uapi/asm/ptrace_v32.h similarity index 100% rename from arch/cris/include/arch-v32/arch/ptrace.h rename to arch/cris/include/uapi/asm/ptrace_v32.h diff --git a/arch/cris/include/uapi/asm/resource.h b/arch/cris/include/uapi/asm/resource.h deleted file mode 100644 index b5d29448de4e..000000000000 --- a/arch/cris/include/uapi/asm/resource.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef _CRIS_RESOURCE_H -#define _CRIS_RESOURCE_H - -#include - -#endif diff --git a/arch/cris/include/uapi/asm/sembuf.h b/arch/cris/include/uapi/asm/sembuf.h deleted file mode 100644 index 7fed9843796d..000000000000 --- a/arch/cris/include/uapi/asm/sembuf.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef _CRIS_SEMBUF_H -#define _CRIS_SEMBUF_H - -/* - * The semid64_ds structure for CRIS architecture. - * Note extra padding because this structure is passed back and forth - * between kernel and user space. - * - * Pad space is left for: - * - 64-bit time_t to solve y2038 problem - * - 2 miscellaneous 32-bit values - */ - -struct semid64_ds { - struct ipc64_perm sem_perm; /* permissions .. see ipc.h */ - __kernel_time_t sem_otime; /* last semop time */ - unsigned long __unused1; - __kernel_time_t sem_ctime; /* last change time */ - unsigned long __unused2; - unsigned long sem_nsems; /* no. of semaphores in array */ - unsigned long __unused3; - unsigned long __unused4; -}; - -#endif /* _CRIS_SEMBUF_H */ diff --git a/arch/cris/include/uapi/asm/shmbuf.h b/arch/cris/include/uapi/asm/shmbuf.h deleted file mode 100644 index 3239e3f000e8..000000000000 --- a/arch/cris/include/uapi/asm/shmbuf.h +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef _CRIS_SHMBUF_H -#define _CRIS_SHMBUF_H - -/* - * The shmid64_ds structure for CRIS architecture (same as for i386) - * Note extra padding because this structure is passed back and forth - * between kernel and user space. - * - * Pad space is left for: - * - 64-bit time_t to solve y2038 problem - * - 2 miscellaneous 32-bit values - */ - -struct shmid64_ds { - struct ipc64_perm shm_perm; /* operation perms */ - size_t shm_segsz; /* size of segment (bytes) */ - __kernel_time_t shm_atime; /* last attach time */ - unsigned long __unused1; - __kernel_time_t shm_dtime; /* last detach time */ - unsigned long __unused2; - __kernel_time_t shm_ctime; /* last change time */ - unsigned long __unused3; - __kernel_pid_t shm_cpid; /* pid of creator */ - __kernel_pid_t shm_lpid; /* pid of last operator */ - unsigned long shm_nattch; /* no. of current attaches */ - unsigned long __unused4; - unsigned long __unused5; -}; - -struct shminfo64 { - unsigned long shmmax; - unsigned long shmmin; - unsigned long shmmni; - unsigned long shmseg; - unsigned long shmall; - unsigned long __unused1; - unsigned long __unused2; - unsigned long __unused3; - unsigned long __unused4; -}; - -#endif /* _CRIS_SHMBUF_H */ diff --git a/arch/cris/include/uapi/asm/siginfo.h b/arch/cris/include/uapi/asm/siginfo.h deleted file mode 100644 index c1cd6d16928b..000000000000 --- a/arch/cris/include/uapi/asm/siginfo.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef _CRIS_SIGINFO_H -#define _CRIS_SIGINFO_H - -#include - -#endif diff --git a/arch/cris/include/uapi/asm/socket.h b/arch/cris/include/uapi/asm/socket.h deleted file mode 100644 index e2503d9f1869..000000000000 --- a/arch/cris/include/uapi/asm/socket.h +++ /dev/null @@ -1,92 +0,0 @@ -#ifndef _ASM_SOCKET_H -#define _ASM_SOCKET_H - -/* almost the same as asm-i386/socket.h */ - -#include - -/* For setsockoptions(2) */ -#define SOL_SOCKET 1 - -#define SO_DEBUG 1 -#define SO_REUSEADDR 2 -#define SO_TYPE 3 -#define SO_ERROR 4 -#define SO_DONTROUTE 5 -#define SO_BROADCAST 6 -#define SO_SNDBUF 7 -#define SO_RCVBUF 8 -#define SO_SNDBUFFORCE 32 -#define SO_RCVBUFFORCE 33 -#define SO_KEEPALIVE 9 -#define SO_OOBINLINE 10 -#define SO_NO_CHECK 11 -#define SO_PRIORITY 12 -#define SO_LINGER 13 -#define SO_BSDCOMPAT 14 -#define SO_REUSEPORT 15 -#define SO_PASSCRED 16 -#define SO_PEERCRED 17 -#define SO_RCVLOWAT 18 -#define SO_SNDLOWAT 19 -#define SO_RCVTIMEO 20 -#define SO_SNDTIMEO 21 - -/* Security levels - as per NRL IPv6 - don't actually do anything */ -#define SO_SECURITY_AUTHENTICATION 22 -#define SO_SECURITY_ENCRYPTION_TRANSPORT 23 -#define SO_SECURITY_ENCRYPTION_NETWORK 24 - -#define SO_BINDTODEVICE 25 - -/* Socket filtering */ -#define SO_ATTACH_FILTER 26 -#define SO_DETACH_FILTER 27 -#define SO_GET_FILTER SO_ATTACH_FILTER - -#define SO_PEERNAME 28 -#define SO_TIMESTAMP 29 -#define SCM_TIMESTAMP SO_TIMESTAMP - -#define SO_ACCEPTCONN 30 - -#define SO_PEERSEC 31 -#define SO_PASSSEC 34 -#define SO_TIMESTAMPNS 35 -#define SCM_TIMESTAMPNS SO_TIMESTAMPNS - -#define SO_MARK 36 - -#define SO_TIMESTAMPING 37 -#define SCM_TIMESTAMPING SO_TIMESTAMPING - -#define SO_PROTOCOL 38 -#define SO_DOMAIN 39 - -#define SO_RXQ_OVFL 40 - -#define SO_WIFI_STATUS 41 -#define SCM_WIFI_STATUS SO_WIFI_STATUS -#define SO_PEEK_OFF 42 - -/* Instruct lower device to use last 4-bytes of skb data as FCS */ -#define SO_NOFCS 43 - -#define SO_LOCK_FILTER 44 - -#define SO_SELECT_ERR_QUEUE 45 - -#define SO_BUSY_POLL 46 - -#define SO_MAX_PACING_RATE 47 - -#define SO_BPF_EXTENSIONS 48 - -#define SO_INCOMING_CPU 49 - -#define SO_ATTACH_BPF 50 -#define SO_DETACH_BPF SO_DETACH_FILTER - -#endif /* _ASM_SOCKET_H */ - - diff --git a/arch/cris/include/uapi/asm/sockios.h b/arch/cris/include/uapi/asm/sockios.h deleted file mode 100644 index cfe7bfecf599..000000000000 --- a/arch/cris/include/uapi/asm/sockios.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef __ARCH_CRIS_SOCKIOS__ -#define __ARCH_CRIS_SOCKIOS__ - -/* Socket-level I/O control calls. */ -#define FIOSETOWN 0x8901 -#define SIOCSPGRP 0x8902 -#define FIOGETOWN 0x8903 -#define SIOCGPGRP 0x8904 -#define SIOCATMARK 0x8905 -#define SIOCGSTAMP 0x8906 /* Get stamp (timeval) */ -#define SIOCGSTAMPNS 0x8907 /* Get stamp (timespec) */ - -#endif diff --git a/arch/cris/include/uapi/asm/statfs.h b/arch/cris/include/uapi/asm/statfs.h deleted file mode 100644 index fdaf921844bc..000000000000 --- a/arch/cris/include/uapi/asm/statfs.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef _CRIS_STATFS_H -#define _CRIS_STATFS_H - -#include - -#endif diff --git a/arch/cris/include/uapi/asm/types.h b/arch/cris/include/uapi/asm/types.h deleted file mode 100644 index 9ec9d4c5ac4d..000000000000 --- a/arch/cris/include/uapi/asm/types.h +++ /dev/null @@ -1 +0,0 @@ -#include diff --git a/arch/cris/include/uapi/asm/unistd.h b/arch/cris/include/uapi/asm/unistd.h index f3287face443..062b648b27e1 100644 --- a/arch/cris/include/uapi/asm/unistd.h +++ b/arch/cris/include/uapi/asm/unistd.h @@ -356,5 +356,13 @@ #define __NR_process_vm_writev 349 #define __NR_kcmp 350 #define __NR_finit_module 351 +#define __NR_sched_setattr 352 +#define __NR_sched_getattr 353 +#define __NR_renameat2 354 +#define __NR_seccomp 355 +#define __NR_getrandom 356 +#define __NR_memfd_create 357 +#define __NR_bpf 358 +#define __NR_execveat 359 #endif /* _UAPI_ASM_CRIS_UNISTD_H_ */ diff --git a/arch/cris/kernel/Makefile b/arch/cris/kernel/Makefile index edef71f12bb8..5fae398ca915 100644 --- a/arch/cris/kernel/Makefile +++ b/arch/cris/kernel/Makefile @@ -8,6 +8,7 @@ extra-y := vmlinux.lds obj-y := process.o traps.o irq.o ptrace.o setup.o time.o sys_cris.o obj-y += devicetree.o +obj-y += stacktrace.o obj-$(CONFIG_MODULES) += crisksyms.o obj-$(CONFIG_MODULES) += module.o diff --git a/arch/cris/kernel/irq.c b/arch/cris/kernel/irq.c index dd0be5de55d5..694850e8f077 100644 --- a/arch/cris/kernel/irq.c +++ b/arch/cris/kernel/irq.c @@ -45,7 +45,11 @@ asmlinkage void do_IRQ(int irq, struct pt_regs * regs) { unsigned long sp; - struct pt_regs *old_regs = set_irq_regs(regs); + struct pt_regs *old_regs; + + trace_hardirqs_off(); + + old_regs = set_irq_regs(regs); irq_enter(); sp = rdsp(); if (unlikely((sp & (PAGE_SIZE - 1)) < (PAGE_SIZE/8))) { diff --git a/arch/cris/kernel/stacktrace.c b/arch/cris/kernel/stacktrace.c new file mode 100644 index 000000000000..99838c74456d --- /dev/null +++ b/arch/cris/kernel/stacktrace.c @@ -0,0 +1,76 @@ +#include +#include +#include +#include + +void walk_stackframe(unsigned long sp, + int (*fn)(unsigned long addr, void *data), + void *data) +{ + unsigned long high = ALIGN(sp, THREAD_SIZE); + + for (; sp <= high - 4; sp += 4) { + unsigned long addr = *(unsigned long *) sp; + + if (!kernel_text_address(addr)) + continue; + + if (fn(addr, data)) + break; + } +} + +struct stack_trace_data { + struct stack_trace *trace; + unsigned int no_sched_functions; + unsigned int skip; +}; + +#ifdef CONFIG_STACKTRACE + +static int save_trace(unsigned long addr, void *d) +{ + struct stack_trace_data *data = d; + struct stack_trace *trace = data->trace; + + if (data->no_sched_functions && in_sched_functions(addr)) + return 0; + + if (data->skip) { + data->skip--; + return 0; + } + + trace->entries[trace->nr_entries++] = addr; + + return trace->nr_entries >= trace->max_entries; +} + +void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) +{ + struct stack_trace_data data; + unsigned long sp; + + data.trace = trace; + data.skip = trace->skip; + + if (tsk != current) { + data.no_sched_functions = 1; + sp = tsk->thread.ksp; + } else { + data.no_sched_functions = 0; + sp = rdsp(); + } + + walk_stackframe(sp, save_trace, &data); + if (trace->nr_entries < trace->max_entries) + trace->entries[trace->nr_entries++] = ULONG_MAX; +} + +void save_stack_trace(struct stack_trace *trace) +{ + save_stack_trace_tsk(current, trace); +} +EXPORT_SYMBOL_GPL(save_stack_trace); + +#endif /* CONFIG_STACKTRACE */ diff --git a/arch/frv/include/asm/atomic.h b/arch/frv/include/asm/atomic.h index 102190a61d65..0da689def4cc 100644 --- a/arch/frv/include/asm/atomic.h +++ b/arch/frv/include/asm/atomic.h @@ -15,7 +15,6 @@ #define _ASM_ATOMIC_H #include -#include #include #include @@ -23,6 +22,8 @@ #error not SMP safe #endif +#include + /* * Atomic operations that C can't guarantee us. Useful for * resource counting etc.. @@ -34,56 +35,26 @@ #define atomic_read(v) ACCESS_ONCE((v)->counter) #define atomic_set(v, i) (((v)->counter) = (i)) -#ifndef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS +static inline int atomic_inc_return(atomic_t *v) +{ + return __atomic_add_return(1, &v->counter); +} + +static inline int atomic_dec_return(atomic_t *v) +{ + return __atomic_sub_return(1, &v->counter); +} + static inline int atomic_add_return(int i, atomic_t *v) { - unsigned long val; - - asm("0: \n" - " orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */ - " ckeq icc3,cc7 \n" - " ld.p %M0,%1 \n" /* LD.P/ORCR must be atomic */ - " orcr cc7,cc7,cc3 \n" /* set CC3 to true */ - " add%I2 %1,%2,%1 \n" - " cst.p %1,%M0 ,cc3,#1 \n" - " corcc gr29,gr29,gr0 ,cc3,#1 \n" /* clear ICC3.Z if store happens */ - " beq icc3,#0,0b \n" - : "+U"(v->counter), "=&r"(val) - : "NPr"(i) - : "memory", "cc7", "cc3", "icc3" - ); - - return val; + return __atomic_add_return(i, &v->counter); } static inline int atomic_sub_return(int i, atomic_t *v) { - unsigned long val; - - asm("0: \n" - " orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */ - " ckeq icc3,cc7 \n" - " ld.p %M0,%1 \n" /* LD.P/ORCR must be atomic */ - " orcr cc7,cc7,cc3 \n" /* set CC3 to true */ - " sub%I2 %1,%2,%1 \n" - " cst.p %1,%M0 ,cc3,#1 \n" - " corcc gr29,gr29,gr0 ,cc3,#1 \n" /* clear ICC3.Z if store happens */ - " beq icc3,#0,0b \n" - : "+U"(v->counter), "=&r"(val) - : "NPr"(i) - : "memory", "cc7", "cc3", "icc3" - ); - - return val; + return __atomic_sub_return(i, &v->counter); } -#else - -extern int atomic_add_return(int i, atomic_t *v); -extern int atomic_sub_return(int i, atomic_t *v); - -#endif - static inline int atomic_add_negative(int i, atomic_t *v) { return atomic_add_return(i, v) < 0; @@ -101,17 +72,14 @@ static inline void atomic_sub(int i, atomic_t *v) static inline void atomic_inc(atomic_t *v) { - atomic_add_return(1, v); + atomic_inc_return(v); } static inline void atomic_dec(atomic_t *v) { - atomic_sub_return(1, v); + atomic_dec_return(v); } -#define atomic_dec_return(v) atomic_sub_return(1, (v)) -#define atomic_inc_return(v) atomic_add_return(1, (v)) - #define atomic_sub_and_test(i,v) (atomic_sub_return((i), (v)) == 0) #define atomic_dec_and_test(v) (atomic_sub_return(1, (v)) == 0) #define atomic_inc_and_test(v) (atomic_add_return(1, (v)) == 0) @@ -120,18 +88,19 @@ static inline void atomic_dec(atomic_t *v) * 64-bit atomic ops */ typedef struct { - volatile long long counter; + long long counter; } atomic64_t; #define ATOMIC64_INIT(i) { (i) } -static inline long long atomic64_read(atomic64_t *v) +static inline long long atomic64_read(const atomic64_t *v) { long long counter; asm("ldd%I1 %M1,%0" : "=e"(counter) : "m"(v->counter)); + return counter; } @@ -142,10 +111,25 @@ static inline void atomic64_set(atomic64_t *v, long long i) : "e"(i)); } -extern long long atomic64_inc_return(atomic64_t *v); -extern long long atomic64_dec_return(atomic64_t *v); -extern long long atomic64_add_return(long long i, atomic64_t *v); -extern long long atomic64_sub_return(long long i, atomic64_t *v); +static inline long long atomic64_inc_return(atomic64_t *v) +{ + return __atomic64_add_return(1, &v->counter); +} + +static inline long long atomic64_dec_return(atomic64_t *v) +{ + return __atomic64_sub_return(1, &v->counter); +} + +static inline long long atomic64_add_return(long long i, atomic64_t *v) +{ + return __atomic64_add_return(i, &v->counter); +} + +static inline long long atomic64_sub_return(long long i, atomic64_t *v) +{ + return __atomic64_sub_return(i, &v->counter); +} static inline long long atomic64_add_negative(long long i, atomic64_t *v) { @@ -176,6 +160,7 @@ static inline void atomic64_dec(atomic64_t *v) #define atomic64_dec_and_test(v) (atomic64_dec_return((v)) == 0) #define atomic64_inc_and_test(v) (atomic64_inc_return((v)) == 0) + #define atomic_cmpxchg(v, old, new) (cmpxchg(&(v)->counter, old, new)) #define atomic_xchg(v, new) (xchg(&(v)->counter, new)) #define atomic64_cmpxchg(v, old, new) (__cmpxchg_64(old, new, &(v)->counter)) @@ -196,5 +181,21 @@ static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) return c; } +#define ATOMIC_OP(op) \ +static inline void atomic_##op(int i, atomic_t *v) \ +{ \ + (void)__atomic32_fetch_##op(i, &v->counter); \ +} \ + \ +static inline void atomic64_##op(long long i, atomic64_t *v) \ +{ \ + (void)__atomic64_fetch_##op(i, &v->counter); \ +} + +ATOMIC_OP(or) +ATOMIC_OP(and) +ATOMIC_OP(xor) + +#undef ATOMIC_OP #endif /* _ASM_ATOMIC_H */ diff --git a/arch/frv/include/asm/atomic_defs.h b/arch/frv/include/asm/atomic_defs.h new file mode 100644 index 000000000000..36e126d2f801 --- /dev/null +++ b/arch/frv/include/asm/atomic_defs.h @@ -0,0 +1,172 @@ + +#include + +#ifdef __ATOMIC_LIB__ + +#ifdef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS + +#define ATOMIC_QUALS +#define ATOMIC_EXPORT(x) EXPORT_SYMBOL(x) + +#else /* !OUTOFLINE && LIB */ + +#define ATOMIC_OP_RETURN(op) +#define ATOMIC_FETCH_OP(op) + +#endif /* OUTOFLINE */ + +#else /* !__ATOMIC_LIB__ */ + +#define ATOMIC_EXPORT(x) + +#ifdef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS + +#define ATOMIC_OP_RETURN(op) \ +extern int __atomic_##op##_return(int i, int *v); \ +extern long long __atomic64_##op##_return(long long i, long long *v); + +#define ATOMIC_FETCH_OP(op) \ +extern int __atomic32_fetch_##op(int i, int *v); \ +extern long long __atomic64_fetch_##op(long long i, long long *v); + +#else /* !OUTOFLINE && !LIB */ + +#define ATOMIC_QUALS static inline + +#endif /* OUTOFLINE */ +#endif /* __ATOMIC_LIB__ */ + + +/* + * Note on the 64 bit inline asm variants... + * + * CSTD is a conditional instruction and needs a constrained memory reference. + * Normally 'U' provides the correct constraints for conditional instructions + * and this is used for the 32 bit version, however 'U' does not appear to work + * for 64 bit values (gcc-4.9) + * + * The exact constraint is that conditional instructions cannot deal with an + * immediate displacement in the memory reference, so what we do is we read the + * address through a volatile cast into a local variable in order to insure we + * _have_ to compute the correct address without displacement. This allows us + * to use the regular 'm' for the memory address. + * + * Furthermore, the %Ln operand, which prints the low word register (r+1), + * really only works for registers, this means we cannot allow immediate values + * for the 64 bit versions -- like we do for the 32 bit ones. + * + */ + +#ifndef ATOMIC_OP_RETURN +#define ATOMIC_OP_RETURN(op) \ +ATOMIC_QUALS int __atomic_##op##_return(int i, int *v) \ +{ \ + int val; \ + \ + asm volatile( \ + "0: \n" \ + " orcc gr0,gr0,gr0,icc3 \n" \ + " ckeq icc3,cc7 \n" \ + " ld.p %M0,%1 \n" \ + " orcr cc7,cc7,cc3 \n" \ + " "#op"%I2 %1,%2,%1 \n" \ + " cst.p %1,%M0 ,cc3,#1 \n" \ + " corcc gr29,gr29,gr0 ,cc3,#1 \n" \ + " beq icc3,#0,0b \n" \ + : "+U"(*v), "=&r"(val) \ + : "NPr"(i) \ + : "memory", "cc7", "cc3", "icc3" \ + ); \ + \ + return val; \ +} \ +ATOMIC_EXPORT(__atomic_##op##_return); \ + \ +ATOMIC_QUALS long long __atomic64_##op##_return(long long i, long long *v) \ +{ \ + long long *__v = READ_ONCE(v); \ + long long val; \ + \ + asm volatile( \ + "0: \n" \ + " orcc gr0,gr0,gr0,icc3 \n" \ + " ckeq icc3,cc7 \n" \ + " ldd.p %M0,%1 \n" \ + " orcr cc7,cc7,cc3 \n" \ + " "#op"cc %L1,%L2,%L1,icc0 \n" \ + " "#op"x %1,%2,%1,icc0 \n" \ + " cstd.p %1,%M0 ,cc3,#1 \n" \ + " corcc gr29,gr29,gr0 ,cc3,#1 \n" \ + " beq icc3,#0,0b \n" \ + : "+m"(*__v), "=&e"(val) \ + : "e"(i) \ + : "memory", "cc7", "cc3", "icc0", "icc3" \ + ); \ + \ + return val; \ +} \ +ATOMIC_EXPORT(__atomic64_##op##_return); +#endif + +#ifndef ATOMIC_FETCH_OP +#define ATOMIC_FETCH_OP(op) \ +ATOMIC_QUALS int __atomic32_fetch_##op(int i, int *v) \ +{ \ + int old, tmp; \ + \ + asm volatile( \ + "0: \n" \ + " orcc gr0,gr0,gr0,icc3 \n" \ + " ckeq icc3,cc7 \n" \ + " ld.p %M0,%1 \n" \ + " orcr cc7,cc7,cc3 \n" \ + " "#op"%I3 %1,%3,%2 \n" \ + " cst.p %2,%M0 ,cc3,#1 \n" \ + " corcc gr29,gr29,gr0 ,cc3,#1 \n" \ + " beq icc3,#0,0b \n" \ + : "+U"(*v), "=&r"(old), "=r"(tmp) \ + : "NPr"(i) \ + : "memory", "cc7", "cc3", "icc3" \ + ); \ + \ + return old; \ +} \ +ATOMIC_EXPORT(__atomic32_fetch_##op); \ + \ +ATOMIC_QUALS long long __atomic64_fetch_##op(long long i, long long *v) \ +{ \ + long long *__v = READ_ONCE(v); \ + long long old, tmp; \ + \ + asm volatile( \ + "0: \n" \ + " orcc gr0,gr0,gr0,icc3 \n" \ + " ckeq icc3,cc7 \n" \ + " ldd.p %M0,%1 \n" \ + " orcr cc7,cc7,cc3 \n" \ + " "#op" %L1,%L3,%L2 \n" \ + " "#op" %1,%3,%2 \n" \ + " cstd.p %2,%M0 ,cc3,#1 \n" \ + " corcc gr29,gr29,gr0 ,cc3,#1 \n" \ + " beq icc3,#0,0b \n" \ + : "+m"(*__v), "=&e"(old), "=e"(tmp) \ + : "e"(i) \ + : "memory", "cc7", "cc3", "icc3" \ + ); \ + \ + return old; \ +} \ +ATOMIC_EXPORT(__atomic64_fetch_##op); +#endif + +ATOMIC_FETCH_OP(or) +ATOMIC_FETCH_OP(and) +ATOMIC_FETCH_OP(xor) + +ATOMIC_OP_RETURN(add) +ATOMIC_OP_RETURN(sub) + +#undef ATOMIC_FETCH_OP +#undef ATOMIC_OP_RETURN +#undef ATOMIC_QUALS +#undef ATOMIC_EXPORT diff --git a/arch/frv/include/asm/bitops.h b/arch/frv/include/asm/bitops.h index 96de220ef131..0df8e95e3715 100644 --- a/arch/frv/include/asm/bitops.h +++ b/arch/frv/include/asm/bitops.h @@ -25,109 +25,30 @@ #include -#ifndef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS -static inline -unsigned long atomic_test_and_ANDNOT_mask(unsigned long mask, volatile unsigned long *v) -{ - unsigned long old, tmp; - - asm volatile( - "0: \n" - " orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */ - " ckeq icc3,cc7 \n" - " ld.p %M0,%1 \n" /* LD.P/ORCR are atomic */ - " orcr cc7,cc7,cc3 \n" /* set CC3 to true */ - " and%I3 %1,%3,%2 \n" - " cst.p %2,%M0 ,cc3,#1 \n" /* if store happens... */ - " corcc gr29,gr29,gr0 ,cc3,#1 \n" /* ... clear ICC3.Z */ - " beq icc3,#0,0b \n" - : "+U"(*v), "=&r"(old), "=r"(tmp) - : "NPr"(~mask) - : "memory", "cc7", "cc3", "icc3" - ); - - return old; -} - -static inline -unsigned long atomic_test_and_OR_mask(unsigned long mask, volatile unsigned long *v) -{ - unsigned long old, tmp; - - asm volatile( - "0: \n" - " orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */ - " ckeq icc3,cc7 \n" - " ld.p %M0,%1 \n" /* LD.P/ORCR are atomic */ - " orcr cc7,cc7,cc3 \n" /* set CC3 to true */ - " or%I3 %1,%3,%2 \n" - " cst.p %2,%M0 ,cc3,#1 \n" /* if store happens... */ - " corcc gr29,gr29,gr0 ,cc3,#1 \n" /* ... clear ICC3.Z */ - " beq icc3,#0,0b \n" - : "+U"(*v), "=&r"(old), "=r"(tmp) - : "NPr"(mask) - : "memory", "cc7", "cc3", "icc3" - ); - - return old; -} - -static inline -unsigned long atomic_test_and_XOR_mask(unsigned long mask, volatile unsigned long *v) -{ - unsigned long old, tmp; - - asm volatile( - "0: \n" - " orcc gr0,gr0,gr0,icc3 \n" /* set ICC3.Z */ - " ckeq icc3,cc7 \n" - " ld.p %M0,%1 \n" /* LD.P/ORCR are atomic */ - " orcr cc7,cc7,cc3 \n" /* set CC3 to true */ - " xor%I3 %1,%3,%2 \n" - " cst.p %2,%M0 ,cc3,#1 \n" /* if store happens... */ - " corcc gr29,gr29,gr0 ,cc3,#1 \n" /* ... clear ICC3.Z */ - " beq icc3,#0,0b \n" - : "+U"(*v), "=&r"(old), "=r"(tmp) - : "NPr"(mask) - : "memory", "cc7", "cc3", "icc3" - ); - - return old; -} - -#else - -extern unsigned long atomic_test_and_ANDNOT_mask(unsigned long mask, volatile unsigned long *v); -extern unsigned long atomic_test_and_OR_mask(unsigned long mask, volatile unsigned long *v); -extern unsigned long atomic_test_and_XOR_mask(unsigned long mask, volatile unsigned long *v); - -#endif - -#define atomic_clear_mask(mask, v) atomic_test_and_ANDNOT_mask((mask), (v)) -#define atomic_set_mask(mask, v) atomic_test_and_OR_mask((mask), (v)) +#include static inline int test_and_clear_bit(unsigned long nr, volatile void *addr) { - volatile unsigned long *ptr = addr; - unsigned long mask = 1UL << (nr & 31); + unsigned int *ptr = (void *)addr; + unsigned int mask = 1UL << (nr & 31); ptr += nr >> 5; - return (atomic_test_and_ANDNOT_mask(mask, ptr) & mask) != 0; + return (__atomic32_fetch_and(~mask, ptr) & mask) != 0; } static inline int test_and_set_bit(unsigned long nr, volatile void *addr) { - volatile unsigned long *ptr = addr; - unsigned long mask = 1UL << (nr & 31); + unsigned int *ptr = (void *)addr; + unsigned int mask = 1UL << (nr & 31); ptr += nr >> 5; - return (atomic_test_and_OR_mask(mask, ptr) & mask) != 0; + return (__atomic32_fetch_or(mask, ptr) & mask) != 0; } static inline int test_and_change_bit(unsigned long nr, volatile void *addr) { - volatile unsigned long *ptr = addr; - unsigned long mask = 1UL << (nr & 31); + unsigned int *ptr = (void *)addr; + unsigned int mask = 1UL << (nr & 31); ptr += nr >> 5; - return (atomic_test_and_XOR_mask(mask, ptr) & mask) != 0; + return (__atomic32_fetch_xor(mask, ptr) & mask) != 0; } static inline void clear_bit(unsigned long nr, volatile void *addr) diff --git a/arch/frv/kernel/dma.c b/arch/frv/kernel/dma.c index 156184e17e57..370dc9fa0b11 100644 --- a/arch/frv/kernel/dma.c +++ b/arch/frv/kernel/dma.c @@ -109,13 +109,13 @@ static struct frv_dma_channel frv_dma_channels[FRV_DMA_NCHANS] = { static DEFINE_RWLOCK(frv_dma_channels_lock); -unsigned long frv_dma_inprogress; +unsigned int frv_dma_inprogress; #define frv_clear_dma_inprogress(channel) \ - atomic_clear_mask(1 << (channel), &frv_dma_inprogress); + (void)__atomic32_fetch_and(~(1 << (channel)), &frv_dma_inprogress); #define frv_set_dma_inprogress(channel) \ - atomic_set_mask(1 << (channel), &frv_dma_inprogress); + (void)__atomic32_fetch_or(1 << (channel), &frv_dma_inprogress); /*****************************************************************************/ /* diff --git a/arch/frv/kernel/frv_ksyms.c b/arch/frv/kernel/frv_ksyms.c index 86c516d96dcd..cdb4ce9960eb 100644 --- a/arch/frv/kernel/frv_ksyms.c +++ b/arch/frv/kernel/frv_ksyms.c @@ -58,11 +58,6 @@ EXPORT_SYMBOL(__outsl_ns); EXPORT_SYMBOL(__insl_ns); #ifdef CONFIG_FRV_OUTOFLINE_ATOMIC_OPS -EXPORT_SYMBOL(atomic_test_and_ANDNOT_mask); -EXPORT_SYMBOL(atomic_test_and_OR_mask); -EXPORT_SYMBOL(atomic_test_and_XOR_mask); -EXPORT_SYMBOL(atomic_add_return); -EXPORT_SYMBOL(atomic_sub_return); EXPORT_SYMBOL(__xchg_32); EXPORT_SYMBOL(__cmpxchg_32); #endif diff --git a/arch/frv/lib/Makefile b/arch/frv/lib/Makefile index 4ff2fb1e6b16..970e8b4f1a02 100644 --- a/arch/frv/lib/Makefile +++ b/arch/frv/lib/Makefile @@ -5,4 +5,4 @@ lib-y := \ __ashldi3.o __lshrdi3.o __muldi3.o __ashrdi3.o __negdi2.o __ucmpdi2.o \ checksum.o memcpy.o memset.o atomic-ops.o atomic64-ops.o \ - outsl_ns.o outsl_sw.o insl_ns.o insl_sw.o cache.o + outsl_ns.o outsl_sw.o insl_ns.o insl_sw.o cache.o atomic-lib.o diff --git a/arch/frv/lib/atomic-lib.c b/arch/frv/lib/atomic-lib.c new file mode 100644 index 000000000000..4d1b887c248b --- /dev/null +++ b/arch/frv/lib/atomic-lib.c @@ -0,0 +1,7 @@ + +#include +#include + +#define __ATOMIC_LIB__ + +#include diff --git a/arch/frv/lib/atomic-ops.S b/arch/frv/lib/atomic-ops.S index 5e9e6ab5dd0e..b7439a960b5b 100644 --- a/arch/frv/lib/atomic-ops.S +++ b/arch/frv/lib/atomic-ops.S @@ -17,116 +17,6 @@ .text .balign 4 -############################################################################### -# -# unsigned long atomic_test_and_ANDNOT_mask(unsigned long mask, volatile unsigned long *v); -# -############################################################################### - .globl atomic_test_and_ANDNOT_mask - .type atomic_test_and_ANDNOT_mask,@function -atomic_test_and_ANDNOT_mask: - not.p gr8,gr10 -0: - orcc gr0,gr0,gr0,icc3 /* set ICC3.Z */ - ckeq icc3,cc7 - ld.p @(gr9,gr0),gr8 /* LD.P/ORCR must be atomic */ - orcr cc7,cc7,cc3 /* set CC3 to true */ - and gr8,gr10,gr11 - cst.p gr11,@(gr9,gr0) ,cc3,#1 - corcc gr29,gr29,gr0 ,cc3,#1 /* clear ICC3.Z if store happens */ - beq icc3,#0,0b - bralr - - .size atomic_test_and_ANDNOT_mask, .-atomic_test_and_ANDNOT_mask - -############################################################################### -# -# unsigned long atomic_test_and_OR_mask(unsigned long mask, volatile unsigned long *v); -# -############################################################################### - .globl atomic_test_and_OR_mask - .type atomic_test_and_OR_mask,@function -atomic_test_and_OR_mask: - or.p gr8,gr8,gr10 -0: - orcc gr0,gr0,gr0,icc3 /* set ICC3.Z */ - ckeq icc3,cc7 - ld.p @(gr9,gr0),gr8 /* LD.P/ORCR must be atomic */ - orcr cc7,cc7,cc3 /* set CC3 to true */ - or gr8,gr10,gr11 - cst.p gr11,@(gr9,gr0) ,cc3,#1 - corcc gr29,gr29,gr0 ,cc3,#1 /* clear ICC3.Z if store happens */ - beq icc3,#0,0b - bralr - - .size atomic_test_and_OR_mask, .-atomic_test_and_OR_mask - -############################################################################### -# -# unsigned long atomic_test_and_XOR_mask(unsigned long mask, volatile unsigned long *v); -# -############################################################################### - .globl atomic_test_and_XOR_mask - .type atomic_test_and_XOR_mask,@function -atomic_test_and_XOR_mask: - or.p gr8,gr8,gr10 -0: - orcc gr0,gr0,gr0,icc3 /* set ICC3.Z */ - ckeq icc3,cc7 - ld.p @(gr9,gr0),gr8 /* LD.P/ORCR must be atomic */ - orcr cc7,cc7,cc3 /* set CC3 to true */ - xor gr8,gr10,gr11 - cst.p gr11,@(gr9,gr0) ,cc3,#1 - corcc gr29,gr29,gr0 ,cc3,#1 /* clear ICC3.Z if store happens */ - beq icc3,#0,0b - bralr - - .size atomic_test_and_XOR_mask, .-atomic_test_and_XOR_mask - -############################################################################### -# -# int atomic_add_return(int i, atomic_t *v) -# -############################################################################### - .globl atomic_add_return - .type atomic_add_return,@function -atomic_add_return: - or.p gr8,gr8,gr10 -0: - orcc gr0,gr0,gr0,icc3 /* set ICC3.Z */ - ckeq icc3,cc7 - ld.p @(gr9,gr0),gr8 /* LD.P/ORCR must be atomic */ - orcr cc7,cc7,cc3 /* set CC3 to true */ - add gr8,gr10,gr8 - cst.p gr8,@(gr9,gr0) ,cc3,#1 - corcc gr29,gr29,gr0 ,cc3,#1 /* clear ICC3.Z if store happens */ - beq icc3,#0,0b - bralr - - .size atomic_add_return, .-atomic_add_return - -############################################################################### -# -# int atomic_sub_return(int i, atomic_t *v) -# -############################################################################### - .globl atomic_sub_return - .type atomic_sub_return,@function -atomic_sub_return: - or.p gr8,gr8,gr10 -0: - orcc gr0,gr0,gr0,icc3 /* set ICC3.Z */ - ckeq icc3,cc7 - ld.p @(gr9,gr0),gr8 /* LD.P/ORCR must be atomic */ - orcr cc7,cc7,cc3 /* set CC3 to true */ - sub gr8,gr10,gr8 - cst.p gr8,@(gr9,gr0) ,cc3,#1 - corcc gr29,gr29,gr0 ,cc3,#1 /* clear ICC3.Z if store happens */ - beq icc3,#0,0b - bralr - - .size atomic_sub_return, .-atomic_sub_return - ############################################################################### # # uint32_t __xchg_32(uint32_t i, uint32_t *v) diff --git a/arch/frv/lib/atomic64-ops.S b/arch/frv/lib/atomic64-ops.S index b6194eeac127..c4c472308a33 100644 --- a/arch/frv/lib/atomic64-ops.S +++ b/arch/frv/lib/atomic64-ops.S @@ -18,100 +18,6 @@ .balign 4 -############################################################################### -# -# long long atomic64_inc_return(atomic64_t *v) -# -############################################################################### - .globl atomic64_inc_return - .type atomic64_inc_return,@function -atomic64_inc_return: - or.p gr8,gr8,gr10 -0: - orcc gr0,gr0,gr0,icc3 /* set ICC3.Z */ - ckeq icc3,cc7 - ldd.p @(gr10,gr0),gr8 /* LDD.P/ORCR must be atomic */ - orcr cc7,cc7,cc3 /* set CC3 to true */ - addicc gr9,#1,gr9,icc0 - addxi gr8,#0,gr8,icc0 - cstd.p gr8,@(gr10,gr0) ,cc3,#1 - corcc gr29,gr29,gr0 ,cc3,#1 /* clear ICC3.Z if store happens */ - beq icc3,#0,0b - bralr - - .size atomic64_inc_return, .-atomic64_inc_return - -############################################################################### -# -# long long atomic64_dec_return(atomic64_t *v) -# -############################################################################### - .globl atomic64_dec_return - .type atomic64_dec_return,@function -atomic64_dec_return: - or.p gr8,gr8,gr10 -0: - orcc gr0,gr0,gr0,icc3 /* set ICC3.Z */ - ckeq icc3,cc7 - ldd.p @(gr10,gr0),gr8 /* LDD.P/ORCR must be atomic */ - orcr cc7,cc7,cc3 /* set CC3 to true */ - subicc gr9,#1,gr9,icc0 - subxi gr8,#0,gr8,icc0 - cstd.p gr8,@(gr10,gr0) ,cc3,#1 - corcc gr29,gr29,gr0 ,cc3,#1 /* clear ICC3.Z if store happens */ - beq icc3,#0,0b - bralr - - .size atomic64_dec_return, .-atomic64_dec_return - -############################################################################### -# -# long long atomic64_add_return(long long i, atomic64_t *v) -# -############################################################################### - .globl atomic64_add_return - .type atomic64_add_return,@function -atomic64_add_return: - or.p gr8,gr8,gr4 - or gr9,gr9,gr5 -0: - orcc gr0,gr0,gr0,icc3 /* set ICC3.Z */ - ckeq icc3,cc7 - ldd.p @(gr10,gr0),gr8 /* LDD.P/ORCR must be atomic */ - orcr cc7,cc7,cc3 /* set CC3 to true */ - addcc gr9,gr5,gr9,icc0 - addx gr8,gr4,gr8,icc0 - cstd.p gr8,@(gr10,gr0) ,cc3,#1 - corcc gr29,gr29,gr0 ,cc3,#1 /* clear ICC3.Z if store happens */ - beq icc3,#0,0b - bralr - - .size atomic64_add_return, .-atomic64_add_return - -############################################################################### -# -# long long atomic64_sub_return(long long i, atomic64_t *v) -# -############################################################################### - .globl atomic64_sub_return - .type atomic64_sub_return,@function -atomic64_sub_return: - or.p gr8,gr8,gr4 - or gr9,gr9,gr5 -0: - orcc gr0,gr0,gr0,icc3 /* set ICC3.Z */ - ckeq icc3,cc7 - ldd.p @(gr10,gr0),gr8 /* LDD.P/ORCR must be atomic */ - orcr cc7,cc7,cc3 /* set CC3 to true */ - subcc gr9,gr5,gr9,icc0 - subx gr8,gr4,gr8,icc0 - cstd.p gr8,@(gr10,gr0) ,cc3,#1 - corcc gr29,gr29,gr0 ,cc3,#1 /* clear ICC3.Z if store happens */ - beq icc3,#0,0b - bralr - - .size atomic64_sub_return, .-atomic64_sub_return - ############################################################################### # # uint64_t __xchg_64(uint64_t i, uint64_t *v) diff --git a/arch/h8300/boot/compressed/misc.c b/arch/h8300/boot/compressed/misc.c index 704274127c07..c4f2cfcb117b 100644 --- a/arch/h8300/boot/compressed/misc.c +++ b/arch/h8300/boot/compressed/misc.c @@ -70,5 +70,5 @@ void decompress_kernel(void) free_mem_ptr = (unsigned long)&_end; free_mem_end_ptr = free_mem_ptr + HEAP_SIZE; - decompress(input_data, input_len, NULL, NULL, output, NULL, error); + __decompress(input_data, input_len, NULL, NULL, output, 0, NULL, error); } diff --git a/arch/h8300/include/asm/atomic.h b/arch/h8300/include/asm/atomic.h index 7ca73f8546cc..702ee539f87d 100644 --- a/arch/h8300/include/asm/atomic.h +++ b/arch/h8300/include/asm/atomic.h @@ -16,83 +16,52 @@ #include -static inline int atomic_add_return(int i, atomic_t *v) -{ - h8300flags flags; - int ret; - - flags = arch_local_irq_save(); - ret = v->counter += i; - arch_local_irq_restore(flags); - return ret; +#define ATOMIC_OP_RETURN(op, c_op) \ +static inline int atomic_##op##_return(int i, atomic_t *v) \ +{ \ + h8300flags flags; \ + int ret; \ + \ + flags = arch_local_irq_save(); \ + ret = v->counter c_op i; \ + arch_local_irq_restore(flags); \ + return ret; \ } -#define atomic_add(i, v) atomic_add_return(i, v) +#define ATOMIC_OP(op, c_op) \ +static inline void atomic_##op(int i, atomic_t *v) \ +{ \ + h8300flags flags; \ + \ + flags = arch_local_irq_save(); \ + v->counter c_op i; \ + arch_local_irq_restore(flags); \ +} + +ATOMIC_OP_RETURN(add, +=) +ATOMIC_OP_RETURN(sub, -=) + +ATOMIC_OP(and, &=) +ATOMIC_OP(or, |=) +ATOMIC_OP(xor, ^=) + +#undef ATOMIC_OP_RETURN +#undef ATOMIC_OP + +#define atomic_add(i, v) (void)atomic_add_return(i, v) #define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) -static inline int atomic_sub_return(int i, atomic_t *v) -{ - h8300flags flags; - int ret; +#define atomic_sub(i, v) (void)atomic_sub_return(i, v) +#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0) - flags = arch_local_irq_save(); - ret = v->counter -= i; - arch_local_irq_restore(flags); - return ret; -} +#define atomic_inc_return(v) atomic_add_return(1, v) +#define atomic_dec_return(v) atomic_sub_return(1, v) -#define atomic_sub(i, v) atomic_sub_return(i, v) -#define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0) +#define atomic_inc(v) (void)atomic_inc_return(v) +#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) -static inline int atomic_inc_return(atomic_t *v) -{ - h8300flags flags; - int ret; - - flags = arch_local_irq_save(); - v->counter++; - ret = v->counter; - arch_local_irq_restore(flags); - return ret; -} - -#define atomic_inc(v) atomic_inc_return(v) - -/* - * atomic_inc_and_test - increment and test - * @v: pointer of type atomic_t - * - * Atomically increments @v by 1 - * and returns true if the result is zero, or false for all - * other cases. - */ -#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) - -static inline int atomic_dec_return(atomic_t *v) -{ - h8300flags flags; - int ret; - - flags = arch_local_irq_save(); - --v->counter; - ret = v->counter; - arch_local_irq_restore(flags); - return ret; -} - -#define atomic_dec(v) atomic_dec_return(v) - -static inline int atomic_dec_and_test(atomic_t *v) -{ - h8300flags flags; - int ret; - - flags = arch_local_irq_save(); - --v->counter; - ret = v->counter; - arch_local_irq_restore(flags); - return ret == 0; -} +#define atomic_dec(v) (void)atomic_dec_return(v) +#define atomic_dec_and_test(v) (atomic_dec_return(v) == 0) static inline int atomic_cmpxchg(atomic_t *v, int old, int new) { @@ -120,40 +89,4 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) return ret; } -static inline void atomic_clear_mask(unsigned long mask, unsigned long *v) -{ - unsigned char ccr; - unsigned long tmp; - - __asm__ __volatile__("stc ccr,%w3\n\t" - "orc #0x80,ccr\n\t" - "mov.l %0,%1\n\t" - "and.l %2,%1\n\t" - "mov.l %1,%0\n\t" - "ldc %w3,ccr" - : "=m"(*v), "=r"(tmp) - : "g"(~(mask)), "r"(ccr)); -} - -static inline void atomic_set_mask(unsigned long mask, unsigned long *v) -{ - unsigned char ccr; - unsigned long tmp; - - __asm__ __volatile__("stc ccr,%w3\n\t" - "orc #0x80,ccr\n\t" - "mov.l %0,%1\n\t" - "or.l %2,%1\n\t" - "mov.l %1,%0\n\t" - "ldc %w3,ccr" - : "=m"(*v), "=r"(tmp) - : "g"(~(mask)), "r"(ccr)); -} - -/* Atomic operations are already serializing */ -#define smp_mb__before_atomic_dec() barrier() -#define smp_mb__after_atomic_dec() barrier() -#define smp_mb__before_atomic_inc() barrier() -#define smp_mb__after_atomic_inc() barrier() - #endif /* __ARCH_H8300_ATOMIC __ */ diff --git a/arch/h8300/include/asm/dma-mapping.h b/arch/h8300/include/asm/dma-mapping.h index 6e67a90902f2..d9b5b806afe6 100644 --- a/arch/h8300/include/asm/dma-mapping.h +++ b/arch/h8300/include/asm/dma-mapping.h @@ -1,8 +1,6 @@ #ifndef _H8300_DMA_MAPPING_H #define _H8300_DMA_MAPPING_H -#include - extern struct dma_map_ops h8300_dma_map_ops; static inline struct dma_map_ops *get_dma_ops(struct device *dev) @@ -12,46 +10,4 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev) #include -static inline int dma_supported(struct device *dev, u64 mask) -{ - return 0; -} - -static inline int dma_set_mask(struct device *dev, u64 mask) -{ - return 0; -} - -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) -#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) - -#define dma_alloc_coherent(d, s, h, f) dma_alloc_attrs(d, s, h, f, NULL) - -static inline void *dma_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - void *memory; - - memory = ops->alloc(dev, size, dma_handle, flag, attrs); - return memory; -} - -#define dma_free_coherent(d, s, c, h) dma_free_attrs(d, s, c, h, NULL) - -static inline void dma_free_attrs(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t dma_handle, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - - ops->free(dev, size, cpu_addr, dma_handle, attrs); -} - -static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - return 0; -} - #endif diff --git a/arch/hexagon/include/asm/atomic.h b/arch/hexagon/include/asm/atomic.h index 93d07025f183..811d61f6422d 100644 --- a/arch/hexagon/include/asm/atomic.h +++ b/arch/hexagon/include/asm/atomic.h @@ -132,6 +132,10 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP diff --git a/arch/hexagon/include/asm/dma-mapping.h b/arch/hexagon/include/asm/dma-mapping.h index 16965427f6b4..268fde8a4575 100644 --- a/arch/hexagon/include/asm/dma-mapping.h +++ b/arch/hexagon/include/asm/dma-mapping.h @@ -31,12 +31,10 @@ struct device; extern int bad_dma_address; +#define DMA_ERROR_CODE bad_dma_address extern struct dma_map_ops *dma_ops; -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) -#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) - static inline struct dma_map_ops *get_dma_ops(struct device *dev) { if (unlikely(dev == NULL)) @@ -45,8 +43,8 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev) return dma_ops; } +#define HAVE_ARCH_DMA_SUPPORTED 1 extern int dma_supported(struct device *dev, u64 mask); -extern int dma_set_mask(struct device *dev, u64 mask); extern int dma_is_consistent(struct device *dev, dma_addr_t dma_handle); extern void dma_cache_sync(struct device *dev, void *vaddr, size_t size, enum dma_data_direction direction); @@ -60,47 +58,4 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) return addr + size - 1 <= *dev->dma_mask; } -static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - struct dma_map_ops *dma_ops = get_dma_ops(dev); - - if (dma_ops->mapping_error) - return dma_ops->mapping_error(dev, dma_addr); - - return (dma_addr == bad_dma_address); -} - -#define dma_alloc_coherent(d,s,h,f) dma_alloc_attrs(d,s,h,f,NULL) - -static inline void *dma_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) -{ - void *ret; - struct dma_map_ops *ops = get_dma_ops(dev); - - BUG_ON(!dma_ops); - - ret = ops->alloc(dev, size, dma_handle, flag, attrs); - - debug_dma_alloc_coherent(dev, size, *dma_handle, ret); - - return ret; -} - -#define dma_free_coherent(d,s,c,h) dma_free_attrs(d,s,c,h,NULL) - -static inline void dma_free_attrs(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t dma_handle, - struct dma_attrs *attrs) -{ - struct dma_map_ops *dma_ops = get_dma_ops(dev); - - BUG_ON(!dma_ops); - - dma_ops->free(dev, size, cpu_addr, dma_handle, attrs); - - debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); -} - #endif diff --git a/arch/hexagon/include/uapi/asm/signal.h b/arch/hexagon/include/uapi/asm/signal.h index 98106e55ad4f..24b998888916 100644 --- a/arch/hexagon/include/uapi/asm/signal.h +++ b/arch/hexagon/include/uapi/asm/signal.h @@ -19,8 +19,6 @@ #ifndef _ASM_SIGNAL_H #define _ASM_SIGNAL_H -#include - extern unsigned long __rt_sigtramp_template[2]; void do_signal(struct pt_regs *regs); diff --git a/arch/hexagon/kernel/dma.c b/arch/hexagon/kernel/dma.c index b74f9bae31a3..9e3ddf792bd3 100644 --- a/arch/hexagon/kernel/dma.c +++ b/arch/hexagon/kernel/dma.c @@ -44,17 +44,6 @@ int dma_supported(struct device *dev, u64 mask) } EXPORT_SYMBOL(dma_supported); -int dma_set_mask(struct device *dev, u64 mask) -{ - if (!dev->dma_mask || !dma_supported(dev, mask)) - return -EIO; - - *dev->dma_mask = mask; - - return 0; -} -EXPORT_SYMBOL(dma_set_mask); - static struct gen_pool *coherent_pool; diff --git a/arch/hexagon/kernel/time.c b/arch/hexagon/kernel/time.c index 17fbf45bf150..a6a1d1f8309a 100644 --- a/arch/hexagon/kernel/time.c +++ b/arch/hexagon/kernel/time.c @@ -97,20 +97,6 @@ static int set_next_event(unsigned long delta, struct clock_event_device *evt) return 0; } -/* - * Sets the mode (periodic, shutdown, oneshot, etc) of a timer. - */ -static void set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) -{ - switch (mode) { - case CLOCK_EVT_MODE_SHUTDOWN: - /* XXX implement me */ - default: - break; - } -} - #ifdef CONFIG_SMP /* Broadcast mechanism */ static void broadcast(const struct cpumask *mask) @@ -119,13 +105,13 @@ static void broadcast(const struct cpumask *mask) } #endif +/* XXX Implement set_state_shutdown() */ static struct clock_event_device hexagon_clockevent_dev = { .name = "clockevent", .features = CLOCK_EVT_FEAT_ONESHOT, .rating = 400, .irq = RTOS_TIMER_INT, .set_next_event = set_next_event, - .set_mode = set_mode, #ifdef CONFIG_SMP .broadcast = broadcast, #endif @@ -146,7 +132,6 @@ void setup_percpu_clockdev(void) dummy_clock_dev->features = CLOCK_EVT_FEAT_DUMMY; dummy_clock_dev->cpumask = cpumask_of(cpu); - dummy_clock_dev->mode = CLOCK_EVT_MODE_UNUSED; clockevents_register_device(dummy_clock_dev); } diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index 42a91a7aa2b0..eb0249e37981 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -518,6 +518,7 @@ source "drivers/sn/Kconfig" config KEXEC bool "kexec system call" depends on !IA64_HP_SIM && (!SMP || HOTPLUG_CPU) + select KEXEC_CORE help kexec is a system call that implements the ability to shutdown your current kernel, and to start another kernel. It is like a reboot diff --git a/arch/ia64/hp/common/sba_iommu.c b/arch/ia64/hp/common/sba_iommu.c index 344387a55406..a6d6190c9d24 100644 --- a/arch/ia64/hp/common/sba_iommu.c +++ b/arch/ia64/hp/common/sba_iommu.c @@ -1140,13 +1140,9 @@ sba_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, #ifdef CONFIG_NUMA { - int node = ioc->node; struct page *page; - if (node == NUMA_NO_NODE) - node = numa_node_id(); - - page = alloc_pages_exact_node(node, flags, get_order(size)); + page = alloc_pages_node(ioc->node, flags, get_order(size)); if (unlikely(!page)) return NULL; diff --git a/arch/ia64/include/asm/atomic.h b/arch/ia64/include/asm/atomic.h index 0bf03501fe5c..be4beeb77d57 100644 --- a/arch/ia64/include/asm/atomic.h +++ b/arch/ia64/include/asm/atomic.h @@ -45,8 +45,6 @@ ia64_atomic_##op (int i, atomic_t *v) \ ATOMIC_OP(add, +) ATOMIC_OP(sub, -) -#undef ATOMIC_OP - #define atomic_add_return(i,v) \ ({ \ int __ia64_aar_i = (i); \ @@ -71,6 +69,16 @@ ATOMIC_OP(sub, -) : ia64_atomic_sub(__ia64_asr_i, v); \ }) +ATOMIC_OP(and, &) +ATOMIC_OP(or, |) +ATOMIC_OP(xor, ^) + +#define atomic_and(i,v) (void)ia64_atomic_and(i,v) +#define atomic_or(i,v) (void)ia64_atomic_or(i,v) +#define atomic_xor(i,v) (void)ia64_atomic_xor(i,v) + +#undef ATOMIC_OP + #define ATOMIC64_OP(op, c_op) \ static __inline__ long \ ia64_atomic64_##op (__s64 i, atomic64_t *v) \ @@ -89,8 +97,6 @@ ia64_atomic64_##op (__s64 i, atomic64_t *v) \ ATOMIC64_OP(add, +) ATOMIC64_OP(sub, -) -#undef ATOMIC64_OP - #define atomic64_add_return(i,v) \ ({ \ long __ia64_aar_i = (i); \ @@ -115,6 +121,16 @@ ATOMIC64_OP(sub, -) : ia64_atomic64_sub(__ia64_asr_i, v); \ }) +ATOMIC64_OP(and, &) +ATOMIC64_OP(or, |) +ATOMIC64_OP(xor, ^) + +#define atomic64_and(i,v) (void)ia64_atomic64_and(i,v) +#define atomic64_or(i,v) (void)ia64_atomic64_or(i,v) +#define atomic64_xor(i,v) (void)ia64_atomic64_xor(i,v) + +#undef ATOMIC64_OP + #define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new)) #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) diff --git a/arch/ia64/include/asm/barrier.h b/arch/ia64/include/asm/barrier.h index 843ba435e43b..df896a1c41d3 100644 --- a/arch/ia64/include/asm/barrier.h +++ b/arch/ia64/include/asm/barrier.h @@ -66,12 +66,12 @@ do { \ compiletime_assert_atomic_type(*p); \ barrier(); \ - ACCESS_ONCE(*p) = (v); \ + WRITE_ONCE(*p, v); \ } while (0) #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ barrier(); \ ___p1; \ diff --git a/arch/ia64/include/asm/dma-mapping.h b/arch/ia64/include/asm/dma-mapping.h index cf3ab7e784b5..9beccf8010bd 100644 --- a/arch/ia64/include/asm/dma-mapping.h +++ b/arch/ia64/include/asm/dma-mapping.h @@ -23,60 +23,10 @@ extern void machvec_dma_sync_single(struct device *, dma_addr_t, size_t, extern void machvec_dma_sync_sg(struct device *, struct scatterlist *, int, enum dma_data_direction); -#define dma_alloc_coherent(d,s,h,f) dma_alloc_attrs(d,s,h,f,NULL) - -static inline void *dma_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *daddr, gfp_t gfp, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = platform_dma_get_ops(dev); - void *caddr; - - caddr = ops->alloc(dev, size, daddr, gfp, attrs); - debug_dma_alloc_coherent(dev, size, *daddr, caddr); - return caddr; -} - -#define dma_free_coherent(d,s,c,h) dma_free_attrs(d,s,c,h,NULL) - -static inline void dma_free_attrs(struct device *dev, size_t size, - void *caddr, dma_addr_t daddr, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = platform_dma_get_ops(dev); - debug_dma_free_coherent(dev, size, caddr, daddr); - ops->free(dev, size, caddr, daddr, attrs); -} - -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) -#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) - #define get_dma_ops(dev) platform_dma_get_ops(dev) #include -static inline int dma_mapping_error(struct device *dev, dma_addr_t daddr) -{ - struct dma_map_ops *ops = platform_dma_get_ops(dev); - debug_dma_mapping_error(dev, daddr); - return ops->mapping_error(dev, daddr); -} - -static inline int dma_supported(struct device *dev, u64 mask) -{ - struct dma_map_ops *ops = platform_dma_get_ops(dev); - return ops->dma_supported(dev, mask); -} - -static inline int -dma_set_mask (struct device *dev, u64 mask) -{ - if (!dev->dma_mask || !dma_supported(dev, mask)) - return -EIO; - *dev->dma_mask = mask; - return 0; -} - static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) { if (!dev->dma_mask) diff --git a/arch/ia64/include/asm/io.h b/arch/ia64/include/asm/io.h index 80a7e34be009..9041bbe2b7b4 100644 --- a/arch/ia64/include/asm/io.h +++ b/arch/ia64/include/asm/io.h @@ -435,6 +435,7 @@ static inline void __iomem * ioremap_cache (unsigned long phys_addr, unsigned lo { return ioremap(phys_addr, size); } +#define ioremap_cache ioremap_cache /* diff --git a/arch/ia64/kernel/cyclone.c b/arch/ia64/kernel/cyclone.c index 4826ff957a3d..5fa3848ba224 100644 --- a/arch/ia64/kernel/cyclone.c +++ b/arch/ia64/kernel/cyclone.c @@ -4,7 +4,7 @@ #include #include #include -#include +#include /* IBM Summit (EXA) Cyclone counter code*/ #define CYCLONE_CBAR_ADDR 0xFEB00CD0 diff --git a/arch/ia64/kernel/iosapic.c b/arch/ia64/kernel/iosapic.c index bc9501e36e77..d2fae054d988 100644 --- a/arch/ia64/kernel/iosapic.c +++ b/arch/ia64/kernel/iosapic.c @@ -610,9 +610,9 @@ register_intr (unsigned int gsi, int irq, unsigned char delivery, chip->name, irq_type->name); chip = irq_type; } - __irq_set_chip_handler_name_locked(irq, chip, trigger == IOSAPIC_EDGE ? - handle_edge_irq : handle_level_irq, - NULL); + irq_set_chip_handler_name_locked(irq_get_irq_data(irq), chip, + trigger == IOSAPIC_EDGE ? handle_edge_irq : handle_level_irq, + NULL); return 0; } @@ -838,7 +838,7 @@ iosapic_unregister_intr (unsigned int gsi) if (iosapic_intr_info[irq].count == 0) { #ifdef CONFIG_SMP /* Clear affinity */ - cpumask_setall(irq_get_irq_data(irq)->affinity); + cpumask_setall(irq_get_affinity_mask(irq)); #endif /* Clear the interrupt information */ iosapic_intr_info[irq].dest = 0; diff --git a/arch/ia64/kernel/irq.c b/arch/ia64/kernel/irq.c index 812a1e6b3179..de4fc00dea98 100644 --- a/arch/ia64/kernel/irq.c +++ b/arch/ia64/kernel/irq.c @@ -67,7 +67,7 @@ static char irq_redir [NR_IRQS]; // = { [0 ... NR_IRQS-1] = 1 }; void set_irq_affinity_info (unsigned int irq, int hwid, int redir) { if (irq < NR_IRQS) { - cpumask_copy(irq_get_irq_data(irq)->affinity, + cpumask_copy(irq_get_affinity_mask(irq), cpumask_of(cpu_logical_id(hwid))); irq_redir[irq] = (char) (redir & 0xff); } @@ -119,8 +119,8 @@ static void migrate_irqs(void) if (irqd_is_per_cpu(data)) continue; - if (cpumask_any_and(data->affinity, cpu_online_mask) - >= nr_cpu_ids) { + if (cpumask_any_and(irq_data_get_affinity_mask(data), + cpu_online_mask) >= nr_cpu_ids) { /* * Save it for phase 2 processing */ diff --git a/arch/ia64/kernel/msi_ia64.c b/arch/ia64/kernel/msi_ia64.c index d70bf15c690a..af4eaec0f7c3 100644 --- a/arch/ia64/kernel/msi_ia64.c +++ b/arch/ia64/kernel/msi_ia64.c @@ -23,7 +23,7 @@ static int ia64_set_msi_irq_affinity(struct irq_data *idata, if (irq_prepare_move(irq, cpu)) return -1; - __get_cached_msi_msg(idata->msi_desc, &msg); + __get_cached_msi_msg(irq_data_get_msi_desc(idata), &msg); addr = msg.address_lo; addr &= MSI_ADDR_DEST_ID_MASK; @@ -36,7 +36,7 @@ static int ia64_set_msi_irq_affinity(struct irq_data *idata, msg.data = data; pci_write_msi_msg(irq, &msg); - cpumask_copy(idata->affinity, cpumask_of(cpu)); + cpumask_copy(irq_data_get_affinity_mask(idata), cpumask_of(cpu)); return 0; } @@ -148,7 +148,7 @@ static int dmar_msi_set_affinity(struct irq_data *data, msg.address_lo |= MSI_ADDR_DEST_ID_CPU(cpu_physical_id(cpu)); dmar_msi_write(irq, &msg); - cpumask_copy(data->affinity, mask); + cpumask_copy(irq_data_get_affinity_mask(data), mask); return 0; } diff --git a/arch/ia64/kernel/uncached.c b/arch/ia64/kernel/uncached.c index 20e8a9b21d75..f3976da36721 100644 --- a/arch/ia64/kernel/uncached.c +++ b/arch/ia64/kernel/uncached.c @@ -97,7 +97,7 @@ static int uncached_add_chunk(struct uncached_pool *uc_pool, int nid) /* attempt to allocate a granule's worth of cached memory pages */ - page = alloc_pages_exact_node(nid, + page = __alloc_pages_node(nid, GFP_KERNEL | __GFP_ZERO | __GFP_THISNODE, IA64_GRANULE_SHIFT-PAGE_SHIFT); if (!page) { diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c index 97e48b0eefc7..1841ef69183d 100644 --- a/arch/ia64/mm/init.c +++ b/arch/ia64/mm/init.c @@ -645,7 +645,7 @@ mem_init (void) } #ifdef CONFIG_MEMORY_HOTPLUG -int arch_add_memory(int nid, u64 start, u64 size) +int arch_add_memory(int nid, u64 start, u64 size, bool for_device) { pg_data_t *pgdat; struct zone *zone; @@ -656,7 +656,7 @@ int arch_add_memory(int nid, u64 start, u64 size) pgdat = NODE_DATA(nid); zone = pgdat->node_zones + - zone_for_memory(nid, start, size, ZONE_NORMAL); + zone_for_memory(nid, start, size, ZONE_NORMAL, for_device); ret = __add_pages(nid, zone, start_pfn, nr_pages); if (ret) diff --git a/arch/ia64/mm/tlb.c b/arch/ia64/mm/tlb.c index ed6129768681..46ecc5d948aa 100644 --- a/arch/ia64/mm/tlb.c +++ b/arch/ia64/mm/tlb.c @@ -444,7 +444,7 @@ int ia64_itr_entry(u64 target_mask, u64 va, u64 pte, u64 log_size) if (p->pte & 0x1) if (is_tr_overlap(p, va, log_size)) { printk(KERN_DEBUG "Overlapped Entry" - "Inserted for TR Reigster!!\n"); + "Inserted for TR Register!!\n"); goto out; } } @@ -456,7 +456,7 @@ int ia64_itr_entry(u64 target_mask, u64 va, u64 pte, u64 log_size) if (p->pte & 0x1) if (is_tr_overlap(p, va, log_size)) { printk(KERN_DEBUG "Overlapped Entry" - "Inserted for TR Reigster!!\n"); + "Inserted for TR Register!!\n"); goto out; } } diff --git a/arch/ia64/sn/kernel/msi_sn.c b/arch/ia64/sn/kernel/msi_sn.c index a0eb27b66d13..fb25065b22c6 100644 --- a/arch/ia64/sn/kernel/msi_sn.c +++ b/arch/ia64/sn/kernel/msi_sn.c @@ -175,7 +175,7 @@ static int sn_set_msi_irq_affinity(struct irq_data *data, * Release XIO resources for the old MSI PCI address */ - __get_cached_msi_msg(data->msi_desc, &msg); + __get_cached_msi_msg(irq_data_get_msi_desc(data), &msg); sn_pdev = (struct pcidev_info *)sn_irq_info->irq_pciioinfo; pdev = sn_pdev->pdi_linux_pcidev; provider = SN_PCIDEV_BUSPROVIDER(pdev); @@ -206,7 +206,7 @@ static int sn_set_msi_irq_affinity(struct irq_data *data, msg.address_lo = (u32)(bus_addr & 0x00000000ffffffff); pci_write_msi_msg(irq, &msg); - cpumask_copy(data->affinity, cpu_mask); + cpumask_copy(irq_data_get_affinity_mask(data), cpu_mask); return 0; } diff --git a/arch/ia64/sn/pci/pci_dma.c b/arch/ia64/sn/pci/pci_dma.c index d0853e8e8623..8f59907007cb 100644 --- a/arch/ia64/sn/pci/pci_dma.c +++ b/arch/ia64/sn/pci/pci_dma.c @@ -92,7 +92,7 @@ static void *sn_dma_alloc_coherent(struct device *dev, size_t size, */ node = pcibus_to_node(pdev->bus); if (likely(node >=0)) { - struct page *p = alloc_pages_exact_node(node, + struct page *p = __alloc_pages_node(node, flags, get_order(size)); if (likely(p)) diff --git a/arch/m32r/boot/compressed/misc.c b/arch/m32r/boot/compressed/misc.c index 28a09529f206..3a7692745868 100644 --- a/arch/m32r/boot/compressed/misc.c +++ b/arch/m32r/boot/compressed/misc.c @@ -86,6 +86,7 @@ decompress_kernel(int mmu_on, unsigned char *zimage_data, free_mem_end_ptr = free_mem_ptr + BOOT_HEAP_SIZE; puts("\nDecompressing Linux... "); - decompress(input_data, input_len, NULL, NULL, output_data, NULL, error); + __decompress(input_data, input_len, NULL, NULL, output_data, 0, + NULL, error); puts("done.\nBooting the kernel.\n"); } diff --git a/arch/m32r/include/asm/atomic.h b/arch/m32r/include/asm/atomic.h index 31bb74adba08..025e2a170493 100644 --- a/arch/m32r/include/asm/atomic.h +++ b/arch/m32r/include/asm/atomic.h @@ -94,6 +94,10 @@ static __inline__ int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -239,45 +243,4 @@ static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) return c; } - -static __inline__ void atomic_clear_mask(unsigned long mask, atomic_t *addr) -{ - unsigned long flags; - unsigned long tmp; - - local_irq_save(flags); - __asm__ __volatile__ ( - "# atomic_clear_mask \n\t" - DCACHE_CLEAR("%0", "r5", "%1") - M32R_LOCK" %0, @%1; \n\t" - "and %0, %2; \n\t" - M32R_UNLOCK" %0, @%1; \n\t" - : "=&r" (tmp) - : "r" (addr), "r" (~mask) - : "memory" - __ATOMIC_CLOBBER - ); - local_irq_restore(flags); -} - -static __inline__ void atomic_set_mask(unsigned long mask, atomic_t *addr) -{ - unsigned long flags; - unsigned long tmp; - - local_irq_save(flags); - __asm__ __volatile__ ( - "# atomic_set_mask \n\t" - DCACHE_CLEAR("%0", "r5", "%1") - M32R_LOCK" %0, @%1; \n\t" - "or %0, %2; \n\t" - M32R_UNLOCK" %0, @%1; \n\t" - : "=&r" (tmp) - : "r" (addr), "r" (mask) - : "memory" - __ATOMIC_CLOBBER - ); - local_irq_restore(flags); -} - #endif /* _ASM_M32R_ATOMIC_H */ diff --git a/arch/m32r/kernel/smp.c b/arch/m32r/kernel/smp.c index c18ddc74ef9a..62d6961e7f2b 100644 --- a/arch/m32r/kernel/smp.c +++ b/arch/m32r/kernel/smp.c @@ -156,7 +156,7 @@ void smp_flush_cache_all(void) cpumask_clear_cpu(smp_processor_id(), &cpumask); spin_lock(&flushcache_lock); mask=cpumask_bits(&cpumask); - atomic_set_mask(*mask, (atomic_t *)&flushcache_cpumask); + atomic_or(*mask, (atomic_t *)&flushcache_cpumask); send_IPI_mask(&cpumask, INVALIDATE_CACHE_IPI, 0); _flush_cache_copyback_all(); while (flushcache_cpumask) @@ -407,7 +407,7 @@ static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, flush_vma = vma; flush_va = va; mask=cpumask_bits(&cpumask); - atomic_set_mask(*mask, (atomic_t *)&flush_cpumask); + atomic_or(*mask, (atomic_t *)&flush_cpumask); /* * We have to send the IPI only to diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig index 2dd8f63bfbbb..498b567f007b 100644 --- a/arch/m68k/Kconfig +++ b/arch/m68k/Kconfig @@ -95,6 +95,7 @@ config MMU_SUN3 config KEXEC bool "kexec system call" depends on M68KCLASSIC + select KEXEC_CORE help kexec is a system call that implements the ability to shutdown your current kernel, and to start another kernel. It is like a reboot diff --git a/arch/m68k/coldfire/intc-5272.c b/arch/m68k/coldfire/intc-5272.c index d1e2fbad327c..47371de60427 100644 --- a/arch/m68k/coldfire/intc-5272.c +++ b/arch/m68k/coldfire/intc-5272.c @@ -143,8 +143,10 @@ static int intc_irq_set_type(struct irq_data *d, unsigned int type) * We need to be careful with the masking/acking due to the side effects * of masking an interrupt. */ -static void intc_external_irq(unsigned int irq, struct irq_desc *desc) +static void intc_external_irq(unsigned int __irq, struct irq_desc *desc) { + unsigned int irq = irq_desc_get_irq(desc); + irq_desc_get_chip(desc)->irq_ack(&desc->irq_data); handle_simple_irq(irq, desc); } diff --git a/arch/m68k/coldfire/m5272.c b/arch/m68k/coldfire/m5272.c index b15219ed22bf..c525e4c08f84 100644 --- a/arch/m68k/coldfire/m5272.c +++ b/arch/m68k/coldfire/m5272.c @@ -126,7 +126,7 @@ static struct fixed_phy_status nettel_fixed_phy_status __initdata = { static int __init init_BSP(void) { m5272_uarts_init(); - fixed_phy_add(PHY_POLL, 0, &nettel_fixed_phy_status); + fixed_phy_add(PHY_POLL, 0, &nettel_fixed_phy_status, -1); return 0; } diff --git a/arch/m68k/coldfire/m54xx.c b/arch/m68k/coldfire/m54xx.c index 075aaabd1360..f7836c6a6b60 100644 --- a/arch/m68k/coldfire/m54xx.c +++ b/arch/m68k/coldfire/m54xx.c @@ -25,6 +25,7 @@ #include #ifdef CONFIG_MMU #include +#include #endif /***************************************************************************/ @@ -91,13 +92,13 @@ static void __init mcf54xx_bootmem_alloc(void) m68k_memory[0].size = _ramend - _rambase; /* compute total pages in system */ - num_pages = (_ramend - _rambase) >> PAGE_SHIFT; + num_pages = PFN_DOWN(_ramend - _rambase); /* page numbers */ memstart = PAGE_ALIGN(_ramstart); - min_low_pfn = _rambase >> PAGE_SHIFT; - start_pfn = memstart >> PAGE_SHIFT; - max_low_pfn = _ramend >> PAGE_SHIFT; + min_low_pfn = PFN_DOWN(_rambase); + start_pfn = PFN_DOWN(memstart); + max_low_pfn = PFN_DOWN(_ramend); high_memory = (void *)_ramend; m68k_virt_to_node_shift = fls(_ramend - _rambase - 1) - 6; diff --git a/arch/m68k/coldfire/pit.c b/arch/m68k/coldfire/pit.c index 493b3111d4c1..d86a9ffb3f13 100644 --- a/arch/m68k/coldfire/pit.c +++ b/arch/m68k/coldfire/pit.c @@ -42,37 +42,28 @@ static u32 pit_cnt; * This is also called after resume to bring the PIT into operation again. */ -static void init_cf_pit_timer(enum clock_event_mode mode, - struct clock_event_device *evt) +static int cf_pit_set_periodic(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: + __raw_writew(MCFPIT_PCSR_DISABLE, TA(MCFPIT_PCSR)); + __raw_writew(PIT_CYCLES_PER_JIFFY, TA(MCFPIT_PMR)); + __raw_writew(MCFPIT_PCSR_EN | MCFPIT_PCSR_PIE | + MCFPIT_PCSR_OVW | MCFPIT_PCSR_RLD | + MCFPIT_PCSR_CLK64, TA(MCFPIT_PCSR)); + return 0; +} - __raw_writew(MCFPIT_PCSR_DISABLE, TA(MCFPIT_PCSR)); - __raw_writew(PIT_CYCLES_PER_JIFFY, TA(MCFPIT_PMR)); - __raw_writew(MCFPIT_PCSR_EN | MCFPIT_PCSR_PIE | \ - MCFPIT_PCSR_OVW | MCFPIT_PCSR_RLD | \ - MCFPIT_PCSR_CLK64, TA(MCFPIT_PCSR)); - break; +static int cf_pit_set_oneshot(struct clock_event_device *evt) +{ + __raw_writew(MCFPIT_PCSR_DISABLE, TA(MCFPIT_PCSR)); + __raw_writew(MCFPIT_PCSR_EN | MCFPIT_PCSR_PIE | + MCFPIT_PCSR_OVW | MCFPIT_PCSR_CLK64, TA(MCFPIT_PCSR)); + return 0; +} - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - - __raw_writew(MCFPIT_PCSR_DISABLE, TA(MCFPIT_PCSR)); - break; - - case CLOCK_EVT_MODE_ONESHOT: - - __raw_writew(MCFPIT_PCSR_DISABLE, TA(MCFPIT_PCSR)); - __raw_writew(MCFPIT_PCSR_EN | MCFPIT_PCSR_PIE | \ - MCFPIT_PCSR_OVW | MCFPIT_PCSR_CLK64, \ - TA(MCFPIT_PCSR)); - break; - - case CLOCK_EVT_MODE_RESUME: - /* Nothing to do here */ - break; - } +static int cf_pit_shutdown(struct clock_event_device *evt) +{ + __raw_writew(MCFPIT_PCSR_DISABLE, TA(MCFPIT_PCSR)); + return 0; } /* @@ -88,12 +79,15 @@ static int cf_pit_next_event(unsigned long delta, } struct clock_event_device cf_pit_clockevent = { - .name = "pit", - .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .set_mode = init_cf_pit_timer, - .set_next_event = cf_pit_next_event, - .shift = 32, - .irq = MCF_IRQ_PIT1, + .name = "pit", + .features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT, + .set_state_shutdown = cf_pit_shutdown, + .set_state_periodic = cf_pit_set_periodic, + .set_state_oneshot = cf_pit_set_oneshot, + .set_next_event = cf_pit_next_event, + .shift = 32, + .irq = MCF_IRQ_PIT1, }; diff --git a/arch/m68k/emu/nfblock.c b/arch/m68k/emu/nfblock.c index 2d75ae246167..f2a00c591bf7 100644 --- a/arch/m68k/emu/nfblock.c +++ b/arch/m68k/emu/nfblock.c @@ -76,7 +76,7 @@ static void nfhd_make_request(struct request_queue *queue, struct bio *bio) bvec_to_phys(&bvec)); sec += len; } - bio_endio(bio, 0); + bio_endio(bio); } static int nfhd_getgeo(struct block_device *bdev, struct hd_geometry *geo) diff --git a/arch/m68k/include/asm/atomic.h b/arch/m68k/include/asm/atomic.h index e85f047fb072..039fac120cc0 100644 --- a/arch/m68k/include/asm/atomic.h +++ b/arch/m68k/include/asm/atomic.h @@ -77,6 +77,10 @@ static inline int atomic_##op##_return(int i, atomic_t * v) \ ATOMIC_OPS(add, +=, add) ATOMIC_OPS(sub, -=, sub) +ATOMIC_OP(and, &=, and) +ATOMIC_OP(or, |=, or) +ATOMIC_OP(xor, ^=, eor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -170,16 +174,6 @@ static inline int atomic_add_negative(int i, atomic_t *v) return c != 0; } -static inline void atomic_clear_mask(unsigned long mask, unsigned long *v) -{ - __asm__ __volatile__("andl %1,%0" : "+m" (*v) : ASM_DI (~(mask))); -} - -static inline void atomic_set_mask(unsigned long mask, unsigned long *v) -{ - __asm__ __volatile__("orl %1,%0" : "+m" (*v) : ASM_DI (mask)); -} - static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) { int c, old; diff --git a/arch/m68k/mac/oss.c b/arch/m68k/mac/oss.c index bb11dceed7ed..191610d97689 100644 --- a/arch/m68k/mac/oss.c +++ b/arch/m68k/mac/oss.c @@ -63,13 +63,15 @@ void __init oss_nubus_init(void) * Handle miscellaneous OSS interrupts. */ -static void oss_irq(unsigned int irq, struct irq_desc *desc) +static void oss_irq(unsigned int __irq, struct irq_desc *desc) { int events = oss->irq_pending & - (OSS_IP_IOPSCC | OSS_IP_SCSI | OSS_IP_IOPISM); + (OSS_IP_IOPSCC | OSS_IP_SCSI | OSS_IP_IOPISM); #ifdef DEBUG_IRQS if ((console_loglevel == 10) && !(events & OSS_IP_SCSI)) { + unsigned int irq = irq_desc_get_irq(desc); + printk("oss_irq: irq %u events = 0x%04X\n", irq, (int) oss->irq_pending); } diff --git a/arch/m68k/mac/psc.c b/arch/m68k/mac/psc.c index 272dde481d17..3b9e302e7a37 100644 --- a/arch/m68k/mac/psc.c +++ b/arch/m68k/mac/psc.c @@ -113,9 +113,10 @@ void __init psc_init(void) * PSC interrupt handler. It's a lot like the VIA interrupt handler. */ -static void psc_irq(unsigned int irq, struct irq_desc *desc) +static void psc_irq(unsigned int __irq, struct irq_desc *desc) { unsigned int offset = (unsigned int)irq_desc_get_handler_data(desc); + unsigned int irq = irq_desc_get_irq(desc); int pIFR = pIFRbase + offset; int pIER = pIERbase + offset; int irq_num; diff --git a/arch/metag/include/asm/atomic_lnkget.h b/arch/metag/include/asm/atomic_lnkget.h index 948d8688643c..21c4c268b86c 100644 --- a/arch/metag/include/asm/atomic_lnkget.h +++ b/arch/metag/include/asm/atomic_lnkget.h @@ -74,44 +74,14 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -static inline void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - int temp; - - asm volatile ( - "1: LNKGETD %0, [%1]\n" - " AND %0, %0, %2\n" - " LNKSETD [%1] %0\n" - " DEFR %0, TXSTAT\n" - " ANDT %0, %0, #HI(0x3f000000)\n" - " CMPT %0, #HI(0x02000000)\n" - " BNZ 1b\n" - : "=&d" (temp) - : "da" (&v->counter), "bd" (~mask) - : "cc"); -} - -static inline void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - int temp; - - asm volatile ( - "1: LNKGETD %0, [%1]\n" - " OR %0, %0, %2\n" - " LNKSETD [%1], %0\n" - " DEFR %0, TXSTAT\n" - " ANDT %0, %0, #HI(0x3f000000)\n" - " CMPT %0, #HI(0x02000000)\n" - " BNZ 1b\n" - : "=&d" (temp) - : "da" (&v->counter), "bd" (mask) - : "cc"); -} - static inline int atomic_cmpxchg(atomic_t *v, int old, int new) { int result, temp; diff --git a/arch/metag/include/asm/atomic_lock1.h b/arch/metag/include/asm/atomic_lock1.h index f5d5898c1020..f8efe380fe8b 100644 --- a/arch/metag/include/asm/atomic_lock1.h +++ b/arch/metag/include/asm/atomic_lock1.h @@ -68,31 +68,14 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add, +=) ATOMIC_OPS(sub, -=) +ATOMIC_OP(and, &=) +ATOMIC_OP(or, |=) +ATOMIC_OP(xor, ^=) #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -static inline void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - unsigned long flags; - - __global_lock1(flags); - fence(); - v->counter &= ~mask; - __global_unlock1(flags); -} - -static inline void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - unsigned long flags; - - __global_lock1(flags); - fence(); - v->counter |= mask; - __global_unlock1(flags); -} - static inline int atomic_cmpxchg(atomic_t *v, int old, int new) { int ret; diff --git a/arch/metag/include/asm/barrier.h b/arch/metag/include/asm/barrier.h index 5a696e507930..172b7e5efc53 100644 --- a/arch/metag/include/asm/barrier.h +++ b/arch/metag/include/asm/barrier.h @@ -90,12 +90,12 @@ static inline void fence(void) do { \ compiletime_assert_atomic_type(*p); \ smp_mb(); \ - ACCESS_ONCE(*p) = (v); \ + WRITE_ONCE(*p, v); \ } while (0) #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ smp_mb(); \ ___p1; \ diff --git a/arch/metag/include/asm/elf.h b/arch/metag/include/asm/elf.h index d2baf6961794..87b0cf1e0acb 100644 --- a/arch/metag/include/asm/elf.h +++ b/arch/metag/include/asm/elf.h @@ -11,7 +11,7 @@ #define R_METAG_RELBRANCH 4 #define R_METAG_GETSETOFF 5 -/* Backward compatability */ +/* Backward compatibility */ #define R_METAG_REG32OP1 6 #define R_METAG_REG32OP2 7 #define R_METAG_REG32OP3 8 diff --git a/arch/metag/include/asm/ftrace.h b/arch/metag/include/asm/ftrace.h index 2901f0f7d944..a2269d60a945 100644 --- a/arch/metag/include/asm/ftrace.h +++ b/arch/metag/include/asm/ftrace.h @@ -6,7 +6,7 @@ #ifndef __ASSEMBLY__ extern void mcount_wrapper(void); -#define MCOUNT_ADDR ((long)(mcount_wrapper)) +#define MCOUNT_ADDR ((unsigned long)(mcount_wrapper)) static inline unsigned long ftrace_call_adjust(unsigned long addr) { diff --git a/arch/metag/kernel/irq.c b/arch/metag/kernel/irq.c index 4f8f1f87ef11..a336094a7a6c 100644 --- a/arch/metag/kernel/irq.c +++ b/arch/metag/kernel/irq.c @@ -270,23 +270,25 @@ void migrate_irqs(void) for_each_active_irq(i) { struct irq_data *data = irq_get_irq_data(i); + struct cpumask *mask; unsigned int newcpu; if (irqd_is_per_cpu(data)) continue; - if (!cpumask_test_cpu(cpu, data->affinity)) + mask = irq_data_get_affinity_mask(data); + if (!cpumask_test_cpu(cpu, mask)) continue; - newcpu = cpumask_any_and(data->affinity, cpu_online_mask); + newcpu = cpumask_any_and(mask, cpu_online_mask); if (newcpu >= nr_cpu_ids) { pr_info_ratelimited("IRQ%u no longer affine to CPU%u\n", i, cpu); - cpumask_setall(data->affinity); + cpumask_setall(mask); } - irq_set_affinity(i, data->affinity); + irq_set_affinity(i, mask); } } #endif /* CONFIG_HOTPLUG_CPU */ diff --git a/arch/microblaze/include/asm/dma-mapping.h b/arch/microblaze/include/asm/dma-mapping.h index ab353723076a..24b12970c9cf 100644 --- a/arch/microblaze/include/asm/dma-mapping.h +++ b/arch/microblaze/include/asm/dma-mapping.h @@ -27,7 +27,6 @@ #include #include #include -#include #include #define DMA_ERROR_CODE (~(dma_addr_t)0x0) @@ -45,31 +44,6 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev) return &dma_direct_ops; } -static inline int dma_supported(struct device *dev, u64 mask) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - - if (unlikely(!ops)) - return 0; - if (!ops->dma_supported) - return 1; - return ops->dma_supported(dev, mask); -} - -static inline int dma_set_mask(struct device *dev, u64 dma_mask) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - - if (unlikely(ops == NULL)) - return -EIO; - if (ops->set_dma_mask) - return ops->set_dma_mask(dev, dma_mask); - if (!dev->dma_mask || !dma_supported(dev, dma_mask)) - return -EIO; - *dev->dma_mask = dma_mask; - return 0; -} - #include static inline void __dma_sync(unsigned long paddr, @@ -88,50 +62,6 @@ static inline void __dma_sync(unsigned long paddr, } } -static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - - debug_dma_mapping_error(dev, dma_addr); - if (ops->mapping_error) - return ops->mapping_error(dev, dma_addr); - - return (dma_addr == DMA_ERROR_CODE); -} - -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) -#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) - -#define dma_alloc_coherent(d, s, h, f) dma_alloc_attrs(d, s, h, f, NULL) - -static inline void *dma_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - void *memory; - - BUG_ON(!ops); - - memory = ops->alloc(dev, size, dma_handle, flag, attrs); - - debug_dma_alloc_coherent(dev, size, *dma_handle, memory); - return memory; -} - -#define dma_free_coherent(d,s,c,h) dma_free_attrs(d, s, c, h, NULL) - -static inline void dma_free_attrs(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t dma_handle, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - - BUG_ON(!ops); - debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); - ops->free(dev, size, cpu_addr, dma_handle, attrs); -} - static inline void dma_cache_sync(struct device *dev, void *vaddr, size_t size, enum dma_data_direction direction) { diff --git a/arch/microblaze/include/asm/ftrace.h b/arch/microblaze/include/asm/ftrace.h index fd2fa2eca62f..da0144f40d99 100644 --- a/arch/microblaze/include/asm/ftrace.h +++ b/arch/microblaze/include/asm/ftrace.h @@ -3,7 +3,7 @@ #ifdef CONFIG_FUNCTION_TRACER -#define MCOUNT_ADDR ((long)(_mcount)) +#define MCOUNT_ADDR ((unsigned long)(_mcount)) #define MCOUNT_INSN_SIZE 8 /* sizeof mcount call */ #ifndef __ASSEMBLY__ diff --git a/arch/microblaze/include/uapi/asm/elf.h b/arch/microblaze/include/uapi/asm/elf.h index be1731d5e2fa..e9bcdb6e0086 100644 --- a/arch/microblaze/include/uapi/asm/elf.h +++ b/arch/microblaze/include/uapi/asm/elf.h @@ -11,12 +11,13 @@ #ifndef _UAPI_ASM_MICROBLAZE_ELF_H #define _UAPI_ASM_MICROBLAZE_ELF_H +#include + /* * Note there is no "official" ELF designation for Microblaze. * I've snaffled the value from the microblaze binutils source code * /binutils/microblaze/include/elf/microblaze.h */ -#define EM_MICROBLAZE 189 #define EM_MICROBLAZE_OLD 0xbaab #define ELF_ARCH EM_MICROBLAZE diff --git a/arch/microblaze/kernel/dma.c b/arch/microblaze/kernel/dma.c index bf4dec229437..c89da6312954 100644 --- a/arch/microblaze/kernel/dma.c +++ b/arch/microblaze/kernel/dma.c @@ -61,8 +61,7 @@ static int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, /* FIXME this part of code is untested */ for_each_sg(sgl, sg, nents, i) { sg->dma_address = sg_phys(sg); - __dma_sync(page_to_phys(sg_page(sg)) + sg->offset, - sg->length, direction); + __dma_sync(sg_phys(sg), sg->length, direction); } return nents; diff --git a/arch/microblaze/kernel/intc.c b/arch/microblaze/kernel/intc.c index 719feee1e043..90bec7d71f85 100644 --- a/arch/microblaze/kernel/intc.c +++ b/arch/microblaze/kernel/intc.c @@ -11,12 +11,11 @@ #include #include +#include #include #include #include -#include "../../drivers/irqchip/irqchip.h" - static void __iomem *intc_baseaddr; /* No one else should require these constants, so define them locally here. */ diff --git a/arch/microblaze/kernel/timer.c b/arch/microblaze/kernel/timer.c index c8977450e28c..67e2ef48d2d0 100644 --- a/arch/microblaze/kernel/timer.c +++ b/arch/microblaze/kernel/timer.c @@ -122,37 +122,29 @@ static int xilinx_timer_set_next_event(unsigned long delta, return 0; } -static void xilinx_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int xilinx_timer_shutdown(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - pr_info("%s: periodic\n", __func__); - xilinx_timer0_start_periodic(freq_div_hz); - break; - case CLOCK_EVT_MODE_ONESHOT: - pr_info("%s: oneshot\n", __func__); - break; - case CLOCK_EVT_MODE_UNUSED: - pr_info("%s: unused\n", __func__); - break; - case CLOCK_EVT_MODE_SHUTDOWN: - pr_info("%s: shutdown\n", __func__); - xilinx_timer0_stop(); - break; - case CLOCK_EVT_MODE_RESUME: - pr_info("%s: resume\n", __func__); - break; - } + pr_info("%s\n", __func__); + xilinx_timer0_stop(); + return 0; +} + +static int xilinx_timer_set_periodic(struct clock_event_device *evt) +{ + pr_info("%s\n", __func__); + xilinx_timer0_start_periodic(freq_div_hz); + return 0; } static struct clock_event_device clockevent_xilinx_timer = { - .name = "xilinx_clockevent", - .features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_PERIODIC, - .shift = 8, - .rating = 300, - .set_next_event = xilinx_timer_set_next_event, - .set_mode = xilinx_timer_set_mode, + .name = "xilinx_clockevent", + .features = CLOCK_EVT_FEAT_ONESHOT | + CLOCK_EVT_FEAT_PERIODIC, + .shift = 8, + .rating = 300, + .set_next_event = xilinx_timer_set_next_event, + .set_state_shutdown = xilinx_timer_shutdown, + .set_state_periodic = xilinx_timer_set_periodic, }; static inline void timer_ack(void) diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index c6d28bce0b40..e3aa5b0b4ef1 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -1,8 +1,10 @@ config MIPS bool default y + select ARCH_SUPPORTS_UPROBES select ARCH_MIGHT_HAVE_PC_PARPORT select ARCH_MIGHT_HAVE_PC_SERIO + select ARCH_USE_CMPXCHG_LOCKREF if 64BIT select HAVE_CONTEXT_TRACKING select HAVE_GENERIC_DMA_COHERENT select HAVE_IDE @@ -13,7 +15,6 @@ config MIPS select HAVE_ARCH_SECCOMP_FILTER select HAVE_ARCH_TRACEHOOK select HAVE_BPF_JIT if !CPU_MICROMIPS - select ARCH_HAVE_CUSTOM_GPIO_H select HAVE_FUNCTION_TRACER select HAVE_DYNAMIC_FTRACE select HAVE_FTRACE_MCOUNT_RECORD @@ -409,6 +410,7 @@ config MIPS_MALTA select CEVT_R4K select CSRC_R4K select CLKSRC_MIPS_GIC + select COMMON_CLK select DMA_MAYBE_COHERENT select GENERIC_ISA_DMA select HAVE_PCSPKR_PLATFORM @@ -459,6 +461,7 @@ config MIPS_SEAD3 select CEVT_R4K select CSRC_R4K select CLKSRC_MIPS_GIC + select COMMON_CLK select CPU_MIPSR2_IRQ_VI select CPU_MIPSR2_IRQ_EI select DMA_NONCOHERENT @@ -899,6 +902,7 @@ config NLM_XLP_BOARD select SYS_SUPPORTS_32BIT_KERNEL select SYS_SUPPORTS_64BIT_KERNEL select ARCH_PHYS_ADDR_T_64BIT + select ARCH_REQUIRE_GPIOLIB select SYS_SUPPORTS_BIG_ENDIAN select SYS_SUPPORTS_LITTLE_ENDIAN select SYS_SUPPORTS_HIGHMEM @@ -948,6 +952,7 @@ source "arch/mips/jazz/Kconfig" source "arch/mips/jz4740/Kconfig" source "arch/mips/lantiq/Kconfig" source "arch/mips/lasat/Kconfig" +source "arch/mips/pistachio/Kconfig" source "arch/mips/pmcs-msp71xx/Kconfig" source "arch/mips/ralink/Kconfig" source "arch/mips/sgi-ip27/Kconfig" @@ -1041,6 +1046,9 @@ config FW_CFE config ARCH_DMA_ADDR_T_64BIT def_bool (HIGHMEM && ARCH_PHYS_ADDR_T_64BIT) || 64BIT +config ARCH_SUPPORTS_UPROBES + bool + config DMA_MAYBE_COHERENT select DMA_NONCOHERENT bool @@ -1071,10 +1079,6 @@ config HOTPLUG_CPU config SYS_SUPPORTS_HOTPLUG_CPU bool -config I8259 - bool - select IRQ_DOMAIN - config MIPS_BONITO64 bool @@ -1368,7 +1372,7 @@ config CPU_MIPS32_R2 otherwise CPU_MIPS32_R1 is a safe bet for any MIPS32 system. config CPU_MIPS32_R6 - bool "MIPS32 Release 6 (EXPERIMENTAL)" + bool "MIPS32 Release 6" depends on SYS_HAS_CPU_MIPS32_R6 select CPU_HAS_PREFETCH select CPU_SUPPORTS_32BIT_KERNEL @@ -1419,7 +1423,7 @@ config CPU_MIPS64_R2 otherwise CPU_MIPS64_R1 is a safe bet for any MIPS64 system. config CPU_MIPS64_R6 - bool "MIPS64 Release 6 (EXPERIMENTAL)" + bool "MIPS64 Release 6" depends on SYS_HAS_CPU_MIPS64_R6 select CPU_HAS_PREFETCH select CPU_SUPPORTS_32BIT_KERNEL @@ -1969,6 +1973,7 @@ config 32BIT select TRAD_SIGNALS help Select this option if you want to build a 32-bit kernel. + config 64BIT bool "64-bit kernel" depends on CPU_SUPPORTS_64BIT_KERNEL && SYS_SUPPORTS_64BIT_KERNEL @@ -2114,7 +2119,7 @@ config CPU_R4K_CACHE_TLB config MIPS_MT_SMP bool "MIPS MT SMP support (1 TC on each available VPE)" - depends on SYS_SUPPORTS_MULTITHREADING + depends on SYS_SUPPORTS_MULTITHREADING && !CPU_MIPSR6 select CPU_MIPSR2_IRQ_VI select CPU_MIPSR2_IRQ_EI select SYNC_R4K @@ -2215,7 +2220,7 @@ config MIPS_VPE_APSP_API_MT config MIPS_CMP bool "MIPS CMP framework support (DEPRECATED)" - depends on SYS_SUPPORTS_MIPS_CMP + depends on SYS_SUPPORTS_MIPS_CMP && !CPU_MIPSR6 select MIPS_GIC_IPI select SMP select SYNC_R4K @@ -2232,7 +2237,7 @@ config MIPS_CMP config MIPS_CPS bool "MIPS Coherent Processing System support" - depends on SYS_SUPPORTS_MIPS_CPS + depends on SYS_SUPPORTS_MIPS_CPS && !CPU_MIPSR6 select MIPS_CM select MIPS_CPC select MIPS_CPS_PM if HOTPLUG_CPU @@ -2307,7 +2312,7 @@ config CPU_MICROMIPS endchoice config CPU_HAS_MSA - bool "Support for the MIPS SIMD Architecture (EXPERIMENTAL)" + bool "Support for the MIPS SIMD Architecture" depends on CPU_SUPPORTS_MSA depends on 64BIT || MIPS_O32_FP64_SUPPORT help @@ -2592,6 +2597,7 @@ source "kernel/Kconfig.preempt" config KEXEC bool "Kexec system call" + select KEXEC_CORE help kexec is a system call that implements the ability to shutdown your current kernel, and to start another kernel. It is like a reboot @@ -2647,7 +2653,7 @@ config SECCOMP If unsure, say Y. Only embedded should say N here. config MIPS_O32_FP64_SUPPORT - bool "Support for O32 binaries using 64-bit FP (EXPERIMENTAL)" + bool "Support for O32 binaries using 64-bit FP" depends on 32BIT || MIPS32_O32 help When this is enabled, the kernel will support use of 64-bit floating diff --git a/arch/mips/Kconfig.debug b/arch/mips/Kconfig.debug index 3a2b775e8458..e250524021ac 100644 --- a/arch/mips/Kconfig.debug +++ b/arch/mips/Kconfig.debug @@ -87,15 +87,6 @@ config SB1XXX_CORELIS Select compile flags that produce code that can be processed by the Corelis mksym utility and UDB Emulator. -config RUNTIME_DEBUG - bool "Enable run-time debugging" - depends on DEBUG_KERNEL - help - If you say Y here, some debugging macros will do run-time checking. - If you say N here, those macros will mostly turn to no-ops. See - arch/mips/include/asm/debug.h for debugging macros. - If unsure, say N. - config DEBUG_ZBOOT bool "Enable compressed kernel support debugging" depends on DEBUG_KERNEL && SYS_SUPPORTS_ZBOOT diff --git a/arch/mips/alchemy/Kconfig b/arch/mips/alchemy/Kconfig index b9628983d620..7fa24881b708 100644 --- a/arch/mips/alchemy/Kconfig +++ b/arch/mips/alchemy/Kconfig @@ -6,13 +6,6 @@ config ALCHEMY_GPIOINT_AU1000 config ALCHEMY_GPIOINT_AU1300 bool -# select this in your board config if you don't want to use the gpio -# namespace as documented in the manuals. In this case however you need -# to create the necessary gpio_* functions in your board code/headers! -# see arch/mips/include/asm/mach-au1x00/gpio.h for more information. -config ALCHEMY_GPIO_INDIRECT - def_bool n - choice prompt "Machine type" depends on MIPS_ALCHEMY diff --git a/arch/mips/alchemy/board-gpr.c b/arch/mips/alchemy/board-gpr.c index acf9a2a37f5a..79efe4c6e636 100644 --- a/arch/mips/alchemy/board-gpr.c +++ b/arch/mips/alchemy/board-gpr.c @@ -34,6 +34,7 @@ #include #include #include +#include #include const char *get_system_type(void) diff --git a/arch/mips/alchemy/board-mtx1.c b/arch/mips/alchemy/board-mtx1.c index 1e3b102389ef..85bb75669b0d 100644 --- a/arch/mips/alchemy/board-mtx1.c +++ b/arch/mips/alchemy/board-mtx1.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include diff --git a/arch/mips/alchemy/common/Makefile b/arch/mips/alchemy/common/Makefile index f64744f3b59f..23800b8e67e5 100644 --- a/arch/mips/alchemy/common/Makefile +++ b/arch/mips/alchemy/common/Makefile @@ -5,10 +5,5 @@ # Makefile for the Alchemy Au1xx0 CPUs, generic files. # -obj-y += prom.o time.o clock.o platform.o power.o \ +obj-y += prom.o time.o clock.o platform.o power.o gpiolib.o \ setup.o sleeper.o dma.o dbdma.o vss.o irq.o usb.o - -# optional gpiolib support -ifeq ($(CONFIG_ALCHEMY_GPIO_INDIRECT),) - obj-$(CONFIG_GPIOLIB) += gpiolib.o -endif diff --git a/arch/mips/alchemy/common/irq.c b/arch/mips/alchemy/common/irq.c index 6cb60abfdcc9..4c496c50edf6 100644 --- a/arch/mips/alchemy/common/irq.c +++ b/arch/mips/alchemy/common/irq.c @@ -491,7 +491,7 @@ static int au1x_ic_settype(struct irq_data *d, unsigned int flow_type) default: ret = -EINVAL; } - __irq_set_chip_handler_name_locked(d->irq, chip, handler, name); + irq_set_chip_handler_name_locked(d, chip, handler, name); wmb(); @@ -703,7 +703,7 @@ static int au1300_gpic_settype(struct irq_data *d, unsigned int type) return -EINVAL; } - __irq_set_chip_handler_name_locked(d->irq, &au1300_gpic, hdl, name); + irq_set_chip_handler_name_locked(d, &au1300_gpic, hdl, name); au1300_gpic_chgcfg(d->irq - ALCHEMY_GPIC_INT_BASE, GPIC_CFG_IC_MASK, s); diff --git a/arch/mips/alchemy/common/time.c b/arch/mips/alchemy/common/time.c index 50e17e13c18b..f99d3ec17a45 100644 --- a/arch/mips/alchemy/common/time.c +++ b/arch/mips/alchemy/common/time.c @@ -69,11 +69,6 @@ static int au1x_rtcmatch2_set_next_event(unsigned long delta, return 0; } -static void au1x_rtcmatch2_set_mode(enum clock_event_mode mode, - struct clock_event_device *cd) -{ -} - static irqreturn_t au1x_rtcmatch2_irq(int irq, void *dev_id) { struct clock_event_device *cd = dev_id; @@ -86,7 +81,6 @@ static struct clock_event_device au1x_rtcmatch2_clockdev = { .features = CLOCK_EVT_FEAT_ONESHOT, .rating = 1500, .set_next_event = au1x_rtcmatch2_set_next_event, - .set_mode = au1x_rtcmatch2_set_mode, .cpumask = cpu_all_mask, }; diff --git a/arch/mips/alchemy/devboards/bcsr.c b/arch/mips/alchemy/devboards/bcsr.c index c98c9ea3372c..324ad72d7c36 100644 --- a/arch/mips/alchemy/devboards/bcsr.c +++ b/arch/mips/alchemy/devboards/bcsr.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include @@ -88,10 +89,11 @@ EXPORT_SYMBOL_GPL(bcsr_mod); static void bcsr_csc_handler(unsigned int irq, struct irq_desc *d) { unsigned short bisr = __raw_readw(bcsr_virt + BCSR_REG_INTSTAT); + struct irq_chip *chip = irq_desc_get_chip(d); - disable_irq_nosync(irq); + chained_irq_enter(chip, d); generic_handle_irq(bcsr_csc_base + __ffs(bisr)); - enable_irq(irq); + chained_irq_exit(chip, d); } static void bcsr_irq_mask(struct irq_data *d) diff --git a/arch/mips/alchemy/devboards/db1000.c b/arch/mips/alchemy/devboards/db1000.c index 001102e197f1..bdeed9d13c6f 100644 --- a/arch/mips/alchemy/devboards/db1000.c +++ b/arch/mips/alchemy/devboards/db1000.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/mips/alchemy/devboards/db1300.c b/arch/mips/alchemy/devboards/db1300.c index 1c64fdbe4c81..b58077008a53 100644 --- a/arch/mips/alchemy/devboards/db1300.c +++ b/arch/mips/alchemy/devboards/db1300.c @@ -24,6 +24,7 @@ #include #include +#include #include #include #include diff --git a/arch/mips/alchemy/devboards/db1550.c b/arch/mips/alchemy/devboards/db1550.c index 0fd5177e35ab..5740bcfdfc7f 100644 --- a/arch/mips/alchemy/devboards/db1550.c +++ b/arch/mips/alchemy/devboards/db1550.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/mips/alchemy/devboards/pm.c b/arch/mips/alchemy/devboards/pm.c index bfeb8f3c0be6..93024dc6b314 100644 --- a/arch/mips/alchemy/devboards/pm.c +++ b/arch/mips/alchemy/devboards/pm.c @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include /* diff --git a/arch/mips/ar7/gpio.c b/arch/mips/ar7/gpio.c index d8dbd8f0c1d2..f4930456eb8e 100644 --- a/arch/mips/ar7/gpio.c +++ b/arch/mips/ar7/gpio.c @@ -21,7 +21,10 @@ #include #include -#include +#include + +#define AR7_GPIO_MAX 32 +#define TITAN_GPIO_MAX 51 struct ar7_gpio_chip { void __iomem *regs; diff --git a/arch/mips/ar7/platform.c b/arch/mips/ar7/platform.c index be9ff1673ded..58fca9ad5fcc 100644 --- a/arch/mips/ar7/platform.c +++ b/arch/mips/ar7/platform.c @@ -39,7 +39,6 @@ #include #include -#include #include /***************************************************************************** @@ -679,7 +678,8 @@ static int __init ar7_register_devices(void) } if (ar7_has_high_cpmac()) { - res = fixed_phy_add(PHY_POLL, cpmac_high.id, &fixed_phy_status); + res = fixed_phy_add(PHY_POLL, cpmac_high.id, + &fixed_phy_status, -1); if (!res) { cpmac_get_mac(1, cpmac_high_data.dev_addr); @@ -692,7 +692,7 @@ static int __init ar7_register_devices(void) } else cpmac_low_data.phy_mask = 0xffffffff; - res = fixed_phy_add(PHY_POLL, cpmac_low.id, &fixed_phy_status); + res = fixed_phy_add(PHY_POLL, cpmac_low.id, &fixed_phy_status, -1); if (!res) { cpmac_get_mac(0, cpmac_low_data.dev_addr); res = platform_device_register(&cpmac_low); diff --git a/arch/mips/ar7/setup.c b/arch/mips/ar7/setup.c index 820b7a313d9b..7bb9a670bb73 100644 --- a/arch/mips/ar7/setup.c +++ b/arch/mips/ar7/setup.c @@ -23,7 +23,6 @@ #include #include #include -#include static void ar7_machine_restart(char *command) { diff --git a/arch/mips/ath79/Makefile b/arch/mips/ath79/Makefile index 5c9ff692ff3c..fcc382cfc770 100644 --- a/arch/mips/ath79/Makefile +++ b/arch/mips/ath79/Makefile @@ -8,7 +8,7 @@ # under the terms of the GNU General Public License version 2 as published # by the Free Software Foundation. -obj-y := prom.o setup.o irq.o common.o clock.o gpio.o +obj-y := prom.o setup.o irq.o common.o clock.o obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-$(CONFIG_PCI) += pci.o diff --git a/arch/mips/ath79/common.h b/arch/mips/ath79/common.h index e5ea71277f0c..ca7cc19adfea 100644 --- a/arch/mips/ath79/common.h +++ b/arch/mips/ath79/common.h @@ -25,9 +25,6 @@ unsigned long ath79_get_sys_clk_rate(const char *id); void ath79_ddr_ctrl_init(void); void ath79_ddr_wb_flush(unsigned int reg); -void ath79_gpio_function_enable(u32 mask); -void ath79_gpio_function_disable(u32 mask); -void ath79_gpio_function_setup(u32 set, u32 clear); void ath79_gpio_init(void); #endif /* __ATH79_COMMON_H */ diff --git a/arch/mips/ath79/irq.c b/arch/mips/ath79/irq.c index afb009603f7f..807132b838b2 100644 --- a/arch/mips/ath79/irq.c +++ b/arch/mips/ath79/irq.c @@ -17,7 +17,6 @@ #include #include #include -#include "../../../drivers/irqchip/irqchip.h" #include #include @@ -124,8 +123,6 @@ static void ar934x_ip2_irq_dispatch(unsigned int irq, struct irq_desc *desc) { u32 status; - disable_irq_nosync(irq); - status = ath79_reset_rr(AR934X_RESET_REG_PCIE_WMAC_INT_STATUS); if (status & AR934X_PCIE_WMAC_INT_PCIE_ALL) { @@ -137,8 +134,6 @@ static void ar934x_ip2_irq_dispatch(unsigned int irq, struct irq_desc *desc) } else { spurious_interrupt(); } - - enable_irq(irq); } static void ar934x_ip2_irq_init(void) @@ -157,14 +152,12 @@ static void qca955x_ip2_irq_dispatch(unsigned int irq, struct irq_desc *desc) { u32 status; - disable_irq_nosync(irq); - status = ath79_reset_rr(QCA955X_RESET_REG_EXT_INT_STATUS); status &= QCA955X_EXT_INT_PCIE_RC1_ALL | QCA955X_EXT_INT_WMAC_ALL; if (status == 0) { spurious_interrupt(); - goto enable; + return; } if (status & QCA955X_EXT_INT_PCIE_RC1_ALL) { @@ -176,17 +169,12 @@ static void qca955x_ip2_irq_dispatch(unsigned int irq, struct irq_desc *desc) /* TODO: flush DDR? */ generic_handle_irq(ATH79_IP2_IRQ(1)); } - -enable: - enable_irq(irq); } static void qca955x_ip3_irq_dispatch(unsigned int irq, struct irq_desc *desc) { u32 status; - disable_irq_nosync(irq); - status = ath79_reset_rr(QCA955X_RESET_REG_EXT_INT_STATUS); status &= QCA955X_EXT_INT_PCIE_RC2_ALL | QCA955X_EXT_INT_USB1 | @@ -194,7 +182,7 @@ static void qca955x_ip3_irq_dispatch(unsigned int irq, struct irq_desc *desc) if (status == 0) { spurious_interrupt(); - goto enable; + return; } if (status & QCA955X_EXT_INT_USB1) { @@ -211,9 +199,6 @@ static void qca955x_ip3_irq_dispatch(unsigned int irq, struct irq_desc *desc) /* TODO: flush DDR? */ generic_handle_irq(ATH79_IP3_IRQ(2)); } - -enable: - enable_irq(irq); } static void qca955x_irq_init(void) diff --git a/arch/mips/bcm47xx/buttons.c b/arch/mips/bcm47xx/buttons.c index 08a4abf09a33..52caa75bfe4e 100644 --- a/arch/mips/bcm47xx/buttons.c +++ b/arch/mips/bcm47xx/buttons.c @@ -396,10 +396,9 @@ static int __init bcm47xx_buttons_copy(const struct gpio_keys_button *buttons, { size_t size = nbuttons * sizeof(*buttons); - bcm47xx_button_pdata.buttons = kmalloc(size, GFP_KERNEL); + bcm47xx_button_pdata.buttons = kmemdup(buttons, size, GFP_KERNEL); if (!bcm47xx_button_pdata.buttons) return -ENOMEM; - memcpy(bcm47xx_button_pdata.buttons, buttons, size); bcm47xx_button_pdata.nbuttons = nbuttons; return 0; diff --git a/arch/mips/bcm47xx/setup.c b/arch/mips/bcm47xx/setup.c index 98c075f81795..17503a05938e 100644 --- a/arch/mips/bcm47xx/setup.c +++ b/arch/mips/bcm47xx/setup.c @@ -263,7 +263,7 @@ static int __init bcm47xx_register_bus_complete(void) bcm47xx_leds_register(); bcm47xx_workarounds(); - fixed_phy_add(PHY_POLL, 0, &bcm47xx_fixed_phy_status); + fixed_phy_add(PHY_POLL, 0, &bcm47xx_fixed_phy_status, -1); return 0; } device_initcall(bcm47xx_register_bus_complete); diff --git a/arch/mips/bcm63xx/irq.c b/arch/mips/bcm63xx/irq.c index e3e808a6c542..1a47ec2a0906 100644 --- a/arch/mips/bcm63xx/irq.c +++ b/arch/mips/bcm63xx/irq.c @@ -60,7 +60,7 @@ static inline int enable_irq_for_cpu(int cpu, struct irq_data *d, if (m) enable &= cpumask_test_cpu(cpu, m); else if (irqd_affinity_was_set(d)) - enable &= cpumask_test_cpu(cpu, d->affinity); + enable &= cpumask_test_cpu(cpu, irq_data_get_affinity_mask(d)); #endif return enable; } @@ -365,9 +365,9 @@ static int bcm63xx_external_irq_set_type(struct irq_data *d, irqd_set_trigger_type(d, flow_type); if (flow_type & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_LEVEL_HIGH)) - __irq_set_handler_locked(d->irq, handle_level_irq); + irq_set_handler_locked(d, handle_level_irq); else - __irq_set_handler_locked(d->irq, handle_edge_irq); + irq_set_handler_locked(d, handle_edge_irq); return IRQ_SET_MASK_OK_NOCOPY; } diff --git a/arch/mips/bmips/irq.c b/arch/mips/bmips/irq.c index 14552e58ff7e..e7fc6f9348ba 100644 --- a/arch/mips/bmips/irq.c +++ b/arch/mips/bmips/irq.c @@ -34,5 +34,5 @@ void __init arch_init_irq(void) irqchip_init(); } -OF_DECLARE_2(irqchip, mips_cpu_intc, "mti,cpu-interrupt-controller", +IRQCHIP_DECLARE(mips_cpu_intc, "mti,cpu-interrupt-controller", mips_cpu_irq_of_init); diff --git a/arch/mips/boot/compressed/Makefile b/arch/mips/boot/compressed/Makefile index dc91bde10d62..d5bdee115f22 100644 --- a/arch/mips/boot/compressed/Makefile +++ b/arch/mips/boot/compressed/Makefile @@ -78,7 +78,7 @@ endif vmlinuzobjs-y += $(obj)/piggy.o -quiet_cmd_zld = LD $@ +quiet_cmd_zld = LD $@ cmd_zld = $(LD) $(LDFLAGS) -Ttext $(VMLINUZ_LOAD_ADDRESS) -T $< $(vmlinuzobjs-y) -o $@ quiet_cmd_strip = STRIP $@ cmd_strip = $(STRIP) -s $@ diff --git a/arch/mips/boot/compressed/decompress.c b/arch/mips/boot/compressed/decompress.c index 54831069a206..080cd53bac36 100644 --- a/arch/mips/boot/compressed/decompress.c +++ b/arch/mips/boot/compressed/decompress.c @@ -111,8 +111,8 @@ void decompress_kernel(unsigned long boot_heap_start) puts("\n"); /* Decompress the kernel with according algorithm */ - decompress((char *)zimage_start, zimage_size, 0, 0, - (void *)VMLINUX_LOAD_ADDRESS_ULL, 0, error); + __decompress((char *)zimage_start, zimage_size, 0, 0, + (void *)VMLINUX_LOAD_ADDRESS_ULL, 0, 0, error); /* FIXME: should we flush cache here? */ puts("Now, booting the kernel...\n"); diff --git a/arch/mips/boot/dts/netlogic/xlp_evp.dts b/arch/mips/boot/dts/netlogic/xlp_evp.dts index 89ad04808c02..ec16ec2d8d02 100644 --- a/arch/mips/boot/dts/netlogic/xlp_evp.dts +++ b/arch/mips/boot/dts/netlogic/xlp_evp.dts @@ -110,6 +110,18 @@ partition@f00000 { read-only; }; }; + + gpio: xlp_gpio@34100 { + compatible = "netlogic,xlp832-gpio"; + reg = <0 0x34100 0x1000>; + #gpio-cells = <2>; + gpio-controller; + + #interrupt-cells = <2>; + interrupt-parent = <&pic>; + interrupts = <39>; + interrupt-controller; + }; }; chosen { diff --git a/arch/mips/boot/dts/netlogic/xlp_fvp.dts b/arch/mips/boot/dts/netlogic/xlp_fvp.dts index 63e62b7bd758..4bcebe641d8e 100644 --- a/arch/mips/boot/dts/netlogic/xlp_fvp.dts +++ b/arch/mips/boot/dts/netlogic/xlp_fvp.dts @@ -110,6 +110,18 @@ partition@f00000 { read-only; }; }; + + gpio: xlp_gpio@34100 { + compatible = "netlogic,xlp208-gpio"; + reg = <0 0x34100 0x1000>; + #gpio-cells = <2>; + gpio-controller; + + #interrupt-cells = <2>; + interrupt-parent = <&pic>; + interrupts = <39>; + interrupt-controller; + }; }; chosen { diff --git a/arch/mips/boot/dts/netlogic/xlp_gvp.dts b/arch/mips/boot/dts/netlogic/xlp_gvp.dts index bb4ecd1d47fc..b3ccb82ad7e4 100644 --- a/arch/mips/boot/dts/netlogic/xlp_gvp.dts +++ b/arch/mips/boot/dts/netlogic/xlp_gvp.dts @@ -69,6 +69,17 @@ partition@f00000 { }; }; + gpio: xlp_gpio@114100 { + compatible = "netlogic,xlp980-gpio"; + reg = <0 0x114100 0x1000>; + #gpio-cells = <2>; + gpio-controller; + + #interrupt-cells = <2>; + interrupt-parent = <&pic>; + interrupts = <39>; + interrupt-controller; + }; }; chosen { diff --git a/arch/mips/boot/dts/netlogic/xlp_rvp.dts b/arch/mips/boot/dts/netlogic/xlp_rvp.dts index 7188aed2ea2e..3783639a318a 100644 --- a/arch/mips/boot/dts/netlogic/xlp_rvp.dts +++ b/arch/mips/boot/dts/netlogic/xlp_rvp.dts @@ -69,6 +69,17 @@ partition@f00000 { }; }; + gpio: xlp_gpio@114100 { + compatible = "netlogic,xlp532-gpio"; + reg = <0 0x114100 0x1000>; + #gpio-cells = <2>; + gpio-controller; + + #interrupt-cells = <2>; + interrupt-parent = <&pic>; + interrupts = <39>; + interrupt-controller; + }; }; chosen { diff --git a/arch/mips/boot/dts/netlogic/xlp_svp.dts b/arch/mips/boot/dts/netlogic/xlp_svp.dts index 1ebd00edaacc..44d6640c1441 100644 --- a/arch/mips/boot/dts/netlogic/xlp_svp.dts +++ b/arch/mips/boot/dts/netlogic/xlp_svp.dts @@ -110,6 +110,18 @@ partition@f00000 { read-only; }; }; + + gpio: xlp_gpio@34100 { + compatible = "netlogic,xlp316-gpio"; + reg = <0 0x34100 0x1000>; + #gpio-cells = <2>; + gpio-controller; + + #interrupt-cells = <2>; + interrupt-parent = <&pic>; + interrupts = <39>; + interrupt-controller; + }; }; chosen { diff --git a/arch/mips/cavium-octeon/dma-octeon.c b/arch/mips/cavium-octeon/dma-octeon.c index d8960d46417b..2cd45f5f9481 100644 --- a/arch/mips/cavium-octeon/dma-octeon.c +++ b/arch/mips/cavium-octeon/dma-octeon.c @@ -161,9 +161,6 @@ static void *octeon_dma_alloc_coherent(struct device *dev, size_t size, { void *ret; - if (dma_alloc_from_coherent(dev, size, dma_handle, &ret)) - return ret; - /* ignore region specifiers */ gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM); @@ -194,11 +191,6 @@ static void *octeon_dma_alloc_coherent(struct device *dev, size_t size, static void octeon_dma_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, struct dma_attrs *attrs) { - int order = get_order(size); - - if (dma_release_from_coherent(dev, order, vaddr)) - return; - swiotlb_free_coherent(dev, size, vaddr, dma_handle); } diff --git a/arch/mips/cavium-octeon/executive/cvmx-helper-board.c b/arch/mips/cavium-octeon/executive/cvmx-helper-board.c index 9eb0feef4417..36e30d65ba05 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-helper-board.c +++ b/arch/mips/cavium-octeon/executive/cvmx-helper-board.c @@ -195,6 +195,12 @@ int cvmx_helper_board_get_mii_address(int ipd_port) return 8; else return -1; + case CVMX_BOARD_TYPE_KONTRON_S1901: + if (ipd_port == CVMX_HELPER_BOARD_MGMT_IPD_PORT) + return 1; + else + return -1; + } /* Some unknown board. Somebody forgot to update this function... */ diff --git a/arch/mips/cavium-octeon/executive/cvmx-helper-util.c b/arch/mips/cavium-octeon/executive/cvmx-helper-util.c index 453d7f66459a..b45b2975746d 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-helper-util.c +++ b/arch/mips/cavium-octeon/executive/cvmx-helper-util.c @@ -95,9 +95,9 @@ int cvmx_helper_dump_packet(cvmx_wqe_t *work) uint8_t *data_address; uint8_t *end_of_data; - cvmx_dprintf("Packet Length: %u\n", work->len); - cvmx_dprintf(" Input Port: %u\n", work->ipprt); - cvmx_dprintf(" QoS: %u\n", work->qos); + cvmx_dprintf("Packet Length: %u\n", work->word1.len); + cvmx_dprintf(" Input Port: %u\n", cvmx_wqe_get_port(work)); + cvmx_dprintf(" QoS: %u\n", cvmx_wqe_get_qos(work)); cvmx_dprintf(" Buffers: %u\n", work->word2.s.bufs); if (work->word2.s.bufs == 0) { @@ -127,7 +127,7 @@ int cvmx_helper_dump_packet(cvmx_wqe_t *work) } } else buffer_ptr = work->packet_ptr; - remaining_bytes = work->len; + remaining_bytes = work->word1.len; while (remaining_bytes) { start_of_buffer = @@ -382,6 +382,10 @@ int cvmx_helper_get_ipd_port(int interface, int port) return port + 32; case 3: return port + 36; + case 4: + return port + 40; + case 5: + return port + 44; } return -1; } @@ -404,6 +408,10 @@ int cvmx_helper_get_interface_num(int ipd_port) return 2; else if (ipd_port < 40) return 3; + else if (ipd_port < 44) + return 4; + else if (ipd_port < 48) + return 5; else cvmx_dprintf("cvmx_helper_get_interface_num: Illegal IPD " "port number\n"); @@ -428,6 +436,10 @@ int cvmx_helper_get_interface_index_num(int ipd_port) return ipd_port & 3; else if (ipd_port < 40) return ipd_port & 3; + else if (ipd_port < 44) + return ipd_port & 3; + else if (ipd_port < 48) + return ipd_port & 3; else cvmx_dprintf("cvmx_helper_get_interface_index_num: " "Illegal IPD port number\n"); diff --git a/arch/mips/cavium-octeon/executive/cvmx-helper-xaui.c b/arch/mips/cavium-octeon/executive/cvmx-helper-xaui.c index 7653b7e92197..a56ee590de1f 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-helper-xaui.c +++ b/arch/mips/cavium-octeon/executive/cvmx-helper-xaui.c @@ -124,6 +124,13 @@ int __cvmx_helper_xaui_enable(int interface) union cvmx_gmxx_tx_int_en gmx_tx_int_en; union cvmx_pcsxx_int_en_reg pcsx_int_en_reg; + /* Setup PKND */ + if (octeon_has_feature(OCTEON_FEATURE_PKND)) { + gmx_cfg.u64 = cvmx_read_csr(CVMX_GMXX_PRTX_CFG(0, interface)); + gmx_cfg.s.pknd = cvmx_helper_get_ipd_port(interface, 0); + cvmx_write_csr(CVMX_GMXX_PRTX_CFG(0, interface), gmx_cfg.u64); + } + /* (1) Interface has already been enabled. */ /* (2) Disable GMX. */ @@ -151,7 +158,12 @@ int __cvmx_helper_xaui_enable(int interface) /* (4)c Aply reset sequence */ xauiCtl.u64 = cvmx_read_csr(CVMX_PCSXX_CONTROL1_REG(interface)); xauiCtl.s.lo_pwr = 0; - xauiCtl.s.reset = 1; + + /* Issuing a reset here seems to hang some CN68XX chips. */ + if (!OCTEON_IS_MODEL(OCTEON_CN68XX_PASS1_X) && + !OCTEON_IS_MODEL(OCTEON_CN68XX_PASS2_X)) + xauiCtl.s.reset = 1; + cvmx_write_csr(CVMX_PCSXX_CONTROL1_REG(interface), xauiCtl.u64); /* Wait for PCS to come out of reset */ diff --git a/arch/mips/cavium-octeon/executive/cvmx-helper.c b/arch/mips/cavium-octeon/executive/cvmx-helper.c index 7e5cf7a5e2f3..376701f41cc2 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-helper.c +++ b/arch/mips/cavium-octeon/executive/cvmx-helper.c @@ -83,6 +83,8 @@ static cvmx_helper_link_info_t */ int cvmx_helper_get_number_of_interfaces(void) { + if (OCTEON_IS_MODEL(OCTEON_CN68XX)) + return 9; if (OCTEON_IS_MODEL(OCTEON_CN56XX) || OCTEON_IS_MODEL(OCTEON_CN52XX)) return 4; else @@ -656,6 +658,21 @@ static int __cvmx_helper_global_setup_pko(void) fau_to.s.tout_val = 0xfff; fau_to.s.tout_enb = 0; cvmx_write_csr(CVMX_IOB_FAU_TIMEOUT, fau_to.u64); + + if (OCTEON_IS_MODEL(OCTEON_CN68XX)) { + union cvmx_pko_reg_min_pkt min_pkt; + + min_pkt.u64 = 0; + min_pkt.s.size1 = 59; + min_pkt.s.size2 = 59; + min_pkt.s.size3 = 59; + min_pkt.s.size4 = 59; + min_pkt.s.size5 = 59; + min_pkt.s.size6 = 59; + min_pkt.s.size7 = 59; + cvmx_write_csr(CVMX_PKO_REG_MIN_PKT, min_pkt.u64); + } + return 0; } diff --git a/arch/mips/cavium-octeon/executive/cvmx-pko.c b/arch/mips/cavium-octeon/executive/cvmx-pko.c index 008b881cdf64..87be167a7a6a 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-pko.c +++ b/arch/mips/cavium-octeon/executive/cvmx-pko.c @@ -39,6 +39,143 @@ * Internal state of packet output */ +static int __cvmx_pko_int(int interface, int index) +{ + switch (interface) { + case 0: + return index; + case 1: + return 4; + case 2: + return index + 0x08; + case 3: + return index + 0x0c; + case 4: + return index + 0x10; + case 5: + return 0x1c; + case 6: + return 0x1d; + case 7: + return 0x1e; + case 8: + return 0x1f; + default: + return -1; + } +} + +static void __cvmx_pko_iport_config(int pko_port) +{ + int queue; + const int num_queues = 1; + const int base_queue = pko_port; + const int static_priority_end = 1; + const int static_priority_base = 1; + + for (queue = 0; queue < num_queues; queue++) { + union cvmx_pko_mem_iqueue_ptrs config; + cvmx_cmd_queue_result_t cmd_res; + uint64_t *buf_ptr; + + config.u64 = 0; + config.s.index = queue; + config.s.qid = base_queue + queue; + config.s.ipid = pko_port; + config.s.tail = (queue == (num_queues - 1)); + config.s.s_tail = (queue == static_priority_end); + config.s.static_p = (static_priority_base >= 0); + config.s.static_q = (queue <= static_priority_end); + config.s.qos_mask = 0xff; + + cmd_res = cvmx_cmd_queue_initialize( + CVMX_CMD_QUEUE_PKO(base_queue + queue), + CVMX_PKO_MAX_QUEUE_DEPTH, + CVMX_FPA_OUTPUT_BUFFER_POOL, + (CVMX_FPA_OUTPUT_BUFFER_POOL_SIZE - + CVMX_PKO_COMMAND_BUFFER_SIZE_ADJUST * 8)); + + WARN(cmd_res, + "%s: cmd_res=%d pko_port=%d base_queue=%d num_queues=%d queue=%d\n", + __func__, (int)cmd_res, pko_port, base_queue, + num_queues, queue); + + buf_ptr = (uint64_t *)cvmx_cmd_queue_buffer( + CVMX_CMD_QUEUE_PKO(base_queue + queue)); + config.s.buf_ptr = cvmx_ptr_to_phys(buf_ptr) >> 7; + CVMX_SYNCWS; + cvmx_write_csr(CVMX_PKO_MEM_IQUEUE_PTRS, config.u64); + } +} + +static void __cvmx_pko_queue_alloc_o68(void) +{ + int port; + + for (port = 0; port < 48; port++) + __cvmx_pko_iport_config(port); +} + +static void __cvmx_pko_port_map_o68(void) +{ + int port; + int interface, index; + cvmx_helper_interface_mode_t mode; + union cvmx_pko_mem_iport_ptrs config; + + /* + * Initialize every iport with the invalid eid. + */ + config.u64 = 0; + config.s.eid = 31; /* Invalid */ + for (port = 0; port < 128; port++) { + config.s.ipid = port; + cvmx_write_csr(CVMX_PKO_MEM_IPORT_PTRS, config.u64); + } + + /* + * Set up PKO_MEM_IPORT_PTRS + */ + for (port = 0; port < 48; port++) { + interface = cvmx_helper_get_interface_num(port); + index = cvmx_helper_get_interface_index_num(port); + mode = cvmx_helper_interface_get_mode(interface); + if (mode == CVMX_HELPER_INTERFACE_MODE_DISABLED) + continue; + + config.s.ipid = port; + config.s.qos_mask = 0xff; + config.s.crc = 1; + config.s.min_pkt = 1; + config.s.intr = __cvmx_pko_int(interface, index); + config.s.eid = config.s.intr; + config.s.pipe = (mode == CVMX_HELPER_INTERFACE_MODE_LOOP) ? + index : port; + cvmx_write_csr(CVMX_PKO_MEM_IPORT_PTRS, config.u64); + } +} + +static void __cvmx_pko_chip_init(void) +{ + int i; + + if (OCTEON_IS_MODEL(OCTEON_CN68XX)) { + __cvmx_pko_port_map_o68(); + __cvmx_pko_queue_alloc_o68(); + return; + } + + /* + * Initialize queues + */ + for (i = 0; i < CVMX_PKO_MAX_OUTPUT_QUEUES; i++) { + const uint64_t priority = 8; + + cvmx_pko_config_port(CVMX_PKO_MEM_QUEUE_PTRS_ILLEGAL_PID, i, 1, + &priority); + } +} + /** * Call before any other calls to initialize the packet * output system. This does chip global config, and should only be @@ -47,8 +184,6 @@ void cvmx_pko_initialize_global(void) { - int i; - uint64_t priority = 8; union cvmx_pko_reg_cmd_buf config; /* @@ -62,9 +197,10 @@ void cvmx_pko_initialize_global(void) cvmx_write_csr(CVMX_PKO_REG_CMD_BUF, config.u64); - for (i = 0; i < CVMX_PKO_MAX_OUTPUT_QUEUES; i++) - cvmx_pko_config_port(CVMX_PKO_MEM_QUEUE_PTRS_ILLEGAL_PID, i, 1, - &priority); + /* + * Chip-specific setup. + */ + __cvmx_pko_chip_init(); /* * If we aren't using all of the queues optimize PKO's @@ -212,6 +348,9 @@ cvmx_pko_status_t cvmx_pko_config_port(uint64_t port, uint64_t base_queue, int static_priority_base = -1; int static_priority_end = -1; + if (OCTEON_IS_MODEL(OCTEON_CN68XX)) + return CVMX_PKO_SUCCESS; + if ((port >= CVMX_PKO_NUM_OUTPUT_PORTS) && (port != CVMX_PKO_MEM_QUEUE_PTRS_ILLEGAL_PID)) { cvmx_dprintf("ERROR: cvmx_pko_config_port: Invalid port %llu\n", diff --git a/arch/mips/cavium-octeon/octeon-irq.c b/arch/mips/cavium-octeon/octeon-irq.c index d8124a3c5a85..f26c3c661cca 100644 --- a/arch/mips/cavium-octeon/octeon-irq.c +++ b/arch/mips/cavium-octeon/octeon-irq.c @@ -225,13 +225,14 @@ static int next_cpu_for_irq(struct irq_data *data) #ifdef CONFIG_SMP int cpu; - int weight = cpumask_weight(data->affinity); + struct cpumask *mask = irq_data_get_affinity_mask(data); + int weight = cpumask_weight(mask); struct octeon_ciu_chip_data *cd = irq_data_get_irq_chip_data(data); if (weight > 1) { cpu = cd->current_cpu; for (;;) { - cpu = cpumask_next(cpu, data->affinity); + cpu = cpumask_next(cpu, mask); if (cpu >= nr_cpu_ids) { cpu = -1; continue; @@ -240,7 +241,7 @@ static int next_cpu_for_irq(struct irq_data *data) } } } else if (weight == 1) { - cpu = cpumask_first(data->affinity); + cpu = cpumask_first(mask); } else { cpu = smp_processor_id(); } @@ -662,6 +663,11 @@ static int octeon_irq_ciu_gpio_set_type(struct irq_data *data, unsigned int t) irqd_set_trigger_type(data, t); octeon_irq_gpio_setup(data); + if (irqd_get_trigger_type(data) & IRQ_TYPE_EDGE_BOTH) + irq_set_handler_locked(data, handle_edge_irq); + else + irq_set_handler_locked(data, handle_level_irq); + return IRQ_SET_MASK_OK; } @@ -696,32 +702,23 @@ static void octeon_irq_ciu_gpio_ack(struct irq_data *data) cvmx_write_csr(CVMX_GPIO_INT_CLR, mask); } -static void octeon_irq_handle_trigger(unsigned int irq, struct irq_desc *desc) -{ - struct irq_data *data = irq_desc_get_irq_data(desc); - - if (irqd_get_trigger_type(data) & IRQ_TYPE_EDGE_BOTH) - handle_edge_irq(irq, desc); - else - handle_level_irq(irq, desc); -} - #ifdef CONFIG_SMP static void octeon_irq_cpu_offline_ciu(struct irq_data *data) { int cpu = smp_processor_id(); cpumask_t new_affinity; + struct cpumask *mask = irq_data_get_affinity_mask(data); - if (!cpumask_test_cpu(cpu, data->affinity)) + if (!cpumask_test_cpu(cpu, mask)) return; - if (cpumask_weight(data->affinity) > 1) { + if (cpumask_weight(mask) > 1) { /* * It has multi CPU affinity, just remove this CPU * from the affinity set. */ - cpumask_copy(&new_affinity, data->affinity); + cpumask_copy(&new_affinity, mask); cpumask_clear_cpu(cpu, &new_affinity); } else { /* Otherwise, put it on lowest numbered online CPU. */ @@ -1227,8 +1224,13 @@ static int octeon_irq_gpio_map(struct irq_domain *d, octeon_irq_ciu_to_irq[line][bit] != 0) return -EINVAL; + /* + * Default to handle_level_irq. If the DT contains a different + * trigger type, it will call the irq_set_type callback and + * the handler gets updated. + */ r = octeon_irq_set_ciu_mapping(virq, line, bit, hw, - octeon_irq_gpio_chip, octeon_irq_handle_trigger); + octeon_irq_gpio_chip, handle_level_irq); return r; } diff --git a/arch/mips/configs/pistachio_defconfig b/arch/mips/configs/pistachio_defconfig index 1646cce032c3..642b50946943 100644 --- a/arch/mips/configs/pistachio_defconfig +++ b/arch/mips/configs/pistachio_defconfig @@ -320,7 +320,6 @@ CONFIG_KEYS=y CONFIG_SECURITY=y CONFIG_SECURITY_NETWORK=y CONFIG_SECURITY_YAMA=y -CONFIG_SECURITY_YAMA_STACKED=y CONFIG_DEFAULT_SECURITY_DAC=y CONFIG_CRYPTO_AUTHENC=y CONFIG_CRYPTO_HMAC=y diff --git a/arch/mips/include/asm/Kbuild b/arch/mips/include/asm/Kbuild index 1f8546081d20..40ec4ca3f946 100644 --- a/arch/mips/include/asm/Kbuild +++ b/arch/mips/include/asm/Kbuild @@ -16,6 +16,5 @@ generic-y += sections.h generic-y += segment.h generic-y += serial.h generic-y += trace_clock.h -generic-y += ucontext.h generic-y += user.h generic-y += xor.h diff --git a/arch/mips/include/asm/abi.h b/arch/mips/include/asm/abi.h index 7186bb51b89b..37f84078e78a 100644 --- a/arch/mips/include/asm/abi.h +++ b/arch/mips/include/asm/abi.h @@ -20,6 +20,10 @@ struct mips_abi { struct pt_regs *regs, sigset_t *set); const unsigned long rt_signal_return_offset; const unsigned long restart; + + unsigned off_sc_fpregs; + unsigned off_sc_fpc_csr; + unsigned off_sc_used_math; }; #endif /* _ASM_ABI_H */ diff --git a/arch/mips/include/asm/asmmacro.h b/arch/mips/include/asm/asmmacro.h index 76317a70200d..867f924b05c7 100644 --- a/arch/mips/include/asm/asmmacro.h +++ b/arch/mips/include/asm/asmmacro.h @@ -232,6 +232,30 @@ .set pop .endm + .macro ld_b wd, off, base + .set push + .set mips32r2 + .set msa + ld.b $w\wd, \off(\base) + .set pop + .endm + + .macro ld_h wd, off, base + .set push + .set mips32r2 + .set msa + ld.h $w\wd, \off(\base) + .set pop + .endm + + .macro ld_w wd, off, base + .set push + .set mips32r2 + .set msa + ld.w $w\wd, \off(\base) + .set pop + .endm + .macro ld_d wd, off, base .set push .set mips32r2 @@ -241,6 +265,30 @@ .set pop .endm + .macro st_b wd, off, base + .set push + .set mips32r2 + .set msa + st.b $w\wd, \off(\base) + .set pop + .endm + + .macro st_h wd, off, base + .set push + .set mips32r2 + .set msa + st.h $w\wd, \off(\base) + .set pop + .endm + + .macro st_w wd, off, base + .set push + .set mips32r2 + .set msa + st.w $w\wd, \off(\base) + .set pop + .endm + .macro st_d wd, off, base .set push .set mips32r2 @@ -290,7 +338,13 @@ #ifdef CONFIG_CPU_MICROMIPS #define CFC_MSA_INSN 0x587e0056 #define CTC_MSA_INSN 0x583e0816 +#define LDB_MSA_INSN 0x58000807 +#define LDH_MSA_INSN 0x58000817 +#define LDW_MSA_INSN 0x58000827 #define LDD_MSA_INSN 0x58000837 +#define STB_MSA_INSN 0x5800080f +#define STH_MSA_INSN 0x5800081f +#define STW_MSA_INSN 0x5800082f #define STD_MSA_INSN 0x5800083f #define COPY_UW_MSA_INSN 0x58f00056 #define COPY_UD_MSA_INSN 0x58f80056 @@ -299,7 +353,13 @@ #else #define CFC_MSA_INSN 0x787e0059 #define CTC_MSA_INSN 0x783e0819 +#define LDB_MSA_INSN 0x78000820 +#define LDH_MSA_INSN 0x78000821 +#define LDW_MSA_INSN 0x78000822 #define LDD_MSA_INSN 0x78000823 +#define STB_MSA_INSN 0x78000824 +#define STH_MSA_INSN 0x78000825 +#define STW_MSA_INSN 0x78000826 #define STD_MSA_INSN 0x78000827 #define COPY_UW_MSA_INSN 0x78f00059 #define COPY_UD_MSA_INSN 0x78f80059 @@ -329,6 +389,33 @@ .set pop .endm + .macro ld_b wd, off, base + .set push + .set noat + SET_HARDFLOAT + addu $1, \base, \off + .word LDB_MSA_INSN | (\wd << 6) + .set pop + .endm + + .macro ld_h wd, off, base + .set push + .set noat + SET_HARDFLOAT + addu $1, \base, \off + .word LDH_MSA_INSN | (\wd << 6) + .set pop + .endm + + .macro ld_w wd, off, base + .set push + .set noat + SET_HARDFLOAT + addu $1, \base, \off + .word LDW_MSA_INSN | (\wd << 6) + .set pop + .endm + .macro ld_d wd, off, base .set push .set noat @@ -338,6 +425,33 @@ .set pop .endm + .macro st_b wd, off, base + .set push + .set noat + SET_HARDFLOAT + addu $1, \base, \off + .word STB_MSA_INSN | (\wd << 6) + .set pop + .endm + + .macro st_h wd, off, base + .set push + .set noat + SET_HARDFLOAT + addu $1, \base, \off + .word STH_MSA_INSN | (\wd << 6) + .set pop + .endm + + .macro st_w wd, off, base + .set push + .set noat + SET_HARDFLOAT + addu $1, \base, \off + .word STW_MSA_INSN | (\wd << 6) + .set pop + .endm + .macro st_d wd, off, base .set push .set noat diff --git a/arch/mips/include/asm/atomic.h b/arch/mips/include/asm/atomic.h index 26d436336f2e..4c42fd9af777 100644 --- a/arch/mips/include/asm/atomic.h +++ b/arch/mips/include/asm/atomic.h @@ -137,6 +137,10 @@ static __inline__ int atomic_##op##_return(int i, atomic_t * v) \ ATOMIC_OPS(add, +=, addu) ATOMIC_OPS(sub, -=, subu) +ATOMIC_OP(and, &=, and) +ATOMIC_OP(or, |=, or) +ATOMIC_OP(xor, ^=, xor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -416,6 +420,9 @@ static __inline__ long atomic64_##op##_return(long i, atomic64_t * v) \ ATOMIC64_OPS(add, +=, daddu) ATOMIC64_OPS(sub, -=, dsubu) +ATOMIC64_OP(and, &=, and) +ATOMIC64_OP(or, |=, or) +ATOMIC64_OP(xor, ^=, xor) #undef ATOMIC64_OPS #undef ATOMIC64_OP_RETURN diff --git a/arch/mips/include/asm/barrier.h b/arch/mips/include/asm/barrier.h index 7ecba84656d4..752e0b86c171 100644 --- a/arch/mips/include/asm/barrier.h +++ b/arch/mips/include/asm/barrier.h @@ -133,12 +133,12 @@ do { \ compiletime_assert_atomic_type(*p); \ smp_mb(); \ - ACCESS_ONCE(*p) = (v); \ + WRITE_ONCE(*p, v); \ } while (0) #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ smp_mb(); \ ___p1; \ diff --git a/arch/mips/include/asm/cdmm.h b/arch/mips/include/asm/cdmm.h index 16e22ce9719f..bece2064cc8c 100644 --- a/arch/mips/include/asm/cdmm.h +++ b/arch/mips/include/asm/cdmm.h @@ -53,7 +53,7 @@ struct mips_cdmm_driver { * mips_cdmm_phys_base() - Choose a physical base address for CDMM region. * * Picking a suitable physical address at which to map the CDMM region is - * platform specific, so this weak function can be defined by platform code to + * platform specific, so this function can be defined by platform code to * pick a suitable value if none is configured by the bootloader. * * This address must be 32kB aligned, and the region occupies a maximum of 32kB @@ -61,7 +61,7 @@ struct mips_cdmm_driver { * * Returns: Physical base address for CDMM region, or 0 on failure. */ -phys_addr_t __weak mips_cdmm_phys_base(void); +phys_addr_t mips_cdmm_phys_base(void); extern struct bus_type mips_cdmm_bustype; void __iomem *mips_cdmm_early_probe(unsigned int dev_type); diff --git a/arch/mips/include/asm/cevt-r4k.h b/arch/mips/include/asm/cevt-r4k.h index f0edf6fcd002..2e13a038d260 100644 --- a/arch/mips/include/asm/cevt-r4k.h +++ b/arch/mips/include/asm/cevt-r4k.h @@ -21,7 +21,6 @@ DECLARE_PER_CPU(struct clock_event_device, mips_clockevent_device); void mips_event_handler(struct clock_event_device *dev); int c0_compare_int_usable(void); -void mips_set_clock_mode(enum clock_event_mode, struct clock_event_device *); irqreturn_t c0_compare_interrupt(int, void *); extern struct irqaction c0_compare_irqaction; diff --git a/arch/mips/include/asm/cpu-features.h b/arch/mips/include/asm/cpu-features.h index f25de771f7ed..9801ac982655 100644 --- a/arch/mips/include/asm/cpu-features.h +++ b/arch/mips/include/asm/cpu-features.h @@ -411,4 +411,8 @@ # define cpu_has_cdmm (cpu_data[0].options & MIPS_CPU_CDMM) #endif +#ifndef cpu_has_small_pages +# define cpu_has_small_pages (cpu_data[0].options & MIPS_CPU_SP) +#endif + #endif /* __ASM_CPU_FEATURES_H */ diff --git a/arch/mips/include/asm/cpu-type.h b/arch/mips/include/asm/cpu-type.h index d41e8e284825..abee2bfd10dc 100644 --- a/arch/mips/include/asm/cpu-type.h +++ b/arch/mips/include/asm/cpu-type.h @@ -77,6 +77,10 @@ static inline int __pure __get_cpu_type(const int cpu_type) */ #endif +#ifdef CONFIG_SYS_HAS_CPU_MIPS64_R6 + case CPU_I6400: +#endif + #ifdef CONFIG_SYS_HAS_CPU_R3000 case CPU_R2000: case CPU_R3000: diff --git a/arch/mips/include/asm/cpu.h b/arch/mips/include/asm/cpu.h index e46e40602af3..cd89e9855775 100644 --- a/arch/mips/include/asm/cpu.h +++ b/arch/mips/include/asm/cpu.h @@ -120,6 +120,7 @@ #define PRID_IMP_PROAPTIV_MP 0xa300 #define PRID_IMP_M5150 0xa700 #define PRID_IMP_P5600 0xa800 +#define PRID_IMP_I6400 0xa900 /* * These are the PRID's for when 23:16 == PRID_COMP_SIBYTE @@ -307,6 +308,7 @@ enum cpu_type_enum { CPU_ALCHEMY, CPU_PR4450, CPU_BMIPS32, CPU_BMIPS3300, CPU_BMIPS4350, CPU_BMIPS4380, CPU_BMIPS5000, CPU_JZRISC, CPU_LOONGSON1, CPU_M14KC, CPU_M14KEC, CPU_INTERAPTIV, CPU_P5600, CPU_PROAPTIV, CPU_1074K, CPU_M5150, + CPU_I6400, /* * MIPS64 class processors @@ -382,6 +384,7 @@ enum cpu_type_enum { #define MIPS_CPU_XPA 0x2000000000ull /* CPU supports Extended Physical Addressing */ #define MIPS_CPU_CDMM 0x4000000000ull /* CPU has Common Device Memory Map */ #define MIPS_CPU_BP_GHIST 0x8000000000ull /* R12K+ Branch Prediction Global History */ +#define MIPS_CPU_SP 0x10000000000ull /* Small (1KB) page support */ /* * CPU ASE encodings diff --git a/arch/mips/include/asm/debug.h b/arch/mips/include/asm/debug.h deleted file mode 100644 index 1fd5a2b39445..000000000000 --- a/arch/mips/include/asm/debug.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Debug macros for run-time debugging. - * Turned on/off with CONFIG_RUNTIME_DEBUG option. - * - * Copyright (C) 2001 MontaVista Software Inc. - * Author: Jun Sun, jsun@mvista.com or jsun@junsun.net - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - */ - -#ifndef _ASM_DEBUG_H -#define _ASM_DEBUG_H - - -/* - * run-time macros for catching spurious errors. Eable CONFIG_RUNTIME_DEBUG in - * kernel hacking config menu to use them. - * - * Use them as run-time debugging aid. NEVER USE THEM AS ERROR HANDLING CODE!!! - */ - -#ifdef CONFIG_RUNTIME_DEBUG - -#include - -#define db_assert(x) if (!(x)) { \ - panic("assertion failed at %s:%d: %s", __FILE__, __LINE__, #x); } -#define db_warn(x) if (!(x)) { \ - printk(KERN_WARNING "warning at %s:%d: %s", __FILE__, __LINE__, #x); } -#define db_verify(x, y) db_assert(x y) -#define db_verify_warn(x, y) db_warn(x y) -#define db_run(x) do { x; } while (0) - -#else - -#define db_assert(x) -#define db_warn(x) -#define db_verify(x, y) x -#define db_verify_warn(x, y) x -#define db_run(x) - -#endif - -#endif /* _ASM_DEBUG_H */ diff --git a/arch/mips/include/asm/dma-mapping.h b/arch/mips/include/asm/dma-mapping.h index 360b3387182a..e604f760c4a0 100644 --- a/arch/mips/include/asm/dma-mapping.h +++ b/arch/mips/include/asm/dma-mapping.h @@ -4,7 +4,6 @@ #include #include #include -#include #ifndef CONFIG_SGI_IP27 /* Kludge to fix 2.6.39 build for IP27 */ #include @@ -32,73 +31,7 @@ static inline void dma_mark_clean(void *addr, size_t size) {} #include -static inline int dma_supported(struct device *dev, u64 mask) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - return ops->dma_supported(dev, mask); -} - -static inline int dma_mapping_error(struct device *dev, u64 mask) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - - debug_dma_mapping_error(dev, mask); - return ops->mapping_error(dev, mask); -} - -static inline int -dma_set_mask(struct device *dev, u64 mask) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - - if(!dev->dma_mask || !dma_supported(dev, mask)) - return -EIO; - - if (ops->set_dma_mask) - return ops->set_dma_mask(dev, mask); - - *dev->dma_mask = mask; - - return 0; -} - extern void dma_cache_sync(struct device *dev, void *vaddr, size_t size, enum dma_data_direction direction); -#define dma_alloc_coherent(d,s,h,f) dma_alloc_attrs(d,s,h,f,NULL) - -static inline void *dma_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, - struct dma_attrs *attrs) -{ - void *ret; - struct dma_map_ops *ops = get_dma_ops(dev); - - ret = ops->alloc(dev, size, dma_handle, gfp, attrs); - - debug_dma_alloc_coherent(dev, size, *dma_handle, ret); - - return ret; -} - -#define dma_free_coherent(d,s,c,h) dma_free_attrs(d,s,c,h,NULL) - -static inline void dma_free_attrs(struct device *dev, size_t size, - void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - - ops->free(dev, size, vaddr, dma_handle, attrs); - - debug_dma_free_coherent(dev, size, vaddr, dma_handle); -} - - -void *dma_alloc_noncoherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag); - -void dma_free_noncoherent(struct device *dev, size_t size, - void *vaddr, dma_addr_t dma_handle); - #endif /* _ASM_DMA_MAPPING_H */ diff --git a/arch/mips/include/asm/elf.h b/arch/mips/include/asm/elf.h index f19e890b99d2..53b26933b12c 100644 --- a/arch/mips/include/asm/elf.h +++ b/arch/mips/include/asm/elf.h @@ -382,7 +382,9 @@ do { \ instruction set this cpu supports. This could be done in userspace, but it's not easy, and we've already done it here. */ -#define ELF_HWCAP (0) +#define ELF_HWCAP (elf_hwcap) +extern unsigned int elf_hwcap; +#include /* * This yields a string that ld.so will use to load implementation diff --git a/arch/mips/include/asm/fpu.h b/arch/mips/include/asm/fpu.h index 1b0625189835..9cbf383b8834 100644 --- a/arch/mips/include/asm/fpu.h +++ b/arch/mips/include/asm/fpu.h @@ -164,25 +164,30 @@ static inline int own_fpu(int restore) return ret; } -static inline void lose_fpu(int save) +static inline void lose_fpu_inatomic(int save, struct task_struct *tsk) { - preempt_disable(); if (is_msa_enabled()) { if (save) { - save_msa(current); - current->thread.fpu.fcr31 = + save_msa(tsk); + tsk->thread.fpu.fcr31 = read_32bit_cp1_register(CP1_STATUS); } disable_msa(); - clear_thread_flag(TIF_USEDMSA); + clear_tsk_thread_flag(tsk, TIF_USEDMSA); __disable_fpu(); } else if (is_fpu_owner()) { if (save) - _save_fp(current); + _save_fp(tsk); __disable_fpu(); } - KSTK_STATUS(current) &= ~ST0_CU1; - clear_thread_flag(TIF_USEDFPU); + KSTK_STATUS(tsk) &= ~ST0_CU1; + clear_tsk_thread_flag(tsk, TIF_USEDFPU); +} + +static inline void lose_fpu(int save) +{ + preempt_disable(); + lose_fpu_inatomic(save, current); preempt_enable(); } diff --git a/arch/mips/include/asm/gpio.h b/arch/mips/include/asm/gpio.h deleted file mode 100644 index 06e46faf862d..000000000000 --- a/arch/mips/include/asm/gpio.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef __ASM_MIPS_GPIO_H -#define __ASM_MIPS_GPIO_H - -#include - -#endif /* __ASM_MIPS_GPIO_H */ diff --git a/arch/mips/include/asm/irq.h b/arch/mips/include/asm/irq.h index f0db99f8defe..15e0fecbc300 100644 --- a/arch/mips/include/asm/irq.h +++ b/arch/mips/include/asm/irq.h @@ -49,7 +49,7 @@ extern int cp0_compare_irq_shift; extern int cp0_perfcount_irq; extern int cp0_fdc_irq; -extern int __weak get_c0_fdc_int(void); +extern int get_c0_fdc_int(void); void arch_trigger_all_cpu_backtrace(bool); #define arch_trigger_all_cpu_backtrace arch_trigger_all_cpu_backtrace diff --git a/arch/mips/include/asm/jump_label.h b/arch/mips/include/asm/jump_label.h index 608aa57799c8..e77672539e8e 100644 --- a/arch/mips/include/asm/jump_label.h +++ b/arch/mips/include/asm/jump_label.h @@ -26,14 +26,29 @@ #define NOP_INSN "nop" #endif -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("1:\t" NOP_INSN "\n\t" "nop\n\t" ".pushsection __jump_table, \"aw\"\n\t" WORD_INSN " 1b, %l[l_yes], %0\n\t" ".popsection\n\t" - : : "i" (key) : : l_yes); + : : "i" (&((char *)key)[branch]) : : l_yes); + + return false; +l_yes: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("1:\tj %l[l_yes]\n\t" + "nop\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + WORD_INSN " 1b, %l[l_yes], %0\n\t" + ".popsection\n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); + return false; l_yes: return true; diff --git a/arch/mips/include/asm/kdebug.h b/arch/mips/include/asm/kdebug.h index cba22ab7ad4d..8e3d08e739c1 100644 --- a/arch/mips/include/asm/kdebug.h +++ b/arch/mips/include/asm/kdebug.h @@ -11,7 +11,9 @@ enum die_val { DIE_PAGE_FAULT, DIE_BREAK, DIE_SSTEPBP, - DIE_MSAFP + DIE_MSAFP, + DIE_UPROBE, + DIE_UPROBE_XOL, }; #endif /* _ASM_MIPS_KDEBUG_H */ diff --git a/arch/mips/include/asm/linkage.h b/arch/mips/include/asm/linkage.h index 2767dda9e309..99651b0ea7c7 100644 --- a/arch/mips/include/asm/linkage.h +++ b/arch/mips/include/asm/linkage.h @@ -5,7 +5,6 @@ #include #endif -#define __weak __attribute__((weak)) #define cond_syscall(x) asm(".weak\t" #x "\n" #x "\t=\tsys_ni_syscall") #define SYSCALL_ALIAS(alias, name) \ asm ( #alias " = " #name "\n\t.globl " #alias) diff --git a/arch/mips/include/asm/maar.h b/arch/mips/include/asm/maar.h index 6c62b0f899c0..b02891f9caaf 100644 --- a/arch/mips/include/asm/maar.h +++ b/arch/mips/include/asm/maar.h @@ -26,7 +26,7 @@ * * Return: The number of MAAR pairs configured. */ -unsigned __weak platform_maar_init(unsigned num_pairs); +unsigned platform_maar_init(unsigned num_pairs); /** * write_maar_pair() - write to a pair of MAARs diff --git a/arch/mips/include/asm/mach-ar7/ar7.h b/arch/mips/include/asm/mach-ar7/ar7.h index a47ea0c85248..468cbd61b906 100644 --- a/arch/mips/include/asm/mach-ar7/ar7.h +++ b/arch/mips/include/asm/mach-ar7/ar7.h @@ -203,4 +203,8 @@ static inline void ar7_device_off(u32 bit) int __init ar7_gpio_init(void); void __init ar7_init_clocks(void); +/* Board specific GPIO functions */ +int ar7_gpio_enable(unsigned gpio); +int ar7_gpio_disable(unsigned gpio); + #endif /* __AR7_H__ */ diff --git a/arch/mips/include/asm/mach-ar7/gpio.h b/arch/mips/include/asm/mach-ar7/gpio.h deleted file mode 100644 index c177cd1eed25..000000000000 --- a/arch/mips/include/asm/mach-ar7/gpio.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (C) 2007-2009 Florian Fainelli - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - */ - -#ifndef __AR7_GPIO_H__ -#define __AR7_GPIO_H__ - -#include - -#define AR7_GPIO_MAX 32 -#define TITAN_GPIO_MAX 51 -#define NR_BUILTIN_GPIO TITAN_GPIO_MAX - -#define gpio_to_irq(gpio) -1 - -#define gpio_get_value __gpio_get_value -#define gpio_set_value __gpio_set_value - -#define gpio_cansleep __gpio_cansleep - -/* Board specific GPIO functions */ -int ar7_gpio_enable(unsigned gpio); -int ar7_gpio_disable(unsigned gpio); - -#include - -#endif diff --git a/arch/mips/include/asm/mach-ath25/gpio.h b/arch/mips/include/asm/mach-ath25/gpio.h deleted file mode 100644 index 713564b8e8ef..000000000000 --- a/arch/mips/include/asm/mach-ath25/gpio.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef __ASM_MACH_ATH25_GPIO_H -#define __ASM_MACH_ATH25_GPIO_H - -#include - -#define gpio_get_value __gpio_get_value -#define gpio_set_value __gpio_set_value -#define gpio_cansleep __gpio_cansleep -#define gpio_to_irq __gpio_to_irq - -static inline int irq_to_gpio(unsigned irq) -{ - return -EINVAL; -} - -#endif /* __ASM_MACH_ATH25_GPIO_H */ diff --git a/arch/mips/include/asm/mach-ath79/gpio.h b/arch/mips/include/asm/mach-ath79/gpio.h deleted file mode 100644 index 60dcb62785b4..000000000000 --- a/arch/mips/include/asm/mach-ath79/gpio.h +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Atheros AR71XX/AR724X/AR913X GPIO API definitions - * - * Copyright (C) 2008-2010 Gabor Juhos - * Copyright (C) 2008 Imre Kaloz - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 as published - * by the Free Software Foundation. - * - */ - -#ifndef __ASM_MACH_ATH79_GPIO_H -#define __ASM_MACH_ATH79_GPIO_H - -#define ARCH_NR_GPIOS 64 -#include - -int gpio_to_irq(unsigned gpio); -int irq_to_gpio(unsigned irq); -int gpio_get_value(unsigned gpio); -void gpio_set_value(unsigned gpio, int value); - -#define gpio_cansleep __gpio_cansleep - -#endif /* __ASM_MACH_ATH79_GPIO_H */ diff --git a/arch/mips/include/asm/mach-au1x00/gpio-au1000.h b/arch/mips/include/asm/mach-au1x00/gpio-au1000.h index 9785e4ebb450..adde1fa5097e 100644 --- a/arch/mips/include/asm/mach-au1x00/gpio-au1000.h +++ b/arch/mips/include/asm/mach-au1x00/gpio-au1000.h @@ -266,6 +266,17 @@ static inline int alchemy_gpio1_to_irq(int gpio) return -ENXIO; } +/* On Au1000, Au1500 and Au1100 GPIOs won't work as inputs before + * SYS_PININPUTEN is written to at least once. On Au1550/Au1200/Au1300 this + * register enables use of GPIOs as wake source. + */ +static inline void alchemy_gpio1_input_enable(void) +{ + void __iomem *base = (void __iomem *)KSEG1ADDR(AU1000_SYS_PHYS_ADDR); + __raw_writel(0, base + 0x110); /* the write op is key */ + wmb(); +} + /* * GPIO2 block macros for common linux GPIO functions. The 'gpio' * parameter must be in range of ALCHEMY_GPIO2_BASE..ALCHEMY_GPIO2_MAX. @@ -518,141 +529,4 @@ static inline int alchemy_irq_to_gpio(int irq) return -ENXIO; } -/**********************************************************************/ - -/* Linux gpio framework integration. - * - * 4 use cases of Au1000-Au1200 GPIOS: - *(1) GPIOLIB=y, ALCHEMY_GPIO_INDIRECT=y: - * Board must register gpiochips. - *(2) GPIOLIB=y, ALCHEMY_GPIO_INDIRECT=n: - * 2 (1 for Au1000) gpio_chips are registered. - * - *(3) GPIOLIB=n, ALCHEMY_GPIO_INDIRECT=y: - * the boards' gpio.h must provide the linux gpio wrapper functions, - * - *(4) GPIOLIB=n, ALCHEMY_GPIO_INDIRECT=n: - * inlinable gpio functions are provided which enable access to the - * Au1000 gpios only by using the numbers straight out of the data- - * sheets. - - * Cases 1 and 3 are intended for boards which want to provide their own - * GPIO namespace and -operations (i.e. for example you have 8 GPIOs - * which are in part provided by spare Au1000 GPIO pins and in part by - * an external FPGA but you still want them to be accssible in linux - * as gpio0-7. The board can of course use the alchemy_gpioX_* functions - * as required). - */ - -#ifndef CONFIG_GPIOLIB - -#ifdef CONFIG_ALCHEMY_GPIOINT_AU1000 - -#ifndef CONFIG_ALCHEMY_GPIO_INDIRECT /* case (4) */ - -static inline int gpio_direction_input(int gpio) -{ - return alchemy_gpio_direction_input(gpio); -} - -static inline int gpio_direction_output(int gpio, int v) -{ - return alchemy_gpio_direction_output(gpio, v); -} - -static inline int gpio_get_value(int gpio) -{ - return alchemy_gpio_get_value(gpio); -} - -static inline void gpio_set_value(int gpio, int v) -{ - alchemy_gpio_set_value(gpio, v); -} - -static inline int gpio_get_value_cansleep(unsigned gpio) -{ - return gpio_get_value(gpio); -} - -static inline void gpio_set_value_cansleep(unsigned gpio, int value) -{ - gpio_set_value(gpio, value); -} - -static inline int gpio_is_valid(int gpio) -{ - return alchemy_gpio_is_valid(gpio); -} - -static inline int gpio_cansleep(int gpio) -{ - return alchemy_gpio_cansleep(gpio); -} - -static inline int gpio_to_irq(int gpio) -{ - return alchemy_gpio_to_irq(gpio); -} - -static inline int irq_to_gpio(int irq) -{ - return alchemy_irq_to_gpio(irq); -} - -static inline int gpio_request(unsigned gpio, const char *label) -{ - return 0; -} - -static inline int gpio_request_one(unsigned gpio, - unsigned long flags, const char *label) -{ - return 0; -} - -static inline int gpio_request_array(struct gpio *array, size_t num) -{ - return 0; -} - -static inline void gpio_free(unsigned gpio) -{ -} - -static inline void gpio_free_array(struct gpio *array, size_t num) -{ -} - -static inline int gpio_set_debounce(unsigned gpio, unsigned debounce) -{ - return -ENOSYS; -} - -static inline int gpio_export(unsigned gpio, bool direction_may_change) -{ - return -ENOSYS; -} - -static inline int gpio_export_link(struct device *dev, const char *name, - unsigned gpio) -{ - return -ENOSYS; -} - -static inline int gpio_sysfs_set_active_low(unsigned gpio, int value) -{ - return -ENOSYS; -} - -static inline void gpio_unexport(unsigned gpio) -{ -} - -#endif /* !CONFIG_ALCHEMY_GPIO_INDIRECT */ - -#endif /* CONFIG_ALCHEMY_GPIOINT_AU1000 */ - -#endif /* !CONFIG_GPIOLIB */ - #endif /* _ALCHEMY_GPIO_AU1000_H_ */ diff --git a/arch/mips/include/asm/mach-au1x00/gpio.h b/arch/mips/include/asm/mach-au1x00/gpio.h deleted file mode 100644 index 22e7ff17fc48..000000000000 --- a/arch/mips/include/asm/mach-au1x00/gpio.h +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Alchemy GPIO support. - * - * With CONFIG_GPIOLIB=y different types of on-chip GPIO can be supported within - * the same kernel image. - * With CONFIG_GPIOLIB=n, your board must select ALCHEMY_GPIOINT_AU1XXX for the - * appropriate CPU type (AU1000 currently). - */ - -#ifndef _ALCHEMY_GPIO_H_ -#define _ALCHEMY_GPIO_H_ - -#include -#include -#include - -/* On Au1000, Au1500 and Au1100 GPIOs won't work as inputs before - * SYS_PININPUTEN is written to at least once. On Au1550/Au1200/Au1300 this - * register enables use of GPIOs as wake source. - */ -static inline void alchemy_gpio1_input_enable(void) -{ - void __iomem *base = (void __iomem *)KSEG1ADDR(AU1000_SYS_PHYS_ADDR); - __raw_writel(0, base + 0x110); /* the write op is key */ - wmb(); -} - - -/* Linux gpio framework integration. -* -* 4 use cases of Alchemy GPIOS: -*(1) GPIOLIB=y, ALCHEMY_GPIO_INDIRECT=y: -* Board must register gpiochips. -*(2) GPIOLIB=y, ALCHEMY_GPIO_INDIRECT=n: -* A gpiochip for the 75 GPIOs is registered. -* -*(3) GPIOLIB=n, ALCHEMY_GPIO_INDIRECT=y: -* the boards' gpio.h must provide the linux gpio wrapper functions, -* -*(4) GPIOLIB=n, ALCHEMY_GPIO_INDIRECT=n: -* inlinable gpio functions are provided which enable access to the -* Au1300 gpios only by using the numbers straight out of the data- -* sheets. - -* Cases 1 and 3 are intended for boards which want to provide their own -* GPIO namespace and -operations (i.e. for example you have 8 GPIOs -* which are in part provided by spare Au1300 GPIO pins and in part by -* an external FPGA but you still want them to be accssible in linux -* as gpio0-7. The board can of course use the alchemy_gpioX_* functions -* as required). -*/ - -#ifdef CONFIG_GPIOLIB - -/* wraps the cpu-dependent irq_to_gpio functions */ -/* FIXME: gpiolib needs an irq_to_gpio hook */ -static inline int __au_irq_to_gpio(unsigned int irq) -{ - switch (alchemy_get_cputype()) { - case ALCHEMY_CPU_AU1000...ALCHEMY_CPU_AU1200: - return alchemy_irq_to_gpio(irq); - case ALCHEMY_CPU_AU1300: - return au1300_irq_to_gpio(irq); - } - return -EINVAL; -} - - -/* using gpiolib to provide up to 2 gpio_chips for on-chip gpios */ -#ifndef CONFIG_ALCHEMY_GPIO_INDIRECT /* case (2) */ - -/* get everything through gpiolib */ -#define gpio_to_irq __gpio_to_irq -#define gpio_get_value __gpio_get_value -#define gpio_set_value __gpio_set_value -#define gpio_cansleep __gpio_cansleep -#define irq_to_gpio __au_irq_to_gpio - -#include - -#endif /* !CONFIG_ALCHEMY_GPIO_INDIRECT */ - - -#endif /* CONFIG_GPIOLIB */ - -#endif /* _ALCHEMY_GPIO_H_ */ diff --git a/arch/mips/include/asm/mach-bcm47xx/gpio.h b/arch/mips/include/asm/mach-bcm47xx/gpio.h deleted file mode 100644 index 90daefa24a4d..000000000000 --- a/arch/mips/include/asm/mach-bcm47xx/gpio.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef __ASM_MIPS_MACH_BCM47XX_GPIO_H -#define __ASM_MIPS_MACH_BCM47XX_GPIO_H - -#include - -#define gpio_get_value __gpio_get_value -#define gpio_set_value __gpio_set_value - -#define gpio_cansleep __gpio_cansleep -#define gpio_to_irq __gpio_to_irq - -static inline int irq_to_gpio(unsigned int irq) -{ - return -EINVAL; -} - -#endif diff --git a/arch/mips/include/asm/mach-bcm63xx/gpio.h b/arch/mips/include/asm/mach-bcm63xx/gpio.h deleted file mode 100644 index 1eb534de8e3b..000000000000 --- a/arch/mips/include/asm/mach-bcm63xx/gpio.h +++ /dev/null @@ -1,15 +0,0 @@ -#ifndef __ASM_MIPS_MACH_BCM63XX_GPIO_H -#define __ASM_MIPS_MACH_BCM63XX_GPIO_H - -#include - -#define gpio_to_irq(gpio) -1 - -#define gpio_get_value __gpio_get_value -#define gpio_set_value __gpio_set_value - -#define gpio_cansleep __gpio_cansleep - -#include - -#endif /* __ASM_MIPS_MACH_BCM63XX_GPIO_H */ diff --git a/arch/mips/include/asm/mach-cavium-octeon/gpio.h b/arch/mips/include/asm/mach-cavium-octeon/gpio.h deleted file mode 100644 index 34e9f7aabab4..000000000000 --- a/arch/mips/include/asm/mach-cavium-octeon/gpio.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef __ASM_MACH_CAVIUM_OCTEON_GPIO_H -#define __ASM_MACH_CAVIUM_OCTEON_GPIO_H - -#ifdef CONFIG_GPIOLIB -#define gpio_get_value __gpio_get_value -#define gpio_set_value __gpio_set_value -#define gpio_cansleep __gpio_cansleep -#else -int gpio_request(unsigned gpio, const char *label); -void gpio_free(unsigned gpio); -int gpio_direction_input(unsigned gpio); -int gpio_direction_output(unsigned gpio, int value); -int gpio_get_value(unsigned gpio); -void gpio_set_value(unsigned gpio, int value); -#endif - -#include - -#define gpio_to_irq __gpio_to_irq - -#endif /* __ASM_MACH_GENERIC_GPIO_H */ diff --git a/arch/mips/include/asm/mach-generic/gpio.h b/arch/mips/include/asm/mach-generic/gpio.h deleted file mode 100644 index b4e70208da64..000000000000 --- a/arch/mips/include/asm/mach-generic/gpio.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef __ASM_MACH_GENERIC_GPIO_H -#define __ASM_MACH_GENERIC_GPIO_H - -#ifdef CONFIG_GPIOLIB -#define gpio_get_value __gpio_get_value -#define gpio_set_value __gpio_set_value -#define gpio_cansleep __gpio_cansleep -#else -int gpio_request(unsigned gpio, const char *label); -void gpio_free(unsigned gpio); -int gpio_direction_input(unsigned gpio); -int gpio_direction_output(unsigned gpio, int value); -int gpio_get_value(unsigned gpio); -void gpio_set_value(unsigned gpio, int value); -#endif -int gpio_to_irq(unsigned gpio); -int irq_to_gpio(unsigned irq); - -#include /* cansleep wrappers */ - -#endif /* __ASM_MACH_GENERIC_GPIO_H */ diff --git a/arch/mips/include/asm/mach-jz4740/gpio.h b/arch/mips/include/asm/mach-jz4740/gpio.h index eaacba79cf18..bf8c3e1860e7 100644 --- a/arch/mips/include/asm/mach-jz4740/gpio.h +++ b/arch/mips/include/asm/mach-jz4740/gpio.h @@ -73,8 +73,6 @@ int jz_gpio_port_direction_output(int port, uint32_t mask); void jz_gpio_port_set_value(int port, uint32_t value, uint32_t mask); uint32_t jz_gpio_port_get_value(int port, uint32_t mask); -#include - #define JZ_GPIO_PORTA(x) ((x) + 32 * 0) #define JZ_GPIO_PORTB(x) ((x) + 32 * 1) #define JZ_GPIO_PORTC(x) ((x) + 32 * 2) diff --git a/arch/mips/include/asm/mach-lantiq/gpio.h b/arch/mips/include/asm/mach-lantiq/gpio.h deleted file mode 100644 index 9ba1caebca5f..000000000000 --- a/arch/mips/include/asm/mach-lantiq/gpio.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef __ASM_MIPS_MACH_LANTIQ_GPIO_H -#define __ASM_MIPS_MACH_LANTIQ_GPIO_H - -#define gpio_to_irq __gpio_to_irq - -#define gpio_get_value __gpio_get_value -#define gpio_set_value __gpio_set_value - -#define gpio_cansleep __gpio_cansleep - -#include - -#endif diff --git a/arch/mips/include/asm/mach-loongson64/gpio.h b/arch/mips/include/asm/mach-loongson64/gpio.h deleted file mode 100644 index b3b216904a9a..000000000000 --- a/arch/mips/include/asm/mach-loongson64/gpio.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Loongson GPIO Support - * - * Copyright (c) 2008 Richard Liu, STMicroelectronics - * Copyright (c) 2008-2010 Arnaud Patard - * Copyright (c) 2014 Huacai Chen - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - */ - -#ifndef __LOONGSON_GPIO_H -#define __LOONGSON_GPIO_H - -#include - -#define gpio_get_value __gpio_get_value -#define gpio_set_value __gpio_set_value -#define gpio_cansleep __gpio_cansleep - -/* The chip can do interrupt - * but it has not been tested and doc not clear - */ -static inline int gpio_to_irq(int gpio) -{ - return -EINVAL; -} - -static inline int irq_to_gpio(int gpio) -{ - return -EINVAL; -} - -#endif /* __LOONGSON_GPIO_H */ diff --git a/arch/mips/include/asm/mach-pistachio/gpio.h b/arch/mips/include/asm/mach-pistachio/gpio.h deleted file mode 100644 index 6c1649c27b8d..000000000000 --- a/arch/mips/include/asm/mach-pistachio/gpio.h +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Pistachio IRQ setup - * - * Copyright (C) 2014 Google, Inc. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - */ - -#ifndef __ASM_MACH_PISTACHIO_GPIO_H -#define __ASM_MACH_PISTACHIO_GPIO_H - -#include - -#define gpio_get_value __gpio_get_value -#define gpio_set_value __gpio_set_value -#define gpio_cansleep __gpio_cansleep -#define gpio_to_irq __gpio_to_irq - -#endif /* __ASM_MACH_PISTACHIO_GPIO_H */ diff --git a/arch/mips/include/asm/mach-rc32434/gpio.h b/arch/mips/include/asm/mach-rc32434/gpio.h index 4dee0a34250c..db211212ce79 100644 --- a/arch/mips/include/asm/mach-rc32434/gpio.h +++ b/arch/mips/include/asm/mach-rc32434/gpio.h @@ -13,18 +13,6 @@ #ifndef _RC32434_GPIO_H_ #define _RC32434_GPIO_H_ -#include -#include - -#define NR_BUILTIN_GPIO 32 - -#define gpio_get_value __gpio_get_value -#define gpio_set_value __gpio_set_value -#define gpio_cansleep __gpio_cansleep - -#define gpio_to_irq(gpio) (8 + 4 * 32 + gpio) -#define irq_to_gpio(irq) (irq - (8 + 4 * 32)) - struct rb532_gpio_reg { u32 gpiofunc; /* GPIO Function Register * gpiofunc[x]==0 bit = gpio diff --git a/arch/mips/include/asm/mips-cm.h b/arch/mips/include/asm/mips-cm.h index edc7ee95269e..d75b75e78ebb 100644 --- a/arch/mips/include/asm/mips-cm.h +++ b/arch/mips/include/asm/mips-cm.h @@ -33,6 +33,29 @@ extern void __iomem *mips_cm_l2sync_base; */ extern phys_addr_t __mips_cm_phys_base(void); +/* + * mips_cm_is64 - determine CM register width + * + * The CM register width is processor and CM specific. A 64-bit processor + * usually has a 64-bit CM and a 32-bit one has a 32-bit CM but a 64-bit + * processor could come with a 32-bit CM. Moreover, accesses on 64-bit CMs + * can be done either using regular 64-bit load/store instructions, or 32-bit + * load/store instruction on 32-bit register pairs. We opt for using 64-bit + * accesses on 64-bit CMs and kernels and 32-bit in any other case. + * + * It's set to 0 for 32-bit accesses and 1 for 64-bit accesses. + */ +extern int mips_cm_is64; + +/** + * mips_cm_error_report - Report CM cache errors + */ +#ifdef CONFIG_MIPS_CM +extern void mips_cm_error_report(void); +#else +static inline void mips_cm_error_report(void) {} +#endif + /** * mips_cm_probe - probe for a Coherence Manager * @@ -90,20 +113,46 @@ static inline bool mips_cm_has_l2sync(void) /* Macros to ease the creation of register access functions */ #define BUILD_CM_R_(name, off) \ -static inline u32 __iomem *addr_gcr_##name(void) \ +static inline unsigned long __iomem *addr_gcr_##name(void) \ { \ - return (u32 __iomem *)(mips_cm_base + (off)); \ + return (unsigned long __iomem *)(mips_cm_base + (off)); \ } \ \ -static inline u32 read_gcr_##name(void) \ +static inline u32 read32_gcr_##name(void) \ { \ return __raw_readl(addr_gcr_##name()); \ +} \ + \ +static inline u64 read64_gcr_##name(void) \ +{ \ + return __raw_readq(addr_gcr_##name()); \ +} \ + \ +static inline unsigned long read_gcr_##name(void) \ +{ \ + if (mips_cm_is64) \ + return read64_gcr_##name(); \ + else \ + return read32_gcr_##name(); \ } #define BUILD_CM__W(name, off) \ -static inline void write_gcr_##name(u32 value) \ +static inline void write32_gcr_##name(u32 value) \ { \ __raw_writel(value, addr_gcr_##name()); \ +} \ + \ +static inline void write64_gcr_##name(u64 value) \ +{ \ + __raw_writeq(value, addr_gcr_##name()); \ +} \ + \ +static inline void write_gcr_##name(unsigned long value) \ +{ \ + if (mips_cm_is64) \ + write64_gcr_##name(value); \ + else \ + write32_gcr_##name(value); \ } #define BUILD_CM_RW(name, off) \ @@ -144,6 +193,7 @@ BUILD_CM_RW(reg3_base, MIPS_CM_GCB_OFS + 0xc0) BUILD_CM_RW(reg3_mask, MIPS_CM_GCB_OFS + 0xc8) BUILD_CM_R_(gic_status, MIPS_CM_GCB_OFS + 0xd0) BUILD_CM_R_(cpc_status, MIPS_CM_GCB_OFS + 0xf0) +BUILD_CM_RW(l2_config, MIPS_CM_GCB_OFS + 0x130) /* Core Local & Core Other register accessor functions */ BUILD_CM_Cx_RW(reset_release, 0x00) @@ -189,6 +239,13 @@ BUILD_CM_Cx_R_(tcid_8_priority, 0x80) #define CM_GCR_REV_MINOR_SHF 0 #define CM_GCR_REV_MINOR_MSK (_ULCAST_(0xff) << 0) +#define CM_ENCODE_REV(major, minor) \ + (((major) << CM_GCR_REV_MAJOR_SHF) | \ + ((minor) << CM_GCR_REV_MINOR_SHF)) + +#define CM_REV_CM2 CM_ENCODE_REV(6, 0) +#define CM_REV_CM3 CM_ENCODE_REV(8, 0) + /* GCR_ERROR_CAUSE register fields */ #define CM_GCR_ERROR_CAUSE_ERRTYPE_SHF 27 #define CM_GCR_ERROR_CAUSE_ERRTYPE_MSK (_ULCAST_(0x1f) << 27) @@ -249,6 +306,16 @@ BUILD_CM_Cx_R_(tcid_8_priority, 0x80) #define CM_GCR_CPC_STATUS_EX_SHF 0 #define CM_GCR_CPC_STATUS_EX_MSK (_ULCAST_(0x1) << 0) +/* GCR_L2_CONFIG register fields */ +#define CM_GCR_L2_CONFIG_BYPASS_SHF 20 +#define CM_GCR_L2_CONFIG_BYPASS_MSK (_ULCAST_(0x1) << 20) +#define CM_GCR_L2_CONFIG_SET_SIZE_SHF 12 +#define CM_GCR_L2_CONFIG_SET_SIZE_MSK (_ULCAST_(0xf) << 12) +#define CM_GCR_L2_CONFIG_LINE_SIZE_SHF 8 +#define CM_GCR_L2_CONFIG_LINE_SIZE_MSK (_ULCAST_(0xf) << 8) +#define CM_GCR_L2_CONFIG_ASSOC_SHF 0 +#define CM_GCR_L2_CONFIG_ASSOC_MSK (_ULCAST_(0xff) << 0) + /* GCR_Cx_COHERENCE register fields */ #define CM_GCR_Cx_COHERENCE_COHDOMAINEN_SHF 0 #define CM_GCR_Cx_COHERENCE_COHDOMAINEN_MSK (_ULCAST_(0xff) << 0) @@ -324,4 +391,18 @@ static inline int mips_cm_l2sync(void) return 0; } +/** + * mips_cm_revision() - return CM revision + * + * Return: The revision of the CM, from GCR_REV, or 0 if no CM is present. The + * return value should be checked against the CM_REV_* macros. + */ +static inline int mips_cm_revision(void) +{ + if (!mips_cm_present()) + return 0; + + return read_gcr_rev(); +} + #endif /* __MIPS_ASM_MIPS_CM_H__ */ diff --git a/arch/mips/include/asm/mips-cpc.h b/arch/mips/include/asm/mips-cpc.h index 1cebe8c79051..f386f32702f1 100644 --- a/arch/mips/include/asm/mips-cpc.h +++ b/arch/mips/include/asm/mips-cpc.h @@ -27,16 +27,6 @@ extern void __iomem *mips_cpc_base; */ extern phys_addr_t mips_cpc_default_phys_base(void); -/** - * mips_cpc_phys_base - retrieve the physical base address of the CPC - * - * This function returns the physical base address of the Cluster Power - * Controller memory mapped registers, or 0 if no Cluster Power Controller - * is present. It may be overriden by individual platforms which determine - * this address in a different way. - */ -extern phys_addr_t __weak mips_cpc_phys_base(void); - /** * mips_cpc_probe - probe for a Cluster Power Controller * diff --git a/arch/mips/include/asm/mipsregs.h b/arch/mips/include/asm/mipsregs.h index c5b0956a8530..d3cd8eac81e3 100644 --- a/arch/mips/include/asm/mipsregs.h +++ b/arch/mips/include/asm/mipsregs.h @@ -112,6 +112,30 @@ #define CP0_TX39_CACHE $7 +/* Generic EntryLo bit definitions */ +#define ENTRYLO_G (_ULCAST_(1) << 0) +#define ENTRYLO_V (_ULCAST_(1) << 1) +#define ENTRYLO_D (_ULCAST_(1) << 2) +#define ENTRYLO_C_SHIFT 3 +#define ENTRYLO_C (_ULCAST_(7) << ENTRYLO_C_SHIFT) + +/* R3000 EntryLo bit definitions */ +#define R3K_ENTRYLO_G (_ULCAST_(1) << 8) +#define R3K_ENTRYLO_V (_ULCAST_(1) << 9) +#define R3K_ENTRYLO_D (_ULCAST_(1) << 10) +#define R3K_ENTRYLO_N (_ULCAST_(1) << 11) + +/* MIPS32/64 EntryLo bit definitions */ +#ifdef CONFIG_64BIT +/* as read by dmfc0 */ +#define MIPS_ENTRYLO_XI (_ULCAST_(1) << 62) +#define MIPS_ENTRYLO_RI (_ULCAST_(1) << 63) +#else +/* as read by mfc0 */ +#define MIPS_ENTRYLO_XI (_ULCAST_(1) << 30) +#define MIPS_ENTRYLO_RI (_ULCAST_(1) << 31) +#endif + /* * Values for PageMask register */ @@ -203,6 +227,9 @@ #define PG_ESP (_ULCAST_(1) << 28) #define PG_IEC (_ULCAST_(1) << 27) +/* MIPS32/64 EntryHI bit definitions */ +#define MIPS_ENTRYHI_EHINV (_ULCAST_(1) << 10) + /* * R4x00 interrupt enable / cause bits */ @@ -579,6 +606,8 @@ #define MIPS_CONF7_IAR (_ULCAST_(1) << 10) #define MIPS_CONF7_AR (_ULCAST_(1) << 16) +/* FTLB probability bits for R6 */ +#define MIPS_CONF7_FTLBP_SHIFT (18) /* MAAR bit definitions */ #define MIPS_MAAR_ADDR ((BIT_ULL(BITS_PER_LONG - 12) - 1) << 12) @@ -586,31 +615,6 @@ #define MIPS_MAAR_S (_ULCAST_(1) << 1) #define MIPS_MAAR_V (_ULCAST_(1) << 0) -/* EntryHI bit definition */ -#define MIPS_ENTRYHI_EHINV (_ULCAST_(1) << 10) - -/* R3000 EntryLo bit definitions */ -#define R3K_ENTRYLO_G (_ULCAST_(1) << 8) -#define R3K_ENTRYLO_V (_ULCAST_(1) << 9) -#define R3K_ENTRYLO_D (_ULCAST_(1) << 10) -#define R3K_ENTRYLO_N (_ULCAST_(1) << 11) - -/* R4000 compatible EntryLo bit definitions */ -#define MIPS_ENTRYLO_G (_ULCAST_(1) << 0) -#define MIPS_ENTRYLO_V (_ULCAST_(1) << 1) -#define MIPS_ENTRYLO_D (_ULCAST_(1) << 2) -#define MIPS_ENTRYLO_C_SHIFT 3 -#define MIPS_ENTRYLO_C (_ULCAST_(7) << MIPS_ENTRYLO_C_SHIFT) -#ifdef CONFIG_64BIT -/* as read by dmfc0 */ -#define MIPS_ENTRYLO_XI (_ULCAST_(1) << 62) -#define MIPS_ENTRYLO_RI (_ULCAST_(1) << 63) -#else -/* as read by mfc0 */ -#define MIPS_ENTRYLO_XI (_ULCAST_(1) << 30) -#define MIPS_ENTRYLO_RI (_ULCAST_(1) << 31) -#endif - /* CMGCRBase bit definitions */ #define MIPS_CMGCRB_BASE 11 #define MIPS_CMGCRF_BASE (~_ULCAST_((1 << MIPS_CMGCRB_BASE) - 1)) @@ -932,7 +936,7 @@ do { \ */ #define __read_32bit_c0_register(source, sel) \ -({ int __res; \ +({ unsigned int __res; \ if (sel == 0) \ __asm__ __volatile__( \ "mfc0\t%0, " #source "\n\t" \ @@ -1014,7 +1018,7 @@ do { \ * On RM7000/RM9000 these are uses to access cop0 set 1 registers */ #define __read_32bit_c0_ctrl_register(source) \ -({ int __res; \ +({ unsigned int __res; \ __asm__ __volatile__( \ "cfc0\t%0, " #source "\n\t" \ : "=r" (__res)); \ @@ -1471,7 +1475,7 @@ do { \ */ #define _read_32bit_cp1_register(source, gas_hardfloat) \ ({ \ - int __res; \ + unsigned int __res; \ \ __asm__ __volatile__( \ " .set push \n" \ diff --git a/arch/mips/include/asm/msa.h b/arch/mips/include/asm/msa.h index af5638b12c75..bbb85fe21642 100644 --- a/arch/mips/include/asm/msa.h +++ b/arch/mips/include/asm/msa.h @@ -14,10 +14,90 @@ #ifndef __ASSEMBLY__ +#include + extern void _save_msa(struct task_struct *); extern void _restore_msa(struct task_struct *); extern void _init_msa_upper(void); +extern void read_msa_wr_b(unsigned idx, union fpureg *to); +extern void read_msa_wr_h(unsigned idx, union fpureg *to); +extern void read_msa_wr_w(unsigned idx, union fpureg *to); +extern void read_msa_wr_d(unsigned idx, union fpureg *to); + +/** + * read_msa_wr() - Read a single MSA vector register + * @idx: The index of the vector register to read + * @to: The FPU register union to store the registers value in + * @fmt: The format of the data in the vector register + * + * Read the value of MSA vector register idx into the FPU register + * union to, using the format fmt. + */ +static inline void read_msa_wr(unsigned idx, union fpureg *to, + enum msa_2b_fmt fmt) +{ + switch (fmt) { + case msa_fmt_b: + read_msa_wr_b(idx, to); + break; + + case msa_fmt_h: + read_msa_wr_h(idx, to); + break; + + case msa_fmt_w: + read_msa_wr_w(idx, to); + break; + + case msa_fmt_d: + read_msa_wr_d(idx, to); + break; + + default: + BUG(); + } +} + +extern void write_msa_wr_b(unsigned idx, union fpureg *from); +extern void write_msa_wr_h(unsigned idx, union fpureg *from); +extern void write_msa_wr_w(unsigned idx, union fpureg *from); +extern void write_msa_wr_d(unsigned idx, union fpureg *from); + +/** + * write_msa_wr() - Write a single MSA vector register + * @idx: The index of the vector register to write + * @from: The FPU register union to take the registers value from + * @fmt: The format of the data in the vector register + * + * Write the value from the FPU register union from into MSA vector + * register idx, using the format fmt. + */ +static inline void write_msa_wr(unsigned idx, union fpureg *from, + enum msa_2b_fmt fmt) +{ + switch (fmt) { + case msa_fmt_b: + write_msa_wr_b(idx, from); + break; + + case msa_fmt_h: + write_msa_wr_h(idx, from); + break; + + case msa_fmt_w: + write_msa_wr_w(idx, from); + break; + + case msa_fmt_d: + write_msa_wr_d(idx, from); + break; + + default: + BUG(); + } +} + static inline void enable_msa(void) { if (cpu_has_msa) { diff --git a/arch/mips/include/asm/octeon/cvmx-bootinfo.h b/arch/mips/include/asm/octeon/cvmx-bootinfo.h index c373d95b5e2c..d92cf59bdae6 100644 --- a/arch/mips/include/asm/octeon/cvmx-bootinfo.h +++ b/arch/mips/include/asm/octeon/cvmx-bootinfo.h @@ -284,6 +284,7 @@ enum cvmx_board_types_enum { CVMX_BOARD_TYPE_CUST_PRIVATE_MIN = 20001, CVMX_BOARD_TYPE_UBNT_E100 = 20002, CVMX_BOARD_TYPE_CUST_DSR1000N = 20006, + CVMX_BOARD_TYPE_KONTRON_S1901 = 21901, CVMX_BOARD_TYPE_CUST_PRIVATE_MAX = 30000, /* The remaining range is reserved for future use. */ @@ -384,6 +385,7 @@ static inline const char *cvmx_board_type_to_string(enum ENUM_BRD_TYPE_CASE(CVMX_BOARD_TYPE_CUST_PRIVATE_MIN) ENUM_BRD_TYPE_CASE(CVMX_BOARD_TYPE_UBNT_E100) ENUM_BRD_TYPE_CASE(CVMX_BOARD_TYPE_CUST_DSR1000N) + ENUM_BRD_TYPE_CASE(CVMX_BOARD_TYPE_KONTRON_S1901) ENUM_BRD_TYPE_CASE(CVMX_BOARD_TYPE_CUST_PRIVATE_MAX) } return "Unsupported Board"; diff --git a/arch/mips/include/asm/octeon/cvmx-pip.h b/arch/mips/include/asm/octeon/cvmx-pip.h index df69bfd2b006..c210154ad941 100644 --- a/arch/mips/include/asm/octeon/cvmx-pip.h +++ b/arch/mips/include/asm/octeon/cvmx-pip.h @@ -37,7 +37,7 @@ #include #include -#define CVMX_PIP_NUM_INPUT_PORTS 40 +#define CVMX_PIP_NUM_INPUT_PORTS 48 #define CVMX_PIP_NUM_WATCHERS 4 /* diff --git a/arch/mips/include/asm/octeon/cvmx-pko.h b/arch/mips/include/asm/octeon/cvmx-pko.h index 3da59bb8ce24..5f47f76ed510 100644 --- a/arch/mips/include/asm/octeon/cvmx-pko.h +++ b/arch/mips/include/asm/octeon/cvmx-pko.h @@ -542,6 +542,9 @@ static inline int cvmx_pko_get_base_queue_per_core(int port, int core) */ static inline int cvmx_pko_get_base_queue(int port) { + if (OCTEON_IS_MODEL(OCTEON_CN68XX)) + return port; + return cvmx_pko_get_base_queue_per_core(port, 0); } diff --git a/arch/mips/include/asm/octeon/cvmx-pow-defs.h b/arch/mips/include/asm/octeon/cvmx-pow-defs.h index 9020ef443736..6a3db4b068ff 100644 --- a/arch/mips/include/asm/octeon/cvmx-pow-defs.h +++ b/arch/mips/include/asm/octeon/cvmx-pow-defs.h @@ -52,6 +52,12 @@ #define CVMX_POW_WQ_INT_THRX(offset) (CVMX_ADD_IO_SEG(0x0001670000000080ull) + ((offset) & 15) * 8) #define CVMX_POW_WS_PCX(offset) (CVMX_ADD_IO_SEG(0x0001670000000280ull) + ((offset) & 15) * 8) +#define CVMX_SSO_WQ_INT (CVMX_ADD_IO_SEG(0x0001670000001000ull)) +#define CVMX_SSO_WQ_IQ_DIS (CVMX_ADD_IO_SEG(0x0001670000001010ull)) +#define CVMX_SSO_WQ_INT_PC (CVMX_ADD_IO_SEG(0x0001670000001020ull)) +#define CVMX_SSO_PPX_GRP_MSK(offset) (CVMX_ADD_IO_SEG(0x0001670000006000ull) + ((offset) & 31) * 8) +#define CVMX_SSO_WQ_INT_THRX(offset) (CVMX_ADD_IO_SEG(0x0001670000007000ull) + ((offset) & 63) * 8) + union cvmx_pow_bist_stat { uint64_t u64; struct cvmx_pow_bist_stat_s { @@ -1286,4 +1292,27 @@ union cvmx_pow_ws_pcx { struct cvmx_pow_ws_pcx_s cnf71xx; }; +union cvmx_sso_wq_int_thrx { + uint64_t u64; + struct { +#ifdef __BIG_ENDIAN_BITFIELD + uint64_t reserved_33_63:31; + uint64_t tc_en:1; + uint64_t tc_thr:4; + uint64_t reserved_26_27:2; + uint64_t ds_thr:12; + uint64_t reserved_12_13:2; + uint64_t iq_thr:12; +#else + uint64_t iq_thr:12; + uint64_t reserved_12_13:2; + uint64_t ds_thr:12; + uint64_t reserved_26_27:2; + uint64_t tc_thr:4; + uint64_t tc_en:1; + uint64_t reserved_33_63:31; +#endif + } s; +}; + #endif diff --git a/arch/mips/include/asm/octeon/cvmx-pow.h b/arch/mips/include/asm/octeon/cvmx-pow.h index d5565d758ddd..51531563f8dc 100644 --- a/arch/mips/include/asm/octeon/cvmx-pow.h +++ b/arch/mips/include/asm/octeon/cvmx-pow.h @@ -1810,10 +1810,11 @@ static inline void cvmx_pow_work_submit(cvmx_wqe_t *wqp, uint32_t tag, cvmx_addr_t ptr; cvmx_pow_tag_req_t tag_req; - wqp->qos = qos; - wqp->tag = tag; - wqp->tag_type = tag_type; - wqp->grp = grp; + wqp->word1.tag = tag; + wqp->word1.tag_type = tag_type; + + cvmx_wqe_set_qos(wqp, qos); + cvmx_wqe_set_grp(wqp, grp); tag_req.u64 = 0; tag_req.s.op = CVMX_POW_TAG_OP_ADDWQ; diff --git a/arch/mips/include/asm/octeon/cvmx-wqe.h b/arch/mips/include/asm/octeon/cvmx-wqe.h index 2d6d0c7127a7..0d697aa786d4 100644 --- a/arch/mips/include/asm/octeon/cvmx-wqe.h +++ b/arch/mips/include/asm/octeon/cvmx-wqe.h @@ -193,6 +193,53 @@ typedef union { uint64_t bufs:8; #endif } s; + struct { +#ifdef __BIG_ENDIAN_BITFIELD + uint64_t bufs:8; + uint64_t ip_offset:8; + uint64_t vlan_valid:1; + uint64_t vlan_stacked:1; + uint64_t unassigned:1; + uint64_t vlan_cfi:1; + uint64_t vlan_id:12; + uint64_t port:12; /* MAC/PIP port number. */ + uint64_t dec_ipcomp:1; + uint64_t tcp_or_udp:1; + uint64_t dec_ipsec:1; + uint64_t is_v6:1; + uint64_t software:1; + uint64_t L4_error:1; + uint64_t is_frag:1; + uint64_t IP_exc:1; + uint64_t is_bcast:1; + uint64_t is_mcast:1; + uint64_t not_IP:1; + uint64_t rcv_error:1; + uint64_t err_code:8; +#else + uint64_t err_code:8; + uint64_t rcv_error:1; + uint64_t not_IP:1; + uint64_t is_mcast:1; + uint64_t is_bcast:1; + uint64_t IP_exc:1; + uint64_t is_frag:1; + uint64_t L4_error:1; + uint64_t software:1; + uint64_t is_v6:1; + uint64_t dec_ipsec:1; + uint64_t tcp_or_udp:1; + uint64_t dec_ipcomp:1; + uint64_t port:12; + uint64_t vlan_id:12; + uint64_t vlan_cfi:1; + uint64_t unassigned:1; + uint64_t vlan_stacked:1; + uint64_t vlan_valid:1; + uint64_t ip_offset:8; + uint64_t bufs:8; +#endif + } s_cn68xx; /* use this to get at the 16 vlan bits */ struct { @@ -355,6 +402,146 @@ typedef union { } cvmx_pip_wqe_word2; +union cvmx_pip_wqe_word0 { + struct { +#ifdef __BIG_ENDIAN_BITFIELD + /** + * raw chksum result generated by the HW + */ + uint16_t hw_chksum; + /** + * Field unused by hardware - available for software + */ + uint8_t unused; + /** + * Next pointer used by hardware for list maintenance. + * May be written/read by HW before the work queue + * entry is scheduled to a PP (Only 36 bits used in + * Octeon 1) + */ + uint64_t next_ptr:40; +#else + uint64_t next_ptr:40; + uint8_t unused; + uint16_t hw_chksum; +#endif + } cn38xx; + struct { +#ifdef __BIG_ENDIAN_BITFIELD + uint64_t l4ptr:8; /* 56..63 */ + uint64_t unused0:8; /* 48..55 */ + uint64_t l3ptr:8; /* 40..47 */ + uint64_t l2ptr:8; /* 32..39 */ + uint64_t unused1:18; /* 14..31 */ + uint64_t bpid:6; /* 8..13 */ + uint64_t unused2:2; /* 6..7 */ + uint64_t pknd:6; /* 0..5 */ +#else + uint64_t pknd:6; /* 0..5 */ + uint64_t unused2:2; /* 6..7 */ + uint64_t bpid:6; /* 8..13 */ + uint64_t unused1:18; /* 14..31 */ + uint64_t l2ptr:8; /* 32..39 */ + uint64_t l3ptr:8; /* 40..47 */ + uint64_t unused0:8; /* 48..55 */ + uint64_t l4ptr:8; /* 56..63 */ +#endif + } cn68xx; +}; + +union cvmx_wqe_word0 { + uint64_t u64; + union cvmx_pip_wqe_word0 pip; +}; + +union cvmx_wqe_word1 { + uint64_t u64; + struct { +#ifdef __BIG_ENDIAN_BITFIELD + uint64_t len:16; + uint64_t varies:14; + /** + * the type of the tag (ORDERED, ATOMIC, NULL) + */ + uint64_t tag_type:2; + uint64_t tag:32; +#else + uint64_t tag:32; + uint64_t tag_type:2; + uint64_t varies:14; + uint64_t len:16; +#endif + }; + struct { +#ifdef __BIG_ENDIAN_BITFIELD + uint64_t len:16; + uint64_t zero_0:1; + /** + * HW sets this to what it thought the priority of + * the input packet was + */ + uint64_t qos:3; + + uint64_t zero_1:1; + /** + * the group that the work queue entry will be scheduled to + */ + uint64_t grp:6; + uint64_t zero_2:3; + uint64_t tag_type:2; + uint64_t tag:32; +#else + uint64_t tag:32; + uint64_t tag_type:2; + uint64_t zero_2:3; + uint64_t grp:6; + uint64_t zero_1:1; + uint64_t qos:3; + uint64_t zero_0:1; + uint64_t len:16; +#endif + } cn68xx; + struct { +#ifdef __BIG_ENDIAN_BITFIELD + /** + * HW sets to the total number of bytes in the packet + */ + uint64_t len:16; + /** + * HW sets this to input physical port + */ + uint64_t ipprt:6; + + /** + * HW sets this to what it thought the priority of + * the input packet was + */ + uint64_t qos:3; + + /** + * the group that the work queue entry will be scheduled to + */ + uint64_t grp:4; + /** + * the type of the tag (ORDERED, ATOMIC, NULL) + */ + uint64_t tag_type:3; + /** + * the synchronization/ordering tag + */ + uint64_t tag:32; +#else + uint64_t tag:32; + uint64_t tag_type:2; + uint64_t zero_2:1; + uint64_t grp:4; + uint64_t qos:3; + uint64_t ipprt:6; + uint64_t len:16; +#endif + } cn38xx; +}; + /** * Work queue entry format * @@ -366,70 +553,13 @@ typedef struct { * WORD 0 * HW WRITE: the following 64 bits are filled by HW when a packet arrives */ - -#ifdef __BIG_ENDIAN_BITFIELD - /** - * raw chksum result generated by the HW - */ - uint16_t hw_chksum; - /** - * Field unused by hardware - available for software - */ - uint8_t unused; - /** - * Next pointer used by hardware for list maintenance. - * May be written/read by HW before the work queue - * entry is scheduled to a PP - * (Only 36 bits used in Octeon 1) - */ - uint64_t next_ptr:40; -#else - uint64_t next_ptr:40; - uint8_t unused; - uint16_t hw_chksum; -#endif + union cvmx_wqe_word0 word0; /***************************************************************** * WORD 1 * HW WRITE: the following 64 bits are filled by HW when a packet arrives */ - -#ifdef __BIG_ENDIAN_BITFIELD - /** - * HW sets to the total number of bytes in the packet - */ - uint64_t len:16; - /** - * HW sets this to input physical port - */ - uint64_t ipprt:6; - - /** - * HW sets this to what it thought the priority of the input packet was - */ - uint64_t qos:3; - - /** - * the group that the work queue entry will be scheduled to - */ - uint64_t grp:4; - /** - * the type of the tag (ORDERED, ATOMIC, NULL) - */ - uint64_t tag_type:3; - /** - * the synchronization/ordering tag - */ - uint64_t tag:32; -#else - uint64_t tag:32; - uint64_t tag_type:2; - uint64_t zero_2:1; - uint64_t grp:4; - uint64_t qos:3; - uint64_t ipprt:6; - uint64_t len:16; -#endif + union cvmx_wqe_word1 word1; /** * WORD 2 HW WRITE: the following 64-bits are filled in by @@ -465,4 +595,64 @@ typedef struct { } CVMX_CACHE_LINE_ALIGNED cvmx_wqe_t; +static inline int cvmx_wqe_get_port(cvmx_wqe_t *work) +{ + int port; + + if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) + port = work->word2.s_cn68xx.port; + else + port = work->word1.cn38xx.ipprt; + + return port; +} + +static inline void cvmx_wqe_set_port(cvmx_wqe_t *work, int port) +{ + if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) + work->word2.s_cn68xx.port = port; + else + work->word1.cn38xx.ipprt = port; +} + +static inline int cvmx_wqe_get_grp(cvmx_wqe_t *work) +{ + int grp; + + if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) + grp = work->word1.cn68xx.grp; + else + grp = work->word1.cn38xx.grp; + + return grp; +} + +static inline void cvmx_wqe_set_grp(cvmx_wqe_t *work, int grp) +{ + if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) + work->word1.cn68xx.grp = grp; + else + work->word1.cn38xx.grp = grp; +} + +static inline int cvmx_wqe_get_qos(cvmx_wqe_t *work) +{ + int qos; + + if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) + qos = work->word1.cn68xx.qos; + else + qos = work->word1.cn38xx.qos; + + return qos; +} + +static inline void cvmx_wqe_set_qos(cvmx_wqe_t *work, int qos) +{ + if (octeon_has_feature(OCTEON_FEATURE_CN68XX_WQE)) + work->word1.cn68xx.qos = qos; + else + work->word1.cn38xx.qos = qos; +} + #endif /* __CVMX_WQE_H__ */ diff --git a/arch/mips/include/asm/pgtable-bits.h b/arch/mips/include/asm/pgtable-bits.h index c28a8499aec7..ff7ad91c85db 100644 --- a/arch/mips/include/asm/pgtable-bits.h +++ b/arch/mips/include/asm/pgtable-bits.h @@ -133,20 +133,13 @@ #define _PAGE_HUGE (1 << _PAGE_HUGE_SHIFT) #define _PAGE_SPLITTING_SHIFT (_PAGE_HUGE_SHIFT + 1) #define _PAGE_SPLITTING (1 << _PAGE_SPLITTING_SHIFT) - -/* Only R2 or newer cores have the XI bit */ -#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6) -#define _PAGE_NO_EXEC_SHIFT (_PAGE_SPLITTING_SHIFT + 1) -#else -#define _PAGE_GLOBAL_SHIFT (_PAGE_SPLITTING_SHIFT + 1) -#define _PAGE_GLOBAL (1 << _PAGE_GLOBAL_SHIFT) -#endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */ - #endif /* CONFIG_64BIT && CONFIG_MIPS_HUGE_TLB_SUPPORT */ #if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6) /* XI - page cannot be executed */ -#ifndef _PAGE_NO_EXEC_SHIFT +#ifdef _PAGE_SPLITTING_SHIFT +#define _PAGE_NO_EXEC_SHIFT (_PAGE_SPLITTING_SHIFT + 1) +#else #define _PAGE_NO_EXEC_SHIFT (_PAGE_MODIFIED_SHIFT + 1) #endif #define _PAGE_NO_EXEC (cpu_has_rixi ? (1 << _PAGE_NO_EXEC_SHIFT) : 0) @@ -156,14 +149,16 @@ #define _PAGE_READ (cpu_has_rixi ? 0 : (1 << _PAGE_READ_SHIFT)) #define _PAGE_NO_READ_SHIFT _PAGE_READ_SHIFT #define _PAGE_NO_READ (cpu_has_rixi ? (1 << _PAGE_READ_SHIFT) : 0) +#endif /* defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6) */ +#if defined(_PAGE_NO_READ_SHIFT) #define _PAGE_GLOBAL_SHIFT (_PAGE_NO_READ_SHIFT + 1) -#define _PAGE_GLOBAL (1 << _PAGE_GLOBAL_SHIFT) - -#else /* !CONFIG_CPU_MIPSR2 && !CONFIG_CPU_MIPSR6 */ +#elif defined(_PAGE_SPLITTING_SHIFT) +#define _PAGE_GLOBAL_SHIFT (_PAGE_SPLITTING_SHIFT + 1) +#else #define _PAGE_GLOBAL_SHIFT (_PAGE_MODIFIED_SHIFT + 1) +#endif #define _PAGE_GLOBAL (1 << _PAGE_GLOBAL_SHIFT) -#endif /* CONFIG_CPU_MIPSR2 || CONFIG_CPU_MIPSR6 */ #define _PAGE_VALID_SHIFT (_PAGE_GLOBAL_SHIFT + 1) #define _PAGE_VALID (1 << _PAGE_VALID_SHIFT) @@ -249,7 +244,7 @@ static inline uint64_t pte_to_entrylo(unsigned long pte_val) #define _CACHE_CACHABLE_NONCOHERENT (3<<_CACHE_SHIFT) /* LOONGSON */ #define _CACHE_CACHABLE_COHERENT (3<<_CACHE_SHIFT) /* LOONGSON-3 */ -#elif defined(CONFIG_MACH_JZ4740) +#elif defined(CONFIG_MACH_INGENIC) /* Ingenic uses the WA bit to achieve write-combine memory writes */ #define _CACHE_UNCACHED_ACCELERATED (1<<_CACHE_SHIFT) diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h index ae8569475264..8957f15e21ec 100644 --- a/arch/mips/include/asm/pgtable.h +++ b/arch/mips/include/asm/pgtable.h @@ -393,6 +393,8 @@ static inline pgprot_t pgprot_noncached(pgprot_t _prot) return __pgprot(prot); } +#define pgprot_writecombine pgprot_writecombine + static inline pgprot_t pgprot_writecombine(pgprot_t _prot) { unsigned long prot = pgprot_val(_prot); diff --git a/arch/mips/include/asm/processor.h b/arch/mips/include/asm/processor.h index 9b3b48e21c22..59ee6dcf6eed 100644 --- a/arch/mips/include/asm/processor.h +++ b/arch/mips/include/asm/processor.h @@ -275,6 +275,7 @@ struct thread_struct { unsigned long cp0_badvaddr; /* Last user fault */ unsigned long cp0_baduaddr; /* Last kernel fault accessing USEG */ unsigned long error_code; + unsigned long trap_nr; #ifdef CONFIG_CPU_CAVIUM_OCTEON struct octeon_cop2_state cp2 __attribute__ ((__aligned__(128))); struct octeon_cvmseg_state cvmseg __attribute__ ((__aligned__(128))); @@ -341,6 +342,7 @@ struct thread_struct { .cp0_badvaddr = 0, \ .cp0_baduaddr = 0, \ .error_code = 0, \ + .trap_nr = 0, \ /* \ * Platform specific cop2 registers(null if no COP2) \ */ \ diff --git a/arch/mips/include/asm/ptrace.h b/arch/mips/include/asm/ptrace.h index ffc320389f40..f6fc6aac5496 100644 --- a/arch/mips/include/asm/ptrace.h +++ b/arch/mips/include/asm/ptrace.h @@ -14,11 +14,16 @@ #include #include #include +#include +#include #include /* * This struct defines the way the registers are stored on the stack during a * system call/exception. As usual the registers k0/k1 aren't being saved. + * + * If you add a register here, also add it to regoffset_table[] in + * arch/mips/kernel/ptrace.c. */ struct pt_regs { #ifdef CONFIG_32BIT @@ -43,8 +48,83 @@ struct pt_regs { unsigned long long mpl[6]; /* MTM{0-5} */ unsigned long long mtp[6]; /* MTP{0-5} */ #endif + unsigned long __last[0]; } __aligned(8); +static inline unsigned long kernel_stack_pointer(struct pt_regs *regs) +{ + return regs->regs[31]; +} + +/* + * Don't use asm-generic/ptrace.h it defines FP accessors that don't make + * sense on MIPS. We rather want an error if they get invoked. + */ + +static inline void instruction_pointer_set(struct pt_regs *regs, + unsigned long val) +{ + regs->cp0_epc = val; +} + +/* Query offset/name of register from its name/offset */ +extern int regs_query_register_offset(const char *name); +#define MAX_REG_OFFSET (offsetof(struct pt_regs, __last)) + +/** + * regs_get_register() - get register value from its offset + * @regs: pt_regs from which register value is gotten. + * @offset: offset number of the register. + * + * regs_get_register returns the value of a register. The @offset is the + * offset of the register in struct pt_regs address which specified by @regs. + * If @offset is bigger than MAX_REG_OFFSET, this returns 0. + */ +static inline unsigned long regs_get_register(struct pt_regs *regs, + unsigned int offset) +{ + if (unlikely(offset > MAX_REG_OFFSET)) + return 0; + + return *(unsigned long *)((unsigned long)regs + offset); +} + +/** + * regs_within_kernel_stack() - check the address in the stack + * @regs: pt_regs which contains kernel stack pointer. + * @addr: address which is checked. + * + * regs_within_kernel_stack() checks @addr is within the kernel stack page(s). + * If @addr is within the kernel stack, it returns true. If not, returns false. + */ +static inline int regs_within_kernel_stack(struct pt_regs *regs, + unsigned long addr) +{ + return ((addr & ~(THREAD_SIZE - 1)) == + (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))); +} + +/** + * regs_get_kernel_stack_nth() - get Nth entry of the stack + * @regs: pt_regs which contains kernel stack pointer. + * @n: stack entry number. + * + * regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which + * is specified by @regs. If the @n th entry is NOT in the kernel stack, + * this returns 0. + */ +static inline unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, + unsigned int n) +{ + unsigned long *addr = (unsigned long *)kernel_stack_pointer(regs); + + addr += n; + if (regs_within_kernel_stack(regs, (unsigned long)addr)) + return *addr; + else + return 0; +} + struct task_struct; extern int ptrace_getregs(struct task_struct *child, diff --git a/arch/mips/include/asm/signal.h b/arch/mips/include/asm/signal.h index 8efe5a9e2c3e..003e273eff4c 100644 --- a/arch/mips/include/asm/signal.h +++ b/arch/mips/include/asm/signal.h @@ -23,4 +23,7 @@ #define __ARCH_HAS_IRIX_SIGACTION +extern int protected_save_fp_context(void __user *sc); +extern int protected_restore_fp_context(void __user *sc); + #endif /* _ASM_SIGNAL_H */ diff --git a/arch/mips/include/asm/spinlock.h b/arch/mips/include/asm/spinlock.h index 9de4ba43dcd1..40196bebe849 100644 --- a/arch/mips/include/asm/spinlock.h +++ b/arch/mips/include/asm/spinlock.h @@ -42,6 +42,11 @@ static inline int arch_spin_is_locked(arch_spinlock_t *lock) return ((counters >> 16) ^ counters) & 0xffff; } +static inline int arch_spin_value_unlocked(arch_spinlock_t lock) +{ + return lock.h.serving_now == lock.h.ticket; +} + #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) #define arch_spin_unlock_wait(x) \ while (arch_spin_is_locked(x)) { cpu_relax(); } diff --git a/arch/mips/include/asm/switch_to.h b/arch/mips/include/asm/switch_to.h index 9733cd0266e4..28b5d84a5022 100644 --- a/arch/mips/include/asm/switch_to.h +++ b/arch/mips/include/asm/switch_to.h @@ -16,29 +16,21 @@ #include #include #include -#include +#include struct task_struct; -enum { - FP_SAVE_NONE = 0, - FP_SAVE_VECTOR = -1, - FP_SAVE_SCALAR = 1, -}; - /** * resume - resume execution of a task * @prev: The task previously executed. * @next: The task to begin executing. * @next_ti: task_thread_info(next). - * @fp_save: Which, if any, FP context to save for prev. * * This function is used whilst scheduling to save the context of prev & load * the context of next. Returns prev. */ extern asmlinkage struct task_struct *resume(struct task_struct *prev, - struct task_struct *next, struct thread_info *next_ti, - s32 fp_save); + struct task_struct *next, struct thread_info *next_ti); extern unsigned int ll_bit; extern struct task_struct *ll_task; @@ -91,8 +83,8 @@ do { if (cpu_has_rw_llb) { \ */ #define switch_to(prev, next, last) \ do { \ - s32 __fpsave = FP_SAVE_NONE; \ __mips_mt_fpaff_switch_to(prev); \ + lose_fpu_inatomic(1, prev); \ if (cpu_has_dsp) { \ __save_dsp(prev); \ __restore_dsp(next); \ @@ -111,15 +103,10 @@ do { \ clear_c0_status(ST0_CU2); \ } \ __clear_software_ll_bit(); \ - if (test_and_clear_tsk_thread_flag(prev, TIF_USEDFPU)) \ - __fpsave = FP_SAVE_SCALAR; \ - if (test_and_clear_tsk_thread_flag(prev, TIF_USEDMSA)) \ - __fpsave = FP_SAVE_VECTOR; \ if (cpu_has_userlocal) \ write_c0_userlocal(task_thread_info(next)->tp_value); \ __restore_watch(); \ - disable_msa(); \ - (last) = resume(prev, next, task_thread_info(next), __fpsave); \ + (last) = resume(prev, next, task_thread_info(next)); \ } while (0) #endif /* _ASM_SWITCH_TO_H */ diff --git a/arch/mips/include/asm/thread_info.h b/arch/mips/include/asm/thread_info.h index 9c0014e87c17..e309d8fcb516 100644 --- a/arch/mips/include/asm/thread_info.h +++ b/arch/mips/include/asm/thread_info.h @@ -99,6 +99,7 @@ static inline struct thread_info *current_thread_info(void) #define TIF_SYSCALL_AUDIT 3 /* syscall auditing active */ #define TIF_SECCOMP 4 /* secure computing */ #define TIF_NOTIFY_RESUME 5 /* callback before returning to user */ +#define TIF_UPROBE 6 /* breakpointed or singlestepping */ #define TIF_RESTORE_SIGMASK 9 /* restore signal mask in do_signal() */ #define TIF_USEDFPU 16 /* FPU was used by this task this quantum (SMP) */ #define TIF_MEMDIE 18 /* is terminating due to OOM killer */ @@ -122,6 +123,7 @@ static inline struct thread_info *current_thread_info(void) #define _TIF_SYSCALL_AUDIT (1< +#include + +#include +#include + +/* + * We want this to be defined as union mips_instruction but that makes the + * generic code blow up. + */ +typedef u32 uprobe_opcode_t; + +/* + * Classic MIPS (note this implementation doesn't consider microMIPS yet) + * instructions are always 4 bytes but in order to deal with branches and + * their delay slots, we treat instructions as having 8 bytes maximum. + */ +#define MAX_UINSN_BYTES 8 +#define UPROBE_XOL_SLOT_BYTES 128 /* Max. cache line size */ + +#define UPROBE_BRK_UPROBE 0x000d000d /* break 13 */ +#define UPROBE_BRK_UPROBE_XOL 0x000e000d /* break 14 */ + +#define UPROBE_SWBP_INSN UPROBE_BRK_UPROBE +#define UPROBE_SWBP_INSN_SIZE 4 + +struct arch_uprobe { + unsigned long resume_epc; + u32 insn[2]; + u32 ixol[2]; + union mips_instruction orig_inst[MAX_UINSN_BYTES / 4]; +}; + +struct arch_uprobe_task { + unsigned long saved_trap_nr; +}; + +extern int arch_uprobe_analyze_insn(struct arch_uprobe *aup, + struct mm_struct *mm, unsigned long addr); +extern int arch_uprobe_pre_xol(struct arch_uprobe *aup, struct pt_regs *regs); +extern int arch_uprobe_post_xol(struct arch_uprobe *aup, struct pt_regs *regs); +extern bool arch_uprobe_xol_was_trapped(struct task_struct *tsk); +extern int arch_uprobe_exception_notify(struct notifier_block *self, + unsigned long val, void *data); +extern void arch_uprobe_abort_xol(struct arch_uprobe *aup, + struct pt_regs *regs); +extern unsigned long arch_uretprobe_hijack_return_addr( + unsigned long trampoline_vaddr, struct pt_regs *regs); + +#endif /* __ASM_UPROBES_H */ diff --git a/arch/mips/include/asm/vpe.h b/arch/mips/include/asm/vpe.h index 7849f3978fea..80e70dbd1f64 100644 --- a/arch/mips/include/asm/vpe.h +++ b/arch/mips/include/asm/vpe.h @@ -122,7 +122,7 @@ void release_vpe(struct vpe *v); void *alloc_progmem(unsigned long len); void release_progmem(void *ptr); -int __weak vpe_run(struct vpe *v); +int vpe_run(struct vpe *v); void cleanup_tc(struct tc *tc); int __init vpe_module_init(void); diff --git a/arch/mips/include/uapi/asm/break.h b/arch/mips/include/uapi/asm/break.h index 002c39ea20c3..9c4265cbf151 100644 --- a/arch/mips/include/uapi/asm/break.h +++ b/arch/mips/include/uapi/asm/break.h @@ -21,6 +21,8 @@ #define BRK_DIVZERO 7 /* Divide by zero check */ #define BRK_RANGE 8 /* Range error check */ #define BRK_BUG 12 /* Used by BUG() */ +#define BRK_UPROBE 13 /* See */ +#define BRK_UPROBE_XOL 14 /* See */ #define BRK_MEMU 514 /* Used by FPU emulator */ #define BRK_KPROBE_BP 515 /* Kprobe break */ #define BRK_KPROBE_SSTEPBP 516 /* Kprobe single step software implementation */ diff --git a/arch/mips/include/uapi/asm/hwcap.h b/arch/mips/include/uapi/asm/hwcap.h new file mode 100644 index 000000000000..c7484a7ca686 --- /dev/null +++ b/arch/mips/include/uapi/asm/hwcap.h @@ -0,0 +1,8 @@ +#ifndef _UAPI_ASM_HWCAP_H +#define _UAPI_ASM_HWCAP_H + +/* HWCAP flags */ +#define HWCAP_MIPS_R6 (1 << 0) +#define HWCAP_MIPS_MSA (1 << 1) + +#endif /* _UAPI_ASM_HWCAP_H */ diff --git a/arch/mips/include/uapi/asm/inst.h b/arch/mips/include/uapi/asm/inst.h index fc0cf5ac0cf7..9b44d5a816fa 100644 --- a/arch/mips/include/uapi/asm/inst.h +++ b/arch/mips/include/uapi/asm/inst.h @@ -26,7 +26,7 @@ enum major_op { cop0_op, cop1_op, cop2_op, cop1x_op, beql_op, bnel_op, blezl_op, bgtzl_op, daddi_op, cbcond1_op = daddi_op, daddiu_op, ldl_op, ldr_op, - spec2_op, jalx_op, mdmx_op, spec3_op, + spec2_op, jalx_op, mdmx_op, msa_op = mdmx_op, spec3_op, lb_op, lh_op, lwl_op, lw_op, lbu_op, lhu_op, lwr_op, lwu_op, sb_op, sh_op, swl_op, sw_op, @@ -167,8 +167,13 @@ enum cop1_sdw_func { fround_op = 0x0c, ftrunc_op = 0x0d, fceil_op = 0x0e, ffloor_op = 0x0f, fmovc_op = 0x11, fmovz_op = 0x12, - fmovn_op = 0x13, frecip_op = 0x15, - frsqrt_op = 0x16, fcvts_op = 0x20, + fmovn_op = 0x13, fseleqz_op = 0x14, + frecip_op = 0x15, frsqrt_op = 0x16, + fselnez_op = 0x17, fmaddf_op = 0x18, + fmsubf_op = 0x19, frint_op = 0x1a, + fclass_op = 0x1b, fmin_op = 0x1c, + fmina_op = 0x1d, fmax_op = 0x1e, + fmaxa_op = 0x1f, fcvts_op = 0x20, fcvtd_op = 0x21, fcvte_op = 0x22, fcvtw_op = 0x24, fcvtl_op = 0x25, fcmp_op = 0x30 @@ -220,6 +225,24 @@ enum bshfl_func { seh_op = 0x18, }; +/* + * func field for MSA MI10 format. + */ +enum msa_mi10_func { + msa_ld_op = 8, + msa_st_op = 9, +}; + +/* + * MSA 2 bit format fields. + */ +enum msa_2b_fmt { + msa_fmt_b = 0, + msa_fmt_h = 1, + msa_fmt_w = 2, + msa_fmt_d = 3, +}; + /* * (microMIPS) Major opcodes. */ @@ -611,6 +634,16 @@ struct v_format { /* MDMX vector format */ ;))))))) }; +struct msa_mi10_format { /* MSA MI10 */ + __BITFIELD_FIELD(unsigned int opcode : 6, + __BITFIELD_FIELD(signed int s10 : 10, + __BITFIELD_FIELD(unsigned int rs : 5, + __BITFIELD_FIELD(unsigned int wd : 5, + __BITFIELD_FIELD(unsigned int func : 4, + __BITFIELD_FIELD(unsigned int df : 2, + ;)))))) +}; + struct spec3_format { /* SPEC3 */ __BITFIELD_FIELD(unsigned int opcode:6, __BITFIELD_FIELD(unsigned int rs:5, @@ -888,6 +921,7 @@ union mips_instruction { struct p_format p_format; struct f_format f_format; struct ma_format ma_format; + struct msa_mi10_format msa_mi10_format; struct b_format b_format; struct ps_format ps_format; struct v_format v_format; diff --git a/arch/mips/include/uapi/asm/sigcontext.h b/arch/mips/include/uapi/asm/sigcontext.h index 9081d88ae44f..5cbd9ae6421f 100644 --- a/arch/mips/include/uapi/asm/sigcontext.h +++ b/arch/mips/include/uapi/asm/sigcontext.h @@ -12,6 +12,18 @@ #include #include +/* scalar FP context was used */ +#define USED_FP (1 << 0) + +/* the value of Status.FR when context was saved */ +#define USED_FR1 (1 << 1) + +/* FR=1, but with odd singles in bits 63:32 of preceding even double */ +#define USED_HYBRID_FPRS (1 << 2) + +/* extended context was used, see struct extcontext for details */ +#define USED_EXTCONTEXT (1 << 3) + #if _MIPS_SIM == _MIPS_SIM_ABI32 /* diff --git a/arch/mips/include/uapi/asm/swab.h b/arch/mips/include/uapi/asm/swab.h index 8f2d184dbe9f..c4ddc4f0d2dc 100644 --- a/arch/mips/include/uapi/asm/swab.h +++ b/arch/mips/include/uapi/asm/swab.h @@ -16,11 +16,13 @@ #if (defined(__mips_isa_rev) && (__mips_isa_rev >= 2)) || \ defined(_MIPS_ARCH_LOONGSON3A) -static inline __attribute_const__ __u16 __arch_swab16(__u16 x) +static inline __attribute__((nomips16)) __attribute_const__ + __u16 __arch_swab16(__u16 x) { __asm__( " .set push \n" " .set arch=mips32r2 \n" + " .set nomips16 \n" " wsbh %0, %1 \n" " .set pop \n" : "=r" (x) @@ -30,11 +32,13 @@ static inline __attribute_const__ __u16 __arch_swab16(__u16 x) } #define __arch_swab16 __arch_swab16 -static inline __attribute_const__ __u32 __arch_swab32(__u32 x) +static inline __attribute__((nomips16)) __attribute_const__ + __u32 __arch_swab32(__u32 x) { __asm__( " .set push \n" " .set arch=mips32r2 \n" + " .set nomips16 \n" " wsbh %0, %1 \n" " rotr %0, %0, 16 \n" " .set pop \n" @@ -50,11 +54,13 @@ static inline __attribute_const__ __u32 __arch_swab32(__u32 x) * 64-bit kernel on r2 CPUs. */ #ifdef __mips64 -static inline __attribute_const__ __u64 __arch_swab64(__u64 x) +static inline __attribute__((nomips16)) __attribute_const__ + __u64 __arch_swab64(__u64 x) { __asm__( " .set push \n" " .set arch=mips64r2 \n" + " .set nomips16 \n" " dsbh %0, %1 \n" " dshd %0, %0 \n" " .set pop \n" diff --git a/arch/mips/include/uapi/asm/ucontext.h b/arch/mips/include/uapi/asm/ucontext.h new file mode 100644 index 000000000000..2320144ce858 --- /dev/null +++ b/arch/mips/include/uapi/asm/ucontext.h @@ -0,0 +1,65 @@ +#ifndef __MIPS_UAPI_ASM_UCONTEXT_H +#define __MIPS_UAPI_ASM_UCONTEXT_H + +/** + * struct extcontext - extended context header structure + * @magic: magic value identifying the type of extended context + * @size: the size in bytes of the enclosing structure + * + * Extended context structures provide context which does not fit within struct + * sigcontext. They are placed sequentially in memory at the end of struct + * ucontext and struct sigframe, with each extended context structure beginning + * with a header defined by this struct. The type of context represented is + * indicated by the magic field. Userland may check each extended context + * structure against magic values that it recognises. The size field allows any + * unrecognised context to be skipped, allowing for future expansion. The end + * of the extended context data is indicated by the magic value + * END_EXTCONTEXT_MAGIC. + */ +struct extcontext { + unsigned int magic; + unsigned int size; +}; + +/** + * struct msa_extcontext - MSA extended context structure + * @ext: the extended context header, with magic == MSA_EXTCONTEXT_MAGIC + * @wr: the most significant 64 bits of each MSA vector register + * @csr: the value of the MSA control & status register + * + * If MSA context is live for a task at the time a signal is delivered to it, + * this structure will hold the MSA context of the task as it was prior to the + * signal delivery. + */ +struct msa_extcontext { + struct extcontext ext; +#define MSA_EXTCONTEXT_MAGIC 0x784d5341 /* xMSA */ + + unsigned long long wr[32]; + unsigned int csr; +}; + +#define END_EXTCONTEXT_MAGIC 0x78454e44 /* xEND */ + +/** + * struct ucontext - user context structure + * @uc_flags: + * @uc_link: + * @uc_stack: + * @uc_mcontext: holds basic processor state + * @uc_sigmask: + * @uc_extcontext: holds extended processor state + */ +struct ucontext { + /* Historic fields matching asm-generic */ + unsigned long uc_flags; + struct ucontext *uc_link; + stack_t uc_stack; + struct sigcontext uc_mcontext; + sigset_t uc_sigmask; + + /* Extended context structures may follow ucontext */ + unsigned long long uc_extcontext[0]; +}; + +#endif /* __MIPS_UAPI_ASM_UCONTEXT_H */ diff --git a/arch/mips/jazz/irq.c b/arch/mips/jazz/irq.c index e1ea4f625f7a..5d6828b2a750 100644 --- a/arch/mips/jazz/irq.c +++ b/arch/mips/jazz/irq.c @@ -110,18 +110,11 @@ asmlinkage void plat_irq_dispatch(void) } } -static void r4030_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) -{ - /* Nothing to do ... */ -} - struct clock_event_device r4030_clockevent = { .name = "r4030", .features = CLOCK_EVT_FEAT_PERIODIC, .rating = 300, .irq = JAZZ_TIMER_IRQ, - .set_mode = r4030_set_mode, }; static irqreturn_t r4030_timer_interrupt(int irq, void *dev_id) diff --git a/arch/mips/jz4740/gpio.c b/arch/mips/jz4740/gpio.c index 54c80d42a88d..6cd69fdaa1c5 100644 --- a/arch/mips/jz4740/gpio.c +++ b/arch/mips/jz4740/gpio.c @@ -231,6 +231,13 @@ static int jz_gpio_direction_input(struct gpio_chip *chip, unsigned gpio) return 0; } +static int jz_gpio_to_irq(struct gpio_chip *chip, unsigned gpio) +{ + struct jz_gpio_chip *jz_gpio = gpio_chip_to_jz_gpio_chip(chip); + + return jz_gpio->irq_base + gpio; +} + int jz_gpio_port_direction_input(int port, uint32_t mask) { writel(mask, GPIO_TO_REG(port, JZ_REG_GPIO_DIRECTION_CLEAR)); @@ -262,18 +269,6 @@ uint32_t jz_gpio_port_get_value(int port, uint32_t mask) } EXPORT_SYMBOL(jz_gpio_port_get_value); -int gpio_to_irq(unsigned gpio) -{ - return JZ4740_IRQ_GPIO(0) + gpio; -} -EXPORT_SYMBOL_GPL(gpio_to_irq); - -int irq_to_gpio(unsigned irq) -{ - return irq - JZ4740_IRQ_GPIO(0); -} -EXPORT_SYMBOL_GPL(irq_to_gpio); - #define IRQ_TO_BIT(irq) BIT(irq_to_gpio(irq) & 0x1f) static void jz_gpio_check_trigger_both(struct jz_gpio_chip *chip, unsigned int irq) @@ -403,6 +398,7 @@ static int jz_gpio_irq_set_wake(struct irq_data *data, unsigned int on) .get = jz_gpio_get_value, \ .direction_output = jz_gpio_direction_output, \ .direction_input = jz_gpio_direction_input, \ + .to_irq = jz_gpio_to_irq, \ .base = JZ4740_GPIO_BASE_ ## _bank, \ .ngpio = JZ4740_GPIO_NUM_ ## _bank, \ }, \ @@ -423,8 +419,8 @@ static void jz4740_gpio_chip_init(struct jz_gpio_chip *chip, unsigned int id) chip->base = ioremap(JZ4740_GPIO_BASE_ADDR + (id * 0x100), 0x100); chip->irq = JZ4740_IRQ_INTC_GPIO(id); - irq_set_handler_data(chip->irq, chip); - irq_set_chained_handler(chip->irq, jz_gpio_irq_demux_handler); + irq_set_chained_handler_and_data(chip->irq, + jz_gpio_irq_demux_handler, chip); gc = irq_alloc_generic_chip(chip->gpio_chip.label, 1, chip->irq_base, chip->base, handle_level_irq); diff --git a/arch/mips/jz4740/time.c b/arch/mips/jz4740/time.c index 7ab47fee1be8..1f7ca2c9f262 100644 --- a/arch/mips/jz4740/time.c +++ b/arch/mips/jz4740/time.c @@ -58,7 +58,7 @@ static irqreturn_t jz4740_clockevent_irq(int irq, void *devid) jz4740_timer_ack_full(TIMER_CLOCKEVENT); - if (cd->mode != CLOCK_EVT_MODE_PERIODIC) + if (!clockevent_state_periodic(cd)) jz4740_timer_disable(TIMER_CLOCKEVENT); cd->event_handler(cd); @@ -66,24 +66,29 @@ static irqreturn_t jz4740_clockevent_irq(int irq, void *devid) return IRQ_HANDLED; } -static void jz4740_clockevent_set_mode(enum clock_event_mode mode, - struct clock_event_device *cd) +static int jz4740_clockevent_set_periodic(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - jz4740_timer_set_count(TIMER_CLOCKEVENT, 0); - jz4740_timer_set_period(TIMER_CLOCKEVENT, jz4740_jiffies_per_tick); - case CLOCK_EVT_MODE_RESUME: - jz4740_timer_irq_full_enable(TIMER_CLOCKEVENT); - jz4740_timer_enable(TIMER_CLOCKEVENT); - break; - case CLOCK_EVT_MODE_ONESHOT: - case CLOCK_EVT_MODE_SHUTDOWN: - jz4740_timer_disable(TIMER_CLOCKEVENT); - break; - default: - break; - } + jz4740_timer_set_count(TIMER_CLOCKEVENT, 0); + jz4740_timer_set_period(TIMER_CLOCKEVENT, jz4740_jiffies_per_tick); + jz4740_timer_irq_full_enable(TIMER_CLOCKEVENT); + jz4740_timer_enable(TIMER_CLOCKEVENT); + + return 0; +} + +static int jz4740_clockevent_resume(struct clock_event_device *evt) +{ + jz4740_timer_irq_full_enable(TIMER_CLOCKEVENT); + jz4740_timer_enable(TIMER_CLOCKEVENT); + + return 0; +} + +static int jz4740_clockevent_shutdown(struct clock_event_device *evt) +{ + jz4740_timer_disable(TIMER_CLOCKEVENT); + + return 0; } static int jz4740_clockevent_set_next(unsigned long evt, @@ -100,7 +105,10 @@ static struct clock_event_device jz4740_clockevent = { .name = "jz4740-timer", .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, .set_next_event = jz4740_clockevent_set_next, - .set_mode = jz4740_clockevent_set_mode, + .set_state_shutdown = jz4740_clockevent_shutdown, + .set_state_periodic = jz4740_clockevent_set_periodic, + .set_state_oneshot = jz4740_clockevent_shutdown, + .tick_resume = jz4740_clockevent_resume, .rating = 200, #ifdef CONFIG_MACH_JZ4740 .irq = JZ4740_IRQ_TCU0, diff --git a/arch/mips/kernel/Makefile b/arch/mips/kernel/Makefile index 3f5cf8aff6f3..d982be1ea1c3 100644 --- a/arch/mips/kernel/Makefile +++ b/arch/mips/kernel/Makefile @@ -61,7 +61,6 @@ obj-$(CONFIG_MIPS_VPE_APSP_API) += rtlx.o obj-$(CONFIG_MIPS_VPE_APSP_API_CMP) += rtlx-cmp.o obj-$(CONFIG_MIPS_VPE_APSP_API_MT) += rtlx-mt.o -obj-$(CONFIG_I8259) += i8259.o obj-$(CONFIG_IRQ_CPU_RM7K) += irq-rm7000.o obj-$(CONFIG_MIPS_MSC) += irq-msc01.o obj-$(CONFIG_IRQ_TXX9) += irq_txx9.o @@ -100,6 +99,7 @@ obj-$(CONFIG_PERF_EVENTS) += perf_event.o obj-$(CONFIG_HW_PERF_EVENTS) += perf_event_mipsxx.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o +obj-$(CONFIG_UPROBES) += uprobes.o obj-$(CONFIG_MIPS_CM) += mips-cm.o obj-$(CONFIG_MIPS_CPC) += mips-cpc.o diff --git a/arch/mips/kernel/asm-offsets.c b/arch/mips/kernel/asm-offsets.c index 072fab13645d..154e2039ea5e 100644 --- a/arch/mips/kernel/asm-offsets.c +++ b/arch/mips/kernel/asm-offsets.c @@ -128,6 +128,7 @@ void output_thread_defines(void) thread.cp0_baduaddr); OFFSET(THREAD_ECODE, task_struct, \ thread.error_code); + OFFSET(THREAD_TRAPNO, task_struct, thread.trap_nr); BLANK(); } @@ -245,17 +246,6 @@ void output_sc_defines(void) } #endif -#ifdef CONFIG_MIPS32_COMPAT -void output_sc32_defines(void) -{ - COMMENT("Linux 32-bit sigcontext offsets."); - OFFSET(SC32_FPREGS, sigcontext32, sc_fpregs); - OFFSET(SC32_FPC_CSR, sigcontext32, sc_fpc_csr); - OFFSET(SC32_FPC_EIR, sigcontext32, sc_fpc_eir); - BLANK(); -} -#endif - void output_signal_defined(void) { COMMENT("Linux signal numbers."); diff --git a/arch/mips/kernel/cevt-bcm1480.c b/arch/mips/kernel/cevt-bcm1480.c index 7976457184b1..940ac00e9129 100644 --- a/arch/mips/kernel/cevt-bcm1480.c +++ b/arch/mips/kernel/cevt-bcm1480.c @@ -40,8 +40,8 @@ * The general purpose timer ticks at 1MHz independent if * the rest of the system */ -static void sibyte_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) + +static int sibyte_set_periodic(struct clock_event_device *evt) { unsigned int cpu = smp_processor_id(); void __iomem *cfg, *init; @@ -49,24 +49,22 @@ static void sibyte_set_mode(enum clock_event_mode mode, cfg = IOADDR(A_SCD_TIMER_REGISTER(cpu, R_SCD_TIMER_CFG)); init = IOADDR(A_SCD_TIMER_REGISTER(cpu, R_SCD_TIMER_INIT)); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - __raw_writeq(0, cfg); - __raw_writeq((V_SCD_TIMER_FREQ / HZ) - 1, init); - __raw_writeq(M_SCD_TIMER_ENABLE | M_SCD_TIMER_MODE_CONTINUOUS, - cfg); - break; + __raw_writeq(0, cfg); + __raw_writeq((V_SCD_TIMER_FREQ / HZ) - 1, init); + __raw_writeq(M_SCD_TIMER_ENABLE | M_SCD_TIMER_MODE_CONTINUOUS, cfg); + return 0; +} - case CLOCK_EVT_MODE_ONESHOT: - /* Stop the timer until we actually program a shot */ - case CLOCK_EVT_MODE_SHUTDOWN: - __raw_writeq(0, cfg); - break; +static int sibyte_shutdown(struct clock_event_device *evt) +{ + unsigned int cpu = smp_processor_id(); + void __iomem *cfg; - case CLOCK_EVT_MODE_UNUSED: /* shuddup gcc */ - case CLOCK_EVT_MODE_RESUME: - ; - } + cfg = IOADDR(A_SCD_TIMER_REGISTER(cpu, R_SCD_TIMER_CFG)); + + /* Stop the timer until we actually program a shot */ + __raw_writeq(0, cfg); + return 0; } static int sibyte_next_event(unsigned long delta, struct clock_event_device *cd) @@ -91,7 +89,7 @@ static irqreturn_t sibyte_counter_handler(int irq, void *dev_id) void __iomem *cfg; unsigned long tmode; - if (cd->mode == CLOCK_EVT_MODE_PERIODIC) + if (clockevent_state_periodic(cd)) tmode = M_SCD_TIMER_ENABLE | M_SCD_TIMER_MODE_CONTINUOUS; else tmode = 0; @@ -130,7 +128,9 @@ void sb1480_clockevent_init(void) cd->irq = irq; cd->cpumask = cpumask_of(cpu); cd->set_next_event = sibyte_next_event; - cd->set_mode = sibyte_set_mode; + cd->set_state_shutdown = sibyte_shutdown; + cd->set_state_periodic = sibyte_set_periodic; + cd->set_state_oneshot = sibyte_shutdown; clockevents_register_device(cd); bcm1480_mask_irq(cpu, irq); diff --git a/arch/mips/kernel/cevt-ds1287.c b/arch/mips/kernel/cevt-ds1287.c index ff1f01b72270..77a5ddf53f57 100644 --- a/arch/mips/kernel/cevt-ds1287.c +++ b/arch/mips/kernel/cevt-ds1287.c @@ -59,27 +59,32 @@ static int ds1287_set_next_event(unsigned long delta, return -EINVAL; } -static void ds1287_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int ds1287_shutdown(struct clock_event_device *evt) { u8 val; spin_lock(&rtc_lock); val = CMOS_READ(RTC_REG_B); - - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - val |= RTC_PIE; - break; - default: - val &= ~RTC_PIE; - break; - } - + val &= ~RTC_PIE; CMOS_WRITE(val, RTC_REG_B); spin_unlock(&rtc_lock); + return 0; +} + +static int ds1287_set_periodic(struct clock_event_device *evt) +{ + u8 val; + + spin_lock(&rtc_lock); + + val = CMOS_READ(RTC_REG_B); + val |= RTC_PIE; + CMOS_WRITE(val, RTC_REG_B); + + spin_unlock(&rtc_lock); + return 0; } static void ds1287_event_handler(struct clock_event_device *dev) @@ -87,11 +92,13 @@ static void ds1287_event_handler(struct clock_event_device *dev) } static struct clock_event_device ds1287_clockevent = { - .name = "ds1287", - .features = CLOCK_EVT_FEAT_PERIODIC, - .set_next_event = ds1287_set_next_event, - .set_mode = ds1287_set_mode, - .event_handler = ds1287_event_handler, + .name = "ds1287", + .features = CLOCK_EVT_FEAT_PERIODIC, + .set_next_event = ds1287_set_next_event, + .set_state_shutdown = ds1287_shutdown, + .set_state_periodic = ds1287_set_periodic, + .tick_resume = ds1287_shutdown, + .event_handler = ds1287_event_handler, }; static irqreturn_t ds1287_interrupt(int irq, void *dev_id) diff --git a/arch/mips/kernel/cevt-gt641xx.c b/arch/mips/kernel/cevt-gt641xx.c index f069460751ab..66040051151d 100644 --- a/arch/mips/kernel/cevt-gt641xx.c +++ b/arch/mips/kernel/cevt-gt641xx.c @@ -64,8 +64,7 @@ static int gt641xx_timer0_set_next_event(unsigned long delta, return 0; } -static void gt641xx_timer0_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int gt641xx_timer0_shutdown(struct clock_event_device *evt) { u32 ctrl; @@ -73,21 +72,39 @@ static void gt641xx_timer0_set_mode(enum clock_event_mode mode, ctrl = GT_READ(GT_TC_CONTROL_OFS); ctrl &= ~(GT_TC_CONTROL_ENTC0_MSK | GT_TC_CONTROL_SELTC0_MSK); - - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - ctrl |= GT_TC_CONTROL_ENTC0_MSK | GT_TC_CONTROL_SELTC0_MSK; - break; - case CLOCK_EVT_MODE_ONESHOT: - ctrl |= GT_TC_CONTROL_ENTC0_MSK; - break; - default: - break; - } - GT_WRITE(GT_TC_CONTROL_OFS, ctrl); raw_spin_unlock(>641xx_timer_lock); + return 0; +} + +static int gt641xx_timer0_set_oneshot(struct clock_event_device *evt) +{ + u32 ctrl; + + raw_spin_lock(>641xx_timer_lock); + + ctrl = GT_READ(GT_TC_CONTROL_OFS); + ctrl &= ~GT_TC_CONTROL_SELTC0_MSK; + ctrl |= GT_TC_CONTROL_ENTC0_MSK; + GT_WRITE(GT_TC_CONTROL_OFS, ctrl); + + raw_spin_unlock(>641xx_timer_lock); + return 0; +} + +static int gt641xx_timer0_set_periodic(struct clock_event_device *evt) +{ + u32 ctrl; + + raw_spin_lock(>641xx_timer_lock); + + ctrl = GT_READ(GT_TC_CONTROL_OFS); + ctrl |= GT_TC_CONTROL_ENTC0_MSK | GT_TC_CONTROL_SELTC0_MSK; + GT_WRITE(GT_TC_CONTROL_OFS, ctrl); + + raw_spin_unlock(>641xx_timer_lock); + return 0; } static void gt641xx_timer0_event_handler(struct clock_event_device *dev) @@ -95,12 +112,16 @@ static void gt641xx_timer0_event_handler(struct clock_event_device *dev) } static struct clock_event_device gt641xx_timer0_clockevent = { - .name = "gt641xx-timer0", - .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .irq = GT641XX_TIMER0_IRQ, - .set_next_event = gt641xx_timer0_set_next_event, - .set_mode = gt641xx_timer0_set_mode, - .event_handler = gt641xx_timer0_event_handler, + .name = "gt641xx-timer0", + .features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT, + .irq = GT641XX_TIMER0_IRQ, + .set_next_event = gt641xx_timer0_set_next_event, + .set_state_shutdown = gt641xx_timer0_shutdown, + .set_state_periodic = gt641xx_timer0_set_periodic, + .set_state_oneshot = gt641xx_timer0_set_oneshot, + .tick_resume = gt641xx_timer0_shutdown, + .event_handler = gt641xx_timer0_event_handler, }; static irqreturn_t gt641xx_timer0_interrupt(int irq, void *dev_id) diff --git a/arch/mips/kernel/cevt-r4k.c b/arch/mips/kernel/cevt-r4k.c index d70c4d893219..8dfe6a6e1480 100644 --- a/arch/mips/kernel/cevt-r4k.c +++ b/arch/mips/kernel/cevt-r4k.c @@ -28,12 +28,6 @@ static int mips_next_event(unsigned long delta, return res; } -void mips_set_clock_mode(enum clock_event_mode mode, - struct clock_event_device *evt) -{ - /* Nothing to do ... */ -} - DEFINE_PER_CPU(struct clock_event_device, mips_clockevent_device); int cp0_timer_irq_installed; @@ -174,6 +168,11 @@ int c0_compare_int_usable(void) return 1; } +unsigned int __weak get_c0_compare_int(void) +{ + return MIPS_CPU_IRQ_BASE + cp0_compare_irq; +} + int r4k_clockevent_init(void) { unsigned int cpu = smp_processor_id(); @@ -189,11 +188,9 @@ int r4k_clockevent_init(void) /* * With vectored interrupts things are getting platform specific. * get_c0_compare_int is a hook to allow a platform to return the - * interrupt number of it's liking. + * interrupt number of its liking. */ - irq = MIPS_CPU_IRQ_BASE + cp0_compare_irq; - if (get_c0_compare_int) - irq = get_c0_compare_int(); + irq = get_c0_compare_int(); cd = &per_cpu(mips_clockevent_device, cpu); @@ -212,7 +209,6 @@ int r4k_clockevent_init(void) cd->irq = irq; cd->cpumask = cpumask_of(cpu); cd->set_next_event = mips_next_event; - cd->set_mode = mips_set_clock_mode; cd->event_handler = mips_event_handler; clockevents_register_device(cd); diff --git a/arch/mips/kernel/cevt-sb1250.c b/arch/mips/kernel/cevt-sb1250.c index 5ea6d6b1de15..3d860efd63b9 100644 --- a/arch/mips/kernel/cevt-sb1250.c +++ b/arch/mips/kernel/cevt-sb1250.c @@ -38,8 +38,20 @@ * The general purpose timer ticks at 1MHz independent if * the rest of the system */ -static void sibyte_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) + +static int sibyte_shutdown(struct clock_event_device *evt) +{ + void __iomem *cfg; + + cfg = IOADDR(A_SCD_TIMER_REGISTER(smp_processor_id(), R_SCD_TIMER_CFG)); + + /* Stop the timer until we actually program a shot */ + __raw_writeq(0, cfg); + + return 0; +} + +static int sibyte_set_periodic(struct clock_event_device *evt) { unsigned int cpu = smp_processor_id(); void __iomem *cfg, *init; @@ -47,24 +59,11 @@ static void sibyte_set_mode(enum clock_event_mode mode, cfg = IOADDR(A_SCD_TIMER_REGISTER(cpu, R_SCD_TIMER_CFG)); init = IOADDR(A_SCD_TIMER_REGISTER(cpu, R_SCD_TIMER_INIT)); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - __raw_writeq(0, cfg); - __raw_writeq((V_SCD_TIMER_FREQ / HZ) - 1, init); - __raw_writeq(M_SCD_TIMER_ENABLE | M_SCD_TIMER_MODE_CONTINUOUS, - cfg); - break; + __raw_writeq(0, cfg); + __raw_writeq((V_SCD_TIMER_FREQ / HZ) - 1, init); + __raw_writeq(M_SCD_TIMER_ENABLE | M_SCD_TIMER_MODE_CONTINUOUS, cfg); - case CLOCK_EVT_MODE_ONESHOT: - /* Stop the timer until we actually program a shot */ - case CLOCK_EVT_MODE_SHUTDOWN: - __raw_writeq(0, cfg); - break; - - case CLOCK_EVT_MODE_UNUSED: /* shuddup gcc */ - case CLOCK_EVT_MODE_RESUME: - ; - } + return 0; } static int sibyte_next_event(unsigned long delta, struct clock_event_device *cd) @@ -89,7 +88,7 @@ static irqreturn_t sibyte_counter_handler(int irq, void *dev_id) void __iomem *cfg; unsigned long tmode; - if (cd->mode == CLOCK_EVT_MODE_PERIODIC) + if (clockevent_state_periodic(cd)) tmode = M_SCD_TIMER_ENABLE | M_SCD_TIMER_MODE_CONTINUOUS; else tmode = 0; @@ -129,7 +128,9 @@ void sb1250_clockevent_init(void) cd->irq = irq; cd->cpumask = cpumask_of(cpu); cd->set_next_event = sibyte_next_event; - cd->set_mode = sibyte_set_mode; + cd->set_state_shutdown = sibyte_shutdown; + cd->set_state_periodic = sibyte_set_periodic; + cd->set_state_oneshot = sibyte_shutdown; clockevents_register_device(cd); sb1250_mask_irq(cpu, irq); diff --git a/arch/mips/kernel/cevt-txx9.c b/arch/mips/kernel/cevt-txx9.c index 723932441ecc..537eefdf838f 100644 --- a/arch/mips/kernel/cevt-txx9.c +++ b/arch/mips/kernel/cevt-txx9.c @@ -85,36 +85,54 @@ static void txx9tmr_stop_and_clear(struct txx9_tmr_reg __iomem *tmrptr) __raw_writel(0, &tmrptr->tisr); } -static void txx9tmr_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int txx9tmr_set_state_periodic(struct clock_event_device *evt) { struct txx9_clock_event_device *txx9_cd = container_of(evt, struct txx9_clock_event_device, cd); struct txx9_tmr_reg __iomem *tmrptr = txx9_cd->tmrptr; txx9tmr_stop_and_clear(tmrptr); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - __raw_writel(TXx9_TMITMR_TIIE | TXx9_TMITMR_TZCE, - &tmrptr->itmr); - /* start timer */ - __raw_writel(((u64)(NSEC_PER_SEC / HZ) * evt->mult) >> - evt->shift, - &tmrptr->cpra); - __raw_writel(TCR_BASE | TXx9_TMTCR_TCE, &tmrptr->tcr); - break; - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - __raw_writel(0, &tmrptr->itmr); - break; - case CLOCK_EVT_MODE_ONESHOT: - __raw_writel(TXx9_TMITMR_TIIE, &tmrptr->itmr); - break; - case CLOCK_EVT_MODE_RESUME: - __raw_writel(TIMER_CCD, &tmrptr->ccdr); - __raw_writel(0, &tmrptr->itmr); - break; - } + + __raw_writel(TXx9_TMITMR_TIIE | TXx9_TMITMR_TZCE, &tmrptr->itmr); + /* start timer */ + __raw_writel(((u64)(NSEC_PER_SEC / HZ) * evt->mult) >> evt->shift, + &tmrptr->cpra); + __raw_writel(TCR_BASE | TXx9_TMTCR_TCE, &tmrptr->tcr); + return 0; +} + +static int txx9tmr_set_state_oneshot(struct clock_event_device *evt) +{ + struct txx9_clock_event_device *txx9_cd = + container_of(evt, struct txx9_clock_event_device, cd); + struct txx9_tmr_reg __iomem *tmrptr = txx9_cd->tmrptr; + + txx9tmr_stop_and_clear(tmrptr); + __raw_writel(TXx9_TMITMR_TIIE, &tmrptr->itmr); + return 0; +} + +static int txx9tmr_set_state_shutdown(struct clock_event_device *evt) +{ + struct txx9_clock_event_device *txx9_cd = + container_of(evt, struct txx9_clock_event_device, cd); + struct txx9_tmr_reg __iomem *tmrptr = txx9_cd->tmrptr; + + txx9tmr_stop_and_clear(tmrptr); + __raw_writel(0, &tmrptr->itmr); + return 0; +} + +static int txx9tmr_tick_resume(struct clock_event_device *evt) +{ + struct txx9_clock_event_device *txx9_cd = + container_of(evt, struct txx9_clock_event_device, cd); + struct txx9_tmr_reg __iomem *tmrptr = txx9_cd->tmrptr; + + txx9tmr_stop_and_clear(tmrptr); + __raw_writel(TIMER_CCD, &tmrptr->ccdr); + __raw_writel(0, &tmrptr->itmr); + return 0; } static int txx9tmr_set_next_event(unsigned long delta, @@ -133,12 +151,15 @@ static int txx9tmr_set_next_event(unsigned long delta, static struct txx9_clock_event_device txx9_clock_event_device = { .cd = { - .name = "TXx9", - .features = CLOCK_EVT_FEAT_PERIODIC | - CLOCK_EVT_FEAT_ONESHOT, - .rating = 200, - .set_mode = txx9tmr_set_mode, - .set_next_event = txx9tmr_set_next_event, + .name = "TXx9", + .features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT, + .rating = 200, + .set_state_shutdown = txx9tmr_set_state_shutdown, + .set_state_periodic = txx9tmr_set_state_periodic, + .set_state_oneshot = txx9tmr_set_state_oneshot, + .tick_resume = txx9tmr_tick_resume, + .set_next_event = txx9tmr_set_next_event, }, }; diff --git a/arch/mips/kernel/cps-vec.S b/arch/mips/kernel/cps-vec.S index 1b6ca634e646..9f71c06aebf6 100644 --- a/arch/mips/kernel/cps-vec.S +++ b/arch/mips/kernel/cps-vec.S @@ -152,7 +152,7 @@ dcache_done: /* Enter the coherent domain */ li t0, 0xff - PTR_S t0, GCR_CL_COHERENCE_OFS(v1) + sw t0, GCR_CL_COHERENCE_OFS(v1) ehb /* Jump to kseg0 */ @@ -302,7 +302,7 @@ LEAF(mips_cps_boot_vpes) PTR_L t0, 0(t0) /* Calculate a pointer to this cores struct core_boot_config */ - PTR_L t0, GCR_CL_ID_OFS(t0) + lw t0, GCR_CL_ID_OFS(t0) li t1, COREBOOTCFG_SIZE mul t0, t0, t1 PTR_LA t1, mips_cps_core_bootcfg diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index dbe0792fc9c1..571a8e6ea5bd 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -32,6 +32,9 @@ #include #include +/* Hardware capabilities */ +unsigned int elf_hwcap __read_mostly; + /* * Get the FPU Implementation/Revision. */ @@ -188,7 +191,7 @@ __setup("nohtw", htw_disable); static int mips_ftlb_disabled; static int mips_has_ftlb_configured; -static void set_ftlb_enable(struct cpuinfo_mips *c, int enable); +static int set_ftlb_enable(struct cpuinfo_mips *c, int enable); static int __init ftlb_disable(char *s) { @@ -202,7 +205,10 @@ static int __init ftlb_disable(char *s) return 1; /* Disable it in the boot cpu */ - set_ftlb_enable(&cpu_data[0], 0); + if (set_ftlb_enable(&cpu_data[0], 0)) { + pr_warn("Can't turn FTLB off\n"); + return 1; + } back_to_back_c0_hazard(); @@ -364,30 +370,41 @@ static unsigned int calculate_ftlb_probability(struct cpuinfo_mips *c) return 3; } -static void set_ftlb_enable(struct cpuinfo_mips *c, int enable) +static int set_ftlb_enable(struct cpuinfo_mips *c, int enable) { - unsigned int config6; + unsigned int config; /* It's implementation dependent how the FTLB can be enabled */ switch (c->cputype) { case CPU_PROAPTIV: case CPU_P5600: /* proAptiv & related cores use Config6 to enable the FTLB */ - config6 = read_c0_config6(); + config = read_c0_config6(); /* Clear the old probability value */ - config6 &= ~(3 << MIPS_CONF6_FTLBP_SHIFT); + config &= ~(3 << MIPS_CONF6_FTLBP_SHIFT); if (enable) /* Enable FTLB */ - write_c0_config6(config6 | + write_c0_config6(config | (calculate_ftlb_probability(c) << MIPS_CONF6_FTLBP_SHIFT) | MIPS_CONF6_FTLBEN); else /* Disable FTLB */ - write_c0_config6(config6 & ~MIPS_CONF6_FTLBEN); - back_to_back_c0_hazard(); + write_c0_config6(config & ~MIPS_CONF6_FTLBEN); break; + case CPU_I6400: + /* I6400 & related cores use Config7 to configure FTLB */ + config = read_c0_config7(); + /* Clear the old probability value */ + config &= ~(3 << MIPS_CONF7_FTLBP_SHIFT); + write_c0_config7(config | (calculate_ftlb_probability(c) + << MIPS_CONF7_FTLBP_SHIFT)); + break; + default: + return 1; } + + return 0; } static inline unsigned int decode_config0(struct cpuinfo_mips *c) @@ -524,6 +541,8 @@ static inline unsigned int decode_config3(struct cpuinfo_mips *c) } if (config3 & MIPS_CONF3_CDMM) c->options |= MIPS_CPU_CDMM; + if (config3 & MIPS_CONF3_SP) + c->options |= MIPS_CPU_SP; return config3 & MIPS_CONF_M; } @@ -540,7 +559,16 @@ static inline unsigned int decode_config4(struct cpuinfo_mips *c) if (cpu_has_tlb) { if (((config4 & MIPS_CONF4_IE) >> 29) == 2) c->options |= MIPS_CPU_TLBINV; - mmuextdef = config4 & MIPS_CONF4_MMUEXTDEF; + /* + * This is a bit ugly. R6 has dropped that field from + * config4 and the only valid configuration is VTLB+FTLB so + * set a good value for mmuextdef for that case. + */ + if (cpu_has_mips_r6) + mmuextdef = MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT; + else + mmuextdef = config4 & MIPS_CONF4_MMUEXTDEF; + switch (mmuextdef) { case MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT: c->tlbsize += (config4 & MIPS_CONF4_MMUSIZEEXT) * 0x40; @@ -1121,6 +1149,10 @@ static inline void cpu_probe_mips(struct cpuinfo_mips *c, unsigned int cpu) c->cputype = CPU_P5600; __cpu_name[cpu] = "MIPS P5600"; break; + case PRID_IMP_I6400: + c->cputype = CPU_I6400; + __cpu_name[cpu] = "MIPS I6400"; + break; case PRID_IMP_M5150: c->cputype = CPU_M5150; __cpu_name[cpu] = "MIPS M5150"; @@ -1492,10 +1524,14 @@ void cpu_probe(void) else c->srsets = 1; + if (cpu_has_mips_r6) + elf_hwcap |= HWCAP_MIPS_R6; + if (cpu_has_msa) { c->msa_id = cpu_get_msa_id(); WARN(c->msa_id & MSA_IR_WRPF, "Vector register partitioning unimplemented!"); + elf_hwcap |= HWCAP_MIPS_MSA; } cpu_probe_vmbits(c); diff --git a/arch/mips/kernel/idle.c b/arch/mips/kernel/idle.c index e4f62b7875d2..ab1478d5a4db 100644 --- a/arch/mips/kernel/idle.c +++ b/arch/mips/kernel/idle.c @@ -196,6 +196,7 @@ void __init check_wait(void) case CPU_INTERAPTIV: case CPU_M5150: case CPU_QEMU_GENERIC: + case CPU_I6400: cpu_wait = r4k_wait; if (read_c0_config7() & MIPS_CONF7_WII) cpu_wait = r4k_wait_irqoff; diff --git a/arch/mips/kernel/jump_label.c b/arch/mips/kernel/jump_label.c index dda800e9e731..3e586daa3a32 100644 --- a/arch/mips/kernel/jump_label.c +++ b/arch/mips/kernel/jump_label.c @@ -51,7 +51,7 @@ void arch_jump_label_transform(struct jump_entry *e, /* Target must have the right alignment and ISA must be preserved. */ BUG_ON((e->target & J_ALIGN_MASK) != J_ISA_BIT); - if (type == JUMP_LABEL_ENABLE) { + if (type == JUMP_LABEL_JMP) { insn.j_format.opcode = J_ISA_BIT ? mm_j32_op : j_op; insn.j_format.target = e->target >> J_RANGE_SHIFT; } else { diff --git a/arch/mips/kernel/mips-cm.c b/arch/mips/kernel/mips-cm.c index 85bbe9b96759..b8ceee576cdf 100644 --- a/arch/mips/kernel/mips-cm.c +++ b/arch/mips/kernel/mips-cm.c @@ -15,11 +15,131 @@ void __iomem *mips_cm_base; void __iomem *mips_cm_l2sync_base; +int mips_cm_is64; + +static char *cm2_tr[8] = { + "mem", "gcr", "gic", "mmio", + "0x04", "cpc", "0x06", "0x07" +}; + +/* CM3 Tag ECC transation type */ +static char *cm3_tr[16] = { + [0x0] = "ReqNoData", + [0x1] = "0x1", + [0x2] = "ReqWData", + [0x3] = "0x3", + [0x4] = "IReqNoResp", + [0x5] = "IReqWResp", + [0x6] = "IReqNoRespDat", + [0x7] = "IReqWRespDat", + [0x8] = "RespNoData", + [0x9] = "RespDataFol", + [0xa] = "RespWData", + [0xb] = "RespDataOnly", + [0xc] = "IRespNoData", + [0xd] = "IRespDataFol", + [0xe] = "IRespWData", + [0xf] = "IRespDataOnly" +}; + +static char *cm2_cmd[32] = { + [0x00] = "0x00", + [0x01] = "Legacy Write", + [0x02] = "Legacy Read", + [0x03] = "0x03", + [0x04] = "0x04", + [0x05] = "0x05", + [0x06] = "0x06", + [0x07] = "0x07", + [0x08] = "Coherent Read Own", + [0x09] = "Coherent Read Share", + [0x0a] = "Coherent Read Discard", + [0x0b] = "Coherent Ready Share Always", + [0x0c] = "Coherent Upgrade", + [0x0d] = "Coherent Writeback", + [0x0e] = "0x0e", + [0x0f] = "0x0f", + [0x10] = "Coherent Copyback", + [0x11] = "Coherent Copyback Invalidate", + [0x12] = "Coherent Invalidate", + [0x13] = "Coherent Write Invalidate", + [0x14] = "Coherent Completion Sync", + [0x15] = "0x15", + [0x16] = "0x16", + [0x17] = "0x17", + [0x18] = "0x18", + [0x19] = "0x19", + [0x1a] = "0x1a", + [0x1b] = "0x1b", + [0x1c] = "0x1c", + [0x1d] = "0x1d", + [0x1e] = "0x1e", + [0x1f] = "0x1f" +}; + +/* CM3 Tag ECC command type */ +static char *cm3_cmd[16] = { + [0x0] = "Legacy Read", + [0x1] = "Legacy Write", + [0x2] = "Coherent Read Own", + [0x3] = "Coherent Read Share", + [0x4] = "Coherent Read Discard", + [0x5] = "Coherent Evicted", + [0x6] = "Coherent Upgrade", + [0x7] = "Coherent Upgrade for Store Conditional", + [0x8] = "Coherent Writeback", + [0x9] = "Coherent Write Invalidate", + [0xa] = "0xa", + [0xb] = "0xb", + [0xc] = "0xc", + [0xd] = "0xd", + [0xe] = "0xe", + [0xf] = "0xf" +}; + +/* CM3 Tag ECC command group */ +static char *cm3_cmd_group[8] = { + [0x0] = "Normal", + [0x1] = "Registers", + [0x2] = "TLB", + [0x3] = "0x3", + [0x4] = "L1I", + [0x5] = "L1D", + [0x6] = "L3", + [0x7] = "L2" +}; + +static char *cm2_core[8] = { + "Invalid/OK", "Invalid/Data", + "Shared/OK", "Shared/Data", + "Modified/OK", "Modified/Data", + "Exclusive/OK", "Exclusive/Data" +}; + +static char *cm2_causes[32] = { + "None", "GC_WR_ERR", "GC_RD_ERR", "COH_WR_ERR", + "COH_RD_ERR", "MMIO_WR_ERR", "MMIO_RD_ERR", "0x07", + "0x08", "0x09", "0x0a", "0x0b", + "0x0c", "0x0d", "0x0e", "0x0f", + "0x10", "0x11", "0x12", "0x13", + "0x14", "0x15", "0x16", "INTVN_WR_ERR", + "INTVN_RD_ERR", "0x19", "0x1a", "0x1b", + "0x1c", "0x1d", "0x1e", "0x1f" +}; + +static char *cm3_causes[32] = { + "0x0", "MP_CORRECTABLE_ECC_ERR", "MP_REQUEST_DECODE_ERR", + "MP_UNCORRECTABLE_ECC_ERR", "MP_PARITY_ERR", "MP_COHERENCE_ERR", + "CMBIU_REQUEST_DECODE_ERR", "CMBIU_PARITY_ERR", "CMBIU_AXI_RESP_ERR", + "0x9", "RBI_BUS_ERR", "0xb", "0xc", "0xd", "0xe", "0xf", "0x10", + "0x11", "0x12", "0x13", "0x14", "0x15", "0x16", "0x17", "0x18", + "0x19", "0x1a", "0x1b", "0x1c", "0x1d", "0x1e", "0x1f" +}; phys_addr_t __mips_cm_phys_base(void) { u32 config3 = read_c0_config3(); - u32 cmgcr; + unsigned long cmgcr; /* Check the CMGCRBase register is implemented */ if (!(config3 & MIPS_CONF3_CMGCR)) @@ -81,6 +201,13 @@ int mips_cm_probe(void) phys_addr_t addr; u32 base_reg; + /* + * No need to probe again if we have already been + * here before. + */ + if (mips_cm_base) + return 0; + addr = mips_cm_phys_base(); BUG_ON((addr & CM_GCR_BASE_GCRBASE_MSK) != addr); if (!addr) @@ -117,5 +244,133 @@ int mips_cm_probe(void) /* probe for an L2-only sync region */ mips_cm_probe_l2sync(); + /* determine register width for this CM */ + mips_cm_is64 = config_enabled(CONFIG_64BIT) && (mips_cm_revision() >= CM_REV_CM3); + return 0; } + +void mips_cm_error_report(void) +{ + unsigned long revision = mips_cm_revision(); + /* + * CM3 has a 64-bit Error cause register with 0:57 containing the error + * info and 63:58 the error type. For old CMs, everything is contained + * in a single 32-bit register (0:26 and 31:27 respectively). Even + * though the cm_error is u64, we will simply ignore the upper word + * for CM2. + */ + u64 cm_error = read_gcr_error_cause(); + int cm_error_cause_sft = CM_GCR_ERROR_CAUSE_ERRTYPE_SHF + + ((revision >= CM_REV_CM3) ? 31 : 0); + unsigned long cm_addr = read_gcr_error_addr(); + unsigned long cm_other = read_gcr_error_mult(); + int ocause, cause; + char buf[256]; + + if (!mips_cm_present()) + return; + + cause = cm_error >> cm_error_cause_sft; + + if (!cause) + /* All good */ + return; + + ocause = cm_other >> CM_GCR_ERROR_MULT_ERR2ND_SHF; + if (revision < CM_REV_CM3) { /* CM2 */ + if (cause < 16) { + unsigned long cca_bits = (cm_error >> 15) & 7; + unsigned long tr_bits = (cm_error >> 12) & 7; + unsigned long cmd_bits = (cm_error >> 7) & 0x1f; + unsigned long stag_bits = (cm_error >> 3) & 15; + unsigned long sport_bits = (cm_error >> 0) & 7; + + snprintf(buf, sizeof(buf), + "CCA=%lu TR=%s MCmd=%s STag=%lu " + "SPort=%lu\n", cca_bits, cm2_tr[tr_bits], + cm2_cmd[cmd_bits], stag_bits, sport_bits); + } else { + /* glob state & sresp together */ + unsigned long c3_bits = (cm_error >> 18) & 7; + unsigned long c2_bits = (cm_error >> 15) & 7; + unsigned long c1_bits = (cm_error >> 12) & 7; + unsigned long c0_bits = (cm_error >> 9) & 7; + unsigned long sc_bit = (cm_error >> 8) & 1; + unsigned long cmd_bits = (cm_error >> 3) & 0x1f; + unsigned long sport_bits = (cm_error >> 0) & 7; + + snprintf(buf, sizeof(buf), + "C3=%s C2=%s C1=%s C0=%s SC=%s " + "MCmd=%s SPort=%lu\n", + cm2_core[c3_bits], cm2_core[c2_bits], + cm2_core[c1_bits], cm2_core[c0_bits], + sc_bit ? "True" : "False", + cm2_cmd[cmd_bits], sport_bits); + } + pr_err("CM_ERROR=%08llx %s <%s>\n", cm_error, + cm2_causes[cause], buf); + pr_err("CM_ADDR =%08lx\n", cm_addr); + pr_err("CM_OTHER=%08lx %s\n", cm_other, cm2_causes[ocause]); + } else { /* CM3 */ + /* Used by cause == {1,2,3} */ + unsigned long core_id_bits = (cm_error >> 22) & 0xf; + unsigned long vp_id_bits = (cm_error >> 18) & 0xf; + unsigned long cmd_bits = (cm_error >> 14) & 0xf; + unsigned long cmd_group_bits = (cm_error >> 11) & 0xf; + unsigned long cm3_cca_bits = (cm_error >> 8) & 7; + unsigned long mcp_bits = (cm_error >> 5) & 0xf; + unsigned long cm3_tr_bits = (cm_error >> 1) & 0xf; + unsigned long sched_bit = cm_error & 0x1; + + if (cause == 1 || cause == 3) { /* Tag ECC */ + unsigned long tag_ecc = (cm_error >> 57) & 0x1; + unsigned long tag_way_bits = (cm_error >> 29) & 0xffff; + unsigned long dword_bits = (cm_error >> 49) & 0xff; + unsigned long data_way_bits = (cm_error >> 45) & 0xf; + unsigned long data_sets_bits = (cm_error >> 29) & 0xfff; + unsigned long bank_bit = (cm_error >> 28) & 0x1; + snprintf(buf, sizeof(buf), + "%s ECC Error: Way=%lu (DWORD=%lu, Sets=%lu)" + "Bank=%lu CoreID=%lu VPID=%lu Command=%s" + "Command Group=%s CCA=%lu MCP=%d" + "Transaction type=%s Scheduler=%lu\n", + tag_ecc ? "TAG" : "DATA", + tag_ecc ? (unsigned long)ffs(tag_way_bits) - 1 : + data_way_bits, bank_bit, dword_bits, + data_sets_bits, + core_id_bits, vp_id_bits, + cm3_cmd[cmd_bits], + cm3_cmd_group[cmd_group_bits], + cm3_cca_bits, 1 << mcp_bits, + cm3_tr[cm3_tr_bits], sched_bit); + } else if (cause == 2) { + unsigned long data_error_type = (cm_error >> 41) & 0xfff; + unsigned long data_decode_cmd = (cm_error >> 37) & 0xf; + unsigned long data_decode_group = (cm_error >> 34) & 0x7; + unsigned long data_decode_destination_id = (cm_error >> 28) & 0x3f; + + snprintf(buf, sizeof(buf), + "Decode Request Error: Type=%lu, Command=%lu" + "Command Group=%lu Destination ID=%lu" + "CoreID=%lu VPID=%lu Command=%s" + "Command Group=%s CCA=%lu MCP=%d" + "Transaction type=%s Scheduler=%lu\n", + data_error_type, data_decode_cmd, + data_decode_group, data_decode_destination_id, + core_id_bits, vp_id_bits, + cm3_cmd[cmd_bits], + cm3_cmd_group[cmd_group_bits], + cm3_cca_bits, 1 << mcp_bits, + cm3_tr[cm3_tr_bits], sched_bit); + } + + pr_err("CM_ERROR=%llx %s <%s>\n", cm_error, + cm3_causes[cause], buf); + pr_err("CM_ADDR =%lx\n", cm_addr); + pr_err("CM_OTHER=%lx %s\n", cm_other, cm3_causes[ocause]); + } + + /* reprime cause register */ + write_gcr_error_cause(0); +} diff --git a/arch/mips/kernel/mips-cpc.c b/arch/mips/kernel/mips-cpc.c index 11964501c4b0..8af4d627b68b 100644 --- a/arch/mips/kernel/mips-cpc.c +++ b/arch/mips/kernel/mips-cpc.c @@ -21,9 +21,16 @@ static DEFINE_PER_CPU_ALIGNED(spinlock_t, cpc_core_lock); static DEFINE_PER_CPU_ALIGNED(unsigned long, cpc_core_lock_flags); -phys_addr_t __weak mips_cpc_phys_base(void) +/** + * mips_cpc_phys_base - retrieve the physical base address of the CPC + * + * This function returns the physical base address of the Cluster Power + * Controller memory mapped registers, or 0 if no Cluster Power Controller + * is present. + */ +static phys_addr_t mips_cpc_phys_base(void) { - u32 cpc_base; + unsigned long cpc_base; if (!mips_cm_present()) return 0; diff --git a/arch/mips/kernel/perf_event_mipsxx.c b/arch/mips/kernel/perf_event_mipsxx.c index cc1b6fadf089..d7b8dd43147a 100644 --- a/arch/mips/kernel/perf_event_mipsxx.c +++ b/arch/mips/kernel/perf_event_mipsxx.c @@ -1556,6 +1556,7 @@ static const struct mips_perf_event *mipsxx_pmu_map_raw_event(u64 config) #endif break; case CPU_P5600: + case CPU_I6400: /* 8-bit event numbers */ raw_id = config & 0x1ff; base_id = raw_id & 0xff; @@ -1717,6 +1718,11 @@ init_hw_perf_events(void) mipspmu.general_event_map = &mipsxxcore_event_map2; mipspmu.cache_event_map = &mipsxxcore_cache_map2; break; + case CPU_I6400: + mipspmu.name = "mips/I6400"; + mipspmu.general_event_map = &mipsxxcore_event_map2; + mipspmu.cache_event_map = &mipsxxcore_cache_map2; + break; case CPU_1004K: mipspmu.name = "mips/1004K"; mipspmu.general_event_map = &mipsxxcore_event_map; diff --git a/arch/mips/kernel/pm-cps.c b/arch/mips/kernel/pm-cps.c index 06147179a175..f63a289977cc 100644 --- a/arch/mips/kernel/pm-cps.c +++ b/arch/mips/kernel/pm-cps.c @@ -267,6 +267,7 @@ static int __init cps_gen_flush_fsb(u32 **pp, struct uasm_label **pl, /* CPUs which do not require the workaround */ case CPU_P5600: + case CPU_I6400: return 0; default: @@ -671,6 +672,7 @@ static int __init cps_pm_init(void) case CPU_PROAPTIV: case CPU_M5150: case CPU_P5600: + case CPU_I6400: stype_intervention = 0x2; stype_memory = 0x3; stype_ordering = 0x10; diff --git a/arch/mips/kernel/ptrace.c b/arch/mips/kernel/ptrace.c index e933a309f2ea..4f0ac78d17f1 100644 --- a/arch/mips/kernel/ptrace.c +++ b/arch/mips/kernel/ptrace.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -490,6 +491,93 @@ enum mips_regset { REGSET_FPR, }; +struct pt_regs_offset { + const char *name; + int offset; +}; + +#define REG_OFFSET_NAME(reg, r) { \ + .name = #reg, \ + .offset = offsetof(struct pt_regs, r) \ +} + +#define REG_OFFSET_END { \ + .name = NULL, \ + .offset = 0 \ +} + +static const struct pt_regs_offset regoffset_table[] = { + REG_OFFSET_NAME(r0, regs[0]), + REG_OFFSET_NAME(r1, regs[1]), + REG_OFFSET_NAME(r2, regs[2]), + REG_OFFSET_NAME(r3, regs[3]), + REG_OFFSET_NAME(r4, regs[4]), + REG_OFFSET_NAME(r5, regs[5]), + REG_OFFSET_NAME(r6, regs[6]), + REG_OFFSET_NAME(r7, regs[7]), + REG_OFFSET_NAME(r8, regs[8]), + REG_OFFSET_NAME(r9, regs[9]), + REG_OFFSET_NAME(r10, regs[10]), + REG_OFFSET_NAME(r11, regs[11]), + REG_OFFSET_NAME(r12, regs[12]), + REG_OFFSET_NAME(r13, regs[13]), + REG_OFFSET_NAME(r14, regs[14]), + REG_OFFSET_NAME(r15, regs[15]), + REG_OFFSET_NAME(r16, regs[16]), + REG_OFFSET_NAME(r17, regs[17]), + REG_OFFSET_NAME(r18, regs[18]), + REG_OFFSET_NAME(r19, regs[19]), + REG_OFFSET_NAME(r20, regs[20]), + REG_OFFSET_NAME(r21, regs[21]), + REG_OFFSET_NAME(r22, regs[22]), + REG_OFFSET_NAME(r23, regs[23]), + REG_OFFSET_NAME(r24, regs[24]), + REG_OFFSET_NAME(r25, regs[25]), + REG_OFFSET_NAME(r26, regs[26]), + REG_OFFSET_NAME(r27, regs[27]), + REG_OFFSET_NAME(r28, regs[28]), + REG_OFFSET_NAME(r29, regs[29]), + REG_OFFSET_NAME(r30, regs[30]), + REG_OFFSET_NAME(r31, regs[31]), + REG_OFFSET_NAME(c0_status, cp0_status), + REG_OFFSET_NAME(hi, hi), + REG_OFFSET_NAME(lo, lo), +#ifdef CONFIG_CPU_HAS_SMARTMIPS + REG_OFFSET_NAME(acx, acx), +#endif + REG_OFFSET_NAME(c0_badvaddr, cp0_badvaddr), + REG_OFFSET_NAME(c0_cause, cp0_cause), + REG_OFFSET_NAME(c0_epc, cp0_epc), +#ifdef CONFIG_MIPS_MT_SMTC + REG_OFFSET_NAME(c0_tcstatus, cp0_tcstatus), +#endif +#ifdef CONFIG_CPU_CAVIUM_OCTEON + REG_OFFSET_NAME(mpl0, mpl[0]), + REG_OFFSET_NAME(mpl1, mpl[1]), + REG_OFFSET_NAME(mpl2, mpl[2]), + REG_OFFSET_NAME(mtp0, mtp[0]), + REG_OFFSET_NAME(mtp1, mtp[1]), + REG_OFFSET_NAME(mtp2, mtp[2]), +#endif + REG_OFFSET_END, +}; + +/** + * regs_query_register_offset() - query register offset from its name + * @name: the name of a register + * + * regs_query_register_offset() returns the offset of a register in struct + * pt_regs from its name. If the name is invalid, this returns -EINVAL; + */ +int regs_query_register_offset(const char *name) +{ + const struct pt_regs_offset *roff; + for (roff = regoffset_table; roff->name != NULL; roff++) + if (!strcmp(roff->name, name)) + return roff->offset; + return -EINVAL; +} + #if defined(CONFIG_32BIT) || defined(CONFIG_MIPS32_O32) static const struct user_regset mips_regsets[] = { diff --git a/arch/mips/kernel/r4k_fpu.S b/arch/mips/kernel/r4k_fpu.S index 1d88af26ba82..f09546ee2cdc 100644 --- a/arch/mips/kernel/r4k_fpu.S +++ b/arch/mips/kernel/r4k_fpu.S @@ -13,6 +13,7 @@ * Copyright (C) 1999, 2001 Silicon Graphics, Inc. */ #include +#include #include #include #include @@ -35,6 +36,14 @@ .set noreorder +/** + * _save_fp_context() - save FP context from the FPU + * @a0 - pointer to fpregs field of sigcontext + * @a1 - pointer to fpc_csr field of sigcontext + * + * Save FP context, including the 32 FP data registers and the FP + * control & status register, from the FPU to signal context. + */ LEAF(_save_fp_context) .set push SET_HARDFLOAT @@ -54,117 +63,60 @@ LEAF(_save_fp_context) nop #endif /* Store the 16 odd double precision registers */ - EX sdc1 $f1, SC_FPREGS+8(a0) - EX sdc1 $f3, SC_FPREGS+24(a0) - EX sdc1 $f5, SC_FPREGS+40(a0) - EX sdc1 $f7, SC_FPREGS+56(a0) - EX sdc1 $f9, SC_FPREGS+72(a0) - EX sdc1 $f11, SC_FPREGS+88(a0) - EX sdc1 $f13, SC_FPREGS+104(a0) - EX sdc1 $f15, SC_FPREGS+120(a0) - EX sdc1 $f17, SC_FPREGS+136(a0) - EX sdc1 $f19, SC_FPREGS+152(a0) - EX sdc1 $f21, SC_FPREGS+168(a0) - EX sdc1 $f23, SC_FPREGS+184(a0) - EX sdc1 $f25, SC_FPREGS+200(a0) - EX sdc1 $f27, SC_FPREGS+216(a0) - EX sdc1 $f29, SC_FPREGS+232(a0) - EX sdc1 $f31, SC_FPREGS+248(a0) + EX sdc1 $f1, 8(a0) + EX sdc1 $f3, 24(a0) + EX sdc1 $f5, 40(a0) + EX sdc1 $f7, 56(a0) + EX sdc1 $f9, 72(a0) + EX sdc1 $f11, 88(a0) + EX sdc1 $f13, 104(a0) + EX sdc1 $f15, 120(a0) + EX sdc1 $f17, 136(a0) + EX sdc1 $f19, 152(a0) + EX sdc1 $f21, 168(a0) + EX sdc1 $f23, 184(a0) + EX sdc1 $f25, 200(a0) + EX sdc1 $f27, 216(a0) + EX sdc1 $f29, 232(a0) + EX sdc1 $f31, 248(a0) 1: .set pop #endif .set push SET_HARDFLOAT /* Store the 16 even double precision registers */ - EX sdc1 $f0, SC_FPREGS+0(a0) - EX sdc1 $f2, SC_FPREGS+16(a0) - EX sdc1 $f4, SC_FPREGS+32(a0) - EX sdc1 $f6, SC_FPREGS+48(a0) - EX sdc1 $f8, SC_FPREGS+64(a0) - EX sdc1 $f10, SC_FPREGS+80(a0) - EX sdc1 $f12, SC_FPREGS+96(a0) - EX sdc1 $f14, SC_FPREGS+112(a0) - EX sdc1 $f16, SC_FPREGS+128(a0) - EX sdc1 $f18, SC_FPREGS+144(a0) - EX sdc1 $f20, SC_FPREGS+160(a0) - EX sdc1 $f22, SC_FPREGS+176(a0) - EX sdc1 $f24, SC_FPREGS+192(a0) - EX sdc1 $f26, SC_FPREGS+208(a0) - EX sdc1 $f28, SC_FPREGS+224(a0) - EX sdc1 $f30, SC_FPREGS+240(a0) - EX sw t1, SC_FPC_CSR(a0) + EX sdc1 $f0, 0(a0) + EX sdc1 $f2, 16(a0) + EX sdc1 $f4, 32(a0) + EX sdc1 $f6, 48(a0) + EX sdc1 $f8, 64(a0) + EX sdc1 $f10, 80(a0) + EX sdc1 $f12, 96(a0) + EX sdc1 $f14, 112(a0) + EX sdc1 $f16, 128(a0) + EX sdc1 $f18, 144(a0) + EX sdc1 $f20, 160(a0) + EX sdc1 $f22, 176(a0) + EX sdc1 $f24, 192(a0) + EX sdc1 $f26, 208(a0) + EX sdc1 $f28, 224(a0) + EX sdc1 $f30, 240(a0) + EX sw t1, 0(a1) jr ra li v0, 0 # success .set pop END(_save_fp_context) -#ifdef CONFIG_MIPS32_COMPAT - /* Save 32-bit process floating point context */ -LEAF(_save_fp_context32) - .set push - .set MIPS_ISA_ARCH_LEVEL_RAW - SET_HARDFLOAT - cfc1 t1, fcr31 - -#ifndef CONFIG_CPU_MIPS64_R6 - mfc0 t0, CP0_STATUS - sll t0, t0, 5 - bgez t0, 1f # skip storing odd if FR=0 - nop -#endif - - /* Store the 16 odd double precision registers */ - EX sdc1 $f1, SC32_FPREGS+8(a0) - EX sdc1 $f3, SC32_FPREGS+24(a0) - EX sdc1 $f5, SC32_FPREGS+40(a0) - EX sdc1 $f7, SC32_FPREGS+56(a0) - EX sdc1 $f9, SC32_FPREGS+72(a0) - EX sdc1 $f11, SC32_FPREGS+88(a0) - EX sdc1 $f13, SC32_FPREGS+104(a0) - EX sdc1 $f15, SC32_FPREGS+120(a0) - EX sdc1 $f17, SC32_FPREGS+136(a0) - EX sdc1 $f19, SC32_FPREGS+152(a0) - EX sdc1 $f21, SC32_FPREGS+168(a0) - EX sdc1 $f23, SC32_FPREGS+184(a0) - EX sdc1 $f25, SC32_FPREGS+200(a0) - EX sdc1 $f27, SC32_FPREGS+216(a0) - EX sdc1 $f29, SC32_FPREGS+232(a0) - EX sdc1 $f31, SC32_FPREGS+248(a0) - - /* Store the 16 even double precision registers */ -1: EX sdc1 $f0, SC32_FPREGS+0(a0) - EX sdc1 $f2, SC32_FPREGS+16(a0) - EX sdc1 $f4, SC32_FPREGS+32(a0) - EX sdc1 $f6, SC32_FPREGS+48(a0) - EX sdc1 $f8, SC32_FPREGS+64(a0) - EX sdc1 $f10, SC32_FPREGS+80(a0) - EX sdc1 $f12, SC32_FPREGS+96(a0) - EX sdc1 $f14, SC32_FPREGS+112(a0) - EX sdc1 $f16, SC32_FPREGS+128(a0) - EX sdc1 $f18, SC32_FPREGS+144(a0) - EX sdc1 $f20, SC32_FPREGS+160(a0) - EX sdc1 $f22, SC32_FPREGS+176(a0) - EX sdc1 $f24, SC32_FPREGS+192(a0) - EX sdc1 $f26, SC32_FPREGS+208(a0) - EX sdc1 $f28, SC32_FPREGS+224(a0) - EX sdc1 $f30, SC32_FPREGS+240(a0) - EX sw t1, SC32_FPC_CSR(a0) - cfc1 t0, $0 # implementation/version - EX sw t0, SC32_FPC_EIR(a0) - .set pop - - jr ra - li v0, 0 # success - END(_save_fp_context32) -#endif - -/* - * Restore FPU state: - * - fp gp registers - * - cp1 status/control register +/** + * _restore_fp_context() - restore FP context to the FPU + * @a0 - pointer to fpregs field of sigcontext + * @a1 - pointer to fpc_csr field of sigcontext + * + * Restore FP context, including the 32 FP data registers and the FP + * control & status register, from signal context to the FPU. */ LEAF(_restore_fp_context) - EX lw t1, SC_FPC_CSR(a0) + EX lw t1, 0(a1) #if defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPS32_R2) || \ defined(CONFIG_CPU_MIPS32_R6) @@ -178,101 +130,231 @@ LEAF(_restore_fp_context) bgez t0, 1f # skip loading odd if FR=0 nop #endif - EX ldc1 $f1, SC_FPREGS+8(a0) - EX ldc1 $f3, SC_FPREGS+24(a0) - EX ldc1 $f5, SC_FPREGS+40(a0) - EX ldc1 $f7, SC_FPREGS+56(a0) - EX ldc1 $f9, SC_FPREGS+72(a0) - EX ldc1 $f11, SC_FPREGS+88(a0) - EX ldc1 $f13, SC_FPREGS+104(a0) - EX ldc1 $f15, SC_FPREGS+120(a0) - EX ldc1 $f17, SC_FPREGS+136(a0) - EX ldc1 $f19, SC_FPREGS+152(a0) - EX ldc1 $f21, SC_FPREGS+168(a0) - EX ldc1 $f23, SC_FPREGS+184(a0) - EX ldc1 $f25, SC_FPREGS+200(a0) - EX ldc1 $f27, SC_FPREGS+216(a0) - EX ldc1 $f29, SC_FPREGS+232(a0) - EX ldc1 $f31, SC_FPREGS+248(a0) + EX ldc1 $f1, 8(a0) + EX ldc1 $f3, 24(a0) + EX ldc1 $f5, 40(a0) + EX ldc1 $f7, 56(a0) + EX ldc1 $f9, 72(a0) + EX ldc1 $f11, 88(a0) + EX ldc1 $f13, 104(a0) + EX ldc1 $f15, 120(a0) + EX ldc1 $f17, 136(a0) + EX ldc1 $f19, 152(a0) + EX ldc1 $f21, 168(a0) + EX ldc1 $f23, 184(a0) + EX ldc1 $f25, 200(a0) + EX ldc1 $f27, 216(a0) + EX ldc1 $f29, 232(a0) + EX ldc1 $f31, 248(a0) 1: .set pop #endif .set push SET_HARDFLOAT - EX ldc1 $f0, SC_FPREGS+0(a0) - EX ldc1 $f2, SC_FPREGS+16(a0) - EX ldc1 $f4, SC_FPREGS+32(a0) - EX ldc1 $f6, SC_FPREGS+48(a0) - EX ldc1 $f8, SC_FPREGS+64(a0) - EX ldc1 $f10, SC_FPREGS+80(a0) - EX ldc1 $f12, SC_FPREGS+96(a0) - EX ldc1 $f14, SC_FPREGS+112(a0) - EX ldc1 $f16, SC_FPREGS+128(a0) - EX ldc1 $f18, SC_FPREGS+144(a0) - EX ldc1 $f20, SC_FPREGS+160(a0) - EX ldc1 $f22, SC_FPREGS+176(a0) - EX ldc1 $f24, SC_FPREGS+192(a0) - EX ldc1 $f26, SC_FPREGS+208(a0) - EX ldc1 $f28, SC_FPREGS+224(a0) - EX ldc1 $f30, SC_FPREGS+240(a0) + EX ldc1 $f0, 0(a0) + EX ldc1 $f2, 16(a0) + EX ldc1 $f4, 32(a0) + EX ldc1 $f6, 48(a0) + EX ldc1 $f8, 64(a0) + EX ldc1 $f10, 80(a0) + EX ldc1 $f12, 96(a0) + EX ldc1 $f14, 112(a0) + EX ldc1 $f16, 128(a0) + EX ldc1 $f18, 144(a0) + EX ldc1 $f20, 160(a0) + EX ldc1 $f22, 176(a0) + EX ldc1 $f24, 192(a0) + EX ldc1 $f26, 208(a0) + EX ldc1 $f28, 224(a0) + EX ldc1 $f30, 240(a0) ctc1 t1, fcr31 .set pop jr ra li v0, 0 # success END(_restore_fp_context) -#ifdef CONFIG_MIPS32_COMPAT -LEAF(_restore_fp_context32) - /* Restore an o32 sigcontext. */ - .set push - SET_HARDFLOAT - EX lw t1, SC32_FPC_CSR(a0) +#ifdef CONFIG_CPU_HAS_MSA -#ifndef CONFIG_CPU_MIPS64_R6 - mfc0 t0, CP0_STATUS - sll t0, t0, 5 - bgez t0, 1f # skip loading odd if FR=0 - nop -#endif - - EX ldc1 $f1, SC32_FPREGS+8(a0) - EX ldc1 $f3, SC32_FPREGS+24(a0) - EX ldc1 $f5, SC32_FPREGS+40(a0) - EX ldc1 $f7, SC32_FPREGS+56(a0) - EX ldc1 $f9, SC32_FPREGS+72(a0) - EX ldc1 $f11, SC32_FPREGS+88(a0) - EX ldc1 $f13, SC32_FPREGS+104(a0) - EX ldc1 $f15, SC32_FPREGS+120(a0) - EX ldc1 $f17, SC32_FPREGS+136(a0) - EX ldc1 $f19, SC32_FPREGS+152(a0) - EX ldc1 $f21, SC32_FPREGS+168(a0) - EX ldc1 $f23, SC32_FPREGS+184(a0) - EX ldc1 $f25, SC32_FPREGS+200(a0) - EX ldc1 $f27, SC32_FPREGS+216(a0) - EX ldc1 $f29, SC32_FPREGS+232(a0) - EX ldc1 $f31, SC32_FPREGS+248(a0) - -1: EX ldc1 $f0, SC32_FPREGS+0(a0) - EX ldc1 $f2, SC32_FPREGS+16(a0) - EX ldc1 $f4, SC32_FPREGS+32(a0) - EX ldc1 $f6, SC32_FPREGS+48(a0) - EX ldc1 $f8, SC32_FPREGS+64(a0) - EX ldc1 $f10, SC32_FPREGS+80(a0) - EX ldc1 $f12, SC32_FPREGS+96(a0) - EX ldc1 $f14, SC32_FPREGS+112(a0) - EX ldc1 $f16, SC32_FPREGS+128(a0) - EX ldc1 $f18, SC32_FPREGS+144(a0) - EX ldc1 $f20, SC32_FPREGS+160(a0) - EX ldc1 $f22, SC32_FPREGS+176(a0) - EX ldc1 $f24, SC32_FPREGS+192(a0) - EX ldc1 $f26, SC32_FPREGS+208(a0) - EX ldc1 $f28, SC32_FPREGS+224(a0) - EX ldc1 $f30, SC32_FPREGS+240(a0) - ctc1 t1, fcr31 + .macro op_one_wr op, idx, base + .align 4 +\idx: \op \idx, 0, \base jr ra - li v0, 0 # success - .set pop - END(_restore_fp_context32) + nop + .endm + + .macro op_msa_wr name, op +LEAF(\name) + .set push + .set noreorder + sll t0, a0, 4 + PTR_LA t1, 0f + PTR_ADDU t0, t0, t1 + jr t0 + nop + op_one_wr \op, 0, a1 + op_one_wr \op, 1, a1 + op_one_wr \op, 2, a1 + op_one_wr \op, 3, a1 + op_one_wr \op, 4, a1 + op_one_wr \op, 5, a1 + op_one_wr \op, 6, a1 + op_one_wr \op, 7, a1 + op_one_wr \op, 8, a1 + op_one_wr \op, 9, a1 + op_one_wr \op, 10, a1 + op_one_wr \op, 11, a1 + op_one_wr \op, 12, a1 + op_one_wr \op, 13, a1 + op_one_wr \op, 14, a1 + op_one_wr \op, 15, a1 + op_one_wr \op, 16, a1 + op_one_wr \op, 17, a1 + op_one_wr \op, 18, a1 + op_one_wr \op, 19, a1 + op_one_wr \op, 20, a1 + op_one_wr \op, 21, a1 + op_one_wr \op, 22, a1 + op_one_wr \op, 23, a1 + op_one_wr \op, 24, a1 + op_one_wr \op, 25, a1 + op_one_wr \op, 26, a1 + op_one_wr \op, 27, a1 + op_one_wr \op, 28, a1 + op_one_wr \op, 29, a1 + op_one_wr \op, 30, a1 + op_one_wr \op, 31, a1 + .set pop + END(\name) + .endm + + op_msa_wr read_msa_wr_b, st_b + op_msa_wr read_msa_wr_h, st_h + op_msa_wr read_msa_wr_w, st_w + op_msa_wr read_msa_wr_d, st_d + + op_msa_wr write_msa_wr_b, ld_b + op_msa_wr write_msa_wr_h, ld_h + op_msa_wr write_msa_wr_w, ld_w + op_msa_wr write_msa_wr_d, ld_d + +#endif /* CONFIG_CPU_HAS_MSA */ + +#ifdef CONFIG_CPU_HAS_MSA + + .macro save_msa_upper wr, off, base + .set push + .set noat +#ifdef CONFIG_64BIT + copy_u_d \wr, 1 + EX sd $1, \off(\base) +#elif defined(CONFIG_CPU_LITTLE_ENDIAN) + copy_u_w \wr, 2 + EX sw $1, \off(\base) + copy_u_w \wr, 3 + EX sw $1, (\off+4)(\base) +#else /* CONFIG_CPU_BIG_ENDIAN */ + copy_u_w \wr, 2 + EX sw $1, (\off+4)(\base) + copy_u_w \wr, 3 + EX sw $1, \off(\base) #endif + .set pop + .endm + +LEAF(_save_msa_all_upper) + save_msa_upper 0, 0x00, a0 + save_msa_upper 1, 0x08, a0 + save_msa_upper 2, 0x10, a0 + save_msa_upper 3, 0x18, a0 + save_msa_upper 4, 0x20, a0 + save_msa_upper 5, 0x28, a0 + save_msa_upper 6, 0x30, a0 + save_msa_upper 7, 0x38, a0 + save_msa_upper 8, 0x40, a0 + save_msa_upper 9, 0x48, a0 + save_msa_upper 10, 0x50, a0 + save_msa_upper 11, 0x58, a0 + save_msa_upper 12, 0x60, a0 + save_msa_upper 13, 0x68, a0 + save_msa_upper 14, 0x70, a0 + save_msa_upper 15, 0x78, a0 + save_msa_upper 16, 0x80, a0 + save_msa_upper 17, 0x88, a0 + save_msa_upper 18, 0x90, a0 + save_msa_upper 19, 0x98, a0 + save_msa_upper 20, 0xa0, a0 + save_msa_upper 21, 0xa8, a0 + save_msa_upper 22, 0xb0, a0 + save_msa_upper 23, 0xb8, a0 + save_msa_upper 24, 0xc0, a0 + save_msa_upper 25, 0xc8, a0 + save_msa_upper 26, 0xd0, a0 + save_msa_upper 27, 0xd8, a0 + save_msa_upper 28, 0xe0, a0 + save_msa_upper 29, 0xe8, a0 + save_msa_upper 30, 0xf0, a0 + save_msa_upper 31, 0xf8, a0 + jr ra + li v0, 0 + END(_save_msa_all_upper) + + .macro restore_msa_upper wr, off, base + .set push + .set noat +#ifdef CONFIG_64BIT + EX ld $1, \off(\base) + insert_d \wr, 1 +#elif defined(CONFIG_CPU_LITTLE_ENDIAN) + EX lw $1, \off(\base) + insert_w \wr, 2 + EX lw $1, (\off+4)(\base) + insert_w \wr, 3 +#else /* CONFIG_CPU_BIG_ENDIAN */ + EX lw $1, (\off+4)(\base) + insert_w \wr, 2 + EX lw $1, \off(\base) + insert_w \wr, 3 +#endif + .set pop + .endm + +LEAF(_restore_msa_all_upper) + restore_msa_upper 0, 0x00, a0 + restore_msa_upper 1, 0x08, a0 + restore_msa_upper 2, 0x10, a0 + restore_msa_upper 3, 0x18, a0 + restore_msa_upper 4, 0x20, a0 + restore_msa_upper 5, 0x28, a0 + restore_msa_upper 6, 0x30, a0 + restore_msa_upper 7, 0x38, a0 + restore_msa_upper 8, 0x40, a0 + restore_msa_upper 9, 0x48, a0 + restore_msa_upper 10, 0x50, a0 + restore_msa_upper 11, 0x58, a0 + restore_msa_upper 12, 0x60, a0 + restore_msa_upper 13, 0x68, a0 + restore_msa_upper 14, 0x70, a0 + restore_msa_upper 15, 0x78, a0 + restore_msa_upper 16, 0x80, a0 + restore_msa_upper 17, 0x88, a0 + restore_msa_upper 18, 0x90, a0 + restore_msa_upper 19, 0x98, a0 + restore_msa_upper 20, 0xa0, a0 + restore_msa_upper 21, 0xa8, a0 + restore_msa_upper 22, 0xb0, a0 + restore_msa_upper 23, 0xb8, a0 + restore_msa_upper 24, 0xc0, a0 + restore_msa_upper 25, 0xc8, a0 + restore_msa_upper 26, 0xd0, a0 + restore_msa_upper 27, 0xd8, a0 + restore_msa_upper 28, 0xe0, a0 + restore_msa_upper 29, 0xe8, a0 + restore_msa_upper 30, 0xf0, a0 + restore_msa_upper 31, 0xf8, a0 + jr ra + li v0, 0 + END(_restore_msa_all_upper) + +#endif /* CONFIG_CPU_HAS_MSA */ .set reorder diff --git a/arch/mips/kernel/r4k_switch.S b/arch/mips/kernel/r4k_switch.S index 04cbbde3521b..92cd0516ecf5 100644 --- a/arch/mips/kernel/r4k_switch.S +++ b/arch/mips/kernel/r4k_switch.S @@ -34,7 +34,7 @@ #ifndef USE_ALTERNATE_RESUME_IMPL /* * task_struct *resume(task_struct *prev, task_struct *next, - * struct thread_info *next_ti, s32 fp_save) + * struct thread_info *next_ti) */ .align 5 LEAF(resume) @@ -43,45 +43,6 @@ cpu_save_nonscratch a0 LONG_S ra, THREAD_REG31(a0) - /* - * Check whether we need to save any FP context. FP context is saved - * iff the process has used the context with the scalar FPU or the MSA - * ASE in the current time slice, as indicated by _TIF_USEDFPU and - * _TIF_USEDMSA respectively. switch_to will have set fp_save - * accordingly to an FP_SAVE_ enum value. - */ - beqz a3, 2f - - /* - * We do. Clear the saved CU1 bit for prev, such that next time it is - * scheduled it will start in userland with the FPU disabled. If the - * task uses the FPU then it will be enabled again via the do_cpu trap. - * This allows us to lazily restore the FP context. - */ - PTR_L t3, TASK_THREAD_INFO(a0) - LONG_L t0, ST_OFF(t3) - li t1, ~ST0_CU1 - and t0, t0, t1 - LONG_S t0, ST_OFF(t3) - - /* Check whether we're saving scalar or vector context. */ - bgtz a3, 1f - - /* Save 128b MSA vector context + scalar FP control & status. */ - .set push - SET_HARDFLOAT - cfc1 t1, fcr31 - msa_save_all a0 - .set pop /* SET_HARDFLOAT */ - - sw t1, THREAD_FCR31(a0) - b 2f - -1: /* Save 32b/64b scalar FP context. */ - fpu_save_double a0 t0 t1 # c0_status passed in t0 - # clobbers t1 -2: - #if defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_SMP) PTR_LA t8, __stack_chk_guard LONG_L t9, TASK_STACK_CANARY(a1) diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index 008b3378653a..35b8316002f8 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -476,7 +476,7 @@ static void __init bootmem_init(void) * o bootmem_init() * o sparse_init() * o paging_init() - * o dma_continguous_reserve() + * o dma_contiguous_reserve() * * At this stage the bootmem allocator is ready to use. * diff --git a/arch/mips/kernel/signal-common.h b/arch/mips/kernel/signal-common.h index 0b85f827cd18..f50d48435c68 100644 --- a/arch/mips/kernel/signal-common.h +++ b/arch/mips/kernel/signal-common.h @@ -31,4 +31,13 @@ extern int fpcsr_pending(unsigned int __user *fpcsr); #define lock_fpu_owner() ({ preempt_disable(); pagefault_disable(); }) #define unlock_fpu_owner() ({ pagefault_enable(); preempt_enable(); }) +/* Assembly functions to move context to/from the FPU */ +extern asmlinkage int +_save_fp_context(void __user *fpregs, void __user *csr); +extern asmlinkage int +_restore_fp_context(void __user *fpregs, void __user *csr); + +extern asmlinkage int _save_msa_all_upper(void __user *buf); +extern asmlinkage int _restore_msa_all_upper(void __user *buf); + #endif /* __SIGNAL_COMMON_H */ diff --git a/arch/mips/kernel/signal.c b/arch/mips/kernel/signal.c index 6a28c792d862..2fec67bfc457 100644 --- a/arch/mips/kernel/signal.c +++ b/arch/mips/kernel/signal.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -38,20 +39,21 @@ #include #include #include +#include #include "signal-common.h" -static int (*save_fp_context)(struct sigcontext __user *sc); -static int (*restore_fp_context)(struct sigcontext __user *sc); - -extern asmlinkage int _save_fp_context(struct sigcontext __user *sc); -extern asmlinkage int _restore_fp_context(struct sigcontext __user *sc); +static int (*save_fp_context)(void __user *sc); +static int (*restore_fp_context)(void __user *sc); struct sigframe { u32 sf_ass[4]; /* argument save space for o32 */ u32 sf_pad[2]; /* Was: signal trampoline */ + + /* Matches struct ucontext from its uc_mcontext field onwards */ struct sigcontext sf_sc; sigset_t sf_mask; + unsigned long long sf_extcontext[0]; }; struct rt_sigframe { @@ -65,43 +67,255 @@ struct rt_sigframe { * Thread saved context copy to/from a signal context presumed to be on the * user stack, and therefore accessed with appropriate macros from uaccess.h. */ -static int copy_fp_to_sigcontext(struct sigcontext __user *sc) +static int copy_fp_to_sigcontext(void __user *sc) { + struct mips_abi *abi = current->thread.abi; + uint64_t __user *fpregs = sc + abi->off_sc_fpregs; + uint32_t __user *csr = sc + abi->off_sc_fpc_csr; int i; int err = 0; + int inc = test_thread_flag(TIF_32BIT_FPREGS) ? 2 : 1; - for (i = 0; i < NUM_FPU_REGS; i++) { + for (i = 0; i < NUM_FPU_REGS; i += inc) { err |= __put_user(get_fpr64(¤t->thread.fpu.fpr[i], 0), - &sc->sc_fpregs[i]); + &fpregs[i]); } - err |= __put_user(current->thread.fpu.fcr31, &sc->sc_fpc_csr); + err |= __put_user(current->thread.fpu.fcr31, csr); return err; } -static int copy_fp_from_sigcontext(struct sigcontext __user *sc) +static int copy_fp_from_sigcontext(void __user *sc) { + struct mips_abi *abi = current->thread.abi; + uint64_t __user *fpregs = sc + abi->off_sc_fpregs; + uint32_t __user *csr = sc + abi->off_sc_fpc_csr; int i; int err = 0; + int inc = test_thread_flag(TIF_32BIT_FPREGS) ? 2 : 1; u64 fpr_val; - for (i = 0; i < NUM_FPU_REGS; i++) { - err |= __get_user(fpr_val, &sc->sc_fpregs[i]); + for (i = 0; i < NUM_FPU_REGS; i += inc) { + err |= __get_user(fpr_val, &fpregs[i]); set_fpr64(¤t->thread.fpu.fpr[i], 0, fpr_val); } - err |= __get_user(current->thread.fpu.fcr31, &sc->sc_fpc_csr); + err |= __get_user(current->thread.fpu.fcr31, csr); return err; } +/* + * Wrappers for the assembly _{save,restore}_fp_context functions. + */ +static int save_hw_fp_context(void __user *sc) +{ + struct mips_abi *abi = current->thread.abi; + uint64_t __user *fpregs = sc + abi->off_sc_fpregs; + uint32_t __user *csr = sc + abi->off_sc_fpc_csr; + + return _save_fp_context(fpregs, csr); +} + +static int restore_hw_fp_context(void __user *sc) +{ + struct mips_abi *abi = current->thread.abi; + uint64_t __user *fpregs = sc + abi->off_sc_fpregs; + uint32_t __user *csr = sc + abi->off_sc_fpc_csr; + + return _restore_fp_context(fpregs, csr); +} + +/* + * Extended context handling. + */ + +static inline void __user *sc_to_extcontext(void __user *sc) +{ + struct ucontext __user *uc; + + /* + * We can just pretend the sigcontext is always embedded in a struct + * ucontext here, because the offset from sigcontext to extended + * context is the same in the struct sigframe case. + */ + uc = container_of(sc, struct ucontext, uc_mcontext); + return &uc->uc_extcontext; +} + +static int save_msa_extcontext(void __user *buf) +{ + struct msa_extcontext __user *msa = buf; + uint64_t val; + int i, err; + + if (!thread_msa_context_live()) + return 0; + + /* + * Ensure that we can't lose the live MSA context between checking + * for it & writing it to memory. + */ + preempt_disable(); + + if (is_msa_enabled()) { + /* + * There are no EVA versions of the vector register load/store + * instructions, so MSA context has to be saved to kernel memory + * and then copied to user memory. The save to kernel memory + * should already have been done when handling scalar FP + * context. + */ + BUG_ON(config_enabled(CONFIG_EVA)); + + err = __put_user(read_msa_csr(), &msa->csr); + err |= _save_msa_all_upper(&msa->wr); + + preempt_enable(); + } else { + preempt_enable(); + + err = __put_user(current->thread.fpu.msacsr, &msa->csr); + + for (i = 0; i < NUM_FPU_REGS; i++) { + val = get_fpr64(¤t->thread.fpu.fpr[i], 1); + err |= __put_user(val, &msa->wr[i]); + } + } + + err |= __put_user(MSA_EXTCONTEXT_MAGIC, &msa->ext.magic); + err |= __put_user(sizeof(*msa), &msa->ext.size); + + return err ? -EFAULT : sizeof(*msa); +} + +static int restore_msa_extcontext(void __user *buf, unsigned int size) +{ + struct msa_extcontext __user *msa = buf; + unsigned long long val; + unsigned int csr; + int i, err; + + if (size != sizeof(*msa)) + return -EINVAL; + + err = get_user(csr, &msa->csr); + if (err) + return err; + + preempt_disable(); + + if (is_msa_enabled()) { + /* + * There are no EVA versions of the vector register load/store + * instructions, so MSA context has to be copied to kernel + * memory and later loaded to registers. The same is true of + * scalar FP context, so FPU & MSA should have already been + * disabled whilst handling scalar FP context. + */ + BUG_ON(config_enabled(CONFIG_EVA)); + + write_msa_csr(csr); + err |= _restore_msa_all_upper(&msa->wr); + preempt_enable(); + } else { + preempt_enable(); + + current->thread.fpu.msacsr = csr; + + for (i = 0; i < NUM_FPU_REGS; i++) { + err |= __get_user(val, &msa->wr[i]); + set_fpr64(¤t->thread.fpu.fpr[i], 1, val); + } + } + + return err; +} + +static int save_extcontext(void __user *buf) +{ + int sz; + + sz = save_msa_extcontext(buf); + if (sz < 0) + return sz; + buf += sz; + + /* If no context was saved then trivially return */ + if (!sz) + return 0; + + /* Write the end marker */ + if (__put_user(END_EXTCONTEXT_MAGIC, (u32 *)buf)) + return -EFAULT; + + sz += sizeof(((struct extcontext *)NULL)->magic); + return sz; +} + +static int restore_extcontext(void __user *buf) +{ + struct extcontext ext; + int err; + + while (1) { + err = __get_user(ext.magic, (unsigned int *)buf); + if (err) + return err; + + if (ext.magic == END_EXTCONTEXT_MAGIC) + return 0; + + err = __get_user(ext.size, (unsigned int *)(buf + + offsetof(struct extcontext, size))); + if (err) + return err; + + switch (ext.magic) { + case MSA_EXTCONTEXT_MAGIC: + err = restore_msa_extcontext(buf, ext.size); + break; + + default: + err = -EINVAL; + break; + } + + if (err) + return err; + + buf += ext.size; + } +} + /* * Helper routines */ -static int protected_save_fp_context(struct sigcontext __user *sc) +int protected_save_fp_context(void __user *sc) { + struct mips_abi *abi = current->thread.abi; + uint64_t __user *fpregs = sc + abi->off_sc_fpregs; + uint32_t __user *csr = sc + abi->off_sc_fpc_csr; + uint32_t __user *used_math = sc + abi->off_sc_used_math; + unsigned int used, ext_sz; int err; -#ifndef CONFIG_EVA + + used = used_math() ? USED_FP : 0; + if (!used) + goto fp_done; + + if (!test_thread_flag(TIF_32BIT_FPREGS)) + used |= USED_FR1; + if (test_thread_flag(TIF_HYBRID_FPREGS)) + used |= USED_HYBRID_FPRS; + + /* + * EVA does not have userland equivalents of ldc1 or sdc1, so + * save to the kernel FP context & copy that to userland below. + */ + if (config_enabled(CONFIG_EVA)) + lose_fpu(1); + while (1) { lock_fpu_owner(); if (is_fpu_owner()) { @@ -114,27 +328,57 @@ static int protected_save_fp_context(struct sigcontext __user *sc) if (likely(!err)) break; /* touch the sigcontext and try again */ - err = __put_user(0, &sc->sc_fpregs[0]) | - __put_user(0, &sc->sc_fpregs[31]) | - __put_user(0, &sc->sc_fpc_csr); + err = __put_user(0, &fpregs[0]) | + __put_user(0, &fpregs[31]) | + __put_user(0, csr); if (err) - break; /* really bad sigcontext */ + return err; /* really bad sigcontext */ } -#else - /* - * EVA does not have FPU EVA instructions so saving fpu context directly - * does not work. - */ - lose_fpu(1); - err = save_fp_context(sc); /* this might fail */ -#endif - return err; + +fp_done: + ext_sz = err = save_extcontext(sc_to_extcontext(sc)); + if (err < 0) + return err; + used |= ext_sz ? USED_EXTCONTEXT : 0; + + return __put_user(used, used_math); } -static int protected_restore_fp_context(struct sigcontext __user *sc) +int protected_restore_fp_context(void __user *sc) { - int err, tmp __maybe_unused; -#ifndef CONFIG_EVA + struct mips_abi *abi = current->thread.abi; + uint64_t __user *fpregs = sc + abi->off_sc_fpregs; + uint32_t __user *csr = sc + abi->off_sc_fpc_csr; + uint32_t __user *used_math = sc + abi->off_sc_used_math; + unsigned int used; + int err, sig = 0, tmp __maybe_unused; + + err = __get_user(used, used_math); + conditional_used_math(used & USED_FP); + + /* + * The signal handler may have used FPU; give it up if the program + * doesn't want it following sigreturn. + */ + if (err || !(used & USED_FP)) + lose_fpu(0); + if (err) + return err; + if (!(used & USED_FP)) + goto fp_done; + + err = sig = fpcsr_pending(csr); + if (err < 0) + return err; + + /* + * EVA does not have userland equivalents of ldc1 or sdc1, so we + * disable the FPU here such that the code below simply copies to + * the kernel FP context. + */ + if (config_enabled(CONFIG_EVA)) + lose_fpu(0); + while (1) { lock_fpu_owner(); if (is_fpu_owner()) { @@ -147,28 +391,24 @@ static int protected_restore_fp_context(struct sigcontext __user *sc) if (likely(!err)) break; /* touch the sigcontext and try again */ - err = __get_user(tmp, &sc->sc_fpregs[0]) | - __get_user(tmp, &sc->sc_fpregs[31]) | - __get_user(tmp, &sc->sc_fpc_csr); + err = __get_user(tmp, &fpregs[0]) | + __get_user(tmp, &fpregs[31]) | + __get_user(tmp, csr); if (err) break; /* really bad sigcontext */ } -#else - /* - * EVA does not have FPU EVA instructions so restoring fpu context - * directly does not work. - */ - lose_fpu(0); - err = restore_fp_context(sc); /* this might fail */ -#endif - return err; + +fp_done: + if (used & USED_EXTCONTEXT) + err |= restore_extcontext(sc_to_extcontext(sc)); + + return err ?: sig; } int setup_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc) { int err = 0; int i; - unsigned int used_math; err |= __put_user(regs->cp0_epc, &sc->sc_pc); @@ -191,19 +431,38 @@ int setup_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc) err |= __put_user(rddsp(DSP_MASK), &sc->sc_dsp); } - used_math = !!used_math(); - err |= __put_user(used_math, &sc->sc_used_math); - if (used_math) { - /* - * Save FPU state to signal context. Signal handler - * will "inherit" current FPU state. - */ - err |= protected_save_fp_context(sc); - } + /* + * Save FPU state to signal context. Signal handler + * will "inherit" current FPU state. + */ + err |= protected_save_fp_context(sc); + return err; } +static size_t extcontext_max_size(void) +{ + size_t sz = 0; + + /* + * The assumption here is that between this point & the point at which + * the extended context is saved the size of the context should only + * ever be able to shrink (if the task is preempted), but never grow. + * That is, what this function returns is an upper bound on the size of + * the extended context for the current task at the current time. + */ + + if (thread_msa_context_live()) + sz += sizeof(struct msa_extcontext); + + /* If any context is saved then we'll append the end marker */ + if (sz) + sz += sizeof(((struct extcontext *)NULL)->magic); + + return sz; +} + int fpcsr_pending(unsigned int __user *fpcsr) { int err, sig = 0; @@ -223,21 +482,8 @@ int fpcsr_pending(unsigned int __user *fpcsr) return err ?: sig; } -static int -check_and_restore_fp_context(struct sigcontext __user *sc) -{ - int err, sig; - - err = sig = fpcsr_pending(&sc->sc_fpc_csr); - if (err > 0) - err = 0; - err |= protected_restore_fp_context(sc); - return err ?: sig; -} - int restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc) { - unsigned int used_math; unsigned long treg; int err = 0; int i; @@ -265,19 +511,7 @@ int restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc) for (i = 1; i < 32; i++) err |= __get_user(regs->regs[i], &sc->sc_regs[i]); - err |= __get_user(used_math, &sc->sc_used_math); - conditional_used_math(used_math); - - if (used_math) { - /* restore fpu context if we have used it before */ - if (!err) - err = check_and_restore_fp_context(sc); - } else { - /* signal handler may have used FPU. Give it up. */ - lose_fpu(0); - } - - return err; + return err ?: protected_restore_fp_context(sc); } void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, @@ -285,6 +519,9 @@ void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, { unsigned long sp; + /* Leave space for potential extended context */ + frame_size += extcontext_max_size(); + /* Default to using normal stack */ sp = regs->regs[29]; @@ -520,7 +757,11 @@ struct mips_abi mips_abi = { .setup_rt_frame = setup_rt_frame, .rt_signal_return_offset = offsetof(struct mips_vdso, rt_signal_trampoline), - .restart = __NR_restart_syscall + .restart = __NR_restart_syscall, + + .off_sc_fpregs = offsetof(struct sigcontext, sc_fpregs), + .off_sc_fpc_csr = offsetof(struct sigcontext, sc_fpc_csr), + .off_sc_used_math = offsetof(struct sigcontext, sc_used_math), }; static void handle_signal(struct ksignal *ksig, struct pt_regs *regs) @@ -616,6 +857,9 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, void *unused, user_exit(); + if (thread_info_flags & _TIF_UPROBE) + uprobe_notify_resume(regs); + /* deal with pending signal delivery */ if (thread_info_flags & _TIF_SIGPENDING) do_signal(regs); @@ -629,43 +873,46 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, void *unused, } #ifdef CONFIG_SMP -#ifndef CONFIG_EVA -static int smp_save_fp_context(struct sigcontext __user *sc) +static int smp_save_fp_context(void __user *sc) { return raw_cpu_has_fpu - ? _save_fp_context(sc) + ? save_hw_fp_context(sc) : copy_fp_to_sigcontext(sc); } -static int smp_restore_fp_context(struct sigcontext __user *sc) +static int smp_restore_fp_context(void __user *sc) { return raw_cpu_has_fpu - ? _restore_fp_context(sc) + ? restore_hw_fp_context(sc) : copy_fp_from_sigcontext(sc); } -#endif /* CONFIG_EVA */ #endif static int signal_setup(void) { -#ifndef CONFIG_EVA + /* + * The offset from sigcontext to extended context should be the same + * regardless of the type of signal, such that userland can always know + * where to look if it wishes to find the extended context structures. + */ + BUILD_BUG_ON((offsetof(struct sigframe, sf_extcontext) - + offsetof(struct sigframe, sf_sc)) != + (offsetof(struct rt_sigframe, rs_uc.uc_extcontext) - + offsetof(struct rt_sigframe, rs_uc.uc_mcontext))); + #ifdef CONFIG_SMP /* For now just do the cpu_has_fpu check when the functions are invoked */ save_fp_context = smp_save_fp_context; restore_fp_context = smp_restore_fp_context; #else if (cpu_has_fpu) { - save_fp_context = _save_fp_context; - restore_fp_context = _restore_fp_context; + save_fp_context = save_hw_fp_context; + restore_fp_context = restore_hw_fp_context; } else { save_fp_context = copy_fp_to_sigcontext; restore_fp_context = copy_fp_from_sigcontext; } #endif /* CONFIG_SMP */ -#else - save_fp_context = copy_fp_to_sigcontext; - restore_fp_context = copy_fp_from_sigcontext; -#endif return 0; } diff --git a/arch/mips/kernel/signal32.c b/arch/mips/kernel/signal32.c index 5d7f2634996f..f7e89524e316 100644 --- a/arch/mips/kernel/signal32.c +++ b/arch/mips/kernel/signal32.c @@ -36,12 +36,6 @@ #include "signal-common.h" -static int (*save_fp_context32)(struct sigcontext32 __user *sc); -static int (*restore_fp_context32)(struct sigcontext32 __user *sc); - -extern asmlinkage int _save_fp_context32(struct sigcontext32 __user *sc); -extern asmlinkage int _restore_fp_context32(struct sigcontext32 __user *sc); - /* * Including would give use the 64-bit syscall numbers ... */ @@ -74,99 +68,11 @@ struct rt_sigframe32 { struct ucontext32 rs_uc; }; -/* - * Thread saved context copy to/from a signal context presumed to be on the - * user stack, and therefore accessed with appropriate macros from uaccess.h. - */ -static int copy_fp_to_sigcontext32(struct sigcontext32 __user *sc) -{ - int i; - int err = 0; - int inc = test_thread_flag(TIF_32BIT_FPREGS) ? 2 : 1; - - for (i = 0; i < NUM_FPU_REGS; i += inc) { - err |= - __put_user(get_fpr64(¤t->thread.fpu.fpr[i], 0), - &sc->sc_fpregs[i]); - } - err |= __put_user(current->thread.fpu.fcr31, &sc->sc_fpc_csr); - - return err; -} - -static int copy_fp_from_sigcontext32(struct sigcontext32 __user *sc) -{ - int i; - int err = 0; - int inc = test_thread_flag(TIF_32BIT_FPREGS) ? 2 : 1; - u64 fpr_val; - - for (i = 0; i < NUM_FPU_REGS; i += inc) { - err |= __get_user(fpr_val, &sc->sc_fpregs[i]); - set_fpr64(¤t->thread.fpu.fpr[i], 0, fpr_val); - } - err |= __get_user(current->thread.fpu.fcr31, &sc->sc_fpc_csr); - - return err; -} - -/* - * sigcontext handlers - */ -static int protected_save_fp_context32(struct sigcontext32 __user *sc) -{ - int err; - while (1) { - lock_fpu_owner(); - if (is_fpu_owner()) { - err = save_fp_context32(sc); - unlock_fpu_owner(); - } else { - unlock_fpu_owner(); - err = copy_fp_to_sigcontext32(sc); - } - if (likely(!err)) - break; - /* touch the sigcontext and try again */ - err = __put_user(0, &sc->sc_fpregs[0]) | - __put_user(0, &sc->sc_fpregs[31]) | - __put_user(0, &sc->sc_fpc_csr); - if (err) - break; /* really bad sigcontext */ - } - return err; -} - -static int protected_restore_fp_context32(struct sigcontext32 __user *sc) -{ - int err, tmp __maybe_unused; - while (1) { - lock_fpu_owner(); - if (is_fpu_owner()) { - err = restore_fp_context32(sc); - unlock_fpu_owner(); - } else { - unlock_fpu_owner(); - err = copy_fp_from_sigcontext32(sc); - } - if (likely(!err)) - break; - /* touch the sigcontext and try again */ - err = __get_user(tmp, &sc->sc_fpregs[0]) | - __get_user(tmp, &sc->sc_fpregs[31]) | - __get_user(tmp, &sc->sc_fpc_csr); - if (err) - break; /* really bad sigcontext */ - } - return err; -} - static int setup_sigcontext32(struct pt_regs *regs, struct sigcontext32 __user *sc) { int err = 0; int i; - u32 used_math; err |= __put_user(regs->cp0_epc, &sc->sc_pc); @@ -186,35 +92,18 @@ static int setup_sigcontext32(struct pt_regs *regs, err |= __put_user(mflo3(), &sc->sc_lo3); } - used_math = !!used_math(); - err |= __put_user(used_math, &sc->sc_used_math); + /* + * Save FPU state to signal context. Signal handler + * will "inherit" current FPU state. + */ + err |= protected_save_fp_context(sc); - if (used_math) { - /* - * Save FPU state to signal context. Signal handler - * will "inherit" current FPU state. - */ - err |= protected_save_fp_context32(sc); - } return err; } -static int -check_and_restore_fp_context32(struct sigcontext32 __user *sc) -{ - int err, sig; - - err = sig = fpcsr_pending(&sc->sc_fpc_csr); - if (err > 0) - err = 0; - err |= protected_restore_fp_context32(sc); - return err ?: sig; -} - static int restore_sigcontext32(struct pt_regs *regs, struct sigcontext32 __user *sc) { - u32 used_math; int err = 0; s32 treg; int i; @@ -238,70 +127,7 @@ static int restore_sigcontext32(struct pt_regs *regs, for (i = 1; i < 32; i++) err |= __get_user(regs->regs[i], &sc->sc_regs[i]); - err |= __get_user(used_math, &sc->sc_used_math); - conditional_used_math(used_math); - - if (used_math) { - /* restore fpu context if we have used it before */ - if (!err) - err = check_and_restore_fp_context32(sc); - } else { - /* signal handler may have used FPU. Give it up. */ - lose_fpu(0); - } - - return err; -} - -/* - * - */ -extern void __put_sigset_unknown_nsig(void); -extern void __get_sigset_unknown_nsig(void); - -static inline int put_sigset(const sigset_t *kbuf, compat_sigset_t __user *ubuf) -{ - int err = 0; - - if (!access_ok(VERIFY_WRITE, ubuf, sizeof(*ubuf))) - return -EFAULT; - - switch (_NSIG_WORDS) { - default: - __put_sigset_unknown_nsig(); - case 2: - err |= __put_user(kbuf->sig[1] >> 32, &ubuf->sig[3]); - err |= __put_user(kbuf->sig[1] & 0xffffffff, &ubuf->sig[2]); - case 1: - err |= __put_user(kbuf->sig[0] >> 32, &ubuf->sig[1]); - err |= __put_user(kbuf->sig[0] & 0xffffffff, &ubuf->sig[0]); - } - - return err; -} - -static inline int get_sigset(sigset_t *kbuf, const compat_sigset_t __user *ubuf) -{ - int err = 0; - unsigned long sig[4]; - - if (!access_ok(VERIFY_READ, ubuf, sizeof(*ubuf))) - return -EFAULT; - - switch (_NSIG_WORDS) { - default: - __get_sigset_unknown_nsig(); - case 2: - err |= __get_user(sig[3], &ubuf->sig[3]); - err |= __get_user(sig[2], &ubuf->sig[2]); - kbuf->sig[1] = sig[2] | (sig[3] << 32); - case 1: - err |= __get_user(sig[1], &ubuf->sig[1]); - err |= __get_user(sig[0], &ubuf->sig[0]); - kbuf->sig[0] = sig[0] | (sig[1] << 32); - } - - return err; + return err ?: protected_restore_fp_context(sc); } /* @@ -585,20 +411,9 @@ struct mips_abi mips_abi_32 = { .setup_rt_frame = setup_rt_frame_32, .rt_signal_return_offset = offsetof(struct mips_vdso, o32_rt_signal_trampoline), - .restart = __NR_O32_restart_syscall + .restart = __NR_O32_restart_syscall, + + .off_sc_fpregs = offsetof(struct sigcontext32, sc_fpregs), + .off_sc_fpc_csr = offsetof(struct sigcontext32, sc_fpc_csr), + .off_sc_used_math = offsetof(struct sigcontext32, sc_used_math), }; - -static int signal32_init(void) -{ - if (cpu_has_fpu) { - save_fp_context32 = _save_fp_context32; - restore_fp_context32 = _restore_fp_context32; - } else { - save_fp_context32 = copy_fp_to_sigcontext32; - restore_fp_context32 = copy_fp_from_sigcontext32; - } - - return 0; -} - -arch_initcall(signal32_init); diff --git a/arch/mips/kernel/signal_n32.c b/arch/mips/kernel/signal_n32.c index f1d4751eead0..0d017fdcaf07 100644 --- a/arch/mips/kernel/signal_n32.c +++ b/arch/mips/kernel/signal_n32.c @@ -153,5 +153,9 @@ struct mips_abi mips_abi_n32 = { .setup_rt_frame = setup_rt_frame_n32, .rt_signal_return_offset = offsetof(struct mips_vdso, n32_rt_signal_trampoline), - .restart = __NR_N32_restart_syscall + .restart = __NR_N32_restart_syscall, + + .off_sc_fpregs = offsetof(struct sigcontext, sc_fpregs), + .off_sc_fpc_csr = offsetof(struct sigcontext, sc_fpc_csr), + .off_sc_used_math = offsetof(struct sigcontext, sc_used_math), }; diff --git a/arch/mips/kernel/spram.c b/arch/mips/kernel/spram.c index d1168d7c31e8..8489c88f9932 100644 --- a/arch/mips/kernel/spram.c +++ b/arch/mips/kernel/spram.c @@ -209,6 +209,7 @@ void spram_config(void) case CPU_PROAPTIV: case CPU_P5600: case CPU_QEMU_GENERIC: + case CPU_I6400: config0 = read_c0_config(); /* FIXME: addresses are Malta specific */ if (config0 & (1<<24)) { diff --git a/arch/mips/kernel/sysrq.c b/arch/mips/kernel/sysrq.c index 5b539f5fc9d9..5f055393092d 100644 --- a/arch/mips/kernel/sysrq.c +++ b/arch/mips/kernel/sysrq.c @@ -21,24 +21,12 @@ static DEFINE_SPINLOCK(show_lock); static void sysrq_tlbdump_single(void *dummy) { - const int field = 2 * sizeof(unsigned long); unsigned long flags; spin_lock_irqsave(&show_lock, flags); pr_info("CPU%d:\n", smp_processor_id()); - pr_info("Index : %0x\n", read_c0_index()); - pr_info("Pagemask: %0x\n", read_c0_pagemask()); - pr_info("EntryHi : %0*lx\n", field, read_c0_entryhi()); - pr_info("EntryLo0: %0*lx\n", field, read_c0_entrylo0()); - pr_info("EntryLo1: %0*lx\n", field, read_c0_entrylo1()); - pr_info("Wired : %0x\n", read_c0_wired()); - pr_info("Pagegrain: %0x\n", read_c0_pagegrain()); - if (cpu_has_htw) { - pr_info("PWField : %0*lx\n", field, read_c0_pwfield()); - pr_info("PWSize : %0*lx\n", field, read_c0_pwsize()); - pr_info("PWCtl : %0x\n", read_c0_pwctl()); - } + dump_tlb_regs(); pr_info("\n"); dump_tlb_all(); pr_info("\n"); diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index 8ea28e6ab37d..fdb392b27e81 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -370,11 +370,6 @@ void show_registers(struct pt_regs *regs) set_fs(old_fs); } -static int regs_to_trapnr(struct pt_regs *regs) -{ - return (regs->cp0_cause >> 2) & 0x1f; -} - static DEFINE_RAW_SPINLOCK(die_lock); void __noreturn die(const char *str, struct pt_regs *regs) @@ -384,7 +379,7 @@ void __noreturn die(const char *str, struct pt_regs *regs) oops_enter(); - if (notify_die(DIE_OOPS, str, regs, 0, regs_to_trapnr(regs), + if (notify_die(DIE_OOPS, str, regs, 0, current->thread.trap_nr, SIGSEGV) == NOTIFY_STOP) sig = 0; @@ -470,7 +465,7 @@ asmlinkage void do_be(struct pt_regs *regs) printk(KERN_ALERT "%s bus error, epc == %0*lx, ra == %0*lx\n", data ? "Data" : "Instruction", field, regs->cp0_epc, field, regs->regs[31]); - if (notify_die(DIE_OOPS, "bus error", regs, 0, regs_to_trapnr(regs), + if (notify_die(DIE_OOPS, "bus error", regs, 0, current->thread.trap_nr, SIGBUS) == NOTIFY_STOP) goto out; @@ -826,7 +821,7 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31) int sig; prev_state = exception_enter(); - if (notify_die(DIE_FP, "FP exception", regs, 0, regs_to_trapnr(regs), + if (notify_die(DIE_FP, "FP exception", regs, 0, current->thread.trap_nr, SIGFPE) == NOTIFY_STOP) goto out; @@ -882,11 +877,12 @@ void do_trap_or_bp(struct pt_regs *regs, unsigned int code, char b[40]; #ifdef CONFIG_KGDB_LOW_LEVEL_TRAP - if (kgdb_ll_trap(DIE_TRAP, str, regs, code, regs_to_trapnr(regs), SIGTRAP) == NOTIFY_STOP) + if (kgdb_ll_trap(DIE_TRAP, str, regs, code, current->thread.trap_nr, + SIGTRAP) == NOTIFY_STOP) return; #endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */ - if (notify_die(DIE_TRAP, str, regs, code, regs_to_trapnr(regs), + if (notify_die(DIE_TRAP, str, regs, code, current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) return; @@ -948,6 +944,7 @@ asmlinkage void do_bp(struct pt_regs *regs) set_fs(KERNEL_DS); prev_state = exception_enter(); + current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f; if (get_isa16_mode(regs->cp0_epc)) { u16 instr[2]; @@ -987,15 +984,27 @@ asmlinkage void do_bp(struct pt_regs *regs) * pertain to them. */ switch (bcode) { + case BRK_UPROBE: + if (notify_die(DIE_UPROBE, "uprobe", regs, bcode, + current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) + goto out; + else + break; + case BRK_UPROBE_XOL: + if (notify_die(DIE_UPROBE_XOL, "uprobe_xol", regs, bcode, + current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) + goto out; + else + break; case BRK_KPROBE_BP: if (notify_die(DIE_BREAK, "debug", regs, bcode, - regs_to_trapnr(regs), SIGTRAP) == NOTIFY_STOP) + current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) goto out; else break; case BRK_KPROBE_SSTEPBP: if (notify_die(DIE_SSTEPBP, "single_step", regs, bcode, - regs_to_trapnr(regs), SIGTRAP) == NOTIFY_STOP) + current->thread.trap_nr, SIGTRAP) == NOTIFY_STOP) goto out; else break; @@ -1028,6 +1037,7 @@ asmlinkage void do_tr(struct pt_regs *regs) set_fs(get_ds()); prev_state = exception_enter(); + current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f; if (get_isa16_mode(regs->cp0_epc)) { if (__get_user(instr[0], (u16 __user *)(epc + 0)) || __get_user(instr[1], (u16 __user *)(epc + 2))) @@ -1094,8 +1104,9 @@ asmlinkage void do_ri(struct pt_regs *regs) no_r2_instr: prev_state = exception_enter(); + current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f; - if (notify_die(DIE_RI, "RI Fault", regs, 0, regs_to_trapnr(regs), + if (notify_die(DIE_RI, "RI Fault", regs, 0, current->thread.trap_nr, SIGILL) == NOTIFY_STOP) goto out; @@ -1444,8 +1455,9 @@ asmlinkage void do_msa_fpe(struct pt_regs *regs, unsigned int msacsr) enum ctx_state prev_state; prev_state = exception_enter(); + current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f; if (notify_die(DIE_MSAFP, "MSA FP exception", regs, 0, - regs_to_trapnr(regs), SIGFPE) == NOTIFY_STOP) + current->thread.trap_nr, SIGFPE) == NOTIFY_STOP) goto out; /* Clear MSACSR.Cause before enabling interrupts */ @@ -1523,7 +1535,6 @@ asmlinkage void do_watch(struct pt_regs *regs) asmlinkage void do_mcheck(struct pt_regs *regs) { - const int field = 2 * sizeof(unsigned long); int multi_match = regs->cp0_status & ST0_TS; enum ctx_state prev_state; mm_segment_t old_fs = get_fs(); @@ -1532,19 +1543,8 @@ asmlinkage void do_mcheck(struct pt_regs *regs) show_regs(regs); if (multi_match) { - pr_err("Index : %0x\n", read_c0_index()); - pr_err("Pagemask: %0x\n", read_c0_pagemask()); - pr_err("EntryHi : %0*lx\n", field, read_c0_entryhi()); - pr_err("EntryLo0: %0*lx\n", field, read_c0_entrylo0()); - pr_err("EntryLo1: %0*lx\n", field, read_c0_entrylo1()); - pr_err("Wired : %0x\n", read_c0_wired()); - pr_err("Pagegrain: %0x\n", read_c0_pagegrain()); - if (cpu_has_htw) { - pr_err("PWField : %0*lx\n", field, read_c0_pwfield()); - pr_err("PWSize : %0*lx\n", field, read_c0_pwsize()); - pr_err("PWCtl : %0x\n", read_c0_pwctl()); - } - pr_err("\n"); + dump_tlb_regs(); + pr_info("\n"); dump_tlb_all(); } @@ -1651,6 +1651,7 @@ static inline void parity_protection_init(void) case CPU_PROAPTIV: case CPU_P5600: case CPU_QEMU_GENERIC: + case CPU_I6400: { #define ERRCTL_PE 0x80000000 #define ERRCTL_L2P 0x00800000 diff --git a/arch/mips/kernel/unaligned.c b/arch/mips/kernel/unaligned.c index eb3efd137fd1..990354dd6bde 100644 --- a/arch/mips/kernel/unaligned.c +++ b/arch/mips/kernel/unaligned.c @@ -891,6 +891,9 @@ static void emulate_load_store_insn(struct pt_regs *regs, #ifdef CONFIG_EVA mm_segment_t seg; #endif + union fpureg *fpr; + enum msa_2b_fmt df; + unsigned int wd; origpc = (unsigned long)pc; orig31 = regs->regs[31]; @@ -1202,6 +1205,75 @@ static void emulate_load_store_insn(struct pt_regs *regs, break; return; + case msa_op: + if (!cpu_has_msa) + goto sigill; + + /* + * If we've reached this point then userland should have taken + * the MSA disabled exception & initialised vector context at + * some point in the past. + */ + BUG_ON(!thread_msa_context_live()); + + df = insn.msa_mi10_format.df; + wd = insn.msa_mi10_format.wd; + fpr = ¤t->thread.fpu.fpr[wd]; + + switch (insn.msa_mi10_format.func) { + case msa_ld_op: + if (!access_ok(VERIFY_READ, addr, sizeof(*fpr))) + goto sigbus; + + /* + * Disable preemption to avoid a race between copying + * state from userland, migrating to another CPU and + * updating the hardware vector register below. + */ + preempt_disable(); + + res = __copy_from_user_inatomic(fpr, addr, + sizeof(*fpr)); + if (res) + goto fault; + + /* + * Update the hardware register if it is in use by the + * task in this quantum, in order to avoid having to + * save & restore the whole vector context. + */ + if (test_thread_flag(TIF_USEDMSA)) + write_msa_wr(wd, fpr, df); + + preempt_enable(); + break; + + case msa_st_op: + if (!access_ok(VERIFY_WRITE, addr, sizeof(*fpr))) + goto sigbus; + + /* + * Update from the hardware register if it is in use by + * the task in this quantum, in order to avoid having to + * save & restore the whole vector context. + */ + preempt_disable(); + if (test_thread_flag(TIF_USEDMSA)) + read_msa_wr(wd, fpr, df); + preempt_enable(); + + res = __copy_to_user_inatomic(addr, fpr, sizeof(*fpr)); + if (res) + goto fault; + break; + + default: + goto sigbus; + } + + compute_return_epc(regs); + break; + #ifndef CONFIG_CPU_MIPSR6 /* * COP2 is available to implementor for application specific use. @@ -2240,5 +2312,5 @@ static int __init debugfs_unaligned(void) return -ENOMEM; return 0; } -__initcall(debugfs_unaligned); +arch_initcall(debugfs_unaligned); #endif diff --git a/arch/mips/kernel/uprobes.c b/arch/mips/kernel/uprobes.c new file mode 100644 index 000000000000..8452d933a645 --- /dev/null +++ b/arch/mips/kernel/uprobes.c @@ -0,0 +1,341 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +static inline int insn_has_delay_slot(const union mips_instruction insn) +{ + switch (insn.i_format.opcode) { + /* + * jr and jalr are in r_format format. + */ + case spec_op: + switch (insn.r_format.func) { + case jalr_op: + case jr_op: + return 1; + } + break; + + /* + * This group contains: + * bltz_op, bgez_op, bltzl_op, bgezl_op, + * bltzal_op, bgezal_op, bltzall_op, bgezall_op. + */ + case bcond_op: + switch (insn.i_format.rt) { + case bltz_op: + case bltzl_op: + case bgez_op: + case bgezl_op: + case bltzal_op: + case bltzall_op: + case bgezal_op: + case bgezall_op: + case bposge32_op: + return 1; + } + break; + + /* + * These are unconditional and in j_format. + */ + case jal_op: + case j_op: + case beq_op: + case beql_op: + case bne_op: + case bnel_op: + case blez_op: /* not really i_format */ + case blezl_op: + case bgtz_op: + case bgtzl_op: + return 1; + + /* + * And now the FPA/cp1 branch instructions. + */ + case cop1_op: +#ifdef CONFIG_CPU_CAVIUM_OCTEON + case lwc2_op: /* This is bbit0 on Octeon */ + case ldc2_op: /* This is bbit032 on Octeon */ + case swc2_op: /* This is bbit1 on Octeon */ + case sdc2_op: /* This is bbit132 on Octeon */ +#endif + return 1; + } + + return 0; +} + +/** + * arch_uprobe_analyze_insn - instruction analysis including validity and fixups. + * @mm: the probed address space. + * @arch_uprobe: the probepoint information. + * @addr: virtual address at which to install the probepoint + * Return 0 on success or a -ve number on error. + */ +int arch_uprobe_analyze_insn(struct arch_uprobe *aup, + struct mm_struct *mm, unsigned long addr) +{ + union mips_instruction inst; + + /* + * For the time being this also blocks attempts to use uprobes with + * MIPS16 and microMIPS. + */ + if (addr & 0x03) + return -EINVAL; + + inst.word = aup->insn[0]; + aup->ixol[0] = aup->insn[insn_has_delay_slot(inst)]; + aup->ixol[1] = UPROBE_BRK_UPROBE_XOL; /* NOP */ + + return 0; +} + +/** + * is_trap_insn - check if the instruction is a trap variant + * @insn: instruction to be checked. + * Returns true if @insn is a trap variant. + * + * This definition overrides the weak definition in kernel/events/uprobes.c. + * and is needed for the case where an architecture has multiple trap + * instructions (like PowerPC or MIPS). We treat BREAK just like the more + * modern conditional trap instructions. + */ +bool is_trap_insn(uprobe_opcode_t *insn) +{ + union mips_instruction inst; + + inst.word = *insn; + + switch (inst.i_format.opcode) { + case spec_op: + switch (inst.r_format.func) { + case break_op: + case teq_op: + case tge_op: + case tgeu_op: + case tlt_op: + case tltu_op: + case tne_op: + return 1; + } + break; + + case bcond_op: /* Yes, really ... */ + switch (inst.u_format.rt) { + case teqi_op: + case tgei_op: + case tgeiu_op: + case tlti_op: + case tltiu_op: + case tnei_op: + return 1; + } + break; + } + + return 0; +} + +#define UPROBE_TRAP_NR ULONG_MAX + +/* + * arch_uprobe_pre_xol - prepare to execute out of line. + * @auprobe: the probepoint information. + * @regs: reflects the saved user state of current task. + */ +int arch_uprobe_pre_xol(struct arch_uprobe *aup, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + union mips_instruction insn; + + /* + * Now find the EPC where to resume after the breakpoint has been + * dealt with. This may require emulation of a branch. + */ + aup->resume_epc = regs->cp0_epc + 4; + if (insn_has_delay_slot((union mips_instruction) aup->insn[0])) { + unsigned long epc; + + epc = regs->cp0_epc; + __compute_return_epc_for_insn(regs, insn); + aup->resume_epc = regs->cp0_epc; + } + + utask->autask.saved_trap_nr = current->thread.trap_nr; + current->thread.trap_nr = UPROBE_TRAP_NR; + regs->cp0_epc = current->utask->xol_vaddr; + + return 0; +} + +int arch_uprobe_post_xol(struct arch_uprobe *aup, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + current->thread.trap_nr = utask->autask.saved_trap_nr; + regs->cp0_epc = aup->resume_epc; + + return 0; +} + +/* + * If xol insn itself traps and generates a signal(Say, + * SIGILL/SIGSEGV/etc), then detect the case where a singlestepped + * instruction jumps back to its own address. It is assumed that anything + * like do_page_fault/do_trap/etc sets thread.trap_nr != -1. + * + * arch_uprobe_pre_xol/arch_uprobe_post_xol save/restore thread.trap_nr, + * arch_uprobe_xol_was_trapped() simply checks that ->trap_nr is not equal to + * UPROBE_TRAP_NR == -1 set by arch_uprobe_pre_xol(). + */ +bool arch_uprobe_xol_was_trapped(struct task_struct *tsk) +{ + if (tsk->thread.trap_nr != UPROBE_TRAP_NR) + return true; + + return false; +} + +int arch_uprobe_exception_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + struct die_args *args = data; + struct pt_regs *regs = args->regs; + + /* regs == NULL is a kernel bug */ + if (WARN_ON(!regs)) + return NOTIFY_DONE; + + /* We are only interested in userspace traps */ + if (!user_mode(regs)) + return NOTIFY_DONE; + + switch (val) { + case DIE_BREAK: + if (uprobe_pre_sstep_notifier(regs)) + return NOTIFY_STOP; + break; + case DIE_UPROBE_XOL: + if (uprobe_post_sstep_notifier(regs)) + return NOTIFY_STOP; + default: + break; + } + + return 0; +} + +/* + * This function gets called when XOL instruction either gets trapped or + * the thread has a fatal signal. Reset the instruction pointer to its + * probed address for the potential restart or for post mortem analysis. + */ +void arch_uprobe_abort_xol(struct arch_uprobe *aup, + struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + instruction_pointer_set(regs, utask->vaddr); +} + +unsigned long arch_uretprobe_hijack_return_addr( + unsigned long trampoline_vaddr, struct pt_regs *regs) +{ + unsigned long ra; + + ra = regs->regs[31]; + + /* Replace the return address with the trampoline address */ + regs->regs[31] = ra; + + return ra; +} + +/** + * set_swbp - store breakpoint at a given address. + * @auprobe: arch specific probepoint information. + * @mm: the probed process address space. + * @vaddr: the virtual address to insert the opcode. + * + * For mm @mm, store the breakpoint instruction at @vaddr. + * Return 0 (success) or a negative errno. + * + * This version overrides the weak version in kernel/events/uprobes.c. + * It is required to handle MIPS16 and microMIPS. + */ +int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, + unsigned long vaddr) +{ + return uprobe_write_opcode(mm, vaddr, UPROBE_SWBP_INSN); +} + +/** + * set_orig_insn - Restore the original instruction. + * @mm: the probed process address space. + * @auprobe: arch specific probepoint information. + * @vaddr: the virtual address to insert the opcode. + * + * For mm @mm, restore the original opcode (opcode) at @vaddr. + * Return 0 (success) or a negative errno. + * + * This overrides the weak version in kernel/events/uprobes.c. + */ +int set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, + unsigned long vaddr) +{ + return uprobe_write_opcode(mm, vaddr, + *(uprobe_opcode_t *)&auprobe->orig_inst[0].word); +} + +void __weak arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, + void *src, unsigned long len) +{ + void *kaddr; + + /* Initialize the slot */ + kaddr = kmap_atomic(page); + memcpy(kaddr + (vaddr & ~PAGE_MASK), src, len); + kunmap_atomic(kaddr); + + /* + * The MIPS version of flush_icache_range will operate safely on + * user space addresses and more importantly, it doesn't require a + * VMA argument. + */ + flush_icache_range(vaddr, vaddr + len); +} + +/** + * uprobe_get_swbp_addr - compute address of swbp given post-swbp regs + * @regs: Reflects the saved state of the task after it has hit a breakpoint + * instruction. + * Return the address of the breakpoint instruction. + * + * This overrides the weak version in kernel/events/uprobes.c. + */ +unsigned long uprobe_get_swbp_addr(struct pt_regs *regs) +{ + return instruction_pointer(regs); +} + +/* + * See if the instruction can be emulated. + * Returns true if instruction was emulated, false otherwise. + * + * For now we always emulate so this function just returns 0. + */ +bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + return 0; +} diff --git a/arch/mips/kernel/vpe.c b/arch/mips/kernel/vpe.c index 11da314565cc..9067b651c7a2 100644 --- a/arch/mips/kernel/vpe.c +++ b/arch/mips/kernel/vpe.c @@ -817,6 +817,7 @@ static int vpe_open(struct inode *inode, struct file *filp) static int vpe_release(struct inode *inode, struct file *filp) { +#if defined(CONFIG_MIPS_VPE_LOADER_MT) || defined(CONFIG_MIPS_VPE_LOADER_CMP) struct vpe *v; Elf_Ehdr *hdr; int ret = 0; @@ -827,7 +828,7 @@ static int vpe_release(struct inode *inode, struct file *filp) hdr = (Elf_Ehdr *) v->pbuffer; if (memcmp(hdr->e_ident, ELFMAG, SELFMAG) == 0) { - if ((vpe_elfload(v) >= 0) && vpe_run) { + if (vpe_elfload(v) >= 0) { vpe_run(v); } else { pr_warn("VPE loader: ELF load failed.\n"); @@ -850,6 +851,10 @@ static int vpe_release(struct inode *inode, struct file *filp) v->plen = 0; return ret; +#else + pr_warn("VPE loader: ELF load failed.\n"); + return -ENOEXEC; +#endif } static ssize_t vpe_write(struct file *file, const char __user *buffer, diff --git a/arch/mips/lasat/sysctl.c b/arch/mips/lasat/sysctl.c index a57959e648a6..c710d969938d 100644 --- a/arch/mips/lasat/sysctl.c +++ b/arch/mips/lasat/sysctl.c @@ -270,4 +270,4 @@ static int __init lasat_register_sysctl(void) return 0; } -__initcall(lasat_register_sysctl); +arch_initcall(lasat_register_sysctl); diff --git a/arch/mips/lib/dump_tlb.c b/arch/mips/lib/dump_tlb.c index 167f35634709..92a37319efbe 100644 --- a/arch/mips/lib/dump_tlb.c +++ b/arch/mips/lib/dump_tlb.c @@ -13,6 +13,33 @@ #include #include +void dump_tlb_regs(void) +{ + const int field = 2 * sizeof(unsigned long); + + pr_info("Index : %0x\n", read_c0_index()); + pr_info("PageMask : %0x\n", read_c0_pagemask()); + pr_info("EntryHi : %0*lx\n", field, read_c0_entryhi()); + pr_info("EntryLo0 : %0*lx\n", field, read_c0_entrylo0()); + pr_info("EntryLo1 : %0*lx\n", field, read_c0_entrylo1()); + pr_info("Wired : %0x\n", read_c0_wired()); + switch (current_cpu_type()) { + case CPU_R10000: + case CPU_R12000: + case CPU_R14000: + case CPU_R16000: + pr_info("FrameMask: %0x\n", read_c0_framemask()); + break; + } + if (cpu_has_small_pages || cpu_has_rixi || cpu_has_xpa) + pr_info("PageGrain: %0x\n", read_c0_pagegrain()); + if (cpu_has_htw) { + pr_info("PWField : %0*lx\n", field, read_c0_pwfield()); + pr_info("PWSize : %0*lx\n", field, read_c0_pwsize()); + pr_info("PWCtl : %0x\n", read_c0_pwctl()); + } +} + static inline const char *msk2str(unsigned int mask) { switch (mask) { @@ -87,7 +114,7 @@ static void dump_tlb(int first, int last) * leave only a single G bit set after a machine check exception * due to duplicate TLB entry. */ - if (!((entrylo0 | entrylo1) & MIPS_ENTRYLO_G) && + if (!((entrylo0 | entrylo1) & ENTRYLO_G) && (entryhi & 0xff) != asid) continue; @@ -96,8 +123,8 @@ static void dump_tlb(int first, int last) */ printk("Index: %2d pgmask=%s ", i, msk2str(pagemask)); - c0 = (entrylo0 & MIPS_ENTRYLO_C) >> MIPS_ENTRYLO_C_SHIFT; - c1 = (entrylo1 & MIPS_ENTRYLO_C) >> MIPS_ENTRYLO_C_SHIFT; + c0 = (entrylo0 & ENTRYLO_C) >> ENTRYLO_C_SHIFT; + c1 = (entrylo1 & ENTRYLO_C) >> ENTRYLO_C_SHIFT; printk("va=%0*lx asid=%02lx\n", vwidth, (entryhi & ~0x1fffUL), @@ -114,9 +141,9 @@ static void dump_tlb(int first, int last) (entrylo0 & MIPS_ENTRYLO_XI) ? 1 : 0); printk("pa=%0*llx c=%d d=%d v=%d g=%d] [", pwidth, pa, c0, - (entrylo0 & MIPS_ENTRYLO_D) ? 1 : 0, - (entrylo0 & MIPS_ENTRYLO_V) ? 1 : 0, - (entrylo0 & MIPS_ENTRYLO_G) ? 1 : 0); + (entrylo0 & ENTRYLO_D) ? 1 : 0, + (entrylo0 & ENTRYLO_V) ? 1 : 0, + (entrylo0 & ENTRYLO_G) ? 1 : 0); /* RI/XI are in awkward places, so mask them off separately */ pa = entrylo1 & ~(MIPS_ENTRYLO_RI | MIPS_ENTRYLO_XI); if (xpa) @@ -128,9 +155,9 @@ static void dump_tlb(int first, int last) (entrylo1 & MIPS_ENTRYLO_XI) ? 1 : 0); printk("pa=%0*llx c=%d d=%d v=%d g=%d]\n", pwidth, pa, c1, - (entrylo1 & MIPS_ENTRYLO_D) ? 1 : 0, - (entrylo1 & MIPS_ENTRYLO_V) ? 1 : 0, - (entrylo1 & MIPS_ENTRYLO_G) ? 1 : 0); + (entrylo1 & ENTRYLO_D) ? 1 : 0, + (entrylo1 & ENTRYLO_V) ? 1 : 0, + (entrylo1 & ENTRYLO_G) ? 1 : 0); } printk("\n"); diff --git a/arch/mips/lib/r3k_dump_tlb.c b/arch/mips/lib/r3k_dump_tlb.c index 8e0d3cff8ae4..cfcbb5218b59 100644 --- a/arch/mips/lib/r3k_dump_tlb.c +++ b/arch/mips/lib/r3k_dump_tlb.c @@ -14,6 +14,17 @@ #include #include +extern int r3k_have_wired_reg; + +void dump_tlb_regs(void) +{ + pr_info("Index : %0x\n", read_c0_index()); + pr_info("EntryHi : %0lx\n", read_c0_entryhi()); + pr_info("EntryLo : %0lx\n", read_c0_entrylo0()); + if (r3k_have_wired_reg) + pr_info("Wired : %0x\n", read_c0_wired()); +} + static void dump_tlb(int first, int last) { int i; diff --git a/arch/mips/loongson32/common/time.c b/arch/mips/loongson32/common/time.c index df0f850d6a5f..0996b025eeef 100644 --- a/arch/mips/loongson32/common/time.c +++ b/arch/mips/loongson32/common/time.c @@ -126,26 +126,34 @@ static irqreturn_t ls1x_clockevent_isr(int irq, void *devid) return IRQ_HANDLED; } -static void ls1x_clockevent_set_mode(enum clock_event_mode mode, - struct clock_event_device *cd) +static int ls1x_clockevent_set_state_periodic(struct clock_event_device *cd) { raw_spin_lock(&ls1x_timer_lock); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - ls1x_pwmtimer_set_period(ls1x_jiffies_per_tick); - ls1x_pwmtimer_restart(); - case CLOCK_EVT_MODE_RESUME: - __raw_writel(INT_EN | CNT_EN, timer_base + PWM_CTRL); - break; - case CLOCK_EVT_MODE_ONESHOT: - case CLOCK_EVT_MODE_SHUTDOWN: - __raw_writel(__raw_readl(timer_base + PWM_CTRL) & ~CNT_EN, - timer_base + PWM_CTRL); - break; - default: - break; - } + ls1x_pwmtimer_set_period(ls1x_jiffies_per_tick); + ls1x_pwmtimer_restart(); + __raw_writel(INT_EN | CNT_EN, timer_base + PWM_CTRL); raw_spin_unlock(&ls1x_timer_lock); + + return 0; +} + +static int ls1x_clockevent_tick_resume(struct clock_event_device *cd) +{ + raw_spin_lock(&ls1x_timer_lock); + __raw_writel(INT_EN | CNT_EN, timer_base + PWM_CTRL); + raw_spin_unlock(&ls1x_timer_lock); + + return 0; +} + +static int ls1x_clockevent_set_state_shutdown(struct clock_event_device *cd) +{ + raw_spin_lock(&ls1x_timer_lock); + __raw_writel(__raw_readl(timer_base + PWM_CTRL) & ~CNT_EN, + timer_base + PWM_CTRL); + raw_spin_unlock(&ls1x_timer_lock); + + return 0; } static int ls1x_clockevent_set_next(unsigned long evt, @@ -160,12 +168,15 @@ static int ls1x_clockevent_set_next(unsigned long evt, } static struct clock_event_device ls1x_clockevent = { - .name = "ls1x-pwmtimer", - .features = CLOCK_EVT_FEAT_PERIODIC, - .rating = 300, - .irq = LS1X_TIMER_IRQ, - .set_next_event = ls1x_clockevent_set_next, - .set_mode = ls1x_clockevent_set_mode, + .name = "ls1x-pwmtimer", + .features = CLOCK_EVT_FEAT_PERIODIC, + .rating = 300, + .irq = LS1X_TIMER_IRQ, + .set_next_event = ls1x_clockevent_set_next, + .set_state_shutdown = ls1x_clockevent_set_state_shutdown, + .set_state_periodic = ls1x_clockevent_set_state_periodic, + .set_state_oneshot = ls1x_clockevent_set_state_shutdown, + .tick_resume = ls1x_clockevent_tick_resume, }; static struct irqaction ls1x_pwmtimer_irqaction = { diff --git a/arch/mips/loongson64/common/cs5536/cs5536_mfgpt.c b/arch/mips/loongson64/common/cs5536/cs5536_mfgpt.c index 875037063a80..da77d412514c 100644 --- a/arch/mips/loongson64/common/cs5536/cs5536_mfgpt.c +++ b/arch/mips/loongson64/common/cs5536/cs5536_mfgpt.c @@ -51,40 +51,36 @@ void enable_mfgpt0_counter(void) } EXPORT_SYMBOL(enable_mfgpt0_counter); -static void init_mfgpt_timer(enum clock_event_mode mode, - struct clock_event_device *evt) +static int mfgpt_timer_set_periodic(struct clock_event_device *evt) { raw_spin_lock(&mfgpt_lock); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - outw(COMPARE, MFGPT0_CMP2); /* set comparator2 */ - outw(0, MFGPT0_CNT); /* set counter to 0 */ - enable_mfgpt0_counter(); - break; + outw(COMPARE, MFGPT0_CMP2); /* set comparator2 */ + outw(0, MFGPT0_CNT); /* set counter to 0 */ + enable_mfgpt0_counter(); - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - if (evt->mode == CLOCK_EVT_MODE_PERIODIC || - evt->mode == CLOCK_EVT_MODE_ONESHOT) - disable_mfgpt0_counter(); - break; - - case CLOCK_EVT_MODE_ONESHOT: - /* The oneshot mode have very high deviation, Not use it! */ - break; - - case CLOCK_EVT_MODE_RESUME: - /* Nothing to do here */ - break; - } raw_spin_unlock(&mfgpt_lock); + return 0; +} + +static int mfgpt_timer_shutdown(struct clock_event_device *evt) +{ + if (clockevent_state_periodic(evt) || clockevent_state_oneshot(evt)) { + raw_spin_lock(&mfgpt_lock); + disable_mfgpt0_counter(); + raw_spin_unlock(&mfgpt_lock); + } + + return 0; } static struct clock_event_device mfgpt_clockevent = { .name = "mfgpt", .features = CLOCK_EVT_FEAT_PERIODIC, - .set_mode = init_mfgpt_timer, + + /* The oneshot mode have very high deviation, don't use it! */ + .set_state_shutdown = mfgpt_timer_shutdown, + .set_state_periodic = mfgpt_timer_set_periodic, .irq = CS5536_MFGPT_INTR, }; diff --git a/arch/mips/loongson64/common/dma-swiotlb.c b/arch/mips/loongson64/common/dma-swiotlb.c index 2c6b989c1bc4..4ffa6fc81c8f 100644 --- a/arch/mips/loongson64/common/dma-swiotlb.c +++ b/arch/mips/loongson64/common/dma-swiotlb.c @@ -14,9 +14,6 @@ static void *loongson_dma_alloc_coherent(struct device *dev, size_t size, { void *ret; - if (dma_alloc_from_coherent(dev, size, dma_handle, &ret)) - return ret; - /* ignore region specifiers */ gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM); @@ -46,11 +43,6 @@ static void *loongson_dma_alloc_coherent(struct device *dev, size_t size, static void loongson_dma_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, struct dma_attrs *attrs) { - int order = get_order(size); - - if (dma_release_from_coherent(dev, order, vaddr)) - return; - swiotlb_free_coherent(dev, size, vaddr, dma_handle); } @@ -93,6 +85,9 @@ static void loongson_dma_sync_sg_for_device(struct device *dev, static int loongson_dma_set_mask(struct device *dev, u64 mask) { + if (!dev->dma_mask || !dma_supported(dev, mask)) + return -EIO; + if (mask > DMA_BIT_MASK(loongson_sysconf.dma_mask_bits)) { *dev->dma_mask = DMA_BIT_MASK(loongson_sysconf.dma_mask_bits); return -EIO; diff --git a/arch/mips/loongson64/loongson-3/hpet.c b/arch/mips/loongson64/loongson-3/hpet.c index 5c21cd3bd339..bf9f1a77f0e5 100644 --- a/arch/mips/loongson64/loongson-3/hpet.c +++ b/arch/mips/loongson64/loongson-3/hpet.c @@ -78,55 +78,77 @@ static void hpet_enable_legacy_int(void) /* Do nothing on Loongson-3 */ } -static void hpet_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int hpet_set_state_periodic(struct clock_event_device *evt) { - int cfg = 0; + int cfg; spin_lock(&hpet_lock); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - pr_info("set clock event to periodic mode!\n"); - /* stop counter */ - hpet_stop_counter(); - /* enables the timer0 to generate a periodic interrupt */ - cfg = hpet_read(HPET_T0_CFG); - cfg &= ~HPET_TN_LEVEL; - cfg |= HPET_TN_ENABLE | HPET_TN_PERIODIC | - HPET_TN_SETVAL | HPET_TN_32BIT; - hpet_write(HPET_T0_CFG, cfg); + pr_info("set clock event to periodic mode!\n"); + /* stop counter */ + hpet_stop_counter(); - /* set the comparator */ - hpet_write(HPET_T0_CMP, HPET_COMPARE_VAL); - udelay(1); - hpet_write(HPET_T0_CMP, HPET_COMPARE_VAL); + /* enables the timer0 to generate a periodic interrupt */ + cfg = hpet_read(HPET_T0_CFG); + cfg &= ~HPET_TN_LEVEL; + cfg |= HPET_TN_ENABLE | HPET_TN_PERIODIC | HPET_TN_SETVAL | + HPET_TN_32BIT; + hpet_write(HPET_T0_CFG, cfg); + + /* set the comparator */ + hpet_write(HPET_T0_CMP, HPET_COMPARE_VAL); + udelay(1); + hpet_write(HPET_T0_CMP, HPET_COMPARE_VAL); + + /* start counter */ + hpet_start_counter(); - /* start counter */ - hpet_start_counter(); - break; - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - cfg = hpet_read(HPET_T0_CFG); - cfg &= ~HPET_TN_ENABLE; - hpet_write(HPET_T0_CFG, cfg); - break; - case CLOCK_EVT_MODE_ONESHOT: - pr_info("set clock event to one shot mode!\n"); - cfg = hpet_read(HPET_T0_CFG); - /* set timer0 type - * 1 : periodic interrupt - * 0 : non-periodic(oneshot) interrupt - */ - cfg &= ~HPET_TN_PERIODIC; - cfg |= HPET_TN_ENABLE | HPET_TN_32BIT; - hpet_write(HPET_T0_CFG, cfg); - break; - case CLOCK_EVT_MODE_RESUME: - hpet_enable_legacy_int(); - break; - } spin_unlock(&hpet_lock); + return 0; +} + +static int hpet_set_state_shutdown(struct clock_event_device *evt) +{ + int cfg; + + spin_lock(&hpet_lock); + + cfg = hpet_read(HPET_T0_CFG); + cfg &= ~HPET_TN_ENABLE; + hpet_write(HPET_T0_CFG, cfg); + + spin_unlock(&hpet_lock); + return 0; +} + +static int hpet_set_state_oneshot(struct clock_event_device *evt) +{ + int cfg; + + spin_lock(&hpet_lock); + + pr_info("set clock event to one shot mode!\n"); + cfg = hpet_read(HPET_T0_CFG); + /* + * set timer0 type + * 1 : periodic interrupt + * 0 : non-periodic(oneshot) interrupt + */ + cfg &= ~HPET_TN_PERIODIC; + cfg |= HPET_TN_ENABLE | HPET_TN_32BIT; + hpet_write(HPET_T0_CFG, cfg); + + spin_unlock(&hpet_lock); + return 0; +} + +static int hpet_tick_resume(struct clock_event_device *evt) +{ + spin_lock(&hpet_lock); + hpet_enable_legacy_int(); + spin_unlock(&hpet_lock); + + return 0; } static int hpet_next_event(unsigned long delta, @@ -206,7 +228,10 @@ void __init setup_hpet_timer(void) cd->name = "hpet"; cd->rating = 320; cd->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; - cd->set_mode = hpet_set_mode; + cd->set_state_shutdown = hpet_set_state_shutdown; + cd->set_state_periodic = hpet_set_state_periodic; + cd->set_state_oneshot = hpet_set_state_oneshot; + cd->tick_resume = hpet_tick_resume; cd->set_next_event = hpet_next_event; cd->irq = HPET_T0_IRQ; cd->cpumask = cpumask_of(cpu); diff --git a/arch/mips/math-emu/Makefile b/arch/mips/math-emu/Makefile index 2e5f96275c38..a19641d3ac23 100644 --- a/arch/mips/math-emu/Makefile +++ b/arch/mips/math-emu/Makefile @@ -4,9 +4,9 @@ obj-y += cp1emu.o ieee754dp.o ieee754sp.o ieee754.o \ dp_div.o dp_mul.o dp_sub.o dp_add.o dp_fsp.o dp_cmp.o dp_simple.o \ - dp_tint.o dp_fint.o \ + dp_tint.o dp_fint.o dp_maddf.o dp_msubf.o dp_2008class.o dp_fmin.o dp_fmax.o \ sp_div.o sp_mul.o sp_sub.o sp_add.o sp_fdp.o sp_cmp.o sp_simple.o \ - sp_tint.o sp_fint.o \ + sp_tint.o sp_fint.o sp_maddf.o sp_msubf.o sp_2008class.o sp_fmin.o sp_fmax.o \ dsemul.o lib-y += ieee754d.o \ diff --git a/arch/mips/math-emu/cp1emu.c b/arch/mips/math-emu/cp1emu.c index 712f17a2ecf2..32f0e19a0d7f 100644 --- a/arch/mips/math-emu/cp1emu.c +++ b/arch/mips/math-emu/cp1emu.c @@ -1137,7 +1137,7 @@ static int cop1Emulate(struct pt_regs *xcp, struct mips_fpu_struct *ctx, break; case mfhc_op: - if (!cpu_has_mips_r2) + if (!cpu_has_mips_r2_r6) goto sigill; /* copregister rd -> gpr[rt] */ @@ -1148,7 +1148,7 @@ static int cop1Emulate(struct pt_regs *xcp, struct mips_fpu_struct *ctx, break; case mthc_op: - if (!cpu_has_mips_r2) + if (!cpu_has_mips_r2_r6) goto sigill; /* copregister rd <- gpr[rt] */ @@ -1181,6 +1181,24 @@ static int cop1Emulate(struct pt_regs *xcp, struct mips_fpu_struct *ctx, } break; + case bc1eqz_op: + case bc1nez_op: + if (!cpu_has_mips_r6 || delay_slot(xcp)) + return SIGILL; + + cond = likely = 0; + switch (MIPSInst_RS(ir)) { + case bc1eqz_op: + if (get_fpr32(¤t->thread.fpu.fpr[MIPSInst_RT(ir)], 0) & 0x1) + cond = 1; + break; + case bc1nez_op: + if (!(get_fpr32(¤t->thread.fpu.fpr[MIPSInst_RT(ir)], 0) & 0x1)) + cond = 1; + break; + } + goto branch_common; + case bc_op: if (delay_slot(xcp)) return SIGILL; @@ -1207,7 +1225,7 @@ static int cop1Emulate(struct pt_regs *xcp, struct mips_fpu_struct *ctx, case bct_op: break; } - +branch_common: set_delay_slot(xcp); if (cond) { /* @@ -1376,6 +1394,14 @@ static const unsigned char cmptab[8] = { IEEE754_CLT | IEEE754_CEQ | IEEE754_CUN, /* cmp_ule (sig) cmp_ngt */ }; +static const unsigned char negative_cmptab[8] = { + 0, /* Reserved */ + IEEE754_CLT | IEEE754_CGT | IEEE754_CEQ, + IEEE754_CLT | IEEE754_CGT | IEEE754_CUN, + IEEE754_CLT | IEEE754_CGT, + /* Reserved */ +}; + /* * Additional MIPS4 instructions @@ -1717,6 +1743,126 @@ static int fpu_emu(struct pt_regs *xcp, struct mips_fpu_struct *ctx, SPFROMREG(rv.s, MIPSInst_FS(ir)); break; + case fseleqz_op: + if (!cpu_has_mips_r6) + return SIGILL; + + SPFROMREG(rv.s, MIPSInst_FT(ir)); + if (rv.w & 0x1) + rv.w = 0; + else + SPFROMREG(rv.s, MIPSInst_FS(ir)); + break; + + case fselnez_op: + if (!cpu_has_mips_r6) + return SIGILL; + + SPFROMREG(rv.s, MIPSInst_FT(ir)); + if (rv.w & 0x1) + SPFROMREG(rv.s, MIPSInst_FS(ir)); + else + rv.w = 0; + break; + + case fmaddf_op: { + union ieee754sp ft, fs, fd; + + if (!cpu_has_mips_r6) + return SIGILL; + + SPFROMREG(ft, MIPSInst_FT(ir)); + SPFROMREG(fs, MIPSInst_FS(ir)); + SPFROMREG(fd, MIPSInst_FD(ir)); + rv.s = ieee754sp_maddf(fd, fs, ft); + break; + } + + case fmsubf_op: { + union ieee754sp ft, fs, fd; + + if (!cpu_has_mips_r6) + return SIGILL; + + SPFROMREG(ft, MIPSInst_FT(ir)); + SPFROMREG(fs, MIPSInst_FS(ir)); + SPFROMREG(fd, MIPSInst_FD(ir)); + rv.s = ieee754sp_msubf(fd, fs, ft); + break; + } + + case frint_op: { + union ieee754sp fs; + + if (!cpu_has_mips_r6) + return SIGILL; + + SPFROMREG(fs, MIPSInst_FS(ir)); + rv.l = ieee754sp_tlong(fs); + rv.s = ieee754sp_flong(rv.l); + goto copcsr; + } + + case fclass_op: { + union ieee754sp fs; + + if (!cpu_has_mips_r6) + return SIGILL; + + SPFROMREG(fs, MIPSInst_FS(ir)); + rv.w = ieee754sp_2008class(fs); + rfmt = w_fmt; + break; + } + + case fmin_op: { + union ieee754sp fs, ft; + + if (!cpu_has_mips_r6) + return SIGILL; + + SPFROMREG(ft, MIPSInst_FT(ir)); + SPFROMREG(fs, MIPSInst_FS(ir)); + rv.s = ieee754sp_fmin(fs, ft); + break; + } + + case fmina_op: { + union ieee754sp fs, ft; + + if (!cpu_has_mips_r6) + return SIGILL; + + SPFROMREG(ft, MIPSInst_FT(ir)); + SPFROMREG(fs, MIPSInst_FS(ir)); + rv.s = ieee754sp_fmina(fs, ft); + break; + } + + case fmax_op: { + union ieee754sp fs, ft; + + if (!cpu_has_mips_r6) + return SIGILL; + + SPFROMREG(ft, MIPSInst_FT(ir)); + SPFROMREG(fs, MIPSInst_FS(ir)); + rv.s = ieee754sp_fmax(fs, ft); + break; + } + + case fmaxa_op: { + union ieee754sp fs, ft; + + if (!cpu_has_mips_r6) + return SIGILL; + + SPFROMREG(ft, MIPSInst_FT(ir)); + SPFROMREG(fs, MIPSInst_FS(ir)); + rv.s = ieee754sp_fmaxa(fs, ft); + break; + } + case fabs_op: handler.u = ieee754sp_abs; goto scopuop; @@ -1820,7 +1966,7 @@ static int fpu_emu(struct pt_regs *xcp, struct mips_fpu_struct *ctx, goto copcsr; default: - if (MIPSInst_FUNC(ir) >= fcmp_op) { + if (!NO_R6EMU && MIPSInst_FUNC(ir) >= fcmp_op) { unsigned cmpop = MIPSInst_FUNC(ir) - fcmp_op; union ieee754sp fs, ft; @@ -1914,6 +2060,127 @@ static int fpu_emu(struct pt_regs *xcp, struct mips_fpu_struct *ctx, return 0; DPFROMREG(rv.d, MIPSInst_FS(ir)); break; + + case fseleqz_op: + if (!cpu_has_mips_r6) + return SIGILL; + + DPFROMREG(rv.d, MIPSInst_FT(ir)); + if (rv.l & 0x1) + rv.l = 0; + else + DPFROMREG(rv.d, MIPSInst_FS(ir)); + break; + + case fselnez_op: + if (!cpu_has_mips_r6) + return SIGILL; + + DPFROMREG(rv.d, MIPSInst_FT(ir)); + if (rv.l & 0x1) + DPFROMREG(rv.d, MIPSInst_FS(ir)); + else + rv.l = 0; + break; + + case fmaddf_op: { + union ieee754dp ft, fs, fd; + + if (!cpu_has_mips_r6) + return SIGILL; + + DPFROMREG(ft, MIPSInst_FT(ir)); + DPFROMREG(fs, MIPSInst_FS(ir)); + DPFROMREG(fd, MIPSInst_FD(ir)); + rv.d = ieee754dp_maddf(fd, fs, ft); + break; + } + + case fmsubf_op: { + union ieee754dp ft, fs, fd; + + if (!cpu_has_mips_r6) + return SIGILL; + + DPFROMREG(ft, MIPSInst_FT(ir)); + DPFROMREG(fs, MIPSInst_FS(ir)); + DPFROMREG(fd, MIPSInst_FD(ir)); + rv.d = ieee754dp_msubf(fd, fs, ft); + break; + } + + case frint_op: { + union ieee754dp fs; + + if (!cpu_has_mips_r6) + return SIGILL; + + DPFROMREG(fs, MIPSInst_FS(ir)); + rv.l = ieee754dp_tlong(fs); + rv.d = ieee754dp_flong(rv.l); + goto copcsr; + } + + case fclass_op: { + union ieee754dp fs; + + if (!cpu_has_mips_r6) + return SIGILL; + + DPFROMREG(fs, MIPSInst_FS(ir)); + rv.w = ieee754dp_2008class(fs); + rfmt = w_fmt; + break; + } + + case fmin_op: { + union ieee754dp fs, ft; + + if (!cpu_has_mips_r6) + return SIGILL; + + DPFROMREG(ft, MIPSInst_FT(ir)); + DPFROMREG(fs, MIPSInst_FS(ir)); + rv.d = ieee754dp_fmin(fs, ft); + break; + } + + case fmina_op: { + union ieee754dp fs, ft; + + if (!cpu_has_mips_r6) + return SIGILL; + + DPFROMREG(ft, MIPSInst_FT(ir)); + DPFROMREG(fs, MIPSInst_FS(ir)); + rv.d = ieee754dp_fmina(fs, ft); + break; + } + + case fmax_op: { + union ieee754dp fs, ft; + + if (!cpu_has_mips_r6) + return SIGILL; + + DPFROMREG(ft, MIPSInst_FT(ir)); + DPFROMREG(fs, MIPSInst_FS(ir)); + rv.d = ieee754dp_fmax(fs, ft); + break; + } + + case fmaxa_op: { + union ieee754dp fs, ft; + + if (!cpu_has_mips_r6) + return SIGILL; + + DPFROMREG(ft, MIPSInst_FT(ir)); + DPFROMREG(fs, MIPSInst_FS(ir)); + rv.d = ieee754dp_fmaxa(fs, ft); + break; + } + case fabs_op: handler.u = ieee754dp_abs; goto dcopuop; @@ -1997,7 +2264,7 @@ static int fpu_emu(struct pt_regs *xcp, struct mips_fpu_struct *ctx, goto copcsr; default: - if (MIPSInst_FUNC(ir) >= fcmp_op) { + if (!NO_R6EMU && MIPSInst_FUNC(ir) >= fcmp_op) { unsigned cmpop = MIPSInst_FUNC(ir) - fcmp_op; union ieee754dp fs, ft; @@ -2021,8 +2288,11 @@ static int fpu_emu(struct pt_regs *xcp, struct mips_fpu_struct *ctx, break; } break; + } + + case w_fmt: { + union ieee754dp fs; - case w_fmt: switch (MIPSInst_FUNC(ir)) { case fcvts_op: /* convert word to single precision real */ @@ -2036,10 +2306,65 @@ static int fpu_emu(struct pt_regs *xcp, struct mips_fpu_struct *ctx, rv.d = ieee754dp_fint(fs.bits); rfmt = d_fmt; goto copcsr; - default: - return SIGILL; + default: { + /* Emulating the new CMP.condn.fmt R6 instruction */ +#define CMPOP_MASK 0x7 +#define SIGN_BIT (0x1 << 3) +#define PREDICATE_BIT (0x1 << 4) + + int cmpop = MIPSInst_FUNC(ir) & CMPOP_MASK; + int sig = MIPSInst_FUNC(ir) & SIGN_BIT; + union ieee754sp fs, ft; + + /* This is an R6 only instruction */ + if (!cpu_has_mips_r6 || + (MIPSInst_FUNC(ir) & 0x20)) + return SIGILL; + + /* fmt is w_fmt for single precision so fix it */ + rfmt = s_fmt; + /* default to false */ + rv.w = 0; + + /* CMP.condn.S */ + SPFROMREG(fs, MIPSInst_FS(ir)); + SPFROMREG(ft, MIPSInst_FT(ir)); + + /* positive predicates */ + if (!(MIPSInst_FUNC(ir) & PREDICATE_BIT)) { + if (ieee754sp_cmp(fs, ft, cmptab[cmpop], + sig)) + rv.w = -1; /* true, all 1s */ + if ((sig) && + ieee754_cxtest(IEEE754_INVALID_OPERATION)) + rcsr = FPU_CSR_INV_X | FPU_CSR_INV_S; + else + goto copcsr; + } else { + /* negative predicates */ + switch (cmpop) { + case 1: + case 2: + case 3: + if (ieee754sp_cmp(fs, ft, + negative_cmptab[cmpop], + sig)) + rv.w = -1; /* true, all 1s */ + if (sig && + ieee754_cxtest(IEEE754_INVALID_OPERATION)) + rcsr = FPU_CSR_INV_X | FPU_CSR_INV_S; + else + goto copcsr; + break; + default: + /* Reserved R6 ops */ + pr_err("Reserved MIPS R6 CMP.condn.S operation\n"); + return SIGILL; + } + } + break; + } } - break; } case l_fmt: @@ -2060,11 +2385,60 @@ static int fpu_emu(struct pt_regs *xcp, struct mips_fpu_struct *ctx, rv.d = ieee754dp_flong(bits); rfmt = d_fmt; goto copcsr; - default: - return SIGILL; - } - break; + default: { + /* Emulating the new CMP.condn.fmt R6 instruction */ + int cmpop = MIPSInst_FUNC(ir) & CMPOP_MASK; + int sig = MIPSInst_FUNC(ir) & SIGN_BIT; + union ieee754dp fs, ft; + if (!cpu_has_mips_r6 || + (MIPSInst_FUNC(ir) & 0x20)) + return SIGILL; + + /* fmt is l_fmt for double precision so fix it */ + rfmt = d_fmt; + /* default to false */ + rv.l = 0; + + /* CMP.condn.D */ + DPFROMREG(fs, MIPSInst_FS(ir)); + DPFROMREG(ft, MIPSInst_FT(ir)); + + /* positive predicates */ + if (!(MIPSInst_FUNC(ir) & PREDICATE_BIT)) { + if (ieee754dp_cmp(fs, ft, + cmptab[cmpop], sig)) + rv.l = -1LL; /* true, all 1s */ + if (sig && + ieee754_cxtest(IEEE754_INVALID_OPERATION)) + rcsr = FPU_CSR_INV_X | FPU_CSR_INV_S; + else + goto copcsr; + } else { + /* negative predicates */ + switch (cmpop) { + case 1: + case 2: + case 3: + if (ieee754dp_cmp(fs, ft, + negative_cmptab[cmpop], + sig)) + rv.l = -1LL; /* true, all 1s */ + if (sig && + ieee754_cxtest(IEEE754_INVALID_OPERATION)) + rcsr = FPU_CSR_INV_X | FPU_CSR_INV_S; + else + goto copcsr; + break; + default: + /* Reserved R6 ops */ + pr_err("Reserved MIPS R6 CMP.condn.D operation\n"); + return SIGILL; + } + } + break; + } + } default: return SIGILL; } diff --git a/arch/mips/math-emu/dp_2008class.c b/arch/mips/math-emu/dp_2008class.c new file mode 100644 index 000000000000..9dc39fc4835e --- /dev/null +++ b/arch/mips/math-emu/dp_2008class.c @@ -0,0 +1,55 @@ +/* + * IEEE754 floating point arithmetic + * double precision: CLASS.f + * FPR[fd] = class(FPR[fs]) + * + * MIPS floating point support + * Copyright (C) 2015 Imagination Technologies, Ltd. + * Author: Markos Chandras + * + * This program is free software; you can distribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; version 2 of the License. + */ + +#include "ieee754dp.h" + +int ieee754dp_2008class(union ieee754dp x) +{ + COMPXDP; + + EXPLODEXDP; + + /* + * 10 bit mask as follows: + * + * bit0 = SNAN + * bit1 = QNAN + * bit2 = -INF + * bit3 = -NORM + * bit4 = -DNORM + * bit5 = -ZERO + * bit6 = INF + * bit7 = NORM + * bit8 = DNORM + * bit9 = ZERO + */ + + switch(xc) { + case IEEE754_CLASS_SNAN: + return 0x01; + case IEEE754_CLASS_QNAN: + return 0x02; + case IEEE754_CLASS_INF: + return 0x04 << (xs ? 0 : 4); + case IEEE754_CLASS_NORM: + return 0x08 << (xs ? 0 : 4); + case IEEE754_CLASS_DNORM: + return 0x10 << (xs ? 0 : 4); + case IEEE754_CLASS_ZERO: + return 0x20 << (xs ? 0 : 4); + default: + pr_err("Unknown class: %d\n", xc); + return 0; + } +} diff --git a/arch/mips/math-emu/dp_fmax.c b/arch/mips/math-emu/dp_fmax.c new file mode 100644 index 000000000000..fd71b8daaaf2 --- /dev/null +++ b/arch/mips/math-emu/dp_fmax.c @@ -0,0 +1,213 @@ +/* + * IEEE754 floating point arithmetic + * double precision: MIN{,A}.f + * MIN : Scalar Floating-Point Minimum + * MINA: Scalar Floating-Point argument with Minimum Absolute Value + * + * MIN.D : FPR[fd] = minNum(FPR[fs],FPR[ft]) + * MINA.D: FPR[fd] = maxNumMag(FPR[fs],FPR[ft]) + * + * MIPS floating point support + * Copyright (C) 2015 Imagination Technologies, Ltd. + * Author: Markos Chandras + * + * This program is free software; you can distribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; version 2 of the License. + */ + +#include "ieee754dp.h" + +union ieee754dp ieee754dp_fmax(union ieee754dp x, union ieee754dp y) +{ + COMPXDP; + COMPYDP; + + EXPLODEXDP; + EXPLODEYDP; + + FLUSHXDP; + FLUSHYDP; + + ieee754_clearcx(); + + switch (CLPAIR(xc, yc)) { + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_SNAN): + return ieee754dp_nanxcpt(y); + + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_INF): + return ieee754dp_nanxcpt(x); + + /* numbers are preferred to NaNs */ + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_QNAN): + return x; + + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_INF): + return y; + + /* + * Infinity and zero handling + */ + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_ZERO): + return xs ? y : x; + + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_DNORM): + return ys ? x : y; + + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_ZERO): + if (xs == ys) + return x; + return ieee754dp_zero(1); + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): + DPDNORMX; + + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): + DPDNORMY; + break; + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_NORM): + DPDNORMX; + } + + /* Finally get to do some computation */ + + assert(xm & DP_HIDDEN_BIT); + assert(ym & DP_HIDDEN_BIT); + + /* Compare signs */ + if (xs > ys) + return y; + else if (xs < ys) + return x; + + /* Compare exponent */ + if (xe > ye) + return x; + else if (xe < ye) + return y; + + /* Compare mantissa */ + if (xm <= ym) + return y; + return x; +} + +union ieee754dp ieee754dp_fmaxa(union ieee754dp x, union ieee754dp y) +{ + COMPXDP; + COMPYDP; + + EXPLODEXDP; + EXPLODEYDP; + + FLUSHXDP; + FLUSHYDP; + + ieee754_clearcx(); + + switch (CLPAIR(xc, yc)) { + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_SNAN): + return ieee754dp_nanxcpt(y); + + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_INF): + return ieee754dp_nanxcpt(x); + + /* numbers are preferred to NaNs */ + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_QNAN): + return x; + + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_INF): + return y; + + /* + * Infinity and zero handling + */ + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_ZERO): + return x; + + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_DNORM): + return y; + + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_ZERO): + if (xs == ys) + return x; + return ieee754dp_zero(1); + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): + DPDNORMX; + + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): + DPDNORMY; + break; + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_NORM): + DPDNORMX; + } + + /* Finally get to do some computation */ + + assert(xm & DP_HIDDEN_BIT); + assert(ym & DP_HIDDEN_BIT); + + /* Compare exponent */ + if (xe > ye) + return x; + else if (xe < ye) + return y; + + /* Compare mantissa */ + if (xm <= ym) + return y; + return x; +} diff --git a/arch/mips/math-emu/dp_fmin.c b/arch/mips/math-emu/dp_fmin.c new file mode 100644 index 000000000000..c1072b0dfb95 --- /dev/null +++ b/arch/mips/math-emu/dp_fmin.c @@ -0,0 +1,213 @@ +/* + * IEEE754 floating point arithmetic + * double precision: MIN{,A}.f + * MIN : Scalar Floating-Point Minimum + * MINA: Scalar Floating-Point argument with Minimum Absolute Value + * + * MIN.D : FPR[fd] = minNum(FPR[fs],FPR[ft]) + * MINA.D: FPR[fd] = maxNumMag(FPR[fs],FPR[ft]) + * + * MIPS floating point support + * Copyright (C) 2015 Imagination Technologies, Ltd. + * Author: Markos Chandras + * + * This program is free software; you can distribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; version 2 of the License. + */ + +#include "ieee754dp.h" + +union ieee754dp ieee754dp_fmin(union ieee754dp x, union ieee754dp y) +{ + COMPXDP; + COMPYDP; + + EXPLODEXDP; + EXPLODEYDP; + + FLUSHXDP; + FLUSHYDP; + + ieee754_clearcx(); + + switch (CLPAIR(xc, yc)) { + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_SNAN): + return ieee754dp_nanxcpt(y); + + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_INF): + return ieee754dp_nanxcpt(x); + + /* numbers are preferred to NaNs */ + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_QNAN): + return x; + + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_INF): + return y; + + /* + * Infinity and zero handling + */ + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_ZERO): + return xs ? x : y; + + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_DNORM): + return ys ? y : x; + + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_ZERO): + if (xs == ys) + return x; + return ieee754dp_zero(1); + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): + DPDNORMX; + + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): + DPDNORMY; + break; + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_NORM): + DPDNORMX; + } + + /* Finally get to do some computation */ + + assert(xm & DP_HIDDEN_BIT); + assert(ym & DP_HIDDEN_BIT); + + /* Compare signs */ + if (xs > ys) + return x; + else if (xs < ys) + return y; + + /* Compare exponent */ + if (xe > ye) + return y; + else if (xe < ye) + return x; + + /* Compare mantissa */ + if (xm <= ym) + return x; + return y; +} + +union ieee754dp ieee754dp_fmina(union ieee754dp x, union ieee754dp y) +{ + COMPXDP; + COMPYDP; + + EXPLODEXDP; + EXPLODEYDP; + + FLUSHXDP; + FLUSHYDP; + + ieee754_clearcx(); + + switch (CLPAIR(xc, yc)) { + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_SNAN): + return ieee754dp_nanxcpt(y); + + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_INF): + return ieee754dp_nanxcpt(x); + + /* numbers are preferred to NaNs */ + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_QNAN): + return x; + + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_INF): + return y; + + /* + * Infinity and zero handling + */ + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_ZERO): + return x; + + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_DNORM): + return y; + + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_ZERO): + if (xs == ys) + return x; + return ieee754dp_zero(1); + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): + DPDNORMX; + + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): + DPDNORMY; + break; + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_NORM): + DPDNORMX; + } + + /* Finally get to do some computation */ + + assert(xm & DP_HIDDEN_BIT); + assert(ym & DP_HIDDEN_BIT); + + /* Compare exponent */ + if (xe > ye) + return y; + else if (xe < ye) + return x; + + /* Compare mantissa */ + if (xm <= ym) + return x; + return y; +} diff --git a/arch/mips/math-emu/dp_maddf.c b/arch/mips/math-emu/dp_maddf.c new file mode 100644 index 000000000000..119eda9fa1ea --- /dev/null +++ b/arch/mips/math-emu/dp_maddf.c @@ -0,0 +1,265 @@ +/* + * IEEE754 floating point arithmetic + * double precision: MADDF.f (Fused Multiply Add) + * MADDF.fmt: FPR[fd] = FPR[fd] + (FPR[fs] x FPR[ft]) + * + * MIPS floating point support + * Copyright (C) 2015 Imagination Technologies, Ltd. + * Author: Markos Chandras + * + * This program is free software; you can distribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; version 2 of the License. + */ + +#include "ieee754dp.h" + +union ieee754dp ieee754dp_maddf(union ieee754dp z, union ieee754dp x, + union ieee754dp y) +{ + int re; + int rs; + u64 rm; + unsigned lxm; + unsigned hxm; + unsigned lym; + unsigned hym; + u64 lrm; + u64 hrm; + u64 t; + u64 at; + int s; + + COMPXDP; + COMPYDP; + + u64 zm; int ze; int zs __maybe_unused; int zc; + + EXPLODEXDP; + EXPLODEYDP; + EXPLODEDP(z, zc, zs, ze, zm) + + FLUSHXDP; + FLUSHYDP; + FLUSHDP(z, zc, zs, ze, zm); + + ieee754_clearcx(); + + switch (zc) { + case IEEE754_CLASS_SNAN: + ieee754_setcx(IEEE754_INVALID_OPERATION); + return ieee754dp_nanxcpt(z); + case IEEE754_CLASS_DNORM: + DPDNORMx(zm, ze); + /* QNAN is handled separately below */ + } + + switch (CLPAIR(xc, yc)) { + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_SNAN): + return ieee754dp_nanxcpt(y); + + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_INF): + return ieee754dp_nanxcpt(x); + + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_QNAN): + return y; + + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_INF): + return x; + + + /* + * Infinity handling + */ + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_INF): + if (zc == IEEE754_CLASS_QNAN) + return z; + ieee754_setcx(IEEE754_INVALID_OPERATION); + return ieee754dp_indef(); + + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_INF): + if (zc == IEEE754_CLASS_QNAN) + return z; + return ieee754dp_inf(xs ^ ys); + + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_ZERO): + if (zc == IEEE754_CLASS_INF) + return ieee754dp_inf(zs); + /* Multiplication is 0 so just return z */ + return z; + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): + DPDNORMX; + + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): + if (zc == IEEE754_CLASS_QNAN) + return z; + else if (zc == IEEE754_CLASS_INF) + return ieee754dp_inf(zs); + DPDNORMY; + break; + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_NORM): + if (zc == IEEE754_CLASS_QNAN) + return z; + else if (zc == IEEE754_CLASS_INF) + return ieee754dp_inf(zs); + DPDNORMX; + break; + + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_NORM): + if (zc == IEEE754_CLASS_QNAN) + return z; + else if (zc == IEEE754_CLASS_INF) + return ieee754dp_inf(zs); + /* fall through to real computations */ + } + + /* Finally get to do some computation */ + + /* + * Do the multiplication bit first + * + * rm = xm * ym, re = xe + ye basically + * + * At this point xm and ym should have been normalized. + */ + assert(xm & DP_HIDDEN_BIT); + assert(ym & DP_HIDDEN_BIT); + + re = xe + ye; + rs = xs ^ ys; + + /* shunt to top of word */ + xm <<= 64 - (DP_FBITS + 1); + ym <<= 64 - (DP_FBITS + 1); + + /* + * Multiply 32 bits xm, ym to give high 32 bits rm with stickness. + */ + + /* 32 * 32 => 64 */ +#define DPXMULT(x, y) ((u64)(x) * (u64)y) + + lxm = xm; + hxm = xm >> 32; + lym = ym; + hym = ym >> 32; + + lrm = DPXMULT(lxm, lym); + hrm = DPXMULT(hxm, hym); + + t = DPXMULT(lxm, hym); + + at = lrm + (t << 32); + hrm += at < lrm; + lrm = at; + + hrm = hrm + (t >> 32); + + t = DPXMULT(hxm, lym); + + at = lrm + (t << 32); + hrm += at < lrm; + lrm = at; + + hrm = hrm + (t >> 32); + + rm = hrm | (lrm != 0); + + /* + * Sticky shift down to normal rounding precision. + */ + if ((s64) rm < 0) { + rm = (rm >> (64 - (DP_FBITS + 1 + 3))) | + ((rm << (DP_FBITS + 1 + 3)) != 0); + re++; + } else { + rm = (rm >> (64 - (DP_FBITS + 1 + 3 + 1))) | + ((rm << (DP_FBITS + 1 + 3 + 1)) != 0); + } + assert(rm & (DP_HIDDEN_BIT << 3)); + + /* And now the addition */ + assert(zm & DP_HIDDEN_BIT); + + /* + * Provide guard,round and stick bit space. + */ + zm <<= 3; + + if (ze > re) { + /* + * Have to shift y fraction right to align. + */ + s = ze - re; + rm = XDPSRS(rm, s); + re += s; + } else if (re > ze) { + /* + * Have to shift x fraction right to align. + */ + s = re - ze; + zm = XDPSRS(zm, s); + ze += s; + } + assert(ze == re); + assert(ze <= DP_EMAX); + + if (zs == rs) { + /* + * Generate 28 bit result of adding two 27 bit numbers + * leaving result in xm, xs and xe. + */ + zm = zm + rm; + + if (zm >> (DP_FBITS + 1 + 3)) { /* carry out */ + zm = XDPSRS1(zm); + ze++; + } + } else { + if (zm >= rm) { + zm = zm - rm; + } else { + zm = rm - zm; + zs = rs; + } + if (zm == 0) + return ieee754dp_zero(ieee754_csr.rm == FPU_CSR_RD); + + /* + * Normalize to rounding precision. + */ + while ((zm >> (DP_FBITS + 3)) == 0) { + zm <<= 1; + ze--; + } + } + + return ieee754dp_format(zs, ze, zm); +} diff --git a/arch/mips/math-emu/dp_msubf.c b/arch/mips/math-emu/dp_msubf.c new file mode 100644 index 000000000000..12241262f856 --- /dev/null +++ b/arch/mips/math-emu/dp_msubf.c @@ -0,0 +1,269 @@ +/* + * IEEE754 floating point arithmetic + * double precision: MSUB.f (Fused Multiply Subtract) + * MSUBF.fmt: FPR[fd] = FPR[fd] - (FPR[fs] x FPR[ft]) + * + * MIPS floating point support + * Copyright (C) 2015 Imagination Technologies, Ltd. + * Author: Markos Chandras + * + * This program is free software; you can distribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; version 2 of the License. + */ + +#include "ieee754dp.h" + +union ieee754dp ieee754dp_msubf(union ieee754dp z, union ieee754dp x, + union ieee754dp y) +{ + int re; + int rs; + u64 rm; + unsigned lxm; + unsigned hxm; + unsigned lym; + unsigned hym; + u64 lrm; + u64 hrm; + u64 t; + u64 at; + int s; + + COMPXDP; + COMPYDP; + + u64 zm; int ze; int zs __maybe_unused; int zc; + + EXPLODEXDP; + EXPLODEYDP; + EXPLODEDP(z, zc, zs, ze, zm) + + FLUSHXDP; + FLUSHYDP; + FLUSHDP(z, zc, zs, ze, zm); + + ieee754_clearcx(); + + switch (zc) { + case IEEE754_CLASS_SNAN: + ieee754_setcx(IEEE754_INVALID_OPERATION); + return ieee754dp_nanxcpt(z); + case IEEE754_CLASS_DNORM: + DPDNORMx(zm, ze); + /* QNAN is handled separately below */ + } + + switch (CLPAIR(xc, yc)) { + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_SNAN): + return ieee754dp_nanxcpt(y); + + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_INF): + return ieee754dp_nanxcpt(x); + + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_QNAN): + return y; + + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_INF): + return x; + + + /* + * Infinity handling + */ + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_INF): + if (zc == IEEE754_CLASS_QNAN) + return z; + ieee754_setcx(IEEE754_INVALID_OPERATION); + return ieee754dp_indef(); + + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_INF): + if (zc == IEEE754_CLASS_QNAN) + return z; + return ieee754dp_inf(xs ^ ys); + + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_ZERO): + if (zc == IEEE754_CLASS_INF) + return ieee754dp_inf(zs); + /* Multiplication is 0 so just return z */ + return z; + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): + DPDNORMX; + + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): + if (zc == IEEE754_CLASS_QNAN) + return z; + else if (zc == IEEE754_CLASS_INF) + return ieee754dp_inf(zs); + DPDNORMY; + break; + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_NORM): + if (zc == IEEE754_CLASS_QNAN) + return z; + else if (zc == IEEE754_CLASS_INF) + return ieee754dp_inf(zs); + DPDNORMX; + break; + + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_NORM): + if (zc == IEEE754_CLASS_QNAN) + return z; + else if (zc == IEEE754_CLASS_INF) + return ieee754dp_inf(zs); + /* fall through to real computations */ + } + + /* Finally get to do some computation */ + + /* + * Do the multiplication bit first + * + * rm = xm * ym, re = xe + ye basically + * + * At this point xm and ym should have been normalized. + */ + assert(xm & DP_HIDDEN_BIT); + assert(ym & DP_HIDDEN_BIT); + + re = xe + ye; + rs = xs ^ ys; + + /* shunt to top of word */ + xm <<= 64 - (DP_FBITS + 1); + ym <<= 64 - (DP_FBITS + 1); + + /* + * Multiply 32 bits xm, ym to give high 32 bits rm with stickness. + */ + + /* 32 * 32 => 64 */ +#define DPXMULT(x, y) ((u64)(x) * (u64)y) + + lxm = xm; + hxm = xm >> 32; + lym = ym; + hym = ym >> 32; + + lrm = DPXMULT(lxm, lym); + hrm = DPXMULT(hxm, hym); + + t = DPXMULT(lxm, hym); + + at = lrm + (t << 32); + hrm += at < lrm; + lrm = at; + + hrm = hrm + (t >> 32); + + t = DPXMULT(hxm, lym); + + at = lrm + (t << 32); + hrm += at < lrm; + lrm = at; + + hrm = hrm + (t >> 32); + + rm = hrm | (lrm != 0); + + /* + * Sticky shift down to normal rounding precision. + */ + if ((s64) rm < 0) { + rm = (rm >> (64 - (DP_FBITS + 1 + 3))) | + ((rm << (DP_FBITS + 1 + 3)) != 0); + re++; + } else { + rm = (rm >> (64 - (DP_FBITS + 1 + 3 + 1))) | + ((rm << (DP_FBITS + 1 + 3 + 1)) != 0); + } + assert(rm & (DP_HIDDEN_BIT << 3)); + + /* And now the subtraction */ + + /* flip sign of r and handle as add */ + rs ^= 1; + + assert(zm & DP_HIDDEN_BIT); + + /* + * Provide guard,round and stick bit space. + */ + zm <<= 3; + + if (ze > re) { + /* + * Have to shift y fraction right to align. + */ + s = ze - re; + rm = XDPSRS(rm, s); + re += s; + } else if (re > ze) { + /* + * Have to shift x fraction right to align. + */ + s = re - ze; + zm = XDPSRS(zm, s); + ze += s; + } + assert(ze == re); + assert(ze <= DP_EMAX); + + if (zs == rs) { + /* + * Generate 28 bit result of adding two 27 bit numbers + * leaving result in xm, xs and xe. + */ + zm = zm + rm; + + if (zm >> (DP_FBITS + 1 + 3)) { /* carry out */ + zm = XDPSRS1(zm); + ze++; + } + } else { + if (zm >= rm) { + zm = zm - rm; + } else { + zm = rm - zm; + zs = rs; + } + if (zm == 0) + return ieee754dp_zero(ieee754_csr.rm == FPU_CSR_RD); + + /* + * Normalize to rounding precision. + */ + while ((zm >> (DP_FBITS + 3)) == 0) { + zm <<= 1; + ze--; + } + } + + return ieee754dp_format(zs, ze, zm); +} diff --git a/arch/mips/math-emu/dsemul.c b/arch/mips/math-emu/dsemul.c index e0b5cc27d78b..cbb36c14b155 100644 --- a/arch/mips/math-emu/dsemul.c +++ b/arch/mips/math-emu/dsemul.c @@ -33,7 +33,6 @@ struct emuframe { int mips_dsemul(struct pt_regs *regs, mips_instruction ir, unsigned long cpc) { - extern asmlinkage void handle_dsemulret(void); struct emuframe __user *fr; int err; diff --git a/arch/mips/math-emu/ieee754.h b/arch/mips/math-emu/ieee754.h index a5ca108ce467..df94720714c7 100644 --- a/arch/mips/math-emu/ieee754.h +++ b/arch/mips/math-emu/ieee754.h @@ -75,6 +75,16 @@ int ieee754sp_cmp(union ieee754sp x, union ieee754sp y, int cop, int sig); union ieee754sp ieee754sp_sqrt(union ieee754sp x); +union ieee754sp ieee754sp_maddf(union ieee754sp z, union ieee754sp x, + union ieee754sp y); +union ieee754sp ieee754sp_msubf(union ieee754sp z, union ieee754sp x, + union ieee754sp y); +int ieee754sp_2008class(union ieee754sp x); +union ieee754sp ieee754sp_fmin(union ieee754sp x, union ieee754sp y); +union ieee754sp ieee754sp_fmina(union ieee754sp x, union ieee754sp y); +union ieee754sp ieee754sp_fmax(union ieee754sp x, union ieee754sp y); +union ieee754sp ieee754sp_fmaxa(union ieee754sp x, union ieee754sp y); + /* * double precision (often aka double) */ @@ -99,6 +109,15 @@ int ieee754dp_cmp(union ieee754dp x, union ieee754dp y, int cop, int sig); union ieee754dp ieee754dp_sqrt(union ieee754dp x); +union ieee754dp ieee754dp_maddf(union ieee754dp z, union ieee754dp x, + union ieee754dp y); +union ieee754dp ieee754dp_msubf(union ieee754dp z, union ieee754dp x, + union ieee754dp y); +int ieee754dp_2008class(union ieee754dp x); +union ieee754dp ieee754dp_fmin(union ieee754dp x, union ieee754dp y); +union ieee754dp ieee754dp_fmina(union ieee754dp x, union ieee754dp y); +union ieee754dp ieee754dp_fmax(union ieee754dp x, union ieee754dp y); +union ieee754dp ieee754dp_fmaxa(union ieee754dp x, union ieee754dp y); /* 5 types of floating point number diff --git a/arch/mips/math-emu/ieee754int.h b/arch/mips/math-emu/ieee754int.h index 05389d5e3a93..6383e2c5c1ad 100644 --- a/arch/mips/math-emu/ieee754int.h +++ b/arch/mips/math-emu/ieee754int.h @@ -65,8 +65,8 @@ static inline int ieee754_class_nan(int xc) vc = IEEE754_CLASS_INF; \ else if (vm & SP_MBIT(SP_FBITS-1)) \ vc = IEEE754_CLASS_SNAN; \ - else \ - vc = IEEE754_CLASS_QNAN; \ + else \ + vc = IEEE754_CLASS_QNAN; \ } else if (ve == SP_EMIN-1+SP_EBIAS) { \ if (vm) { \ ve = SP_EMIN; \ @@ -105,8 +105,8 @@ static inline int ieee754_class_nan(int xc) if (vm) { \ ve = DP_EMIN; \ vc = IEEE754_CLASS_DNORM; \ - } else \ - vc = IEEE754_CLASS_ZERO; \ + } else \ + vc = IEEE754_CLASS_ZERO; \ } else { \ ve -= DP_EBIAS; \ vm |= DP_HIDDEN_BIT; \ diff --git a/arch/mips/math-emu/me-debugfs.c b/arch/mips/math-emu/me-debugfs.c index f308e0f05fc5..506a67a98cdf 100644 --- a/arch/mips/math-emu/me-debugfs.c +++ b/arch/mips/math-emu/me-debugfs.c @@ -65,4 +65,4 @@ do { \ return 0; } -__initcall(debugfs_fpuemu); +arch_initcall(debugfs_fpuemu); diff --git a/arch/mips/math-emu/sp_2008class.c b/arch/mips/math-emu/sp_2008class.c new file mode 100644 index 000000000000..ff62606a1465 --- /dev/null +++ b/arch/mips/math-emu/sp_2008class.c @@ -0,0 +1,55 @@ +/* + * IEEE754 floating point arithmetic + * single precision: CLASS.f + * FPR[fd] = class(FPR[fs]) + * + * MIPS floating point support + * Copyright (C) 2015 Imagination Technologies, Ltd. + * Author: Markos Chandras + * + * This program is free software; you can distribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; version 2 of the License. + */ + +#include "ieee754sp.h" + +int ieee754sp_2008class(union ieee754sp x) +{ + COMPXSP; + + EXPLODEXSP; + + /* + * 10 bit mask as follows: + * + * bit0 = SNAN + * bit1 = QNAN + * bit2 = -INF + * bit3 = -NORM + * bit4 = -DNORM + * bit5 = -ZERO + * bit6 = INF + * bit7 = NORM + * bit8 = DNORM + * bit9 = ZERO + */ + + switch(xc) { + case IEEE754_CLASS_SNAN: + return 0x01; + case IEEE754_CLASS_QNAN: + return 0x02; + case IEEE754_CLASS_INF: + return 0x04 << (xs ? 0 : 4); + case IEEE754_CLASS_NORM: + return 0x08 << (xs ? 0 : 4); + case IEEE754_CLASS_DNORM: + return 0x10 << (xs ? 0 : 4); + case IEEE754_CLASS_ZERO: + return 0x20 << (xs ? 0 : 4); + default: + pr_err("Unknown class: %d\n", xc); + return 0; + } +} diff --git a/arch/mips/math-emu/sp_fmax.c b/arch/mips/math-emu/sp_fmax.c new file mode 100644 index 000000000000..4d000844e48e --- /dev/null +++ b/arch/mips/math-emu/sp_fmax.c @@ -0,0 +1,213 @@ +/* + * IEEE754 floating point arithmetic + * single precision: MAX{,A}.f + * MAX : Scalar Floating-Point Maximum + * MAXA: Scalar Floating-Point argument with Maximum Absolute Value + * + * MAX.S : FPR[fd] = maxNum(FPR[fs],FPR[ft]) + * MAXA.S: FPR[fd] = maxNumMag(FPR[fs],FPR[ft]) + * + * MIPS floating point support + * Copyright (C) 2015 Imagination Technologies, Ltd. + * Author: Markos Chandras + * + * This program is free software; you can distribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; version 2 of the License. + */ + +#include "ieee754sp.h" + +union ieee754sp ieee754sp_fmax(union ieee754sp x, union ieee754sp y) +{ + COMPXSP; + COMPYSP; + + EXPLODEXSP; + EXPLODEYSP; + + FLUSHXSP; + FLUSHYSP; + + ieee754_clearcx(); + + switch (CLPAIR(xc, yc)) { + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_SNAN): + return ieee754sp_nanxcpt(y); + + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_INF): + return ieee754sp_nanxcpt(x); + + /* numbers are preferred to NaNs */ + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_QNAN): + return x; + + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_INF): + return y; + + /* + * Infinity and zero handling + */ + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_ZERO): + return xs ? y : x; + + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_DNORM): + return ys ? x : y; + + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_ZERO): + if (xs == ys) + return x; + return ieee754sp_zero(1); + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): + SPDNORMX; + + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): + SPDNORMY; + break; + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_NORM): + SPDNORMX; + } + + /* Finally get to do some computation */ + + assert(xm & SP_HIDDEN_BIT); + assert(ym & SP_HIDDEN_BIT); + + /* Compare signs */ + if (xs > ys) + return y; + else if (xs < ys) + return x; + + /* Compare exponent */ + if (xe > ye) + return x; + else if (xe < ye) + return y; + + /* Compare mantissa */ + if (xm <= ym) + return y; + return x; +} + +union ieee754sp ieee754sp_fmaxa(union ieee754sp x, union ieee754sp y) +{ + COMPXSP; + COMPYSP; + + EXPLODEXSP; + EXPLODEYSP; + + FLUSHXSP; + FLUSHYSP; + + ieee754_clearcx(); + + switch (CLPAIR(xc, yc)) { + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_SNAN): + return ieee754sp_nanxcpt(y); + + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_INF): + return ieee754sp_nanxcpt(x); + + /* numbers are preferred to NaNs */ + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_QNAN): + return x; + + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_INF): + return y; + + /* + * Infinity and zero handling + */ + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_ZERO): + return x; + + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_DNORM): + return y; + + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_ZERO): + if (xs == ys) + return x; + return ieee754sp_zero(1); + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): + SPDNORMX; + + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): + SPDNORMY; + break; + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_NORM): + SPDNORMX; + } + + /* Finally get to do some computation */ + + assert(xm & SP_HIDDEN_BIT); + assert(ym & SP_HIDDEN_BIT); + + /* Compare exponent */ + if (xe > ye) + return x; + else if (xe < ye) + return y; + + /* Compare mantissa */ + if (xm <= ym) + return y; + return x; +} diff --git a/arch/mips/math-emu/sp_fmin.c b/arch/mips/math-emu/sp_fmin.c new file mode 100644 index 000000000000..4eb1bb9e9dec --- /dev/null +++ b/arch/mips/math-emu/sp_fmin.c @@ -0,0 +1,213 @@ +/* + * IEEE754 floating point arithmetic + * single precision: MIN{,A}.f + * MIN : Scalar Floating-Point Minimum + * MINA: Scalar Floating-Point argument with Minimum Absolute Value + * + * MIN.S : FPR[fd] = minNum(FPR[fs],FPR[ft]) + * MINA.S: FPR[fd] = maxNumMag(FPR[fs],FPR[ft]) + * + * MIPS floating point support + * Copyright (C) 2015 Imagination Technologies, Ltd. + * Author: Markos Chandras + * + * This program is free software; you can distribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; version 2 of the License. + */ + +#include "ieee754sp.h" + +union ieee754sp ieee754sp_fmin(union ieee754sp x, union ieee754sp y) +{ + COMPXSP; + COMPYSP; + + EXPLODEXSP; + EXPLODEYSP; + + FLUSHXSP; + FLUSHYSP; + + ieee754_clearcx(); + + switch (CLPAIR(xc, yc)) { + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_SNAN): + return ieee754sp_nanxcpt(y); + + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_INF): + return ieee754sp_nanxcpt(x); + + /* numbers are preferred to NaNs */ + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_QNAN): + return x; + + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_INF): + return y; + + /* + * Infinity and zero handling + */ + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_ZERO): + return xs ? x : y; + + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_DNORM): + return ys ? y : x; + + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_ZERO): + if (xs == ys) + return x; + return ieee754sp_zero(1); + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): + SPDNORMX; + + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): + SPDNORMY; + break; + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_NORM): + SPDNORMX; + } + + /* Finally get to do some computation */ + + assert(xm & SP_HIDDEN_BIT); + assert(ym & SP_HIDDEN_BIT); + + /* Compare signs */ + if (xs > ys) + return x; + else if (xs < ys) + return y; + + /* Compare exponent */ + if (xe > ye) + return y; + else if (xe < ye) + return x; + + /* Compare mantissa */ + if (xm <= ym) + return x; + return y; +} + +union ieee754sp ieee754sp_fmina(union ieee754sp x, union ieee754sp y) +{ + COMPXSP; + COMPYSP; + + EXPLODEXSP; + EXPLODEYSP; + + FLUSHXSP; + FLUSHYSP; + + ieee754_clearcx(); + + switch (CLPAIR(xc, yc)) { + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_SNAN): + return ieee754sp_nanxcpt(y); + + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_INF): + return ieee754sp_nanxcpt(x); + + /* numbers are preferred to NaNs */ + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_QNAN): + return x; + + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_INF): + return y; + + /* + * Infinity and zero handling + */ + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_ZERO): + return x; + + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_DNORM): + return y; + + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_ZERO): + if (xs == ys) + return x; + return ieee754sp_zero(1); + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): + SPDNORMX; + + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): + SPDNORMY; + break; + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_NORM): + SPDNORMX; + } + + /* Finally get to do some computation */ + + assert(xm & SP_HIDDEN_BIT); + assert(ym & SP_HIDDEN_BIT); + + /* Compare exponent */ + if (xe > ye) + return y; + else if (xe < ye) + return x; + + /* Compare mantissa */ + if (xm <= ym) + return x; + return y; +} diff --git a/arch/mips/math-emu/sp_maddf.c b/arch/mips/math-emu/sp_maddf.c new file mode 100644 index 000000000000..dd1dd83e34eb --- /dev/null +++ b/arch/mips/math-emu/sp_maddf.c @@ -0,0 +1,255 @@ +/* + * IEEE754 floating point arithmetic + * single precision: MADDF.f (Fused Multiply Add) + * MADDF.fmt: FPR[fd] = FPR[fd] + (FPR[fs] x FPR[ft]) + * + * MIPS floating point support + * Copyright (C) 2015 Imagination Technologies, Ltd. + * Author: Markos Chandras + * + * This program is free software; you can distribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; version 2 of the License. + */ + +#include "ieee754sp.h" + +union ieee754sp ieee754sp_maddf(union ieee754sp z, union ieee754sp x, + union ieee754sp y) +{ + int re; + int rs; + unsigned rm; + unsigned short lxm; + unsigned short hxm; + unsigned short lym; + unsigned short hym; + unsigned lrm; + unsigned hrm; + unsigned t; + unsigned at; + int s; + + COMPXSP; + COMPYSP; + u32 zm; int ze; int zs __maybe_unused; int zc; + + EXPLODEXSP; + EXPLODEYSP; + EXPLODESP(z, zc, zs, ze, zm) + + FLUSHXSP; + FLUSHYSP; + FLUSHSP(z, zc, zs, ze, zm); + + ieee754_clearcx(); + + switch (zc) { + case IEEE754_CLASS_SNAN: + ieee754_setcx(IEEE754_INVALID_OPERATION); + return ieee754sp_nanxcpt(z); + case IEEE754_CLASS_DNORM: + SPDNORMx(zm, ze); + /* QNAN is handled separately below */ + } + + switch (CLPAIR(xc, yc)) { + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_SNAN): + return ieee754sp_nanxcpt(y); + + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_INF): + return ieee754sp_nanxcpt(x); + + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_QNAN): + return y; + + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_INF): + return x; + + /* + * Infinity handling + */ + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_INF): + if (zc == IEEE754_CLASS_QNAN) + return z; + ieee754_setcx(IEEE754_INVALID_OPERATION); + return ieee754sp_indef(); + + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_INF): + if (zc == IEEE754_CLASS_QNAN) + return z; + return ieee754sp_inf(xs ^ ys); + + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_ZERO): + if (zc == IEEE754_CLASS_INF) + return ieee754sp_inf(zs); + /* Multiplication is 0 so just return z */ + return z; + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): + SPDNORMX; + + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): + if (zc == IEEE754_CLASS_QNAN) + return z; + else if (zc == IEEE754_CLASS_INF) + return ieee754sp_inf(zs); + SPDNORMY; + break; + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_NORM): + if (zc == IEEE754_CLASS_QNAN) + return z; + else if (zc == IEEE754_CLASS_INF) + return ieee754sp_inf(zs); + SPDNORMX; + break; + + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_NORM): + if (zc == IEEE754_CLASS_QNAN) + return z; + else if (zc == IEEE754_CLASS_INF) + return ieee754sp_inf(zs); + /* fall through to real computations */ + } + + /* Finally get to do some computation */ + + /* + * Do the multiplication bit first + * + * rm = xm * ym, re = xe + ye basically + * + * At this point xm and ym should have been normalized. + */ + + /* rm = xm * ym, re = xe+ye basically */ + assert(xm & SP_HIDDEN_BIT); + assert(ym & SP_HIDDEN_BIT); + + re = xe + ye; + rs = xs ^ ys; + + /* shunt to top of word */ + xm <<= 32 - (SP_FBITS + 1); + ym <<= 32 - (SP_FBITS + 1); + + /* + * Multiply 32 bits xm, ym to give high 32 bits rm with stickness. + */ + lxm = xm & 0xffff; + hxm = xm >> 16; + lym = ym & 0xffff; + hym = ym >> 16; + + lrm = lxm * lym; /* 16 * 16 => 32 */ + hrm = hxm * hym; /* 16 * 16 => 32 */ + + t = lxm * hym; /* 16 * 16 => 32 */ + at = lrm + (t << 16); + hrm += at < lrm; + lrm = at; + hrm = hrm + (t >> 16); + + t = hxm * lym; /* 16 * 16 => 32 */ + at = lrm + (t << 16); + hrm += at < lrm; + lrm = at; + hrm = hrm + (t >> 16); + + rm = hrm | (lrm != 0); + + /* + * Sticky shift down to normal rounding precision. + */ + if ((int) rm < 0) { + rm = (rm >> (32 - (SP_FBITS + 1 + 3))) | + ((rm << (SP_FBITS + 1 + 3)) != 0); + re++; + } else { + rm = (rm >> (32 - (SP_FBITS + 1 + 3 + 1))) | + ((rm << (SP_FBITS + 1 + 3 + 1)) != 0); + } + assert(rm & (SP_HIDDEN_BIT << 3)); + + /* And now the addition */ + + assert(zm & SP_HIDDEN_BIT); + + /* + * Provide guard,round and stick bit space. + */ + zm <<= 3; + + if (ze > re) { + /* + * Have to shift y fraction right to align. + */ + s = ze - re; + SPXSRSYn(s); + } else if (re > ze) { + /* + * Have to shift x fraction right to align. + */ + s = re - ze; + SPXSRSYn(s); + } + assert(ze == re); + assert(ze <= SP_EMAX); + + if (zs == rs) { + /* + * Generate 28 bit result of adding two 27 bit numbers + * leaving result in zm, zs and ze. + */ + zm = zm + rm; + + if (zm >> (SP_FBITS + 1 + 3)) { /* carry out */ + SPXSRSX1(); + } + } else { + if (zm >= rm) { + zm = zm - rm; + } else { + zm = rm - zm; + zs = rs; + } + if (zm == 0) + return ieee754sp_zero(ieee754_csr.rm == FPU_CSR_RD); + + /* + * Normalize in extended single precision + */ + while ((zm >> (SP_MBITS + 3)) == 0) { + zm <<= 1; + ze--; + } + + } + return ieee754sp_format(zs, ze, zm); +} diff --git a/arch/mips/math-emu/sp_msubf.c b/arch/mips/math-emu/sp_msubf.c new file mode 100644 index 000000000000..81c38b980d69 --- /dev/null +++ b/arch/mips/math-emu/sp_msubf.c @@ -0,0 +1,258 @@ +/* + * IEEE754 floating point arithmetic + * single precision: MSUB.f (Fused Multiply Subtract) + * MSUBF.fmt: FPR[fd] = FPR[fd] - (FPR[fs] x FPR[ft]) + * + * MIPS floating point support + * Copyright (C) 2015 Imagination Technologies, Ltd. + * Author: Markos Chandras + * + * This program is free software; you can distribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; version 2 of the License. + */ + +#include "ieee754sp.h" + +union ieee754sp ieee754sp_msubf(union ieee754sp z, union ieee754sp x, + union ieee754sp y) +{ + int re; + int rs; + unsigned rm; + unsigned short lxm; + unsigned short hxm; + unsigned short lym; + unsigned short hym; + unsigned lrm; + unsigned hrm; + unsigned t; + unsigned at; + int s; + + COMPXSP; + COMPYSP; + u32 zm; int ze; int zs __maybe_unused; int zc; + + EXPLODEXSP; + EXPLODEYSP; + EXPLODESP(z, zc, zs, ze, zm) + + FLUSHXSP; + FLUSHYSP; + FLUSHSP(z, zc, zs, ze, zm); + + ieee754_clearcx(); + + switch (zc) { + case IEEE754_CLASS_SNAN: + ieee754_setcx(IEEE754_INVALID_OPERATION); + return ieee754sp_nanxcpt(z); + case IEEE754_CLASS_DNORM: + SPDNORMx(zm, ze); + /* QNAN is handled separately below */ + } + + switch (CLPAIR(xc, yc)) { + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_SNAN): + return ieee754sp_nanxcpt(y); + + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_SNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_SNAN, IEEE754_CLASS_INF): + return ieee754sp_nanxcpt(x); + + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_QNAN): + return y; + + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_QNAN): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_QNAN, IEEE754_CLASS_INF): + return x; + + /* + * Infinity handling + */ + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_INF): + if (zc == IEEE754_CLASS_QNAN) + return z; + ieee754_setcx(IEEE754_INVALID_OPERATION); + return ieee754sp_indef(); + + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_INF): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_INF, IEEE754_CLASS_INF): + if (zc == IEEE754_CLASS_QNAN) + return z; + return ieee754sp_inf(xs ^ ys); + + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_NORM): + case CLPAIR(IEEE754_CLASS_ZERO, IEEE754_CLASS_DNORM): + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_ZERO): + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_ZERO): + if (zc == IEEE754_CLASS_INF) + return ieee754sp_inf(zs); + /* Multiplication is 0 so just return z */ + return z; + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): + SPDNORMX; + + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): + if (zc == IEEE754_CLASS_QNAN) + return z; + else if (zc == IEEE754_CLASS_INF) + return ieee754sp_inf(zs); + SPDNORMY; + break; + + case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_NORM): + if (zc == IEEE754_CLASS_QNAN) + return z; + else if (zc == IEEE754_CLASS_INF) + return ieee754sp_inf(zs); + SPDNORMX; + break; + + case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_NORM): + if (zc == IEEE754_CLASS_QNAN) + return z; + else if (zc == IEEE754_CLASS_INF) + return ieee754sp_inf(zs); + /* fall through to real compuation */ + } + + /* Finally get to do some computation */ + + /* + * Do the multiplication bit first + * + * rm = xm * ym, re = xe + ye basically + * + * At this point xm and ym should have been normalized. + */ + + /* rm = xm * ym, re = xe+ye basically */ + assert(xm & SP_HIDDEN_BIT); + assert(ym & SP_HIDDEN_BIT); + + re = xe + ye; + rs = xs ^ ys; + + /* shunt to top of word */ + xm <<= 32 - (SP_FBITS + 1); + ym <<= 32 - (SP_FBITS + 1); + + /* + * Multiply 32 bits xm, ym to give high 32 bits rm with stickness. + */ + lxm = xm & 0xffff; + hxm = xm >> 16; + lym = ym & 0xffff; + hym = ym >> 16; + + lrm = lxm * lym; /* 16 * 16 => 32 */ + hrm = hxm * hym; /* 16 * 16 => 32 */ + + t = lxm * hym; /* 16 * 16 => 32 */ + at = lrm + (t << 16); + hrm += at < lrm; + lrm = at; + hrm = hrm + (t >> 16); + + t = hxm * lym; /* 16 * 16 => 32 */ + at = lrm + (t << 16); + hrm += at < lrm; + lrm = at; + hrm = hrm + (t >> 16); + + rm = hrm | (lrm != 0); + + /* + * Sticky shift down to normal rounding precision. + */ + if ((int) rm < 0) { + rm = (rm >> (32 - (SP_FBITS + 1 + 3))) | + ((rm << (SP_FBITS + 1 + 3)) != 0); + re++; + } else { + rm = (rm >> (32 - (SP_FBITS + 1 + 3 + 1))) | + ((rm << (SP_FBITS + 1 + 3 + 1)) != 0); + } + assert(rm & (SP_HIDDEN_BIT << 3)); + + /* And now the subtraction */ + + /* Flip sign of r and handle as add */ + rs ^= 1; + + assert(zm & SP_HIDDEN_BIT); + + /* + * Provide guard,round and stick bit space. + */ + zm <<= 3; + + if (ze > re) { + /* + * Have to shift y fraction right to align. + */ + s = ze - re; + SPXSRSYn(s); + } else if (re > ze) { + /* + * Have to shift x fraction right to align. + */ + s = re - ze; + SPXSRSYn(s); + } + assert(ze == re); + assert(ze <= SP_EMAX); + + if (zs == rs) { + /* + * Generate 28 bit result of adding two 27 bit numbers + * leaving result in zm, zs and ze. + */ + zm = zm + rm; + + if (zm >> (SP_FBITS + 1 + 3)) { /* carry out */ + SPXSRSX1(); /* shift preserving sticky */ + } + } else { + if (zm >= rm) { + zm = zm - rm; + } else { + zm = rm - zm; + zs = rs; + } + if (zm == 0) + return ieee754sp_zero(ieee754_csr.rm == FPU_CSR_RD); + + /* + * Normalize in extended single precision + */ + while ((zm >> (SP_MBITS + 3)) == 0) { + zm <<= 1; + ze--; + } + + } + return ieee754sp_format(zs, ze, zm); +} diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index fbea4432f3f2..5d3a25e1cfae 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -1276,6 +1276,7 @@ static void probe_pcache(void) case CPU_PROAPTIV: case CPU_M5150: case CPU_QEMU_GENERIC: + case CPU_I6400: if (!(read_c0_config7() & MIPS_CONF7_IAR) && (c->icache.waysize > PAGE_SIZE)) c->icache.flags |= MIPS_CACHE_ALIASES; diff --git a/arch/mips/mm/dma-default.c b/arch/mips/mm/dma-default.c index eeaf0245c3b1..a914dc1cb6d1 100644 --- a/arch/mips/mm/dma-default.c +++ b/arch/mips/mm/dma-default.c @@ -112,7 +112,7 @@ static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp) return gfp | dma_flag; } -void *dma_alloc_noncoherent(struct device *dev, size_t size, +static void *mips_dma_alloc_noncoherent(struct device *dev, size_t size, dma_addr_t * dma_handle, gfp_t gfp) { void *ret; @@ -128,7 +128,6 @@ void *dma_alloc_noncoherent(struct device *dev, size_t size, return ret; } -EXPORT_SYMBOL(dma_alloc_noncoherent); static void *mips_dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t * dma_handle, gfp_t gfp, struct dma_attrs *attrs) @@ -137,8 +136,12 @@ static void *mips_dma_alloc_coherent(struct device *dev, size_t size, struct page *page = NULL; unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; - if (dma_alloc_from_coherent(dev, size, dma_handle, &ret)) - return ret; + /* + * XXX: seems like the coherent and non-coherent implementations could + * be consolidated. + */ + if (dma_get_attr(DMA_ATTR_NON_CONSISTENT, attrs)) + return mips_dma_alloc_noncoherent(dev, size, dma_handle, gfp); gfp = massage_gfp_flags(dev, gfp); @@ -164,24 +167,24 @@ static void *mips_dma_alloc_coherent(struct device *dev, size_t size, } -void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr, - dma_addr_t dma_handle) +static void mips_dma_free_noncoherent(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma_handle) { plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL); free_pages((unsigned long) vaddr, get_order(size)); } -EXPORT_SYMBOL(dma_free_noncoherent); static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, struct dma_attrs *attrs) { unsigned long addr = (unsigned long) vaddr; - int order = get_order(size); unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; struct page *page = NULL; - if (dma_release_from_coherent(dev, order, vaddr)) + if (dma_get_attr(DMA_ATTR_NON_CONSISTENT, attrs)) { + mips_dma_free_noncoherent(dev, size, vaddr, dma_handle); return; + } plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL); @@ -194,6 +197,40 @@ static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr, __free_pages(page, get_order(size)); } +static int mips_dma_mmap(struct device *dev, struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + struct dma_attrs *attrs) +{ + unsigned long user_count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; + unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; + unsigned long addr = (unsigned long)cpu_addr; + unsigned long off = vma->vm_pgoff; + unsigned long pfn; + int ret = -ENXIO; + + if (!plat_device_is_coherent(dev) && !hw_coherentio) + addr = CAC_ADDR(addr); + + pfn = page_to_pfn(virt_to_page((void *)addr)); + + if (dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs)) + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + else + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + + if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret)) + return ret; + + if (off < count && user_count <= (count - off)) { + ret = remap_pfn_range(vma, vma->vm_start, + pfn + off, + user_count << PAGE_SHIFT, + vma->vm_page_prot); + } + + return ret; +} + static inline void __dma_sync_virtual(void *addr, size_t size, enum dma_data_direction direction) { @@ -380,6 +417,7 @@ EXPORT_SYMBOL(dma_cache_sync); static struct dma_map_ops mips_default_dma_map_ops = { .alloc = mips_dma_alloc_coherent, .free = mips_dma_free_coherent, + .mmap = mips_dma_mmap, .map_page = mips_dma_map_page, .unmap_page = mips_dma_unmap_page, .map_sg = mips_dma_map_sg, diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c index 852a41c6da45..4b88fa031891 100644 --- a/arch/mips/mm/fault.c +++ b/arch/mips/mm/fault.c @@ -57,12 +57,10 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write, #ifdef CONFIG_KPROBES /* - * This is to notify the fault handler of the kprobes. The - * exception code is redundant as it is also carried in REGS, - * but we pass it anyhow. + * This is to notify the fault handler of the kprobes. */ if (notify_die(DIE_PAGE_FAULT, "page fault", regs, -1, - (regs->cp0_cause >> 2) & 0x1f, SIGSEGV) == NOTIFY_STOP) + current->thread.trap_nr, SIGSEGV) == NOTIFY_STOP) return; #endif @@ -224,6 +222,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write, print_vma_addr(" ", regs->regs[31]); pr_info("\n"); } + current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f; info.si_signo = SIGSEGV; info.si_errno = 0; /* info.si_code has been set above */ @@ -282,6 +281,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write, field, (unsigned long) regs->cp0_epc, field, (unsigned long) regs->regs[31]); #endif + current->thread.trap_nr = (regs->cp0_cause >> 2) & 0x1f; tsk->thread.cp0_badvaddr = address; info.si_signo = SIGBUS; info.si_errno = 0; diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c index 198a3147dd7d..66d0f49c5bec 100644 --- a/arch/mips/mm/init.c +++ b/arch/mips/mm/init.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -333,9 +334,40 @@ static inline void mem_init_free_highmem(void) #endif } -unsigned __weak platform_maar_init(unsigned num_maars) +unsigned __weak platform_maar_init(unsigned num_pairs) { - return 0; + struct maar_config cfg[BOOT_MEM_MAP_MAX]; + unsigned i, num_configured, num_cfg = 0; + phys_addr_t skip; + + for (i = 0; i < boot_mem_map.nr_map; i++) { + switch (boot_mem_map.map[i].type) { + case BOOT_MEM_RAM: + case BOOT_MEM_INIT_RAM: + break; + default: + continue; + } + + skip = 0x10000 - (boot_mem_map.map[i].addr & 0xffff); + + cfg[num_cfg].lower = boot_mem_map.map[i].addr; + cfg[num_cfg].lower += skip; + + cfg[num_cfg].upper = cfg[num_cfg].lower; + cfg[num_cfg].upper += boot_mem_map.map[i].size - 1; + cfg[num_cfg].upper -= skip; + + cfg[num_cfg].attrs = MIPS_MAAR_S; + num_cfg++; + } + + num_configured = maar_config(cfg, num_cfg, num_pairs); + if (num_configured < num_cfg) + pr_warn("Not enough MAAR pairs (%u) for all bootmem regions (%u)\n", + num_pairs, num_cfg); + + return num_configured; } static void maar_init(void) diff --git a/arch/mips/mm/sc-mips.c b/arch/mips/mm/sc-mips.c index 4ceafd13870c..53ea8391f9bb 100644 --- a/arch/mips/mm/sc-mips.c +++ b/arch/mips/mm/sc-mips.c @@ -14,6 +14,7 @@ #include #include #include +#include /* * MIPS32/MIPS64 L2 cache handling @@ -94,6 +95,38 @@ static inline int mips_sc_is_activated(struct cpuinfo_mips *c) return 1; } +static int __init mips_sc_probe_cm3(void) +{ + struct cpuinfo_mips *c = ¤t_cpu_data; + unsigned long cfg = read_gcr_l2_config(); + unsigned long sets, line_sz, assoc; + + if (cfg & CM_GCR_L2_CONFIG_BYPASS_MSK) + return 0; + + sets = cfg & CM_GCR_L2_CONFIG_SET_SIZE_MSK; + sets >>= CM_GCR_L2_CONFIG_SET_SIZE_SHF; + c->scache.sets = 64 << sets; + + line_sz = cfg & CM_GCR_L2_CONFIG_LINE_SIZE_MSK; + line_sz >>= CM_GCR_L2_CONFIG_LINE_SIZE_SHF; + c->scache.linesz = 2 << line_sz; + + assoc = cfg & CM_GCR_L2_CONFIG_ASSOC_MSK; + assoc >>= CM_GCR_L2_CONFIG_ASSOC_SHF; + c->scache.ways = assoc + 1; + c->scache.waysize = c->scache.sets * c->scache.linesz; + c->scache.waybit = __ffs(c->scache.waysize); + + c->scache.flags &= ~MIPS_CACHE_NOT_PRESENT; + + return 1; +} + +void __weak platform_early_l2_init(void) +{ +} + static inline int __init mips_sc_probe(void) { struct cpuinfo_mips *c = ¤t_cpu_data; @@ -103,6 +136,15 @@ static inline int __init mips_sc_probe(void) /* Mark as not present until probe completed */ c->scache.flags |= MIPS_CACHE_NOT_PRESENT; + /* + * Do we need some platform specific probing before + * we configure L2? + */ + platform_early_l2_init(); + + if (mips_cm_revision() >= CM_REV_CM3) + return mips_sc_probe_cm3(); + /* Ignore anything but MIPSxx processors */ if (!(c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R1 | diff --git a/arch/mips/mm/tlb-r3k.c b/arch/mips/mm/tlb-r3k.c index 2b75b8f880ed..b4f366f7c0f5 100644 --- a/arch/mips/mm/tlb-r3k.c +++ b/arch/mips/mm/tlb-r3k.c @@ -36,7 +36,7 @@ extern void build_tlb_refill_handler(void); "nop\n\t" \ ".set pop\n\t") -static int r3k_have_wired_reg; /* Should be in cpu_data? */ +int r3k_have_wired_reg; /* Should be in cpu_data? */ /* TLB operations. */ static void local_flush_tlb_from(int entry) diff --git a/arch/mips/mti-malta/malta-init.c b/arch/mips/mti-malta/malta-init.c index cec3e187c48f..53c24784a2f7 100644 --- a/arch/mips/mti-malta/malta-init.c +++ b/arch/mips/mti-malta/malta-init.c @@ -303,3 +303,10 @@ void __init prom_init(void) if (!register_vsmp_smp_ops()) return; } + +void platform_early_l2_init(void) +{ + /* L2 configuration lives in the CM3 */ + if (mips_cm_revision() >= CM_REV_CM3) + mips_cm_probe(); +} diff --git a/arch/mips/mti-malta/malta-int.c b/arch/mips/mti-malta/malta-int.c index fa8f591f3713..c6a6c7afddab 100644 --- a/arch/mips/mti-malta/malta-int.c +++ b/arch/mips/mti-malta/malta-int.c @@ -382,122 +382,12 @@ void malta_be_init(void) /* Could change CM error mask register. */ } - -static char *tr[8] = { - "mem", "gcr", "gic", "mmio", - "0x04", "0x05", "0x06", "0x07" -}; - -static char *mcmd[32] = { - [0x00] = "0x00", - [0x01] = "Legacy Write", - [0x02] = "Legacy Read", - [0x03] = "0x03", - [0x04] = "0x04", - [0x05] = "0x05", - [0x06] = "0x06", - [0x07] = "0x07", - [0x08] = "Coherent Read Own", - [0x09] = "Coherent Read Share", - [0x0a] = "Coherent Read Discard", - [0x0b] = "Coherent Ready Share Always", - [0x0c] = "Coherent Upgrade", - [0x0d] = "Coherent Writeback", - [0x0e] = "0x0e", - [0x0f] = "0x0f", - [0x10] = "Coherent Copyback", - [0x11] = "Coherent Copyback Invalidate", - [0x12] = "Coherent Invalidate", - [0x13] = "Coherent Write Invalidate", - [0x14] = "Coherent Completion Sync", - [0x15] = "0x15", - [0x16] = "0x16", - [0x17] = "0x17", - [0x18] = "0x18", - [0x19] = "0x19", - [0x1a] = "0x1a", - [0x1b] = "0x1b", - [0x1c] = "0x1c", - [0x1d] = "0x1d", - [0x1e] = "0x1e", - [0x1f] = "0x1f" -}; - -static char *core[8] = { - "Invalid/OK", "Invalid/Data", - "Shared/OK", "Shared/Data", - "Modified/OK", "Modified/Data", - "Exclusive/OK", "Exclusive/Data" -}; - -static char *causes[32] = { - "None", "GC_WR_ERR", "GC_RD_ERR", "COH_WR_ERR", - "COH_RD_ERR", "MMIO_WR_ERR", "MMIO_RD_ERR", "0x07", - "0x08", "0x09", "0x0a", "0x0b", - "0x0c", "0x0d", "0x0e", "0x0f", - "0x10", "0x11", "0x12", "0x13", - "0x14", "0x15", "0x16", "INTVN_WR_ERR", - "INTVN_RD_ERR", "0x19", "0x1a", "0x1b", - "0x1c", "0x1d", "0x1e", "0x1f" -}; - int malta_be_handler(struct pt_regs *regs, int is_fixup) { /* This duplicates the handling in do_be which seems wrong */ int retval = is_fixup ? MIPS_BE_FIXUP : MIPS_BE_FATAL; - if (mips_cm_present()) { - unsigned long cm_error = read_gcr_error_cause(); - unsigned long cm_addr = read_gcr_error_addr(); - unsigned long cm_other = read_gcr_error_mult(); - unsigned long cause, ocause; - char buf[256]; - - cause = cm_error & CM_GCR_ERROR_CAUSE_ERRTYPE_MSK; - if (cause != 0) { - cause >>= CM_GCR_ERROR_CAUSE_ERRTYPE_SHF; - if (cause < 16) { - unsigned long cca_bits = (cm_error >> 15) & 7; - unsigned long tr_bits = (cm_error >> 12) & 7; - unsigned long cmd_bits = (cm_error >> 7) & 0x1f; - unsigned long stag_bits = (cm_error >> 3) & 15; - unsigned long sport_bits = (cm_error >> 0) & 7; - - snprintf(buf, sizeof(buf), - "CCA=%lu TR=%s MCmd=%s STag=%lu " - "SPort=%lu\n", - cca_bits, tr[tr_bits], mcmd[cmd_bits], - stag_bits, sport_bits); - } else { - /* glob state & sresp together */ - unsigned long c3_bits = (cm_error >> 18) & 7; - unsigned long c2_bits = (cm_error >> 15) & 7; - unsigned long c1_bits = (cm_error >> 12) & 7; - unsigned long c0_bits = (cm_error >> 9) & 7; - unsigned long sc_bit = (cm_error >> 8) & 1; - unsigned long cmd_bits = (cm_error >> 3) & 0x1f; - unsigned long sport_bits = (cm_error >> 0) & 7; - snprintf(buf, sizeof(buf), - "C3=%s C2=%s C1=%s C0=%s SC=%s " - "MCmd=%s SPort=%lu\n", - core[c3_bits], core[c2_bits], - core[c1_bits], core[c0_bits], - sc_bit ? "True" : "False", - mcmd[cmd_bits], sport_bits); - } - - ocause = (cm_other & CM_GCR_ERROR_MULT_ERR2ND_MSK) >> - CM_GCR_ERROR_MULT_ERR2ND_SHF; - - pr_err("CM_ERROR=%08lx %s <%s>\n", cm_error, - causes[cause], buf); - pr_err("CM_ADDR =%08lx\n", cm_addr); - pr_err("CM_OTHER=%08lx %s\n", cm_other, causes[ocause]); - - /* reprime cause register */ - write_gcr_error_cause(0); - } - } + mips_cm_error_report(); return retval; } diff --git a/arch/mips/mti-malta/malta-memory.c b/arch/mips/mti-malta/malta-memory.c index b769657be4d4..dadeb8379182 100644 --- a/arch/mips/mti-malta/malta-memory.c +++ b/arch/mips/mti-malta/malta-memory.c @@ -179,31 +179,6 @@ void __init prom_free_prom_memory(void) } } -unsigned platform_maar_init(unsigned num_pairs) -{ - phys_addr_t mem_end = (physical_memsize & ~0xffffull) - 1; - struct maar_config cfg[] = { - /* DRAM preceding I/O */ - { 0x00000000, 0x0fffffff, MIPS_MAAR_S }, - - /* DRAM following I/O */ - { 0x20000000, mem_end, MIPS_MAAR_S }, - - /* DRAM alias in upper half of physical */ - { 0x80000000, 0x80000000 + mem_end, MIPS_MAAR_S }, - }; - unsigned i, num_cfg = ARRAY_SIZE(cfg); - - /* If DRAM fits before I/O, drop the region following it */ - if (physical_memsize <= 0x10000000) { - num_cfg--; - for (i = 1; i < num_cfg; i++) - cfg[i] = cfg[i + 1]; - } - - return maar_config(cfg, num_cfg, num_pairs); -} - phys_addr_t mips_cdmm_phys_base(void) { /* This address is "typically unused" */ diff --git a/arch/mips/netlogic/common/irq.c b/arch/mips/netlogic/common/irq.c index 5f5d18b0e94d..3660dc67d544 100644 --- a/arch/mips/netlogic/common/irq.c +++ b/arch/mips/netlogic/common/irq.c @@ -87,7 +87,7 @@ struct nlm_pic_irq { static void xlp_pic_enable(struct irq_data *d) { unsigned long flags; - struct nlm_pic_irq *pd = irq_data_get_irq_handler_data(d); + struct nlm_pic_irq *pd = irq_data_get_irq_chip_data(d); BUG_ON(!pd); spin_lock_irqsave(&pd->node->piclock, flags); @@ -97,7 +97,7 @@ static void xlp_pic_enable(struct irq_data *d) static void xlp_pic_disable(struct irq_data *d) { - struct nlm_pic_irq *pd = irq_data_get_irq_handler_data(d); + struct nlm_pic_irq *pd = irq_data_get_irq_chip_data(d); unsigned long flags; BUG_ON(!pd); @@ -108,7 +108,7 @@ static void xlp_pic_disable(struct irq_data *d) static void xlp_pic_mask_ack(struct irq_data *d) { - struct nlm_pic_irq *pd = irq_data_get_irq_handler_data(d); + struct nlm_pic_irq *pd = irq_data_get_irq_chip_data(d); clear_c0_eimr(pd->picirq); ack_c0_eirr(pd->picirq); @@ -116,7 +116,7 @@ static void xlp_pic_mask_ack(struct irq_data *d) static void xlp_pic_unmask(struct irq_data *d) { - struct nlm_pic_irq *pd = irq_data_get_irq_handler_data(d); + struct nlm_pic_irq *pd = irq_data_get_irq_chip_data(d); BUG_ON(!pd); @@ -193,7 +193,7 @@ void nlm_setup_pic_irq(int node, int picirq, int irq, int irt) pic_data->picirq = picirq; pic_data->node = nlm_get_node(node); irq_set_chip_and_handler(xirq, &xlp_pic, handle_level_irq); - irq_set_handler_data(xirq, pic_data); + irq_set_chip_data(xirq, pic_data); } void nlm_set_pic_extra_ack(int node, int irq, void (*xack)(struct irq_data *)) @@ -202,7 +202,7 @@ void nlm_set_pic_extra_ack(int node, int irq, void (*xack)(struct irq_data *)) int xirq; xirq = nlm_irq_to_xirq(node, irq); - pic_data = irq_get_handler_data(xirq); + pic_data = irq_get_chip_data(xirq); if (WARN_ON(!pic_data)) return; pic_data->extra_ack = xack; diff --git a/arch/mips/netlogic/common/nlm-dma.c b/arch/mips/netlogic/common/nlm-dma.c index f3d4ae87abc7..3758715d4ab6 100644 --- a/arch/mips/netlogic/common/nlm-dma.c +++ b/arch/mips/netlogic/common/nlm-dma.c @@ -47,11 +47,6 @@ static char *nlm_swiotlb; static void *nlm_dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, struct dma_attrs *attrs) { - void *ret; - - if (dma_alloc_from_coherent(dev, size, dma_handle, &ret)) - return ret; - /* ignore region specifiers */ gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM); @@ -69,11 +64,6 @@ static void *nlm_dma_alloc_coherent(struct device *dev, size_t size, static void nlm_dma_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, struct dma_attrs *attrs) { - int order = get_order(size); - - if (dma_release_from_coherent(dev, order, vaddr)) - return; - swiotlb_free_coherent(dev, size, vaddr, dma_handle); } diff --git a/arch/mips/netlogic/common/smp.c b/arch/mips/netlogic/common/smp.c index f5fff228b347..0136b4f9c9cd 100644 --- a/arch/mips/netlogic/common/smp.c +++ b/arch/mips/netlogic/common/smp.c @@ -82,8 +82,9 @@ void nlm_send_ipi_mask(const struct cpumask *mask, unsigned int action) } /* IRQ_IPI_SMP_FUNCTION Handler */ -void nlm_smp_function_ipi_handler(unsigned int irq, struct irq_desc *desc) +void nlm_smp_function_ipi_handler(unsigned int __irq, struct irq_desc *desc) { + unsigned int irq = irq_desc_get_irq(desc); clear_c0_eimr(irq); ack_c0_eirr(irq); generic_smp_call_function_interrupt(); @@ -91,8 +92,9 @@ void nlm_smp_function_ipi_handler(unsigned int irq, struct irq_desc *desc) } /* IRQ_IPI_SMP_RESCHEDULE handler */ -void nlm_smp_resched_ipi_handler(unsigned int irq, struct irq_desc *desc) +void nlm_smp_resched_ipi_handler(unsigned int __irq, struct irq_desc *desc) { + unsigned int irq = irq_desc_get_irq(desc); clear_c0_eimr(irq); ack_c0_eirr(irq); scheduler_ipi(); diff --git a/arch/mips/netlogic/xlp/ahci-init-xlp2.c b/arch/mips/netlogic/xlp/ahci-init-xlp2.c index 7b066a44e679..c11b9c7dc7c8 100644 --- a/arch/mips/netlogic/xlp/ahci-init-xlp2.c +++ b/arch/mips/netlogic/xlp/ahci-init-xlp2.c @@ -152,7 +152,7 @@ static const u8 sata_phy_config1[] = { 0xC9, 0xC9, 0x07, 0x07, 0x18, 0x18, 0x01, 0x01, 0x22, 0x00 }; -/* SATA PHY config for register block 2 0x0x8065 .. 0x0x80A4 */ +/* SATA PHY config for register block 2 0x8065 .. 0x80A4 */ static const u8 sata_phy_config2[] = { 0xAA, 0x00, 0x4C, 0xC9, 0xC9, 0x07, 0x07, 0x18, 0x18, 0x05, 0x0C, 0x10, 0x00, 0x10, 0x00, 0xFF, diff --git a/arch/mips/netlogic/xlp/nlm_hal.c b/arch/mips/netlogic/xlp/nlm_hal.c index a8f4144a0297..80ec929747c3 100644 --- a/arch/mips/netlogic/xlp/nlm_hal.c +++ b/arch/mips/netlogic/xlp/nlm_hal.c @@ -91,6 +91,8 @@ static int xlp9xx_irq_to_irt(int irq) return 134; case PIC_SATA_IRQ: return 143; + case PIC_NAND_IRQ: + return 151; case PIC_SPI_IRQ: return 152; case PIC_MMC_IRQ: diff --git a/arch/mips/oprofile/common.c b/arch/mips/oprofile/common.c index 81f58958cf08..3c9ec3ddca84 100644 --- a/arch/mips/oprofile/common.c +++ b/arch/mips/oprofile/common.c @@ -91,6 +91,7 @@ int __init oprofile_arch_init(struct oprofile_operations *ops) case CPU_INTERAPTIV: case CPU_PROAPTIV: case CPU_P5600: + case CPU_I6400: case CPU_M5150: case CPU_LOONGSON1: case CPU_SB1: diff --git a/arch/mips/oprofile/op_model_mipsxx.c b/arch/mips/oprofile/op_model_mipsxx.c index 6a6e2cc55b89..8f988a61b7a8 100644 --- a/arch/mips/oprofile/op_model_mipsxx.c +++ b/arch/mips/oprofile/op_model_mipsxx.c @@ -392,6 +392,10 @@ static int __init mipsxx_init(void) op_model_mipsxx_ops.cpu_type = "mips/P5600"; break; + case CPU_I6400: + op_model_mipsxx_ops.cpu_type = "mips/I6400"; + break; + case CPU_M5150: op_model_mipsxx_ops.cpu_type = "mips/M5150"; break; diff --git a/arch/mips/pci/msi-octeon.c b/arch/mips/pci/msi-octeon.c index cffaaf4aae3c..2a5bb849b10e 100644 --- a/arch/mips/pci/msi-octeon.c +++ b/arch/mips/pci/msi-octeon.c @@ -200,7 +200,7 @@ int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) if (type == PCI_CAP_ID_MSI && nvec > 1) return 1; - list_for_each_entry(entry, &dev->msi_list, list) { + for_each_pci_msi_entry(entry, dev) { ret = arch_setup_msi_irq(dev, entry); if (ret < 0) return ret; diff --git a/arch/mips/pci/msi-xlp.c b/arch/mips/pci/msi-xlp.c index 3407495fcbe2..bb14335f804b 100644 --- a/arch/mips/pci/msi-xlp.c +++ b/arch/mips/pci/msi-xlp.c @@ -131,7 +131,7 @@ struct xlp_msi_data { */ static void xlp_msi_enable(struct irq_data *d) { - struct xlp_msi_data *md = irq_data_get_irq_handler_data(d); + struct xlp_msi_data *md = irq_data_get_irq_chip_data(d); unsigned long flags; int vec; @@ -148,7 +148,7 @@ static void xlp_msi_enable(struct irq_data *d) static void xlp_msi_disable(struct irq_data *d) { - struct xlp_msi_data *md = irq_data_get_irq_handler_data(d); + struct xlp_msi_data *md = irq_data_get_irq_chip_data(d); unsigned long flags; int vec; @@ -165,7 +165,7 @@ static void xlp_msi_disable(struct irq_data *d) static void xlp_msi_mask_ack(struct irq_data *d) { - struct xlp_msi_data *md = irq_data_get_irq_handler_data(d); + struct xlp_msi_data *md = irq_data_get_irq_chip_data(d); int link, vec; link = nlm_irq_msilink(d->irq); @@ -211,7 +211,7 @@ static void xlp_msix_mask_ack(struct irq_data *d) msixvec = nlm_irq_msixvec(d->irq); link = nlm_irq_msixlink(msixvec); pci_msi_mask_irq(d); - md = irq_data_get_irq_handler_data(d); + md = irq_data_get_irq_chip_data(d); /* Ack MSI on bridge */ if (cpu_is_xlp9xx()) { @@ -302,7 +302,7 @@ static int xlp_setup_msi(uint64_t lnkbase, int node, int link, /* Get MSI data for the link */ lirq = PIC_PCIE_LINK_MSI_IRQ(link); xirq = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0)); - md = irq_get_handler_data(xirq); + md = irq_get_chip_data(xirq); msiaddr = MSI_LINK_ADDR(node, link); spin_lock_irqsave(&md->msi_lock, flags); @@ -409,7 +409,7 @@ static int xlp_setup_msix(uint64_t lnkbase, int node, int link, /* Get MSI data for the link */ lirq = PIC_PCIE_MSIX_IRQ(link); xirq = nlm_irq_to_xirq(node, nlm_link_msixirq(link, 0)); - md = irq_get_handler_data(xirq); + md = irq_get_chip_data(xirq); msixaddr = MSIX_LINK_ADDR(node, link); spin_lock_irqsave(&md->msi_lock, flags); @@ -485,7 +485,7 @@ void __init xlp_init_node_msi_irqs(int node, int link) irq = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0)); for (i = irq; i < irq + XLP_MSIVEC_PER_LINK; i++) { irq_set_chip_and_handler(i, &xlp_msi_chip, handle_level_irq); - irq_set_handler_data(i, md); + irq_set_chip_data(i, md); } for (i = 0; i < XLP_MSIXVEC_PER_LINK ; i++) { @@ -508,7 +508,7 @@ void __init xlp_init_node_msi_irqs(int node, int link) /* Initialize MSI-X extended irq space for the link */ irq = nlm_irq_to_xirq(node, nlm_link_msixirq(link, i)); irq_set_chip_and_handler(irq, &xlp_msix_chip, handle_level_irq); - irq_set_handler_data(irq, md); + irq_set_chip_data(irq, md); } } @@ -520,7 +520,7 @@ void nlm_dispatch_msi(int node, int lirq) link = lirq - PIC_PCIE_LINK_MSI_IRQ_BASE; irqbase = nlm_irq_to_xirq(node, nlm_link_msiirq(link, 0)); - md = irq_get_handler_data(irqbase); + md = irq_get_chip_data(irqbase); if (cpu_is_xlp9xx()) status = nlm_read_reg(md->lnkbase, PCIE_9XX_MSI_STATUS) & md->msi_enabled_mask; @@ -550,7 +550,7 @@ void nlm_dispatch_msix(int node, int lirq) link = lirq - PIC_PCIE_MSIX_IRQ_BASE; irqbase = nlm_irq_to_xirq(node, nlm_link_msixirq(link, 0)); - md = irq_get_handler_data(irqbase); + md = irq_get_chip_data(irqbase); if (cpu_is_xlp9xx()) status = nlm_read_reg(md->lnkbase, PCIE_9XX_MSIX_STATUSX(link)); else diff --git a/arch/mips/pci/ops-emma2rh.c b/arch/mips/pci/ops-emma2rh.c index 710aef5c070e..2dc97c45685e 100644 --- a/arch/mips/pci/ops-emma2rh.c +++ b/arch/mips/pci/ops-emma2rh.c @@ -25,7 +25,6 @@ #include #include -#include #include @@ -40,10 +39,9 @@ static int check_args(struct pci_bus *bus, u32 devfn, u32 * bus_num) { /* check if the bus is top-level */ - if (bus->parent != NULL) { + if (bus->parent != NULL) *bus_num = bus->number; - db_assert(bus_num != NULL); - } else + else *bus_num = 0; if (*bus_num == 0) { diff --git a/arch/mips/pci/pci-ar71xx.c b/arch/mips/pci/pci-ar71xx.c index 283157f8dc64..ad35a5e6a56c 100644 --- a/arch/mips/pci/pci-ar71xx.c +++ b/arch/mips/pci/pci-ar71xx.c @@ -312,8 +312,8 @@ static void ar71xx_pci_irq_init(struct ar71xx_pci_controller *apc) irq_set_chip_data(i, apc); } - irq_set_handler_data(apc->irq, apc); - irq_set_chained_handler(apc->irq, ar71xx_pci_irq_handler); + irq_set_chained_handler_and_data(apc->irq, ar71xx_pci_irq_handler, + apc); } static void ar71xx_pci_reset(void) diff --git a/arch/mips/pci/pci-ar724x.c b/arch/mips/pci/pci-ar724x.c index 0af362b5af92..907d11dd921b 100644 --- a/arch/mips/pci/pci-ar724x.c +++ b/arch/mips/pci/pci-ar724x.c @@ -321,8 +321,8 @@ static void ar724x_pci_irq_init(struct ar724x_pci_controller *apc, irq_set_chip_data(i, apc); } - irq_set_handler_data(apc->irq, apc); - irq_set_chained_handler(apc->irq, ar724x_pci_irq_handler); + irq_set_chained_handler_and_data(apc->irq, ar724x_pci_irq_handler, + apc); } static int ar724x_pci_probe(struct platform_device *pdev) diff --git a/arch/mips/pci/pci-lantiq.c b/arch/mips/pci/pci-lantiq.c index c5347d99cf3a..6a15dbd085aa 100644 --- a/arch/mips/pci/pci-lantiq.c +++ b/arch/mips/pci/pci-lantiq.c @@ -20,7 +20,6 @@ #include #include -#include #include #include diff --git a/arch/mips/pci/pci-rt3883.c b/arch/mips/pci/pci-rt3883.c index 80fafe646e74..53c8efaf1572 100644 --- a/arch/mips/pci/pci-rt3883.c +++ b/arch/mips/pci/pci-rt3883.c @@ -129,7 +129,7 @@ static void rt3883_pci_write_cfg32(struct rt3883_pci_controller *rpc, rt3883_pci_w32(rpc, val, RT3883_PCI_REG_CFGDATA); } -static void rt3883_pci_irq_handler(unsigned int irq, struct irq_desc *desc) +static void rt3883_pci_irq_handler(unsigned int __irq, struct irq_desc *desc) { struct rt3883_pci_controller *rpc; u32 pending; @@ -145,7 +145,7 @@ static void rt3883_pci_irq_handler(unsigned int irq, struct irq_desc *desc) } while (pending) { - unsigned bit = __ffs(pending); + unsigned irq, bit = __ffs(pending); irq = irq_find_mapping(rpc->irq_domain, bit); generic_handle_irq(irq); @@ -225,8 +225,7 @@ static int rt3883_pci_irq_init(struct device *dev, return -ENODEV; } - irq_set_handler_data(irq, rpc); - irq_set_chained_handler(irq, rt3883_pci_irq_handler); + irq_set_chained_handler_and_data(irq, rt3883_pci_irq_handler, rpc); return 0; } diff --git a/arch/mips/pistachio/Kconfig b/arch/mips/pistachio/Kconfig new file mode 100644 index 000000000000..97731ea94322 --- /dev/null +++ b/arch/mips/pistachio/Kconfig @@ -0,0 +1,13 @@ +config PISTACHIO_GPTIMER_CLKSRC + bool "Enable General Purpose Timer based clocksource" + depends on MACH_PISTACHIO + select CLKSRC_PISTACHIO + select MIPS_EXTERNAL_TIMER + help + This option enables a clocksource driver based on a Pistachio + SoC General Purpose external timer. + + If you want to enable the CPUFreq, you need to enable + this option. + + If you don't want to enable CPUFreq, you can leave this disabled. diff --git a/arch/mips/pmcs-msp71xx/msp_irq_cic.c b/arch/mips/pmcs-msp71xx/msp_irq_cic.c index 1207ec4dfb77..8b9cf6463040 100644 --- a/arch/mips/pmcs-msp71xx/msp_irq_cic.c +++ b/arch/mips/pmcs-msp71xx/msp_irq_cic.c @@ -88,7 +88,8 @@ static void unmask_cic_irq(struct irq_data *d) * Make sure we have IRQ affinity. It may have changed while * we were processing the IRQ. */ - if (!cpumask_test_cpu(smp_processor_id(), d->affinity)) + if (!cpumask_test_cpu(smp_processor_id(), + irq_data_get_affinity_mask(d))) return; #endif diff --git a/arch/mips/ralink/cevt-rt3352.c b/arch/mips/ralink/cevt-rt3352.c index 24bf057a3613..a8e70a9f274b 100644 --- a/arch/mips/ralink/cevt-rt3352.c +++ b/arch/mips/ralink/cevt-rt3352.c @@ -36,8 +36,8 @@ struct systick_device { int freq_scale; }; -static void systick_set_clock_mode(enum clock_event_mode mode, - struct clock_event_device *evt); +static int systick_set_oneshot(struct clock_event_device *evt); +static int systick_shutdown(struct clock_event_device *evt); static int systick_next_event(unsigned long delta, struct clock_event_device *evt) @@ -73,11 +73,12 @@ static struct systick_device systick = { * cevt-r4k uses 300, make sure systick * gets used if available */ - .rating = 310, - .features = CLOCK_EVT_FEAT_ONESHOT, - .set_next_event = systick_next_event, - .set_mode = systick_set_clock_mode, - .event_handler = systick_event_handler, + .rating = 310, + .features = CLOCK_EVT_FEAT_ONESHOT, + .set_next_event = systick_next_event, + .set_state_shutdown = systick_shutdown, + .set_state_oneshot = systick_set_oneshot, + .event_handler = systick_event_handler, }, }; @@ -87,33 +88,33 @@ static struct irqaction systick_irqaction = { .dev_id = &systick.dev, }; -static void systick_set_clock_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int systick_shutdown(struct clock_event_device *evt) { struct systick_device *sdev; sdev = container_of(evt, struct systick_device, dev); - switch (mode) { - case CLOCK_EVT_MODE_ONESHOT: - if (!sdev->irq_requested) - setup_irq(systick.dev.irq, &systick_irqaction); - sdev->irq_requested = 1; - iowrite32(CFG_EXT_STK_EN | CFG_CNT_EN, - systick.membase + SYSTICK_CONFIG); - break; + if (sdev->irq_requested) + free_irq(systick.dev.irq, &systick_irqaction); + sdev->irq_requested = 0; + iowrite32(0, systick.membase + SYSTICK_CONFIG); - case CLOCK_EVT_MODE_SHUTDOWN: - if (sdev->irq_requested) - free_irq(systick.dev.irq, &systick_irqaction); - sdev->irq_requested = 0; - iowrite32(0, systick.membase + SYSTICK_CONFIG); - break; + return 0; +} - default: - pr_err("%s: Unhandeled mips clock_mode\n", systick.dev.name); - break; - } +static int systick_set_oneshot(struct clock_event_device *evt) +{ + struct systick_device *sdev; + + sdev = container_of(evt, struct systick_device, dev); + + if (!sdev->irq_requested) + setup_irq(systick.dev.irq, &systick_irqaction); + sdev->irq_requested = 1; + iowrite32(CFG_EXT_STK_EN | CFG_CNT_EN, + systick.membase + SYSTICK_CONFIG); + + return 0; } static void __init ralink_systick_init(struct device_node *np) diff --git a/arch/mips/rb532/devices.c b/arch/mips/rb532/devices.c index e31e8cdcb296..9bd7a2de0765 100644 --- a/arch/mips/rb532/devices.c +++ b/arch/mips/rb532/devices.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/mips/rb532/gpio.c b/arch/mips/rb532/gpio.c index 5aa3df853082..650d5d39f34d 100644 --- a/arch/mips/rb532/gpio.c +++ b/arch/mips/rb532/gpio.c @@ -140,6 +140,11 @@ static int rb532_gpio_direction_output(struct gpio_chip *chip, return 0; } +static int rb532_gpio_to_irq(struct gpio_chip *chip, unsigned gpio) +{ + return 8 + 4 * 32 + gpio; +} + static struct rb532_gpio_chip rb532_gpio_chip[] = { [0] = { .chip = { @@ -148,6 +153,7 @@ static struct rb532_gpio_chip rb532_gpio_chip[] = { .direction_output = rb532_gpio_direction_output, .get = rb532_gpio_get, .set = rb532_gpio_set, + .to_irq = rb532_gpio_to_irq, .base = 0, .ngpio = 32, }, diff --git a/arch/mips/sgi-ip27/ip27-timer.c b/arch/mips/sgi-ip27/ip27-timer.c index a6d10f607f34..42d6cb9f956e 100644 --- a/arch/mips/sgi-ip27/ip27-timer.c +++ b/arch/mips/sgi-ip27/ip27-timer.c @@ -64,12 +64,6 @@ static int rt_next_event(unsigned long delta, struct clock_event_device *evt) return LOCAL_HUB_L(PI_RT_COUNT) >= cnt ? -ETIME : 0; } -static void rt_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) -{ - /* Nothing to do ... */ -} - unsigned int rt_timer_irq; static DEFINE_PER_CPU(struct clock_event_device, hub_rt_clockevent); @@ -124,7 +118,6 @@ void hub_rt_clock_event_init(void) cd->irq = irq; cd->cpumask = cpumask_of(cpu); cd->set_next_event = rt_next_event; - cd->set_mode = rt_set_mode; clockevents_register_device(cd); } diff --git a/arch/mips/sibyte/common/bus_watcher.c b/arch/mips/sibyte/common/bus_watcher.c index 41a1d2242211..a4e55999ecb4 100644 --- a/arch/mips/sibyte/common/bus_watcher.c +++ b/arch/mips/sibyte/common/bus_watcher.c @@ -250,4 +250,4 @@ int __init sibyte_bus_watcher(void) return 0; } -__initcall(sibyte_bus_watcher); +device_initcall(sibyte_bus_watcher); diff --git a/arch/mips/sni/time.c b/arch/mips/sni/time.c index cf8ec568b9df..fb4b3520cdc6 100644 --- a/arch/mips/sni/time.c +++ b/arch/mips/sni/time.c @@ -14,44 +14,33 @@ #define SNI_COUNTER2_DIV 64 #define SNI_COUNTER0_DIV ((SNI_CLOCK_TICK_RATE / SNI_COUNTER2_DIV) / HZ) -static void a20r_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int a20r_set_periodic(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - *(volatile u8 *)(A20R_PT_CLOCK_BASE + 12) = 0x34; - wmb(); - *(volatile u8 *)(A20R_PT_CLOCK_BASE + 0) = SNI_COUNTER0_DIV; - wmb(); - *(volatile u8 *)(A20R_PT_CLOCK_BASE + 0) = SNI_COUNTER0_DIV >> 8; - wmb(); + *(volatile u8 *)(A20R_PT_CLOCK_BASE + 12) = 0x34; + wmb(); + *(volatile u8 *)(A20R_PT_CLOCK_BASE + 0) = SNI_COUNTER0_DIV; + wmb(); + *(volatile u8 *)(A20R_PT_CLOCK_BASE + 0) = SNI_COUNTER0_DIV >> 8; + wmb(); - *(volatile u8 *)(A20R_PT_CLOCK_BASE + 12) = 0xb4; - wmb(); - *(volatile u8 *)(A20R_PT_CLOCK_BASE + 8) = SNI_COUNTER2_DIV; - wmb(); - *(volatile u8 *)(A20R_PT_CLOCK_BASE + 8) = SNI_COUNTER2_DIV >> 8; - wmb(); - - break; - case CLOCK_EVT_MODE_ONESHOT: - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - break; - case CLOCK_EVT_MODE_RESUME: - break; - } + *(volatile u8 *)(A20R_PT_CLOCK_BASE + 12) = 0xb4; + wmb(); + *(volatile u8 *)(A20R_PT_CLOCK_BASE + 8) = SNI_COUNTER2_DIV; + wmb(); + *(volatile u8 *)(A20R_PT_CLOCK_BASE + 8) = SNI_COUNTER2_DIV >> 8; + wmb(); + return 0; } static struct clock_event_device a20r_clockevent_device = { - .name = "a20r-timer", - .features = CLOCK_EVT_FEAT_PERIODIC, + .name = "a20r-timer", + .features = CLOCK_EVT_FEAT_PERIODIC, /* .mult, .shift, .max_delta_ns and .min_delta_ns left uninitialized */ - .rating = 300, - .irq = SNI_A20R_IRQ_TIMER, - .set_mode = a20r_set_mode, + .rating = 300, + .irq = SNI_A20R_IRQ_TIMER, + .set_state_periodic = a20r_set_periodic, }; static irqreturn_t a20r_interrupt(int irq, void *dev_id) diff --git a/arch/mips/txx9/generic/setup.c b/arch/mips/txx9/generic/setup.c index 2791b8641df6..9d9962ab7d25 100644 --- a/arch/mips/txx9/generic/setup.c +++ b/arch/mips/txx9/generic/setup.c @@ -117,22 +117,6 @@ void clk_put(struct clk *clk) } EXPORT_SYMBOL(clk_put); -/* GPIO support */ - -#ifdef CONFIG_GPIOLIB -int gpio_to_irq(unsigned gpio) -{ - return -EINVAL; -} -EXPORT_SYMBOL(gpio_to_irq); - -int irq_to_gpio(unsigned irq) -{ - return -EINVAL; -} -EXPORT_SYMBOL(irq_to_gpio); -#endif - #define BOARD_VEC(board) extern struct txx9_board_vec board; #include #undef BOARD_VEC diff --git a/arch/mn10300/include/asm/atomic.h b/arch/mn10300/include/asm/atomic.h index 5be655e83e70..375e59140c9c 100644 --- a/arch/mn10300/include/asm/atomic.h +++ b/arch/mn10300/include/asm/atomic.h @@ -89,6 +89,10 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -127,73 +131,6 @@ static inline void atomic_dec(atomic_t *v) #define atomic_xchg(ptr, v) (xchg(&(ptr)->counter, (v))) #define atomic_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), (old), (new))) -/** - * atomic_clear_mask - Atomically clear bits in memory - * @mask: Mask of the bits to be cleared - * @v: pointer to word in memory - * - * Atomically clears the bits set in mask from the memory word specified. - */ -static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr) -{ -#ifdef CONFIG_SMP - int status; - - asm volatile( - "1: mov %3,(_AAR,%2) \n" - " mov (_ADR,%2),%0 \n" - " and %4,%0 \n" - " mov %0,(_ADR,%2) \n" - " mov (_ADR,%2),%0 \n" /* flush */ - " mov (_ASR,%2),%0 \n" - " or %0,%0 \n" - " bne 1b \n" - : "=&r"(status), "=m"(*addr) - : "a"(ATOMIC_OPS_BASE_ADDR), "r"(addr), "r"(~mask) - : "memory", "cc"); -#else - unsigned long flags; - - mask = ~mask; - flags = arch_local_cli_save(); - *addr &= mask; - arch_local_irq_restore(flags); -#endif -} - -/** - * atomic_set_mask - Atomically set bits in memory - * @mask: Mask of the bits to be set - * @v: pointer to word in memory - * - * Atomically sets the bits set in mask from the memory word specified. - */ -static inline void atomic_set_mask(unsigned long mask, unsigned long *addr) -{ -#ifdef CONFIG_SMP - int status; - - asm volatile( - "1: mov %3,(_AAR,%2) \n" - " mov (_ADR,%2),%0 \n" - " or %4,%0 \n" - " mov %0,(_ADR,%2) \n" - " mov (_ADR,%2),%0 \n" /* flush */ - " mov (_ASR,%2),%0 \n" - " or %0,%0 \n" - " bne 1b \n" - : "=&r"(status), "=m"(*addr) - : "a"(ATOMIC_OPS_BASE_ADDR), "r"(addr), "r"(mask) - : "memory", "cc"); -#else - unsigned long flags; - - flags = arch_local_cli_save(); - *addr |= mask; - arch_local_irq_restore(flags); -#endif -} - #endif /* __KERNEL__ */ #endif /* CONFIG_SMP */ #endif /* _ASM_ATOMIC_H */ diff --git a/arch/mn10300/kernel/cevt-mn10300.c b/arch/mn10300/kernel/cevt-mn10300.c index 60f64ca1752a..d9b34dd44f04 100644 --- a/arch/mn10300/kernel/cevt-mn10300.c +++ b/arch/mn10300/kernel/cevt-mn10300.c @@ -41,12 +41,6 @@ static int next_event(unsigned long delta, return 0; } -static void set_clock_mode(enum clock_event_mode mode, - struct clock_event_device *evt) -{ - /* Nothing to do ... */ -} - static DEFINE_PER_CPU(struct clock_event_device, mn10300_clockevent_device); static DEFINE_PER_CPU(struct irqaction, timer_irq); @@ -108,7 +102,6 @@ int __init init_clockevents(void) cd->rating = 200; cd->cpumask = cpumask_of(smp_processor_id()); - cd->set_mode = set_clock_mode; cd->event_handler = event_handler; cd->set_next_event = next_event; @@ -123,7 +116,7 @@ int __init init_clockevents(void) { struct irq_data *data; data = irq_get_irq_data(cd->irq); - cpumask_copy(data->affinity, cpumask_of(cpu)); + cpumask_copy(irq_data_get_affinity_mask(data), cpumask_of(cpu)); iact->flags |= IRQF_NOBALANCING; } #endif diff --git a/arch/mn10300/kernel/irq.c b/arch/mn10300/kernel/irq.c index 480de70f4059..c716437baa2c 100644 --- a/arch/mn10300/kernel/irq.c +++ b/arch/mn10300/kernel/irq.c @@ -87,7 +87,8 @@ static void mn10300_cpupic_mask_ack(struct irq_data *d) tmp2 = GxICR(irq); irq_affinity_online[irq] = - cpumask_any_and(d->affinity, cpu_online_mask); + cpumask_any_and(irq_data_get_affinity_mask(d), + cpu_online_mask); CROSS_GxICR(irq, irq_affinity_online[irq]) = (tmp & (GxICR_LEVEL | GxICR_ENABLE)) | GxICR_DETECT; tmp = CROSS_GxICR(irq, irq_affinity_online[irq]); @@ -124,7 +125,7 @@ static void mn10300_cpupic_unmask_clear(struct irq_data *d) } else { tmp = GxICR(irq); - irq_affinity_online[irq] = cpumask_any_and(d->affinity, + irq_affinity_online[irq] = cpumask_any_and(irq_data_get_affinity_mask(d), cpu_online_mask); CROSS_GxICR(irq, irq_affinity_online[irq]) = (tmp & GxICR_LEVEL) | GxICR_ENABLE | GxICR_DETECT; tmp = CROSS_GxICR(irq, irq_affinity_online[irq]); @@ -316,15 +317,16 @@ void migrate_irqs(void) self = smp_processor_id(); for (irq = 0; irq < NR_IRQS; irq++) { struct irq_data *data = irq_get_irq_data(irq); + struct cpumask *mask = irq_data_get_affinity_mask(data); if (irqd_is_per_cpu(data)) continue; - if (cpumask_test_cpu(self, data->affinity) && + if (cpumask_test_cpu(self, mask) && !cpumask_intersects(&irq_affinity[irq], cpu_online_mask)) { int cpu_id; cpu_id = cpumask_first(cpu_online_mask); - cpumask_set_cpu(cpu_id, data->affinity); + cpumask_set_cpu(cpu_id, mask); } /* We need to operate irq_affinity_online atomically. */ arch_local_cli_save(flags); @@ -335,8 +337,7 @@ void migrate_irqs(void) GxICR(irq) = x & GxICR_LEVEL; tmp = GxICR(irq); - new = cpumask_any_and(data->affinity, - cpu_online_mask); + new = cpumask_any_and(mask, cpu_online_mask); irq_affinity_online[irq] = new; CROSS_GxICR(irq, new) = diff --git a/arch/mn10300/mm/tlb-smp.c b/arch/mn10300/mm/tlb-smp.c index e5d0ef722bfa..9a39ea9031d4 100644 --- a/arch/mn10300/mm/tlb-smp.c +++ b/arch/mn10300/mm/tlb-smp.c @@ -119,7 +119,7 @@ static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, flush_mm = mm; flush_va = va; #if NR_CPUS <= BITS_PER_LONG - atomic_set_mask(cpumask.bits[0], &flush_cpumask.bits[0]); + atomic_or(cpumask.bits[0], (atomic_t *)&flush_cpumask.bits[0]); #else #error Not supported. #endif diff --git a/arch/nios2/boot/dts/10m50_devboard.dts b/arch/nios2/boot/dts/10m50_devboard.dts new file mode 100755 index 000000000000..3e411c644824 --- /dev/null +++ b/arch/nios2/boot/dts/10m50_devboard.dts @@ -0,0 +1,248 @@ +/* + * Copyright (C) 2015 Altera Corporation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see . + */ + +/dts-v1/; + +/ { + model = "Altera NiosII Max10"; + compatible = "altr,niosii-max10"; + #address-cells = <1>; + #size-cells = <1>; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu: cpu@0 { + device_type = "cpu"; + compatible = "altr,nios2-1.1"; + reg = <0x00000000>; + interrupt-controller; + #interrupt-cells = <1>; + altr,exception-addr = <0xc8000120>; + altr,fast-tlb-miss-addr = <0xc0000100>; + altr,has-div = <1>; + altr,has-initda = <1>; + altr,has-mmu = <1>; + altr,has-mul = <1>; + altr,implementation = "fast"; + altr,pid-num-bits = <8>; + altr,reset-addr = <0xd4000000>; + altr,tlb-num-entries = <256>; + altr,tlb-num-ways = <16>; + altr,tlb-ptr-sz = <8>; + clock-frequency = <75000000>; + dcache-line-size = <32>; + dcache-size = <32768>; + icache-line-size = <32>; + icache-size = <32768>; + }; + }; + + memory { + device_type = "memory"; + reg = <0x08000000 0x08000000>, + <0x00000000 0x00000400>; + }; + + sopc0: sopc@0 { + device_type = "soc"; + ranges; + #address-cells = <1>; + #size-cells = <1>; + compatible = "altr,avalon", "simple-bus"; + bus-frequency = <75000000>; + + jtag_uart: serial@18001530 { + compatible = "altr,juart-1.0"; + reg = <0x18001530 0x00000008>; + interrupt-parent = <&cpu>; + interrupts = <7>; + }; + + a_16550_uart_0: serial@18001600 { + compatible = "altr,16550-FIFO32", "ns16550a"; + reg = <0x18001600 0x00000200>; + interrupt-parent = <&cpu>; + interrupts = <1>; + auto-flow-control = <1>; + clock-frequency = <50000000>; + fifo-size = <32>; + reg-io-width = <4>; + reg-shift = <2>; + }; + + sysid: sysid@18001528 { + compatible = "altr,sysid-1.0"; + reg = <0x18001528 0x00000008>; + id = <4207856382>; + timestamp = <1431309290>; + }; + + rgmii_0_eth_tse_0: ethernet@400 { + compatible = "altr,tse-msgdma-1.0", "altr,tse-1.0"; + reg = <0x00000400 0x00000400>, + <0x00000820 0x00000020>, + <0x00000800 0x00000020>, + <0x000008c0 0x00000008>, + <0x00000840 0x00000020>, + <0x00000860 0x00000020>; + reg-names = "control_port", "rx_csr", "rx_desc", "rx_resp", "tx_csr", "tx_desc"; + interrupt-parent = <&cpu>; + interrupts = <2 3>; + interrupt-names = "rx_irq", "tx_irq"; + rx-fifo-depth = <8192>; + tx-fifo-depth = <8192>; + address-bits = <48>; + max-frame-size = <1518>; + local-mac-address = [00 00 00 00 00 00]; + altr,has-supplementary-unicast; + altr,enable-sup-addr = <1>; + altr,has-hash-multicast-filter; + altr,enable-hash = <1>; + phy-mode = "rgmii-id"; + phy-handle = <&phy0>; + rgmii_0_eth_tse_0_mdio: mdio { + compatible = "altr,tse-mdio"; + #address-cells = <1>; + #size-cells = <0>; + phy0: ethernet-phy@0 { + reg = <0>; + device_type = "ethernet-phy"; + }; + }; + }; + + enet_pll: clock@0 { + compatible = "altr,pll-1.0"; + #clock-cells = <1>; + + enet_pll_c0: enet_pll_c0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <125000000>; + clock-output-names = "enet_pll-c0"; + }; + + enet_pll_c1: enet_pll_c1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <25000000>; + clock-output-names = "enet_pll-c1"; + }; + + enet_pll_c2: enet_pll_c2 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <2500000>; + clock-output-names = "enet_pll-c2"; + }; + }; + + sys_pll: clock@1 { + compatible = "altr,pll-1.0"; + #clock-cells = <1>; + + sys_pll_c0: sys_pll_c0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <100000000>; + clock-output-names = "sys_pll-c0"; + }; + + sys_pll_c1: sys_pll_c1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <50000000>; + clock-output-names = "sys_pll-c1"; + }; + + sys_pll_c2: sys_pll_c2 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <75000000>; + clock-output-names = "sys_pll-c2"; + }; + }; + + sys_clk_timer: timer@18001440 { + compatible = "altr,timer-1.0"; + reg = <0x18001440 0x00000020>; + interrupt-parent = <&cpu>; + interrupts = <0>; + clock-frequency = <75000000>; + }; + + led_pio: gpio@180014d0 { + compatible = "altr,pio-1.0"; + reg = <0x180014d0 0x00000010>; + altr,gpio-bank-width = <4>; + resetvalue = <15>; + #gpio-cells = <2>; + gpio-controller; + }; + + button_pio: gpio@180014c0 { + compatible = "altr,pio-1.0"; + reg = <0x180014c0 0x00000010>; + interrupt-parent = <&cpu>; + interrupts = <6>; + altr,gpio-bank-width = <3>; + altr,interrupt-type = <2>; + edge_type = <1>; + level_trigger = <0>; + resetvalue = <0>; + #gpio-cells = <2>; + gpio-controller; + }; + + sys_clk_timer_1: timer@880 { + compatible = "altr,timer-1.0"; + reg = <0x00000880 0x00000020>; + interrupt-parent = <&cpu>; + interrupts = <5>; + clock-frequency = <75000000>; + }; + + fpga_leds: leds { + compatible = "gpio-leds"; + + led_fpga0: fpga0 { + label = "fpga_led0"; + gpios = <&led_pio 0 1>; + }; + + led_fpga1: fpga1 { + label = "fpga_led1"; + gpios = <&led_pio 1 1>; + }; + + led_fpga2: fpga2 { + label = "fpga_led2"; + gpios = <&led_pio 2 1>; + }; + + led_fpga3: fpga3 { + label = "fpga_led3"; + gpios = <&led_pio 3 1>; + }; + }; + }; + + chosen { + bootargs = "debug console=ttyS0,115200"; + }; +}; diff --git a/arch/nios2/configs/10m50_defconfig b/arch/nios2/configs/10m50_defconfig new file mode 100755 index 000000000000..8b2a30b3b34f --- /dev/null +++ b/arch/nios2/configs/10m50_defconfig @@ -0,0 +1,81 @@ +CONFIG_SYSVIPC=y +CONFIG_NO_HZ_IDLE=y +CONFIG_BSD_PROCESS_ACCT=y +CONFIG_LOG_BUF_SHIFT=14 +CONFIG_SYSCTL_SYSCALL=y +# CONFIG_ELF_CORE is not set +# CONFIG_EPOLL is not set +# CONFIG_SIGNALFD is not set +# CONFIG_TIMERFD is not set +# CONFIG_EVENTFD is not set +# CONFIG_SHMEM is not set +# CONFIG_AIO is not set +CONFIG_EMBEDDED=y +CONFIG_SLAB=y +CONFIG_MODULES=y +CONFIG_MODULE_UNLOAD=y +CONFIG_NIOS2_MEM_BASE=0x8000000 +CONFIG_NIOS2_HW_MUL_SUPPORT=y +CONFIG_NIOS2_HW_DIV_SUPPORT=y +CONFIG_CUSTOM_CACHE_SETTINGS=y +CONFIG_NIOS2_DCACHE_SIZE=0x8000 +CONFIG_NIOS2_ICACHE_SIZE=0x8000 +# CONFIG_NIOS2_CMDLINE_IGNORE_DTB is not set +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_UNIX=y +CONFIG_INET=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_IP_PNP_BOOTP=y +CONFIG_IP_PNP_RARP=y +# CONFIG_INET_XFRM_MODE_TRANSPORT is not set +# CONFIG_INET_XFRM_MODE_TUNNEL is not set +# CONFIG_INET_XFRM_MODE_BEET is not set +# CONFIG_INET_LRO is not set +# CONFIG_IPV6 is not set +# CONFIG_WIRELESS is not set +CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +# CONFIG_FW_LOADER is not set +CONFIG_MTD=y +CONFIG_MTD_CMDLINE_PARTS=y +CONFIG_MTD_BLOCK=y +CONFIG_MTD_CFI=y +CONFIG_MTD_CFI_INTELEXT=y +CONFIG_MTD_CFI_AMDSTD=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_NETDEVICES=y +CONFIG_ALTERA_TSE=y +CONFIG_MARVELL_PHY=y +# CONFIG_WLAN is not set +# CONFIG_INPUT_MOUSE is not set +# CONFIG_SERIO_SERPORT is not set +# CONFIG_VT is not set +CONFIG_SERIAL_8250=y +# CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_SERIAL_OF_PLATFORM=y +CONFIG_SERIAL_ALTERA_JTAGUART=y +# CONFIG_HW_RANDOM is not set +CONFIG_GPIOLIB=y +CONFIG_GPIO_SYSFS=y +CONFIG_GPIO_ALTERA=y +# CONFIG_HWMON is not set +# CONFIG_USB_SUPPORT is not set +CONFIG_NEW_LEDS=y +CONFIG_LEDS_CLASS=y +CONFIG_LEDS_GPIO=y +CONFIG_LEDS_TRIGGERS=y +CONFIG_LEDS_TRIGGER_HEARTBEAT=y +# CONFIG_DNOTIFY is not set +# CONFIG_INOTIFY_USER is not set +CONFIG_JFFS2_FS=y +CONFIG_NFS_FS=y +CONFIG_NFS_V3_ACL=y +CONFIG_ROOT_NFS=y +CONFIG_SUNRPC_DEBUG=y +CONFIG_DEBUG_INFO=y +# CONFIG_ENABLE_WARN_DEPRECATED is not set diff --git a/arch/nios2/kernel/misaligned.c b/arch/nios2/kernel/misaligned.c index 4e5907a0cabe..23e0544e117c 100644 --- a/arch/nios2/kernel/misaligned.c +++ b/arch/nios2/kernel/misaligned.c @@ -32,8 +32,6 @@ #define INST_STW 0x15 #define INST_LDW 0x17 -static unsigned long ma_user, ma_kern, ma_skipped, ma_half, ma_word; - static unsigned int ma_usermode; #define UM_WARN 0x01 #define UM_FIXUP 0x02 @@ -53,7 +51,6 @@ static int reg_offsets[32]; static inline u32 get_reg_val(struct pt_regs *fp, int reg) { u8 *p = ((u8 *)fp) + reg_offsets[reg]; - return *(u32 *)p; } @@ -71,14 +68,13 @@ asmlinkage void handle_unaligned_c(struct pt_regs *fp, int cause) u32 isn, addr, val; int in_kernel; u8 a, b, d0, d1, d2, d3; - u16 imm16; + s16 imm16; unsigned int fault; /* back up one instruction */ fp->ea -= 4; if (fixup_exception(fp)) { - ma_skipped++; return; } @@ -103,18 +99,11 @@ asmlinkage void handle_unaligned_c(struct pt_regs *fp, int cause) fault |= __get_user(d1, (u8 *)(addr+1)); val = (d1 << 8) | d0; put_reg_val(fp, b, val); - ma_half++; break; case INST_STH: val = get_reg_val(fp, b); d1 = val >> 8; d0 = val >> 0; - - pr_debug("sth: ra=%d (%08x) rb=%d (%08x), imm16 %04x addr %08x val %08x\n", - a, get_reg_val(fp, a), - b, get_reg_val(fp, b), - imm16, addr, val); - if (in_kernel) { *(u8 *)(addr+0) = d0; *(u8 *)(addr+1) = d1; @@ -122,14 +111,12 @@ asmlinkage void handle_unaligned_c(struct pt_regs *fp, int cause) fault |= __put_user(d0, (u8 *)(addr+0)); fault |= __put_user(d1, (u8 *)(addr+1)); } - ma_half++; break; case INST_LDH: fault |= __get_user(d0, (u8 *)(addr+0)); fault |= __get_user(d1, (u8 *)(addr+1)); val = (short)((d1 << 8) | d0); put_reg_val(fp, b, val); - ma_half++; break; case INST_STW: val = get_reg_val(fp, b); @@ -148,7 +135,6 @@ asmlinkage void handle_unaligned_c(struct pt_regs *fp, int cause) fault |= __put_user(d2, (u8 *)(addr+2)); fault |= __put_user(d3, (u8 *)(addr+3)); } - ma_word++; break; case INST_LDW: fault |= __get_user(d0, (u8 *)(addr+0)); @@ -157,7 +143,6 @@ asmlinkage void handle_unaligned_c(struct pt_regs *fp, int cause) fault |= __get_user(d3, (u8 *)(addr+3)); val = (d3 << 24) | (d2 << 16) | (d1 << 8) | d0; put_reg_val(fp, b, val); - ma_word++; break; } } @@ -186,7 +171,6 @@ asmlinkage void handle_unaligned_c(struct pt_regs *fp, int cause) * note exception and skip bad instruction (return) */ if (in_kernel) { - ma_kern++; fp->ea += 4; if (ma_usermode & KM_WARN) { @@ -200,8 +184,6 @@ asmlinkage void handle_unaligned_c(struct pt_regs *fp, int cause) return; } - ma_user++; - /* * user mode - * possibly warn, diff --git a/arch/nios2/kernel/time.c b/arch/nios2/kernel/time.c index 9e3cc8a40ee9..bbc3f9157f9c 100644 --- a/arch/nios2/kernel/time.c +++ b/arch/nios2/kernel/time.c @@ -130,7 +130,7 @@ static void nios2_timer_stop(struct nios2_timer *timer) } static void nios2_timer_config(struct nios2_timer *timer, unsigned long period, - enum clock_event_mode mode) + bool periodic) { u16 ctrl; @@ -148,7 +148,7 @@ static void nios2_timer_config(struct nios2_timer *timer, unsigned long period, timer_writew(timer, period >> 16, ALTERA_TIMER_PERIODH_REG); ctrl |= ALTERA_TIMER_CONTROL_START_MSK | ALTERA_TIMER_CONTROL_ITO_MSK; - if (mode == CLOCK_EVT_MODE_PERIODIC) + if (periodic) ctrl |= ALTERA_TIMER_CONTROL_CONT_MSK; else ctrl &= ~ALTERA_TIMER_CONTROL_CONT_MSK; @@ -160,32 +160,38 @@ static int nios2_timer_set_next_event(unsigned long delta, { struct nios2_clockevent_dev *nios2_ced = to_nios2_clkevent(evt); - nios2_timer_config(&nios2_ced->timer, delta, evt->mode); + nios2_timer_config(&nios2_ced->timer, delta, false); return 0; } -static void nios2_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int nios2_timer_shutdown(struct clock_event_device *evt) +{ + struct nios2_clockevent_dev *nios2_ced = to_nios2_clkevent(evt); + struct nios2_timer *timer = &nios2_ced->timer; + + nios2_timer_stop(timer); + return 0; +} + +static int nios2_timer_set_periodic(struct clock_event_device *evt) { unsigned long period; struct nios2_clockevent_dev *nios2_ced = to_nios2_clkevent(evt); struct nios2_timer *timer = &nios2_ced->timer; - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - period = DIV_ROUND_UP(timer->freq, HZ); - nios2_timer_config(timer, period, CLOCK_EVT_MODE_PERIODIC); - break; - case CLOCK_EVT_MODE_ONESHOT: - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - nios2_timer_stop(timer); - break; - case CLOCK_EVT_MODE_RESUME: - nios2_timer_start(timer); - break; - } + period = DIV_ROUND_UP(timer->freq, HZ); + nios2_timer_config(timer, period, true); + return 0; +} + +static int nios2_timer_resume(struct clock_event_device *evt) +{ + struct nios2_clockevent_dev *nios2_ced = to_nios2_clkevent(evt); + struct nios2_timer *timer = &nios2_ced->timer; + + nios2_timer_start(timer); + return 0; } irqreturn_t timer_interrupt(int irq, void *dev_id) @@ -218,7 +224,10 @@ static struct nios2_clockevent_dev nios2_ce = { .rating = 250, .shift = 32, .set_next_event = nios2_timer_set_next_event, - .set_mode = nios2_timer_set_mode, + .set_state_shutdown = nios2_timer_shutdown, + .set_state_periodic = nios2_timer_set_periodic, + .set_state_oneshot = nios2_timer_shutdown, + .tick_resume = nios2_timer_resume, }, }; diff --git a/arch/openrisc/include/asm/dma-mapping.h b/arch/openrisc/include/asm/dma-mapping.h index fab8628e1b6e..413bfcf86384 100644 --- a/arch/openrisc/include/asm/dma-mapping.h +++ b/arch/openrisc/include/asm/dma-mapping.h @@ -23,7 +23,6 @@ */ #include -#include #include #include @@ -36,75 +35,13 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev) return &or1k_dma_map_ops; } -#include - -#define dma_alloc_coherent(d,s,h,f) dma_alloc_attrs(d,s,h,f,NULL) - -static inline void *dma_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - void *memory; - - memory = ops->alloc(dev, size, dma_handle, gfp, attrs); - - debug_dma_alloc_coherent(dev, size, *dma_handle, memory); - - return memory; -} - -#define dma_free_coherent(d,s,c,h) dma_free_attrs(d,s,c,h,NULL) - -static inline void dma_free_attrs(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t dma_handle, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - - debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); - - ops->free(dev, size, cpu_addr, dma_handle, attrs); -} - -static inline void *dma_alloc_noncoherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp) -{ - struct dma_attrs attrs; - - dma_set_attr(DMA_ATTR_NON_CONSISTENT, &attrs); - - return dma_alloc_attrs(dev, size, dma_handle, gfp, &attrs); -} - -static inline void dma_free_noncoherent(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t dma_handle) -{ - struct dma_attrs attrs; - - dma_set_attr(DMA_ATTR_NON_CONSISTENT, &attrs); - - dma_free_attrs(dev, size, cpu_addr, dma_handle, &attrs); -} - +#define HAVE_ARCH_DMA_SUPPORTED 1 static inline int dma_supported(struct device *dev, u64 dma_mask) { /* Support 32 bit DMA mask exclusively */ return dma_mask == DMA_BIT_MASK(32); } -static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - return 0; -} +#include -static inline int dma_set_mask(struct device *dev, u64 dma_mask) -{ - if (!dev->dma_mask || !dma_supported(dev, dma_mask)) - return -EIO; - - *dev->dma_mask = dma_mask; - - return 0; -} #endif /* __ASM_OPENRISC_DMA_MAPPING_H */ diff --git a/arch/openrisc/kernel/time.c b/arch/openrisc/kernel/time.c index 7c52e9494a8d..50e970183dcd 100644 --- a/arch/openrisc/kernel/time.c +++ b/arch/openrisc/kernel/time.c @@ -48,29 +48,6 @@ static int openrisc_timer_set_next_event(unsigned long delta, return 0; } -static void openrisc_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) -{ - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - pr_debug(KERN_INFO "%s: periodic\n", __func__); - BUG(); - break; - case CLOCK_EVT_MODE_ONESHOT: - pr_debug(KERN_INFO "%s: oneshot\n", __func__); - break; - case CLOCK_EVT_MODE_UNUSED: - pr_debug(KERN_INFO "%s: unused\n", __func__); - break; - case CLOCK_EVT_MODE_SHUTDOWN: - pr_debug(KERN_INFO "%s: shutdown\n", __func__); - break; - case CLOCK_EVT_MODE_RESUME: - pr_debug(KERN_INFO "%s: resume\n", __func__); - break; - } -} - /* This is the clock event device based on the OR1K tick timer. * As the timer is being used as a continuous clock-source (required for HR * timers) we cannot enable the PERIODIC feature. The tick timer can run using @@ -82,7 +59,6 @@ static struct clock_event_device clockevent_openrisc_timer = { .features = CLOCK_EVT_FEAT_ONESHOT, .rating = 300, .set_next_event = openrisc_timer_set_next_event, - .set_mode = openrisc_timer_set_mode, }; static inline void timer_ack(void) diff --git a/arch/parisc/configs/c8000_defconfig b/arch/parisc/configs/c8000_defconfig index 269c23d23fcb..1a8f6f95689e 100644 --- a/arch/parisc/configs/c8000_defconfig +++ b/arch/parisc/configs/c8000_defconfig @@ -242,7 +242,6 @@ CONFIG_LOCKUP_DETECTOR=y CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=y CONFIG_PANIC_ON_OOPS=y CONFIG_DEBUG_RT_MUTEXES=y -CONFIG_RT_MUTEX_TESTER=y CONFIG_PROVE_RCU_DELAY=y CONFIG_DEBUG_BLOCK_EXT_DEVT=y CONFIG_LATENCYTOP=y diff --git a/arch/parisc/configs/generic-32bit_defconfig b/arch/parisc/configs/generic-32bit_defconfig index 33b148f825ba..0ffb08ff5125 100644 --- a/arch/parisc/configs/generic-32bit_defconfig +++ b/arch/parisc/configs/generic-32bit_defconfig @@ -295,7 +295,6 @@ CONFIG_DEBUG_SHIRQ=y CONFIG_DETECT_HUNG_TASK=y CONFIG_TIMER_STATS=y CONFIG_DEBUG_RT_MUTEXES=y -CONFIG_RT_MUTEX_TESTER=y CONFIG_DEBUG_SPINLOCK=y CONFIG_DEBUG_MUTEXES=y CONFIG_RCU_CPU_STALL_INFO=y diff --git a/arch/parisc/include/asm/atomic.h b/arch/parisc/include/asm/atomic.h index 226f8ca993f6..2536965d00ea 100644 --- a/arch/parisc/include/asm/atomic.h +++ b/arch/parisc/include/asm/atomic.h @@ -126,6 +126,10 @@ static __inline__ int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add, +=) ATOMIC_OPS(sub, -=) +ATOMIC_OP(and, &=) +ATOMIC_OP(or, |=) +ATOMIC_OP(xor, ^=) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -185,6 +189,9 @@ static __inline__ s64 atomic64_##op##_return(s64 i, atomic64_t *v) \ ATOMIC64_OPS(add, +=) ATOMIC64_OPS(sub, -=) +ATOMIC64_OP(and, &=) +ATOMIC64_OP(or, |=) +ATOMIC64_OP(xor, ^=) #undef ATOMIC64_OPS #undef ATOMIC64_OP_RETURN diff --git a/arch/parisc/include/asm/io.h b/arch/parisc/include/asm/io.h index 8cd0abf28ffb..1a16f1d1075f 100644 --- a/arch/parisc/include/asm/io.h +++ b/arch/parisc/include/asm/io.h @@ -137,6 +137,8 @@ static inline void __iomem * ioremap(unsigned long offset, unsigned long size) return __ioremap(offset, size, _PAGE_NO_CACHE); } #define ioremap_nocache(off, sz) ioremap((off), (sz)) +#define ioremap_wc ioremap_nocache +#define ioremap_uc ioremap_nocache extern void iounmap(const volatile void __iomem *addr); diff --git a/arch/parisc/kernel/irq.c b/arch/parisc/kernel/irq.c index f3191db6e2e9..ba5e1c7b1f17 100644 --- a/arch/parisc/kernel/irq.c +++ b/arch/parisc/kernel/irq.c @@ -131,7 +131,7 @@ static int cpu_set_affinity_irq(struct irq_data *d, const struct cpumask *dest, if (cpu_dest < 0) return -1; - cpumask_copy(d->affinity, dest); + cpumask_copy(irq_data_get_affinity_mask(d), dest); return 0; } @@ -339,7 +339,7 @@ unsigned long txn_affinity_addr(unsigned int irq, int cpu) { #ifdef CONFIG_SMP struct irq_data *d = irq_get_irq_data(irq); - cpumask_copy(d->affinity, cpumask_of(cpu)); + cpumask_copy(irq_data_get_affinity_mask(d), cpumask_of(cpu)); #endif return per_cpu(cpu_data, cpu).txn_addr; @@ -507,8 +507,8 @@ void do_cpu_irq_mask(struct pt_regs *regs) struct pt_regs *old_regs; unsigned long eirr_val; int irq, cpu = smp_processor_id(); + struct irq_data *irq_data; #ifdef CONFIG_SMP - struct irq_desc *desc; cpumask_t dest; #endif @@ -521,10 +521,15 @@ void do_cpu_irq_mask(struct pt_regs *regs) goto set_out; irq = eirr_to_irq(eirr_val); + irq_data = irq_get_irq_data(irq); + + /* Filter out spurious interrupts, mostly from serial port at bootup */ + if (unlikely(!irq_desc_has_action(irq_data_to_desc(irq_data)))) + goto set_out; + #ifdef CONFIG_SMP - desc = irq_to_desc(irq); - cpumask_copy(&dest, desc->irq_data.affinity); - if (irqd_is_per_cpu(&desc->irq_data) && + cpumask_copy(&dest, irq_data_get_affinity_mask(irq_data)); + if (irqd_is_per_cpu(irq_data) && !cpumask_test_cpu(smp_processor_id(), &dest)) { int cpu = cpumask_first(&dest); diff --git a/arch/parisc/kernel/syscall.S b/arch/parisc/kernel/syscall.S index 7ef22e3387e0..0b8d26d3ba43 100644 --- a/arch/parisc/kernel/syscall.S +++ b/arch/parisc/kernel/syscall.S @@ -821,7 +821,7 @@ cas2_action: /* 64bit CAS */ #ifdef CONFIG_64BIT 19: ldd,ma 0(%sr3,%r26), %r29 - sub,= %r29, %r25, %r0 + sub,*= %r29, %r25, %r0 b,n cas2_end 20: std,ma %r24, 0(%sr3,%r26) copy %r0, %r28 diff --git a/arch/parisc/kernel/time.c b/arch/parisc/kernel/time.c index 70e105d62423..400acac0a304 100644 --- a/arch/parisc/kernel/time.c +++ b/arch/parisc/kernel/time.c @@ -202,7 +202,6 @@ static struct clocksource clocksource_cr16 = { .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; -#ifdef CONFIG_SMP int update_cr16_clocksource(void) { /* since the cr16 cycle counters are not synchronized across CPUs, @@ -214,12 +213,6 @@ int update_cr16_clocksource(void) return 0; } -#else -int update_cr16_clocksource(void) -{ - return 0; /* no change */ -} -#endif /*CONFIG_SMP*/ void __init start_cpu_itimer(void) { @@ -231,20 +224,14 @@ void __init start_cpu_itimer(void) per_cpu(cpu_data, cpu).it_value = next_tick; } -static struct platform_device rtc_generic_dev = { - .name = "rtc-generic", - .id = -1, -}; - static int __init rtc_init(void) { - if (platform_device_register(&rtc_generic_dev) < 0) - printk(KERN_ERR "unable to register rtc device...\n"); + struct platform_device *pdev; - /* not necessarily an error */ - return 0; + pdev = platform_device_register_simple("rtc-generic", -1, NULL, 0); + return PTR_ERR_OR_ZERO(pdev); } -module_init(rtc_init); +device_initcall(rtc_init); void read_persistent_clock(struct timespec *ts) { diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c index 15503adddf4f..a762864ec92e 100644 --- a/arch/parisc/mm/fault.c +++ b/arch/parisc/mm/fault.c @@ -207,7 +207,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long code, int fault; unsigned int flags; - if (pagefault_disabled()) + if (faulthandler_disabled()) goto no_context; tsk = current; diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 5ef27113b898..9a7057ec2154 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -82,6 +82,9 @@ config GENERIC_HWEIGHT bool default y +config ARCH_HAS_DMA_SET_COHERENT_MASK + bool + config PPC bool default y @@ -155,6 +158,8 @@ config PPC select HAVE_PERF_EVENTS_NMI if PPC64 select EDAC_SUPPORT select EDAC_ATOMIC_SCRUB + select ARCH_HAS_DMA_SET_COHERENT_MASK + select HAVE_ARCH_SECCOMP_FILTER config GENERIC_CSUM def_bool CPU_LITTLE_ENDIAN @@ -415,6 +420,7 @@ config PPC64_SUPPORTS_MEMORY_FAILURE config KEXEC bool "kexec system call" depends on (PPC_BOOK3S || FSL_BOOKE || (44x && !SMP)) + select KEXEC_CORE help kexec is a system call that implements the ability to shutdown your current kernel, and to start another kernel. It is like a reboot @@ -514,11 +520,6 @@ config NODES_SPAN_OTHER_NODES def_bool y depends on NEED_MULTIPLE_NODES -config PPC_HAS_HASH_64K - bool - depends on PPC64 - default n - config STDBINUTILS bool "Using standard binutils settings" depends on 44x @@ -560,16 +561,16 @@ config PPC_4K_PAGES bool "4k page size" config PPC_16K_PAGES - bool "16k page size" if 44x || PPC_8xx + bool "16k page size" + depends on 44x || PPC_8xx config PPC_64K_PAGES - bool "64k page size" if 44x || PPC_STD_MMU_64 || PPC_BOOK3E_64 - depends on !PPC_FSL_BOOK3E - select PPC_HAS_HASH_64K if PPC_STD_MMU_64 + bool "64k page size" + depends on !PPC_FSL_BOOK3E && (44x || PPC_STD_MMU_64 || PPC_BOOK3E_64) config PPC_256K_PAGES - bool "256k page size" if 44x - depends on !STDBINUTILS + bool "256k page size" + depends on 44x && !STDBINUTILS help Make the page size 256k. diff --git a/arch/powerpc/Makefile b/arch/powerpc/Makefile index 05f464eb6952..b9b4af2af9a5 100644 --- a/arch/powerpc/Makefile +++ b/arch/powerpc/Makefile @@ -67,7 +67,7 @@ UTS_MACHINE := $(OLDARCH) ifeq ($(CONFIG_CPU_LITTLE_ENDIAN),y) override CC += -mlittle-endian -ifneq ($(COMPILER),clang) +ifneq ($(cc-name),clang) override CC += -mno-strict-align endif override AS += -mlittle-endian @@ -288,6 +288,26 @@ PHONY += pseries_le_defconfig pseries_le_defconfig: $(call merge_into_defconfig,pseries_defconfig,le) +PHONY += mpc85xx_defconfig +mpc85xx_defconfig: + $(call merge_into_defconfig,mpc85xx_basic_defconfig,\ + 85xx-32bit 85xx-hw fsl-emb-nonhw) + +PHONY += mpc85xx_smp_defconfig +mpc85xx_smp_defconfig: + $(call merge_into_defconfig,mpc85xx_basic_defconfig,\ + 85xx-32bit 85xx-smp 85xx-hw fsl-emb-nonhw) + +PHONY += corenet32_smp_defconfig +corenet32_smp_defconfig: + $(call merge_into_defconfig,corenet_basic_defconfig,\ + 85xx-32bit 85xx-smp 85xx-hw fsl-emb-nonhw) + +PHONY += corenet64_smp_defconfig +corenet64_smp_defconfig: + $(call merge_into_defconfig,corenet_basic_defconfig,\ + 85xx-64bit 85xx-smp altivec 85xx-hw fsl-emb-nonhw) + define archhelp @echo '* zImage - Build default images selected by kernel config' @echo ' zImage.* - Compressed kernel image (arch/$(ARCH)/boot/zImage.*)' @@ -333,7 +353,7 @@ TOUT := .tmp_gas_check # - Require gcc 4.0 or above on 64-bit # - gcc-4.2.0 has issues compiling modules on 64-bit checkbin: - @if test "${COMPILER}" != "clang" \ + @if test "$(cc-name)" != "clang" \ && test "$(cc-version)" = "0304" ; then \ if ! /bin/echo mftb 5 | $(AS) -v -mppc -many -o $(TOUT) >/dev/null 2>&1 ; then \ echo -n '*** ${VERSION}.${PATCHLEVEL} kernels no longer build '; \ @@ -342,14 +362,14 @@ checkbin: false; \ fi ; \ fi - @if test "${COMPILER}" != "clang" \ + @if test "$(cc-name)" != "clang" \ && test "$(cc-version)" -lt "0400" \ && test "x${CONFIG_PPC64}" = "xy" ; then \ echo -n "Sorry, GCC v4.0 or above is required to build " ; \ echo "the 64-bit powerpc kernel." ; \ false ; \ fi - @if test "${COMPILER}" != "clang" \ + @if test "$(cc-name)" != "clang" \ && test "$(cc-fullversion)" = "040200" \ && test "x${CONFIG_MODULES}${CONFIG_PPC64}" = "xyy" ; then \ echo -n '*** GCC-4.2.0 cannot compile the 64-bit powerpc ' ; \ diff --git a/arch/powerpc/boot/dts/fsl/p1022si-post.dtsi b/arch/powerpc/boot/dts/fsl/p1022si-post.dtsi index ebf202234549..426bf4103b9e 100644 --- a/arch/powerpc/boot/dts/fsl/p1022si-post.dtsi +++ b/arch/powerpc/boot/dts/fsl/p1022si-post.dtsi @@ -175,7 +175,7 @@ dma01: dma-channel@80 { /include/ "pq3-gpio-0.dtsi" - display@10000 { + display: display@10000 { compatible = "fsl,diu", "fsl,p1022-diu"; reg = <0x10000 1000>; interrupts = <64 2 0 0>; diff --git a/arch/powerpc/boot/dts/fsl/p1022si-pre.dtsi b/arch/powerpc/boot/dts/fsl/p1022si-pre.dtsi index 1956dea040cc..de76ae8992c6 100644 --- a/arch/powerpc/boot/dts/fsl/p1022si-pre.dtsi +++ b/arch/powerpc/boot/dts/fsl/p1022si-pre.dtsi @@ -50,6 +50,8 @@ aliases { pci0 = &pci0; pci1 = &pci1; pci2 = &pci2; + vga = &display; + display = &display; }; cpus { diff --git a/arch/powerpc/boot/dts/fsl/t1040si-post.dtsi b/arch/powerpc/boot/dts/fsl/t1040si-post.dtsi index 9e9f7e201d43..9770d0278493 100644 --- a/arch/powerpc/boot/dts/fsl/t1040si-post.dtsi +++ b/arch/powerpc/boot/dts/fsl/t1040si-post.dtsi @@ -484,6 +484,11 @@ serdes: serdes@ea000 { reg = <0xea000 0x4000>; }; + scfg: global-utilities@fc000 { + compatible = "fsl,t1040-scfg"; + reg = <0xfc000 0x1000>; + }; + /include/ "elo3-dma-0.dtsi" /include/ "elo3-dma-1.dtsi" /include/ "qoriq-espi-0.dtsi" diff --git a/arch/powerpc/boot/dts/t1023rdb.dts b/arch/powerpc/boot/dts/t1023rdb.dts index 06b090aba066..d3fa8294cd49 100644 --- a/arch/powerpc/boot/dts/t1023rdb.dts +++ b/arch/powerpc/boot/dts/t1023rdb.dts @@ -60,7 +60,7 @@ nand@1,0 { #address-cells = <1>; #size-cells = <1>; compatible = "fsl,ifc-nand"; - reg = <0x2 0x0 0x10000>; + reg = <0x1 0x0 0x10000>; }; }; @@ -99,6 +99,17 @@ rtc@68 { }; i2c@118100 { + current-sensor@40 { + compatible = "ti,ina220"; + reg = <0x40>; + shunt-resistor = <1000>; + }; + + current-sensor@41 { + compatible = "ti,ina220"; + reg = <0x41>; + shunt-resistor = <1000>; + }; }; }; diff --git a/arch/powerpc/boot/dts/t1024rdb.dts b/arch/powerpc/boot/dts/t1024rdb.dts index 733e723ffed6..bf05e324fda2 100644 --- a/arch/powerpc/boot/dts/t1024rdb.dts +++ b/arch/powerpc/boot/dts/t1024rdb.dts @@ -114,6 +114,12 @@ adt7461@4c { reg = <0x4c>; }; + current-sensor@40 { + compatible = "ti,ina220"; + reg = <0x40>; + shunt-resistor = <1000>; + }; + eeprom@50 { compatible = "atmel,24c256"; reg = <0x50>; diff --git a/arch/powerpc/boot/dts/t1040d4rdb.dts b/arch/powerpc/boot/dts/t1040d4rdb.dts new file mode 100644 index 000000000000..2d1315a1670e --- /dev/null +++ b/arch/powerpc/boot/dts/t1040d4rdb.dts @@ -0,0 +1,46 @@ +/* + * T1040D4RDB Device Tree Source + * + * Copyright 2015 Freescale Semiconductor Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/include/ "fsl/t104xsi-pre.dtsi" +/include/ "t104xd4rdb.dtsi" + +/ { + model = "fsl,T1040D4RDB"; + compatible = "fsl,T1040D4RDB"; + #address-cells = <2>; + #size-cells = <2>; + interrupt-parent = <&mpic>; +}; + +/include/ "fsl/t1040si-post.dtsi" diff --git a/arch/powerpc/boot/dts/t1042d4rdb.dts b/arch/powerpc/boot/dts/t1042d4rdb.dts new file mode 100644 index 000000000000..846f8c87e85a --- /dev/null +++ b/arch/powerpc/boot/dts/t1042d4rdb.dts @@ -0,0 +1,53 @@ +/* + * T1042D4RDB Device Tree Source + * + * Copyright 2015 Freescale Semiconductor Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/include/ "fsl/t104xsi-pre.dtsi" +/include/ "t104xd4rdb.dtsi" + +/ { + model = "fsl,T1042D4RDB"; + compatible = "fsl,T1042D4RDB"; + #address-cells = <2>; + #size-cells = <2>; + interrupt-parent = <&mpic>; + + ifc: localbus@ffe124000 { + cpld@3,0 { + compatible = "fsl,t1040d4rdb-cpld", + "fsl,deepsleep-cpld"; + }; + }; +}; + +/include/ "fsl/t1040si-post.dtsi" diff --git a/arch/powerpc/boot/dts/t104xd4rdb.dtsi b/arch/powerpc/boot/dts/t104xd4rdb.dtsi new file mode 100644 index 000000000000..491367bd3883 --- /dev/null +++ b/arch/powerpc/boot/dts/t104xd4rdb.dtsi @@ -0,0 +1,205 @@ +/* + * T1040D4RDB/T1042D4RDB Device Tree Source + * + * Copyright 2015 Freescale Semiconductor Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor "AS IS" AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/ { + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bman_fbpr: bman-fbpr { + size = <0 0x1000000>; + alignment = <0 0x1000000>; + }; + qman_fqd: qman-fqd { + size = <0 0x400000>; + alignment = <0 0x400000>; + }; + qman_pfdr: qman-pfdr { + size = <0 0x2000000>; + alignment = <0 0x2000000>; + }; + }; + + ifc: localbus@ffe124000 { + reg = <0xf 0xfe124000 0 0x2000>; + ranges = <0 0 0xf 0xe8000000 0x08000000 + 2 0 0xf 0xff800000 0x00010000 + 3 0 0xf 0xffdf0000 0x00008000>; + + nor@0,0 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "cfi-flash"; + reg = <0x0 0x0 0x8000000>; + bank-width = <2>; + device-width = <1>; + }; + + nand@2,0 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "fsl,ifc-nand"; + reg = <0x2 0x0 0x10000>; + }; + + cpld@3,0 { + compatible = "fsl,t1040d4rdb-cpld"; + reg = <3 0 0x300>; + }; + }; + + memory { + device_type = "memory"; + }; + + dcsr: dcsr@f00000000 { + ranges = <0x00000000 0xf 0x00000000 0x01072000>; + }; + + bportals: bman-portals@ff4000000 { + ranges = <0x0 0xf 0xf4000000 0x2000000>; + }; + + qportals: qman-portals@ff6000000 { + ranges = <0x0 0xf 0xf6000000 0x2000000>; + }; + + soc: soc@ffe000000 { + ranges = <0x00000000 0xf 0xfe000000 0x1000000>; + reg = <0xf 0xfe000000 0 0x00001000>; + + spi@110000 { + flash@0 { + #address-cells = <1>; + #size-cells = <1>; + compatible = "micron,n25q512ax3"; + reg = <0>; + /* input clock */ + spi-max-frequency = <10000000>; + }; + }; + i2c@118000 { + hwmon@4c { + compatible = "adi,adt7461"; + reg = <0x4c>; + }; + + rtc@68 { + compatible = "dallas,ds1337"; + reg = <0x68>; + interrupts = <0x2 0x1 0 0>; + }; + }; + + i2c@118100 { + mux@77 { + /* + * Child nodes of mux depend on which i2c + * devices are connected via the mini PCI + * connector slot1, the mini PCI connector + * slot2, the HDMI connector, and the PEX + * slot. Systems with such devices attached + * should provide a wrapper .dts file that + * includes this one, and adds those nodes + */ + compatible = "nxp,pca9546"; + reg = <0x77>; + #address-cells = <1>; + #size-cells = <0>; + }; + }; + + }; + + pci0: pcie@ffe240000 { + reg = <0xf 0xfe240000 0 0x10000>; + ranges = <0x02000000 0 0xe0000000 0xc 0x0 0x0 0x10000000 + 0x01000000 0 0x0 0xf 0xf8000000 0x0 0x00010000>; + pcie@0 { + ranges = <0x02000000 0 0xe0000000 + 0x02000000 0 0xe0000000 + 0 0x10000000 + + 0x01000000 0 0x00000000 + 0x01000000 0 0x00000000 + 0 0x00010000>; + }; + }; + + pci1: pcie@ffe250000 { + reg = <0xf 0xfe250000 0 0x10000>; + ranges = <0x02000000 0 0xe0000000 0xc 0x10000000 0 0x10000000 + 0x01000000 0 0 0xf 0xf8010000 0 0x00010000>; + pcie@0 { + ranges = <0x02000000 0 0xe0000000 + 0x02000000 0 0xe0000000 + 0 0x10000000 + + 0x01000000 0 0x00000000 + 0x01000000 0 0x00000000 + 0 0x00010000>; + }; + }; + + pci2: pcie@ffe260000 { + reg = <0xf 0xfe260000 0 0x10000>; + ranges = <0x02000000 0 0xe0000000 0xc 0x20000000 0 0x10000000 + 0x01000000 0 0x00000000 0xf 0xf8020000 0 0x00010000>; + pcie@0 { + ranges = <0x02000000 0 0xe0000000 + 0x02000000 0 0xe0000000 + 0 0x10000000 + + 0x01000000 0 0x00000000 + 0x01000000 0 0x00000000 + 0 0x00010000>; + }; + }; + + pci3: pcie@ffe270000 { + reg = <0xf 0xfe270000 0 0x10000>; + ranges = <0x02000000 0 0xe0000000 0xc 0x30000000 0 0x10000000 + 0x01000000 0 0x00000000 0xf 0xf8030000 0 0x00010000>; + pcie@0 { + ranges = <0x02000000 0 0xe0000000 + 0x02000000 0 0xe0000000 + 0 0x10000000 + + 0x01000000 0 0x00000000 + 0x01000000 0 0x00000000 + 0 0x00010000>; + }; + }; +}; diff --git a/arch/powerpc/configs/85xx-32bit.config b/arch/powerpc/configs/85xx-32bit.config new file mode 100644 index 000000000000..6b8894d727a2 --- /dev/null +++ b/arch/powerpc/configs/85xx-32bit.config @@ -0,0 +1,5 @@ +CONFIG_HIGHMEM=y +CONFIG_KEXEC=y +CONFIG_PPC_85xx=y +CONFIG_PROC_KCORE=y +CONFIG_PHYS_64BIT=y diff --git a/arch/powerpc/configs/85xx-64bit.config b/arch/powerpc/configs/85xx-64bit.config new file mode 100644 index 000000000000..4aba81222885 --- /dev/null +++ b/arch/powerpc/configs/85xx-64bit.config @@ -0,0 +1,4 @@ +CONFIG_MATH_EMULATION=y +CONFIG_MATH_EMULATION_HW_UNIMPLEMENTED=y +CONFIG_PPC64=y +CONFIG_PPC_BOOK3E_64=y diff --git a/arch/powerpc/configs/85xx-hw.config b/arch/powerpc/configs/85xx-hw.config new file mode 100644 index 000000000000..528ff0e714e6 --- /dev/null +++ b/arch/powerpc/configs/85xx-hw.config @@ -0,0 +1,142 @@ +CONFIG_AQUANTIA_PHY=y +CONFIG_AT803X_PHY=y +CONFIG_ATA=y +CONFIG_BLK_DEV_SD=y +CONFIG_BLK_DEV_SR_VENDOR=y +CONFIG_BLK_DEV_SR=y +CONFIG_BROADCOM_PHY=y +CONFIG_C293_PCIE=y +CONFIG_CHR_DEV_SG=y +CONFIG_CHR_DEV_ST=y +CONFIG_CICADA_PHY=y +CONFIG_CLK_QORIQ=y +CONFIG_CRYPTO_DEV_FSL_CAAM=y +CONFIG_CRYPTO_DEV_TALITOS=y +CONFIG_DAVICOM_PHY=y +CONFIG_DMADEVICES=y +CONFIG_E1000E=y +CONFIG_E1000=y +CONFIG_EDAC_MM_EDAC=y +CONFIG_EDAC_MPC85XX=y +CONFIG_EDAC=y +CONFIG_EEPROM_AT24=y +CONFIG_EEPROM_LEGACY=y +CONFIG_FB_FSL_DIU=y +CONFIG_FS_ENET=y +CONFIG_FSL_CORENET_CF=y +CONFIG_FSL_DMA=y +CONFIG_FSL_HV_MANAGER=y +CONFIG_FSL_PQ_MDIO=y +CONFIG_FSL_RIO=y +CONFIG_FSL_XGMAC_MDIO=y +CONFIG_GIANFAR=y +CONFIG_GPIO_MPC8XXX=y +CONFIG_HID_A4TECH=y +CONFIG_HID_APPLE=y +CONFIG_HID_BELKIN=y +CONFIG_HID_CHERRY=y +CONFIG_HID_CHICONY=y +CONFIG_HID_CYPRESS=y +CONFIG_HID_EZKEY=y +CONFIG_HID_GYRATION=y +CONFIG_HID_LOGITECH=y +CONFIG_HID_MICROSOFT=y +CONFIG_HID_MONTEREY=y +CONFIG_HID_PANTHERLORD=y +CONFIG_HID_PETALYNX=y +CONFIG_HID_SAMSUNG=y +CONFIG_HID_SUNPLUS=y +CONFIG_I2C_CHARDEV=y +CONFIG_I2C_CPM=m +CONFIG_I2C_MPC=y +CONFIG_I2C_MUX_PCA954x=y +CONFIG_I2C_MUX=y +CONFIG_I2C=y +CONFIG_IGB=y +CONFIG_INPUT_FF_MEMLESS=m +# CONFIG_INPUT_KEYBOARD is not set +# CONFIG_INPUT_MOUSEDEV is not set +# CONFIG_INPUT_MOUSE is not set +CONFIG_MARVELL_PHY=y +CONFIG_MDIO_BUS_MUX_GPIO=y +CONFIG_MDIO_BUS_MUX_MMIOREG=y +CONFIG_MMC_SDHCI_OF_ESDHC=y +CONFIG_MMC_SDHCI_PLTFM=y +CONFIG_MMC_SDHCI=y +CONFIG_MMC=y +CONFIG_MTD_BLOCK=y +CONFIG_MTD_CFI_AMDSTD=y +CONFIG_MTD_CFI_INTELEXT=y +CONFIG_MTD_CFI=y +CONFIG_MTD_CMDLINE_PARTS=y +CONFIG_MTD_M25P80=y +CONFIG_MTD_NAND_FSL_ELBC=y +CONFIG_MTD_NAND_FSL_IFC=y +CONFIG_MTD_NAND=y +CONFIG_MTD_PHYSMAP_OF=y +CONFIG_MTD_PHYSMAP=y +CONFIG_MTD_PLATRAM=y +CONFIG_MTD_SPI_NOR=y +CONFIG_NETDEVICES=y +CONFIG_NVRAM=y +CONFIG_PATA_ALI=y +CONFIG_PATA_SIL680=y +CONFIG_PATA_VIA=y +# CONFIG_PCIEASPM is not set +CONFIG_PCIEPORTBUS=y +CONFIG_PCI_MSI=y +CONFIG_PCI=y +CONFIG_PPC_EPAPR_HV_BYTECHAN=y +# CONFIG_PPC_OF_BOOT_TRAMPOLINE is not set +CONFIG_QE_GPIO=y +CONFIG_QUICC_ENGINE=y +CONFIG_RAPIDIO=y +CONFIG_RTC_CLASS=y +CONFIG_RTC_DRV_CMOS=y +CONFIG_RTC_DRV_DS1307=y +CONFIG_RTC_DRV_DS1374=y +CONFIG_RTC_DRV_DS3232=y +CONFIG_SATA_AHCI=y +CONFIG_SATA_FSL=y +CONFIG_SATA_SIL24=y +CONFIG_SATA_SIL=y +CONFIG_SCSI_LOGGING=y +CONFIG_SCSI_SYM53C8XX_2=y +CONFIG_SENSORS_INA2XX=y +CONFIG_SENSORS_LM90=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_SERIAL_8250_DETECT_IRQ=y +CONFIG_SERIAL_8250_MANY_PORTS=y +CONFIG_SERIAL_8250_NR_UARTS=6 +CONFIG_SERIAL_8250_RSA=y +CONFIG_SERIAL_8250_RUNTIME_UARTS=6 +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_QE=m +CONFIG_SERIO_LIBPS2=y +# CONFIG_SND_DRIVERS is not set +CONFIG_SND_INTEL8X0=y +CONFIG_SND_POWERPC_SOC=y +# CONFIG_SND_PPC is not set +CONFIG_SND_SOC=y +# CONFIG_SND_SUPPORT_OLD_API is not set +# CONFIG_SND_USB is not set +CONFIG_SND=y +CONFIG_SOUND=y +CONFIG_SPI_FSL_ESPI=y +CONFIG_SPI_FSL_SPI=y +CONFIG_SPI_GPIO=y +CONFIG_SPI=y +CONFIG_TERANETICS_PHY=y +CONFIG_UCC_GETH=y +CONFIG_USB_EHCI_FSL=y +CONFIG_USB_EHCI_HCD=y +CONFIG_USB_HID=m +CONFIG_USB_MON=y +CONFIG_USB_OHCI_HCD_PPC_OF_BE=y +CONFIG_USB_OHCI_HCD_PPC_OF_LE=y +CONFIG_USB_OHCI_HCD=y +CONFIG_USB_STORAGE=y +CONFIG_USB=y +# CONFIG_VGA_CONSOLE is not set +CONFIG_VIRT_DRIVERS=y +CONFIG_VITESSE_PHY=y diff --git a/arch/powerpc/configs/85xx-smp.config b/arch/powerpc/configs/85xx-smp.config new file mode 100644 index 000000000000..3b4d1e54636d --- /dev/null +++ b/arch/powerpc/configs/85xx-smp.config @@ -0,0 +1,2 @@ +CONFIG_NR_CPUS=24 +CONFIG_SMP=y diff --git a/arch/powerpc/configs/altivec.config b/arch/powerpc/configs/altivec.config new file mode 100644 index 000000000000..58a697cb5a62 --- /dev/null +++ b/arch/powerpc/configs/altivec.config @@ -0,0 +1 @@ +CONFIG_ALTIVEC=y diff --git a/arch/powerpc/configs/corenet32_smp_defconfig b/arch/powerpc/configs/corenet32_smp_defconfig deleted file mode 100644 index 37659937bd12..000000000000 --- a/arch/powerpc/configs/corenet32_smp_defconfig +++ /dev/null @@ -1,185 +0,0 @@ -CONFIG_PPC_85xx=y -CONFIG_SMP=y -CONFIG_NR_CPUS=8 -CONFIG_SYSVIPC=y -CONFIG_POSIX_MQUEUE=y -CONFIG_AUDIT=y -CONFIG_NO_HZ=y -CONFIG_HIGH_RES_TIMERS=y -CONFIG_BSD_PROCESS_ACCT=y -CONFIG_IKCONFIG=y -CONFIG_IKCONFIG_PROC=y -CONFIG_LOG_BUF_SHIFT=14 -CONFIG_BLK_DEV_INITRD=y -CONFIG_KALLSYMS_ALL=y -CONFIG_EMBEDDED=y -CONFIG_PERF_EVENTS=y -CONFIG_SLAB=y -CONFIG_MODULES=y -CONFIG_MODULE_UNLOAD=y -CONFIG_MODULE_FORCE_UNLOAD=y -CONFIG_MODVERSIONS=y -# CONFIG_BLK_DEV_BSG is not set -CONFIG_PARTITION_ADVANCED=y -CONFIG_MAC_PARTITION=y -CONFIG_CORENET_GENERIC=y -CONFIG_HIGHMEM=y -# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set -CONFIG_BINFMT_MISC=m -CONFIG_KEXEC=y -CONFIG_FORCE_MAX_ZONEORDER=13 -CONFIG_PCI=y -CONFIG_PCIEPORTBUS=y -# CONFIG_PCIEASPM is not set -CONFIG_PCI_MSI=y -CONFIG_RAPIDIO=y -CONFIG_FSL_RIO=y -CONFIG_NET=y -CONFIG_PACKET=y -CONFIG_UNIX=y -CONFIG_XFRM_USER=y -CONFIG_XFRM_SUB_POLICY=y -CONFIG_XFRM_STATISTICS=y -CONFIG_NET_KEY=y -CONFIG_NET_KEY_MIGRATE=y -CONFIG_INET=y -CONFIG_IP_MULTICAST=y -CONFIG_IP_ADVANCED_ROUTER=y -CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IP_ROUTE_MULTIPATH=y -CONFIG_IP_ROUTE_VERBOSE=y -CONFIG_IP_PNP=y -CONFIG_IP_PNP_DHCP=y -CONFIG_IP_PNP_BOOTP=y -CONFIG_IP_PNP_RARP=y -CONFIG_NET_IPIP=y -CONFIG_IP_MROUTE=y -CONFIG_IP_PIMSM_V1=y -CONFIG_IP_PIMSM_V2=y -CONFIG_INET_AH=y -CONFIG_INET_ESP=y -CONFIG_INET_IPCOMP=y -# CONFIG_INET_LRO is not set -CONFIG_IPV6=y -CONFIG_IP_SCTP=m -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" -CONFIG_DEVTMPFS=y -CONFIG_DEVTMPFS_MOUNT=y -CONFIG_MTD=y -CONFIG_MTD_CMDLINE_PARTS=y -CONFIG_MTD_BLOCK=y -CONFIG_MTD_CFI=y -CONFIG_MTD_CFI_INTELEXT=y -CONFIG_MTD_CFI_AMDSTD=y -CONFIG_MTD_PHYSMAP_OF=y -CONFIG_MTD_NAND=y -CONFIG_MTD_NAND_FSL_ELBC=y -CONFIG_MTD_NAND_FSL_IFC=y -CONFIG_MTD_SPI_NOR=y -CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_RAM=y -CONFIG_BLK_DEV_RAM_SIZE=131072 -CONFIG_BLK_DEV_SD=y -CONFIG_CHR_DEV_ST=y -CONFIG_BLK_DEV_SR=y -CONFIG_CHR_DEV_SG=y -CONFIG_SCSI_LOGGING=y -CONFIG_SCSI_SYM53C8XX_2=y -CONFIG_ATA=y -CONFIG_SATA_AHCI=y -CONFIG_SATA_FSL=y -CONFIG_SATA_SIL24=y -CONFIG_SATA_SIL=y -CONFIG_PATA_SIL680=y -CONFIG_NETDEVICES=y -CONFIG_FSL_PQ_MDIO=y -CONFIG_FSL_XGMAC_MDIO=y -CONFIG_E1000=y -CONFIG_E1000E=y -CONFIG_AT803X_PHY=y -CONFIG_VITESSE_PHY=y -CONFIG_FIXED_PHY=y -CONFIG_MDIO_BUS_MUX_GPIO=y -CONFIG_MDIO_BUS_MUX_MMIOREG=y -# CONFIG_INPUT_MOUSEDEV is not set -# CONFIG_INPUT_KEYBOARD is not set -# CONFIG_INPUT_MOUSE is not set -CONFIG_SERIO_LIBPS2=y -# CONFIG_LEGACY_PTYS is not set -CONFIG_PPC_EPAPR_HV_BYTECHAN=y -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -CONFIG_SERIAL_8250_MANY_PORTS=y -CONFIG_SERIAL_8250_DETECT_IRQ=y -CONFIG_SERIAL_8250_RSA=y -CONFIG_NVRAM=y -CONFIG_I2C=y -CONFIG_I2C_CHARDEV=y -CONFIG_I2C_MPC=y -CONFIG_I2C_MUX=y -CONFIG_I2C_MUX_PCA954x=y -CONFIG_SPI=y -CONFIG_SPI_GPIO=y -CONFIG_SPI_FSL_SPI=y -CONFIG_SPI_FSL_ESPI=y -CONFIG_SENSORS_LM90=y -CONFIG_SENSORS_INA2XX=y -CONFIG_USB_HID=m -CONFIG_USB=y -CONFIG_USB_MON=y -CONFIG_USB_EHCI_HCD=y -CONFIG_USB_EHCI_FSL=y -CONFIG_USB_OHCI_HCD=y -CONFIG_USB_OHCI_HCD_PPC_OF_BE=y -CONFIG_USB_OHCI_HCD_PPC_OF_LE=y -CONFIG_USB_STORAGE=y -CONFIG_MMC=y -CONFIG_MMC_SDHCI=y -CONFIG_EDAC=y -CONFIG_EDAC_MM_EDAC=y -CONFIG_EDAC_MPC85XX=y -CONFIG_RTC_CLASS=y -CONFIG_RTC_DRV_DS1307=y -CONFIG_RTC_DRV_DS1374=y -CONFIG_RTC_DRV_DS3232=y -CONFIG_UIO=y -CONFIG_VIRT_DRIVERS=y -CONFIG_FSL_HV_MANAGER=y -CONFIG_STAGING=y -CONFIG_FSL_CORENET_CF=y -CONFIG_CLK_QORIQ=y -CONFIG_EXT2_FS=y -CONFIG_EXT3_FS=y -# CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set -CONFIG_ISO9660_FS=m -CONFIG_JOLIET=y -CONFIG_ZISOFS=y -CONFIG_UDF_FS=m -CONFIG_MSDOS_FS=m -CONFIG_VFAT_FS=y -CONFIG_NTFS_FS=y -CONFIG_PROC_KCORE=y -CONFIG_TMPFS=y -CONFIG_HUGETLBFS=y -CONFIG_JFFS2_FS=y -CONFIG_CRAMFS=y -CONFIG_NFS_FS=y -CONFIG_NFS_V4=y -CONFIG_ROOT_NFS=y -CONFIG_NFSD=m -CONFIG_NLS_CODEPAGE_437=y -CONFIG_NLS_CODEPAGE_850=y -CONFIG_NLS_ISO8859_1=y -CONFIG_NLS_UTF8=m -CONFIG_DEBUG_INFO=y -CONFIG_MAGIC_SYSRQ=y -CONFIG_DEBUG_SHIRQ=y -CONFIG_DETECT_HUNG_TASK=y -CONFIG_RCU_TRACE=y -CONFIG_CRYPTO_NULL=y -CONFIG_CRYPTO_PCBC=m -CONFIG_CRYPTO_MD4=y -CONFIG_CRYPTO_SHA256=y -CONFIG_CRYPTO_SHA512=y -# CONFIG_CRYPTO_ANSI_CPRNG is not set -CONFIG_CRYPTO_DEV_FSL_CAAM=y diff --git a/arch/powerpc/configs/corenet_basic_defconfig b/arch/powerpc/configs/corenet_basic_defconfig new file mode 100644 index 000000000000..b568d465e59e --- /dev/null +++ b/arch/powerpc/configs/corenet_basic_defconfig @@ -0,0 +1 @@ +CONFIG_CORENET_GENERIC=y diff --git a/arch/powerpc/configs/corenet64_smp_defconfig b/arch/powerpc/configs/fsl-emb-nonhw.config similarity index 50% rename from arch/powerpc/configs/corenet64_smp_defconfig rename to arch/powerpc/configs/fsl-emb-nonhw.config index 33cd1df818ad..41e4d359524d 100644 --- a/arch/powerpc/configs/corenet64_smp_defconfig +++ b/arch/powerpc/configs/fsl-emb-nonhw.config @@ -1,176 +1,126 @@ -CONFIG_PPC64=y -CONFIG_PPC_BOOK3E_64=y -CONFIG_ALTIVEC=y -CONFIG_SMP=y -CONFIG_NR_CPUS=24 -CONFIG_SYSVIPC=y -CONFIG_FHANDLE=y -CONFIG_IRQ_DOMAIN_DEBUG=y -CONFIG_NO_HZ=y -CONFIG_HIGH_RES_TIMERS=y +CONFIG_ADFS_FS=m +CONFIG_AFFS_FS=m +CONFIG_AUDIT=y +CONFIG_BEFS_FS=m +CONFIG_BFS_FS=m +CONFIG_BINFMT_MISC=m +# CONFIG_BLK_DEV_BSG is not set +CONFIG_BLK_DEV_INITRD=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_NBD=y +CONFIG_BLK_DEV_RAM_SIZE=131072 +CONFIG_BLK_DEV_RAM=y CONFIG_BSD_PROCESS_ACCT=y -CONFIG_IKCONFIG=y -CONFIG_IKCONFIG_PROC=y -CONFIG_LOG_BUF_SHIFT=14 -CONFIG_CGROUPS=y -CONFIG_CPUSETS=y CONFIG_CGROUP_CPUACCT=y CONFIG_CGROUP_SCHED=y -CONFIG_BLK_DEV_INITRD=y +CONFIG_CGROUPS=y +# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set +CONFIG_CRC_T10DIF=y +CONFIG_CPUSETS=y +CONFIG_CRAMFS=y +CONFIG_CRYPTO_MD4=y +CONFIG_CRYPTO_NULL=y +CONFIG_CRYPTO_PCBC=m +CONFIG_CRYPTO_SHA256=y +CONFIG_CRYPTO_SHA512=y +CONFIG_DEBUG_FS=y +CONFIG_DEBUG_INFO=y +CONFIG_DEBUG_SHIRQ=y +CONFIG_DETECT_HUNG_TASK=y +CONFIG_DEVTMPFS_MOUNT=y +CONFIG_DEVTMPFS=y +CONFIG_DUMMY=y +CONFIG_EFS_FS=m CONFIG_EXPERT=y -CONFIG_KALLSYMS_ALL=y -CONFIG_MODULES=y -CONFIG_MODULE_UNLOAD=y -CONFIG_MODULE_FORCE_UNLOAD=y -CONFIG_MODVERSIONS=y -# CONFIG_BLK_DEV_BSG is not set -CONFIG_PARTITION_ADVANCED=y -CONFIG_MAC_PARTITION=y -CONFIG_CORENET_GENERIC=y -# CONFIG_PPC_OF_BOOT_TRAMPOLINE is not set -CONFIG_BINFMT_MISC=m -CONFIG_MATH_EMULATION=y -CONFIG_MATH_EMULATION_HW_UNIMPLEMENTED=y -CONFIG_PCIEPORTBUS=y -CONFIG_PCI_MSI=y -CONFIG_RAPIDIO=y -CONFIG_FSL_RIO=y -CONFIG_NET=y -CONFIG_PACKET=y -CONFIG_UNIX=y -CONFIG_XFRM_USER=y -CONFIG_NET_KEY=y +CONFIG_EXT2_FS=y +# CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set +CONFIG_EXT3_FS=y +CONFIG_FB=y +CONFIG_FHANDLE=y +CONFIG_FIXED_PHY=y +CONFIG_FONT_8x16=y +CONFIG_FONT_8x8=y +CONFIG_FONTS=y +CONFIG_FORCE_MAX_ZONEORDER=13 +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_FRAME_WARN=1024 +CONFIG_FTL=y +CONFIG_HFS_FS=m +CONFIG_HFSPLUS_FS=m +CONFIG_HIGH_RES_TIMERS=y +CONFIG_HPFS_FS=m +CONFIG_HUGETLBFS=y +CONFIG_IKCONFIG_PROC=y +CONFIG_IKCONFIG=y +CONFIG_INET_AH=y +CONFIG_INET_ESP=y +CONFIG_INET_IPCOMP=y +# CONFIG_INET_LRO is not set +# CONFIG_INET_XFRM_MODE_BEET is not set CONFIG_INET=y -CONFIG_IP_MULTICAST=y CONFIG_IP_ADVANCED_ROUTER=y -CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IP_ROUTE_MULTIPATH=y -CONFIG_IP_ROUTE_VERBOSE=y -CONFIG_IP_PNP=y -CONFIG_IP_PNP_DHCP=y -CONFIG_IP_PNP_BOOTP=y -CONFIG_IP_PNP_RARP=y -CONFIG_NET_IPIP=y CONFIG_IP_MROUTE=y +CONFIG_IP_MULTICAST=y +CONFIG_IP_MULTIPLE_TABLES=y CONFIG_IP_PIMSM_V1=y CONFIG_IP_PIMSM_V2=y -CONFIG_INET_ESP=y -# CONFIG_INET_XFRM_MODE_BEET is not set -# CONFIG_INET_LRO is not set -CONFIG_IPV6=y +CONFIG_IP_PNP_BOOTP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_IP_PNP_RARP=y +CONFIG_IP_PNP=y +CONFIG_IP_ROUTE_MULTIPATH=y +CONFIG_IP_ROUTE_VERBOSE=y CONFIG_IP_SCTP=m -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" -CONFIG_DEVTMPFS=y -CONFIG_DEVTMPFS_MOUNT=y -CONFIG_MTD=y -CONFIG_MTD_CMDLINE_PARTS=y -CONFIG_MTD_BLOCK=y -CONFIG_FTL=y -CONFIG_MTD_CFI=y -CONFIG_MTD_CFI_INTELEXT=y -CONFIG_MTD_CFI_AMDSTD=y -CONFIG_MTD_PHYSMAP_OF=y -CONFIG_MTD_NAND=y -CONFIG_MTD_NAND_FSL_ELBC=y -CONFIG_MTD_NAND_FSL_IFC=y -CONFIG_MTD_SPI_NOR=y -CONFIG_MTD_UBI=y -CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_RAM=y -CONFIG_BLK_DEV_RAM_SIZE=131072 -CONFIG_EEPROM_LEGACY=y -CONFIG_BLK_DEV_SD=y -CONFIG_BLK_DEV_SR=y -CONFIG_BLK_DEV_SR_VENDOR=y -CONFIG_CHR_DEV_SG=y -CONFIG_ATA=y -CONFIG_SATA_FSL=y -CONFIG_SATA_SIL24=y -CONFIG_NETDEVICES=y -CONFIG_DUMMY=y -CONFIG_FSL_PQ_MDIO=y -CONFIG_FSL_XGMAC_MDIO=y -CONFIG_E1000E=y -CONFIG_VITESSE_PHY=y -CONFIG_FIXED_PHY=y -CONFIG_MDIO_BUS_MUX_GPIO=y -CONFIG_MDIO_BUS_MUX_MMIOREG=y -CONFIG_INPUT_FF_MEMLESS=m -# CONFIG_INPUT_MOUSEDEV is not set -# CONFIG_INPUT_KEYBOARD is not set -# CONFIG_INPUT_MOUSE is not set -CONFIG_SERIO_LIBPS2=y -CONFIG_PPC_EPAPR_HV_BYTECHAN=y -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -CONFIG_SERIAL_8250_MANY_PORTS=y -CONFIG_SERIAL_8250_DETECT_IRQ=y -CONFIG_SERIAL_8250_RSA=y -CONFIG_I2C=y -CONFIG_I2C_CHARDEV=y -CONFIG_I2C_MPC=y -CONFIG_I2C_MUX=y -CONFIG_I2C_MUX_PCA954x=y -CONFIG_SPI=y -CONFIG_SPI_GPIO=y -CONFIG_SPI_FSL_SPI=y -CONFIG_SPI_FSL_ESPI=y -CONFIG_SENSORS_LM90=y -CONFIG_SENSORS_INA2XX=y -CONFIG_USB_HID=m -CONFIG_USB=y -CONFIG_USB_MON=y -CONFIG_USB_EHCI_HCD=y -CONFIG_USB_EHCI_FSL=y -CONFIG_USB_STORAGE=y -CONFIG_MMC=y -CONFIG_MMC_SDHCI=y -CONFIG_EDAC=y -CONFIG_EDAC_MM_EDAC=y -CONFIG_RTC_CLASS=y -CONFIG_RTC_DRV_DS1307=y -CONFIG_RTC_DRV_DS1374=y -CONFIG_RTC_DRV_DS3232=y -CONFIG_DMADEVICES=y -CONFIG_FSL_DMA=y -CONFIG_VIRT_DRIVERS=y -CONFIG_FSL_HV_MANAGER=y -CONFIG_CLK_QORIQ=y -CONFIG_FSL_CORENET_CF=y -CONFIG_EXT2_FS=y -CONFIG_EXT3_FS=y +CONFIG_IPV6=y +CONFIG_IRQ_DOMAIN_DEBUG=y CONFIG_ISO9660_FS=m -CONFIG_JOLIET=y -CONFIG_ZISOFS=y -CONFIG_UDF_FS=m -CONFIG_MSDOS_FS=m -CONFIG_VFAT_FS=y -CONFIG_NTFS_FS=y -CONFIG_PROC_KCORE=y -CONFIG_TMPFS=y -CONFIG_HUGETLBFS=y -CONFIG_JFFS2_FS=y CONFIG_JFFS2_FS_DEBUG=1 -CONFIG_UBIFS_FS=y +CONFIG_JFFS2_FS=y +CONFIG_JOLIET=y +CONFIG_KALLSYMS_ALL=y +# CONFIG_LEGACY_PTYS is not set +CONFIG_LOG_BUF_SHIFT=14 +CONFIG_MAC_PARTITION=y +CONFIG_MAGIC_SYSRQ=y +CONFIG_MODULE_FORCE_UNLOAD=y +CONFIG_MODULES=y +CONFIG_MODULE_UNLOAD=y +CONFIG_MODVERSIONS=y +CONFIG_MSDOS_FS=m +CONFIG_MTD_UBI=y +CONFIG_MTD=y +CONFIG_NET_IPIP=y +CONFIG_NET_KEY_MIGRATE=y +CONFIG_NET_KEY=y +CONFIG_NET=y +CONFIG_NFSD=y CONFIG_NFS_FS=y CONFIG_NFS_V4=y -CONFIG_ROOT_NFS=y -CONFIG_NFSD=m CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_CODEPAGE_850=y CONFIG_NLS_ISO8859_1=y CONFIG_NLS_UTF8=m -CONFIG_CRC_T10DIF=y -CONFIG_DEBUG_INFO=y -CONFIG_FRAME_WARN=1024 -CONFIG_DEBUG_FS=y -CONFIG_MAGIC_SYSRQ=y -CONFIG_DEBUG_SHIRQ=y -CONFIG_DETECT_HUNG_TASK=y -CONFIG_CRYPTO_NULL=y -CONFIG_CRYPTO_PCBC=m -CONFIG_CRYPTO_MD4=y -CONFIG_CRYPTO_SHA256=y -CONFIG_CRYPTO_SHA512=y -# CONFIG_CRYPTO_ANSI_CPRNG is not set -CONFIG_CRYPTO_DEV_FSL_CAAM=y +CONFIG_NO_HZ=y +CONFIG_NTFS_FS=y +CONFIG_PACKET=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_PERF_EVENTS=y +CONFIG_POSIX_MQUEUE=y +CONFIG_QNX4FS_FS=m +CONFIG_RCU_TRACE=y +CONFIG_ROOT_NFS=y +CONFIG_SYSV_FS=m +CONFIG_SYSVIPC=y +CONFIG_TMPFS=y +CONFIG_UBIFS_FS=y +CONFIG_UDF_FS=m +CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" +CONFIG_UFS_FS=m +CONFIG_UIO=y +CONFIG_UNIX=y +CONFIG_VFAT_FS=y +CONFIG_VXFS_FS=m +CONFIG_XFRM_STATISTICS=y +CONFIG_XFRM_SUB_POLICY=y +CONFIG_XFRM_USER=y +CONFIG_ZISOFS=y diff --git a/arch/powerpc/configs/mpc85xx_basic_defconfig b/arch/powerpc/configs/mpc85xx_basic_defconfig new file mode 100644 index 000000000000..850bd195d0e8 --- /dev/null +++ b/arch/powerpc/configs/mpc85xx_basic_defconfig @@ -0,0 +1,23 @@ +CONFIG_MATH_EMULATION=y +CONFIG_MPC8536_DS=y +CONFIG_MPC8540_ADS=y +CONFIG_MPC8560_ADS=y +CONFIG_MPC85xx_CDS=y +CONFIG_MPC85xx_DS=y +CONFIG_MPC85xx_MDS=y +CONFIG_MPC85xx_RDB=y +CONFIG_KSI8560=y +CONFIG_MVME2500=y +CONFIG_P1010_RDB=y +CONFIG_P1022_DS=y +CONFIG_P1022_RDK=y +CONFIG_P1023_RDB=y +CONFIG_SBC8548=y +CONFIG_SOCRATES=y +CONFIG_STX_GP3=y +CONFIG_TQM8540=y +CONFIG_TQM8541=y +CONFIG_TQM8548=y +CONFIG_TQM8555=y +CONFIG_TQM8560=y +CONFIG_XES_MPC85xx=y diff --git a/arch/powerpc/configs/mpc85xx_defconfig b/arch/powerpc/configs/mpc85xx_defconfig deleted file mode 100644 index 6ecf7bdbc2f9..000000000000 --- a/arch/powerpc/configs/mpc85xx_defconfig +++ /dev/null @@ -1,252 +0,0 @@ -CONFIG_PPC_85xx=y -CONFIG_PHYS_64BIT=y -CONFIG_SYSVIPC=y -CONFIG_POSIX_MQUEUE=y -CONFIG_AUDIT=y -CONFIG_IRQ_DOMAIN_DEBUG=y -CONFIG_NO_HZ=y -CONFIG_HIGH_RES_TIMERS=y -CONFIG_BSD_PROCESS_ACCT=y -CONFIG_IKCONFIG=y -CONFIG_IKCONFIG_PROC=y -CONFIG_LOG_BUF_SHIFT=14 -CONFIG_BLK_DEV_INITRD=y -CONFIG_EXPERT=y -CONFIG_KALLSYMS_ALL=y -CONFIG_MODULES=y -CONFIG_MODULE_UNLOAD=y -CONFIG_MODULE_FORCE_UNLOAD=y -CONFIG_MODVERSIONS=y -# CONFIG_BLK_DEV_BSG is not set -CONFIG_PARTITION_ADVANCED=y -CONFIG_MAC_PARTITION=y -CONFIG_C293_PCIE=y -CONFIG_MPC8540_ADS=y -CONFIG_MPC8560_ADS=y -CONFIG_MPC85xx_CDS=y -CONFIG_MPC85xx_MDS=y -CONFIG_MPC8536_DS=y -CONFIG_MPC85xx_DS=y -CONFIG_MPC85xx_RDB=y -CONFIG_P1010_RDB=y -CONFIG_P1022_DS=y -CONFIG_P1022_RDK=y -CONFIG_P1023_RDB=y -CONFIG_SOCRATES=y -CONFIG_KSI8560=y -CONFIG_XES_MPC85xx=y -CONFIG_STX_GP3=y -CONFIG_TQM8540=y -CONFIG_TQM8541=y -CONFIG_TQM8548=y -CONFIG_TQM8555=y -CONFIG_TQM8560=y -CONFIG_SBC8548=y -CONFIG_MVME2500=y -CONFIG_QUICC_ENGINE=y -CONFIG_QE_GPIO=y -CONFIG_HIGHMEM=y -CONFIG_BINFMT_MISC=m -CONFIG_MATH_EMULATION=y -CONFIG_FORCE_MAX_ZONEORDER=12 -CONFIG_PCI=y -CONFIG_PCIEPORTBUS=y -# CONFIG_PCIEASPM is not set -CONFIG_PCI_MSI=y -CONFIG_RAPIDIO=y -CONFIG_NET=y -CONFIG_PACKET=y -CONFIG_UNIX=y -CONFIG_XFRM_USER=y -CONFIG_NET_KEY=y -CONFIG_INET=y -CONFIG_IP_MULTICAST=y -CONFIG_IP_ADVANCED_ROUTER=y -CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IP_ROUTE_MULTIPATH=y -CONFIG_IP_ROUTE_VERBOSE=y -CONFIG_IP_PNP=y -CONFIG_IP_PNP_DHCP=y -CONFIG_IP_PNP_BOOTP=y -CONFIG_IP_PNP_RARP=y -CONFIG_NET_IPIP=y -CONFIG_IP_MROUTE=y -CONFIG_IP_PIMSM_V1=y -CONFIG_IP_PIMSM_V2=y -CONFIG_INET_ESP=y -# CONFIG_INET_XFRM_MODE_BEET is not set -# CONFIG_INET_LRO is not set -CONFIG_IPV6=y -CONFIG_IP_SCTP=m -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" -CONFIG_DEVTMPFS=y -CONFIG_DEVTMPFS_MOUNT=y -CONFIG_MTD=y -CONFIG_MTD_CMDLINE_PARTS=y -CONFIG_MTD_BLOCK=y -CONFIG_FTL=y -CONFIG_MTD_CFI=y -CONFIG_MTD_CFI_INTELEXT=y -CONFIG_MTD_CFI_AMDSTD=y -CONFIG_MTD_PHYSMAP=y -CONFIG_MTD_PHYSMAP_OF=y -CONFIG_MTD_PLATRAM=y -CONFIG_MTD_M25P80=y -CONFIG_MTD_NAND=y -CONFIG_MTD_NAND_FSL_ELBC=y -CONFIG_MTD_NAND_FSL_IFC=y -CONFIG_MTD_SPI_NOR=y -CONFIG_MTD_UBI=y -CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_NBD=y -CONFIG_BLK_DEV_RAM=y -CONFIG_BLK_DEV_RAM_SIZE=131072 -CONFIG_EEPROM_AT24=y -CONFIG_EEPROM_LEGACY=y -CONFIG_BLK_DEV_SD=y -CONFIG_CHR_DEV_ST=y -CONFIG_BLK_DEV_SR=y -CONFIG_CHR_DEV_SG=y -CONFIG_SCSI_LOGGING=y -CONFIG_ATA=y -CONFIG_SATA_AHCI=y -CONFIG_SATA_FSL=y -CONFIG_SATA_SIL24=y -CONFIG_PATA_ALI=y -CONFIG_PATA_VIA=y -CONFIG_NETDEVICES=y -CONFIG_DUMMY=y -CONFIG_FS_ENET=y -CONFIG_UCC_GETH=y -CONFIG_GIANFAR=y -CONFIG_E1000=y -CONFIG_E1000E=y -CONFIG_IGB=y -CONFIG_AT803X_PHY=y -CONFIG_MARVELL_PHY=y -CONFIG_DAVICOM_PHY=y -CONFIG_CICADA_PHY=y -CONFIG_VITESSE_PHY=y -CONFIG_BROADCOM_PHY=y -CONFIG_FIXED_PHY=y -CONFIG_INPUT_FF_MEMLESS=m -# CONFIG_INPUT_MOUSEDEV is not set -# CONFIG_INPUT_KEYBOARD is not set -# CONFIG_INPUT_MOUSE is not set -CONFIG_SERIO_LIBPS2=y -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -CONFIG_SERIAL_8250_NR_UARTS=6 -CONFIG_SERIAL_8250_RUNTIME_UARTS=6 -CONFIG_SERIAL_8250_MANY_PORTS=y -CONFIG_SERIAL_8250_DETECT_IRQ=y -CONFIG_SERIAL_8250_RSA=y -CONFIG_SERIAL_QE=m -CONFIG_NVRAM=y -CONFIG_I2C_CHARDEV=y -CONFIG_I2C_CPM=m -CONFIG_I2C_MPC=y -CONFIG_SPI=y -CONFIG_SPI_FSL_SPI=y -CONFIG_SPI_FSL_ESPI=y -CONFIG_GPIO_MPC8XXX=y -CONFIG_SENSORS_LM90=y -CONFIG_FB=y -CONFIG_FB_FSL_DIU=y -# CONFIG_VGA_CONSOLE is not set -CONFIG_FRAMEBUFFER_CONSOLE=y -CONFIG_SOUND=y -CONFIG_SND=y -# CONFIG_SND_SUPPORT_OLD_API is not set -# CONFIG_SND_DRIVERS is not set -CONFIG_SND_INTEL8X0=y -# CONFIG_SND_PPC is not set -# CONFIG_SND_USB is not set -CONFIG_SND_SOC=y -CONFIG_SND_POWERPC_SOC=y -CONFIG_HID_A4TECH=y -CONFIG_HID_APPLE=y -CONFIG_HID_BELKIN=y -CONFIG_HID_CHERRY=y -CONFIG_HID_CHICONY=y -CONFIG_HID_CYPRESS=y -CONFIG_HID_EZKEY=y -CONFIG_HID_GYRATION=y -CONFIG_HID_LOGITECH=y -CONFIG_HID_MICROSOFT=y -CONFIG_HID_MONTEREY=y -CONFIG_HID_PANTHERLORD=y -CONFIG_HID_PETALYNX=y -CONFIG_HID_SAMSUNG=y -CONFIG_HID_SUNPLUS=y -CONFIG_USB=y -CONFIG_USB_MON=y -CONFIG_USB_EHCI_HCD=y -CONFIG_USB_EHCI_FSL=y -CONFIG_USB_OHCI_HCD=y -CONFIG_USB_OHCI_HCD_PPC_OF_BE=y -CONFIG_USB_OHCI_HCD_PPC_OF_LE=y -CONFIG_USB_STORAGE=y -CONFIG_MMC=y -CONFIG_MMC_SDHCI=y -CONFIG_MMC_SDHCI_PLTFM=y -CONFIG_MMC_SDHCI_OF_ESDHC=y -CONFIG_EDAC=y -CONFIG_EDAC_MM_EDAC=y -CONFIG_EDAC_MPC85XX=y -CONFIG_RTC_CLASS=y -CONFIG_RTC_DRV_DS1307=y -CONFIG_RTC_DRV_DS1374=y -CONFIG_RTC_DRV_DS3232=y -CONFIG_RTC_DRV_CMOS=y -CONFIG_DMADEVICES=y -CONFIG_FSL_DMA=y -CONFIG_EXT2_FS=y -CONFIG_EXT3_FS=y -# CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set -CONFIG_ISO9660_FS=m -CONFIG_JOLIET=y -CONFIG_ZISOFS=y -CONFIG_UDF_FS=m -CONFIG_MSDOS_FS=m -CONFIG_VFAT_FS=y -CONFIG_NTFS_FS=y -CONFIG_PROC_KCORE=y -CONFIG_TMPFS=y -CONFIG_HUGETLBFS=y -CONFIG_ADFS_FS=m -CONFIG_AFFS_FS=m -CONFIG_HFS_FS=m -CONFIG_HFSPLUS_FS=m -CONFIG_BEFS_FS=m -CONFIG_BFS_FS=m -CONFIG_EFS_FS=m -CONFIG_JFFS2_FS=y -CONFIG_JFFS2_FS_DEBUG=1 -CONFIG_UBIFS_FS=y -CONFIG_CRAMFS=y -CONFIG_VXFS_FS=m -CONFIG_HPFS_FS=m -CONFIG_QNX4FS_FS=m -CONFIG_SYSV_FS=m -CONFIG_UFS_FS=m -CONFIG_NFS_FS=y -CONFIG_NFS_V4=y -CONFIG_ROOT_NFS=y -CONFIG_NFSD=y -CONFIG_NLS_CODEPAGE_437=y -CONFIG_NLS_CODEPAGE_850=y -CONFIG_NLS_ISO8859_1=y -CONFIG_CRC_T10DIF=y -CONFIG_FONTS=y -CONFIG_FONT_8x8=y -CONFIG_FONT_8x16=y -CONFIG_DEBUG_INFO=y -CONFIG_DEBUG_FS=y -CONFIG_DETECT_HUNG_TASK=y -CONFIG_CRYPTO_PCBC=m -CONFIG_CRYPTO_SHA256=y -CONFIG_CRYPTO_SHA512=y -# CONFIG_CRYPTO_ANSI_CPRNG is not set -CONFIG_CRYPTO_DEV_FSL_CAAM=y -CONFIG_CRYPTO_DEV_TALITOS=y diff --git a/arch/powerpc/configs/mpc85xx_smp_defconfig b/arch/powerpc/configs/mpc85xx_smp_defconfig deleted file mode 100644 index b6c7111ea913..000000000000 --- a/arch/powerpc/configs/mpc85xx_smp_defconfig +++ /dev/null @@ -1,244 +0,0 @@ -CONFIG_PPC_85xx=y -CONFIG_PHYS_64BIT=y -CONFIG_SMP=y -CONFIG_NR_CPUS=8 -CONFIG_SYSVIPC=y -CONFIG_POSIX_MQUEUE=y -CONFIG_AUDIT=y -CONFIG_IRQ_DOMAIN_DEBUG=y -CONFIG_NO_HZ=y -CONFIG_HIGH_RES_TIMERS=y -CONFIG_BSD_PROCESS_ACCT=y -CONFIG_IKCONFIG=y -CONFIG_IKCONFIG_PROC=y -CONFIG_LOG_BUF_SHIFT=14 -CONFIG_BLK_DEV_INITRD=y -CONFIG_EXPERT=y -CONFIG_KALLSYMS_ALL=y -CONFIG_MODULES=y -CONFIG_MODULE_UNLOAD=y -CONFIG_MODULE_FORCE_UNLOAD=y -CONFIG_MODVERSIONS=y -# CONFIG_BLK_DEV_BSG is not set -CONFIG_PARTITION_ADVANCED=y -CONFIG_MAC_PARTITION=y -CONFIG_C293_PCIE=y -CONFIG_MPC8540_ADS=y -CONFIG_MPC8560_ADS=y -CONFIG_MPC85xx_CDS=y -CONFIG_MPC85xx_MDS=y -CONFIG_MPC8536_DS=y -CONFIG_MPC85xx_DS=y -CONFIG_MPC85xx_RDB=y -CONFIG_P1010_RDB=y -CONFIG_P1022_DS=y -CONFIG_P1022_RDK=y -CONFIG_P1023_RDB=y -CONFIG_SOCRATES=y -CONFIG_KSI8560=y -CONFIG_XES_MPC85xx=y -CONFIG_STX_GP3=y -CONFIG_TQM8540=y -CONFIG_TQM8541=y -CONFIG_TQM8548=y -CONFIG_TQM8555=y -CONFIG_TQM8560=y -CONFIG_SBC8548=y -CONFIG_QUICC_ENGINE=y -CONFIG_QE_GPIO=y -CONFIG_HIGHMEM=y -CONFIG_BINFMT_MISC=m -CONFIG_MATH_EMULATION=y -CONFIG_FORCE_MAX_ZONEORDER=12 -CONFIG_PCI=y -CONFIG_PCI_MSI=y -CONFIG_RAPIDIO=y -CONFIG_NET=y -CONFIG_PACKET=y -CONFIG_UNIX=y -CONFIG_XFRM_USER=y -CONFIG_NET_KEY=y -CONFIG_INET=y -CONFIG_IP_MULTICAST=y -CONFIG_IP_ADVANCED_ROUTER=y -CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IP_ROUTE_MULTIPATH=y -CONFIG_IP_ROUTE_VERBOSE=y -CONFIG_IP_PNP=y -CONFIG_IP_PNP_DHCP=y -CONFIG_IP_PNP_BOOTP=y -CONFIG_IP_PNP_RARP=y -CONFIG_NET_IPIP=y -CONFIG_IP_MROUTE=y -CONFIG_IP_PIMSM_V1=y -CONFIG_IP_PIMSM_V2=y -CONFIG_INET_ESP=y -# CONFIG_INET_XFRM_MODE_BEET is not set -# CONFIG_INET_LRO is not set -CONFIG_IPV6=y -CONFIG_IP_SCTP=m -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" -CONFIG_DEVTMPFS=y -CONFIG_DEVTMPFS_MOUNT=y -CONFIG_MTD=y -CONFIG_MTD_CMDLINE_PARTS=y -CONFIG_MTD_BLOCK=y -CONFIG_FTL=y -CONFIG_MTD_CFI=y -CONFIG_MTD_CFI_INTELEXT=y -CONFIG_MTD_CFI_AMDSTD=y -CONFIG_MTD_PHYSMAP_OF=y -CONFIG_MTD_NAND=y -CONFIG_MTD_NAND_FSL_ELBC=y -CONFIG_MTD_NAND_FSL_IFC=y -CONFIG_MTD_SPI_NOR=y -CONFIG_MTD_UBI=y -CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_NBD=y -CONFIG_BLK_DEV_RAM=y -CONFIG_BLK_DEV_RAM_SIZE=131072 -CONFIG_EEPROM_AT24=y -CONFIG_EEPROM_LEGACY=y -CONFIG_BLK_DEV_SD=y -CONFIG_CHR_DEV_ST=y -CONFIG_BLK_DEV_SR=y -CONFIG_CHR_DEV_SG=y -CONFIG_SCSI_LOGGING=y -CONFIG_ATA=y -CONFIG_SATA_AHCI=y -CONFIG_SATA_FSL=y -CONFIG_SATA_SIL24=y -CONFIG_PATA_ALI=y -CONFIG_NETDEVICES=y -CONFIG_DUMMY=y -CONFIG_FS_ENET=y -CONFIG_UCC_GETH=y -CONFIG_GIANFAR=y -CONFIG_E1000E=y -CONFIG_AT803X_PHY=y -CONFIG_MARVELL_PHY=y -CONFIG_DAVICOM_PHY=y -CONFIG_CICADA_PHY=y -CONFIG_VITESSE_PHY=y -CONFIG_FIXED_PHY=y -CONFIG_INPUT_FF_MEMLESS=m -# CONFIG_INPUT_MOUSEDEV is not set -# CONFIG_INPUT_KEYBOARD is not set -# CONFIG_INPUT_MOUSE is not set -CONFIG_SERIO_LIBPS2=y -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -CONFIG_SERIAL_8250_NR_UARTS=2 -CONFIG_SERIAL_8250_RUNTIME_UARTS=2 -CONFIG_SERIAL_8250_MANY_PORTS=y -CONFIG_SERIAL_8250_DETECT_IRQ=y -CONFIG_SERIAL_8250_RSA=y -CONFIG_SERIAL_QE=m -CONFIG_NVRAM=y -CONFIG_I2C=y -CONFIG_I2C_CHARDEV=y -CONFIG_I2C_CPM=m -CONFIG_I2C_MPC=y -CONFIG_SPI=y -CONFIG_SPI_FSL_SPI=y -CONFIG_SPI_FSL_ESPI=y -CONFIG_GPIO_MPC8XXX=y -CONFIG_SENSORS_LM90=y -CONFIG_FB=y -CONFIG_FB_FSL_DIU=y -# CONFIG_VGA_CONSOLE is not set -CONFIG_FRAMEBUFFER_CONSOLE=y -CONFIG_SOUND=y -CONFIG_SND=y -# CONFIG_SND_SUPPORT_OLD_API is not set -# CONFIG_SND_DRIVERS is not set -CONFIG_SND_INTEL8X0=y -# CONFIG_SND_PPC is not set -# CONFIG_SND_USB is not set -CONFIG_SND_SOC=y -CONFIG_SND_POWERPC_SOC=y -CONFIG_HID_A4TECH=y -CONFIG_HID_APPLE=y -CONFIG_HID_BELKIN=y -CONFIG_HID_CHERRY=y -CONFIG_HID_CHICONY=y -CONFIG_HID_CYPRESS=y -CONFIG_HID_EZKEY=y -CONFIG_HID_GYRATION=y -CONFIG_HID_LOGITECH=y -CONFIG_HID_MICROSOFT=y -CONFIG_HID_MONTEREY=y -CONFIG_HID_PANTHERLORD=y -CONFIG_HID_PETALYNX=y -CONFIG_HID_SAMSUNG=y -CONFIG_HID_SUNPLUS=y -CONFIG_USB=y -CONFIG_USB_MON=y -CONFIG_USB_EHCI_HCD=y -CONFIG_USB_EHCI_FSL=y -CONFIG_USB_OHCI_HCD=y -CONFIG_USB_OHCI_HCD_PPC_OF_BE=y -CONFIG_USB_OHCI_HCD_PPC_OF_LE=y -CONFIG_USB_STORAGE=y -CONFIG_MMC=y -CONFIG_MMC_SDHCI=y -CONFIG_MMC_SDHCI_PLTFM=y -CONFIG_MMC_SDHCI_OF_ESDHC=y -CONFIG_EDAC=y -CONFIG_EDAC_MM_EDAC=y -CONFIG_RTC_CLASS=y -CONFIG_RTC_DRV_DS1307=y -CONFIG_RTC_DRV_DS1374=y -CONFIG_RTC_DRV_DS3232=y -CONFIG_RTC_DRV_CMOS=y -CONFIG_DMADEVICES=y -CONFIG_FSL_DMA=y -CONFIG_EXT2_FS=y -CONFIG_EXT3_FS=y -# CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set -CONFIG_ISO9660_FS=m -CONFIG_JOLIET=y -CONFIG_ZISOFS=y -CONFIG_UDF_FS=m -CONFIG_MSDOS_FS=m -CONFIG_VFAT_FS=y -CONFIG_NTFS_FS=y -CONFIG_PROC_KCORE=y -CONFIG_TMPFS=y -CONFIG_HUGETLBFS=y -CONFIG_ADFS_FS=m -CONFIG_AFFS_FS=m -CONFIG_HFS_FS=m -CONFIG_HFSPLUS_FS=m -CONFIG_BEFS_FS=m -CONFIG_BFS_FS=m -CONFIG_EFS_FS=m -CONFIG_JFFS2_FS=y -CONFIG_JFFS2_FS_DEBUG=1 -CONFIG_UBIFS_FS=y -CONFIG_CRAMFS=y -CONFIG_VXFS_FS=m -CONFIG_HPFS_FS=m -CONFIG_QNX4FS_FS=m -CONFIG_SYSV_FS=m -CONFIG_UFS_FS=m -CONFIG_NFS_FS=y -CONFIG_NFS_V4=y -CONFIG_ROOT_NFS=y -CONFIG_NFSD=y -CONFIG_NLS_CODEPAGE_437=y -CONFIG_NLS_CODEPAGE_850=y -CONFIG_NLS_ISO8859_1=y -CONFIG_CRC_T10DIF=y -CONFIG_FONTS=y -CONFIG_FONT_8x8=y -CONFIG_FONT_8x16=y -CONFIG_DEBUG_INFO=y -CONFIG_DEBUG_FS=y -CONFIG_DETECT_HUNG_TASK=y -CONFIG_CRYPTO_PCBC=m -CONFIG_CRYPTO_SHA256=y -CONFIG_CRYPTO_SHA512=y -# CONFIG_CRYPTO_ANSI_CPRNG is not set -CONFIG_CRYPTO_DEV_FSL_CAAM=y -CONFIG_CRYPTO_DEV_TALITOS=y diff --git a/arch/powerpc/configs/ppc64_defconfig b/arch/powerpc/configs/ppc64_defconfig index a97efc2146fd..6bc0ee4b1070 100644 --- a/arch/powerpc/configs/ppc64_defconfig +++ b/arch/powerpc/configs/ppc64_defconfig @@ -355,3 +355,6 @@ CONFIG_CRYPTO_DEV_NX_ENCRYPT=m CONFIG_VIRTUALIZATION=y CONFIG_KVM_BOOK3S_64=m CONFIG_KVM_BOOK3S_64_HV=m +CONFIG_NEW_LEDS=y +CONFIG_LEDS_CLASS=m +CONFIG_LEDS_POWERNV=m diff --git a/arch/powerpc/configs/pseries_defconfig b/arch/powerpc/configs/pseries_defconfig index 0d9efcedaf34..7991f37e5fe2 100644 --- a/arch/powerpc/configs/pseries_defconfig +++ b/arch/powerpc/configs/pseries_defconfig @@ -190,7 +190,8 @@ CONFIG_HVC_RTAS=y CONFIG_HVCS=m CONFIG_VIRTIO_CONSOLE=m CONFIG_IBM_BSR=m -CONFIG_GEN_RTC=y +CONFIG_RTC_CLASS=y +CONFIG_RTC_DRV_GENERIC=y CONFIG_RAW_DRIVER=y CONFIG_MAX_RAW_DEVS=1024 CONFIG_FB=y @@ -319,3 +320,6 @@ CONFIG_CRYPTO_DEV_NX_ENCRYPT=m CONFIG_VIRTUALIZATION=y CONFIG_KVM_BOOK3S_64=m CONFIG_KVM_BOOK3S_64_HV=m +CONFIG_NEW_LEDS=y +CONFIG_LEDS_CLASS=m +CONFIG_LEDS_POWERNV=m diff --git a/arch/powerpc/include/asm/Kbuild b/arch/powerpc/include/asm/Kbuild index 050712e1ce41..ab9f4e0ed4cf 100644 --- a/arch/powerpc/include/asm/Kbuild +++ b/arch/powerpc/include/asm/Kbuild @@ -6,5 +6,4 @@ generic-y += local64.h generic-y += mcs_spinlock.h generic-y += preempt.h generic-y += rwsem.h -generic-y += trace_clock.h generic-y += vtime.h diff --git a/arch/powerpc/include/asm/archrandom.h b/arch/powerpc/include/asm/archrandom.h index 0cc6eedc4780..85e88f7a59c0 100644 --- a/arch/powerpc/include/asm/archrandom.h +++ b/arch/powerpc/include/asm/archrandom.h @@ -7,13 +7,22 @@ static inline int arch_get_random_long(unsigned long *v) { - if (ppc_md.get_random_long) - return ppc_md.get_random_long(v); - return 0; } static inline int arch_get_random_int(unsigned int *v) +{ + return 0; +} + +static inline int arch_get_random_seed_long(unsigned long *v) +{ + if (ppc_md.get_random_seed) + return ppc_md.get_random_seed(v); + + return 0; +} +static inline int arch_get_random_seed_int(unsigned int *v) { unsigned long val; int rc; @@ -27,22 +36,13 @@ static inline int arch_get_random_int(unsigned int *v) static inline int arch_has_random(void) { - return !!ppc_md.get_random_long; + return 0; } -static inline int arch_get_random_seed_long(unsigned long *v) -{ - return 0; -} -static inline int arch_get_random_seed_int(unsigned int *v) -{ - return 0; -} static inline int arch_has_random_seed(void) { - return 0; + return !!ppc_md.get_random_seed; } - #endif /* CONFIG_ARCH_RANDOM */ #ifdef CONFIG_PPC_POWERNV diff --git a/arch/powerpc/include/asm/atomic.h b/arch/powerpc/include/asm/atomic.h index 512d2782b043..55f106ed12bf 100644 --- a/arch/powerpc/include/asm/atomic.h +++ b/arch/powerpc/include/asm/atomic.h @@ -67,6 +67,10 @@ static __inline__ int atomic_##op##_return(int a, atomic_t *v) \ ATOMIC_OPS(add, add) ATOMIC_OPS(sub, subf) +ATOMIC_OP(and, and) +ATOMIC_OP(or, or) +ATOMIC_OP(xor, xor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -304,6 +308,9 @@ static __inline__ long atomic64_##op##_return(long a, atomic64_t *v) \ ATOMIC64_OPS(add, add) ATOMIC64_OPS(sub, subf) +ATOMIC64_OP(and, and) +ATOMIC64_OP(or, or) +ATOMIC64_OP(xor, xor) #undef ATOMIC64_OPS #undef ATOMIC64_OP_RETURN diff --git a/arch/powerpc/include/asm/barrier.h b/arch/powerpc/include/asm/barrier.h index 51ccc7232042..0eca6efc0631 100644 --- a/arch/powerpc/include/asm/barrier.h +++ b/arch/powerpc/include/asm/barrier.h @@ -76,12 +76,12 @@ do { \ compiletime_assert_atomic_type(*p); \ smp_lwsync(); \ - ACCESS_ONCE(*p) = (v); \ + WRITE_ONCE(*p, v); \ } while (0) #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ smp_lwsync(); \ ___p1; \ diff --git a/arch/powerpc/include/asm/cacheflush.h b/arch/powerpc/include/asm/cacheflush.h index 30b35fff2dea..6229e6b6037b 100644 --- a/arch/powerpc/include/asm/cacheflush.h +++ b/arch/powerpc/include/asm/cacheflush.h @@ -40,7 +40,12 @@ extern void __flush_dcache_icache(void *page_va); extern void flush_dcache_icache_page(struct page *page); #if defined(CONFIG_PPC32) && !defined(CONFIG_BOOKE) extern void __flush_dcache_icache_phys(unsigned long physaddr); -#endif /* CONFIG_PPC32 && !CONFIG_BOOKE */ +#else +static inline void __flush_dcache_icache_phys(unsigned long physaddr) +{ + BUG(); +} +#endif extern void flush_dcache_range(unsigned long start, unsigned long stop); #ifdef CONFIG_PPC32 diff --git a/arch/powerpc/include/asm/checksum.h b/arch/powerpc/include/asm/checksum.h index 8251a3ba870f..e8d9ef4755a4 100644 --- a/arch/powerpc/include/asm/checksum.h +++ b/arch/powerpc/include/asm/checksum.h @@ -19,15 +19,6 @@ #else extern __sum16 ip_fast_csum(const void *iph, unsigned int ihl); -/* - * computes the checksum of the TCP/UDP pseudo-header - * returns a 16-bit checksum, already complemented - */ -extern __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, - unsigned short len, - unsigned short proto, - __wsum sum); - /* * computes the checksum of a memory block at buff, length len, * and adds in "sum" (32-bit) @@ -127,6 +118,34 @@ static inline __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, #endif } +/* + * computes the checksum of the TCP/UDP pseudo-header + * returns a 16-bit checksum, already complemented + */ +static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, + unsigned short len, + unsigned short proto, + __wsum sum) +{ + return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum)); +} + +#define HAVE_ARCH_CSUM_ADD +static inline __wsum csum_add(__wsum csum, __wsum addend) +{ +#ifdef __powerpc64__ + u64 res = (__force u64)csum; + + res += (__force u64)addend; + return (__force __wsum)((u32)res + (res >> 32)); +#else + asm("addc %0,%0,%1;" + "addze %0,%0;" + : "+r" (csum) : "r" (addend)); + return csum; +#endif +} + #endif #endif /* __KERNEL__ */ #endif diff --git a/arch/powerpc/include/asm/compat.h b/arch/powerpc/include/asm/compat.h index b142b8e0ed9e..4f2df589ec1d 100644 --- a/arch/powerpc/include/asm/compat.h +++ b/arch/powerpc/include/asm/compat.h @@ -174,6 +174,13 @@ typedef struct compat_siginfo { int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ int _fd; } _sigpoll; + + /* SIGSYS */ + struct { + unsigned int _call_addr; /* calling insn */ + int _syscall; /* triggering system call number */ + unsigned int _arch; /* AUDIT_ARCH_* of syscall */ + } _sigsys; } _sifields; } compat_siginfo_t; diff --git a/arch/powerpc/include/asm/device.h b/arch/powerpc/include/asm/device.h index e9bdda88f1fb..406c2b1ff82d 100644 --- a/arch/powerpc/include/asm/device.h +++ b/arch/powerpc/include/asm/device.h @@ -10,6 +10,7 @@ struct dma_map_ops; struct device_node; #ifdef CONFIG_PPC64 struct pci_dn; +struct iommu_table; #endif /* @@ -23,13 +24,15 @@ struct dev_archdata { struct dma_map_ops *dma_ops; /* - * When an iommu is in use, dma_data is used as a ptr to the base of the - * iommu_table. Otherwise, it is a simple numerical offset. + * These two used to be a union. However, with the hybrid ops we need + * both so here we store both a DMA offset for direct mappings and + * an iommu_table for remapped DMA. */ - union { - dma_addr_t dma_offset; - void *iommu_table_base; - } dma_data; + dma_addr_t dma_offset; + +#ifdef CONFIG_PPC64 + struct iommu_table *iommu_table_base; +#endif #ifdef CONFIG_IOMMU_API void *iommu_domain; diff --git a/arch/powerpc/include/asm/dma-mapping.h b/arch/powerpc/include/asm/dma-mapping.h index 9103687b0436..7f522c021dc3 100644 --- a/arch/powerpc/include/asm/dma-mapping.h +++ b/arch/powerpc/include/asm/dma-mapping.h @@ -18,15 +18,17 @@ #include #include +#ifdef CONFIG_PPC64 #define DMA_ERROR_CODE (~(dma_addr_t)0x0) +#endif /* Some dma direct funcs must be visible for use in other dma_ops */ -extern void *dma_direct_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag, +extern void *__dma_direct_alloc_coherent(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t flag, + struct dma_attrs *attrs); +extern void __dma_direct_free_coherent(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma_handle, struct dma_attrs *attrs); -extern void dma_direct_free_coherent(struct device *dev, size_t size, - void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs); extern int dma_direct_mmap_coherent(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t handle, @@ -106,7 +108,7 @@ static inline void set_dma_ops(struct device *dev, struct dma_map_ops *ops) static inline dma_addr_t get_dma_offset(struct device *dev) { if (dev) - return dev->archdata.dma_data.dma_offset; + return dev->archdata.dma_offset; return PCI_DRAM_OFFSET; } @@ -114,77 +116,20 @@ static inline dma_addr_t get_dma_offset(struct device *dev) static inline void set_dma_offset(struct device *dev, dma_addr_t off) { if (dev) - dev->archdata.dma_data.dma_offset = off; + dev->archdata.dma_offset = off; } /* this will be removed soon */ #define flush_write_buffers() +#define HAVE_ARCH_DMA_SET_MASK 1 +extern int dma_set_mask(struct device *dev, u64 dma_mask); + #include -static inline int dma_supported(struct device *dev, u64 mask) -{ - struct dma_map_ops *dma_ops = get_dma_ops(dev); - - if (unlikely(dma_ops == NULL)) - return 0; - if (dma_ops->dma_supported == NULL) - return 1; - return dma_ops->dma_supported(dev, mask); -} - -extern int dma_set_mask(struct device *dev, u64 dma_mask); extern int __dma_set_mask(struct device *dev, u64 dma_mask); extern u64 __dma_get_required_mask(struct device *dev); -#define dma_alloc_coherent(d,s,h,f) dma_alloc_attrs(d,s,h,f,NULL) - -static inline void *dma_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) -{ - struct dma_map_ops *dma_ops = get_dma_ops(dev); - void *cpu_addr; - - BUG_ON(!dma_ops); - - cpu_addr = dma_ops->alloc(dev, size, dma_handle, flag, attrs); - - debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr); - - return cpu_addr; -} - -#define dma_free_coherent(d,s,c,h) dma_free_attrs(d,s,c,h,NULL) - -static inline void dma_free_attrs(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t dma_handle, - struct dma_attrs *attrs) -{ - struct dma_map_ops *dma_ops = get_dma_ops(dev); - - BUG_ON(!dma_ops); - - debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); - - dma_ops->free(dev, size, cpu_addr, dma_handle, attrs); -} - -static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - struct dma_map_ops *dma_ops = get_dma_ops(dev); - - debug_dma_mapping_error(dev, dma_addr); - if (dma_ops->mapping_error) - return dma_ops->mapping_error(dev, dma_addr); - -#ifdef CONFIG_PPC64 - return (dma_addr == DMA_ERROR_CODE); -#else - return 0; -#endif -} - static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) { #ifdef CONFIG_SWIOTLB @@ -210,9 +155,6 @@ static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr) return daddr - get_dma_offset(dev); } -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) -#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) - #define ARCH_HAS_DMA_MMAP_COHERENT static inline void dma_cache_sync(struct device *dev, void *vaddr, size_t size, diff --git a/arch/powerpc/include/asm/ftrace.h b/arch/powerpc/include/asm/ftrace.h index e3661872fbea..ef89b1465573 100644 --- a/arch/powerpc/include/asm/ftrace.h +++ b/arch/powerpc/include/asm/ftrace.h @@ -2,7 +2,7 @@ #define _ASM_POWERPC_FTRACE #ifdef CONFIG_FUNCTION_TRACER -#define MCOUNT_ADDR ((long)(_mcount)) +#define MCOUNT_ADDR ((unsigned long)(_mcount)) #define MCOUNT_INSN_SIZE 4 /* sizeof mcount call */ #ifdef __ASSEMBLY__ diff --git a/arch/powerpc/include/asm/iommu.h b/arch/powerpc/include/asm/iommu.h index ca18cff90900..7b87bab09564 100644 --- a/arch/powerpc/include/asm/iommu.h +++ b/arch/powerpc/include/asm/iommu.h @@ -2,17 +2,17 @@ * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation * Rewrite, cleanup: * Copyright (C) 2004 Olof Johansson , IBM Corporation - * + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. - * + * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * + * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA @@ -131,16 +131,21 @@ int get_iommu_order(unsigned long size, struct iommu_table *tbl) struct scatterlist; -static inline void set_iommu_table_base(struct device *dev, void *base) +#ifdef CONFIG_PPC64 + +static inline void set_iommu_table_base(struct device *dev, + struct iommu_table *base) { - dev->archdata.dma_data.iommu_table_base = base; + dev->archdata.iommu_table_base = base; } static inline void *get_iommu_table_base(struct device *dev) { - return dev->archdata.dma_data.iommu_table_base; + return dev->archdata.iommu_table_base; } +extern int dma_iommu_dma_supported(struct device *dev, u64 mask); + /* Frees table for an individual device node */ extern void iommu_free_table(struct iommu_table *tbl, const char *node_name); @@ -225,6 +230,20 @@ static inline int __init tce_iommu_bus_notifier_init(void) } #endif /* !CONFIG_IOMMU_API */ +#else + +static inline void *get_iommu_table_base(struct device *dev) +{ + return NULL; +} + +static inline int dma_iommu_dma_supported(struct device *dev, u64 mask) +{ + return 0; +} + +#endif /* CONFIG_PPC64 */ + extern int ppc_iommu_map_sg(struct device *dev, struct iommu_table *tbl, struct scatterlist *sglist, int nelems, unsigned long mask, diff --git a/arch/powerpc/include/asm/jump_label.h b/arch/powerpc/include/asm/jump_label.h index efbf9a322a23..47e155f15433 100644 --- a/arch/powerpc/include/asm/jump_label.h +++ b/arch/powerpc/include/asm/jump_label.h @@ -18,14 +18,29 @@ #define JUMP_ENTRY_TYPE stringify_in_c(FTR_ENTRY_LONG) #define JUMP_LABEL_NOP_SIZE 4 -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("1:\n\t" "nop\n\t" ".pushsection __jump_table, \"aw\"\n\t" JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t" ".popsection \n\t" - : : "i" (key) : : l_yes); + : : "i" (&((char *)key)[branch]) : : l_yes); + + return false; +l_yes: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("1:\n\t" + "b %l[l_yes]\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); + return false; l_yes: return true; diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h index b91e74a817d8..9fac01cb89c1 100644 --- a/arch/powerpc/include/asm/kvm_book3s.h +++ b/arch/powerpc/include/asm/kvm_book3s.h @@ -158,6 +158,7 @@ extern pfn_t kvmppc_gpa_to_pfn(struct kvm_vcpu *vcpu, gpa_t gpa, bool writing, bool *writable); extern void kvmppc_add_revmap_chain(struct kvm *kvm, struct revmap_entry *rev, unsigned long *rmap, long pte_index, int realmode); +extern void kvmppc_update_rmap_change(unsigned long *rmap, unsigned long psize); extern void kvmppc_invalidate_hpte(struct kvm *kvm, __be64 *hptep, unsigned long pte_index); void kvmppc_clear_ref_hpte(struct kvm *kvm, __be64 *hptep, @@ -225,12 +226,12 @@ static inline u32 kvmppc_get_cr(struct kvm_vcpu *vcpu) return vcpu->arch.cr; } -static inline void kvmppc_set_xer(struct kvm_vcpu *vcpu, u32 val) +static inline void kvmppc_set_xer(struct kvm_vcpu *vcpu, ulong val) { vcpu->arch.xer = val; } -static inline u32 kvmppc_get_xer(struct kvm_vcpu *vcpu) +static inline ulong kvmppc_get_xer(struct kvm_vcpu *vcpu) { return vcpu->arch.xer; } diff --git a/arch/powerpc/include/asm/kvm_book3s_asm.h b/arch/powerpc/include/asm/kvm_book3s_asm.h index 5bdfb5dd3400..72b6225aca73 100644 --- a/arch/powerpc/include/asm/kvm_book3s_asm.h +++ b/arch/powerpc/include/asm/kvm_book3s_asm.h @@ -25,6 +25,12 @@ #define XICS_MFRR 0xc #define XICS_IPI 2 /* interrupt source # for IPIs */ +/* Maximum number of threads per physical core */ +#define MAX_SMT_THREADS 8 + +/* Maximum number of subcores per physical core */ +#define MAX_SUBCORES 4 + #ifdef __ASSEMBLY__ #ifdef CONFIG_KVM_BOOK3S_HANDLER @@ -65,6 +71,19 @@ kvmppc_resume_\intno: #else /*__ASSEMBLY__ */ +struct kvmppc_vcore; + +/* Struct used for coordinating micro-threading (split-core) mode changes */ +struct kvm_split_mode { + unsigned long rpr; + unsigned long pmmar; + unsigned long ldbar; + u8 subcore_size; + u8 do_nap; + u8 napped[MAX_SMT_THREADS]; + struct kvmppc_vcore *master_vcs[MAX_SUBCORES]; +}; + /* * This struct goes in the PACA on 64-bit processors. It is used * to store host state that needs to be saved when we enter a guest @@ -100,6 +119,7 @@ struct kvmppc_host_state { u64 host_spurr; u64 host_dscr; u64 dec_expires; + struct kvm_split_mode *kvm_split_mode; #endif #ifdef CONFIG_PPC_BOOK3S_64 u64 cfar; @@ -112,7 +132,7 @@ struct kvmppc_book3s_shadow_vcpu { bool in_use; ulong gpr[14]; u32 cr; - u32 xer; + ulong xer; ulong ctr; ulong lr; ulong pc; diff --git a/arch/powerpc/include/asm/kvm_booke.h b/arch/powerpc/include/asm/kvm_booke.h index 3286f0d6a86c..bc6e29e4dfd4 100644 --- a/arch/powerpc/include/asm/kvm_booke.h +++ b/arch/powerpc/include/asm/kvm_booke.h @@ -54,12 +54,12 @@ static inline u32 kvmppc_get_cr(struct kvm_vcpu *vcpu) return vcpu->arch.cr; } -static inline void kvmppc_set_xer(struct kvm_vcpu *vcpu, u32 val) +static inline void kvmppc_set_xer(struct kvm_vcpu *vcpu, ulong val) { vcpu->arch.xer = val; } -static inline u32 kvmppc_get_xer(struct kvm_vcpu *vcpu) +static inline ulong kvmppc_get_xer(struct kvm_vcpu *vcpu) { return vcpu->arch.xer; } diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index d91f65b28e32..98eebbf66340 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h @@ -205,8 +205,10 @@ struct revmap_entry { */ #define KVMPPC_RMAP_LOCK_BIT 63 #define KVMPPC_RMAP_RC_SHIFT 32 +#define KVMPPC_RMAP_CHG_SHIFT 48 #define KVMPPC_RMAP_REFERENCED (HPTE_R_R << KVMPPC_RMAP_RC_SHIFT) #define KVMPPC_RMAP_CHANGED (HPTE_R_C << KVMPPC_RMAP_RC_SHIFT) +#define KVMPPC_RMAP_CHG_ORDER (0x3ful << KVMPPC_RMAP_CHG_SHIFT) #define KVMPPC_RMAP_PRESENT 0x100000000ul #define KVMPPC_RMAP_INDEX 0xfffffffful @@ -278,7 +280,9 @@ struct kvmppc_vcore { u16 last_cpu; u8 vcore_state; u8 in_guest; + struct kvmppc_vcore *master_vcore; struct list_head runnable_threads; + struct list_head preempt_list; spinlock_t lock; wait_queue_head_t wq; spinlock_t stoltb_lock; /* protects stolen_tb and preempt_tb */ @@ -300,12 +304,21 @@ struct kvmppc_vcore { #define VCORE_EXIT_MAP(vc) ((vc)->entry_exit_map >> 8) #define VCORE_IS_EXITING(vc) (VCORE_EXIT_MAP(vc) != 0) -/* Values for vcore_state */ +/* This bit is used when a vcore exit is triggered from outside the vcore */ +#define VCORE_EXIT_REQ 0x10000 + +/* + * Values for vcore_state. + * Note that these are arranged such that lower values + * (< VCORE_SLEEPING) don't require stolen time accounting + * on load/unload, and higher values do. + */ #define VCORE_INACTIVE 0 -#define VCORE_SLEEPING 1 -#define VCORE_PREEMPT 2 -#define VCORE_RUNNING 3 -#define VCORE_EXITING 4 +#define VCORE_PREEMPT 1 +#define VCORE_PIGGYBACK 2 +#define VCORE_SLEEPING 3 +#define VCORE_RUNNING 4 +#define VCORE_EXITING 5 /* * Struct used to manage memory for a virtual processor area @@ -473,7 +486,7 @@ struct kvm_vcpu_arch { ulong ciabr; ulong cfar; ulong ppr; - ulong pspb; + u32 pspb; ulong fscr; ulong shadow_fscr; ulong ebbhr; @@ -619,6 +632,7 @@ struct kvm_vcpu_arch { int trap; int state; int ptid; + int thread_cpu; bool timer_running; wait_queue_head_t cpu_run; diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h index 952579f5e79a..cab6753f1be5 100644 --- a/arch/powerpc/include/asm/machdep.h +++ b/arch/powerpc/include/asm/machdep.h @@ -249,7 +249,7 @@ struct machdep_calls { #endif #ifdef CONFIG_ARCH_RANDOM - int (*get_random_long)(unsigned long *v); + int (*get_random_seed)(unsigned long *v); #endif }; diff --git a/arch/powerpc/include/asm/opal-api.h b/arch/powerpc/include/asm/opal-api.h index e9e4c52f3685..8374afed9d0a 100644 --- a/arch/powerpc/include/asm/opal-api.h +++ b/arch/powerpc/include/asm/opal-api.h @@ -154,7 +154,10 @@ #define OPAL_FLASH_WRITE 111 #define OPAL_FLASH_ERASE 112 #define OPAL_PRD_MSG 113 -#define OPAL_LAST 113 +#define OPAL_LEDS_GET_INDICATOR 114 +#define OPAL_LEDS_SET_INDICATOR 115 +#define OPAL_CEC_REBOOT2 116 +#define OPAL_LAST 116 /* Device tree flags */ @@ -340,6 +343,18 @@ enum OpalPciResetState { OPAL_ASSERT_RESET = 1 }; +enum OpalSlotLedType { + OPAL_SLOT_LED_TYPE_ID = 0, /* IDENTIFY LED */ + OPAL_SLOT_LED_TYPE_FAULT = 1, /* FAULT LED */ + OPAL_SLOT_LED_TYPE_ATTN = 2, /* System Attention LED */ + OPAL_SLOT_LED_TYPE_MAX = 3 +}; + +enum OpalSlotLedState { + OPAL_SLOT_LED_STATE_OFF = 0, /* LED is OFF */ + OPAL_SLOT_LED_STATE_ON = 1 /* LED is ON */ +}; + /* * Address cycle types for LPC accesses. These also correspond * to the content of the first cell of the "reg" property for @@ -361,6 +376,7 @@ enum opal_msg_type { OPAL_MSG_HMI_EVT, OPAL_MSG_DPO, OPAL_MSG_PRD, + OPAL_MSG_OCC, OPAL_MSG_TYPE_MAX, }; @@ -437,6 +453,7 @@ struct OpalMemoryErrorData { /* HMI interrupt event */ enum OpalHMI_Version { OpalHMIEvt_V1 = 1, + OpalHMIEvt_V2 = 2, }; enum OpalHMI_Severity { @@ -467,6 +484,49 @@ enum OpalHMI_ErrType { OpalHMI_ERROR_CAPP_RECOVERY, }; +enum OpalHMI_XstopType { + CHECKSTOP_TYPE_UNKNOWN = 0, + CHECKSTOP_TYPE_CORE = 1, + CHECKSTOP_TYPE_NX = 2, +}; + +enum OpalHMI_CoreXstopReason { + CORE_CHECKSTOP_IFU_REGFILE = 0x00000001, + CORE_CHECKSTOP_IFU_LOGIC = 0x00000002, + CORE_CHECKSTOP_PC_DURING_RECOV = 0x00000004, + CORE_CHECKSTOP_ISU_REGFILE = 0x00000008, + CORE_CHECKSTOP_ISU_LOGIC = 0x00000010, + CORE_CHECKSTOP_FXU_LOGIC = 0x00000020, + CORE_CHECKSTOP_VSU_LOGIC = 0x00000040, + CORE_CHECKSTOP_PC_RECOV_IN_MAINT_MODE = 0x00000080, + CORE_CHECKSTOP_LSU_REGFILE = 0x00000100, + CORE_CHECKSTOP_PC_FWD_PROGRESS = 0x00000200, + CORE_CHECKSTOP_LSU_LOGIC = 0x00000400, + CORE_CHECKSTOP_PC_LOGIC = 0x00000800, + CORE_CHECKSTOP_PC_HYP_RESOURCE = 0x00001000, + CORE_CHECKSTOP_PC_HANG_RECOV_FAILED = 0x00002000, + CORE_CHECKSTOP_PC_AMBI_HANG_DETECTED = 0x00004000, + CORE_CHECKSTOP_PC_DEBUG_TRIG_ERR_INJ = 0x00008000, + CORE_CHECKSTOP_PC_SPRD_HYP_ERR_INJ = 0x00010000, +}; + +enum OpalHMI_NestAccelXstopReason { + NX_CHECKSTOP_SHM_INVAL_STATE_ERR = 0x00000001, + NX_CHECKSTOP_DMA_INVAL_STATE_ERR_1 = 0x00000002, + NX_CHECKSTOP_DMA_INVAL_STATE_ERR_2 = 0x00000004, + NX_CHECKSTOP_DMA_CH0_INVAL_STATE_ERR = 0x00000008, + NX_CHECKSTOP_DMA_CH1_INVAL_STATE_ERR = 0x00000010, + NX_CHECKSTOP_DMA_CH2_INVAL_STATE_ERR = 0x00000020, + NX_CHECKSTOP_DMA_CH3_INVAL_STATE_ERR = 0x00000040, + NX_CHECKSTOP_DMA_CH4_INVAL_STATE_ERR = 0x00000080, + NX_CHECKSTOP_DMA_CH5_INVAL_STATE_ERR = 0x00000100, + NX_CHECKSTOP_DMA_CH6_INVAL_STATE_ERR = 0x00000200, + NX_CHECKSTOP_DMA_CH7_INVAL_STATE_ERR = 0x00000400, + NX_CHECKSTOP_DMA_CRB_UE = 0x00000800, + NX_CHECKSTOP_DMA_CRB_SUE = 0x00001000, + NX_CHECKSTOP_PBI_ISN_UE = 0x00002000, +}; + struct OpalHMIEvent { uint8_t version; /* 0x00 */ uint8_t severity; /* 0x01 */ @@ -477,6 +537,23 @@ struct OpalHMIEvent { __be64 hmer; /* TFMR register. Valid only for TFAC and TFMR_PARITY error type. */ __be64 tfmr; + + /* version 2 and later */ + union { + /* + * checkstop info (Core/NX). + * Valid for OpalHMI_ERROR_MALFUNC_ALERT. + */ + struct { + uint8_t xstop_type; /* enum OpalHMI_XstopType */ + uint8_t reserved_1[3]; + __be32 xstop_reason; + union { + __be32 pir; /* for CHECKSTOP_TYPE_CORE */ + __be32 chip_id; /* for CHECKSTOP_TYPE_NX */ + } u; + } xstop_error; + } u; }; enum { @@ -700,6 +777,17 @@ struct opal_prd_msg_header { struct opal_prd_msg; +#define OCC_RESET 0 +#define OCC_LOAD 1 +#define OCC_THROTTLE 2 +#define OCC_MAX_THROTTLE_STATUS 5 + +struct opal_occ_msg { + __be64 type; + __be64 chip; + __be64 throttle_status; +}; + /* * SG entries * @@ -756,6 +844,52 @@ struct opal_i2c_request { __be64 buffer_ra; /* Buffer real address */ }; +/* + * EPOW status sharing (OPAL and the host) + * + * The host will pass on OPAL, a buffer of length OPAL_SYSEPOW_MAX + * with individual elements being 16 bits wide to fetch the system + * wide EPOW status. Each element in the buffer will contain the + * EPOW status in it's bit representation for a particular EPOW sub + * class as defiend here. So multiple detailed EPOW status bits + * specific for any sub class can be represented in a single buffer + * element as it's bit representation. + */ + +/* System EPOW type */ +enum OpalSysEpow { + OPAL_SYSEPOW_POWER = 0, /* Power EPOW */ + OPAL_SYSEPOW_TEMP = 1, /* Temperature EPOW */ + OPAL_SYSEPOW_COOLING = 2, /* Cooling EPOW */ + OPAL_SYSEPOW_MAX = 3, /* Max EPOW categories */ +}; + +/* Power EPOW */ +enum OpalSysPower { + OPAL_SYSPOWER_UPS = 0x0001, /* System on UPS power */ + OPAL_SYSPOWER_CHNG = 0x0002, /* System power config change */ + OPAL_SYSPOWER_FAIL = 0x0004, /* System impending power failure */ + OPAL_SYSPOWER_INCL = 0x0008, /* System incomplete power */ +}; + +/* Temperature EPOW */ +enum OpalSysTemp { + OPAL_SYSTEMP_AMB = 0x0001, /* System over ambient temperature */ + OPAL_SYSTEMP_INT = 0x0002, /* System over internal temperature */ + OPAL_SYSTEMP_HMD = 0x0004, /* System over ambient humidity */ +}; + +/* Cooling EPOW */ +enum OpalSysCooling { + OPAL_SYSCOOL_INSF = 0x0001, /* System insufficient cooling */ +}; + +/* Argument to OPAL_CEC_REBOOT2() */ +enum { + OPAL_REBOOT_NORMAL = 0, + OPAL_REBOOT_PLATFORM_ERROR = 1, +}; + #endif /* __ASSEMBLY__ */ #endif /* __OPAL_API_H */ diff --git a/arch/powerpc/include/asm/opal.h b/arch/powerpc/include/asm/opal.h index 958e941c0cda..800115910e43 100644 --- a/arch/powerpc/include/asm/opal.h +++ b/arch/powerpc/include/asm/opal.h @@ -44,6 +44,7 @@ int64_t opal_tpo_write(uint64_t token, uint32_t year_mon_day, uint32_t hour_min); int64_t opal_cec_power_down(uint64_t request); int64_t opal_cec_reboot(void); +int64_t opal_cec_reboot2(uint32_t reboot_type, char *diag); int64_t opal_read_nvram(uint64_t buffer, uint64_t size, uint64_t offset); int64_t opal_write_nvram(uint64_t buffer, uint64_t size, uint64_t offset); int64_t opal_handle_interrupt(uint64_t isn, __be64 *outstanding_event_mask); @@ -141,7 +142,8 @@ int64_t opal_pci_fence_phb(uint64_t phb_id); int64_t opal_pci_reinit(uint64_t phb_id, uint64_t reinit_scope, uint64_t data); int64_t opal_pci_mask_pe_error(uint64_t phb_id, uint16_t pe_number, uint8_t error_type, uint8_t mask_action); int64_t opal_set_slot_led_status(uint64_t phb_id, uint64_t slot_id, uint8_t led_type, uint8_t led_action); -int64_t opal_get_epow_status(__be64 *status); +int64_t opal_get_epow_status(__be16 *epow_status, __be16 *num_epow_classes); +int64_t opal_get_dpo_status(__be64 *dpo_timeout); int64_t opal_set_system_attention_led(uint8_t led_action); int64_t opal_pci_next_error(uint64_t phb_id, __be64 *first_frozen_pe, __be16 *pci_error_type, __be16 *severity); @@ -195,6 +197,10 @@ int64_t opal_ipmi_recv(uint64_t interface, struct opal_ipmi_msg *msg, int64_t opal_i2c_request(uint64_t async_token, uint32_t bus_id, struct opal_i2c_request *oreq); int64_t opal_prd_msg(struct opal_prd_msg *msg); +int64_t opal_leds_get_ind(char *loc_code, __be64 *led_mask, + __be64 *led_value, __be64 *max_led_type); +int64_t opal_leds_set_ind(uint64_t token, char *loc_code, const u64 led_mask, + const u64 led_value, __be64 *max_led_type); int64_t opal_flash_read(uint64_t id, uint64_t offset, uint64_t buf, uint64_t size, uint64_t token); diff --git a/arch/powerpc/include/asm/pci-bridge.h b/arch/powerpc/include/asm/pci-bridge.h index 712add590445..37fc53587bb4 100644 --- a/arch/powerpc/include/asm/pci-bridge.h +++ b/arch/powerpc/include/asm/pci-bridge.h @@ -42,6 +42,7 @@ struct pci_controller_ops { #endif int (*dma_set_mask)(struct pci_dev *dev, u64 dma_mask); + u64 (*dma_get_required_mask)(struct pci_dev *dev); void (*shutdown)(struct pci_controller *); }; diff --git a/arch/powerpc/include/asm/pgtable-ppc64.h b/arch/powerpc/include/asm/pgtable-ppc64.h index 3bb7488bd24b..fa1dfb7f7b48 100644 --- a/arch/powerpc/include/asm/pgtable-ppc64.h +++ b/arch/powerpc/include/asm/pgtable-ppc64.h @@ -134,11 +134,11 @@ #define pte_iterate_hashed_end() } while(0) -#ifdef CONFIG_PPC_HAS_HASH_64K -#define pte_pagesize_index(mm, addr, pte) get_slice_psize(mm, addr) -#else +/* + * We expect this to be called only for user addresses or kernel virtual + * addresses other than the linear mapping. + */ #define pte_pagesize_index(mm, addr, pte) MMU_PAGE_4K -#endif #endif /* __real_pte */ diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h index 11a38635dd65..0717693c8428 100644 --- a/arch/powerpc/include/asm/pgtable.h +++ b/arch/powerpc/include/asm/pgtable.h @@ -169,6 +169,17 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, * cases, and 32-bit non-hash with 32-bit PTEs. */ *ptep = pte; + +#ifdef CONFIG_PPC_BOOK3E_64 + /* + * With hardware tablewalk, a sync is needed to ensure that + * subsequent accesses see the PTE we just wrote. Unlike userspace + * mappings, we can't tolerate spurious faults, so make sure + * the new PTE will be seen the first time. + */ + if (is_kernel_addr(addr)) + mb(); +#endif #endif } diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h index 8452335661a5..790f5d1d9a46 100644 --- a/arch/powerpc/include/asm/ppc-opcode.h +++ b/arch/powerpc/include/asm/ppc-opcode.h @@ -287,7 +287,7 @@ /* POWER8 Micro Partition Prefetch (MPP) parameters */ /* Address mask is common for LOGMPP instruction and MPPR SPR */ -#define PPC_MPPE_ADDRESS_MASK 0xffffffffc000 +#define PPC_MPPE_ADDRESS_MASK 0xffffffffc000ULL /* Bits 60 and 61 of MPP SPR should be set to one of the following */ /* Aborting the fetch is indeed setting 00 in the table size bits */ diff --git a/arch/powerpc/include/asm/ppc-pci.h b/arch/powerpc/include/asm/ppc-pci.h index 4122a86d6858..ca0c5bff7849 100644 --- a/arch/powerpc/include/asm/ppc-pci.h +++ b/arch/powerpc/include/asm/ppc-pci.h @@ -61,6 +61,7 @@ int rtas_write_config(struct pci_dn *, int where, int size, u32 val); int rtas_read_config(struct pci_dn *, int where, int size, u32 *val); void eeh_pe_state_mark(struct eeh_pe *pe, int state); void eeh_pe_state_clear(struct eeh_pe *pe, int state); +void eeh_pe_state_mark_with_cfg(struct eeh_pe *pe, int state); void eeh_pe_dev_mode_mark(struct eeh_pe *pe, int mode); void eeh_sysfs_add_device(struct pci_dev *pdev); diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h index 28ded5d9b579..5afea361beaa 100644 --- a/arch/powerpc/include/asm/processor.h +++ b/arch/powerpc/include/asm/processor.h @@ -264,7 +264,6 @@ struct thread_struct { u64 tm_tfhar; /* Transaction fail handler addr */ u64 tm_texasr; /* Transaction exception & summary */ u64 tm_tfiar; /* Transaction fail instr address reg */ - unsigned long tm_orig_msr; /* Thread's MSR on ctx switch */ struct pt_regs ckpt_regs; /* Checkpointed registers */ unsigned long tm_tar; diff --git a/arch/powerpc/include/asm/pte-common.h b/arch/powerpc/include/asm/pte-common.h index b7c8d079c121..71537a319fc8 100644 --- a/arch/powerpc/include/asm/pte-common.h +++ b/arch/powerpc/include/asm/pte-common.h @@ -109,7 +109,8 @@ extern unsigned long bad_call_to_PMD_PAGE_SIZE(void); * the processor might need it for DMA coherency. */ #define _PAGE_BASE_NC (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_PSIZE) -#if defined(CONFIG_SMP) || defined(CONFIG_PPC_STD_MMU) +#if defined(CONFIG_SMP) || defined(CONFIG_PPC_STD_MMU) || \ + defined(CONFIG_PPC_E500MC) #define _PAGE_BASE (_PAGE_BASE_NC | _PAGE_COHERENT) #else #define _PAGE_BASE (_PAGE_BASE_NC) diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h index af56b5c6c81a..aa1cc5f015ee 100644 --- a/arch/powerpc/include/asm/reg.h +++ b/arch/powerpc/include/asm/reg.h @@ -1193,8 +1193,7 @@ #ifdef CONFIG_PPC_BOOK3S_64 #define __mtmsrd(v, l) asm volatile("mtmsrd %0," __stringify(l) \ : : "r" (v) : "memory") -#define mtmsrd(v) __mtmsrd((v), 0) -#define mtmsr(v) mtmsrd(v) +#define mtmsr(v) __mtmsrd((v), 0) #else #define mtmsr(v) asm volatile("mtmsr %0" : \ : "r" ((unsigned long)(v)) \ @@ -1281,6 +1280,15 @@ struct pt_regs; extern void ppc_save_regs(struct pt_regs *regs); +static inline void update_power8_hid0(unsigned long hid0) +{ + /* + * The HID0 update on Power8 should at the very least be + * preceded by a a SYNC instruction followed by an ISYNC + * instruction + */ + asm volatile("sync; mtspr %0,%1; isync":: "i"(SPRN_HID0), "r"(hid0)); +} #endif /* __ASSEMBLY__ */ #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_REG_H */ diff --git a/arch/powerpc/include/asm/rtas.h b/arch/powerpc/include/asm/rtas.h index 7a4ede16b283..b77ef369c0f0 100644 --- a/arch/powerpc/include/asm/rtas.h +++ b/arch/powerpc/include/asm/rtas.h @@ -343,6 +343,7 @@ extern void rtas_power_off(void); extern void rtas_halt(void); extern void rtas_os_term(char *str); extern int rtas_get_sensor(int sensor, int index, int *state); +extern int rtas_get_sensor_fast(int sensor, int index, int *state); extern int rtas_get_power_level(int powerdomain, int *level); extern int rtas_set_power_level(int powerdomain, int level, int *setlevel); extern bool rtas_indicator_present(int token, int *maxindex); diff --git a/arch/powerpc/include/asm/spu_csa.h b/arch/powerpc/include/asm/spu_csa.h index a40fd491250c..51f80b41cda3 100644 --- a/arch/powerpc/include/asm/spu_csa.h +++ b/arch/powerpc/include/asm/spu_csa.h @@ -241,12 +241,6 @@ struct spu_priv2_collapsed { */ struct spu_state { struct spu_lscsa *lscsa; -#ifdef CONFIG_SPU_FS_64K_LS - int use_big_pages; - /* One struct page per 64k page */ -#define SPU_LSCSA_NUM_BIG_PAGES (sizeof(struct spu_lscsa) / 0x10000) - struct page *lscsa_pages[SPU_LSCSA_NUM_BIG_PAGES]; -#endif struct spu_problem_collapsed prob; struct spu_priv1_collapsed priv1; struct spu_priv2_collapsed priv2; diff --git a/arch/powerpc/include/asm/syscall.h b/arch/powerpc/include/asm/syscall.h index ff21b7a2f0cc..ab9f3f0a8637 100644 --- a/arch/powerpc/include/asm/syscall.h +++ b/arch/powerpc/include/asm/syscall.h @@ -22,10 +22,15 @@ extern const unsigned long sys_call_table[]; #endif /* CONFIG_FTRACE_SYSCALLS */ -static inline long syscall_get_nr(struct task_struct *task, - struct pt_regs *regs) +static inline int syscall_get_nr(struct task_struct *task, struct pt_regs *regs) { - return TRAP(regs) == 0xc00 ? regs->gpr[0] : -1L; + /* + * Note that we are returning an int here. That means 0xffffffff, ie. + * 32-bit negative 1, will be interpreted as -1 on a 64-bit kernel. + * This is important for seccomp so that compat tasks can set r0 = -1 + * to reject the syscall. + */ + return TRAP(regs) == 0xc00 ? regs->gpr[0] : -1; } static inline void syscall_rollback(struct task_struct *task, @@ -34,12 +39,6 @@ static inline void syscall_rollback(struct task_struct *task, regs->gpr[3] = regs->orig_gpr3; } -static inline long syscall_get_error(struct task_struct *task, - struct pt_regs *regs) -{ - return (regs->ccr & 0x10000000) ? -regs->gpr[3] : 0; -} - static inline long syscall_get_return_value(struct task_struct *task, struct pt_regs *regs) { @@ -50,9 +49,15 @@ static inline void syscall_set_return_value(struct task_struct *task, struct pt_regs *regs, int error, long val) { + /* + * In the general case it's not obvious that we must deal with CCR + * here, as the syscall exit path will also do that for us. However + * there are some places, eg. the signal code, which check ccr to + * decide if the value in r3 is actually an error. + */ if (error) { regs->ccr |= 0x10000000L; - regs->gpr[3] = -error; + regs->gpr[3] = error; } else { regs->ccr &= ~0x10000000L; regs->gpr[3] = val; @@ -64,19 +69,22 @@ static inline void syscall_get_arguments(struct task_struct *task, unsigned int i, unsigned int n, unsigned long *args) { + unsigned long val, mask = -1UL; + BUG_ON(i + n > 6); -#ifdef CONFIG_PPC64 - if (test_tsk_thread_flag(task, TIF_32BIT)) { - /* - * Zero-extend 32-bit argument values. The high bits are - * garbage ignored by the actual syscall dispatch. - */ - while (n-- > 0) - args[n] = (u32) regs->gpr[3 + i + n]; - return; - } + +#ifdef CONFIG_COMPAT + if (test_tsk_thread_flag(task, TIF_32BIT)) + mask = 0xffffffff; #endif - memcpy(args, ®s->gpr[3 + i], n * sizeof(args[0])); + while (n--) { + if (n == 0 && i == 0) + val = regs->orig_gpr3; + else + val = regs->gpr[3 + i + n]; + + args[n] = val & mask; + } } static inline void syscall_set_arguments(struct task_struct *task, @@ -86,6 +94,10 @@ static inline void syscall_set_arguments(struct task_struct *task, { BUG_ON(i + n > 6); memcpy(®s->gpr[3 + i], args, n * sizeof(args[0])); + + /* Also copy the first argument into orig_gpr3 */ + if (i == 0 && n > 0) + regs->orig_gpr3 = args[0]; } static inline int syscall_get_arch(void) diff --git a/arch/powerpc/include/asm/trace_clock.h b/arch/powerpc/include/asm/trace_clock.h new file mode 100644 index 000000000000..cf1ee75ca069 --- /dev/null +++ b/arch/powerpc/include/asm/trace_clock.h @@ -0,0 +1,19 @@ +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2, as + * published by the Free Software Foundation. + * + * Copyright (C) 2015 Naveen N. Rao, IBM Corporation + */ + +#ifndef _ASM_PPC_TRACE_CLOCK_H +#define _ASM_PPC_TRACE_CLOCK_H + +#include +#include + +extern u64 notrace trace_clock_ppc_tb(void); + +#define ARCH_TRACE_CLOCKS { trace_clock_ppc_tb, "ppc-tb", 0 }, + +#endif /* _ASM_PPC_TRACE_CLOCK_H */ diff --git a/arch/powerpc/include/uapi/asm/Kbuild b/arch/powerpc/include/uapi/asm/Kbuild index f44a027818af..dab3717e3ea0 100644 --- a/arch/powerpc/include/uapi/asm/Kbuild +++ b/arch/powerpc/include/uapi/asm/Kbuild @@ -6,6 +6,7 @@ header-y += bitsperlong.h header-y += bootx.h header-y += byteorder.h header-y += cputable.h +header-y += eeh.h header-y += elf.h header-y += epapr_hcalls.h header-y += errno.h diff --git a/arch/powerpc/include/uapi/asm/errno.h b/arch/powerpc/include/uapi/asm/errno.h index 8c145fd17d86..e8b6b5f7de7c 100644 --- a/arch/powerpc/include/uapi/asm/errno.h +++ b/arch/powerpc/include/uapi/asm/errno.h @@ -6,6 +6,4 @@ #undef EDEADLOCK #define EDEADLOCK 58 /* File locking deadlock error */ -#define _LAST_ERRNO 516 - #endif /* _ASM_POWERPC_ERRNO_H */ diff --git a/arch/powerpc/include/uapi/asm/sigcontext.h b/arch/powerpc/include/uapi/asm/sigcontext.h index 9c1f24fd5d11..3ad0c7f001a9 100644 --- a/arch/powerpc/include/uapi/asm/sigcontext.h +++ b/arch/powerpc/include/uapi/asm/sigcontext.h @@ -28,7 +28,7 @@ struct sigcontext { /* * To maintain compatibility with current implementations the sigcontext is * extended by appending a pointer (v_regs) to a quadword type (elf_vrreg_t) - * followed by an unstructured (vmx_reserve) field of 69 doublewords. This + * followed by an unstructured (vmx_reserve) field of 101 doublewords. This * allows the array of vector registers to be quadword aligned independent of * the alignment of the containing sigcontext or ucontext. It is the * responsibility of the code setting the sigcontext to set this pointer to @@ -80,7 +80,7 @@ struct sigcontext { * registers and vscr/vrsave. */ elf_vrreg_t __user *v_regs; - long vmx_reserve[ELF_NVRREG+ELF_NVRREG+32+1]; + long vmx_reserve[ELF_NVRREG + ELF_NVRREG + 1 + 32]; #endif }; diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile index 12868b1c4e05..ba336930d448 100644 --- a/arch/powerpc/kernel/Makefile +++ b/arch/powerpc/kernel/Makefile @@ -118,6 +118,7 @@ obj-$(CONFIG_PPC_IO_WORKAROUNDS) += io-workarounds.o obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o obj-$(CONFIG_FTRACE_SYSCALLS) += ftrace.o +obj-$(CONFIG_TRACING) += trace_clock.o ifneq ($(CONFIG_PPC_INDIRECT_PIO),y) obj-y += iomap.o diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c index 98230579d99c..221d584d089f 100644 --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -213,7 +213,6 @@ int main(void) offsetof(struct tlb_core_data, esel_max)); DEFINE(TCD_ESEL_FIRST, offsetof(struct tlb_core_data, esel_first)); - DEFINE(TCD_LOCK, offsetof(struct tlb_core_data, lock)); #endif /* CONFIG_PPC_BOOK3E */ #ifdef CONFIG_PPC_STD_MMU_64 @@ -512,6 +511,8 @@ int main(void) DEFINE(VCPU_VPA, offsetof(struct kvm_vcpu, arch.vpa.pinned_addr)); DEFINE(VCPU_VPA_DIRTY, offsetof(struct kvm_vcpu, arch.vpa.dirty)); DEFINE(VCPU_HEIR, offsetof(struct kvm_vcpu, arch.emul_inst)); + DEFINE(VCPU_CPU, offsetof(struct kvm_vcpu, cpu)); + DEFINE(VCPU_THREAD_CPU, offsetof(struct kvm_vcpu, arch.thread_cpu)); #endif #ifdef CONFIG_PPC_BOOK3S DEFINE(VCPU_VCPUID, offsetof(struct kvm_vcpu, vcpu_id)); @@ -674,7 +675,14 @@ int main(void) HSTATE_FIELD(HSTATE_DSCR, host_dscr); HSTATE_FIELD(HSTATE_DABR, dabr); HSTATE_FIELD(HSTATE_DECEXP, dec_expires); + HSTATE_FIELD(HSTATE_SPLIT_MODE, kvm_split_mode); DEFINE(IPI_PRIORITY, IPI_PRIORITY); + DEFINE(KVM_SPLIT_RPR, offsetof(struct kvm_split_mode, rpr)); + DEFINE(KVM_SPLIT_PMMAR, offsetof(struct kvm_split_mode, pmmar)); + DEFINE(KVM_SPLIT_LDBAR, offsetof(struct kvm_split_mode, ldbar)); + DEFINE(KVM_SPLIT_SIZE, offsetof(struct kvm_split_mode, subcore_size)); + DEFINE(KVM_SPLIT_DO_NAP, offsetof(struct kvm_split_mode, do_nap)); + DEFINE(KVM_SPLIT_NAPPED, offsetof(struct kvm_split_mode, napped)); #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ #ifdef CONFIG_PPC_BOOK3S_64 diff --git a/arch/powerpc/kernel/dma-iommu.c b/arch/powerpc/kernel/dma-iommu.c index 4c68bfe4108a..41a7d9d49a5a 100644 --- a/arch/powerpc/kernel/dma-iommu.c +++ b/arch/powerpc/kernel/dma-iommu.c @@ -73,7 +73,7 @@ static void dma_iommu_unmap_sg(struct device *dev, struct scatterlist *sglist, } /* We support DMA to/from any memory page via the iommu */ -static int dma_iommu_dma_supported(struct device *dev, u64 mask) +int dma_iommu_dma_supported(struct device *dev, u64 mask) { struct iommu_table *tbl = get_iommu_table_base(dev); diff --git a/arch/powerpc/kernel/dma-swiotlb.c b/arch/powerpc/kernel/dma-swiotlb.c index 6e8d764ce47b..c6689f658b50 100644 --- a/arch/powerpc/kernel/dma-swiotlb.c +++ b/arch/powerpc/kernel/dma-swiotlb.c @@ -47,8 +47,8 @@ static u64 swiotlb_powerpc_get_required(struct device *dev) * for everything else. */ struct dma_map_ops swiotlb_dma_ops = { - .alloc = dma_direct_alloc_coherent, - .free = dma_direct_free_coherent, + .alloc = __dma_direct_alloc_coherent, + .free = __dma_direct_free_coherent, .mmap = dma_direct_mmap_coherent, .map_sg = swiotlb_map_sg_attrs, .unmap_sg = swiotlb_unmap_sg_attrs, diff --git a/arch/powerpc/kernel/dma.c b/arch/powerpc/kernel/dma.c index 35e4dcc5dce3..59503ed98e5f 100644 --- a/arch/powerpc/kernel/dma.c +++ b/arch/powerpc/kernel/dma.c @@ -16,6 +16,7 @@ #include #include #include +#include /* * Generic direct DMA implementation @@ -39,9 +40,31 @@ static u64 __maybe_unused get_pfn_limit(struct device *dev) return pfn; } -void *dma_direct_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) +static int dma_direct_dma_supported(struct device *dev, u64 mask) +{ +#ifdef CONFIG_PPC64 + u64 limit = get_dma_offset(dev) + (memblock_end_of_DRAM() - 1); + + /* Limit fits in the mask, we are good */ + if (mask >= limit) + return 1; + +#ifdef CONFIG_FSL_SOC + /* Freescale gets another chance via ZONE_DMA/ZONE_DMA32, however + * that will have to be refined if/when they support iommus + */ + return 1; +#endif + /* Sorry ... */ + return 0; +#else + return 1; +#endif +} + +void *__dma_direct_alloc_coherent(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t flag, + struct dma_attrs *attrs) { void *ret; #ifdef CONFIG_NOT_COHERENT_CACHE @@ -96,9 +119,9 @@ void *dma_direct_alloc_coherent(struct device *dev, size_t size, #endif } -void dma_direct_free_coherent(struct device *dev, size_t size, - void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) +void __dma_direct_free_coherent(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma_handle, + struct dma_attrs *attrs) { #ifdef CONFIG_NOT_COHERENT_CACHE __dma_free_coherent(size, vaddr); @@ -107,6 +130,51 @@ void dma_direct_free_coherent(struct device *dev, size_t size, #endif } +static void *dma_direct_alloc_coherent(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t flag, + struct dma_attrs *attrs) +{ + struct iommu_table *iommu; + + /* The coherent mask may be smaller than the real mask, check if + * we can really use the direct ops + */ + if (dma_direct_dma_supported(dev, dev->coherent_dma_mask)) + return __dma_direct_alloc_coherent(dev, size, dma_handle, + flag, attrs); + + /* Ok we can't ... do we have an iommu ? If not, fail */ + iommu = get_iommu_table_base(dev); + if (!iommu) + return NULL; + + /* Try to use the iommu */ + return iommu_alloc_coherent(dev, iommu, size, dma_handle, + dev->coherent_dma_mask, flag, + dev_to_node(dev)); +} + +static void dma_direct_free_coherent(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma_handle, + struct dma_attrs *attrs) +{ + struct iommu_table *iommu; + + /* See comments in dma_direct_alloc_coherent() */ + if (dma_direct_dma_supported(dev, dev->coherent_dma_mask)) + return __dma_direct_free_coherent(dev, size, vaddr, dma_handle, + attrs); + /* Maybe we used an iommu ... */ + iommu = get_iommu_table_base(dev); + + /* If we hit that we should have never allocated in the first + * place so how come we are freeing ? + */ + if (WARN_ON(!iommu)) + return; + iommu_free_coherent(iommu, size, vaddr, dma_handle); +} + int dma_direct_mmap_coherent(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t handle, size_t size, struct dma_attrs *attrs) @@ -147,18 +215,6 @@ static void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sg, { } -static int dma_direct_dma_supported(struct device *dev, u64 mask) -{ -#ifdef CONFIG_PPC64 - /* Could be improved so platforms can set the limit in case - * they have limited DMA windows - */ - return mask >= get_dma_offset(dev) + (memblock_end_of_DRAM() - 1); -#else - return 1; -#endif -} - static u64 dma_direct_get_required_mask(struct device *dev) { u64 end, mask; @@ -230,6 +286,25 @@ struct dma_map_ops dma_direct_ops = { }; EXPORT_SYMBOL(dma_direct_ops); +int dma_set_coherent_mask(struct device *dev, u64 mask) +{ + if (!dma_supported(dev, mask)) { + /* + * We need to special case the direct DMA ops which can + * support a fallback for coherent allocations. There + * is no dma_op->set_coherent_mask() so we have to do + * things the hard way: + */ + if (get_dma_ops(dev) != &dma_direct_ops || + get_iommu_table_base(dev) == NULL || + !dma_iommu_dma_supported(dev, mask)) + return -EIO; + } + dev->coherent_dma_mask = mask; + return 0; +} +EXPORT_SYMBOL_GPL(dma_set_coherent_mask); + #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16) int __dma_set_mask(struct device *dev, u64 dma_mask) @@ -278,6 +353,13 @@ u64 dma_get_required_mask(struct device *dev) if (ppc_md.dma_get_required_mask) return ppc_md.dma_get_required_mask(dev); + if (dev_is_pci(dev)) { + struct pci_dev *pdev = to_pci_dev(dev); + struct pci_controller *phb = pci_bus_to_host(pdev->bus); + if (phb->controller_ops.dma_get_required_mask) + return phb->controller_ops.dma_get_required_mask(pdev); + } + return __dma_get_required_mask(dev); } EXPORT_SYMBOL_GPL(dma_get_required_mask); diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c index af9b597b10af..e968533e3e05 100644 --- a/arch/powerpc/kernel/eeh.c +++ b/arch/powerpc/kernel/eeh.c @@ -308,11 +308,26 @@ void eeh_slot_error_detail(struct eeh_pe *pe, int severity) if (!(pe->type & EEH_PE_PHB)) { if (eeh_has_flag(EEH_ENABLE_IO_FOR_LOG)) eeh_pci_enable(pe, EEH_OPT_THAW_MMIO); - eeh_ops->configure_bridge(pe); - eeh_pe_restore_bars(pe); - pci_regs_buf[0] = 0; - eeh_pe_traverse(pe, eeh_dump_pe_log, &loglen); + /* + * The config space of some PCI devices can't be accessed + * when their PEs are in frozen state. Otherwise, fenced + * PHB might be seen. Those PEs are identified with flag + * EEH_PE_CFG_RESTRICTED, indicating EEH_PE_CFG_BLOCKED + * is set automatically when the PE is put to EEH_PE_ISOLATED. + * + * Restoring BARs possibly triggers PCI config access in + * (OPAL) firmware and then causes fenced PHB. If the + * PCI config is blocked with flag EEH_PE_CFG_BLOCKED, it's + * pointless to restore BARs and dump config space. + */ + eeh_ops->configure_bridge(pe); + if (!(pe->state & EEH_PE_CFG_BLOCKED)) { + eeh_pe_restore_bars(pe); + + pci_regs_buf[0] = 0; + eeh_pe_traverse(pe, eeh_dump_pe_log, &loglen); + } } eeh_ops->get_log(pe, severity, pci_regs_buf, loglen); @@ -750,14 +765,14 @@ int pcibios_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state stat eeh_pe_state_clear(pe, EEH_PE_ISOLATED); break; case pcie_hot_reset: - eeh_pe_state_mark(pe, EEH_PE_ISOLATED); + eeh_pe_state_mark_with_cfg(pe, EEH_PE_ISOLATED); eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE); eeh_pe_dev_traverse(pe, eeh_disable_and_save_dev_state, dev); eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED); eeh_ops->reset(pe, EEH_RESET_HOT); break; case pcie_warm_reset: - eeh_pe_state_mark(pe, EEH_PE_ISOLATED); + eeh_pe_state_mark_with_cfg(pe, EEH_PE_ISOLATED); eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE); eeh_pe_dev_traverse(pe, eeh_disable_and_save_dev_state, dev); eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED); @@ -1116,9 +1131,6 @@ void eeh_add_device_late(struct pci_dev *dev) return; } - if (eeh_has_flag(EEH_PROBE_MODE_DEV)) - eeh_ops->probe(pdn, NULL); - /* * The EEH cache might not be removed correctly because of * unbalanced kref to the device during unplug time, which @@ -1142,6 +1154,9 @@ void eeh_add_device_late(struct pci_dev *dev) dev->dev.archdata.edev = NULL; } + if (eeh_has_flag(EEH_PROBE_MODE_DEV)) + eeh_ops->probe(pdn, NULL); + edev->pdev = dev; dev->dev.archdata.edev = edev; diff --git a/arch/powerpc/kernel/eeh_pe.c b/arch/powerpc/kernel/eeh_pe.c index 35f0b62259bb..8654cb166c19 100644 --- a/arch/powerpc/kernel/eeh_pe.c +++ b/arch/powerpc/kernel/eeh_pe.c @@ -657,6 +657,28 @@ void eeh_pe_state_clear(struct eeh_pe *pe, int state) eeh_pe_traverse(pe, __eeh_pe_state_clear, &state); } +/** + * eeh_pe_state_mark_with_cfg - Mark PE state with unblocked config space + * @pe: PE + * @state: PE state to be set + * + * Set specified flag to PE and its child PEs. The PCI config space + * of some PEs is blocked automatically when EEH_PE_ISOLATED is set, + * which isn't needed in some situations. The function allows to set + * the specified flag to indicated PEs without blocking their PCI + * config space. + */ +void eeh_pe_state_mark_with_cfg(struct eeh_pe *pe, int state) +{ + eeh_pe_traverse(pe, __eeh_pe_state_mark, &state); + if (!(state & EEH_PE_ISOLATED)) + return; + + /* Clear EEH_PE_CFG_BLOCKED, which might be set just now */ + state = EEH_PE_CFG_BLOCKED; + eeh_pe_traverse(pe, __eeh_pe_state_clear, &state); +} + /* * Some PCI bridges (e.g. PLX bridges) have primary/secondary * buses assigned explicitly by firmware, and we probably have diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S index 46fc0f4d8982..2405631e91a2 100644 --- a/arch/powerpc/kernel/entry_32.S +++ b/arch/powerpc/kernel/entry_32.S @@ -20,6 +20,7 @@ */ #include +#include #include #include #include @@ -354,7 +355,7 @@ ret_from_syscall: SYNC MTMSRD(r10) lwz r9,TI_FLAGS(r12) - li r8,-_LAST_ERRNO + li r8,-MAX_ERRNO andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK) bne- syscall_exit_work cmplw 0,r3,r8 @@ -457,6 +458,10 @@ syscall_dotrace: lwz r7,GPR7(r1) lwz r8,GPR8(r1) REST_NVGPRS(r1) + + cmplwi r0,NR_syscalls + /* Return code is already in r3 thanks to do_syscall_trace_enter() */ + bge- ret_from_syscall b syscall_dotrace_cont syscall_exit_work: diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S index 579e0f9a2d57..a94f155db78e 100644 --- a/arch/powerpc/kernel/entry_64.S +++ b/arch/powerpc/kernel/entry_64.S @@ -19,6 +19,7 @@ */ #include +#include #include #include #include @@ -150,8 +151,7 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR) CURRENT_THREAD_INFO(r11, r1) ld r10,TI_FLAGS(r11) andi. r11,r10,_TIF_SYSCALL_DOTRACE - bne syscall_dotrace -.Lsyscall_dotrace_cont: + bne syscall_dotrace /* does not return */ cmpldi 0,r0,NR_syscalls bge- syscall_enosys @@ -207,7 +207,7 @@ system_call: /* label this so stack traces look sane */ #endif /* CONFIG_PPC_BOOK3E */ ld r9,TI_FLAGS(r12) - li r11,-_LAST_ERRNO + li r11,-MAX_ERRNO andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK) bne- syscall_exit_work cmpld r3,r11 @@ -245,22 +245,34 @@ syscall_dotrace: bl save_nvgprs addi r3,r1,STACK_FRAME_OVERHEAD bl do_syscall_trace_enter + /* - * Restore argument registers possibly just changed. - * We use the return value of do_syscall_trace_enter - * for the call number to look up in the table (r0). + * We use the return value of do_syscall_trace_enter() as the syscall + * number. If the syscall was rejected for any reason do_syscall_trace_enter() + * returns an invalid syscall number and the test below against + * NR_syscalls will fail. */ mr r0,r3 + + /* Restore argument registers just clobbered and/or possibly changed. */ ld r3,GPR3(r1) ld r4,GPR4(r1) ld r5,GPR5(r1) ld r6,GPR6(r1) ld r7,GPR7(r1) ld r8,GPR8(r1) + + /* Repopulate r9 and r10 for the system_call path */ addi r9,r1,STACK_FRAME_OVERHEAD CURRENT_THREAD_INFO(r10, r1) ld r10,TI_FLAGS(r10) - b .Lsyscall_dotrace_cont + + cmpldi r0,NR_syscalls + blt+ system_call + + /* Return code is already in r3 thanks to do_syscall_trace_enter() */ + b .Lsyscall_exit + syscall_enosys: li r3,-ENOSYS @@ -277,7 +289,7 @@ syscall_exit_work: beq+ 0f REST_NVGPRS(r1) b 2f -0: cmpld r3,r11 /* r10 is -LAST_ERRNO */ +0: cmpld r3,r11 /* r11 is -MAX_ERRNO */ blt+ 1f andi. r0,r9,_TIF_NOERROR bne- 1f diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S index 3e68d1c69718..f3bd5e747ed8 100644 --- a/arch/powerpc/kernel/exceptions-64e.S +++ b/arch/powerpc/kernel/exceptions-64e.S @@ -1313,11 +1313,14 @@ skpinv: addi r6,r6,1 /* Increment */ sync isync -/* The mapping only needs to be cache-coherent on SMP */ -#ifdef CONFIG_SMP -#define M_IF_SMP MAS2_M +/* + * The mapping only needs to be cache-coherent on SMP, except on + * Freescale e500mc derivatives where it's also needed for coherent DMA. + */ +#if defined(CONFIG_SMP) || defined(CONFIG_PPC_E500MC) +#define M_IF_NEEDED MAS2_M #else -#define M_IF_SMP 0 +#define M_IF_NEEDED 0 #endif /* 6. Setup KERNELBASE mapping in TLB[0] @@ -1332,7 +1335,7 @@ skpinv: addi r6,r6,1 /* Increment */ ori r6,r6,(MAS1_TSIZE(BOOK3E_PAGESZ_1GB))@l mtspr SPRN_MAS1,r6 - LOAD_REG_IMMEDIATE(r6, PAGE_OFFSET | M_IF_SMP) + LOAD_REG_IMMEDIATE(r6, PAGE_OFFSET | M_IF_NEEDED) mtspr SPRN_MAS2,r6 rlwinm r5,r5,0,0,25 diff --git a/arch/powerpc/kernel/fsl_booke_entry_mapping.S b/arch/powerpc/kernel/fsl_booke_entry_mapping.S index f22e7e44fbf3..83dd0f6776b3 100644 --- a/arch/powerpc/kernel/fsl_booke_entry_mapping.S +++ b/arch/powerpc/kernel/fsl_booke_entry_mapping.S @@ -152,11 +152,14 @@ skpinv: addi r6,r6,1 /* Increment */ tlbivax 0,r9 TLBSYNC -/* The mapping only needs to be cache-coherent on SMP */ -#ifdef CONFIG_SMP -#define M_IF_SMP MAS2_M +/* + * The mapping only needs to be cache-coherent on SMP, except on + * Freescale e500mc derivatives where it's also needed for coherent DMA. + */ +#if defined(CONFIG_SMP) || defined(CONFIG_PPC_E500MC) +#define M_IF_NEEDED MAS2_M #else -#define M_IF_SMP 0 +#define M_IF_NEEDED 0 #endif #if defined(ENTRY_MAPPING_BOOT_SETUP) @@ -167,8 +170,8 @@ skpinv: addi r6,r6,1 /* Increment */ lis r6,(MAS1_VALID|MAS1_IPROT)@h ori r6,r6,(MAS1_TSIZE(BOOK3E_PAGESZ_64M))@l mtspr SPRN_MAS1,r6 - lis r6,MAS2_VAL(PAGE_OFFSET, BOOK3E_PAGESZ_64M, M_IF_SMP)@h - ori r6,r6,MAS2_VAL(PAGE_OFFSET, BOOK3E_PAGESZ_64M, M_IF_SMP)@l + lis r6,MAS2_VAL(PAGE_OFFSET, BOOK3E_PAGESZ_64M, M_IF_NEEDED)@h + ori r6,r6,MAS2_VAL(PAGE_OFFSET, BOOK3E_PAGESZ_64M, M_IF_NEEDED)@l mtspr SPRN_MAS2,r6 mtspr SPRN_MAS3,r8 tlbwe diff --git a/arch/powerpc/kernel/jump_label.c b/arch/powerpc/kernel/jump_label.c index a1ed8a8c7cb4..6472472093d0 100644 --- a/arch/powerpc/kernel/jump_label.c +++ b/arch/powerpc/kernel/jump_label.c @@ -17,7 +17,7 @@ void arch_jump_label_transform(struct jump_entry *entry, { u32 *addr = (u32 *)(unsigned long)entry->code; - if (type == JUMP_LABEL_ENABLE) + if (type == JUMP_LABEL_JMP) patch_branch(addr, entry->target, 0); else patch_instruction(addr, PPC_INST_NOP); diff --git a/arch/powerpc/kernel/kvm.c b/arch/powerpc/kernel/kvm.c index 33aa4ddf597d..9ad37f827a97 100644 --- a/arch/powerpc/kernel/kvm.c +++ b/arch/powerpc/kernel/kvm.c @@ -649,7 +649,6 @@ static void kvm_check_ins(u32 *inst, u32 features) kvm_patch_ins_mtsrin(inst, inst_rt, inst_rb); } break; - break; #endif } diff --git a/arch/powerpc/kernel/misc_32.S b/arch/powerpc/kernel/misc_32.S index 7c6bb4b17b49..ed3ab509faca 100644 --- a/arch/powerpc/kernel/misc_32.S +++ b/arch/powerpc/kernel/misc_32.S @@ -595,25 +595,6 @@ _GLOBAL(copy_page) li r11,4 b 2b -/* - * void atomic_clear_mask(atomic_t mask, atomic_t *addr) - * void atomic_set_mask(atomic_t mask, atomic_t *addr); - */ -_GLOBAL(atomic_clear_mask) -10: lwarx r5,0,r4 - andc r5,r5,r3 - PPC405_ERR77(0,r4) - stwcx. r5,0,r4 - bne- 10b - blr -_GLOBAL(atomic_set_mask) -10: lwarx r5,0,r4 - or r5,r5,r3 - PPC405_ERR77(0,r4) - stwcx. r5,0,r4 - bne- 10b - blr - /* * Extended precision shifts. * diff --git a/arch/powerpc/kernel/misc_64.S b/arch/powerpc/kernel/misc_64.S index 4e314b90c75d..6e4168cf4698 100644 --- a/arch/powerpc/kernel/misc_64.S +++ b/arch/powerpc/kernel/misc_64.S @@ -475,9 +475,18 @@ _GLOBAL(kexec_wait) #ifdef CONFIG_KEXEC /* use no memory without kexec */ lwz r4,0(r5) cmpwi 0,r4,0 - bnea 0x60 + beq 99b +#ifdef CONFIG_PPC_BOOK3S_64 + li r10,0x60 + mfmsr r11 + clrrdi r11,r11,1 /* Clear MSR_LE */ + mtsrr0 r10 + mtsrr1 r11 + rfid +#else + ba 0x60 +#endif #endif - b 99b /* this can be in text because we won't change it until we are * running in real anyways diff --git a/arch/powerpc/kernel/nvram_64.c b/arch/powerpc/kernel/nvram_64.c index 1e703f8ebad4..98ba106a59ef 100644 --- a/arch/powerpc/kernel/nvram_64.c +++ b/arch/powerpc/kernel/nvram_64.c @@ -541,10 +541,9 @@ static ssize_t nvram_pstore_read(u64 *id, enum pstore_type_id *type, time->tv_sec = be64_to_cpu(oops_hdr->timestamp); time->tv_nsec = 0; } - *buf = kmalloc(length, GFP_KERNEL); + *buf = kmemdup(buff + hdr_size, length, GFP_KERNEL); if (*buf == NULL) return -ENOMEM; - memcpy(*buf, buff + hdr_size, length); kfree(buff); if (err_type == ERR_TYPE_KERNEL_PANIC_GZ) @@ -582,9 +581,10 @@ static int nvram_pstore_init(void) spin_lock_init(&nvram_pstore_info.buf_lock); rc = pstore_register(&nvram_pstore_info); - if (rc != 0) - pr_err("nvram: pstore_register() failed, defaults to " - "kmsg_dump; returned %d\n", rc); + if (rc && (rc != -EPERM)) + /* Print error only when pstore.backend == nvram */ + pr_err("nvram: pstore_register() failed, returned %d. " + "Defaults to kmsg_dump\n", rc); return rc; } diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c index 02c1d5dcee4d..a1d0632d97c6 100644 --- a/arch/powerpc/kernel/pci-common.c +++ b/arch/powerpc/kernel/pci-common.c @@ -823,23 +823,15 @@ static void pcibios_fixup_resources(struct pci_dev *dev) (reg.start == 0 && !pci_has_flag(PCI_PROBE_ONLY))) { /* Only print message if not re-assigning */ if (!pci_has_flag(PCI_REASSIGN_ALL_RSRC)) - pr_debug("PCI:%s Resource %d %016llx-%016llx [%x] " - "is unassigned\n", - pci_name(dev), i, - (unsigned long long)res->start, - (unsigned long long)res->end, - (unsigned int)res->flags); + pr_debug("PCI:%s Resource %d %pR is unassigned\n", + pci_name(dev), i, res); res->end -= res->start; res->start = 0; res->flags |= IORESOURCE_UNSET; continue; } - pr_debug("PCI:%s Resource %d %016llx-%016llx [%x]\n", - pci_name(dev), i, - (unsigned long long)res->start,\ - (unsigned long long)res->end, - (unsigned int)res->flags); + pr_debug("PCI:%s Resource %d %pR\n", pci_name(dev), i, res); } /* Call machine specific resource fixup */ @@ -943,11 +935,7 @@ static void pcibios_fixup_bridge(struct pci_bus *bus) continue; } - pr_debug("PCI:%s Bus rsrc %d %016llx-%016llx [%x]\n", - pci_name(dev), i, - (unsigned long long)res->start,\ - (unsigned long long)res->end, - (unsigned int)res->flags); + pr_debug("PCI:%s Bus rsrc %d %pR\n", pci_name(dev), i, res); /* Try to detect uninitialized P2P bridge resources, * and clear them out so they get re-assigned later @@ -1126,10 +1114,8 @@ static int reparent_resources(struct resource *parent, *pp = NULL; for (p = res->child; p != NULL; p = p->sibling) { p->parent = res; - pr_debug("PCI: Reparented %s [%llx..%llx] under %s\n", - p->name, - (unsigned long long)p->start, - (unsigned long long)p->end, res->name); + pr_debug("PCI: Reparented %s %pR under %s\n", + p->name, p, res->name); } return 0; } @@ -1198,14 +1184,9 @@ static void pcibios_allocate_bus_resources(struct pci_bus *bus) } } - pr_debug("PCI: %s (bus %d) bridge rsrc %d: %016llx-%016llx " - "[0x%x], parent %p (%s)\n", - bus->self ? pci_name(bus->self) : "PHB", - bus->number, i, - (unsigned long long)res->start, - (unsigned long long)res->end, - (unsigned int)res->flags, - pr, (pr && pr->name) ? pr->name : "nil"); + pr_debug("PCI: %s (bus %d) bridge rsrc %d: %pR, parent %p (%s)\n", + bus->self ? pci_name(bus->self) : "PHB", bus->number, + i, res, pr, (pr && pr->name) ? pr->name : "nil"); if (pr && !(pr->flags & IORESOURCE_UNSET)) { struct pci_dev *dev = bus->self; @@ -1247,11 +1228,8 @@ static inline void alloc_resource(struct pci_dev *dev, int idx) { struct resource *pr, *r = &dev->resource[idx]; - pr_debug("PCI: Allocating %s: Resource %d: %016llx..%016llx [%x]\n", - pci_name(dev), idx, - (unsigned long long)r->start, - (unsigned long long)r->end, - (unsigned int)r->flags); + pr_debug("PCI: Allocating %s: Resource %d: %pR\n", + pci_name(dev), idx, r); pr = pci_find_parent_resource(dev, r); if (!pr || (pr->flags & IORESOURCE_UNSET) || @@ -1259,11 +1237,7 @@ static inline void alloc_resource(struct pci_dev *dev, int idx) printk(KERN_WARNING "PCI: Cannot allocate resource region %d" " of device %s, will remap\n", idx, pci_name(dev)); if (pr) - pr_debug("PCI: parent is %p: %016llx-%016llx [%x]\n", - pr, - (unsigned long long)pr->start, - (unsigned long long)pr->end, - (unsigned int)pr->flags); + pr_debug("PCI: parent is %p: %pR\n", pr, pr); /* We'll assign a new address later */ r->flags |= IORESOURCE_UNSET; r->end -= r->start; @@ -1425,12 +1399,8 @@ void pcibios_claim_one_bus(struct pci_bus *bus) if (r->parent || !r->start || !r->flags) continue; - pr_debug("PCI: Claiming %s: " - "Resource %d: %016llx..%016llx [%x]\n", - pci_name(dev), i, - (unsigned long long)r->start, - (unsigned long long)r->end, - (unsigned int)r->flags); + pr_debug("PCI: Claiming %s: Resource %d: %pR\n", + pci_name(dev), i, r); if (pci_claim_resource(dev, i) == 0) continue; @@ -1514,11 +1484,8 @@ static void pcibios_setup_phb_resources(struct pci_controller *hose, } else { offset = pcibios_io_space_offset(hose); - pr_debug("PCI: PHB IO resource = %08llx-%08llx [%lx] off 0x%08llx\n", - (unsigned long long)res->start, - (unsigned long long)res->end, - (unsigned long)res->flags, - (unsigned long long)offset); + pr_debug("PCI: PHB IO resource = %pR off 0x%08llx\n", + res, (unsigned long long)offset); pci_add_resource_offset(resources, res, offset); } @@ -1535,11 +1502,8 @@ static void pcibios_setup_phb_resources(struct pci_controller *hose, offset = hose->mem_offset[i]; - pr_debug("PCI: PHB MEM resource %d = %08llx-%08llx [%lx] off 0x%08llx\n", i, - (unsigned long long)res->start, - (unsigned long long)res->end, - (unsigned long)res->flags, - (unsigned long long)offset); + pr_debug("PCI: PHB MEM resource %d = %pR off 0x%08llx\n", i, + res, (unsigned long long)offset); pci_add_resource_offset(resources, res, offset); } diff --git a/arch/powerpc/kernel/pci_of_scan.c b/arch/powerpc/kernel/pci_of_scan.c index c8c62c7fc31c..2e710c15893f 100644 --- a/arch/powerpc/kernel/pci_of_scan.c +++ b/arch/powerpc/kernel/pci_of_scan.c @@ -102,7 +102,7 @@ static void of_pci_parse_addrs(struct device_node *node, struct pci_dev *dev) res = &dev->resource[(i - PCI_BASE_ADDRESS_0) >> 2]; } else if (i == dev->rom_base_reg) { res = &dev->resource[PCI_ROM_RESOURCE]; - flags |= IORESOURCE_READONLY | IORESOURCE_CACHEABLE; + flags |= IORESOURCE_READONLY; } else { printk(KERN_ERR "PCI: bad cfg reg num 0x%x\n", i); continue; diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index 64e6e9d9e656..75b6676c1a0b 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -86,7 +86,7 @@ void giveup_fpu_maybe_transactional(struct task_struct *tsk) if (tsk == current && tsk->thread.regs && MSR_TM_ACTIVE(tsk->thread.regs->msr) && !test_thread_flag(TIF_RESTORE_TM)) { - tsk->thread.tm_orig_msr = tsk->thread.regs->msr; + tsk->thread.ckpt_regs.msr = tsk->thread.regs->msr; set_thread_flag(TIF_RESTORE_TM); } @@ -104,7 +104,7 @@ void giveup_altivec_maybe_transactional(struct task_struct *tsk) if (tsk == current && tsk->thread.regs && MSR_TM_ACTIVE(tsk->thread.regs->msr) && !test_thread_flag(TIF_RESTORE_TM)) { - tsk->thread.tm_orig_msr = tsk->thread.regs->msr; + tsk->thread.ckpt_regs.msr = tsk->thread.regs->msr; set_thread_flag(TIF_RESTORE_TM); } @@ -540,7 +540,7 @@ static void tm_reclaim_thread(struct thread_struct *thr, * the thread will no longer be transactional. */ if (test_ti_thread_flag(ti, TIF_RESTORE_TM)) { - msr_diff = thr->tm_orig_msr & ~thr->regs->msr; + msr_diff = thr->ckpt_regs.msr & ~thr->regs->msr; if (msr_diff & MSR_FP) memcpy(&thr->transact_fp, &thr->fp_state, sizeof(struct thread_fp_state)); @@ -591,10 +591,10 @@ static inline void tm_reclaim_task(struct task_struct *tsk) /* Stash the original thread MSR, as giveup_fpu et al will * modify it. We hold onto it to see whether the task used * FP & vector regs. If the TIF_RESTORE_TM flag is set, - * tm_orig_msr is already set. + * ckpt_regs.msr is already set. */ if (!test_ti_thread_flag(task_thread_info(tsk), TIF_RESTORE_TM)) - thr->tm_orig_msr = thr->regs->msr; + thr->ckpt_regs.msr = thr->regs->msr; TM_DEBUG("--- tm_reclaim on pid %d (NIP=%lx, " "ccr=%lx, msr=%lx, trap=%lx)\n", @@ -663,7 +663,7 @@ static inline void tm_recheckpoint_new_task(struct task_struct *new) tm_restore_sprs(&new->thread); return; } - msr = new->thread.tm_orig_msr; + msr = new->thread.ckpt_regs.msr; /* Recheckpoint to restore original checkpointed register state. */ TM_DEBUG("*** tm_recheckpoint of pid %d " "(new->msr 0x%lx, new->origmsr 0x%lx)\n", @@ -723,7 +723,7 @@ void restore_tm_state(struct pt_regs *regs) if (!MSR_TM_ACTIVE(regs->msr)) return; - msr_diff = current->thread.tm_orig_msr & ~regs->msr; + msr_diff = current->thread.ckpt_regs.msr & ~regs->msr; msr_diff &= MSR_FP | MSR_VEC | MSR_VSX; if (msr_diff & MSR_FP) { fp_enable(); diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c index 8b888b12a475..bef76c5033e4 100644 --- a/arch/powerpc/kernel/prom.c +++ b/arch/powerpc/kernel/prom.c @@ -218,22 +218,18 @@ static void __init check_cpu_pa_features(unsigned long node) } #ifdef CONFIG_PPC_STD_MMU_64 -static void __init check_cpu_slb_size(unsigned long node) +static void __init init_mmu_slb_size(unsigned long node) { const __be32 *slb_size_ptr; - slb_size_ptr = of_get_flat_dt_prop(node, "slb-size", NULL); - if (slb_size_ptr != NULL) { + slb_size_ptr = of_get_flat_dt_prop(node, "slb-size", NULL) ? : + of_get_flat_dt_prop(node, "ibm,slb-size", NULL); + + if (slb_size_ptr) mmu_slb_size = be32_to_cpup(slb_size_ptr); - return; - } - slb_size_ptr = of_get_flat_dt_prop(node, "ibm,slb-size", NULL); - if (slb_size_ptr != NULL) { - mmu_slb_size = be32_to_cpup(slb_size_ptr); - } } #else -#define check_cpu_slb_size(node) do { } while(0) +#define init_mmu_slb_size(node) do { } while(0) #endif static struct feature_property { @@ -380,7 +376,7 @@ static int __init early_init_dt_scan_cpus(unsigned long node, check_cpu_feature_properties(node); check_cpu_pa_features(node); - check_cpu_slb_size(node); + init_mmu_slb_size(node); #ifdef CONFIG_PPC64 if (nthreads > 1) @@ -476,9 +472,10 @@ static int __init early_init_dt_scan_drconf_memory(unsigned long node) flags = of_read_number(&dm[3], 1); /* skip DRC index, pad, assoc. list index, flags */ dm += 4; - /* skip this block if the reserved bit is set in flags (0x80) - or if the block is not assigned to this partition (0x8) */ - if ((flags & 0x80) || !(flags & 0x8)) + /* skip this block if the reserved bit is set in flags + or if the block is not assigned to this partition */ + if ((flags & DRCONF_MEM_RESERVED) || + !(flags & DRCONF_MEM_ASSIGNED)) continue; size = memblock_size; rngs = 1; diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c index fcca8077e6a2..15099c41622e 100644 --- a/arch/powerpc/kernel/prom_init.c +++ b/arch/powerpc/kernel/prom_init.c @@ -641,6 +641,15 @@ static void __init early_cmdline_parse(void) #define W(x) ((x) >> 24) & 0xff, ((x) >> 16) & 0xff, \ ((x) >> 8) & 0xff, (x) & 0xff +/* Firmware expects the value to be n - 1, where n is the # of vectors */ +#define NUM_VECTORS(n) ((n) - 1) + +/* + * Firmware expects 1 + n - 2, where n is the length of the option vector in + * bytes. The 1 accounts for the length byte itself, the - 2 .. ? + */ +#define VECTOR_LENGTH(n) (1 + (n) - 2) + unsigned char ibm_architecture_vec[] = { W(0xfffe0000), W(0x003a0000), /* POWER5/POWER5+ */ W(0xffff0000), W(0x003e0000), /* POWER6 */ @@ -651,16 +660,16 @@ unsigned char ibm_architecture_vec[] = { W(0xffffffff), W(0x0f000003), /* all 2.06-compliant */ W(0xffffffff), W(0x0f000002), /* all 2.05-compliant */ W(0xfffffffe), W(0x0f000001), /* all 2.04-compliant and earlier */ - 6 - 1, /* 6 option vectors */ + NUM_VECTORS(6), /* 6 option vectors */ /* option vector 1: processor architectures supported */ - 3 - 2, /* length */ + VECTOR_LENGTH(2), /* length */ 0, /* don't ignore, don't halt */ OV1_PPC_2_00 | OV1_PPC_2_01 | OV1_PPC_2_02 | OV1_PPC_2_03 | OV1_PPC_2_04 | OV1_PPC_2_05 | OV1_PPC_2_06 | OV1_PPC_2_07, /* option vector 2: Open Firmware options supported */ - 34 - 2, /* length */ + VECTOR_LENGTH(33), /* length */ OV2_REAL_MODE, 0, 0, W(0xffffffff), /* real_base */ @@ -674,17 +683,17 @@ unsigned char ibm_architecture_vec[] = { 48, /* max log_2(hash table size) */ /* option vector 3: processor options supported */ - 3 - 2, /* length */ + VECTOR_LENGTH(2), /* length */ 0, /* don't ignore, don't halt */ OV3_FP | OV3_VMX | OV3_DFP, /* option vector 4: IBM PAPR implementation */ - 3 - 2, /* length */ + VECTOR_LENGTH(2), /* length */ 0, /* don't halt */ OV4_MIN_ENT_CAP, /* minimum VP entitled capacity */ /* option vector 5: PAPR/OF options */ - 19 - 2, /* length */ + VECTOR_LENGTH(18), /* length */ 0, /* don't ignore, don't halt */ OV5_FEAT(OV5_LPAR) | OV5_FEAT(OV5_SPLPAR) | OV5_FEAT(OV5_LARGE_PAGES) | OV5_FEAT(OV5_DRCONF_MEMORY) | OV5_FEAT(OV5_DONATE_DEDICATE_CPU) | @@ -717,12 +726,12 @@ unsigned char ibm_architecture_vec[] = { OV5_FEAT(OV5_PFO_HW_RNG) | OV5_FEAT(OV5_PFO_HW_ENCR) | OV5_FEAT(OV5_PFO_HW_842), OV5_FEAT(OV5_SUB_PROCESSORS), + /* option vector 6: IBM PAPR hints */ - 4 - 2, /* length */ + VECTOR_LENGTH(3), /* length */ 0, 0, OV6_LINUX, - }; /* Old method - ELF header with PT_NOTE sections only works on BE */ diff --git a/arch/powerpc/kernel/ptrace.c b/arch/powerpc/kernel/ptrace.c index f21897b42057..737c0d0b53ac 100644 --- a/arch/powerpc/kernel/ptrace.c +++ b/arch/powerpc/kernel/ptrace.c @@ -1762,26 +1762,81 @@ long arch_ptrace(struct task_struct *child, long request, return ret; } -/* - * We must return the syscall number to actually look up in the table. - * This can be -1L to skip running any syscall at all. +#ifdef CONFIG_SECCOMP +static int do_seccomp(struct pt_regs *regs) +{ + if (!test_thread_flag(TIF_SECCOMP)) + return 0; + + /* + * The ABI we present to seccomp tracers is that r3 contains + * the syscall return value and orig_gpr3 contains the first + * syscall parameter. This is different to the ptrace ABI where + * both r3 and orig_gpr3 contain the first syscall parameter. + */ + regs->gpr[3] = -ENOSYS; + + /* + * We use the __ version here because we have already checked + * TIF_SECCOMP. If this fails, there is nothing left to do, we + * have already loaded -ENOSYS into r3, or seccomp has put + * something else in r3 (via SECCOMP_RET_ERRNO/TRACE). + */ + if (__secure_computing()) + return -1; + + /* + * The syscall was allowed by seccomp, restore the register + * state to what ptrace and audit expect. + * Note that we use orig_gpr3, which means a seccomp tracer can + * modify the first syscall parameter (in orig_gpr3) and also + * allow the syscall to proceed. + */ + regs->gpr[3] = regs->orig_gpr3; + + return 0; +} +#else +static inline int do_seccomp(struct pt_regs *regs) { return 0; } +#endif /* CONFIG_SECCOMP */ + +/** + * do_syscall_trace_enter() - Do syscall tracing on kernel entry. + * @regs: the pt_regs of the task to trace (current) + * + * Performs various types of tracing on syscall entry. This includes seccomp, + * ptrace, syscall tracepoints and audit. + * + * The pt_regs are potentially visible to userspace via ptrace, so their + * contents is ABI. + * + * One or more of the tracers may modify the contents of pt_regs, in particular + * to modify arguments or even the syscall number itself. + * + * It's also possible that a tracer can choose to reject the system call. In + * that case this function will return an illegal syscall number, and will put + * an appropriate return value in regs->r3. + * + * Return: the (possibly changed) syscall number. */ long do_syscall_trace_enter(struct pt_regs *regs) { - long ret = 0; + bool abort = false; user_exit(); - secure_computing_strict(regs->gpr[0]); + if (do_seccomp(regs)) + return -1; - if (test_thread_flag(TIF_SYSCALL_TRACE) && - tracehook_report_syscall_entry(regs)) + if (test_thread_flag(TIF_SYSCALL_TRACE)) { /* - * Tracing decided this syscall should not happen. - * We'll return a bogus call number to get an ENOSYS - * error, but leave the original number in regs->gpr[0]. + * The tracer may decide to abort the syscall, if so tracehook + * will return !0. Note that the tracer may also just change + * regs->gpr[0] to an invalid syscall number, that is handled + * below on the exit path. */ - ret = -1L; + abort = tracehook_report_syscall_entry(regs) != 0; + } if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) trace_sys_enter(regs, regs->gpr[0]); @@ -1798,7 +1853,17 @@ long do_syscall_trace_enter(struct pt_regs *regs) regs->gpr[5] & 0xffffffff, regs->gpr[6] & 0xffffffff); - return ret ?: regs->gpr[0]; + if (abort || regs->gpr[0] >= NR_syscalls) { + /* + * If we are aborting explicitly, or if the syscall number is + * now invalid, set the return value to -ENOSYS. + */ + regs->gpr[3] = -ENOSYS; + return -1; + } + + /* Return the possibly modified but valid syscall number */ + return regs->gpr[0]; } void do_syscall_trace_leave(struct pt_regs *regs) diff --git a/arch/powerpc/kernel/rtas.c b/arch/powerpc/kernel/rtas.c index 7a488c108410..84bf934cf748 100644 --- a/arch/powerpc/kernel/rtas.c +++ b/arch/powerpc/kernel/rtas.c @@ -478,8 +478,9 @@ unsigned int rtas_busy_delay_time(int status) if (status == RTAS_BUSY) { ms = 1; - } else if (status >= 9900 && status <= 9905) { - order = status - 9900; + } else if (status >= RTAS_EXTENDED_DELAY_MIN && + status <= RTAS_EXTENDED_DELAY_MAX) { + order = status - RTAS_EXTENDED_DELAY_MIN; for (ms = 1; order > 0; order--) ms *= 10; } @@ -584,6 +585,23 @@ int rtas_get_sensor(int sensor, int index, int *state) } EXPORT_SYMBOL(rtas_get_sensor); +int rtas_get_sensor_fast(int sensor, int index, int *state) +{ + int token = rtas_token("get-sensor-state"); + int rc; + + if (token == RTAS_UNKNOWN_SERVICE) + return -ENOENT; + + rc = rtas_call(token, 2, 2, state, sensor, index); + WARN_ON(rc == RTAS_BUSY || (rc >= RTAS_EXTENDED_DELAY_MIN && + rc <= RTAS_EXTENDED_DELAY_MAX)); + + if (rc < 0) + return rtas_error_rc(rc); + return rc; +} + bool rtas_indicator_present(int token, int *maxindex) { int proplen, count, i; @@ -641,7 +659,8 @@ int rtas_set_indicator_fast(int indicator, int index, int new_value) rc = rtas_call(token, 3, 1, NULL, indicator, index, new_value); - WARN_ON(rc == -2 || (rc >= 9900 && rc <= 9905)); + WARN_ON(rc == RTAS_BUSY || (rc >= RTAS_EXTENDED_DELAY_MIN && + rc <= RTAS_EXTENDED_DELAY_MAX)); if (rc < 0) return rtas_error_rc(rc); diff --git a/arch/powerpc/kernel/signal_32.c b/arch/powerpc/kernel/signal_32.c index da50e0c9c57e..0dbee465af7a 100644 --- a/arch/powerpc/kernel/signal_32.c +++ b/arch/powerpc/kernel/signal_32.c @@ -949,6 +949,11 @@ int copy_siginfo_to_user32(struct compat_siginfo __user *d, const siginfo_t *s) err |= __put_user(s->si_overrun, &d->si_overrun); err |= __put_user(s->si_int, &d->si_int); break; + case __SI_SYS >> 16: + err |= __put_user(ptr_to_compat(s->si_call_addr), &d->si_call_addr); + err |= __put_user(s->si_syscall, &d->si_syscall); + err |= __put_user(s->si_arch, &d->si_arch); + break; case __SI_RT >> 16: /* This is not generated by the kernel as of now. */ case __SI_MESGQ >> 16: err |= __put_user(s->si_int, &d->si_int); diff --git a/arch/powerpc/kernel/signal_64.c b/arch/powerpc/kernel/signal_64.c index c7c24d2e2bdb..20756dfb9f34 100644 --- a/arch/powerpc/kernel/signal_64.c +++ b/arch/powerpc/kernel/signal_64.c @@ -73,6 +73,19 @@ static const char fmt32[] = KERN_INFO \ static const char fmt64[] = KERN_INFO \ "%s[%d]: bad frame in %s: %016lx nip %016lx lr %016lx\n"; +/* + * This computes a quad word aligned pointer inside the vmx_reserve array + * element. For historical reasons sigcontext might not be quad word aligned, + * but the location we write the VMX regs to must be. See the comment in + * sigcontext for more detail. + */ +#ifdef CONFIG_ALTIVEC +static elf_vrreg_t __user *sigcontext_vmx_regs(struct sigcontext __user *sc) +{ + return (elf_vrreg_t __user *) (((unsigned long)sc->vmx_reserve + 15) & ~0xful); +} +#endif + /* * Set up the sigcontext for the signal frame. */ @@ -90,7 +103,7 @@ static long setup_sigcontext(struct sigcontext __user *sc, struct pt_regs *regs, * v_regs pointer or not */ #ifdef CONFIG_ALTIVEC - elf_vrreg_t __user *v_regs = (elf_vrreg_t __user *)(((unsigned long)sc->vmx_reserve + 15) & ~0xful); + elf_vrreg_t __user *v_regs = sigcontext_vmx_regs(sc); #endif unsigned long msr = regs->msr; long err = 0; @@ -181,10 +194,8 @@ static long setup_tm_sigcontexts(struct sigcontext __user *sc, * v_regs pointer or not. */ #ifdef CONFIG_ALTIVEC - elf_vrreg_t __user *v_regs = (elf_vrreg_t __user *) - (((unsigned long)sc->vmx_reserve + 15) & ~0xful); - elf_vrreg_t __user *tm_v_regs = (elf_vrreg_t __user *) - (((unsigned long)tm_sc->vmx_reserve + 15) & ~0xful); + elf_vrreg_t __user *v_regs = sigcontext_vmx_regs(sc); + elf_vrreg_t __user *tm_v_regs = sigcontext_vmx_regs(tm_sc); #endif unsigned long msr = regs->msr; long err = 0; diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index 43922509a483..1be1092c7204 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -99,16 +99,17 @@ static struct clocksource clocksource_timebase = { static int decrementer_set_next_event(unsigned long evt, struct clock_event_device *dev); -static void decrementer_set_mode(enum clock_event_mode mode, - struct clock_event_device *dev); +static int decrementer_shutdown(struct clock_event_device *evt); struct clock_event_device decrementer_clockevent = { - .name = "decrementer", - .rating = 200, - .irq = 0, - .set_next_event = decrementer_set_next_event, - .set_mode = decrementer_set_mode, - .features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP, + .name = "decrementer", + .rating = 200, + .irq = 0, + .set_next_event = decrementer_set_next_event, + .set_state_shutdown = decrementer_shutdown, + .tick_resume = decrementer_shutdown, + .features = CLOCK_EVT_FEAT_ONESHOT | + CLOCK_EVT_FEAT_C3STOP, }; EXPORT_SYMBOL(decrementer_clockevent); @@ -862,11 +863,10 @@ static int decrementer_set_next_event(unsigned long evt, return 0; } -static void decrementer_set_mode(enum clock_event_mode mode, - struct clock_event_device *dev) +static int decrementer_shutdown(struct clock_event_device *dev) { - if (mode != CLOCK_EVT_MODE_ONESHOT) - decrementer_set_next_event(DECREMENTER_MAX, dev); + decrementer_set_next_event(DECREMENTER_MAX, dev); + return 0; } /* Interrupt handler for the timer broadcast IPI */ diff --git a/arch/powerpc/kernel/trace_clock.c b/arch/powerpc/kernel/trace_clock.c new file mode 100644 index 000000000000..49170690946d --- /dev/null +++ b/arch/powerpc/kernel/trace_clock.c @@ -0,0 +1,15 @@ +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2, as + * published by the Free Software Foundation. + * + * Copyright (C) 2015 Naveen N. Rao, IBM Corporation + */ + +#include +#include + +u64 notrace trace_clock_ppc_tb(void) +{ + return get_tb(); +} diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig index 3caec2c42105..c2024ac9d4e8 100644 --- a/arch/powerpc/kvm/Kconfig +++ b/arch/powerpc/kvm/Kconfig @@ -74,14 +74,14 @@ config KVM_BOOK3S_64 If unsure, say N. config KVM_BOOK3S_64_HV - tristate "KVM support for POWER7 and PPC970 using hypervisor mode in host" + tristate "KVM for POWER7 and later using hypervisor mode in host" depends on KVM_BOOK3S_64 && PPC_POWERNV select KVM_BOOK3S_HV_POSSIBLE select MMU_NOTIFIER select CMA ---help--- Support running unmodified book3s_64 guest kernels in - virtual machines on POWER7 and PPC970 processors that have + virtual machines on POWER7 and newer processors that have hypervisor mode available to the host. If you say Y here, KVM will use the hardware virtualization @@ -89,8 +89,8 @@ config KVM_BOOK3S_64_HV guest operating systems will run at full hardware speed using supervisor and user modes. However, this also means that KVM is not usable under PowerVM (pHyp), is only usable - on POWER7 (or later) processors and PPC970-family processors, - and cannot emulate a different processor from the host processor. + on POWER7 or later processors, and cannot emulate a + different processor from the host processor. If unsure, say N. diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c index 05ea8fc7f829..d75bf325f54a 100644 --- a/arch/powerpc/kvm/book3s.c +++ b/arch/powerpc/kvm/book3s.c @@ -240,7 +240,8 @@ void kvmppc_core_queue_inst_storage(struct kvm_vcpu *vcpu, ulong flags) kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_INST_STORAGE); } -int kvmppc_book3s_irqprio_deliver(struct kvm_vcpu *vcpu, unsigned int priority) +static int kvmppc_book3s_irqprio_deliver(struct kvm_vcpu *vcpu, + unsigned int priority) { int deliver = 1; int vec = 0; @@ -902,7 +903,7 @@ int kvmppc_core_check_processor_compat(void) { /* * We always return 0 for book3s. We check - * for compatability while loading the HV + * for compatibility while loading the HV * or PR module */ return 0; diff --git a/arch/powerpc/kvm/book3s_32_mmu_host.c b/arch/powerpc/kvm/book3s_32_mmu_host.c index 2035d16a9262..d5c9bfeb0c9c 100644 --- a/arch/powerpc/kvm/book3s_32_mmu_host.c +++ b/arch/powerpc/kvm/book3s_32_mmu_host.c @@ -26,6 +26,7 @@ #include #include #include +#include "book3s.h" /* #define DEBUG_MMU */ /* #define DEBUG_SR */ diff --git a/arch/powerpc/kvm/book3s_64_mmu_host.c b/arch/powerpc/kvm/book3s_64_mmu_host.c index b982d925c710..79ad35abd196 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_host.c +++ b/arch/powerpc/kvm/book3s_64_mmu_host.c @@ -28,6 +28,7 @@ #include #include #include "trace_pr.h" +#include "book3s.h" #define PTE_SIZE 12 diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c index dab68b7af3f2..1f9c0a17f445 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_hv.c +++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c @@ -761,6 +761,8 @@ static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp, /* Harvest R and C */ rcbits = be64_to_cpu(hptep[1]) & (HPTE_R_R | HPTE_R_C); *rmapp |= rcbits << KVMPPC_RMAP_RC_SHIFT; + if (rcbits & HPTE_R_C) + kvmppc_update_rmap_change(rmapp, psize); if (rcbits & ~rev[i].guest_rpte) { rev[i].guest_rpte = ptel | rcbits; note_hpte_modification(kvm, &rev[i]); @@ -927,8 +929,12 @@ static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp) retry: lock_rmap(rmapp); if (*rmapp & KVMPPC_RMAP_CHANGED) { - *rmapp &= ~KVMPPC_RMAP_CHANGED; + long change_order = (*rmapp & KVMPPC_RMAP_CHG_ORDER) + >> KVMPPC_RMAP_CHG_SHIFT; + *rmapp &= ~(KVMPPC_RMAP_CHANGED | KVMPPC_RMAP_CHG_ORDER); npages_dirty = 1; + if (change_order > PAGE_SHIFT) + npages_dirty = 1ul << (change_order - PAGE_SHIFT); } if (!(*rmapp & KVMPPC_RMAP_PRESENT)) { unlock_rmap(rmapp); diff --git a/arch/powerpc/kvm/book3s_emulate.c b/arch/powerpc/kvm/book3s_emulate.c index 5a2bc4b0dfe5..2afdb9c0937d 100644 --- a/arch/powerpc/kvm/book3s_emulate.c +++ b/arch/powerpc/kvm/book3s_emulate.c @@ -23,6 +23,7 @@ #include #include #include +#include "book3s.h" #define OP_19_XOP_RFID 18 #define OP_19_XOP_RFI 50 diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index a9f753fb73a8..9754e6815e52 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -81,6 +81,12 @@ static DECLARE_BITMAP(default_enabled_hcalls, MAX_HCALL_OPCODE/4 + 1); #define MPP_BUFFER_ORDER 3 #endif +static int dynamic_mt_modes = 6; +module_param(dynamic_mt_modes, int, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC(dynamic_mt_modes, "Set of allowed dynamic micro-threading modes: 0 (= none), 2, 4, or 6 (= 2 or 4)"); +static int target_smt_mode; +module_param(target_smt_mode, int, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC(target_smt_mode, "Target threads per core (0 = max)"); static void kvmppc_end_cede(struct kvm_vcpu *vcpu); static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu); @@ -114,7 +120,7 @@ static bool kvmppc_ipi_thread(int cpu) static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) { - int cpu = vcpu->cpu; + int cpu; wait_queue_head_t *wqp; wqp = kvm_arch_vcpu_wq(vcpu); @@ -123,10 +129,11 @@ static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) ++vcpu->stat.halt_wakeup; } - if (kvmppc_ipi_thread(cpu + vcpu->arch.ptid)) + if (kvmppc_ipi_thread(vcpu->arch.thread_cpu)) return; /* CPU points to the first thread of the core */ + cpu = vcpu->cpu; if (cpu >= 0 && cpu < nr_cpu_ids && cpu_online(cpu)) smp_send_reschedule(cpu); } @@ -164,6 +171,27 @@ static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) * they should never fail.) */ +static void kvmppc_core_start_stolen(struct kvmppc_vcore *vc) +{ + unsigned long flags; + + spin_lock_irqsave(&vc->stoltb_lock, flags); + vc->preempt_tb = mftb(); + spin_unlock_irqrestore(&vc->stoltb_lock, flags); +} + +static void kvmppc_core_end_stolen(struct kvmppc_vcore *vc) +{ + unsigned long flags; + + spin_lock_irqsave(&vc->stoltb_lock, flags); + if (vc->preempt_tb != TB_NIL) { + vc->stolen_tb += mftb() - vc->preempt_tb; + vc->preempt_tb = TB_NIL; + } + spin_unlock_irqrestore(&vc->stoltb_lock, flags); +} + static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu) { struct kvmppc_vcore *vc = vcpu->arch.vcore; @@ -175,14 +203,9 @@ static void kvmppc_core_vcpu_load_hv(struct kvm_vcpu *vcpu, int cpu) * vcpu, and once it is set to this vcpu, only this task * ever sets it to NULL. */ - if (vc->runner == vcpu && vc->vcore_state != VCORE_INACTIVE) { - spin_lock_irqsave(&vc->stoltb_lock, flags); - if (vc->preempt_tb != TB_NIL) { - vc->stolen_tb += mftb() - vc->preempt_tb; - vc->preempt_tb = TB_NIL; - } - spin_unlock_irqrestore(&vc->stoltb_lock, flags); - } + if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING) + kvmppc_core_end_stolen(vc); + spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags); if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST && vcpu->arch.busy_preempt != TB_NIL) { @@ -197,11 +220,9 @@ static void kvmppc_core_vcpu_put_hv(struct kvm_vcpu *vcpu) struct kvmppc_vcore *vc = vcpu->arch.vcore; unsigned long flags; - if (vc->runner == vcpu && vc->vcore_state != VCORE_INACTIVE) { - spin_lock_irqsave(&vc->stoltb_lock, flags); - vc->preempt_tb = mftb(); - spin_unlock_irqrestore(&vc->stoltb_lock, flags); - } + if (vc->runner == vcpu && vc->vcore_state >= VCORE_SLEEPING) + kvmppc_core_start_stolen(vc); + spin_lock_irqsave(&vcpu->arch.tbacct_lock, flags); if (vcpu->arch.state == KVMPPC_VCPU_BUSY_IN_HOST) vcpu->arch.busy_preempt = mftb(); @@ -214,12 +235,12 @@ static void kvmppc_set_msr_hv(struct kvm_vcpu *vcpu, u64 msr) kvmppc_end_cede(vcpu); } -void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr) +static void kvmppc_set_pvr_hv(struct kvm_vcpu *vcpu, u32 pvr) { vcpu->arch.pvr = pvr; } -int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat) +static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat) { unsigned long pcr = 0; struct kvmppc_vcore *vc = vcpu->arch.vcore; @@ -259,7 +280,7 @@ int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat) return 0; } -void kvmppc_dump_regs(struct kvm_vcpu *vcpu) +static void kvmppc_dump_regs(struct kvm_vcpu *vcpu) { int r; @@ -292,7 +313,7 @@ void kvmppc_dump_regs(struct kvm_vcpu *vcpu) vcpu->arch.last_inst); } -struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id) +static struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id) { int r; struct kvm_vcpu *v, *ret = NULL; @@ -641,7 +662,8 @@ static int kvm_arch_vcpu_yield_to(struct kvm_vcpu *target) spin_lock(&vcore->lock); if (target->arch.state == KVMPPC_VCPU_RUNNABLE && - vcore->vcore_state != VCORE_INACTIVE) + vcore->vcore_state != VCORE_INACTIVE && + vcore->runner) target = vcore->runner; spin_unlock(&vcore->lock); @@ -1431,6 +1453,7 @@ static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int core) vcore->lpcr = kvm->arch.lpcr; vcore->first_vcpuid = core * threads_per_subcore; vcore->kvm = kvm; + INIT_LIST_HEAD(&vcore->preempt_list); vcore->mpp_buffer_is_valid = false; @@ -1655,6 +1678,7 @@ static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm, spin_unlock(&vcore->lock); vcpu->arch.vcore = vcore; vcpu->arch.ptid = vcpu->vcpu_id - vcore->first_vcpuid; + vcpu->arch.thread_cpu = -1; vcpu->arch.cpu_type = KVM_CPU_3S_64; kvmppc_sanity_check(vcpu); @@ -1749,6 +1773,7 @@ static int kvmppc_grab_hwthread(int cpu) /* Ensure the thread won't go into the kernel if it wakes */ tpaca->kvm_hstate.kvm_vcpu = NULL; + tpaca->kvm_hstate.kvm_vcore = NULL; tpaca->kvm_hstate.napping = 0; smp_wmb(); tpaca->kvm_hstate.hwthread_req = 1; @@ -1780,26 +1805,32 @@ static void kvmppc_release_hwthread(int cpu) tpaca = &paca[cpu]; tpaca->kvm_hstate.hwthread_req = 0; tpaca->kvm_hstate.kvm_vcpu = NULL; + tpaca->kvm_hstate.kvm_vcore = NULL; + tpaca->kvm_hstate.kvm_split_mode = NULL; } -static void kvmppc_start_thread(struct kvm_vcpu *vcpu) +static void kvmppc_start_thread(struct kvm_vcpu *vcpu, struct kvmppc_vcore *vc) { int cpu; struct paca_struct *tpaca; - struct kvmppc_vcore *vc = vcpu->arch.vcore; + struct kvmppc_vcore *mvc = vc->master_vcore; - if (vcpu->arch.timer_running) { - hrtimer_try_to_cancel(&vcpu->arch.dec_timer); - vcpu->arch.timer_running = 0; + cpu = vc->pcpu; + if (vcpu) { + if (vcpu->arch.timer_running) { + hrtimer_try_to_cancel(&vcpu->arch.dec_timer); + vcpu->arch.timer_running = 0; + } + cpu += vcpu->arch.ptid; + vcpu->cpu = mvc->pcpu; + vcpu->arch.thread_cpu = cpu; } - cpu = vc->pcpu + vcpu->arch.ptid; tpaca = &paca[cpu]; - tpaca->kvm_hstate.kvm_vcore = vc; - tpaca->kvm_hstate.ptid = vcpu->arch.ptid; - vcpu->cpu = vc->pcpu; - /* Order stores to hstate.kvm_vcore etc. before store to kvm_vcpu */ - smp_wmb(); tpaca->kvm_hstate.kvm_vcpu = vcpu; + tpaca->kvm_hstate.ptid = cpu - mvc->pcpu; + /* Order stores to hstate.kvm_vcpu etc. before store to kvm_vcore */ + smp_wmb(); + tpaca->kvm_hstate.kvm_vcore = mvc; if (cpu != smp_processor_id()) kvmppc_ipi_thread(cpu); } @@ -1812,12 +1843,12 @@ static void kvmppc_wait_for_nap(void) for (loops = 0; loops < 1000000; ++loops) { /* * Check if all threads are finished. - * We set the vcpu pointer when starting a thread + * We set the vcore pointer when starting a thread * and the thread clears it when finished, so we look - * for any threads that still have a non-NULL vcpu ptr. + * for any threads that still have a non-NULL vcore ptr. */ for (i = 1; i < threads_per_subcore; ++i) - if (paca[cpu + i].kvm_hstate.kvm_vcpu) + if (paca[cpu + i].kvm_hstate.kvm_vcore) break; if (i == threads_per_subcore) { HMT_medium(); @@ -1827,7 +1858,7 @@ static void kvmppc_wait_for_nap(void) } HMT_medium(); for (i = 1; i < threads_per_subcore; ++i) - if (paca[cpu + i].kvm_hstate.kvm_vcpu) + if (paca[cpu + i].kvm_hstate.kvm_vcore) pr_err("KVM: CPU %d seems to be stuck\n", cpu + i); } @@ -1890,6 +1921,278 @@ static void kvmppc_start_restoring_l2_cache(const struct kvmppc_vcore *vc) mtspr(SPRN_MPPR, mpp_addr | PPC_MPPR_FETCH_WHOLE_TABLE); } +/* + * A list of virtual cores for each physical CPU. + * These are vcores that could run but their runner VCPU tasks are + * (or may be) preempted. + */ +struct preempted_vcore_list { + struct list_head list; + spinlock_t lock; +}; + +static DEFINE_PER_CPU(struct preempted_vcore_list, preempted_vcores); + +static void init_vcore_lists(void) +{ + int cpu; + + for_each_possible_cpu(cpu) { + struct preempted_vcore_list *lp = &per_cpu(preempted_vcores, cpu); + spin_lock_init(&lp->lock); + INIT_LIST_HEAD(&lp->list); + } +} + +static void kvmppc_vcore_preempt(struct kvmppc_vcore *vc) +{ + struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores); + + vc->vcore_state = VCORE_PREEMPT; + vc->pcpu = smp_processor_id(); + if (vc->num_threads < threads_per_subcore) { + spin_lock(&lp->lock); + list_add_tail(&vc->preempt_list, &lp->list); + spin_unlock(&lp->lock); + } + + /* Start accumulating stolen time */ + kvmppc_core_start_stolen(vc); +} + +static void kvmppc_vcore_end_preempt(struct kvmppc_vcore *vc) +{ + struct preempted_vcore_list *lp; + + kvmppc_core_end_stolen(vc); + if (!list_empty(&vc->preempt_list)) { + lp = &per_cpu(preempted_vcores, vc->pcpu); + spin_lock(&lp->lock); + list_del_init(&vc->preempt_list); + spin_unlock(&lp->lock); + } + vc->vcore_state = VCORE_INACTIVE; +} + +/* + * This stores information about the virtual cores currently + * assigned to a physical core. + */ +struct core_info { + int n_subcores; + int max_subcore_threads; + int total_threads; + int subcore_threads[MAX_SUBCORES]; + struct kvm *subcore_vm[MAX_SUBCORES]; + struct list_head vcs[MAX_SUBCORES]; +}; + +/* + * This mapping means subcores 0 and 1 can use threads 0-3 and 4-7 + * respectively in 2-way micro-threading (split-core) mode. + */ +static int subcore_thread_map[MAX_SUBCORES] = { 0, 4, 2, 6 }; + +static void init_core_info(struct core_info *cip, struct kvmppc_vcore *vc) +{ + int sub; + + memset(cip, 0, sizeof(*cip)); + cip->n_subcores = 1; + cip->max_subcore_threads = vc->num_threads; + cip->total_threads = vc->num_threads; + cip->subcore_threads[0] = vc->num_threads; + cip->subcore_vm[0] = vc->kvm; + for (sub = 0; sub < MAX_SUBCORES; ++sub) + INIT_LIST_HEAD(&cip->vcs[sub]); + list_add_tail(&vc->preempt_list, &cip->vcs[0]); +} + +static bool subcore_config_ok(int n_subcores, int n_threads) +{ + /* Can only dynamically split if unsplit to begin with */ + if (n_subcores > 1 && threads_per_subcore < MAX_SMT_THREADS) + return false; + if (n_subcores > MAX_SUBCORES) + return false; + if (n_subcores > 1) { + if (!(dynamic_mt_modes & 2)) + n_subcores = 4; + if (n_subcores > 2 && !(dynamic_mt_modes & 4)) + return false; + } + + return n_subcores * roundup_pow_of_two(n_threads) <= MAX_SMT_THREADS; +} + +static void init_master_vcore(struct kvmppc_vcore *vc) +{ + vc->master_vcore = vc; + vc->entry_exit_map = 0; + vc->in_guest = 0; + vc->napping_threads = 0; + vc->conferring_threads = 0; +} + +/* + * See if the existing subcores can be split into 3 (or fewer) subcores + * of at most two threads each, so we can fit in another vcore. This + * assumes there are at most two subcores and at most 6 threads in total. + */ +static bool can_split_piggybacked_subcores(struct core_info *cip) +{ + int sub, new_sub; + int large_sub = -1; + int thr; + int n_subcores = cip->n_subcores; + struct kvmppc_vcore *vc, *vcnext; + struct kvmppc_vcore *master_vc = NULL; + + for (sub = 0; sub < cip->n_subcores; ++sub) { + if (cip->subcore_threads[sub] <= 2) + continue; + if (large_sub >= 0) + return false; + large_sub = sub; + vc = list_first_entry(&cip->vcs[sub], struct kvmppc_vcore, + preempt_list); + if (vc->num_threads > 2) + return false; + n_subcores += (cip->subcore_threads[sub] - 1) >> 1; + } + if (n_subcores > 3 || large_sub < 0) + return false; + + /* + * Seems feasible, so go through and move vcores to new subcores. + * Note that when we have two or more vcores in one subcore, + * all those vcores must have only one thread each. + */ + new_sub = cip->n_subcores; + thr = 0; + sub = large_sub; + list_for_each_entry_safe(vc, vcnext, &cip->vcs[sub], preempt_list) { + if (thr >= 2) { + list_del(&vc->preempt_list); + list_add_tail(&vc->preempt_list, &cip->vcs[new_sub]); + /* vc->num_threads must be 1 */ + if (++cip->subcore_threads[new_sub] == 1) { + cip->subcore_vm[new_sub] = vc->kvm; + init_master_vcore(vc); + master_vc = vc; + ++cip->n_subcores; + } else { + vc->master_vcore = master_vc; + ++new_sub; + } + } + thr += vc->num_threads; + } + cip->subcore_threads[large_sub] = 2; + cip->max_subcore_threads = 2; + + return true; +} + +static bool can_dynamic_split(struct kvmppc_vcore *vc, struct core_info *cip) +{ + int n_threads = vc->num_threads; + int sub; + + if (!cpu_has_feature(CPU_FTR_ARCH_207S)) + return false; + + if (n_threads < cip->max_subcore_threads) + n_threads = cip->max_subcore_threads; + if (subcore_config_ok(cip->n_subcores + 1, n_threads)) { + cip->max_subcore_threads = n_threads; + } else if (cip->n_subcores <= 2 && cip->total_threads <= 6 && + vc->num_threads <= 2) { + /* + * We may be able to fit another subcore in by + * splitting an existing subcore with 3 or 4 + * threads into two 2-thread subcores, or one + * with 5 or 6 threads into three subcores. + * We can only do this if those subcores have + * piggybacked virtual cores. + */ + if (!can_split_piggybacked_subcores(cip)) + return false; + } else { + return false; + } + + sub = cip->n_subcores; + ++cip->n_subcores; + cip->total_threads += vc->num_threads; + cip->subcore_threads[sub] = vc->num_threads; + cip->subcore_vm[sub] = vc->kvm; + init_master_vcore(vc); + list_del(&vc->preempt_list); + list_add_tail(&vc->preempt_list, &cip->vcs[sub]); + + return true; +} + +static bool can_piggyback_subcore(struct kvmppc_vcore *pvc, + struct core_info *cip, int sub) +{ + struct kvmppc_vcore *vc; + int n_thr; + + vc = list_first_entry(&cip->vcs[sub], struct kvmppc_vcore, + preempt_list); + + /* require same VM and same per-core reg values */ + if (pvc->kvm != vc->kvm || + pvc->tb_offset != vc->tb_offset || + pvc->pcr != vc->pcr || + pvc->lpcr != vc->lpcr) + return false; + + /* P8 guest with > 1 thread per core would see wrong TIR value */ + if (cpu_has_feature(CPU_FTR_ARCH_207S) && + (vc->num_threads > 1 || pvc->num_threads > 1)) + return false; + + n_thr = cip->subcore_threads[sub] + pvc->num_threads; + if (n_thr > cip->max_subcore_threads) { + if (!subcore_config_ok(cip->n_subcores, n_thr)) + return false; + cip->max_subcore_threads = n_thr; + } + + cip->total_threads += pvc->num_threads; + cip->subcore_threads[sub] = n_thr; + pvc->master_vcore = vc; + list_del(&pvc->preempt_list); + list_add_tail(&pvc->preempt_list, &cip->vcs[sub]); + + return true; +} + +/* + * Work out whether it is possible to piggyback the execution of + * vcore *pvc onto the execution of the other vcores described in *cip. + */ +static bool can_piggyback(struct kvmppc_vcore *pvc, struct core_info *cip, + int target_threads) +{ + int sub; + + if (cip->total_threads + pvc->num_threads > target_threads) + return false; + for (sub = 0; sub < cip->n_subcores; ++sub) + if (cip->subcore_threads[sub] && + can_piggyback_subcore(pvc, cip, sub)) + return true; + + if (can_dynamic_split(pvc, cip)) + return true; + + return false; +} + static void prepare_threads(struct kvmppc_vcore *vc) { struct kvm_vcpu *vcpu, *vnext; @@ -1909,12 +2212,45 @@ static void prepare_threads(struct kvmppc_vcore *vc) } } -static void post_guest_process(struct kvmppc_vcore *vc) +static void collect_piggybacks(struct core_info *cip, int target_threads) { + struct preempted_vcore_list *lp = this_cpu_ptr(&preempted_vcores); + struct kvmppc_vcore *pvc, *vcnext; + + spin_lock(&lp->lock); + list_for_each_entry_safe(pvc, vcnext, &lp->list, preempt_list) { + if (!spin_trylock(&pvc->lock)) + continue; + prepare_threads(pvc); + if (!pvc->n_runnable) { + list_del_init(&pvc->preempt_list); + if (pvc->runner == NULL) { + pvc->vcore_state = VCORE_INACTIVE; + kvmppc_core_end_stolen(pvc); + } + spin_unlock(&pvc->lock); + continue; + } + if (!can_piggyback(pvc, cip, target_threads)) { + spin_unlock(&pvc->lock); + continue; + } + kvmppc_core_end_stolen(pvc); + pvc->vcore_state = VCORE_PIGGYBACK; + if (cip->total_threads >= target_threads) + break; + } + spin_unlock(&lp->lock); +} + +static void post_guest_process(struct kvmppc_vcore *vc, bool is_master) +{ + int still_running = 0; u64 now; long ret; struct kvm_vcpu *vcpu, *vnext; + spin_lock(&vc->lock); now = get_tb(); list_for_each_entry_safe(vcpu, vnext, &vc->runnable_threads, arch.run_list) { @@ -1933,17 +2269,36 @@ static void post_guest_process(struct kvmppc_vcore *vc) vcpu->arch.ret = ret; vcpu->arch.trap = 0; - if (vcpu->arch.ceded) { - if (!is_kvmppc_resume_guest(ret)) - kvmppc_end_cede(vcpu); - else + if (is_kvmppc_resume_guest(vcpu->arch.ret)) { + if (vcpu->arch.pending_exceptions) + kvmppc_core_prepare_to_enter(vcpu); + if (vcpu->arch.ceded) kvmppc_set_timer(vcpu); - } - if (!is_kvmppc_resume_guest(vcpu->arch.ret)) { + else + ++still_running; + } else { kvmppc_remove_runnable(vc, vcpu); wake_up(&vcpu->arch.cpu_run); } } + list_del_init(&vc->preempt_list); + if (!is_master) { + if (still_running > 0) { + kvmppc_vcore_preempt(vc); + } else if (vc->runner) { + vc->vcore_state = VCORE_PREEMPT; + kvmppc_core_start_stolen(vc); + } else { + vc->vcore_state = VCORE_INACTIVE; + } + if (vc->n_runnable > 0 && vc->runner == NULL) { + /* make sure there's a candidate runner awake */ + vcpu = list_first_entry(&vc->runnable_threads, + struct kvm_vcpu, arch.run_list); + wake_up(&vcpu->arch.cpu_run); + } + } + spin_unlock(&vc->lock); } /* @@ -1955,6 +2310,15 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) struct kvm_vcpu *vcpu, *vnext; int i; int srcu_idx; + struct core_info core_info; + struct kvmppc_vcore *pvc, *vcnext; + struct kvm_split_mode split_info, *sip; + int split, subcore_size, active; + int sub; + bool thr0_done; + unsigned long cmd_bit, stat_bit; + int pcpu, thr; + int target_threads; /* * Remove from the list any threads that have a signal pending @@ -1969,11 +2333,8 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) /* * Initialize *vc. */ - vc->entry_exit_map = 0; + init_master_vcore(vc); vc->preempt_tb = TB_NIL; - vc->in_guest = 0; - vc->napping_threads = 0; - vc->conferring_threads = 0; /* * Make sure we are running on primary threads, and that secondary @@ -1991,24 +2352,120 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) goto out; } + /* + * See if we could run any other vcores on the physical core + * along with this one. + */ + init_core_info(&core_info, vc); + pcpu = smp_processor_id(); + target_threads = threads_per_subcore; + if (target_smt_mode && target_smt_mode < target_threads) + target_threads = target_smt_mode; + if (vc->num_threads < target_threads) + collect_piggybacks(&core_info, target_threads); - vc->pcpu = smp_processor_id(); - list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) { - kvmppc_start_thread(vcpu); - kvmppc_create_dtl_entry(vcpu, vc); - trace_kvm_guest_enter(vcpu); + /* Decide on micro-threading (split-core) mode */ + subcore_size = threads_per_subcore; + cmd_bit = stat_bit = 0; + split = core_info.n_subcores; + sip = NULL; + if (split > 1) { + /* threads_per_subcore must be MAX_SMT_THREADS (8) here */ + if (split == 2 && (dynamic_mt_modes & 2)) { + cmd_bit = HID0_POWER8_1TO2LPAR; + stat_bit = HID0_POWER8_2LPARMODE; + } else { + split = 4; + cmd_bit = HID0_POWER8_1TO4LPAR; + stat_bit = HID0_POWER8_4LPARMODE; + } + subcore_size = MAX_SMT_THREADS / split; + sip = &split_info; + memset(&split_info, 0, sizeof(split_info)); + split_info.rpr = mfspr(SPRN_RPR); + split_info.pmmar = mfspr(SPRN_PMMAR); + split_info.ldbar = mfspr(SPRN_LDBAR); + split_info.subcore_size = subcore_size; + for (sub = 0; sub < core_info.n_subcores; ++sub) + split_info.master_vcs[sub] = + list_first_entry(&core_info.vcs[sub], + struct kvmppc_vcore, preempt_list); + /* order writes to split_info before kvm_split_mode pointer */ + smp_wmb(); + } + pcpu = smp_processor_id(); + for (thr = 0; thr < threads_per_subcore; ++thr) + paca[pcpu + thr].kvm_hstate.kvm_split_mode = sip; + + /* Initiate micro-threading (split-core) if required */ + if (cmd_bit) { + unsigned long hid0 = mfspr(SPRN_HID0); + + hid0 |= cmd_bit | HID0_POWER8_DYNLPARDIS; + mb(); + mtspr(SPRN_HID0, hid0); + isync(); + for (;;) { + hid0 = mfspr(SPRN_HID0); + if (hid0 & stat_bit) + break; + cpu_relax(); + } } - /* Set this explicitly in case thread 0 doesn't have a vcpu */ - get_paca()->kvm_hstate.kvm_vcore = vc; - get_paca()->kvm_hstate.ptid = 0; + /* Start all the threads */ + active = 0; + for (sub = 0; sub < core_info.n_subcores; ++sub) { + thr = subcore_thread_map[sub]; + thr0_done = false; + active |= 1 << thr; + list_for_each_entry(pvc, &core_info.vcs[sub], preempt_list) { + pvc->pcpu = pcpu + thr; + list_for_each_entry(vcpu, &pvc->runnable_threads, + arch.run_list) { + kvmppc_start_thread(vcpu, pvc); + kvmppc_create_dtl_entry(vcpu, pvc); + trace_kvm_guest_enter(vcpu); + if (!vcpu->arch.ptid) + thr0_done = true; + active |= 1 << (thr + vcpu->arch.ptid); + } + /* + * We need to start the first thread of each subcore + * even if it doesn't have a vcpu. + */ + if (pvc->master_vcore == pvc && !thr0_done) + kvmppc_start_thread(NULL, pvc); + thr += pvc->num_threads; + } + } + + /* + * Ensure that split_info.do_nap is set after setting + * the vcore pointer in the PACA of the secondaries. + */ + smp_mb(); + if (cmd_bit) + split_info.do_nap = 1; /* ask secondaries to nap when done */ + + /* + * When doing micro-threading, poke the inactive threads as well. + * This gets them to the nap instruction after kvm_do_nap, + * which reduces the time taken to unsplit later. + */ + if (split > 1) + for (thr = 1; thr < threads_per_subcore; ++thr) + if (!(active & (1 << thr))) + kvmppc_ipi_thread(pcpu + thr); vc->vcore_state = VCORE_RUNNING; preempt_disable(); trace_kvmppc_run_core(vc, 0); - spin_unlock(&vc->lock); + for (sub = 0; sub < core_info.n_subcores; ++sub) + list_for_each_entry(pvc, &core_info.vcs[sub], preempt_list) + spin_unlock(&pvc->lock); kvm_guest_enter(); @@ -2019,32 +2476,58 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) __kvmppc_vcore_entry(); - spin_lock(&vc->lock); - if (vc->mpp_buffer) kvmppc_start_saving_l2_cache(vc); - /* disable sending of IPIs on virtual external irqs */ - list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) - vcpu->cpu = -1; - /* wait for secondary threads to finish writing their state to memory */ - kvmppc_wait_for_nap(); - for (i = 0; i < threads_per_subcore; ++i) - kvmppc_release_hwthread(vc->pcpu + i); + srcu_read_unlock(&vc->kvm->srcu, srcu_idx); + + spin_lock(&vc->lock); /* prevent other vcpu threads from doing kvmppc_start_thread() now */ vc->vcore_state = VCORE_EXITING; - spin_unlock(&vc->lock); - srcu_read_unlock(&vc->kvm->srcu, srcu_idx); + /* wait for secondary threads to finish writing their state to memory */ + kvmppc_wait_for_nap(); + + /* Return to whole-core mode if we split the core earlier */ + if (split > 1) { + unsigned long hid0 = mfspr(SPRN_HID0); + unsigned long loops = 0; + + hid0 &= ~HID0_POWER8_DYNLPARDIS; + stat_bit = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE; + mb(); + mtspr(SPRN_HID0, hid0); + isync(); + for (;;) { + hid0 = mfspr(SPRN_HID0); + if (!(hid0 & stat_bit)) + break; + cpu_relax(); + ++loops; + } + split_info.do_nap = 0; + } + + /* Let secondaries go back to the offline loop */ + for (i = 0; i < threads_per_subcore; ++i) { + kvmppc_release_hwthread(pcpu + i); + if (sip && sip->napped[i]) + kvmppc_ipi_thread(pcpu + i); + } + + spin_unlock(&vc->lock); /* make sure updates to secondary vcpu structs are visible now */ smp_mb(); kvm_guest_exit(); - preempt_enable(); + for (sub = 0; sub < core_info.n_subcores; ++sub) + list_for_each_entry_safe(pvc, vcnext, &core_info.vcs[sub], + preempt_list) + post_guest_process(pvc, pvc == vc); spin_lock(&vc->lock); - post_guest_process(vc); + preempt_enable(); out: vc->vcore_state = VCORE_INACTIVE; @@ -2055,13 +2538,17 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) * Wait for some other vcpu thread to execute us, and * wake us up when we need to handle something in the host. */ -static void kvmppc_wait_for_exec(struct kvm_vcpu *vcpu, int wait_state) +static void kvmppc_wait_for_exec(struct kvmppc_vcore *vc, + struct kvm_vcpu *vcpu, int wait_state) { DEFINE_WAIT(wait); prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state); - if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) + if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) { + spin_unlock(&vc->lock); schedule(); + spin_lock(&vc->lock); + } finish_wait(&vcpu->arch.cpu_run, &wait); } @@ -2137,9 +2624,21 @@ static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) * this thread straight away and have it join in. */ if (!signal_pending(current)) { - if (vc->vcore_state == VCORE_RUNNING && !VCORE_IS_EXITING(vc)) { + if (vc->vcore_state == VCORE_PIGGYBACK) { + struct kvmppc_vcore *mvc = vc->master_vcore; + if (spin_trylock(&mvc->lock)) { + if (mvc->vcore_state == VCORE_RUNNING && + !VCORE_IS_EXITING(mvc)) { + kvmppc_create_dtl_entry(vcpu, vc); + kvmppc_start_thread(vcpu, vc); + trace_kvm_guest_enter(vcpu); + } + spin_unlock(&mvc->lock); + } + } else if (vc->vcore_state == VCORE_RUNNING && + !VCORE_IS_EXITING(vc)) { kvmppc_create_dtl_entry(vcpu, vc); - kvmppc_start_thread(vcpu); + kvmppc_start_thread(vcpu, vc); trace_kvm_guest_enter(vcpu); } else if (vc->vcore_state == VCORE_SLEEPING) { wake_up(&vc->wq); @@ -2149,10 +2648,11 @@ static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE && !signal_pending(current)) { + if (vc->vcore_state == VCORE_PREEMPT && vc->runner == NULL) + kvmppc_vcore_end_preempt(vc); + if (vc->vcore_state != VCORE_INACTIVE) { - spin_unlock(&vc->lock); - kvmppc_wait_for_exec(vcpu, TASK_INTERRUPTIBLE); - spin_lock(&vc->lock); + kvmppc_wait_for_exec(vc, vcpu, TASK_INTERRUPTIBLE); continue; } list_for_each_entry_safe(v, vn, &vc->runnable_threads, @@ -2179,10 +2679,11 @@ static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) if (n_ceded == vc->n_runnable) { kvmppc_vcore_blocked(vc); } else if (need_resched()) { - vc->vcore_state = VCORE_PREEMPT; + kvmppc_vcore_preempt(vc); /* Let something else run */ cond_resched_lock(&vc->lock); - vc->vcore_state = VCORE_INACTIVE; + if (vc->vcore_state == VCORE_PREEMPT) + kvmppc_vcore_end_preempt(vc); } else { kvmppc_run_core(vc); } @@ -2191,11 +2692,8 @@ static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE && (vc->vcore_state == VCORE_RUNNING || - vc->vcore_state == VCORE_EXITING)) { - spin_unlock(&vc->lock); - kvmppc_wait_for_exec(vcpu, TASK_UNINTERRUPTIBLE); - spin_lock(&vc->lock); - } + vc->vcore_state == VCORE_EXITING)) + kvmppc_wait_for_exec(vc, vcpu, TASK_UNINTERRUPTIBLE); if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) { kvmppc_remove_runnable(vc, vcpu); @@ -2755,6 +3253,8 @@ static int kvmppc_book3s_init_hv(void) init_default_hcalls(); + init_vcore_lists(); + r = kvmppc_mmu_hv_init(); return r; } diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c index ed2589d4593f..fd7006bf6b1a 100644 --- a/arch/powerpc/kvm/book3s_hv_builtin.c +++ b/arch/powerpc/kvm/book3s_hv_builtin.c @@ -110,14 +110,15 @@ void __init kvm_cma_reserve(void) long int kvmppc_rm_h_confer(struct kvm_vcpu *vcpu, int target, unsigned int yield_count) { - struct kvmppc_vcore *vc = vcpu->arch.vcore; + struct kvmppc_vcore *vc = local_paca->kvm_hstate.kvm_vcore; + int ptid = local_paca->kvm_hstate.ptid; int threads_running; int threads_ceded; int threads_conferring; u64 stop = get_tb() + 10 * tb_ticks_per_usec; int rv = H_SUCCESS; /* => don't yield */ - set_bit(vcpu->arch.ptid, &vc->conferring_threads); + set_bit(ptid, &vc->conferring_threads); while ((get_tb() < stop) && !VCORE_IS_EXITING(vc)) { threads_running = VCORE_ENTRY_MAP(vc); threads_ceded = vc->napping_threads; @@ -127,7 +128,7 @@ long int kvmppc_rm_h_confer(struct kvm_vcpu *vcpu, int target, break; } } - clear_bit(vcpu->arch.ptid, &vc->conferring_threads); + clear_bit(ptid, &vc->conferring_threads); return rv; } @@ -238,7 +239,8 @@ void kvmhv_commence_exit(int trap) { struct kvmppc_vcore *vc = local_paca->kvm_hstate.kvm_vcore; int ptid = local_paca->kvm_hstate.ptid; - int me, ee; + struct kvm_split_mode *sip = local_paca->kvm_hstate.kvm_split_mode; + int me, ee, i; /* Set our bit in the threads-exiting-guest map in the 0xff00 bits of vcore->entry_exit_map */ @@ -258,4 +260,26 @@ void kvmhv_commence_exit(int trap) */ if (trap != BOOK3S_INTERRUPT_HV_DECREMENTER) kvmhv_interrupt_vcore(vc, ee & ~(1 << ptid)); + + /* + * If we are doing dynamic micro-threading, interrupt the other + * subcores to pull them out of their guests too. + */ + if (!sip) + return; + + for (i = 0; i < MAX_SUBCORES; ++i) { + vc = sip->master_vcs[i]; + if (!vc) + break; + do { + ee = vc->entry_exit_map; + /* Already asked to exit? */ + if ((ee >> 8) != 0) + break; + } while (cmpxchg(&vc->entry_exit_map, ee, + ee | VCORE_EXIT_REQ) != ee); + if ((ee >> 8) == 0) + kvmhv_interrupt_vcore(vc, ee); + } } diff --git a/arch/powerpc/kvm/book3s_hv_rm_mmu.c b/arch/powerpc/kvm/book3s_hv_rm_mmu.c index b027a89737b6..c1df9bb1e413 100644 --- a/arch/powerpc/kvm/book3s_hv_rm_mmu.c +++ b/arch/powerpc/kvm/book3s_hv_rm_mmu.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -97,25 +98,52 @@ void kvmppc_add_revmap_chain(struct kvm *kvm, struct revmap_entry *rev, } EXPORT_SYMBOL_GPL(kvmppc_add_revmap_chain); +/* Update the changed page order field of an rmap entry */ +void kvmppc_update_rmap_change(unsigned long *rmap, unsigned long psize) +{ + unsigned long order; + + if (!psize) + return; + order = ilog2(psize); + order <<= KVMPPC_RMAP_CHG_SHIFT; + if (order > (*rmap & KVMPPC_RMAP_CHG_ORDER)) + *rmap = (*rmap & ~KVMPPC_RMAP_CHG_ORDER) | order; +} +EXPORT_SYMBOL_GPL(kvmppc_update_rmap_change); + +/* Returns a pointer to the revmap entry for the page mapped by a HPTE */ +static unsigned long *revmap_for_hpte(struct kvm *kvm, unsigned long hpte_v, + unsigned long hpte_gr) +{ + struct kvm_memory_slot *memslot; + unsigned long *rmap; + unsigned long gfn; + + gfn = hpte_rpn(hpte_gr, hpte_page_size(hpte_v, hpte_gr)); + memslot = __gfn_to_memslot(kvm_memslots_raw(kvm), gfn); + if (!memslot) + return NULL; + + rmap = real_vmalloc_addr(&memslot->arch.rmap[gfn - memslot->base_gfn]); + return rmap; +} + /* Remove this HPTE from the chain for a real page */ static void remove_revmap_chain(struct kvm *kvm, long pte_index, struct revmap_entry *rev, unsigned long hpte_v, unsigned long hpte_r) { struct revmap_entry *next, *prev; - unsigned long gfn, ptel, head; - struct kvm_memory_slot *memslot; + unsigned long ptel, head; unsigned long *rmap; unsigned long rcbits; rcbits = hpte_r & (HPTE_R_R | HPTE_R_C); ptel = rev->guest_rpte |= rcbits; - gfn = hpte_rpn(ptel, hpte_page_size(hpte_v, ptel)); - memslot = __gfn_to_memslot(kvm_memslots_raw(kvm), gfn); - if (!memslot) + rmap = revmap_for_hpte(kvm, hpte_v, ptel); + if (!rmap) return; - - rmap = real_vmalloc_addr(&memslot->arch.rmap[gfn - memslot->base_gfn]); lock_rmap(rmap); head = *rmap & KVMPPC_RMAP_INDEX; @@ -131,6 +159,8 @@ static void remove_revmap_chain(struct kvm *kvm, long pte_index, *rmap = (*rmap & ~KVMPPC_RMAP_INDEX) | head; } *rmap |= rcbits << KVMPPC_RMAP_RC_SHIFT; + if (rcbits & HPTE_R_C) + kvmppc_update_rmap_change(rmap, hpte_page_size(hpte_v, hpte_r)); unlock_rmap(rmap); } @@ -421,14 +451,20 @@ long kvmppc_do_h_remove(struct kvm *kvm, unsigned long flags, rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]); v = pte & ~HPTE_V_HVLOCK; if (v & HPTE_V_VALID) { - u64 pte1; - - pte1 = be64_to_cpu(hpte[1]); hpte[0] &= ~cpu_to_be64(HPTE_V_VALID); - rb = compute_tlbie_rb(v, pte1, pte_index); + rb = compute_tlbie_rb(v, be64_to_cpu(hpte[1]), pte_index); do_tlbies(kvm, &rb, 1, global_invalidates(kvm, flags), true); - /* Read PTE low word after tlbie to get final R/C values */ - remove_revmap_chain(kvm, pte_index, rev, v, pte1); + /* + * The reference (R) and change (C) bits in a HPT + * entry can be set by hardware at any time up until + * the HPTE is invalidated and the TLB invalidation + * sequence has completed. This means that when + * removing a HPTE, we need to re-read the HPTE after + * the invalidation sequence has completed in order to + * obtain reliable values of R and C. + */ + remove_revmap_chain(kvm, pte_index, rev, v, + be64_to_cpu(hpte[1])); } r = rev->guest_rpte & ~HPTE_GR_RESERVED; note_hpte_modification(kvm, rev); @@ -655,6 +691,105 @@ long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags, return H_SUCCESS; } +long kvmppc_h_clear_ref(struct kvm_vcpu *vcpu, unsigned long flags, + unsigned long pte_index) +{ + struct kvm *kvm = vcpu->kvm; + __be64 *hpte; + unsigned long v, r, gr; + struct revmap_entry *rev; + unsigned long *rmap; + long ret = H_NOT_FOUND; + + if (pte_index >= kvm->arch.hpt_npte) + return H_PARAMETER; + + rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]); + hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4)); + while (!try_lock_hpte(hpte, HPTE_V_HVLOCK)) + cpu_relax(); + v = be64_to_cpu(hpte[0]); + r = be64_to_cpu(hpte[1]); + if (!(v & (HPTE_V_VALID | HPTE_V_ABSENT))) + goto out; + + gr = rev->guest_rpte; + if (rev->guest_rpte & HPTE_R_R) { + rev->guest_rpte &= ~HPTE_R_R; + note_hpte_modification(kvm, rev); + } + if (v & HPTE_V_VALID) { + gr |= r & (HPTE_R_R | HPTE_R_C); + if (r & HPTE_R_R) { + kvmppc_clear_ref_hpte(kvm, hpte, pte_index); + rmap = revmap_for_hpte(kvm, v, gr); + if (rmap) { + lock_rmap(rmap); + *rmap |= KVMPPC_RMAP_REFERENCED; + unlock_rmap(rmap); + } + } + } + vcpu->arch.gpr[4] = gr; + ret = H_SUCCESS; + out: + unlock_hpte(hpte, v & ~HPTE_V_HVLOCK); + return ret; +} + +long kvmppc_h_clear_mod(struct kvm_vcpu *vcpu, unsigned long flags, + unsigned long pte_index) +{ + struct kvm *kvm = vcpu->kvm; + __be64 *hpte; + unsigned long v, r, gr; + struct revmap_entry *rev; + unsigned long *rmap; + long ret = H_NOT_FOUND; + + if (pte_index >= kvm->arch.hpt_npte) + return H_PARAMETER; + + rev = real_vmalloc_addr(&kvm->arch.revmap[pte_index]); + hpte = (__be64 *)(kvm->arch.hpt_virt + (pte_index << 4)); + while (!try_lock_hpte(hpte, HPTE_V_HVLOCK)) + cpu_relax(); + v = be64_to_cpu(hpte[0]); + r = be64_to_cpu(hpte[1]); + if (!(v & (HPTE_V_VALID | HPTE_V_ABSENT))) + goto out; + + gr = rev->guest_rpte; + if (gr & HPTE_R_C) { + rev->guest_rpte &= ~HPTE_R_C; + note_hpte_modification(kvm, rev); + } + if (v & HPTE_V_VALID) { + /* need to make it temporarily absent so C is stable */ + hpte[0] |= cpu_to_be64(HPTE_V_ABSENT); + kvmppc_invalidate_hpte(kvm, hpte, pte_index); + r = be64_to_cpu(hpte[1]); + gr |= r & (HPTE_R_R | HPTE_R_C); + if (r & HPTE_R_C) { + unsigned long psize = hpte_page_size(v, r); + hpte[1] = cpu_to_be64(r & ~HPTE_R_C); + eieio(); + rmap = revmap_for_hpte(kvm, v, gr); + if (rmap) { + lock_rmap(rmap); + *rmap |= KVMPPC_RMAP_CHANGED; + kvmppc_update_rmap_change(rmap, psize); + unlock_rmap(rmap); + } + } + } + vcpu->arch.gpr[4] = gr; + ret = H_SUCCESS; + out: + unlock_hpte(hpte, v & ~HPTE_V_HVLOCK); + return ret; +} + void kvmppc_invalidate_hpte(struct kvm *kvm, __be64 *hptep, unsigned long pte_index) { diff --git a/arch/powerpc/kvm/book3s_hv_rm_xics.c b/arch/powerpc/kvm/book3s_hv_rm_xics.c index 00e45b6d4f24..24f58076d49e 100644 --- a/arch/powerpc/kvm/book3s_hv_rm_xics.c +++ b/arch/powerpc/kvm/book3s_hv_rm_xics.c @@ -67,14 +67,12 @@ static void icp_rm_set_vcpu_irq(struct kvm_vcpu *vcpu, } /* Check if the core is loaded, if not, too hard */ - cpu = vcpu->cpu; + cpu = vcpu->arch.thread_cpu; if (cpu < 0 || cpu >= nr_cpu_ids) { this_icp->rm_action |= XICS_RM_KICK_VCPU; this_icp->rm_kick_target = vcpu; return; } - /* In SMT cpu will always point to thread 0, we adjust it */ - cpu += vcpu->arch.ptid; smp_mb(); kvmhv_rm_send_ipi(cpu); diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S index faa86e9c0551..2273dcacef39 100644 --- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S +++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S @@ -128,6 +128,10 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) subf r4, r4, r3 mtspr SPRN_DEC, r4 + /* hwthread_req may have got set by cede or no vcpu, so clear it */ + li r0, 0 + stb r0, HSTATE_HWTHREAD_REQ(r13) + /* * For external and machine check interrupts, we need * to call the Linux handler to process the interrupt. @@ -215,7 +219,6 @@ kvm_novcpu_wakeup: ld r5, HSTATE_KVM_VCORE(r13) li r0, 0 stb r0, HSTATE_NAPPING(r13) - stb r0, HSTATE_HWTHREAD_REQ(r13) /* check the wake reason */ bl kvmppc_check_wake_reason @@ -315,10 +318,10 @@ kvm_start_guest: cmpdi r3, 0 bge kvm_no_guest - /* get vcpu pointer, NULL if we have no vcpu to run */ - ld r4,HSTATE_KVM_VCPU(r13) - cmpdi r4,0 - /* if we have no vcpu to run, go back to sleep */ + /* get vcore pointer, NULL if we have nothing to run */ + ld r5,HSTATE_KVM_VCORE(r13) + cmpdi r5,0 + /* if we have no vcore to run, go back to sleep */ beq kvm_no_guest kvm_secondary_got_guest: @@ -327,21 +330,42 @@ kvm_secondary_got_guest: ld r6, PACA_DSCR_DEFAULT(r13) std r6, HSTATE_DSCR(r13) - /* Order load of vcore, ptid etc. after load of vcpu */ + /* On thread 0 of a subcore, set HDEC to max */ + lbz r4, HSTATE_PTID(r13) + cmpwi r4, 0 + bne 63f + lis r6, 0x7fff + ori r6, r6, 0xffff + mtspr SPRN_HDEC, r6 + /* and set per-LPAR registers, if doing dynamic micro-threading */ + ld r6, HSTATE_SPLIT_MODE(r13) + cmpdi r6, 0 + beq 63f + ld r0, KVM_SPLIT_RPR(r6) + mtspr SPRN_RPR, r0 + ld r0, KVM_SPLIT_PMMAR(r6) + mtspr SPRN_PMMAR, r0 + ld r0, KVM_SPLIT_LDBAR(r6) + mtspr SPRN_LDBAR, r0 + isync +63: + /* Order load of vcpu after load of vcore */ lwsync + ld r4, HSTATE_KVM_VCPU(r13) bl kvmppc_hv_entry /* Back from the guest, go back to nap */ - /* Clear our vcpu pointer so we don't come back in early */ + /* Clear our vcpu and vcore pointers so we don't come back in early */ li r0, 0 + std r0, HSTATE_KVM_VCPU(r13) /* - * Once we clear HSTATE_KVM_VCPU(r13), the code in + * Once we clear HSTATE_KVM_VCORE(r13), the code in * kvmppc_run_core() is going to assume that all our vcpu * state is visible in memory. This lwsync makes sure * that that is true. */ lwsync - std r0, HSTATE_KVM_VCPU(r13) + std r0, HSTATE_KVM_VCORE(r13) /* * At this point we have finished executing in the guest. @@ -374,16 +398,71 @@ kvm_no_guest: b power7_wakeup_loss 53: HMT_LOW - ld r4, HSTATE_KVM_VCPU(r13) - cmpdi r4, 0 + ld r5, HSTATE_KVM_VCORE(r13) + cmpdi r5, 0 + bne 60f + ld r3, HSTATE_SPLIT_MODE(r13) + cmpdi r3, 0 + beq kvm_no_guest + lbz r0, KVM_SPLIT_DO_NAP(r3) + cmpwi r0, 0 beq kvm_no_guest HMT_MEDIUM + b kvm_unsplit_nap +60: HMT_MEDIUM b kvm_secondary_got_guest 54: li r0, KVM_HWTHREAD_IN_KVM stb r0, HSTATE_HWTHREAD_STATE(r13) b kvm_no_guest +/* + * Here the primary thread is trying to return the core to + * whole-core mode, so we need to nap. + */ +kvm_unsplit_nap: + /* + * Ensure that secondary doesn't nap when it has + * its vcore pointer set. + */ + sync /* matches smp_mb() before setting split_info.do_nap */ + ld r0, HSTATE_KVM_VCORE(r13) + cmpdi r0, 0 + bne kvm_no_guest + /* clear any pending message */ +BEGIN_FTR_SECTION + lis r6, (PPC_DBELL_SERVER << (63-36))@h + PPC_MSGCLR(6) +END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) + /* Set kvm_split_mode.napped[tid] = 1 */ + ld r3, HSTATE_SPLIT_MODE(r13) + li r0, 1 + lhz r4, PACAPACAINDEX(r13) + clrldi r4, r4, 61 /* micro-threading => P8 => 8 threads/core */ + addi r4, r4, KVM_SPLIT_NAPPED + stbx r0, r3, r4 + /* Check the do_nap flag again after setting napped[] */ + sync + lbz r0, KVM_SPLIT_DO_NAP(r3) + cmpwi r0, 0 + beq 57f + li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4 + mfspr r4, SPRN_LPCR + rlwimi r4, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1) + mtspr SPRN_LPCR, r4 + isync + std r0, HSTATE_SCRATCH0(r13) + ptesync + ld r0, HSTATE_SCRATCH0(r13) +1: cmpd r0, r0 + bne 1b + nap + b . + +57: li r0, 0 + stbx r0, r3, r4 + b kvm_no_guest + /****************************************************************************** * * * Entry code * @@ -854,7 +933,10 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) cmpwi r0, 0 bne 21f HMT_LOW -20: lbz r0, VCORE_IN_GUEST(r5) +20: lwz r3, VCORE_ENTRY_EXIT(r5) + cmpwi r3, 0x100 + bge no_switch_exit + lbz r0, VCORE_IN_GUEST(r5) cmpwi r0, 0 beq 20b HMT_MEDIUM @@ -870,7 +952,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) blt hdec_soon ld r6, VCPU_CTR(r4) - lwz r7, VCPU_XER(r4) + ld r7, VCPU_XER(r4) mtctr r6 mtxer r7 @@ -985,9 +1067,13 @@ secondary_too_late: #endif 11: b kvmhv_switch_to_host +no_switch_exit: + HMT_MEDIUM + li r12, 0 + b 12f hdec_soon: li r12, BOOK3S_INTERRUPT_HV_DECREMENTER - stw r12, VCPU_TRAP(r4) +12: stw r12, VCPU_TRAP(r4) mr r9, r4 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING addi r3, r4, VCPU_TB_RMEXIT @@ -1103,7 +1189,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) mfctr r3 mfxer r4 std r3, VCPU_CTR(r9) - stw r4, VCPU_XER(r9) + std r4, VCPU_XER(r9) /* If this is a page table miss then see if it's theirs or ours */ cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE @@ -1127,6 +1213,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL bne 3f lbz r0, HSTATE_HOST_IPI(r13) + cmpwi r0, 0 beq 4f b guest_exit_cont 3: @@ -1176,6 +1263,11 @@ mc_cont: ld r9, HSTATE_KVM_VCPU(r13) lwz r12, VCPU_TRAP(r9) + /* Stop others sending VCPU interrupts to this physical CPU */ + li r0, -1 + stw r0, VCPU_CPU(r9) + stw r0, VCPU_THREAD_CPU(r9) + /* Save guest CTRL register, set runlatch to 1 */ mfspr r6,SPRN_CTRLF stw r6,VCPU_CTRL(r9) @@ -1540,12 +1632,17 @@ kvmhv_switch_to_host: /* Primary thread waits for all the secondaries to exit guest */ 15: lwz r3,VCORE_ENTRY_EXIT(r5) - srwi r0,r3,8 + rlwinm r0,r3,32-8,0xff clrldi r3,r3,56 cmpw r3,r0 bne 15b isync + /* Did we actually switch to the guest at all? */ + lbz r6, VCORE_IN_GUEST(r5) + cmpwi r6, 0 + beq 19f + /* Primary thread switches back to host partition */ ld r6,KVM_HOST_SDR1(r4) lwz r7,KVM_HOST_LPID(r4) @@ -1589,7 +1686,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) 18: /* Signal secondary CPUs to continue */ stb r0,VCORE_IN_GUEST(r5) - lis r8,0x7fff /* MAX_INT@h */ +19: lis r8,0x7fff /* MAX_INT@h */ mtspr SPRN_HDEC,r8 16: ld r8,KVM_HOST_LPCR(r4) @@ -1675,7 +1772,7 @@ kvmppc_hdsi: bl kvmppc_msr_interrupt fast_interrupt_c_return: 6: ld r7, VCPU_CTR(r9) - lwz r8, VCPU_XER(r9) + ld r8, VCPU_XER(r9) mtctr r7 mtxer r8 mr r4, r9 @@ -1816,8 +1913,8 @@ hcall_real_table: .long DOTSYM(kvmppc_h_remove) - hcall_real_table .long DOTSYM(kvmppc_h_enter) - hcall_real_table .long DOTSYM(kvmppc_h_read) - hcall_real_table - .long 0 /* 0x10 - H_CLEAR_MOD */ - .long 0 /* 0x14 - H_CLEAR_REF */ + .long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table + .long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table .long DOTSYM(kvmppc_h_protect) - hcall_real_table .long DOTSYM(kvmppc_h_get_tce) - hcall_real_table .long DOTSYM(kvmppc_h_put_tce) - hcall_real_table diff --git a/arch/powerpc/kvm/book3s_paired_singles.c b/arch/powerpc/kvm/book3s_paired_singles.c index bd6ab1672ae6..a759d9adb0b6 100644 --- a/arch/powerpc/kvm/book3s_paired_singles.c +++ b/arch/powerpc/kvm/book3s_paired_singles.c @@ -352,7 +352,7 @@ static inline u32 inst_get_field(u32 inst, int msb, int lsb) return kvmppc_get_field(inst, msb + 32, lsb + 32); } -bool kvmppc_inst_is_paired_single(struct kvm_vcpu *vcpu, u32 inst) +static bool kvmppc_inst_is_paired_single(struct kvm_vcpu *vcpu, u32 inst) { if (!(vcpu->arch.hflags & BOOK3S_HFLAG_PAIRED_SINGLE)) return false; diff --git a/arch/powerpc/kvm/book3s_segment.S b/arch/powerpc/kvm/book3s_segment.S index acee37cde840..ca8f174289bb 100644 --- a/arch/powerpc/kvm/book3s_segment.S +++ b/arch/powerpc/kvm/book3s_segment.S @@ -123,7 +123,7 @@ no_dcbz32_on: PPC_LL r8, SVCPU_CTR(r3) PPC_LL r9, SVCPU_LR(r3) lwz r10, SVCPU_CR(r3) - lwz r11, SVCPU_XER(r3) + PPC_LL r11, SVCPU_XER(r3) mtctr r8 mtlr r9 @@ -237,7 +237,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) mfctr r8 mflr r9 - stw r5, SVCPU_XER(r13) + PPC_STL r5, SVCPU_XER(r13) PPC_STL r6, SVCPU_FAULT_DAR(r13) stw r7, SVCPU_FAULT_DSISR(r13) PPC_STL r8, SVCPU_CTR(r13) diff --git a/arch/powerpc/kvm/book3s_xics.c b/arch/powerpc/kvm/book3s_xics.c index c6ca7db64673..905e94a1370f 100644 --- a/arch/powerpc/kvm/book3s_xics.c +++ b/arch/powerpc/kvm/book3s_xics.c @@ -41,7 +41,7 @@ * ======= * * Each ICS has a spin lock protecting the information about the IRQ - * sources and avoiding simultaneous deliveries if the same interrupt. + * sources and avoiding simultaneous deliveries of the same interrupt. * * ICP operations are done via a single compare & swap transaction * (most ICP state fits in the union kvmppc_icp_state) diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c index cc5842657161..ae458f0fd061 100644 --- a/arch/powerpc/kvm/booke.c +++ b/arch/powerpc/kvm/booke.c @@ -933,6 +933,7 @@ static void kvmppc_restart_interrupt(struct kvm_vcpu *vcpu, #endif break; case BOOKE_INTERRUPT_CRITICAL: + kvmppc_fill_pt_regs(®s); unknown_exception(®s); break; case BOOKE_INTERRUPT_DEBUG: diff --git a/arch/powerpc/kvm/e500_mmu.c b/arch/powerpc/kvm/e500_mmu.c index 50860e919cb8..29911a07bcdb 100644 --- a/arch/powerpc/kvm/e500_mmu.c +++ b/arch/powerpc/kvm/e500_mmu.c @@ -377,7 +377,7 @@ int kvmppc_e500_emul_tlbsx(struct kvm_vcpu *vcpu, gva_t ea) | MAS0_NV(vcpu_e500->gtlb_nv[tlbsel]); vcpu->arch.shared->mas1 = (vcpu->arch.shared->mas6 & MAS6_SPID0) - | (vcpu->arch.shared->mas6 & (MAS6_SAS ? MAS1_TS : 0)) + | ((vcpu->arch.shared->mas6 & MAS6_SAS) ? MAS1_TS : 0) | (vcpu->arch.shared->mas4 & MAS4_TSIZED(~0)); vcpu->arch.shared->mas2 &= MAS2_EPN; vcpu->arch.shared->mas2 |= vcpu->arch.shared->mas4 & diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index e5dde32fe71f..2e51289610e4 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -660,7 +660,7 @@ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) return kvmppc_core_pending_dec(vcpu); } -enum hrtimer_restart kvmppc_decrementer_wakeup(struct hrtimer *timer) +static enum hrtimer_restart kvmppc_decrementer_wakeup(struct hrtimer *timer) { struct kvm_vcpu *vcpu; diff --git a/arch/powerpc/lib/checksum_32.S b/arch/powerpc/lib/checksum_32.S index 7874e8a80455..6d67e057f15e 100644 --- a/arch/powerpc/lib/checksum_32.S +++ b/arch/powerpc/lib/checksum_32.S @@ -40,22 +40,6 @@ _GLOBAL(ip_fast_csum) srwi r3,r3,16 blr -/* - * Compute checksum of TCP or UDP pseudo-header: - * csum_tcpudp_magic(saddr, daddr, len, proto, sum) - */ -_GLOBAL(csum_tcpudp_magic) - rlwimi r5,r6,16,0,15 /* put proto in upper half of len */ - addc r0,r3,r4 /* add 4 32-bit words together */ - adde r0,r0,r5 - adde r0,r0,r7 - addze r0,r0 /* add in final carry */ - rlwinm r3,r0,16,0,31 /* fold two halves together */ - add r3,r0,r3 - not r3,r3 - srwi r3,r3,16 - blr - /* * computes the checksum of a memory block at buff, length len, * and adds in "sum" (32-bit) diff --git a/arch/powerpc/lib/checksum_64.S b/arch/powerpc/lib/checksum_64.S index 57a072065057..f3ef35436612 100644 --- a/arch/powerpc/lib/checksum_64.S +++ b/arch/powerpc/lib/checksum_64.S @@ -44,27 +44,6 @@ _GLOBAL(ip_fast_csum) srwi r3,r3,16 blr -/* - * Compute checksum of TCP or UDP pseudo-header: - * csum_tcpudp_magic(r3=saddr, r4=daddr, r5=len, r6=proto, r7=sum) - * No real gain trying to do this specially for 64 bit, but - * the 32 bit addition may spill into the upper bits of - * the doubleword so we still must fold it down from 64. - */ -_GLOBAL(csum_tcpudp_magic) - rlwimi r5,r6,16,0,15 /* put proto in upper half of len */ - addc r0,r3,r4 /* add 4 32-bit words together */ - adde r0,r0,r5 - adde r0,r0,r7 - rldicl r4,r0,32,0 /* fold 64 bit value */ - add r0,r4,r0 - srdi r0,r0,32 - rlwinm r3,r0,16,0,31 /* fold two halves together */ - add r3,r0,r3 - not r3,r3 - srwi r3,r3,16 - blr - /* * Computes the checksum of a memory block at buff, length len, * and adds in "sum" (32-bit). diff --git a/arch/powerpc/lib/copy_32.S b/arch/powerpc/lib/copy_32.S index 6813f80d1eec..2ef50c629470 100644 --- a/arch/powerpc/lib/copy_32.S +++ b/arch/powerpc/lib/copy_32.S @@ -69,9 +69,15 @@ CACHELINE_BYTES = L1_CACHE_BYTES LG_CACHELINE_BYTES = L1_CACHE_SHIFT CACHELINE_MASK = (L1_CACHE_BYTES-1) +/* + * Use dcbz on the complete cache lines in the destination + * to set them to zero. This requires that the destination + * area is cacheable. -- paulus + */ _GLOBAL(memset) rlwimi r4,r4,8,16,23 rlwimi r4,r4,16,0,15 + addi r6,r3,-4 cmplwi 0,r5,4 blt 7f @@ -80,7 +86,29 @@ _GLOBAL(memset) andi. r0,r6,3 add r5,r0,r5 subf r6,r0,r6 - srwi r0,r5,2 + cmplwi 0,r4,0 + bne 2f /* Use normal procedure if r4 is not zero */ + + clrlwi r7,r6,32-LG_CACHELINE_BYTES + add r8,r7,r5 + srwi r9,r8,LG_CACHELINE_BYTES + addic. r9,r9,-1 /* total number of complete cachelines */ + ble 2f + xori r0,r7,CACHELINE_MASK & ~3 + srwi. r0,r0,2 + beq 3f + mtctr r0 +4: stwu r4,4(r6) + bdnz 4b +3: mtctr r9 + li r7,4 +10: dcbz r7,r6 + addi r6,r6,CACHELINE_BYTES + bdnz 10b + clrlwi r5,r8,32-LG_CACHELINE_BYTES + addi r5,r5,4 + +2: srwi r0,r5,2 mtctr r0 bdz 6f 1: stwu r4,4(r6) @@ -94,12 +122,91 @@ _GLOBAL(memset) bdnz 8b blr +/* + * This version uses dcbz on the complete cache lines in the + * destination area to reduce memory traffic. This requires that + * the destination area is cacheable. + * We only use this version if the source and dest don't overlap. + * -- paulus. + */ _GLOBAL(memmove) cmplw 0,r3,r4 bgt backwards_memcpy /* fall through */ _GLOBAL(memcpy) + add r7,r3,r5 /* test if the src & dst overlap */ + add r8,r4,r5 + cmplw 0,r4,r7 + cmplw 1,r3,r8 + crand 0,0,4 /* cr0.lt &= cr1.lt */ + blt generic_memcpy /* if regions overlap */ + + addi r4,r4,-4 + addi r6,r3,-4 + neg r0,r3 + andi. r0,r0,CACHELINE_MASK /* # bytes to start of cache line */ + beq 58f + + cmplw 0,r5,r0 /* is this more than total to do? */ + blt 63f /* if not much to do */ + andi. r8,r0,3 /* get it word-aligned first */ + subf r5,r0,r5 + mtctr r8 + beq+ 61f +70: lbz r9,4(r4) /* do some bytes */ + addi r4,r4,1 + addi r6,r6,1 + stb r9,3(r6) + bdnz 70b +61: srwi. r0,r0,2 + mtctr r0 + beq 58f +72: lwzu r9,4(r4) /* do some words */ + stwu r9,4(r6) + bdnz 72b + +58: srwi. r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */ + clrlwi r5,r5,32-LG_CACHELINE_BYTES + li r11,4 + mtctr r0 + beq 63f +53: + dcbz r11,r6 + COPY_16_BYTES +#if L1_CACHE_BYTES >= 32 + COPY_16_BYTES +#if L1_CACHE_BYTES >= 64 + COPY_16_BYTES + COPY_16_BYTES +#if L1_CACHE_BYTES >= 128 + COPY_16_BYTES + COPY_16_BYTES + COPY_16_BYTES + COPY_16_BYTES +#endif +#endif +#endif + bdnz 53b + +63: srwi. r0,r5,2 + mtctr r0 + beq 64f +30: lwzu r0,4(r4) + stwu r0,4(r6) + bdnz 30b + +64: andi. r0,r5,3 + mtctr r0 + beq+ 65f + addi r4,r4,3 + addi r6,r6,3 +40: lbzu r0,1(r4) + stbu r0,1(r6) + bdnz 40b +65: blr + +_GLOBAL(generic_memcpy) srwi. r7,r5,3 addi r6,r3,-4 addi r4,r4,-4 diff --git a/arch/powerpc/mm/fsl_booke_mmu.c b/arch/powerpc/mm/fsl_booke_mmu.c index 9c90e66cffb6..354ba3c09ef3 100644 --- a/arch/powerpc/mm/fsl_booke_mmu.c +++ b/arch/powerpc/mm/fsl_booke_mmu.c @@ -112,7 +112,7 @@ static void settlbcam(int index, unsigned long virt, phys_addr_t phys, tsize = __ilog2(size) - 10; -#ifdef CONFIG_SMP +#if defined(CONFIG_SMP) || defined(CONFIG_PPC_E500MC) if ((flags & _PAGE_NO_CACHE) == 0) flags |= _PAGE_COHERENT; #endif diff --git a/arch/powerpc/mm/hash_low_64.S b/arch/powerpc/mm/hash_low_64.S index 463174a4a647..3b49e3295901 100644 --- a/arch/powerpc/mm/hash_low_64.S +++ b/arch/powerpc/mm/hash_low_64.S @@ -701,7 +701,7 @@ htab_pte_insert_failure: #endif /* CONFIG_PPC_64K_PAGES */ -#ifdef CONFIG_PPC_HAS_HASH_64K +#ifdef CONFIG_PPC_64K_PAGES /***************************************************************************** * * @@ -993,7 +993,7 @@ ht64_pte_insert_failure: b ht64_bail -#endif /* CONFIG_PPC_HAS_HASH_64K */ +#endif /* CONFIG_PPC_64K_PAGES */ /***************************************************************************** diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index 5ec987f65b2c..aee70171355b 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c @@ -640,7 +640,7 @@ extern u32 ht64_call_hpte_updatepp[]; static void __init htab_finish_init(void) { -#ifdef CONFIG_PPC_HAS_HASH_64K +#ifdef CONFIG_PPC_64K_PAGES patch_branch(ht64_call_hpte_insert1, ppc_function_entry(ppc_md.hpte_insert), BRANCH_SET_LINK); @@ -653,7 +653,7 @@ static void __init htab_finish_init(void) patch_branch(ht64_call_hpte_updatepp, ppc_function_entry(ppc_md.hpte_updatepp), BRANCH_SET_LINK); -#endif /* CONFIG_PPC_HAS_HASH_64K */ +#endif /* CONFIG_PPC_64K_PAGES */ patch_branch(htab_call_hpte_insert1, ppc_function_entry(ppc_md.hpte_insert), @@ -1151,12 +1151,12 @@ int hash_page_mm(struct mm_struct *mm, unsigned long ea, check_paca_psize(ea, mm, psize, user_region); #endif /* CONFIG_PPC_64K_PAGES */ -#ifdef CONFIG_PPC_HAS_HASH_64K +#ifdef CONFIG_PPC_64K_PAGES if (psize == MMU_PAGE_64K) rc = __hash_page_64K(ea, access, vsid, ptep, trap, flags, ssize); else -#endif /* CONFIG_PPC_HAS_HASH_64K */ +#endif /* CONFIG_PPC_64K_PAGES */ { int spp = subpage_protection(mm, ea); if (access & spp) @@ -1264,12 +1264,12 @@ void hash_preload(struct mm_struct *mm, unsigned long ea, update_flags |= HPTE_LOCAL_UPDATE; /* Hash it in */ -#ifdef CONFIG_PPC_HAS_HASH_64K +#ifdef CONFIG_PPC_64K_PAGES if (mm->context.user_psize == MMU_PAGE_64K) rc = __hash_page_64K(ea, access, vsid, ptep, trap, update_flags, ssize); else -#endif /* CONFIG_PPC_HAS_HASH_64K */ +#endif /* CONFIG_PPC_64K_PAGES */ rc = __hash_page_4K(ea, access, vsid, ptep, trap, update_flags, ssize, subpage_protection(mm, ea)); diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index bb0bd7025cb8..06c14523b787 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -808,14 +808,6 @@ static int __init add_huge_page_size(unsigned long long size) if ((mmu_psize = shift_to_mmu_psize(shift)) < 0) return -EINVAL; -#ifdef CONFIG_SPU_FS_64K_LS - /* Disable support for 64K huge pages when 64K SPU local store - * support is enabled as the current implementation conflicts. - */ - if (shift == PAGE_SHIFT_64K) - return -EINVAL; -#endif /* CONFIG_SPU_FS_64K_LS */ - BUG_ON(mmu_psize_defs[mmu_psize].shift != shift); /* Return if huge page size has already been setup */ diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c index 0f11819d8f1d..22d94c3e6fc4 100644 --- a/arch/powerpc/mm/mem.c +++ b/arch/powerpc/mm/mem.c @@ -113,7 +113,7 @@ int memory_add_physaddr_to_nid(u64 start) } #endif -int arch_add_memory(int nid, u64 start, u64 size) +int arch_add_memory(int nid, u64 start, u64 size, bool for_device) { struct pglist_data *pgdata; struct zone *zone; @@ -128,7 +128,7 @@ int arch_add_memory(int nid, u64 start, u64 size) /* this should work for most non-highmem platforms */ zone = pgdata->node_zones + - zone_for_memory(nid, start, size, 0); + zone_for_memory(nid, start, size, 0, for_device); return __add_pages(nid, zone, start_pfn, nr_pages); } @@ -414,17 +414,17 @@ void flush_dcache_icache_page(struct page *page) return; } #endif -#ifdef CONFIG_BOOKE - { +#if defined(CONFIG_8xx) || defined(CONFIG_PPC64) + /* On 8xx there is no need to kmap since highmem is not supported */ + __flush_dcache_icache(page_address(page)); +#else + if (IS_ENABLED(CONFIG_BOOKE) || sizeof(phys_addr_t) > sizeof(void *)) { void *start = kmap_atomic(page); __flush_dcache_icache(start); kunmap_atomic(start); + } else { + __flush_dcache_icache_phys(page_to_pfn(page) << PAGE_SHIFT); } -#elif defined(CONFIG_8xx) || defined(CONFIG_PPC64) - /* On 8xx there is no need to kmap since highmem is not supported */ - __flush_dcache_icache(page_address(page)); -#else - __flush_dcache_icache_phys(page_to_pfn(page) << PAGE_SHIFT); #endif } EXPORT_SYMBOL(flush_dcache_icache_page); diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c index 5e80621d9324..8b9502adaf79 100644 --- a/arch/powerpc/mm/numa.c +++ b/arch/powerpc/mm/numa.c @@ -225,7 +225,7 @@ static void initialize_distance_lookup_table(int nid, for (i = 0; i < distance_ref_points_depth; i++) { const __be32 *entry; - entry = &associativity[be32_to_cpu(distance_ref_points[i])]; + entry = &associativity[be32_to_cpu(distance_ref_points[i]) - 1]; distance_lookup_table[nid][i] = of_read_number(entry, 1); } } @@ -248,8 +248,12 @@ static int associativity_to_nid(const __be32 *associativity) nid = -1; if (nid > 0 && - of_read_number(associativity, 1) >= distance_ref_points_depth) - initialize_distance_lookup_table(nid, associativity); + of_read_number(associativity, 1) >= distance_ref_points_depth) { + /* + * Skip the length field and send start of associativity array + */ + initialize_distance_lookup_table(nid, associativity + 1); + } out: return nid; @@ -507,6 +511,12 @@ static int of_drconf_to_nid_single(struct of_drconf_cell *drmem, if (nid == 0xffff || nid >= MAX_NUMNODES) nid = default_nid; + + if (nid > 0) { + index = drmem->aa_index * aa->array_sz; + initialize_distance_lookup_table(nid, + &aa->arrays[index]); + } } return nid; diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c index 876232d64126..e92cb2146b18 100644 --- a/arch/powerpc/mm/pgtable_64.c +++ b/arch/powerpc/mm/pgtable_64.c @@ -149,17 +149,7 @@ int map_kernel_page(unsigned long ea, unsigned long pa, int flags) #endif /* !CONFIG_PPC_MMU_NOHASH */ } -#ifdef CONFIG_PPC_BOOK3E_64 - /* - * With hardware tablewalk, a sync is needed to ensure that - * subsequent accesses see the PTE we just wrote. Unlike userspace - * mappings, we can't tolerate spurious faults, so make sure - * the new PTE will be seen the first time. - */ - mb(); -#else smp_wmb(); -#endif return 0; } diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c index 6e450ca66526..8a32a2be3c53 100644 --- a/arch/powerpc/mm/slb.c +++ b/arch/powerpc/mm/slb.c @@ -41,9 +41,9 @@ static void slb_allocate(unsigned long ea) (((ssize) == MMU_SEGSIZE_256M)? ESID_MASK: ESID_MASK_1T) static inline unsigned long mk_esid_data(unsigned long ea, int ssize, - unsigned long slot) + unsigned long entry) { - return (ea & slb_esid_mask(ssize)) | SLB_ESID_V | slot; + return (ea & slb_esid_mask(ssize)) | SLB_ESID_V | entry; } static inline unsigned long mk_vsid_data(unsigned long ea, int ssize, @@ -249,11 +249,24 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm) static inline void patch_slb_encoding(unsigned int *insn_addr, unsigned int immed) { - int insn = (*insn_addr & 0xffff0000) | immed; + + /* + * This function patches either an li or a cmpldi instruction with + * a new immediate value. This relies on the fact that both li + * (which is actually addi) and cmpldi both take a 16-bit immediate + * value, and it is situated in the same location in the instruction, + * ie. bits 16-31 (Big endian bit order) or the lower 16 bits. + * The signedness of the immediate operand differs between the two + * instructions however this code is only ever patching a small value, + * much less than 1 << 15, so we can get away with it. + * To patch the value we read the existing instruction, clear the + * immediate value, and or in our new value, then write the instruction + * back. + */ + unsigned int insn = (*insn_addr & 0xffff0000) | immed; patch_instruction(insn_addr, insn); } -extern u32 slb_compare_rr_to_size[]; extern u32 slb_miss_kernel_load_linear[]; extern u32 slb_miss_kernel_load_io[]; extern u32 slb_compare_rr_to_size[]; @@ -309,12 +322,11 @@ void slb_initialize(void) lflags = SLB_VSID_KERNEL | linear_llp; vflags = SLB_VSID_KERNEL | vmalloc_llp; - /* Invalidate the entire SLB (even slot 0) & all the ERATS */ + /* Invalidate the entire SLB (even entry 0) & all the ERATS */ asm volatile("isync":::"memory"); asm volatile("slbmte %0,%0"::"r" (0) : "memory"); asm volatile("isync; slbia; isync":::"memory"); create_shadowed_slbe(PAGE_OFFSET, mmu_kernel_ssize, lflags, 0); - create_shadowed_slbe(VMALLOC_START, mmu_kernel_ssize, vflags, 1); /* For the boot cpu, we're running on the stack in init_thread_union, diff --git a/arch/powerpc/mm/tlb_low_64e.S b/arch/powerpc/mm/tlb_low_64e.S index 765b419883f2..e4185581c5a7 100644 --- a/arch/powerpc/mm/tlb_low_64e.S +++ b/arch/powerpc/mm/tlb_low_64e.S @@ -308,11 +308,11 @@ BEGIN_FTR_SECTION /* CPU_FTR_SMT */ * * MAS6:IND should be already set based on MAS4 */ -1: lbarx r15,0,r11 lhz r10,PACAPACAINDEX(r13) - cmpdi r15,0 - cmpdi cr1,r15,1 /* set cr1.eq = 0 for non-recursive */ addi r10,r10,1 + crclr cr1*4+eq /* set cr1.eq = 0 for non-recursive */ +1: lbarx r15,0,r11 + cmpdi r15,0 bne 2f stbcx. r10,0,r11 bne 1b @@ -320,9 +320,9 @@ BEGIN_FTR_SECTION /* CPU_FTR_SMT */ .subsection 1 2: cmpd cr1,r15,r10 /* recursive lock due to mcheck/crit/etc? */ beq cr1,3b /* unlock will happen if cr1.eq = 0 */ - lbz r15,0(r11) +10: lbz r15,0(r11) cmpdi r15,0 - bne 2b + bne 10b b 1b .previous diff --git a/arch/powerpc/oprofile/op_model_power4.c b/arch/powerpc/oprofile/op_model_power4.c index 962fe7b3e3fb..4b32e9404bbe 100644 --- a/arch/powerpc/oprofile/op_model_power4.c +++ b/arch/powerpc/oprofile/op_model_power4.c @@ -207,7 +207,7 @@ static int power4_start(struct op_counter_config *ctr) unsigned int mmcr0; /* set the PMM bit (see comment below) */ - mtmsrd(mfmsr() | MSR_PMM); + mtmsr(mfmsr() | MSR_PMM); for (i = 0; i < cur_cpu_spec->num_pmcs; ++i) { if (ctr[i].enabled) { @@ -377,7 +377,7 @@ static void power4_handle_interrupt(struct pt_regs *regs, is_kernel = get_kernel(pc, mmcra); /* set the PMM bit (see comment below) */ - mtmsrd(mfmsr() | MSR_PMM); + mtmsr(mfmsr() | MSR_PMM); /* Check that the SIAR valid bit in MMCRA is set to 1. */ if ((mmcra & MMCRA_SIAR_VALID_MASK) == MMCRA_SIAR_VALID_MASK) diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c index d90893b76e7c..b0382f3f1095 100644 --- a/arch/powerpc/perf/core-book3s.c +++ b/arch/powerpc/perf/core-book3s.c @@ -53,7 +53,7 @@ struct cpu_hw_events { /* BHRB bits */ u64 bhrb_filter; /* BHRB HW branch filter */ - int bhrb_users; + unsigned int bhrb_users; void *bhrb_context; struct perf_branch_stack bhrb_stack; struct perf_branch_entry bhrb_entries[BHRB_MAX_ENTRIES]; @@ -369,8 +369,8 @@ static void power_pmu_bhrb_disable(struct perf_event *event) if (!ppmu->bhrb_nr) return; + WARN_ON_ONCE(!cpuhw->bhrb_users); cpuhw->bhrb_users--; - WARN_ON_ONCE(cpuhw->bhrb_users < 0); perf_sched_cb_dec(event->ctx->pmu); if (!cpuhw->disabled && !cpuhw->bhrb_users) { diff --git a/arch/powerpc/perf/hv-24x7.c b/arch/powerpc/perf/hv-24x7.c index df956295c2a7..527c8b98e97e 100644 --- a/arch/powerpc/perf/hv-24x7.c +++ b/arch/powerpc/perf/hv-24x7.c @@ -416,7 +416,7 @@ static struct attribute *event_to_attr(unsigned ix, } static struct attribute *event_to_desc_attr(struct hv_24x7_event_data *event, - int nonce) + int nonce) { int nl, dl; char *name = event_name(event, &nl); @@ -444,7 +444,7 @@ event_to_long_desc_attr(struct hv_24x7_event_data *event, int nonce) } static ssize_t event_data_to_attrs(unsigned ix, struct attribute **attrs, - struct hv_24x7_event_data *event, int nonce) + struct hv_24x7_event_data *event, int nonce) { unsigned i; @@ -512,7 +512,7 @@ static int memord(const void *d1, size_t s1, const void *d2, size_t s2) } static int ev_uniq_ord(const void *v1, size_t s1, unsigned d1, const void *v2, - size_t s2, unsigned d2) + size_t s2, unsigned d2) { int r = memord(v1, s1, v2, s2); @@ -526,7 +526,7 @@ static int ev_uniq_ord(const void *v1, size_t s1, unsigned d1, const void *v2, } static int event_uniq_add(struct rb_root *root, const char *name, int nl, - unsigned domain) + unsigned domain) { struct rb_node **new = &(root->rb_node), *parent = NULL; struct event_uniq *data; @@ -650,8 +650,8 @@ static ssize_t catalog_event_len_validate(struct hv_24x7_event_data *event, #define MAX_4K (SIZE_MAX / 4096) static int create_events_from_catalog(struct attribute ***events_, - struct attribute ***event_descs_, - struct attribute ***event_long_descs_) + struct attribute ***event_descs_, + struct attribute ***event_long_descs_) { unsigned long hret; size_t catalog_len, catalog_page_len, event_entry_count, @@ -1008,8 +1008,8 @@ static const struct attribute_group *attr_groups[] = { }; static void log_24x7_hcall(struct hv_24x7_request_buffer *request_buffer, - struct hv_24x7_data_result_buffer *result_buffer, - unsigned long ret) + struct hv_24x7_data_result_buffer *result_buffer, + unsigned long ret) { struct hv_24x7_request *req; @@ -1026,7 +1026,7 @@ static void log_24x7_hcall(struct hv_24x7_request_buffer *request_buffer, * Start the process for a new H_GET_24x7_DATA hcall. */ static void init_24x7_request(struct hv_24x7_request_buffer *request_buffer, - struct hv_24x7_data_result_buffer *result_buffer) + struct hv_24x7_data_result_buffer *result_buffer) { memset(request_buffer, 0, 4096); @@ -1041,7 +1041,7 @@ static void init_24x7_request(struct hv_24x7_request_buffer *request_buffer, * by 'init_24x7_request()' and 'add_event_to_24x7_request()'. */ static int make_24x7_request(struct hv_24x7_request_buffer *request_buffer, - struct hv_24x7_data_result_buffer *result_buffer) + struct hv_24x7_data_result_buffer *result_buffer) { unsigned long ret; @@ -1104,7 +1104,6 @@ static unsigned long single_24x7_request(struct perf_event *event, u64 *count) unsigned long ret; struct hv_24x7_request_buffer *request_buffer; struct hv_24x7_data_result_buffer *result_buffer; - struct hv_24x7_result *resb; BUILD_BUG_ON(sizeof(*request_buffer) > 4096); BUILD_BUG_ON(sizeof(*result_buffer) > 4096); @@ -1125,8 +1124,7 @@ static unsigned long single_24x7_request(struct perf_event *event, u64 *count) } /* process result from hcall */ - resb = &result_buffer->results[0]; - *count = be64_to_cpu(resb->elements[0].element_data[0]); + *count = be64_to_cpu(result_buffer->results[0].elements[0].element_data[0]); out: put_cpu_var(hv_24x7_reqb); diff --git a/arch/powerpc/platforms/512x/Kconfig b/arch/powerpc/platforms/512x/Kconfig index 5aa3f4b5332c..48bf38d0de35 100644 --- a/arch/powerpc/platforms/512x/Kconfig +++ b/arch/powerpc/platforms/512x/Kconfig @@ -7,8 +7,8 @@ config PPC_MPC512x select PPC_PCI_CHOICE select FSL_PCI if PCI select ARCH_WANT_OPTIONAL_GPIOLIB - select USB_EHCI_BIG_ENDIAN_MMIO - select USB_EHCI_BIG_ENDIAN_DESC + select USB_EHCI_BIG_ENDIAN_MMIO if USB_EHCI_HCD + select USB_EHCI_BIG_ENDIAN_DESC if USB_EHCI_HCD config MPC5121_ADS bool "Freescale MPC5121E ADS" diff --git a/arch/powerpc/platforms/512x/mpc5121_ads_cpld.c b/arch/powerpc/platforms/512x/mpc5121_ads_cpld.c index ca3a062ed1b9..11090ab4bf59 100644 --- a/arch/powerpc/platforms/512x/mpc5121_ads_cpld.c +++ b/arch/powerpc/platforms/512x/mpc5121_ads_cpld.c @@ -123,7 +123,8 @@ cpld_pic_cascade(unsigned int irq, struct irq_desc *desc) } static int -cpld_pic_host_match(struct irq_domain *h, struct device_node *node) +cpld_pic_host_match(struct irq_domain *h, struct device_node *node, + enum irq_domain_bus_token bus_token) { return cpld_pic_node == node; } diff --git a/arch/powerpc/platforms/85xx/c293pcie.c b/arch/powerpc/platforms/85xx/c293pcie.c index 84476b646005..61bc851e9a8e 100644 --- a/arch/powerpc/platforms/85xx/c293pcie.c +++ b/arch/powerpc/platforms/85xx/c293pcie.c @@ -66,10 +66,6 @@ define_machine(c293_pcie) { .probe = c293_pcie_probe, .setup_arch = c293_pcie_setup_arch, .init_IRQ = c293_pcie_pic_init, -#ifdef CONFIG_PCI - .pcibios_fixup_bus = fsl_pcibios_fixup_bus, - .pcibios_fixup_phb = fsl_pcibios_fixup_phb, -#endif .get_irq = mpic_get_irq, .restart = fsl_rstcr_restart, .calibrate_decr = generic_calibrate_decr, diff --git a/arch/powerpc/platforms/85xx/corenet_generic.c b/arch/powerpc/platforms/85xx/corenet_generic.c index bd839dc287fe..b39557120cbb 100644 --- a/arch/powerpc/platforms/85xx/corenet_generic.c +++ b/arch/powerpc/platforms/85xx/corenet_generic.c @@ -153,6 +153,8 @@ static const char * const boards[] __initconst = { "fsl,T1023RDB", "fsl,T1024QDS", "fsl,T1024RDB", + "fsl,T1040D4RDB", + "fsl,T1042D4RDB", "fsl,T1040QDS", "fsl,T1042QDS", "fsl,T1040RDB", diff --git a/arch/powerpc/platforms/cell/Kconfig b/arch/powerpc/platforms/cell/Kconfig index 2f23133ab3d1..b0ac1773cea6 100644 --- a/arch/powerpc/platforms/cell/Kconfig +++ b/arch/powerpc/platforms/cell/Kconfig @@ -57,21 +57,6 @@ config SPU_FS Units on machines implementing the Broadband Processor Architecture. -config SPU_FS_64K_LS - bool "Use 64K pages to map SPE local store" - # we depend on PPC_MM_SLICES for now rather than selecting - # it because we depend on hugetlbfs hooks being present. We - # will fix that when the generic code has been improved to - # not require hijacking hugetlbfs hooks. - depends on SPU_FS && PPC_MM_SLICES && !PPC_64K_PAGES - default y - select PPC_HAS_HASH_64K - help - This option causes SPE local stores to be mapped in process - address spaces using 64K pages while the rest of the kernel - uses 4K pages. This can improve performances of applications - using multiple SPEs by lowering the TLB pressure on them. - config SPU_BASE bool default n diff --git a/arch/powerpc/platforms/cell/axon_msi.c b/arch/powerpc/platforms/cell/axon_msi.c index fe51de4fcf13..306888acb737 100644 --- a/arch/powerpc/platforms/cell/axon_msi.c +++ b/arch/powerpc/platforms/cell/axon_msi.c @@ -213,7 +213,7 @@ static int setup_msi_msg_address(struct pci_dev *dev, struct msi_msg *msg) return -ENODEV; } - entry = list_first_entry(&dev->msi_list, struct msi_desc, list); + entry = first_pci_msi_entry(dev); for (; dn; dn = of_get_next_parent(dn)) { if (entry->msi_attrib.is_64) { @@ -269,7 +269,7 @@ static int axon_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) if (rc) return rc; - list_for_each_entry(entry, &dev->msi_list, list) { + for_each_pci_msi_entry(entry, dev) { virq = irq_create_direct_mapping(msic->irq_domain); if (virq == NO_IRQ) { dev_warn(&dev->dev, @@ -292,7 +292,7 @@ static void axon_msi_teardown_msi_irqs(struct pci_dev *dev) dev_dbg(&dev->dev, "axon_msi: tearing down msi irqs\n"); - list_for_each_entry(entry, &dev->msi_list, list) { + for_each_pci_msi_entry(entry, dev) { if (entry->irq == NO_IRQ) continue; diff --git a/arch/powerpc/platforms/cell/interrupt.c b/arch/powerpc/platforms/cell/interrupt.c index 3af8324c122e..a15f1efc295f 100644 --- a/arch/powerpc/platforms/cell/interrupt.c +++ b/arch/powerpc/platforms/cell/interrupt.c @@ -222,7 +222,8 @@ void iic_request_IPIs(void) #endif /* CONFIG_SMP */ -static int iic_host_match(struct irq_domain *h, struct device_node *node) +static int iic_host_match(struct irq_domain *h, struct device_node *node, + enum irq_domain_bus_token bus_token) { return of_device_is_compatible(node, "IBM,CBEA-Internal-Interrupt-Controller"); diff --git a/arch/powerpc/platforms/cell/ras.c b/arch/powerpc/platforms/cell/ras.c index e865d748179b..2d4f60c0119a 100644 --- a/arch/powerpc/platforms/cell/ras.c +++ b/arch/powerpc/platforms/cell/ras.c @@ -123,7 +123,7 @@ static int __init cbe_ptcal_enable_on_node(int nid, int order) area->nid = nid; area->order = order; - area->pages = alloc_pages_exact_node(area->nid, + area->pages = __alloc_pages_node(area->nid, GFP_KERNEL|__GFP_THISNODE, area->order); diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c index d966bbe58b8f..5038fd578e65 100644 --- a/arch/powerpc/platforms/cell/spufs/file.c +++ b/arch/powerpc/platforms/cell/spufs/file.c @@ -239,23 +239,6 @@ spufs_mem_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) unsigned long address = (unsigned long)vmf->virtual_address; unsigned long pfn, offset; -#ifdef CONFIG_SPU_FS_64K_LS - struct spu_state *csa = &ctx->csa; - int psize; - - /* Check what page size we are using */ - psize = get_slice_psize(vma->vm_mm, address); - - /* Some sanity checking */ - BUG_ON(csa->use_big_pages != (psize == MMU_PAGE_64K)); - - /* Wow, 64K, cool, we need to align the address though */ - if (csa->use_big_pages) { - BUG_ON(vma->vm_start & 0xffff); - address &= ~0xfffful; - } -#endif /* CONFIG_SPU_FS_64K_LS */ - offset = vmf->pgoff << PAGE_SHIFT; if (offset >= LS_SIZE) return VM_FAULT_SIGBUS; @@ -310,22 +293,6 @@ static const struct vm_operations_struct spufs_mem_mmap_vmops = { static int spufs_mem_mmap(struct file *file, struct vm_area_struct *vma) { -#ifdef CONFIG_SPU_FS_64K_LS - struct spu_context *ctx = file->private_data; - struct spu_state *csa = &ctx->csa; - - /* Sanity check VMA alignment */ - if (csa->use_big_pages) { - pr_debug("spufs_mem_mmap 64K, start=0x%lx, end=0x%lx," - " pgoff=0x%lx\n", vma->vm_start, vma->vm_end, - vma->vm_pgoff); - if (vma->vm_start & 0xffff) - return -EINVAL; - if (vma->vm_pgoff & 0xf) - return -EINVAL; - } -#endif /* CONFIG_SPU_FS_64K_LS */ - if (!(vma->vm_flags & VM_SHARED)) return -EINVAL; @@ -336,25 +303,6 @@ static int spufs_mem_mmap(struct file *file, struct vm_area_struct *vma) return 0; } -#ifdef CONFIG_SPU_FS_64K_LS -static unsigned long spufs_get_unmapped_area(struct file *file, - unsigned long addr, unsigned long len, unsigned long pgoff, - unsigned long flags) -{ - struct spu_context *ctx = file->private_data; - struct spu_state *csa = &ctx->csa; - - /* If not using big pages, fallback to normal MM g_u_a */ - if (!csa->use_big_pages) - return current->mm->get_unmapped_area(file, addr, len, - pgoff, flags); - - /* Else, try to obtain a 64K pages slice */ - return slice_get_unmapped_area(addr, len, flags, - MMU_PAGE_64K, 1); -} -#endif /* CONFIG_SPU_FS_64K_LS */ - static const struct file_operations spufs_mem_fops = { .open = spufs_mem_open, .release = spufs_mem_release, @@ -362,9 +310,6 @@ static const struct file_operations spufs_mem_fops = { .write = spufs_mem_write, .llseek = generic_file_llseek, .mmap = spufs_mem_mmap, -#ifdef CONFIG_SPU_FS_64K_LS - .get_unmapped_area = spufs_get_unmapped_area, -#endif }; static int spufs_ps_fault(struct vm_area_struct *vma, diff --git a/arch/powerpc/platforms/cell/spufs/lscsa_alloc.c b/arch/powerpc/platforms/cell/spufs/lscsa_alloc.c index 147069938cfe..b847e9403566 100644 --- a/arch/powerpc/platforms/cell/spufs/lscsa_alloc.c +++ b/arch/powerpc/platforms/cell/spufs/lscsa_alloc.c @@ -31,7 +31,7 @@ #include "spufs.h" -static int spu_alloc_lscsa_std(struct spu_state *csa) +int spu_alloc_lscsa(struct spu_state *csa) { struct spu_lscsa *lscsa; unsigned char *p; @@ -48,7 +48,7 @@ static int spu_alloc_lscsa_std(struct spu_state *csa) return 0; } -static void spu_free_lscsa_std(struct spu_state *csa) +void spu_free_lscsa(struct spu_state *csa) { /* Clear reserved bit before vfree. */ unsigned char *p; @@ -61,123 +61,3 @@ static void spu_free_lscsa_std(struct spu_state *csa) vfree(csa->lscsa); } - -#ifdef CONFIG_SPU_FS_64K_LS - -#define SPU_64K_PAGE_SHIFT 16 -#define SPU_64K_PAGE_ORDER (SPU_64K_PAGE_SHIFT - PAGE_SHIFT) -#define SPU_64K_PAGE_COUNT (1ul << SPU_64K_PAGE_ORDER) - -int spu_alloc_lscsa(struct spu_state *csa) -{ - struct page **pgarray; - unsigned char *p; - int i, j, n_4k; - - /* Check availability of 64K pages */ - if (!spu_64k_pages_available()) - goto fail; - - csa->use_big_pages = 1; - - pr_debug("spu_alloc_lscsa(csa=0x%p), trying to allocate 64K pages\n", - csa); - - /* First try to allocate our 64K pages. We need 5 of them - * with the current implementation. In the future, we should try - * to separate the lscsa with the actual local store image, thus - * allowing us to require only 4 64K pages per context - */ - for (i = 0; i < SPU_LSCSA_NUM_BIG_PAGES; i++) { - /* XXX This is likely to fail, we should use a special pool - * similar to what hugetlbfs does. - */ - csa->lscsa_pages[i] = alloc_pages(GFP_KERNEL, - SPU_64K_PAGE_ORDER); - if (csa->lscsa_pages[i] == NULL) - goto fail; - } - - pr_debug(" success ! creating vmap...\n"); - - /* Now we need to create a vmalloc mapping of these for the kernel - * and SPU context switch code to use. Currently, we stick to a - * normal kernel vmalloc mapping, which in our case will be 4K - */ - n_4k = SPU_64K_PAGE_COUNT * SPU_LSCSA_NUM_BIG_PAGES; - pgarray = kmalloc(sizeof(struct page *) * n_4k, GFP_KERNEL); - if (pgarray == NULL) - goto fail; - for (i = 0; i < SPU_LSCSA_NUM_BIG_PAGES; i++) - for (j = 0; j < SPU_64K_PAGE_COUNT; j++) - /* We assume all the struct page's are contiguous - * which should be hopefully the case for an order 4 - * allocation.. - */ - pgarray[i * SPU_64K_PAGE_COUNT + j] = - csa->lscsa_pages[i] + j; - csa->lscsa = vmap(pgarray, n_4k, VM_USERMAP, PAGE_KERNEL); - kfree(pgarray); - if (csa->lscsa == NULL) - goto fail; - - memset(csa->lscsa, 0, sizeof(struct spu_lscsa)); - - /* Set LS pages reserved to allow for user-space mapping. - * - * XXX isn't that a bit obsolete ? I think we should just - * make sure the page count is high enough. Anyway, won't harm - * for now - */ - for (p = csa->lscsa->ls; p < csa->lscsa->ls + LS_SIZE; p += PAGE_SIZE) - SetPageReserved(vmalloc_to_page(p)); - - pr_debug(" all good !\n"); - - return 0; -fail: - pr_debug("spufs: failed to allocate lscsa 64K pages, falling back\n"); - spu_free_lscsa(csa); - return spu_alloc_lscsa_std(csa); -} - -void spu_free_lscsa(struct spu_state *csa) -{ - unsigned char *p; - int i; - - if (!csa->use_big_pages) { - spu_free_lscsa_std(csa); - return; - } - csa->use_big_pages = 0; - - if (csa->lscsa == NULL) - goto free_pages; - - for (p = csa->lscsa->ls; p < csa->lscsa->ls + LS_SIZE; p += PAGE_SIZE) - ClearPageReserved(vmalloc_to_page(p)); - - vunmap(csa->lscsa); - csa->lscsa = NULL; - - free_pages: - - for (i = 0; i < SPU_LSCSA_NUM_BIG_PAGES; i++) - if (csa->lscsa_pages[i]) - __free_pages(csa->lscsa_pages[i], SPU_64K_PAGE_ORDER); -} - -#else /* CONFIG_SPU_FS_64K_LS */ - -int spu_alloc_lscsa(struct spu_state *csa) -{ - return spu_alloc_lscsa_std(csa); -} - -void spu_free_lscsa(struct spu_state *csa) -{ - spu_free_lscsa_std(csa); -} - -#endif /* !defined(CONFIG_SPU_FS_64K_LS) */ diff --git a/arch/powerpc/platforms/embedded6xx/flipper-pic.c b/arch/powerpc/platforms/embedded6xx/flipper-pic.c index 4cde8e7da4b8..b7866e01483d 100644 --- a/arch/powerpc/platforms/embedded6xx/flipper-pic.c +++ b/arch/powerpc/platforms/embedded6xx/flipper-pic.c @@ -108,7 +108,8 @@ static int flipper_pic_map(struct irq_domain *h, unsigned int virq, return 0; } -static int flipper_pic_match(struct irq_domain *h, struct device_node *np) +static int flipper_pic_match(struct irq_domain *h, struct device_node *np, + enum irq_domain_bus_token bus_token) { return 1; } diff --git a/arch/powerpc/platforms/pasemi/msi.c b/arch/powerpc/platforms/pasemi/msi.c index 27f2b187a91b..e66ef1943338 100644 --- a/arch/powerpc/platforms/pasemi/msi.c +++ b/arch/powerpc/platforms/pasemi/msi.c @@ -66,7 +66,7 @@ static void pasemi_msi_teardown_msi_irqs(struct pci_dev *pdev) pr_debug("pasemi_msi_teardown_msi_irqs, pdev %p\n", pdev); - list_for_each_entry(entry, &pdev->msi_list, list) { + for_each_pci_msi_entry(entry, pdev) { if (entry->irq == NO_IRQ) continue; @@ -94,7 +94,7 @@ static int pasemi_msi_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) msg.address_hi = 0; msg.address_lo = PASEMI_MSI_ADDR; - list_for_each_entry(entry, &pdev->msi_list, list) { + for_each_pci_msi_entry(entry, pdev) { /* Allocate 16 interrupts for now, since that's the grouping for * affinity. This can be changed later if it turns out 32 is too * few MSIs for someone, but restrictions will apply to how the diff --git a/arch/powerpc/platforms/powermac/pic.c b/arch/powerpc/platforms/powermac/pic.c index 59cfc9d63c2d..6f4f8b060def 100644 --- a/arch/powerpc/platforms/powermac/pic.c +++ b/arch/powerpc/platforms/powermac/pic.c @@ -268,7 +268,8 @@ static struct irqaction gatwick_cascade_action = { .name = "cascade", }; -static int pmac_pic_host_match(struct irq_domain *h, struct device_node *node) +static int pmac_pic_host_match(struct irq_domain *h, struct device_node *node, + enum irq_domain_bus_token bus_token) { /* We match all, we don't always have a node anyway */ return 1; diff --git a/arch/powerpc/platforms/powernv/eeh-powernv.c b/arch/powerpc/platforms/powernv/eeh-powernv.c index 7cf0df859d05..3bb6acb76339 100644 --- a/arch/powerpc/platforms/powernv/eeh-powernv.c +++ b/arch/powerpc/platforms/powernv/eeh-powernv.c @@ -1394,11 +1394,19 @@ static int pnv_eeh_next_error(struct eeh_pe **pe) */ if (pnv_eeh_get_pe(hose, be64_to_cpu(frozen_pe_no), pe)) { - /* Try best to clear it */ pr_info("EEH: Clear non-existing PHB#%x-PE#%llx\n", - hose->global_number, frozen_pe_no); + hose->global_number, be64_to_cpu(frozen_pe_no)); pr_info("EEH: PHB location: %s\n", eeh_pe_loc_get(phb_pe)); + + /* Dump PHB diag-data */ + rc = opal_pci_get_phb_diag_data2(phb->opal_id, + phb->diag.blob, PNV_PCI_DIAG_BUF_SIZE); + if (rc == OPAL_SUCCESS) + pnv_pci_dump_phb_diag_data(hose, + phb->diag.blob); + + /* Try best to clear it */ opal_pci_eeh_freeze_clear(phb->opal_id, frozen_pe_no, OPAL_EEH_ACTION_CLEAR_FREEZE_ALL); diff --git a/arch/powerpc/platforms/powernv/opal-hmi.c b/arch/powerpc/platforms/powernv/opal-hmi.c index a8f49d380449..d000f4e21981 100644 --- a/arch/powerpc/platforms/powernv/opal-hmi.c +++ b/arch/powerpc/platforms/powernv/opal-hmi.c @@ -35,9 +35,134 @@ struct OpalHmiEvtNode { struct list_head list; struct OpalHMIEvent hmi_evt; }; + +struct xstop_reason { + uint32_t xstop_reason; + const char *unit_failed; + const char *description; +}; + static LIST_HEAD(opal_hmi_evt_list); static DEFINE_SPINLOCK(opal_hmi_evt_lock); +static void print_core_checkstop_reason(const char *level, + struct OpalHMIEvent *hmi_evt) +{ + int i; + static const struct xstop_reason xstop_reason[] = { + { CORE_CHECKSTOP_IFU_REGFILE, "IFU", + "RegFile core check stop" }, + { CORE_CHECKSTOP_IFU_LOGIC, "IFU", "Logic core check stop" }, + { CORE_CHECKSTOP_PC_DURING_RECOV, "PC", + "Core checkstop during recovery" }, + { CORE_CHECKSTOP_ISU_REGFILE, "ISU", + "RegFile core check stop (mapper error)" }, + { CORE_CHECKSTOP_ISU_LOGIC, "ISU", "Logic core check stop" }, + { CORE_CHECKSTOP_FXU_LOGIC, "FXU", "Logic core check stop" }, + { CORE_CHECKSTOP_VSU_LOGIC, "VSU", "Logic core check stop" }, + { CORE_CHECKSTOP_PC_RECOV_IN_MAINT_MODE, "PC", + "Recovery in maintenance mode" }, + { CORE_CHECKSTOP_LSU_REGFILE, "LSU", + "RegFile core check stop" }, + { CORE_CHECKSTOP_PC_FWD_PROGRESS, "PC", + "Forward Progress Error" }, + { CORE_CHECKSTOP_LSU_LOGIC, "LSU", "Logic core check stop" }, + { CORE_CHECKSTOP_PC_LOGIC, "PC", "Logic core check stop" }, + { CORE_CHECKSTOP_PC_HYP_RESOURCE, "PC", + "Hypervisor Resource error - core check stop" }, + { CORE_CHECKSTOP_PC_HANG_RECOV_FAILED, "PC", + "Hang Recovery Failed (core check stop)" }, + { CORE_CHECKSTOP_PC_AMBI_HANG_DETECTED, "PC", + "Ambiguous Hang Detected (unknown source)" }, + { CORE_CHECKSTOP_PC_DEBUG_TRIG_ERR_INJ, "PC", + "Debug Trigger Error inject" }, + { CORE_CHECKSTOP_PC_SPRD_HYP_ERR_INJ, "PC", + "Hypervisor check stop via SPRC/SPRD" }, + }; + + /* Validity check */ + if (!hmi_evt->u.xstop_error.xstop_reason) { + printk("%s Unknown Core check stop.\n", level); + return; + } + + printk("%s CPU PIR: %08x\n", level, + be32_to_cpu(hmi_evt->u.xstop_error.u.pir)); + for (i = 0; i < ARRAY_SIZE(xstop_reason); i++) + if (be32_to_cpu(hmi_evt->u.xstop_error.xstop_reason) & + xstop_reason[i].xstop_reason) + printk("%s [Unit: %-3s] %s\n", level, + xstop_reason[i].unit_failed, + xstop_reason[i].description); +} + +static void print_nx_checkstop_reason(const char *level, + struct OpalHMIEvent *hmi_evt) +{ + int i; + static const struct xstop_reason xstop_reason[] = { + { NX_CHECKSTOP_SHM_INVAL_STATE_ERR, "DMA & Engine", + "SHM invalid state error" }, + { NX_CHECKSTOP_DMA_INVAL_STATE_ERR_1, "DMA & Engine", + "DMA invalid state error bit 15" }, + { NX_CHECKSTOP_DMA_INVAL_STATE_ERR_2, "DMA & Engine", + "DMA invalid state error bit 16" }, + { NX_CHECKSTOP_DMA_CH0_INVAL_STATE_ERR, "DMA & Engine", + "Channel 0 invalid state error" }, + { NX_CHECKSTOP_DMA_CH1_INVAL_STATE_ERR, "DMA & Engine", + "Channel 1 invalid state error" }, + { NX_CHECKSTOP_DMA_CH2_INVAL_STATE_ERR, "DMA & Engine", + "Channel 2 invalid state error" }, + { NX_CHECKSTOP_DMA_CH3_INVAL_STATE_ERR, "DMA & Engine", + "Channel 3 invalid state error" }, + { NX_CHECKSTOP_DMA_CH4_INVAL_STATE_ERR, "DMA & Engine", + "Channel 4 invalid state error" }, + { NX_CHECKSTOP_DMA_CH5_INVAL_STATE_ERR, "DMA & Engine", + "Channel 5 invalid state error" }, + { NX_CHECKSTOP_DMA_CH6_INVAL_STATE_ERR, "DMA & Engine", + "Channel 6 invalid state error" }, + { NX_CHECKSTOP_DMA_CH7_INVAL_STATE_ERR, "DMA & Engine", + "Channel 7 invalid state error" }, + { NX_CHECKSTOP_DMA_CRB_UE, "DMA & Engine", + "UE error on CRB(CSB address, CCB)" }, + { NX_CHECKSTOP_DMA_CRB_SUE, "DMA & Engine", + "SUE error on CRB(CSB address, CCB)" }, + { NX_CHECKSTOP_PBI_ISN_UE, "PowerBus Interface", + "CRB Kill ISN received while holding ISN with UE error" }, + }; + + /* Validity check */ + if (!hmi_evt->u.xstop_error.xstop_reason) { + printk("%s Unknown NX check stop.\n", level); + return; + } + + printk("%s NX checkstop on CHIP ID: %x\n", level, + be32_to_cpu(hmi_evt->u.xstop_error.u.chip_id)); + for (i = 0; i < ARRAY_SIZE(xstop_reason); i++) + if (be32_to_cpu(hmi_evt->u.xstop_error.xstop_reason) & + xstop_reason[i].xstop_reason) + printk("%s [Unit: %-3s] %s\n", level, + xstop_reason[i].unit_failed, + xstop_reason[i].description); +} + +static void print_checkstop_reason(const char *level, + struct OpalHMIEvent *hmi_evt) +{ + switch (hmi_evt->u.xstop_error.xstop_type) { + case CHECKSTOP_TYPE_CORE: + print_core_checkstop_reason(level, hmi_evt); + break; + case CHECKSTOP_TYPE_NX: + print_nx_checkstop_reason(level, hmi_evt); + break; + case CHECKSTOP_TYPE_UNKNOWN: + printk("%s Unknown Malfunction Alert.\n", level); + break; + } +} + static void print_hmi_event_info(struct OpalHMIEvent *hmi_evt) { const char *level, *sevstr, *error_info; @@ -95,6 +220,13 @@ static void print_hmi_event_info(struct OpalHMIEvent *hmi_evt) (hmi_evt->type == OpalHMI_ERROR_TFMR_PARITY)) printk("%s TFMR: %016llx\n", level, be64_to_cpu(hmi_evt->tfmr)); + + if (hmi_evt->version < OpalHMIEvt_V2) + return; + + /* OpalHMIEvt_V2 and above provides reason for malfunction alert. */ + if (hmi_evt->type == OpalHMI_ERROR_MALFUNC_ALERT) + print_checkstop_reason(level, hmi_evt); } static void hmi_event_handler(struct work_struct *work) @@ -103,6 +235,8 @@ static void hmi_event_handler(struct work_struct *work) struct OpalHMIEvent *hmi_evt; struct OpalHmiEvtNode *msg_node; uint8_t disposition; + struct opal_msg msg; + int unrecoverable = 0; spin_lock_irqsave(&opal_hmi_evt_lock, flags); while (!list_empty(&opal_hmi_evt_list)) { @@ -118,14 +252,53 @@ static void hmi_event_handler(struct work_struct *work) /* * Check if HMI event has been recovered or not. If not - * then we can't continue, invoke panic. + * then kernel can't continue, we need to panic. + * But before we do that, display all the HMI event + * available on the list and set unrecoverable flag to 1. */ if (disposition != OpalHMI_DISPOSITION_RECOVERED) - panic("Unrecoverable HMI exception"); + unrecoverable = 1; spin_lock_irqsave(&opal_hmi_evt_lock, flags); } spin_unlock_irqrestore(&opal_hmi_evt_lock, flags); + + if (unrecoverable) { + int ret; + + /* Pull all HMI events from OPAL before we panic. */ + while (opal_get_msg(__pa(&msg), sizeof(msg)) == OPAL_SUCCESS) { + u32 type; + + type = be32_to_cpu(msg.msg_type); + + /* skip if not HMI event */ + if (type != OPAL_MSG_HMI_EVT) + continue; + + /* HMI event info starts from param[0] */ + hmi_evt = (struct OpalHMIEvent *)&msg.params[0]; + print_hmi_event_info(hmi_evt); + } + + /* + * Unrecoverable HMI exception. We need to inform BMC/OCC + * about this error so that it can collect relevant data + * for error analysis before rebooting. + */ + ret = opal_cec_reboot2(OPAL_REBOOT_PLATFORM_ERROR, + "Unrecoverable HMI exception"); + if (ret == OPAL_UNSUPPORTED) { + pr_emerg("Reboot type %d not supported\n", + OPAL_REBOOT_PLATFORM_ERROR); + } + + /* + * Fall through and panic if opal_cec_reboot2() returns + * OPAL_UNSUPPORTED. + */ + panic("Unrecoverable HMI exception"); + } } static DECLARE_WORK(hmi_event_work, hmi_event_handler); diff --git a/arch/powerpc/platforms/powernv/opal-irqchip.c b/arch/powerpc/platforms/powernv/opal-irqchip.c index e2e7d75f52f3..2c91ee7800b9 100644 --- a/arch/powerpc/platforms/powernv/opal-irqchip.c +++ b/arch/powerpc/platforms/powernv/opal-irqchip.c @@ -134,7 +134,8 @@ static void opal_handle_irq_work(struct irq_work *work) opal_handle_events(be64_to_cpu(last_outstanding_events)); } -static int opal_event_match(struct irq_domain *h, struct device_node *node) +static int opal_event_match(struct irq_domain *h, struct device_node *node, + enum irq_domain_bus_token bus_token) { return h->of_node == node; } diff --git a/arch/powerpc/platforms/powernv/opal-power.c b/arch/powerpc/platforms/powernv/opal-power.c index ac46c2c24f99..58dc3308237f 100644 --- a/arch/powerpc/platforms/powernv/opal-power.c +++ b/arch/powerpc/platforms/powernv/opal-power.c @@ -9,9 +9,12 @@ * 2 of the License, or (at your option) any later version. */ +#define pr_fmt(fmt) "opal-power: " fmt + #include #include #include +#include #include #include @@ -19,30 +22,116 @@ #define SOFT_OFF 0x00 #define SOFT_REBOOT 0x01 -static int opal_power_control_event(struct notifier_block *nb, - unsigned long msg_type, void *msg) +/* Detect EPOW event */ +static bool detect_epow(void) +{ + u16 epow; + int i, rc; + __be16 epow_classes; + __be16 opal_epow_status[OPAL_SYSEPOW_MAX] = {0}; + + /* + * Check for EPOW event. Kernel sends supported EPOW classes info + * to OPAL. OPAL returns EPOW info along with classes present. + */ + epow_classes = cpu_to_be16(OPAL_SYSEPOW_MAX); + rc = opal_get_epow_status(opal_epow_status, &epow_classes); + if (rc != OPAL_SUCCESS) { + pr_err("Failed to get EPOW event information\n"); + return false; + } + + /* Look for EPOW events present */ + for (i = 0; i < be16_to_cpu(epow_classes); i++) { + epow = be16_to_cpu(opal_epow_status[i]); + + /* Filter events which do not need shutdown. */ + if (i == OPAL_SYSEPOW_POWER) + epow &= ~(OPAL_SYSPOWER_CHNG | OPAL_SYSPOWER_FAIL | + OPAL_SYSPOWER_INCL); + if (epow) + return true; + } + + return false; +} + +/* Check for existing EPOW, DPO events */ +static bool poweroff_pending(void) +{ + int rc; + __be64 opal_dpo_timeout; + + /* Check for DPO event */ + rc = opal_get_dpo_status(&opal_dpo_timeout); + if (rc == OPAL_SUCCESS) { + pr_info("Existing DPO event detected.\n"); + return true; + } + + /* Check for EPOW event */ + if (detect_epow()) { + pr_info("Existing EPOW event detected.\n"); + return true; + } + + return false; +} + +/* OPAL power-control events notifier */ +static int opal_power_control_event(struct notifier_block *nb, + unsigned long msg_type, void *msg) { - struct opal_msg *power_msg = msg; uint64_t type; - type = be64_to_cpu(power_msg->params[0]); - - switch (type) { - case SOFT_REBOOT: - pr_info("OPAL: reboot requested\n"); - orderly_reboot(); + switch (msg_type) { + case OPAL_MSG_EPOW: + if (detect_epow()) { + pr_info("EPOW msg received. Powering off system\n"); + orderly_poweroff(true); + } break; - case SOFT_OFF: - pr_info("OPAL: poweroff requested\n"); + case OPAL_MSG_DPO: + pr_info("DPO msg received. Powering off system\n"); orderly_poweroff(true); break; + case OPAL_MSG_SHUTDOWN: + type = be64_to_cpu(((struct opal_msg *)msg)->params[0]); + switch (type) { + case SOFT_REBOOT: + pr_info("Reboot requested\n"); + orderly_reboot(); + break; + case SOFT_OFF: + pr_info("Poweroff requested\n"); + orderly_poweroff(true); + break; + default: + pr_err("Unknown power-control type %llu\n", type); + } + break; default: - pr_err("OPAL: power control type unexpected %016llx\n", type); + pr_err("Unknown OPAL message type %lu\n", msg_type); } return 0; } +/* OPAL EPOW event notifier block */ +static struct notifier_block opal_epow_nb = { + .notifier_call = opal_power_control_event, + .next = NULL, + .priority = 0, +}; + +/* OPAL DPO event notifier block */ +static struct notifier_block opal_dpo_nb = { + .notifier_call = opal_power_control_event, + .next = NULL, + .priority = 0, +}; + +/* OPAL power-control event notifier block */ static struct notifier_block opal_power_control_nb = { .notifier_call = opal_power_control_event, .next = NULL, @@ -51,16 +140,40 @@ static struct notifier_block opal_power_control_nb = { static int __init opal_power_control_init(void) { - int ret; + int ret, supported = 0; + struct device_node *np; + /* Register OPAL power-control events notifier */ ret = opal_message_notifier_register(OPAL_MSG_SHUTDOWN, - &opal_power_control_nb); - if (ret) { - pr_err("%s: Can't register OPAL event notifier (%d)\n", - __func__, ret); - return ret; + &opal_power_control_nb); + if (ret) + pr_err("Failed to register SHUTDOWN notifier, ret = %d\n", ret); + + /* Determine OPAL EPOW, DPO support */ + np = of_find_node_by_path("/ibm,opal/epow"); + if (np) { + supported = of_device_is_compatible(np, "ibm,opal-v3-epow"); + of_node_put(np); } + if (!supported) + return 0; + pr_info("OPAL EPOW, DPO support detected.\n"); + + /* Register EPOW event notifier */ + ret = opal_message_notifier_register(OPAL_MSG_EPOW, &opal_epow_nb); + if (ret) + pr_err("Failed to register EPOW notifier, ret = %d\n", ret); + + /* Register DPO event notifier */ + ret = opal_message_notifier_register(OPAL_MSG_DPO, &opal_dpo_nb); + if (ret) + pr_err("Failed to register DPO notifier, ret = %d\n", ret); + + /* Check for any pending EPOW or DPO events. */ + if (poweroff_pending()) + orderly_poweroff(true); + return 0; } machine_subsys_initcall(powernv, opal_power_control_init); diff --git a/arch/powerpc/platforms/powernv/opal-wrappers.S b/arch/powerpc/platforms/powernv/opal-wrappers.S index d6a7b8252e4d..b7a464fef7a7 100644 --- a/arch/powerpc/platforms/powernv/opal-wrappers.S +++ b/arch/powerpc/platforms/powernv/opal-wrappers.S @@ -202,6 +202,7 @@ OPAL_CALL(opal_rtc_read, OPAL_RTC_READ); OPAL_CALL(opal_rtc_write, OPAL_RTC_WRITE); OPAL_CALL(opal_cec_power_down, OPAL_CEC_POWER_DOWN); OPAL_CALL(opal_cec_reboot, OPAL_CEC_REBOOT); +OPAL_CALL(opal_cec_reboot2, OPAL_CEC_REBOOT2); OPAL_CALL(opal_read_nvram, OPAL_READ_NVRAM); OPAL_CALL(opal_write_nvram, OPAL_WRITE_NVRAM); OPAL_CALL(opal_handle_interrupt, OPAL_HANDLE_INTERRUPT); @@ -249,6 +250,7 @@ OPAL_CALL(opal_pci_reinit, OPAL_PCI_REINIT); OPAL_CALL(opal_pci_mask_pe_error, OPAL_PCI_MASK_PE_ERROR); OPAL_CALL(opal_set_slot_led_status, OPAL_SET_SLOT_LED_STATUS); OPAL_CALL(opal_get_epow_status, OPAL_GET_EPOW_STATUS); +OPAL_CALL(opal_get_dpo_status, OPAL_GET_DPO_STATUS); OPAL_CALL(opal_set_system_attention_led, OPAL_SET_SYSTEM_ATTENTION_LED); OPAL_CALL(opal_pci_next_error, OPAL_PCI_NEXT_ERROR); OPAL_CALL(opal_pci_poll, OPAL_PCI_POLL); @@ -297,3 +299,5 @@ OPAL_CALL(opal_flash_read, OPAL_FLASH_READ); OPAL_CALL(opal_flash_write, OPAL_FLASH_WRITE); OPAL_CALL(opal_flash_erase, OPAL_FLASH_ERASE); OPAL_CALL(opal_prd_msg, OPAL_PRD_MSG); +OPAL_CALL(opal_leds_get_ind, OPAL_LEDS_GET_INDICATOR); +OPAL_CALL(opal_leds_set_ind, OPAL_LEDS_SET_INDICATOR); diff --git a/arch/powerpc/platforms/powernv/opal.c b/arch/powerpc/platforms/powernv/opal.c index f084afa0e3ba..230f3a7cdea4 100644 --- a/arch/powerpc/platforms/powernv/opal.c +++ b/arch/powerpc/platforms/powernv/opal.c @@ -441,6 +441,7 @@ static int opal_recover_mce(struct pt_regs *regs, int opal_machine_check(struct pt_regs *regs) { struct machine_check_event evt; + int ret; if (!get_mce_event(&evt, MCE_EVENT_RELEASE)) return 0; @@ -455,6 +456,40 @@ int opal_machine_check(struct pt_regs *regs) if (opal_recover_mce(regs, &evt)) return 1; + + /* + * Unrecovered machine check, we are heading to panic path. + * + * We may have hit this MCE in very early stage of kernel + * initialization even before opal-prd has started running. If + * this is the case then this MCE error may go un-noticed or + * un-analyzed if we go down panic path. We need to inform + * BMC/OCC about this error so that they can collect relevant + * data for error analysis before rebooting. + * Use opal_cec_reboot2(OPAL_REBOOT_PLATFORM_ERROR) to do so. + * This function may not return on BMC based system. + */ + ret = opal_cec_reboot2(OPAL_REBOOT_PLATFORM_ERROR, + "Unrecoverable Machine Check exception"); + if (ret == OPAL_UNSUPPORTED) { + pr_emerg("Reboot type %d not supported\n", + OPAL_REBOOT_PLATFORM_ERROR); + } + + /* + * We reached here. There can be three possibilities: + * 1. We are running on a firmware level that do not support + * opal_cec_reboot2() + * 2. We are running on a firmware level that do not support + * OPAL_REBOOT_PLATFORM_ERROR reboot type. + * 3. We are running on FSP based system that does not need opal + * to trigger checkstop explicitly for error analysis. The FSP + * PRD component would have already got notified about this + * error through other channels. + * + * In any case, let us just fall through. We anyway heading + * down to panic path. + */ return 0; } @@ -648,7 +683,7 @@ static void opal_init_heartbeat(void) static int __init opal_init(void) { - struct device_node *np, *consoles; + struct device_node *np, *consoles, *leds; int rc; opal_node = of_find_node_by_path("/ibm,opal"); @@ -689,6 +724,13 @@ static int __init opal_init(void) /* Setup a heatbeat thread if requested by OPAL */ opal_init_heartbeat(); + /* Create leds platform devices */ + leds = of_find_node_by_path("/ibm,opal/leds"); + if (leds) { + of_platform_device_create(leds, "opal_leds", NULL); + of_node_put(leds); + } + /* Create "opal" kobject under /sys/firmware */ rc = opal_sysfs_init(); if (rc == 0) { @@ -841,3 +883,6 @@ EXPORT_SYMBOL_GPL(opal_rtc_write); EXPORT_SYMBOL_GPL(opal_tpo_read); EXPORT_SYMBOL_GPL(opal_tpo_write); EXPORT_SYMBOL_GPL(opal_i2c_request); +/* Export these symbols for PowerNV LED class driver */ +EXPORT_SYMBOL_GPL(opal_leds_get_ind); +EXPORT_SYMBOL_GPL(opal_leds_set_ind); diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index 85cbc96eff6c..2927cd5c8303 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -140,11 +140,9 @@ static void pnv_ioda_reserve_pe(struct pnv_phb *phb, int pe_no) return; } - if (test_and_set_bit(pe_no, phb->ioda.pe_alloc)) { - pr_warn("%s: PE %d was assigned on PHB#%x\n", - __func__, pe_no, phb->hose->global_number); - return; - } + if (test_and_set_bit(pe_no, phb->ioda.pe_alloc)) + pr_debug("%s: PE %d was reserved on PHB#%x\n", + __func__, pe_no, phb->hose->global_number); phb->ioda.pe_array[pe_no].phb = phb; phb->ioda.pe_array[pe_no].pe_number = pe_no; @@ -231,61 +229,60 @@ static int pnv_ioda2_init_m64(struct pnv_phb *phb) return -EIO; } -static void pnv_ioda2_reserve_m64_pe(struct pnv_phb *phb) +static void pnv_ioda2_reserve_dev_m64_pe(struct pci_dev *pdev, + unsigned long *pe_bitmap) { - resource_size_t sgsz = phb->ioda.m64_segsize; - struct pci_dev *pdev; + struct pci_controller *hose = pci_bus_to_host(pdev->bus); + struct pnv_phb *phb = hose->private_data; struct resource *r; - int base, step, i; + resource_size_t base, sgsz, start, end; + int segno, i; - /* - * Root bus always has full M64 range and root port has - * M64 range used in reality. So we're checking root port - * instead of root bus. - */ - list_for_each_entry(pdev, &phb->hose->bus->devices, bus_list) { - for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) { - r = &pdev->resource[PCI_BRIDGE_RESOURCES + i]; - if (!r->parent || - !pnv_pci_is_mem_pref_64(r->flags)) - continue; + base = phb->ioda.m64_base; + sgsz = phb->ioda.m64_segsize; + for (i = 0; i <= PCI_ROM_RESOURCE; i++) { + r = &pdev->resource[i]; + if (!r->parent || !pnv_pci_is_mem_pref_64(r->flags)) + continue; - base = (r->start - phb->ioda.m64_base) / sgsz; - for (step = 0; step < resource_size(r) / sgsz; step++) - pnv_ioda_reserve_pe(phb, base + step); + start = _ALIGN_DOWN(r->start - base, sgsz); + end = _ALIGN_UP(r->end - base, sgsz); + for (segno = start / sgsz; segno < end / sgsz; segno++) { + if (pe_bitmap) + set_bit(segno, pe_bitmap); + else + pnv_ioda_reserve_pe(phb, segno); } } } -static int pnv_ioda2_pick_m64_pe(struct pnv_phb *phb, - struct pci_bus *bus, int all) +static void pnv_ioda2_reserve_m64_pe(struct pci_bus *bus, + unsigned long *pe_bitmap, + bool all) { - resource_size_t segsz = phb->ioda.m64_segsize; struct pci_dev *pdev; - struct resource *r; + + list_for_each_entry(pdev, &bus->devices, bus_list) { + pnv_ioda2_reserve_dev_m64_pe(pdev, pe_bitmap); + + if (all && pdev->subordinate) + pnv_ioda2_reserve_m64_pe(pdev->subordinate, + pe_bitmap, all); + } +} + +static int pnv_ioda2_pick_m64_pe(struct pci_bus *bus, bool all) +{ + struct pci_controller *hose = pci_bus_to_host(bus); + struct pnv_phb *phb = hose->private_data; struct pnv_ioda_pe *master_pe, *pe; unsigned long size, *pe_alloc; - bool found; - int start, i, j; + int i; /* Root bus shouldn't use M64 */ if (pci_is_root_bus(bus)) return IODA_INVALID_PE; - /* We support only one M64 window on each bus */ - found = false; - pci_bus_for_each_resource(bus, r, i) { - if (r && r->parent && - pnv_pci_is_mem_pref_64(r->flags)) { - found = true; - break; - } - } - - /* No M64 window found ? */ - if (!found) - return IODA_INVALID_PE; - /* Allocate bitmap */ size = _ALIGN_UP(phb->ioda.total_pe / 8, sizeof(unsigned long)); pe_alloc = kzalloc(size, GFP_KERNEL); @@ -295,35 +292,8 @@ static int pnv_ioda2_pick_m64_pe(struct pnv_phb *phb, return IODA_INVALID_PE; } - /* - * Figure out reserved PE numbers by the PE - * the its child PEs. - */ - start = (r->start - phb->ioda.m64_base) / segsz; - for (i = 0; i < resource_size(r) / segsz; i++) - set_bit(start + i, pe_alloc); - - if (all) - goto done; - - /* - * If the PE doesn't cover all subordinate buses, - * we need subtract from reserved PEs for children. - */ - list_for_each_entry(pdev, &bus->devices, bus_list) { - if (!pdev->subordinate) - continue; - - pci_bus_for_each_resource(pdev->subordinate, r, i) { - if (!r || !r->parent || - !pnv_pci_is_mem_pref_64(r->flags)) - continue; - - start = (r->start - phb->ioda.m64_base) / segsz; - for (j = 0; j < resource_size(r) / segsz ; j++) - clear_bit(start + j, pe_alloc); - } - } + /* Figure out reserved PE numbers by the PE */ + pnv_ioda2_reserve_m64_pe(bus, pe_alloc, all); /* * the current bus might not own M64 window and that's all @@ -339,7 +309,6 @@ static int pnv_ioda2_pick_m64_pe(struct pnv_phb *phb, * Figure out the master PE and put all slave PEs to master * PE's list to form compound PE. */ -done: master_pe = NULL; i = -1; while ((i = find_next_bit(pe_alloc, phb->ioda.total_pe, i + 1)) < @@ -653,7 +622,7 @@ static int pnv_ioda_set_peltv(struct pnv_phb *phb, pdev = pe->pdev->bus->self; #ifdef CONFIG_PCI_IOV else if (pe->flags & PNV_IODA_PE_VF) - pdev = pe->parent_dev->bus->self; + pdev = pe->parent_dev; #endif /* CONFIG_PCI_IOV */ while (pdev) { struct pci_dn *pdn = pci_get_pdn(pdev); @@ -732,7 +701,7 @@ static int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe) parent = parent->bus->self; } - opal_pci_eeh_freeze_set(phb->opal_id, pe->pe_number, + opal_pci_eeh_freeze_clear(phb->opal_id, pe->pe_number, OPAL_EEH_ACTION_CLEAR_FREEZE_ALL); /* Disassociate PE in PELT */ @@ -946,8 +915,9 @@ static int pnv_pci_vf_resource_shift(struct pci_dev *dev, int offset) res2 = *res; res->start += size * offset; - dev_info(&dev->dev, "VF BAR%d: %pR shifted to %pR (enabling %d VFs shifted by %d)\n", - i, &res2, res, num_vfs, offset); + dev_info(&dev->dev, "VF BAR%d: %pR shifted to %pR (%sabling %d VFs shifted by %d)\n", + i, &res2, res, (offset > 0) ? "En" : "Dis", + num_vfs, offset); pci_update_resource(dev, i + PCI_IOV_RESOURCES); } return 0; @@ -1050,7 +1020,7 @@ static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe) * subordinate PCI devices and buses. The second type of PE is normally * orgiriated by PCIe-to-PCI bridge or PLX switch downstream ports. */ -static void pnv_ioda_setup_bus_PE(struct pci_bus *bus, int all) +static void pnv_ioda_setup_bus_PE(struct pci_bus *bus, bool all) { struct pci_controller *hose = pci_bus_to_host(bus); struct pnv_phb *phb = hose->private_data; @@ -1059,7 +1029,7 @@ static void pnv_ioda_setup_bus_PE(struct pci_bus *bus, int all) /* Check if PE is determined by M64 */ if (phb->pick_m64_pe) - pe_num = phb->pick_m64_pe(phb, bus, all); + pe_num = phb->pick_m64_pe(bus, all); /* The PE number isn't pinned by M64 */ if (pe_num == IODA_INVALID_PE) @@ -1117,12 +1087,12 @@ static void pnv_ioda_setup_PEs(struct pci_bus *bus) { struct pci_dev *dev; - pnv_ioda_setup_bus_PE(bus, 0); + pnv_ioda_setup_bus_PE(bus, false); list_for_each_entry(dev, &bus->devices, bus_list) { if (dev->subordinate) { if (pci_pcie_type(dev) == PCI_EXP_TYPE_PCI_BRIDGE) - pnv_ioda_setup_bus_PE(dev->subordinate, 1); + pnv_ioda_setup_bus_PE(dev->subordinate, true); else pnv_ioda_setup_PEs(dev->subordinate); } @@ -1147,7 +1117,7 @@ static void pnv_pci_ioda_setup_PEs(void) /* M64 layout might affect PE allocation */ if (phb->reserve_m64_pe) - phb->reserve_m64_pe(phb); + phb->reserve_m64_pe(hose->bus, NULL, true); pnv_ioda_setup_PEs(hose->bus); } @@ -1590,6 +1560,7 @@ static void pnv_pci_ioda_dma_dev_setup(struct pnv_phb *phb, struct pci_dev *pdev pe = &phb->ioda.pe_array[pdn->pe_number]; WARN_ON(get_dma_ops(&pdev->dev) != &dma_iommu_ops); + set_dma_offset(&pdev->dev, pe->tce_bypass_base); set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]); /* * Note: iommu_add_device() will fail here as @@ -1620,19 +1591,18 @@ static int pnv_pci_ioda_dma_set_mask(struct pci_dev *pdev, u64 dma_mask) if (bypass) { dev_info(&pdev->dev, "Using 64-bit DMA iommu bypass\n"); set_dma_ops(&pdev->dev, &dma_direct_ops); - set_dma_offset(&pdev->dev, pe->tce_bypass_base); } else { dev_info(&pdev->dev, "Using 32-bit DMA via iommu\n"); set_dma_ops(&pdev->dev, &dma_iommu_ops); - set_iommu_table_base(&pdev->dev, pe->table_group.tables[0]); } *pdev->dev.dma_mask = dma_mask; return 0; } -static u64 pnv_pci_ioda_dma_get_required_mask(struct pnv_phb *phb, - struct pci_dev *pdev) +static u64 pnv_pci_ioda_dma_get_required_mask(struct pci_dev *pdev) { + struct pci_controller *hose = pci_bus_to_host(pdev->bus); + struct pnv_phb *phb = hose->private_data; struct pci_dn *pdn = pci_get_pdn(pdev); struct pnv_ioda_pe *pe; u64 end, mask; @@ -1659,6 +1629,7 @@ static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe, list_for_each_entry(dev, &bus->devices, bus_list) { set_iommu_table_base(&dev->dev, pe->table_group.tables[0]); + set_dma_offset(&dev->dev, pe->tce_bypass_base); iommu_add_device(&dev->dev); if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate) @@ -3057,6 +3028,7 @@ static const struct pci_controller_ops pnv_pci_ioda_controller_ops = { .window_alignment = pnv_pci_window_alignment, .reset_secondary_bus = pnv_pci_reset_secondary_bus, .dma_set_mask = pnv_pci_ioda_dma_set_mask, + .dma_get_required_mask = pnv_pci_ioda_dma_get_required_mask, .shutdown = pnv_pci_ioda_shutdown, }; @@ -3203,7 +3175,6 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, /* Setup TCEs */ phb->dma_dev_setup = pnv_pci_ioda_dma_dev_setup; - phb->dma_get_required_mask = pnv_pci_ioda_dma_get_required_mask; /* Setup MSI support */ pnv_pci_init_ioda_msis(phb); diff --git a/arch/powerpc/platforms/powernv/pci.c b/arch/powerpc/platforms/powernv/pci.c index 765d8ed558d0..9b2480b265c0 100644 --- a/arch/powerpc/platforms/powernv/pci.c +++ b/arch/powerpc/platforms/powernv/pci.c @@ -61,7 +61,7 @@ int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) if (pdev->no_64bit_msi && !phb->msi32_support) return -ENODEV; - list_for_each_entry(entry, &pdev->msi_list, list) { + for_each_pci_msi_entry(entry, pdev) { if (!entry->msi_attrib.is_64 && !phb->msi32_support) { pr_warn("%s: Supports only 64-bit MSIs\n", pci_name(pdev)); @@ -103,7 +103,7 @@ void pnv_teardown_msi_irqs(struct pci_dev *pdev) if (WARN_ON(!phb)) return; - list_for_each_entry(entry, &pdev->msi_list, list) { + for_each_pci_msi_entry(entry, pdev) { if (entry->irq == NO_IRQ) continue; irq_set_msi_desc(entry->irq, NULL); @@ -761,17 +761,6 @@ void pnv_pci_dma_dev_setup(struct pci_dev *pdev) phb->dma_dev_setup(phb, pdev); } -u64 pnv_pci_dma_get_required_mask(struct pci_dev *pdev) -{ - struct pci_controller *hose = pci_bus_to_host(pdev->bus); - struct pnv_phb *phb = hose->private_data; - - if (phb && phb->dma_get_required_mask) - return phb->dma_get_required_mask(phb, pdev); - - return __dma_get_required_mask(&pdev->dev); -} - void pnv_pci_shutdown(void) { struct pci_controller *hose; diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h index 8ef2d28aded0..c8ff50e90766 100644 --- a/arch/powerpc/platforms/powernv/pci.h +++ b/arch/powerpc/platforms/powernv/pci.h @@ -105,13 +105,12 @@ struct pnv_phb { unsigned int hwirq, unsigned int virq, unsigned int is_64, struct msi_msg *msg); void (*dma_dev_setup)(struct pnv_phb *phb, struct pci_dev *pdev); - u64 (*dma_get_required_mask)(struct pnv_phb *phb, - struct pci_dev *pdev); void (*fixup_phb)(struct pci_controller *hose); u32 (*bdfn_to_pe)(struct pnv_phb *phb, struct pci_bus *bus, u32 devfn); int (*init_m64)(struct pnv_phb *phb); - void (*reserve_m64_pe)(struct pnv_phb *phb); - int (*pick_m64_pe)(struct pnv_phb *phb, struct pci_bus *bus, int all); + void (*reserve_m64_pe)(struct pci_bus *bus, + unsigned long *pe_bitmap, bool all); + int (*pick_m64_pe)(struct pci_bus *bus, bool all); int (*get_pe_state)(struct pnv_phb *phb, int pe_no); void (*freeze_pe)(struct pnv_phb *phb, int pe_no); int (*unfreeze_pe)(struct pnv_phb *phb, int pe_no, int opt); diff --git a/arch/powerpc/platforms/powernv/powernv.h b/arch/powerpc/platforms/powernv/powernv.h index 9269e30e4ca0..6dbc0a1da1f6 100644 --- a/arch/powerpc/platforms/powernv/powernv.h +++ b/arch/powerpc/platforms/powernv/powernv.h @@ -12,15 +12,9 @@ struct pci_dev; #ifdef CONFIG_PCI extern void pnv_pci_init(void); extern void pnv_pci_shutdown(void); -extern u64 pnv_pci_dma_get_required_mask(struct pci_dev *pdev); #else static inline void pnv_pci_init(void) { } static inline void pnv_pci_shutdown(void) { } - -static inline u64 pnv_pci_dma_get_required_mask(struct pci_dev *pdev) -{ - return 0; -} #endif extern u32 pnv_get_supported_cpuidle_states(void); diff --git a/arch/powerpc/platforms/powernv/rng.c b/arch/powerpc/platforms/powernv/rng.c index 6eb808ff637e..5dcbdea1afac 100644 --- a/arch/powerpc/platforms/powernv/rng.c +++ b/arch/powerpc/platforms/powernv/rng.c @@ -128,7 +128,7 @@ static __init int rng_create(struct device_node *dn) pr_info_once("Registering arch random hook.\n"); - ppc_md.get_random_long = powernv_get_random_long; + ppc_md.get_random_seed = powernv_get_random_long; return 0; } diff --git a/arch/powerpc/platforms/powernv/setup.c b/arch/powerpc/platforms/powernv/setup.c index 53737e019ae3..685b3cbe1362 100644 --- a/arch/powerpc/platforms/powernv/setup.c +++ b/arch/powerpc/platforms/powernv/setup.c @@ -165,14 +165,6 @@ static void pnv_progress(char *s, unsigned short hex) { } -static u64 pnv_dma_get_required_mask(struct device *dev) -{ - if (dev_is_pci(dev)) - return pnv_pci_dma_get_required_mask(to_pci_dev(dev)); - - return __dma_get_required_mask(dev); -} - static void pnv_shutdown(void) { /* Let the PCI code clear up IODA tables */ @@ -243,6 +235,13 @@ static void pnv_kexec_cpu_down(int crash_shutdown, int secondary) } else { /* Primary waits for the secondaries to have reached OPAL */ pnv_kexec_wait_secondaries_down(); + + /* + * We might be running as little-endian - now that interrupts + * are disabled, reset the HILE bit to big-endian so we don't + * take interrupts in the wrong endian later + */ + opal_reinit_cpus(OPAL_REINIT_CPUS_HILE_BE); } } #endif /* CONFIG_KEXEC */ @@ -314,7 +313,6 @@ define_machine(powernv) { .machine_shutdown = pnv_shutdown, .power_save = power7_idle, .calibrate_decr = generic_calibrate_decr, - .dma_get_required_mask = pnv_dma_get_required_mask, #ifdef CONFIG_KEXEC .kexec_cpu_down = pnv_kexec_cpu_down, #endif diff --git a/arch/powerpc/platforms/powernv/subcore.c b/arch/powerpc/platforms/powernv/subcore.c index f60f80ada903..503a73f59359 100644 --- a/arch/powerpc/platforms/powernv/subcore.c +++ b/arch/powerpc/platforms/powernv/subcore.c @@ -190,7 +190,7 @@ static void unsplit_core(void) hid0 = mfspr(SPRN_HID0); hid0 &= ~HID0_POWER8_DYNLPARDIS; - mtspr(SPRN_HID0, hid0); + update_power8_hid0(hid0); update_hid_in_slw(hid0); while (mfspr(SPRN_HID0) & mask) @@ -227,7 +227,7 @@ static void split_core(int new_mode) /* Write new mode */ hid0 = mfspr(SPRN_HID0); hid0 |= HID0_POWER8_DYNLPARDIS | split_parms[i].value; - mtspr(SPRN_HID0, hid0); + update_power8_hid0(hid0); update_hid_in_slw(hid0); /* Wait for it to happen */ diff --git a/arch/powerpc/platforms/ps3/interrupt.c b/arch/powerpc/platforms/ps3/interrupt.c index a6c42f34303a..638c4060938e 100644 --- a/arch/powerpc/platforms/ps3/interrupt.c +++ b/arch/powerpc/platforms/ps3/interrupt.c @@ -678,7 +678,8 @@ static int ps3_host_map(struct irq_domain *h, unsigned int virq, return 0; } -static int ps3_host_match(struct irq_domain *h, struct device_node *np) +static int ps3_host_match(struct irq_domain *h, struct device_node *np, + enum irq_domain_bus_token bus_token) { /* Match all */ return 1; diff --git a/arch/powerpc/platforms/pseries/hotplug-memory.c b/arch/powerpc/platforms/pseries/hotplug-memory.c index 0ced387e1463..e9ff44cd5d86 100644 --- a/arch/powerpc/platforms/pseries/hotplug-memory.c +++ b/arch/powerpc/platforms/pseries/hotplug-memory.c @@ -92,13 +92,12 @@ static struct property *dlpar_clone_drconf_property(struct device_node *dn) return NULL; new_prop->name = kstrdup(prop->name, GFP_KERNEL); - new_prop->value = kmalloc(prop->length, GFP_KERNEL); + new_prop->value = kmemdup(prop->value, prop->length, GFP_KERNEL); if (!new_prop->name || !new_prop->value) { dlpar_free_drconf_property(new_prop); return NULL; } - memcpy(new_prop->value, prop->value, prop->length); new_prop->length = prop->length; /* Convert the property to cpu endian-ness */ diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c index 10510dea16b3..0946b98d75d4 100644 --- a/arch/powerpc/platforms/pseries/iommu.c +++ b/arch/powerpc/platforms/pseries/iommu.c @@ -1253,11 +1253,10 @@ static int dma_set_mask_pSeriesLP(struct device *dev, u64 dma_mask) } } - /* fall back on iommu ops, restore table pointer with ops */ + /* fall back on iommu ops */ if (!ddw_enabled && get_dma_ops(dev) != &dma_iommu_ops) { dev_info(dev, "Restoring 32-bit DMA via iommu\n"); set_dma_ops(dev, &dma_iommu_ops); - pci_dma_dev_setup_pSeriesLP(pdev); } check_mask: diff --git a/arch/powerpc/platforms/pseries/msi.c b/arch/powerpc/platforms/pseries/msi.c index c22bb647cce6..272e9ec1ab54 100644 --- a/arch/powerpc/platforms/pseries/msi.c +++ b/arch/powerpc/platforms/pseries/msi.c @@ -118,7 +118,7 @@ static void rtas_teardown_msi_irqs(struct pci_dev *pdev) { struct msi_desc *entry; - list_for_each_entry(entry, &pdev->msi_list, list) { + for_each_pci_msi_entry(entry, pdev) { if (entry->irq == NO_IRQ) continue; @@ -350,7 +350,7 @@ static int check_msix_entries(struct pci_dev *pdev) * So we must reject such requests. */ expected = 0; - list_for_each_entry(entry, &pdev->msi_list, list) { + for_each_pci_msi_entry(entry, pdev) { if (entry->msi_attrib.entry_nr != expected) { pr_debug("rtas_msi: bad MSI-X entries.\n"); return -EINVAL; @@ -462,7 +462,7 @@ static int rtas_setup_msi_irqs(struct pci_dev *pdev, int nvec_in, int type) } i = 0; - list_for_each_entry(entry, &pdev->msi_list, list) { + for_each_pci_msi_entry(entry, pdev) { hwirq = rtas_query_irq_number(pdn, i++); if (hwirq < 0) { pr_debug("rtas_msi: error (%d) getting hwirq\n", rc); diff --git a/arch/powerpc/platforms/pseries/ras.c b/arch/powerpc/platforms/pseries/ras.c index 02e4a1745516..3b6647e574b6 100644 --- a/arch/powerpc/platforms/pseries/ras.c +++ b/arch/powerpc/platforms/pseries/ras.c @@ -189,7 +189,8 @@ static irqreturn_t ras_epow_interrupt(int irq, void *dev_id) int state; int critical; - status = rtas_get_sensor(EPOW_SENSOR_TOKEN, EPOW_SENSOR_INDEX, &state); + status = rtas_get_sensor_fast(EPOW_SENSOR_TOKEN, EPOW_SENSOR_INDEX, + &state); if (state > 3) critical = 1; /* Time Critical */ diff --git a/arch/powerpc/platforms/pseries/rng.c b/arch/powerpc/platforms/pseries/rng.c index e09608770909..31ca557af60b 100644 --- a/arch/powerpc/platforms/pseries/rng.c +++ b/arch/powerpc/platforms/pseries/rng.c @@ -38,7 +38,7 @@ static __init int rng_init(void) pr_info("Registering arch random hook.\n"); - ppc_md.get_random_long = pseries_get_random_long; + ppc_md.get_random_seed = pseries_get_random_long; return 0; } diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c index df6a7041922b..39a74fad3e04 100644 --- a/arch/powerpc/platforms/pseries/setup.c +++ b/arch/powerpc/platforms/pseries/setup.c @@ -254,19 +254,26 @@ static void __init pseries_discover_pic(void) static int pci_dn_reconfig_notifier(struct notifier_block *nb, unsigned long action, void *data) { struct of_reconfig_data *rd = data; - struct device_node *np = rd->dn; - struct pci_dn *pci = NULL; + struct device_node *parent, *np = rd->dn; + struct pci_dn *pdn; int err = NOTIFY_OK; switch (action) { case OF_RECONFIG_ATTACH_NODE: - pci = np->parent->data; - if (pci) { - update_dn_pci_info(np, pci->phb); - - /* Create EEH device for the OF node */ - eeh_dev_init(PCI_DN(np), pci->phb); + parent = of_get_parent(np); + pdn = parent ? PCI_DN(parent) : NULL; + if (pdn) { + /* Create pdn and EEH device */ + update_dn_pci_info(np, pdn->phb); + eeh_dev_init(PCI_DN(np), pdn->phb); } + + of_node_put(parent); + break; + case OF_RECONFIG_DETACH_NODE: + pdn = PCI_DN(np); + if (pdn) + list_del(&pdn->list); break; default: err = NOTIFY_DONE; diff --git a/arch/powerpc/sysdev/axonram.c b/arch/powerpc/sysdev/axonram.c index ee90db17b097..d2b79bc336c1 100644 --- a/arch/powerpc/sysdev/axonram.c +++ b/arch/powerpc/sysdev/axonram.c @@ -132,7 +132,7 @@ axon_ram_make_request(struct request_queue *queue, struct bio *bio) phys_mem += vec.bv_len; transfered += vec.bv_len; } - bio_endio(bio, 0); + bio_endio(bio); } /** @@ -141,13 +141,14 @@ axon_ram_make_request(struct request_queue *queue, struct bio *bio) */ static long axon_ram_direct_access(struct block_device *device, sector_t sector, - void **kaddr, unsigned long *pfn, long size) + void __pmem **kaddr, unsigned long *pfn) { struct axon_ram_bank *bank = device->bd_disk->private_data; loff_t offset = (loff_t)sector << AXON_RAM_SECTOR_SHIFT; + void *addr = (void *)(bank->ph_addr + offset); - *kaddr = (void *)(bank->ph_addr + offset); - *pfn = virt_to_phys(*kaddr) >> PAGE_SHIFT; + *kaddr = (void __pmem *)addr; + *pfn = virt_to_phys(addr) >> PAGE_SHIFT; return bank->size - offset; } diff --git a/arch/powerpc/sysdev/cpm_common.c b/arch/powerpc/sysdev/cpm_common.c index 4f7869571290..e2ea51961979 100644 --- a/arch/powerpc/sysdev/cpm_common.c +++ b/arch/powerpc/sysdev/cpm_common.c @@ -147,7 +147,7 @@ unsigned long cpm_muram_alloc(unsigned long size, unsigned long align) spin_lock_irqsave(&cpm_muram_lock, flags); cpm_muram_info.alignment = align; start = rh_alloc(&cpm_muram_info, size, "commproc"); - memset(cpm_muram_addr(start), 0, size); + memset_io(cpm_muram_addr(start), 0, size); spin_unlock_irqrestore(&cpm_muram_lock, flags); return start; diff --git a/arch/powerpc/sysdev/dart_iommu.c b/arch/powerpc/sysdev/dart_iommu.c index 90bcdfeedf48..b7348637eae0 100644 --- a/arch/powerpc/sysdev/dart_iommu.c +++ b/arch/powerpc/sysdev/dart_iommu.c @@ -313,20 +313,11 @@ static void iommu_table_dart_setup(void) set_bit(iommu_table_dart.it_size - 1, iommu_table_dart.it_map); } -static void dma_dev_setup_dart(struct device *dev) -{ - /* We only have one iommu table on the mac for now, which makes - * things simple. Setup all PCI devices to point to this table - */ - if (get_dma_ops(dev) == &dma_direct_ops) - set_dma_offset(dev, DART_U4_BYPASS_BASE); - else - set_iommu_table_base(dev, &iommu_table_dart); -} - static void pci_dma_dev_setup_dart(struct pci_dev *dev) { - dma_dev_setup_dart(&dev->dev); + if (dart_is_u4) + set_dma_offset(&dev->dev, DART_U4_BYPASS_BASE); + set_iommu_table_base(&dev->dev, &iommu_table_dart); } static void pci_dma_bus_setup_dart(struct pci_bus *bus) @@ -370,7 +361,6 @@ static int dart_dma_set_mask(struct device *dev, u64 dma_mask) dev_info(dev, "Using 32-bit DMA via iommu\n"); set_dma_ops(dev, &dma_iommu_ops); } - dma_dev_setup_dart(dev); *dev->dma_mask = dma_mask; return 0; diff --git a/arch/powerpc/sysdev/ehv_pic.c b/arch/powerpc/sysdev/ehv_pic.c index 2d20f10a4203..eca0b00794fa 100644 --- a/arch/powerpc/sysdev/ehv_pic.c +++ b/arch/powerpc/sysdev/ehv_pic.c @@ -177,7 +177,8 @@ unsigned int ehv_pic_get_irq(void) return irq_linear_revmap(global_ehv_pic->irqhost, irq); } -static int ehv_pic_host_match(struct irq_domain *h, struct device_node *node) +static int ehv_pic_host_match(struct irq_domain *h, struct device_node *node, + enum irq_domain_bus_token bus_token) { /* Exact match, unless ehv_pic node is NULL */ return h->of_node == NULL || h->of_node == node; diff --git a/arch/powerpc/sysdev/fsl_msi.c b/arch/powerpc/sysdev/fsl_msi.c index 5236e5427c38..5916da1856a7 100644 --- a/arch/powerpc/sysdev/fsl_msi.c +++ b/arch/powerpc/sysdev/fsl_msi.c @@ -129,7 +129,7 @@ static void fsl_teardown_msi_irqs(struct pci_dev *pdev) struct msi_desc *entry; struct fsl_msi *msi_data; - list_for_each_entry(entry, &pdev->msi_list, list) { + for_each_pci_msi_entry(entry, pdev) { if (entry->irq == NO_IRQ) continue; msi_data = irq_get_chip_data(entry->irq); @@ -219,7 +219,7 @@ static int fsl_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) } } - list_for_each_entry(entry, &pdev->msi_list, list) { + for_each_pci_msi_entry(entry, pdev) { /* * Loop over all the MSI devices until we find one that has an * available interrupt. diff --git a/arch/powerpc/sysdev/fsl_pci.c b/arch/powerpc/sysdev/fsl_pci.c index 9a8fcf0d79d7..ebc1f412cf49 100644 --- a/arch/powerpc/sysdev/fsl_pci.c +++ b/arch/powerpc/sysdev/fsl_pci.c @@ -1113,7 +1113,7 @@ static int fsl_pci_pme_probe(struct pci_controller *hose) IRQF_SHARED, "[PCI] PME", hose); if (res < 0) { - dev_err(&dev->dev, "Unable to requiest irq %d for PME\n", pme_irq); + dev_err(&dev->dev, "Unable to request irq %d for PME\n", pme_irq); irq_dispose_mapping(pme_irq); return -ENODEV; diff --git a/arch/powerpc/sysdev/i8259.c b/arch/powerpc/sysdev/i8259.c index 31c33475c7b7..e1a9c2c2d5d3 100644 --- a/arch/powerpc/sysdev/i8259.c +++ b/arch/powerpc/sysdev/i8259.c @@ -162,7 +162,8 @@ static struct resource pic_edgectrl_iores = { .flags = IORESOURCE_BUSY, }; -static int i8259_host_match(struct irq_domain *h, struct device_node *node) +static int i8259_host_match(struct irq_domain *h, struct device_node *node, + enum irq_domain_bus_token bus_token) { return h->of_node == NULL || h->of_node == node; } diff --git a/arch/powerpc/sysdev/ipic.c b/arch/powerpc/sysdev/ipic.c index d78f1364b639..6b2b68914810 100644 --- a/arch/powerpc/sysdev/ipic.c +++ b/arch/powerpc/sysdev/ipic.c @@ -671,7 +671,8 @@ static struct irq_chip ipic_edge_irq_chip = { .irq_set_type = ipic_set_irq_type, }; -static int ipic_host_match(struct irq_domain *h, struct device_node *node) +static int ipic_host_match(struct irq_domain *h, struct device_node *node, + enum irq_domain_bus_token bus_token) { /* Exact match, unless ipic node is NULL */ return h->of_node == NULL || h->of_node == node; diff --git a/arch/powerpc/sysdev/mpic.c b/arch/powerpc/sysdev/mpic.c index c8e73332eaad..97a8ae8f94dd 100644 --- a/arch/powerpc/sysdev/mpic.c +++ b/arch/powerpc/sysdev/mpic.c @@ -1007,7 +1007,8 @@ static struct irq_chip mpic_irq_ht_chip = { #endif /* CONFIG_MPIC_U3_HT_IRQS */ -static int mpic_host_match(struct irq_domain *h, struct device_node *node) +static int mpic_host_match(struct irq_domain *h, struct device_node *node, + enum irq_domain_bus_token bus_token) { /* Exact match, unless mpic node is NULL */ return h->of_node == NULL || h->of_node == node; diff --git a/arch/powerpc/sysdev/mpic_u3msi.c b/arch/powerpc/sysdev/mpic_u3msi.c index fc46ef3b816e..70fbd5694a8b 100644 --- a/arch/powerpc/sysdev/mpic_u3msi.c +++ b/arch/powerpc/sysdev/mpic_u3msi.c @@ -108,7 +108,7 @@ static void u3msi_teardown_msi_irqs(struct pci_dev *pdev) { struct msi_desc *entry; - list_for_each_entry(entry, &pdev->msi_list, list) { + for_each_pci_msi_entry(entry, pdev) { if (entry->irq == NO_IRQ) continue; @@ -140,7 +140,7 @@ static int u3msi_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) return -ENXIO; } - list_for_each_entry(entry, &pdev->msi_list, list) { + for_each_pci_msi_entry(entry, pdev) { hwirq = msi_bitmap_alloc_hwirqs(&msi_mpic->msi_bitmap, 1); if (hwirq < 0) { pr_debug("u3msi: failed allocating hwirq\n"); diff --git a/arch/powerpc/sysdev/ppc4xx_hsta_msi.c b/arch/powerpc/sysdev/ppc4xx_hsta_msi.c index 87f9623ca805..52a93dcae262 100644 --- a/arch/powerpc/sysdev/ppc4xx_hsta_msi.c +++ b/arch/powerpc/sysdev/ppc4xx_hsta_msi.c @@ -51,7 +51,7 @@ static int hsta_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) return -EINVAL; } - list_for_each_entry(entry, &dev->msi_list, list) { + for_each_pci_msi_entry(entry, dev) { irq = msi_bitmap_alloc_hwirqs(&ppc4xx_hsta_msi.bmp, 1); if (irq < 0) { pr_debug("%s: Failed to allocate msi interrupt\n", @@ -109,7 +109,7 @@ static void hsta_teardown_msi_irqs(struct pci_dev *dev) struct msi_desc *entry; int irq; - list_for_each_entry(entry, &dev->msi_list, list) { + for_each_pci_msi_entry(entry, dev) { if (entry->irq == NO_IRQ) continue; @@ -132,7 +132,7 @@ static int hsta_msi_probe(struct platform_device *pdev) struct pci_controller *phb; mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (IS_ERR(mem)) { + if (!mem) { dev_err(dev, "Unable to get mmio space\n"); return -EINVAL; } @@ -157,7 +157,7 @@ static int hsta_msi_probe(struct platform_device *pdev) goto out; ppc4xx_hsta_msi.irq_map = kmalloc(sizeof(int) * irq_count, GFP_KERNEL); - if (IS_ERR(ppc4xx_hsta_msi.irq_map)) { + if (!ppc4xx_hsta_msi.irq_map) { ret = -ENOMEM; goto out1; } diff --git a/arch/powerpc/sysdev/ppc4xx_msi.c b/arch/powerpc/sysdev/ppc4xx_msi.c index 6eb21f2ea585..24d0470c1698 100644 --- a/arch/powerpc/sysdev/ppc4xx_msi.c +++ b/arch/powerpc/sysdev/ppc4xx_msi.c @@ -93,7 +93,7 @@ static int ppc4xx_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) if (!msi_data->msi_virqs) return -ENOMEM; - list_for_each_entry(entry, &dev->msi_list, list) { + for_each_pci_msi_entry(entry, dev) { int_no = msi_bitmap_alloc_hwirqs(&msi_data->bitmap, 1); if (int_no >= 0) break; @@ -127,7 +127,7 @@ void ppc4xx_teardown_msi_irqs(struct pci_dev *dev) dev_dbg(&dev->dev, "PCIE-MSI: tearing down msi irqs\n"); - list_for_each_entry(entry, &dev->msi_list, list) { + for_each_pci_msi_entry(entry, dev) { if (entry->irq == NO_IRQ) continue; irq_set_msi_desc(entry->irq, NULL); diff --git a/arch/powerpc/sysdev/qe_lib/qe_ic.c b/arch/powerpc/sysdev/qe_lib/qe_ic.c index 6512cd8caa51..47b352e4bc74 100644 --- a/arch/powerpc/sysdev/qe_lib/qe_ic.c +++ b/arch/powerpc/sysdev/qe_lib/qe_ic.c @@ -244,7 +244,8 @@ static struct irq_chip qe_ic_irq_chip = { .irq_mask_ack = qe_ic_mask_irq, }; -static int qe_ic_host_match(struct irq_domain *h, struct device_node *node) +static int qe_ic_host_match(struct irq_domain *h, struct device_node *node, + enum irq_domain_bus_token bus_token) { /* Exact match, unless qe_ic node is NULL */ return h->of_node == NULL || h->of_node == node; diff --git a/arch/powerpc/sysdev/xics/ics-opal.c b/arch/powerpc/sysdev/xics/ics-opal.c index 68c7e5cc98e0..11ac964d5175 100644 --- a/arch/powerpc/sysdev/xics/ics-opal.c +++ b/arch/powerpc/sysdev/xics/ics-opal.c @@ -72,7 +72,7 @@ static unsigned int ics_opal_startup(struct irq_data *d) * card, using the MSI mask bits. Firmware doesn't appear to unmask * at that level, so we do it here by hand. */ - if (d->msi_desc) + if (irq_data_get_msi_desc(d)) pci_msi_unmask_irq(d); #endif diff --git a/arch/powerpc/sysdev/xics/ics-rtas.c b/arch/powerpc/sysdev/xics/ics-rtas.c index 0af97deb83f3..d1c625c4cc5a 100644 --- a/arch/powerpc/sysdev/xics/ics-rtas.c +++ b/arch/powerpc/sysdev/xics/ics-rtas.c @@ -75,7 +75,7 @@ static unsigned int ics_rtas_startup(struct irq_data *d) * card, using the MSI mask bits. Firmware doesn't appear to unmask * at that level, so we do it here by hand. */ - if (d->msi_desc) + if (irq_data_get_msi_desc(d)) pci_msi_unmask_irq(d); #endif /* unmask it */ diff --git a/arch/powerpc/sysdev/xics/xics-common.c b/arch/powerpc/sysdev/xics/xics-common.c index 08c248eb491b..47e43b7b076b 100644 --- a/arch/powerpc/sysdev/xics/xics-common.c +++ b/arch/powerpc/sysdev/xics/xics-common.c @@ -298,7 +298,8 @@ int xics_get_irq_server(unsigned int virq, const struct cpumask *cpumask, } #endif /* CONFIG_SMP */ -static int xics_host_match(struct irq_domain *h, struct device_node *node) +static int xics_host_match(struct irq_domain *h, struct device_node *node, + enum irq_domain_bus_token bus_token) { struct ics *ics; diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c index e599259d84fc..6ef1231c6e9c 100644 --- a/arch/powerpc/xmon/xmon.c +++ b/arch/powerpc/xmon/xmon.c @@ -1987,7 +1987,6 @@ memex(void) case '^': adrs -= size; break; - break; case '/': if (nslash > 0) adrs -= 1 << nslash; @@ -2731,7 +2730,7 @@ static void xmon_print_symbol(unsigned long address, const char *mid, void dump_segments(void) { int i; - unsigned long esid,vsid,valid; + unsigned long esid,vsid; unsigned long llp; printf("SLB contents of cpu 0x%x\n", smp_processor_id()); @@ -2739,10 +2738,9 @@ void dump_segments(void) for (i = 0; i < mmu_slb_size; i++) { asm volatile("slbmfee %0,%1" : "=r" (esid) : "r" (i)); asm volatile("slbmfev %0,%1" : "=r" (vsid) : "r" (i)); - valid = (esid & SLB_ESID_V); - if (valid | esid | vsid) { + if (esid || vsid) { printf("%02d %016lx %016lx", i, esid, vsid); - if (valid) { + if (esid & SLB_ESID_V) { llp = vsid & SLB_VSID_LLP; if (vsid & SLB_VSID_B_1T) { printf(" 1T ESID=%9lx VSID=%13lx LLP:%3lx \n", diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 4827870f7a6d..1d57000b1b24 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -48,6 +48,7 @@ config ARCH_SUPPORTS_DEBUG_PAGEALLOC config KEXEC def_bool y + select KEXEC_CORE config AUDIT_ARCH def_bool y diff --git a/arch/s390/boot/compressed/misc.c b/arch/s390/boot/compressed/misc.c index 42506b371b74..4da604ebf6fd 100644 --- a/arch/s390/boot/compressed/misc.c +++ b/arch/s390/boot/compressed/misc.c @@ -167,7 +167,7 @@ unsigned long decompress_kernel(void) #endif puts("Uncompressing Linux... "); - decompress(input_data, input_len, NULL, NULL, output, NULL, error); + __decompress(input_data, input_len, NULL, NULL, output, 0, NULL, error); puts("Ok, booting the kernel.\n"); return (unsigned long) output; } diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h index adbe3802e377..117fa5c921c1 100644 --- a/arch/s390/include/asm/atomic.h +++ b/arch/s390/include/asm/atomic.h @@ -27,6 +27,7 @@ #define __ATOMIC_OR "lao" #define __ATOMIC_AND "lan" #define __ATOMIC_ADD "laa" +#define __ATOMIC_XOR "lax" #define __ATOMIC_BARRIER "bcr 14,0\n" #define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier) \ @@ -49,6 +50,7 @@ #define __ATOMIC_OR "or" #define __ATOMIC_AND "nr" #define __ATOMIC_ADD "ar" +#define __ATOMIC_XOR "xr" #define __ATOMIC_BARRIER "\n" #define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier) \ @@ -118,15 +120,17 @@ static inline void atomic_add(int i, atomic_t *v) #define atomic_dec_return(_v) atomic_sub_return(1, _v) #define atomic_dec_and_test(_v) (atomic_sub_return(1, _v) == 0) -static inline void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - __ATOMIC_LOOP(v, ~mask, __ATOMIC_AND, __ATOMIC_NO_BARRIER); +#define ATOMIC_OP(op, OP) \ +static inline void atomic_##op(int i, atomic_t *v) \ +{ \ + __ATOMIC_LOOP(v, i, __ATOMIC_##OP, __ATOMIC_NO_BARRIER); \ } -static inline void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - __ATOMIC_LOOP(v, mask, __ATOMIC_OR, __ATOMIC_NO_BARRIER); -} +ATOMIC_OP(and, AND) +ATOMIC_OP(or, OR) +ATOMIC_OP(xor, XOR) + +#undef ATOMIC_OP #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) @@ -167,6 +171,7 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) #define __ATOMIC64_OR "laog" #define __ATOMIC64_AND "lang" #define __ATOMIC64_ADD "laag" +#define __ATOMIC64_XOR "laxg" #define __ATOMIC64_BARRIER "bcr 14,0\n" #define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier) \ @@ -189,6 +194,7 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) #define __ATOMIC64_OR "ogr" #define __ATOMIC64_AND "ngr" #define __ATOMIC64_ADD "agr" +#define __ATOMIC64_XOR "xgr" #define __ATOMIC64_BARRIER "\n" #define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier) \ @@ -247,16 +253,6 @@ static inline void atomic64_add(long long i, atomic64_t *v) __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_NO_BARRIER); } -static inline void atomic64_clear_mask(unsigned long mask, atomic64_t *v) -{ - __ATOMIC64_LOOP(v, ~mask, __ATOMIC64_AND, __ATOMIC64_NO_BARRIER); -} - -static inline void atomic64_set_mask(unsigned long mask, atomic64_t *v) -{ - __ATOMIC64_LOOP(v, mask, __ATOMIC64_OR, __ATOMIC64_NO_BARRIER); -} - #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) static inline long long atomic64_cmpxchg(atomic64_t *v, @@ -270,6 +266,17 @@ static inline long long atomic64_cmpxchg(atomic64_t *v, return old; } +#define ATOMIC64_OP(op, OP) \ +static inline void atomic64_##op(long i, atomic64_t *v) \ +{ \ + __ATOMIC64_LOOP(v, i, __ATOMIC64_##OP, __ATOMIC64_NO_BARRIER); \ +} + +ATOMIC64_OP(and, AND) +ATOMIC64_OP(or, OR) +ATOMIC64_OP(xor, XOR) + +#undef ATOMIC64_OP #undef __ATOMIC64_LOOP static inline int atomic64_add_unless(atomic64_t *v, long long i, long long u) diff --git a/arch/s390/include/asm/barrier.h b/arch/s390/include/asm/barrier.h index e6f8615a11eb..d48fe0162331 100644 --- a/arch/s390/include/asm/barrier.h +++ b/arch/s390/include/asm/barrier.h @@ -42,12 +42,12 @@ do { \ compiletime_assert_atomic_type(*p); \ barrier(); \ - ACCESS_ONCE(*p) = (v); \ + WRITE_ONCE(*p, v); \ } while (0) #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ barrier(); \ ___p1; \ diff --git a/arch/s390/include/asm/dma-mapping.h b/arch/s390/include/asm/dma-mapping.h index 9d395961e713..b3fd54d93dd2 100644 --- a/arch/s390/include/asm/dma-mapping.h +++ b/arch/s390/include/asm/dma-mapping.h @@ -18,27 +18,13 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev) return &s390_dma_ops; } -extern int dma_set_mask(struct device *dev, u64 mask); - static inline void dma_cache_sync(struct device *dev, void *vaddr, size_t size, enum dma_data_direction direction) { } -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) -#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) - #include -static inline int dma_supported(struct device *dev, u64 mask) -{ - struct dma_map_ops *dma_ops = get_dma_ops(dev); - - if (dma_ops->dma_supported == NULL) - return 1; - return dma_ops->dma_supported(dev, mask); -} - static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) { if (!dev->dma_mask) @@ -46,45 +32,4 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) return addr + size - 1 <= *dev->dma_mask; } -static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - struct dma_map_ops *dma_ops = get_dma_ops(dev); - - debug_dma_mapping_error(dev, dma_addr); - if (dma_ops->mapping_error) - return dma_ops->mapping_error(dev, dma_addr); - return dma_addr == DMA_ERROR_CODE; -} - -#define dma_alloc_coherent(d, s, h, f) dma_alloc_attrs(d, s, h, f, NULL) - -static inline void *dma_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flags, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - void *cpu_addr; - - BUG_ON(!ops); - - cpu_addr = ops->alloc(dev, size, dma_handle, flags, attrs); - debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr); - - return cpu_addr; -} - -#define dma_free_coherent(d, s, c, h) dma_free_attrs(d, s, c, h, NULL) - -static inline void dma_free_attrs(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t dma_handle, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - - BUG_ON(!ops); - - debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); - ops->free(dev, size, cpu_addr, dma_handle, attrs); -} - #endif /* _ASM_S390_DMA_MAPPING_H */ diff --git a/arch/s390/include/asm/jump_label.h b/arch/s390/include/asm/jump_label.h index 69972b7957ee..7f9fd5e3f1bf 100644 --- a/arch/s390/include/asm/jump_label.h +++ b/arch/s390/include/asm/jump_label.h @@ -12,14 +12,29 @@ * We use a brcl 0,2 instruction for jump labels at compile time so it * can be easily distinguished from a hotpatch generated instruction. */ -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("0: brcl 0,"__stringify(JUMP_LABEL_NOP_OFFSET)"\n" ".pushsection __jump_table, \"aw\"\n" ".balign 8\n" ".quad 0b, %l[label], %0\n" ".popsection\n" - : : "X" (key) : : label); + : : "X" (&((char *)key)[branch]) : : label); + + return false; +label: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("0: brcl 15, %l[label]\n" + ".pushsection __jump_table, \"aw\"\n" + ".balign 8\n" + ".quad 0b, %l[label], %0\n" + ".popsection\n" + : : "X" (&((char *)key)[branch]) : : label); + return false; label: return true; diff --git a/arch/s390/kernel/jump_label.c b/arch/s390/kernel/jump_label.c index c9dac2139f59..083b05f5f5ab 100644 --- a/arch/s390/kernel/jump_label.c +++ b/arch/s390/kernel/jump_label.c @@ -61,7 +61,7 @@ static void __jump_label_transform(struct jump_entry *entry, { struct insn old, new; - if (type == JUMP_LABEL_ENABLE) { + if (type == JUMP_LABEL_JMP) { jump_label_make_nop(entry, &old); jump_label_make_branch(entry, &new); } else { diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 627887b075a7..017c3a9bfc28 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -120,11 +120,6 @@ static int s390_next_event(unsigned long delta, return 0; } -static void s390_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) -{ -} - /* * Set up lowcore and control register of the current cpu to * enable TOD clock and clock comparator interrupts. @@ -148,7 +143,6 @@ void init_cpu_timer(void) cd->rating = 400; cd->cpumask = cpumask_of(cpu); cd->set_next_event = s390_next_event; - cd->set_mode = s390_set_mode; clockevents_register_device(cd); @@ -384,7 +378,7 @@ static void disable_sync_clock(void *dummy) * increase the "sequence" counter to avoid the race of an * etr event and the complete recovery against get_sync_clock. */ - atomic_clear_mask(0x80000000, sw_ptr); + atomic_andnot(0x80000000, sw_ptr); atomic_inc(sw_ptr); } @@ -395,7 +389,7 @@ static void disable_sync_clock(void *dummy) static void enable_sync_clock(void) { atomic_t *sw_ptr = this_cpu_ptr(&clock_sync_word); - atomic_set_mask(0x80000000, sw_ptr); + atomic_or(0x80000000, sw_ptr); } /* diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index b277d50dcf76..5c2c169395c3 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -173,20 +173,20 @@ static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu) static void __set_cpu_idle(struct kvm_vcpu *vcpu) { - atomic_set_mask(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags); + atomic_or(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags); set_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask); } static void __unset_cpu_idle(struct kvm_vcpu *vcpu) { - atomic_clear_mask(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags); + atomic_andnot(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags); clear_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask); } static void __reset_intercept_indicators(struct kvm_vcpu *vcpu) { - atomic_clear_mask(CPUSTAT_IO_INT | CPUSTAT_EXT_INT | CPUSTAT_STOP_INT, - &vcpu->arch.sie_block->cpuflags); + atomic_andnot(CPUSTAT_IO_INT | CPUSTAT_EXT_INT | CPUSTAT_STOP_INT, + &vcpu->arch.sie_block->cpuflags); vcpu->arch.sie_block->lctl = 0x0000; vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT); @@ -199,7 +199,7 @@ static void __reset_intercept_indicators(struct kvm_vcpu *vcpu) static void __set_cpuflag(struct kvm_vcpu *vcpu, u32 flag) { - atomic_set_mask(flag, &vcpu->arch.sie_block->cpuflags); + atomic_or(flag, &vcpu->arch.sie_block->cpuflags); } static void set_intercept_indicators_io(struct kvm_vcpu *vcpu) @@ -928,7 +928,7 @@ void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu) spin_unlock(&li->lock); /* clear pending external calls set by sigp interpretation facility */ - atomic_clear_mask(CPUSTAT_ECALL_PEND, li->cpuflags); + atomic_andnot(CPUSTAT_ECALL_PEND, li->cpuflags); vcpu->kvm->arch.sca->cpu[vcpu->vcpu_id].sigp_ctrl = 0; } @@ -1026,7 +1026,7 @@ static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) li->irq.ext = irq->u.ext; set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs); - atomic_set_mask(CPUSTAT_EXT_INT, li->cpuflags); + atomic_or(CPUSTAT_EXT_INT, li->cpuflags); return 0; } @@ -1041,7 +1041,7 @@ static int __inject_extcall_sigpif(struct kvm_vcpu *vcpu, uint16_t src_id) /* another external call is pending */ return -EBUSY; } - atomic_set_mask(CPUSTAT_ECALL_PEND, &vcpu->arch.sie_block->cpuflags); + atomic_or(CPUSTAT_ECALL_PEND, &vcpu->arch.sie_block->cpuflags); return 0; } @@ -1067,7 +1067,7 @@ static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs)) return -EBUSY; *extcall = irq->u.extcall; - atomic_set_mask(CPUSTAT_EXT_INT, li->cpuflags); + atomic_or(CPUSTAT_EXT_INT, li->cpuflags); return 0; } @@ -1139,7 +1139,7 @@ static int __inject_sigp_emergency(struct kvm_vcpu *vcpu, set_bit(irq->u.emerg.code, li->sigp_emerg_pending); set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs); - atomic_set_mask(CPUSTAT_EXT_INT, li->cpuflags); + atomic_or(CPUSTAT_EXT_INT, li->cpuflags); return 0; } @@ -1183,7 +1183,7 @@ static int __inject_ckc(struct kvm_vcpu *vcpu) 0, 0); set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); - atomic_set_mask(CPUSTAT_EXT_INT, li->cpuflags); + atomic_or(CPUSTAT_EXT_INT, li->cpuflags); return 0; } @@ -1196,7 +1196,7 @@ static int __inject_cpu_timer(struct kvm_vcpu *vcpu) 0, 0); set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); - atomic_set_mask(CPUSTAT_EXT_INT, li->cpuflags); + atomic_or(CPUSTAT_EXT_INT, li->cpuflags); return 0; } @@ -1375,13 +1375,13 @@ static void __floating_irq_kick(struct kvm *kvm, u64 type) spin_lock(&li->lock); switch (type) { case KVM_S390_MCHK: - atomic_set_mask(CPUSTAT_STOP_INT, li->cpuflags); + atomic_or(CPUSTAT_STOP_INT, li->cpuflags); break; case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: - atomic_set_mask(CPUSTAT_IO_INT, li->cpuflags); + atomic_or(CPUSTAT_IO_INT, li->cpuflags); break; default: - atomic_set_mask(CPUSTAT_EXT_INT, li->cpuflags); + atomic_or(CPUSTAT_EXT_INT, li->cpuflags); break; } spin_unlock(&li->lock); diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 98df53c01343..c91eb941b444 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -1333,12 +1333,12 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) save_access_regs(vcpu->arch.host_acrs); restore_access_regs(vcpu->run->s.regs.acrs); gmap_enable(vcpu->arch.gmap); - atomic_set_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); + atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); } void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) { - atomic_clear_mask(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); + atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); gmap_disable(vcpu->arch.gmap); save_fpu_regs(); @@ -1443,9 +1443,9 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) CPUSTAT_STOPPED); if (test_kvm_facility(vcpu->kvm, 78)) - atomic_set_mask(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags); + atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags); else if (test_kvm_facility(vcpu->kvm, 8)) - atomic_set_mask(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags); + atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags); kvm_s390_vcpu_setup_model(vcpu); @@ -1557,24 +1557,24 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu) { - atomic_set_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); + atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); exit_sie(vcpu); } void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu) { - atomic_clear_mask(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); + atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); } static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu) { - atomic_set_mask(PROG_REQUEST, &vcpu->arch.sie_block->prog20); + atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20); exit_sie(vcpu); } static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu) { - atomic_clear_mask(PROG_REQUEST, &vcpu->arch.sie_block->prog20); + atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20); } /* @@ -1583,7 +1583,7 @@ static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu) * return immediately. */ void exit_sie(struct kvm_vcpu *vcpu) { - atomic_set_mask(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags); + atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags); while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE) cpu_relax(); } @@ -1807,19 +1807,19 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, if (dbg->control & KVM_GUESTDBG_ENABLE) { vcpu->guest_debug = dbg->control; /* enforce guest PER */ - atomic_set_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); + atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); if (dbg->control & KVM_GUESTDBG_USE_HW_BP) rc = kvm_s390_import_bp_data(vcpu, dbg); } else { - atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); + atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); vcpu->arch.guestdbg.last_bp = 0; } if (rc) { vcpu->guest_debug = 0; kvm_s390_clear_bp_data(vcpu); - atomic_clear_mask(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); + atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); } return rc; @@ -1894,7 +1894,7 @@ static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) { if (!ibs_enabled(vcpu)) { trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1); - atomic_set_mask(CPUSTAT_IBS, + atomic_or(CPUSTAT_IBS, &vcpu->arch.sie_block->cpuflags); } goto retry; @@ -1903,7 +1903,7 @@ static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) { if (ibs_enabled(vcpu)) { trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0); - atomic_clear_mask(CPUSTAT_IBS, + atomic_andnot(CPUSTAT_IBS, &vcpu->arch.sie_block->cpuflags); } goto retry; @@ -2419,7 +2419,7 @@ void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu) __disable_ibs_on_all_vcpus(vcpu->kvm); } - atomic_clear_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); + atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); /* * Another VCPU might have used IBS while we were offline. * Let's play safe and flush the VCPU at startup. @@ -2445,7 +2445,7 @@ void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu) /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */ kvm_s390_clear_stop_irq(vcpu); - atomic_set_mask(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); + atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); __disable_ibs_on_vcpu(vcpu); for (i = 0; i < online_vcpus; i++) { diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index 0d002a746bec..ae4de559e3a0 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -15,7 +15,7 @@ #include #include -static struct static_key have_mvcos = STATIC_KEY_INIT_FALSE; +static DEFINE_STATIC_KEY_FALSE(have_mvcos); static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr, unsigned long size) @@ -104,7 +104,7 @@ static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr, unsigned long __copy_from_user(void *to, const void __user *from, unsigned long n) { - if (static_key_false(&have_mvcos)) + if (static_branch_likely(&have_mvcos)) return copy_from_user_mvcos(to, from, n); return copy_from_user_mvcp(to, from, n); } @@ -177,7 +177,7 @@ static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x, unsigned long __copy_to_user(void __user *to, const void *from, unsigned long n) { - if (static_key_false(&have_mvcos)) + if (static_branch_likely(&have_mvcos)) return copy_to_user_mvcos(to, from, n); return copy_to_user_mvcs(to, from, n); } @@ -240,7 +240,7 @@ static inline unsigned long copy_in_user_mvc(void __user *to, const void __user unsigned long __copy_in_user(void __user *to, const void __user *from, unsigned long n) { - if (static_key_false(&have_mvcos)) + if (static_branch_likely(&have_mvcos)) return copy_in_user_mvcos(to, from, n); return copy_in_user_mvc(to, from, n); } @@ -312,7 +312,7 @@ static inline unsigned long clear_user_xc(void __user *to, unsigned long size) unsigned long __clear_user(void __user *to, unsigned long size) { - if (static_key_false(&have_mvcos)) + if (static_branch_likely(&have_mvcos)) return clear_user_mvcos(to, size); return clear_user_xc(to, size); } @@ -373,7 +373,7 @@ EXPORT_SYMBOL(__strncpy_from_user); static int __init uaccess_init(void) { if (test_facility(27)) - static_key_slow_inc(&have_mvcos); + static_branch_enable(&have_mvcos); return 0; } early_initcall(uaccess_init); diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 2963b563621c..c3c07d3505ba 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -169,7 +169,7 @@ void __init free_initrd_mem(unsigned long start, unsigned long end) #endif #ifdef CONFIG_MEMORY_HOTPLUG -int arch_add_memory(int nid, u64 start, u64 size) +int arch_add_memory(int nid, u64 start, u64 size, bool for_device) { unsigned long normal_end_pfn = PFN_DOWN(memblock_end_of_DRAM()); unsigned long dma_end_pfn = PFN_DOWN(MAX_DMA_ADDRESS); diff --git a/arch/s390/net/bpf_jit.h b/arch/s390/net/bpf_jit.h index f6498eec9ee1..f010c93a88b1 100644 --- a/arch/s390/net/bpf_jit.h +++ b/arch/s390/net/bpf_jit.h @@ -36,6 +36,8 @@ extern u8 sk_load_word[], sk_load_half[], sk_load_byte[]; * | BPF stack | | * | | | * +---------------+ | + * | 8 byte skbp | | + * R15+170 -> +---------------+ | * | 8 byte hlen | | * R15+168 -> +---------------+ | * | 4 byte align | | @@ -51,11 +53,12 @@ extern u8 sk_load_word[], sk_load_half[], sk_load_byte[]; * We get 160 bytes stack space from calling function, but only use * 12 * 8 byte for old backchain, r15..r6, and tail_call_cnt. */ -#define STK_SPACE (MAX_BPF_STACK + 8 + 4 + 4 + 160) +#define STK_SPACE (MAX_BPF_STACK + 8 + 8 + 4 + 4 + 160) #define STK_160_UNUSED (160 - 12 * 8) #define STK_OFF (STK_SPACE - STK_160_UNUSED) #define STK_OFF_TMP 160 /* Offset of tmp buffer on stack */ #define STK_OFF_HLEN 168 /* Offset of SKB header length on stack */ +#define STK_OFF_SKBP 170 /* Offset of SKB pointer on stack */ #define STK_OFF_R6 (160 - 11 * 8) /* Offset of r6 on stack */ #define STK_OFF_TCCNT (160 - 12 * 8) /* Offset of tail_call_cnt on stack */ diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index 8d2e5165865f..eeda051442c3 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -45,7 +45,7 @@ struct bpf_jit { int labels[1]; /* Labels for local jumps */ }; -#define BPF_SIZE_MAX 4096 /* Max size for program */ +#define BPF_SIZE_MAX 0x7ffff /* Max size for program (20 bit signed displ) */ #define SEEN_SKB 1 /* skb access */ #define SEEN_MEM 2 /* use mem[] for temporary storage */ @@ -53,6 +53,7 @@ struct bpf_jit { #define SEEN_LITERAL 8 /* code uses literals */ #define SEEN_FUNC 16 /* calls C functions */ #define SEEN_TAIL_CALL 32 /* code uses tail calls */ +#define SEEN_SKB_CHANGE 64 /* code changes skb data */ #define SEEN_STACK (SEEN_FUNC | SEEN_MEM | SEEN_SKB) /* @@ -203,19 +204,11 @@ static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) _EMIT6(op1 | __disp, op2); \ }) -#define EMIT6_DISP(op1, op2, b1, b2, b3, disp) \ -({ \ - _EMIT6_DISP(op1 | reg(b1, b2) << 16 | \ - reg_high(b3) << 8, op2, disp); \ - REG_SET_SEEN(b1); \ - REG_SET_SEEN(b2); \ - REG_SET_SEEN(b3); \ -}) - #define _EMIT6_DISP_LH(op1, op2, disp) \ ({ \ - unsigned int __disp_h = ((u32)disp) & 0xff000; \ - unsigned int __disp_l = ((u32)disp) & 0x00fff; \ + u32 _disp = (u32) disp; \ + unsigned int __disp_h = _disp & 0xff000; \ + unsigned int __disp_l = _disp & 0x00fff; \ _EMIT6(op1 | __disp_l, op2 | __disp_h >> 4); \ }) @@ -389,13 +382,33 @@ static void save_restore_regs(struct bpf_jit *jit, int op) } while (re <= 15); } +/* + * For SKB access %b1 contains the SKB pointer. For "bpf_jit.S" + * we store the SKB header length on the stack and the SKB data + * pointer in REG_SKB_DATA. + */ +static void emit_load_skb_data_hlen(struct bpf_jit *jit) +{ + /* Header length: llgf %w1,(%b1) */ + EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_1, + offsetof(struct sk_buff, len)); + /* s %w1,(%b1) */ + EMIT4_DISP(0x5b000000, REG_W1, BPF_REG_1, + offsetof(struct sk_buff, data_len)); + /* stg %w1,ST_OFF_HLEN(%r0,%r15) */ + EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, REG_15, STK_OFF_HLEN); + /* lg %skb_data,data_off(%b1) */ + EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0, + BPF_REG_1, offsetof(struct sk_buff, data)); +} + /* * Emit function prologue * * Save registers and create stack frame if necessary. * See stack frame layout desription in "bpf_jit.h"! */ -static void bpf_jit_prologue(struct bpf_jit *jit) +static void bpf_jit_prologue(struct bpf_jit *jit, bool is_classic) { if (jit->seen & SEEN_TAIL_CALL) { /* xc STK_OFF_TCCNT(4,%r15),STK_OFF_TCCNT(%r15) */ @@ -429,32 +442,21 @@ static void bpf_jit_prologue(struct bpf_jit *jit) EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, REG_15, 152); } - /* - * For SKB access %b1 contains the SKB pointer. For "bpf_jit.S" - * we store the SKB header length on the stack and the SKB data - * pointer in REG_SKB_DATA. - */ - if (jit->seen & SEEN_SKB) { - /* Header length: llgf %w1,(%b1) */ - EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_1, - offsetof(struct sk_buff, len)); - /* s %w1,(%b1) */ - EMIT4_DISP(0x5b000000, REG_W1, BPF_REG_1, - offsetof(struct sk_buff, data_len)); - /* stg %w1,ST_OFF_HLEN(%r0,%r15) */ + if (jit->seen & SEEN_SKB) + emit_load_skb_data_hlen(jit); + if (jit->seen & SEEN_SKB_CHANGE) + /* stg %b1,ST_OFF_SKBP(%r0,%r15) */ EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, REG_15, - STK_OFF_HLEN); - /* lg %skb_data,data_off(%b1) */ - EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0, - BPF_REG_1, offsetof(struct sk_buff, data)); + STK_OFF_SKBP); + /* Clear A (%b0) and X (%b7) registers for converted BPF programs */ + if (is_classic) { + if (REG_SEEN(BPF_REG_A)) + /* lghi %ba,0 */ + EMIT4_IMM(0xa7090000, BPF_REG_A, 0); + if (REG_SEEN(BPF_REG_X)) + /* lghi %bx,0 */ + EMIT4_IMM(0xa7090000, BPF_REG_X, 0); } - /* BPF compatibility: clear A (%b0) and X (%b7) registers */ - if (REG_SEEN(BPF_REG_A)) - /* lghi %ba,0 */ - EMIT4_IMM(0xa7090000, BPF_REG_A, 0); - if (REG_SEEN(BPF_REG_X)) - /* lghi %bx,0 */ - EMIT4_IMM(0xa7090000, BPF_REG_X, 0); } /* @@ -976,12 +978,19 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i REG_SET_SEEN(BPF_REG_5); jit->seen |= SEEN_FUNC; /* lg %w1,(%l) */ - EMIT6_DISP(0xe3000000, 0x0004, REG_W1, REG_0, REG_L, - EMIT_CONST_U64(func)); + EMIT6_DISP_LH(0xe3000000, 0x0004, REG_W1, REG_0, REG_L, + EMIT_CONST_U64(func)); /* basr %r14,%w1 */ EMIT2(0x0d00, REG_14, REG_W1); /* lgr %b0,%r2: load return value into %b0 */ EMIT4(0xb9040000, BPF_REG_0, REG_2); + if (bpf_helper_changes_skb_data((void *)func)) { + jit->seen |= SEEN_SKB_CHANGE; + /* lg %b1,ST_OFF_SKBP(%r15) */ + EMIT6_DISP_LH(0xe3000000, 0x0004, BPF_REG_1, REG_0, + REG_15, STK_OFF_SKBP); + emit_load_skb_data_hlen(jit); + } break; } case BPF_JMP | BPF_CALL | BPF_X: @@ -1023,7 +1032,7 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i MAX_TAIL_CALL_CNT, 0, 0x2); /* - * prog = array->prog[index]; + * prog = array->ptrs[index]; * if (prog == NULL) * goto out; */ @@ -1032,7 +1041,7 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i EMIT6_DISP_LH(0xeb000000, 0x000d, REG_1, BPF_REG_3, REG_0, 3); /* lg %r1,prog(%b2,%r1) */ EMIT6_DISP_LH(0xe3000000, 0x0004, REG_1, BPF_REG_2, - REG_1, offsetof(struct bpf_array, prog)); + REG_1, offsetof(struct bpf_array, ptrs)); /* clgij %r1,0,0x8,label0 */ EMIT6_PCREL_IMM_LABEL(0xec000000, 0x007d, REG_1, 0, 0, 0x8); @@ -1236,7 +1245,7 @@ static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp) jit->lit = jit->lit_start; jit->prg = 0; - bpf_jit_prologue(jit); + bpf_jit_prologue(jit, bpf_prog_was_classic(fp)); for (i = 0; i < fp->len; i += insn_count) { insn_count = bpf_jit_insn(jit, fp, i); if (insn_count < 0) diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index 17c04c7269e7..7ef12a3ace3a 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -409,7 +409,7 @@ int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) /* Request MSI interrupts */ hwirq = 0; - list_for_each_entry(msi, &pdev->msi_list, list) { + for_each_pci_msi_entry(msi, pdev) { rc = -EIO; irq = irq_alloc_desc(0); /* Alloc irq on node 0 */ if (irq < 0) @@ -435,7 +435,7 @@ int arch_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) return (msi_vecs == nvec) ? 0 : msi_vecs; out_msi: - list_for_each_entry(msi, &pdev->msi_list, list) { + for_each_pci_msi_entry(msi, pdev) { if (hwirq-- == 0) break; irq_set_msi_desc(msi->irq, NULL); @@ -465,7 +465,7 @@ void arch_teardown_msi_irqs(struct pci_dev *pdev) return; /* Release MSI interrupts */ - list_for_each_entry(msi, &pdev->msi_list, list) { + for_each_pci_msi_entry(msi, pdev) { if (msi->msi_attrib.is_msix) __pci_msix_desc_mask_irq(msi, 1); else diff --git a/arch/s390/pci/pci_dma.c b/arch/s390/pci/pci_dma.c index 42b76580c8b8..37505b8b4093 100644 --- a/arch/s390/pci/pci_dma.c +++ b/arch/s390/pci/pci_dma.c @@ -262,16 +262,6 @@ static void dma_free_iommu(struct zpci_dev *zdev, unsigned long offset, int size spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, flags); } -int dma_set_mask(struct device *dev, u64 mask) -{ - if (!dev->dma_mask || !dma_supported(dev, mask)) - return -EIO; - - *dev->dma_mask = mask; - return 0; -} -EXPORT_SYMBOL_GPL(dma_set_mask); - static dma_addr_t s390_dma_map_pages(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction direction, diff --git a/arch/score/kernel/time.c b/arch/score/kernel/time.c index 24770cd9b473..679b8d7b0350 100644 --- a/arch/score/kernel/time.c +++ b/arch/score/kernel/time.c @@ -55,31 +55,20 @@ static int score_timer_set_next_event(unsigned long delta, return 0; } -static void score_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *evdev) +static int score_timer_set_periodic(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - outl((TMR_M_PERIODIC | TMR_IE_ENABLE), P_TIMER0_CTRL); - outl(SYSTEM_CLOCK/HZ, P_TIMER0_PRELOAD); - outl(inl(P_TIMER0_CTRL) | TMR_ENABLE, P_TIMER0_CTRL); - break; - case CLOCK_EVT_MODE_ONESHOT: - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_RESUME: - case CLOCK_EVT_MODE_UNUSED: - break; - default: - BUG(); - } + outl((TMR_M_PERIODIC | TMR_IE_ENABLE), P_TIMER0_CTRL); + outl(SYSTEM_CLOCK / HZ, P_TIMER0_PRELOAD); + outl(inl(P_TIMER0_CTRL) | TMR_ENABLE, P_TIMER0_CTRL); + return 0; } static struct clock_event_device score_clockevent = { - .name = "score_clockevent", - .features = CLOCK_EVT_FEAT_PERIODIC, - .shift = 16, - .set_next_event = score_timer_set_next_event, - .set_mode = score_timer_set_mode, + .name = "score_clockevent", + .features = CLOCK_EVT_FEAT_PERIODIC, + .shift = 16, + .set_next_event = score_timer_set_next_event, + .set_state_periodic = score_timer_set_periodic, }; void __init time_init(void) diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index 50057fed819d..d514df7e04dd 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -602,6 +602,7 @@ source kernel/Kconfig.hz config KEXEC bool "kexec system call (EXPERIMENTAL)" depends on SUPERH32 && MMU + select KEXEC_CORE help kexec is a system call that implements the ability to shutdown your current kernel, and to start another kernel. It is like a reboot diff --git a/arch/sh/boards/mach-se/7343/irq.c b/arch/sh/boards/mach-se/7343/irq.c index 1087dba9b015..6f97a8f0d0d6 100644 --- a/arch/sh/boards/mach-se/7343/irq.c +++ b/arch/sh/boards/mach-se/7343/irq.c @@ -31,7 +31,7 @@ struct irq_domain *se7343_irq_domain; static void se7343_irq_demux(unsigned int irq, struct irq_desc *desc) { - struct irq_data *data = irq_get_irq_data(irq); + struct irq_data *data = irq_desc_get_irq_data(desc); struct irq_chip *chip = irq_data_get_irq_chip(data); unsigned long mask; int bit; diff --git a/arch/sh/boards/mach-se/7722/irq.c b/arch/sh/boards/mach-se/7722/irq.c index 00e699232621..60aebd14ccf8 100644 --- a/arch/sh/boards/mach-se/7722/irq.c +++ b/arch/sh/boards/mach-se/7722/irq.c @@ -30,7 +30,7 @@ struct irq_domain *se7722_irq_domain; static void se7722_irq_demux(unsigned int irq, struct irq_desc *desc) { - struct irq_data *data = irq_get_irq_data(irq); + struct irq_data *data = irq_desc_get_irq_data(desc); struct irq_chip *chip = irq_data_get_irq_chip(data); unsigned long mask; int bit; diff --git a/arch/sh/boards/mach-se/7724/irq.c b/arch/sh/boards/mach-se/7724/irq.c index 5d1d3ec9a6cd..9f2033898652 100644 --- a/arch/sh/boards/mach-se/7724/irq.c +++ b/arch/sh/boards/mach-se/7724/irq.c @@ -92,8 +92,9 @@ static struct irq_chip se7724_irq_chip __read_mostly = { .irq_unmask = enable_se7724_irq, }; -static void se7724_irq_demux(unsigned int irq, struct irq_desc *desc) +static void se7724_irq_demux(unsigned int __irq, struct irq_desc *desc) { + unsigned int irq = irq_desc_get_irq(desc); struct fpga_irq set = get_fpga_irq(irq); unsigned short intv = __raw_readw(set.sraddr); unsigned int ext_irq = set.base; diff --git a/arch/sh/boards/mach-x3proto/gpio.c b/arch/sh/boards/mach-x3proto/gpio.c index f035a7ac6456..24555c364d5b 100644 --- a/arch/sh/boards/mach-x3proto/gpio.c +++ b/arch/sh/boards/mach-x3proto/gpio.c @@ -62,7 +62,7 @@ static int x3proto_gpio_to_irq(struct gpio_chip *chip, unsigned gpio) static void x3proto_gpio_irq_handler(unsigned int irq, struct irq_desc *desc) { - struct irq_data *data = irq_get_irq_data(irq); + struct irq_data *data = irq_desc_get_irq_data(desc); struct irq_chip *chip = irq_data_get_irq_chip(data); unsigned long mask; int pin; diff --git a/arch/sh/boot/compressed/misc.c b/arch/sh/boot/compressed/misc.c index 95470a472d2c..208a9753ab38 100644 --- a/arch/sh/boot/compressed/misc.c +++ b/arch/sh/boot/compressed/misc.c @@ -132,7 +132,7 @@ void decompress_kernel(void) puts("Uncompressing Linux... "); cache_control(CACHE_ENABLE); - decompress(input_data, input_len, NULL, NULL, output, NULL, error); + __decompress(input_data, input_len, NULL, NULL, output, 0, NULL, error); cache_control(CACHE_DISABLE); puts("Ok, booting the kernel.\n"); } diff --git a/arch/sh/include/asm/atomic-grb.h b/arch/sh/include/asm/atomic-grb.h index 97a5fda83450..b94df40e5f2d 100644 --- a/arch/sh/include/asm/atomic-grb.h +++ b/arch/sh/include/asm/atomic-grb.h @@ -48,47 +48,12 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -static inline void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - int tmp; - unsigned int _mask = ~mask; - - __asm__ __volatile__ ( - " .align 2 \n\t" - " mova 1f, r0 \n\t" /* r0 = end point */ - " mov r15, r1 \n\t" /* r1 = saved sp */ - " mov #-6, r15 \n\t" /* LOGIN: r15 = size */ - " mov.l @%1, %0 \n\t" /* load old value */ - " and %2, %0 \n\t" /* add */ - " mov.l %0, @%1 \n\t" /* store new value */ - "1: mov r1, r15 \n\t" /* LOGOUT */ - : "=&r" (tmp), - "+r" (v) - : "r" (_mask) - : "memory" , "r0", "r1"); -} - -static inline void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - int tmp; - - __asm__ __volatile__ ( - " .align 2 \n\t" - " mova 1f, r0 \n\t" /* r0 = end point */ - " mov r15, r1 \n\t" /* r1 = saved sp */ - " mov #-6, r15 \n\t" /* LOGIN: r15 = size */ - " mov.l @%1, %0 \n\t" /* load old value */ - " or %2, %0 \n\t" /* or */ - " mov.l %0, @%1 \n\t" /* store new value */ - "1: mov r1, r15 \n\t" /* LOGOUT */ - : "=&r" (tmp), - "+r" (v) - : "r" (mask) - : "memory" , "r0", "r1"); -} - #endif /* __ASM_SH_ATOMIC_GRB_H */ diff --git a/arch/sh/include/asm/atomic-irq.h b/arch/sh/include/asm/atomic-irq.h index 61d107523f06..23fcdad5773e 100644 --- a/arch/sh/include/asm/atomic-irq.h +++ b/arch/sh/include/asm/atomic-irq.h @@ -37,27 +37,12 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add, +=) ATOMIC_OPS(sub, -=) +ATOMIC_OP(and, &=) +ATOMIC_OP(or, |=) +ATOMIC_OP(xor, ^=) #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -static inline void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - unsigned long flags; - - raw_local_irq_save(flags); - v->counter &= ~mask; - raw_local_irq_restore(flags); -} - -static inline void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - unsigned long flags; - - raw_local_irq_save(flags); - v->counter |= mask; - raw_local_irq_restore(flags); -} - #endif /* __ASM_SH_ATOMIC_IRQ_H */ diff --git a/arch/sh/include/asm/atomic-llsc.h b/arch/sh/include/asm/atomic-llsc.h index 8575dccb9ef7..33d34b16d4d6 100644 --- a/arch/sh/include/asm/atomic-llsc.h +++ b/arch/sh/include/asm/atomic-llsc.h @@ -52,37 +52,12 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP -static inline void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ - unsigned long tmp; - - __asm__ __volatile__ ( -"1: movli.l @%2, %0 ! atomic_clear_mask \n" -" and %1, %0 \n" -" movco.l %0, @%2 \n" -" bf 1b \n" - : "=&z" (tmp) - : "r" (~mask), "r" (&v->counter) - : "t"); -} - -static inline void atomic_set_mask(unsigned int mask, atomic_t *v) -{ - unsigned long tmp; - - __asm__ __volatile__ ( -"1: movli.l @%2, %0 ! atomic_set_mask \n" -" or %1, %0 \n" -" movco.l %0, @%2 \n" -" bf 1b \n" - : "=&z" (tmp) - : "r" (mask), "r" (&v->counter) - : "t"); -} - #endif /* __ASM_SH_ATOMIC_LLSC_H */ diff --git a/arch/sh/include/asm/dma-mapping.h b/arch/sh/include/asm/dma-mapping.h index b437f2c780b8..a3745a3fe029 100644 --- a/arch/sh/include/asm/dma-mapping.h +++ b/arch/sh/include/asm/dma-mapping.h @@ -9,86 +9,13 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev) return dma_ops; } -#include +#define DMA_ERROR_CODE 0 + #include -static inline int dma_supported(struct device *dev, u64 mask) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - - if (ops->dma_supported) - return ops->dma_supported(dev, mask); - - return 1; -} - -static inline int dma_set_mask(struct device *dev, u64 mask) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - - if (!dev->dma_mask || !dma_supported(dev, mask)) - return -EIO; - if (ops->set_dma_mask) - return ops->set_dma_mask(dev, mask); - - *dev->dma_mask = mask; - - return 0; -} - void dma_cache_sync(struct device *dev, void *vaddr, size_t size, enum dma_data_direction dir); -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) -#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) - -static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - - debug_dma_mapping_error(dev, dma_addr); - if (ops->mapping_error) - return ops->mapping_error(dev, dma_addr); - - return dma_addr == 0; -} - -#define dma_alloc_coherent(d,s,h,f) dma_alloc_attrs(d,s,h,f,NULL) - -static inline void *dma_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - void *memory; - - if (dma_alloc_from_coherent(dev, size, dma_handle, &memory)) - return memory; - if (!ops->alloc) - return NULL; - - memory = ops->alloc(dev, size, dma_handle, gfp, attrs); - debug_dma_alloc_coherent(dev, size, *dma_handle, memory); - - return memory; -} - -#define dma_free_coherent(d,s,c,h) dma_free_attrs(d,s,c,h,NULL) - -static inline void dma_free_attrs(struct device *dev, size_t size, - void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - - if (dma_release_from_coherent(dev, get_order(size), vaddr)) - return; - - debug_dma_free_coherent(dev, size, vaddr, dma_handle); - if (ops->free) - ops->free(dev, size, vaddr, dma_handle, attrs); -} - /* arch/sh/mm/consistent.c */ extern void *dma_generic_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, gfp_t flag, diff --git a/arch/sh/include/asm/ftrace.h b/arch/sh/include/asm/ftrace.h index e79fb6ebaa42..1f157b86eaa7 100644 --- a/arch/sh/include/asm/ftrace.h +++ b/arch/sh/include/asm/ftrace.h @@ -9,7 +9,7 @@ #ifndef __ASSEMBLY__ extern void mcount(void); -#define MCOUNT_ADDR ((long)(mcount)) +#define MCOUNT_ADDR ((unsigned long)(mcount)) #ifdef CONFIG_DYNAMIC_FTRACE #define CALL_ADDR ((long)(ftrace_call)) diff --git a/arch/sh/include/asm/io.h b/arch/sh/include/asm/io.h index 93ec9066dbef..3280a6bfa503 100644 --- a/arch/sh/include/asm/io.h +++ b/arch/sh/include/asm/io.h @@ -342,6 +342,7 @@ ioremap_cache(phys_addr_t offset, unsigned long size) { return __ioremap_mode(offset, size, PAGE_KERNEL); } +#define ioremap_cache ioremap_cache #ifdef CONFIG_HAVE_IOREMAP_PROT static inline void __iomem * diff --git a/arch/sh/kernel/cpu/sh4a/clock-sh7724.c b/arch/sh/kernel/cpu/sh4a/clock-sh7724.c index c187b9579c21..f27c618de527 100644 --- a/arch/sh/kernel/cpu/sh4a/clock-sh7724.c +++ b/arch/sh/kernel/cpu/sh4a/clock-sh7724.c @@ -343,7 +343,7 @@ static struct clk_lookup lookups[] = { CLKDEV_CON_ID("2ddmac0", &mstp_clks[HWBLK_2DDMAC]), CLKDEV_DEV_ID("sh_fsi.0", &mstp_clks[HWBLK_SPU]), CLKDEV_CON_ID("jpu0", &mstp_clks[HWBLK_JPU]), - CLKDEV_DEV_ID("sh-vou.0", &mstp_clks[HWBLK_VOU]), + CLKDEV_DEV_ID("sh-vou", &mstp_clks[HWBLK_VOU]), CLKDEV_CON_ID("beu0", &mstp_clks[HWBLK_BEU0]), CLKDEV_DEV_ID("sh_mobile_ceu.0", &mstp_clks[HWBLK_CEU0]), CLKDEV_CON_ID("veu0", &mstp_clks[HWBLK_VEU0]), diff --git a/arch/sh/kernel/irq.c b/arch/sh/kernel/irq.c index eb10ff84015c..6c0378c0b8b5 100644 --- a/arch/sh/kernel/irq.c +++ b/arch/sh/kernel/irq.c @@ -227,16 +227,17 @@ void migrate_irqs(void) for_each_active_irq(irq) { struct irq_data *data = irq_get_irq_data(irq); - if (data->node == cpu) { - unsigned int newcpu = cpumask_any_and(data->affinity, + if (irq_data_get_node(data) == cpu) { + struct cpumask *mask = irq_data_get_affinity_mask(data); + unsigned int newcpu = cpumask_any_and(mask, cpu_online_mask); if (newcpu >= nr_cpu_ids) { pr_info_ratelimited("IRQ%u no longer affine to CPU%u\n", irq, cpu); - cpumask_setall(data->affinity); + cpumask_setall(mask); } - irq_set_affinity(irq, data->affinity); + irq_set_affinity(irq, mask); } } } diff --git a/arch/sh/kernel/localtimer.c b/arch/sh/kernel/localtimer.c index b880a7e2ace7..cbb7d4636ec0 100644 --- a/arch/sh/kernel/localtimer.c +++ b/arch/sh/kernel/localtimer.c @@ -39,11 +39,6 @@ void local_timer_interrupt(void) irq_exit(); } -static void dummy_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *clk) -{ -} - void local_timer_setup(unsigned int cpu) { struct clock_event_device *clk = &per_cpu(local_clockevent, cpu); @@ -54,7 +49,6 @@ void local_timer_setup(unsigned int cpu) CLOCK_EVT_FEAT_DUMMY; clk->rating = 400; clk->mult = 1; - clk->set_mode = dummy_timer_set_mode; clk->broadcast = smp_timer_broadcast; clk->cpumask = cpumask_of(cpu); diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c index 2790b6a64157..75491862d900 100644 --- a/arch/sh/mm/init.c +++ b/arch/sh/mm/init.c @@ -485,10 +485,10 @@ void free_initrd_mem(unsigned long start, unsigned long end) #endif #ifdef CONFIG_MEMORY_HOTPLUG -int arch_add_memory(int nid, u64 start, u64 size) +int arch_add_memory(int nid, u64 start, u64 size, bool for_device) { pg_data_t *pgdat; - unsigned long start_pfn = start >> PAGE_SHIFT; + unsigned long start_pfn = PFN_DOWN(start); unsigned long nr_pages = size >> PAGE_SHIFT; int ret; @@ -496,7 +496,8 @@ int arch_add_memory(int nid, u64 start, u64 size) /* We only have ZONE_NORMAL, so this is easy.. */ ret = __add_pages(nid, pgdat->node_zones + - zone_for_memory(nid, start, size, ZONE_NORMAL), + zone_for_memory(nid, start, size, ZONE_NORMAL, + for_device), start_pfn, nr_pages); if (unlikely(ret)) printk("%s: Failed, __add_pages() == %d\n", __func__, ret); @@ -517,7 +518,7 @@ EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid); #ifdef CONFIG_MEMORY_HOTREMOVE int arch_remove_memory(u64 start, u64 size) { - unsigned long start_pfn = start >> PAGE_SHIFT; + unsigned long start_pfn = PFN_DOWN(start); unsigned long nr_pages = size >> PAGE_SHIFT; struct zone *zone; int ret; diff --git a/arch/sh/mm/numa.c b/arch/sh/mm/numa.c index bce52ba66206..05713d190247 100644 --- a/arch/sh/mm/numa.c +++ b/arch/sh/mm/numa.c @@ -33,8 +33,8 @@ void __init setup_bootmem_node(int nid, unsigned long start, unsigned long end) /* Don't allow bogus node assignment */ BUG_ON(nid >= MAX_NUMNODES || nid <= 0); - start_pfn = start >> PAGE_SHIFT; - end_pfn = end >> PAGE_SHIFT; + start_pfn = PFN_DOWN(start); + end_pfn = PFN_DOWN(end); pmb_bolt_mapping((unsigned long)__va(start), start, end - start, PAGE_KERNEL); diff --git a/arch/sparc/include/asm/atomic_32.h b/arch/sparc/include/asm/atomic_32.h index 0e69b7e7a439..7dcbebbcaec6 100644 --- a/arch/sparc/include/asm/atomic_32.h +++ b/arch/sparc/include/asm/atomic_32.h @@ -17,10 +17,12 @@ #include #include - #define ATOMIC_INIT(i) { (i) } int atomic_add_return(int, atomic_t *); +void atomic_and(int, atomic_t *); +void atomic_or(int, atomic_t *); +void atomic_xor(int, atomic_t *); int atomic_cmpxchg(atomic_t *, int, int); int atomic_xchg(atomic_t *, int); int __atomic_add_unless(atomic_t *, int, int); diff --git a/arch/sparc/include/asm/atomic_64.h b/arch/sparc/include/asm/atomic_64.h index 4082749913ce..917084ace49d 100644 --- a/arch/sparc/include/asm/atomic_64.h +++ b/arch/sparc/include/asm/atomic_64.h @@ -33,6 +33,10 @@ long atomic64_##op##_return(long, atomic64_t *); ATOMIC_OPS(add) ATOMIC_OPS(sub) +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP diff --git a/arch/sparc/include/asm/barrier_64.h b/arch/sparc/include/asm/barrier_64.h index 809941e33e12..14a928601657 100644 --- a/arch/sparc/include/asm/barrier_64.h +++ b/arch/sparc/include/asm/barrier_64.h @@ -60,12 +60,12 @@ do { __asm__ __volatile__("ba,pt %%xcc, 1f\n\t" \ do { \ compiletime_assert_atomic_type(*p); \ barrier(); \ - ACCESS_ONCE(*p) = (v); \ + WRITE_ONCE(*p, v); \ } while (0) #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ barrier(); \ ___p1; \ diff --git a/arch/sparc/include/asm/dma-mapping.h b/arch/sparc/include/asm/dma-mapping.h index 7e064c68c5ec..a21da597b0b5 100644 --- a/arch/sparc/include/asm/dma-mapping.h +++ b/arch/sparc/include/asm/dma-mapping.h @@ -7,11 +7,9 @@ #define DMA_ERROR_CODE (~(dma_addr_t)0x0) +#define HAVE_ARCH_DMA_SUPPORTED 1 int dma_supported(struct device *dev, u64 mask); -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) -#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) - static inline void dma_cache_sync(struct device *dev, void *vaddr, size_t size, enum dma_data_direction dir) { @@ -39,39 +37,7 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev) return dma_ops; } -#include - -#define dma_alloc_coherent(d,s,h,f) dma_alloc_attrs(d,s,h,f,NULL) - -static inline void *dma_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - void *cpu_addr; - - cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs); - debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr); - return cpu_addr; -} - -#define dma_free_coherent(d,s,c,h) dma_free_attrs(d,s,c,h,NULL) - -static inline void dma_free_attrs(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t dma_handle, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - - debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); - ops->free(dev, size, cpu_addr, dma_handle, attrs); -} - -static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - debug_dma_mapping_error(dev, dma_addr); - return (dma_addr == DMA_ERROR_CODE); -} +#define HAVE_ARCH_DMA_SET_MASK 1 static inline int dma_set_mask(struct device *dev, u64 mask) { @@ -86,4 +52,6 @@ static inline int dma_set_mask(struct device *dev, u64 mask) return -EINVAL; } +#include + #endif diff --git a/arch/sparc/include/asm/ftrace.h b/arch/sparc/include/asm/ftrace.h index 9ec94ad116fb..3192a8e42fd6 100644 --- a/arch/sparc/include/asm/ftrace.h +++ b/arch/sparc/include/asm/ftrace.h @@ -2,7 +2,7 @@ #define _ASM_SPARC64_FTRACE #ifdef CONFIG_MCOUNT -#define MCOUNT_ADDR ((long)(_mcount)) +#define MCOUNT_ADDR ((unsigned long)(_mcount)) #define MCOUNT_INSN_SIZE 4 /* sizeof mcount call */ #ifndef __ASSEMBLY__ diff --git a/arch/sparc/include/asm/jump_label.h b/arch/sparc/include/asm/jump_label.h index cc9b04a2b11b..62d0354d1727 100644 --- a/arch/sparc/include/asm/jump_label.h +++ b/arch/sparc/include/asm/jump_label.h @@ -7,16 +7,33 @@ #define JUMP_LABEL_NOP_SIZE 4 -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { - asm_volatile_goto("1:\n\t" - "nop\n\t" - "nop\n\t" - ".pushsection __jump_table, \"aw\"\n\t" - ".align 4\n\t" - ".word 1b, %l[l_yes], %c0\n\t" - ".popsection \n\t" - : : "i" (key) : : l_yes); + asm_volatile_goto("1:\n\t" + "nop\n\t" + "nop\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".align 4\n\t" + ".word 1b, %l[l_yes], %c0\n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); + + return false; +l_yes: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("1:\n\t" + "b %l[l_yes]\n\t" + "nop\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".align 4\n\t" + ".word 1b, %l[l_yes], %c0\n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); + return false; l_yes: return true; diff --git a/arch/sparc/include/asm/pgtable_32.h b/arch/sparc/include/asm/pgtable_32.h index f06b36a00a3b..91b963a887b7 100644 --- a/arch/sparc/include/asm/pgtable_32.h +++ b/arch/sparc/include/asm/pgtable_32.h @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/sparc/include/uapi/asm/pstate.h b/arch/sparc/include/uapi/asm/pstate.h index 4b6b998afd99..cf832e14aa05 100644 --- a/arch/sparc/include/uapi/asm/pstate.h +++ b/arch/sparc/include/uapi/asm/pstate.h @@ -88,7 +88,7 @@ #define VERS_MAXTL _AC(0x000000000000ff00,UL) /* Max Trap Level. */ #define VERS_MAXWIN _AC(0x000000000000001f,UL) /* Max RegWindow Idx.*/ -/* Compatability Feature Register (%asr26), SPARC-T4 and later */ +/* Compatibility Feature Register (%asr26), SPARC-T4 and later */ #define CFR_AES _AC(0x0000000000000001,UL) /* Supports AES opcodes */ #define CFR_DES _AC(0x0000000000000002,UL) /* Supports DES opcodes */ #define CFR_KASUMI _AC(0x0000000000000004,UL) /* Supports KASUMI opcodes */ diff --git a/arch/sparc/kernel/irq_64.c b/arch/sparc/kernel/irq_64.c index 4033c23bdfa6..e22416ce56ea 100644 --- a/arch/sparc/kernel/irq_64.c +++ b/arch/sparc/kernel/irq_64.c @@ -210,21 +210,21 @@ struct irq_handler_data { static inline unsigned int irq_data_to_handle(struct irq_data *data) { - struct irq_handler_data *ihd = data->handler_data; + struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data); return ihd->dev_handle; } static inline unsigned int irq_data_to_ino(struct irq_data *data) { - struct irq_handler_data *ihd = data->handler_data; + struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data); return ihd->dev_ino; } static inline unsigned long irq_data_to_sysino(struct irq_data *data) { - struct irq_handler_data *ihd = data->handler_data; + struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data); return ihd->sysino; } @@ -370,13 +370,15 @@ static int irq_choose_cpu(unsigned int irq, const struct cpumask *affinity) static void sun4u_irq_enable(struct irq_data *data) { - struct irq_handler_data *handler_data = data->handler_data; + struct irq_handler_data *handler_data; + handler_data = irq_data_get_irq_handler_data(data); if (likely(handler_data)) { unsigned long cpuid, imap, val; unsigned int tid; - cpuid = irq_choose_cpu(data->irq, data->affinity); + cpuid = irq_choose_cpu(data->irq, + irq_data_get_affinity_mask(data)); imap = handler_data->imap; tid = sun4u_compute_tid(imap, cpuid); @@ -393,8 +395,9 @@ static void sun4u_irq_enable(struct irq_data *data) static int sun4u_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force) { - struct irq_handler_data *handler_data = data->handler_data; + struct irq_handler_data *handler_data; + handler_data = irq_data_get_irq_handler_data(data); if (likely(handler_data)) { unsigned long cpuid, imap, val; unsigned int tid; @@ -438,15 +441,17 @@ static void sun4u_irq_disable(struct irq_data *data) static void sun4u_irq_eoi(struct irq_data *data) { - struct irq_handler_data *handler_data = data->handler_data; + struct irq_handler_data *handler_data; + handler_data = irq_data_get_irq_handler_data(data); if (likely(handler_data)) upa_writeq(ICLR_IDLE, handler_data->iclr); } static void sun4v_irq_enable(struct irq_data *data) { - unsigned long cpuid = irq_choose_cpu(data->irq, data->affinity); + unsigned long cpuid = irq_choose_cpu(data->irq, + irq_data_get_affinity_mask(data)); unsigned int ino = irq_data_to_sysino(data); int err; @@ -508,7 +513,7 @@ static void sun4v_virq_enable(struct irq_data *data) unsigned long cpuid; int err; - cpuid = irq_choose_cpu(data->irq, data->affinity); + cpuid = irq_choose_cpu(data->irq, irq_data_get_affinity_mask(data)); err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid); if (err != HV_EOK) @@ -881,8 +886,8 @@ void fixup_irqs(void) if (desc->action && !irqd_is_per_cpu(data)) { if (data->chip->irq_set_affinity) data->chip->irq_set_affinity(data, - data->affinity, - false); + irq_data_get_affinity_mask(data), + false); } raw_spin_unlock_irqrestore(&desc->lock, flags); } diff --git a/arch/sparc/kernel/jump_label.c b/arch/sparc/kernel/jump_label.c index 48565c11e82a..59bbeff55024 100644 --- a/arch/sparc/kernel/jump_label.c +++ b/arch/sparc/kernel/jump_label.c @@ -16,7 +16,7 @@ void arch_jump_label_transform(struct jump_entry *entry, u32 val; u32 *insn = (u32 *) (unsigned long) entry->code; - if (type == JUMP_LABEL_ENABLE) { + if (type == JUMP_LABEL_JMP) { s32 off = (s32)entry->target - (s32)entry->code; #ifdef CONFIG_SPARC64 diff --git a/arch/sparc/kernel/leon_kernel.c b/arch/sparc/kernel/leon_kernel.c index 9bbb8f2bbfcc..0299f052a2ef 100644 --- a/arch/sparc/kernel/leon_kernel.c +++ b/arch/sparc/kernel/leon_kernel.c @@ -126,7 +126,7 @@ static int leon_set_affinity(struct irq_data *data, const struct cpumask *dest, int oldcpu, newcpu; mask = (unsigned long)data->chip_data; - oldcpu = irq_choose_cpu(data->affinity); + oldcpu = irq_choose_cpu(irq_data_get_affinity_mask(data)); newcpu = irq_choose_cpu(dest); if (oldcpu == newcpu) @@ -149,7 +149,7 @@ static void leon_unmask_irq(struct irq_data *data) int cpu; mask = (unsigned long)data->chip_data; - cpu = irq_choose_cpu(data->affinity); + cpu = irq_choose_cpu(irq_data_get_affinity_mask(data)); spin_lock_irqsave(&leon_irq_lock, flags); oldmask = LEON3_BYPASS_LOAD_PA(LEON_IMASK(cpu)); LEON3_BYPASS_STORE_PA(LEON_IMASK(cpu), (oldmask | mask)); @@ -162,7 +162,7 @@ static void leon_mask_irq(struct irq_data *data) int cpu; mask = (unsigned long)data->chip_data; - cpu = irq_choose_cpu(data->affinity); + cpu = irq_choose_cpu(irq_data_get_affinity_mask(data)); spin_lock_irqsave(&leon_irq_lock, flags); oldmask = LEON3_BYPASS_LOAD_PA(LEON_IMASK(cpu)); LEON3_BYPASS_STORE_PA(LEON_IMASK(cpu), (oldmask & ~mask)); diff --git a/arch/sparc/kernel/pci.c b/arch/sparc/kernel/pci.c index 3a0e1a986bfe..b91d7f146175 100644 --- a/arch/sparc/kernel/pci.c +++ b/arch/sparc/kernel/pci.c @@ -231,8 +231,7 @@ static void pci_parse_of_addrs(struct platform_device *op, res = &dev->resource[(i - PCI_BASE_ADDRESS_0) >> 2]; } else if (i == dev->rom_base_reg) { res = &dev->resource[PCI_ROM_RESOURCE]; - flags |= IORESOURCE_READONLY | IORESOURCE_CACHEABLE - | IORESOURCE_SIZEALIGN; + flags |= IORESOURCE_READONLY | IORESOURCE_SIZEALIGN; } else { printk(KERN_ERR "PCI: bad cfg reg num 0x%x\n", i); continue; @@ -914,7 +913,7 @@ int arch_setup_msi_irq(struct pci_dev *pdev, struct msi_desc *desc) void arch_teardown_msi_irq(unsigned int irq) { struct msi_desc *entry = irq_get_msi_desc(irq); - struct pci_dev *pdev = entry->dev; + struct pci_dev *pdev = msi_desc_to_pci_dev(entry); struct pci_pbm_info *pbm = pdev->dev.archdata.host_controller; if (pbm->teardown_msi_irq) diff --git a/arch/sparc/kernel/sun4d_irq.c b/arch/sparc/kernel/sun4d_irq.c index a1bb2675b280..a87d0e47c168 100644 --- a/arch/sparc/kernel/sun4d_irq.c +++ b/arch/sparc/kernel/sun4d_irq.c @@ -188,7 +188,7 @@ void sun4d_handler_irq(unsigned int pil, struct pt_regs *regs) static void sun4d_mask_irq(struct irq_data *data) { - struct sun4d_handler_data *handler_data = data->handler_data; + struct sun4d_handler_data *handler_data = irq_data_get_irq_handler_data(data); unsigned int real_irq; #ifdef CONFIG_SMP int cpuid = handler_data->cpuid; @@ -206,7 +206,7 @@ static void sun4d_mask_irq(struct irq_data *data) static void sun4d_unmask_irq(struct irq_data *data) { - struct sun4d_handler_data *handler_data = data->handler_data; + struct sun4d_handler_data *handler_data = irq_data_get_irq_handler_data(data); unsigned int real_irq; #ifdef CONFIG_SMP int cpuid = handler_data->cpuid; diff --git a/arch/sparc/kernel/sun4m_irq.c b/arch/sparc/kernel/sun4m_irq.c index 8bb3b3fddea7..da737c712fa8 100644 --- a/arch/sparc/kernel/sun4m_irq.c +++ b/arch/sparc/kernel/sun4m_irq.c @@ -188,9 +188,10 @@ static unsigned long sun4m_imask[0x50] = { static void sun4m_mask_irq(struct irq_data *data) { - struct sun4m_handler_data *handler_data = data->handler_data; + struct sun4m_handler_data *handler_data; int cpu = smp_processor_id(); + handler_data = irq_data_get_irq_handler_data(data); if (handler_data->mask) { unsigned long flags; @@ -206,9 +207,10 @@ static void sun4m_mask_irq(struct irq_data *data) static void sun4m_unmask_irq(struct irq_data *data) { - struct sun4m_handler_data *handler_data = data->handler_data; + struct sun4m_handler_data *handler_data; int cpu = smp_processor_id(); + handler_data = irq_data_get_irq_handler_data(data); if (handler_data->mask) { unsigned long flags; diff --git a/arch/sparc/kernel/sun4m_smp.c b/arch/sparc/kernel/sun4m_smp.c index d3408e72d20c..278c40abce82 100644 --- a/arch/sparc/kernel/sun4m_smp.c +++ b/arch/sparc/kernel/sun4m_smp.c @@ -247,7 +247,7 @@ void smp4m_percpu_timer_interrupt(struct pt_regs *regs) ce = &per_cpu(sparc32_clockevent, cpu); - if (ce->mode & CLOCK_EVT_MODE_PERIODIC) + if (clockevent_state_periodic(ce)) sun4m_clear_profile_irq(cpu); else sparc_config.load_profile_irq(cpu, 0); /* Is this needless? */ diff --git a/arch/sparc/kernel/time_32.c b/arch/sparc/kernel/time_32.c index c9692f387cee..1affabc96b08 100644 --- a/arch/sparc/kernel/time_32.c +++ b/arch/sparc/kernel/time_32.c @@ -101,21 +101,18 @@ irqreturn_t notrace timer_interrupt(int dummy, void *dev_id) return IRQ_HANDLED; } -static void timer_ce_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int timer_ce_shutdown(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - case CLOCK_EVT_MODE_RESUME: - timer_ce_enabled = 1; - break; - case CLOCK_EVT_MODE_SHUTDOWN: - timer_ce_enabled = 0; - break; - default: - break; - } + timer_ce_enabled = 0; smp_mb(); + return 0; +} + +static int timer_ce_set_periodic(struct clock_event_device *evt) +{ + timer_ce_enabled = 1; + smp_mb(); + return 0; } static __init void setup_timer_ce(void) @@ -127,7 +124,9 @@ static __init void setup_timer_ce(void) ce->name = "timer_ce"; ce->rating = 100; ce->features = CLOCK_EVT_FEAT_PERIODIC; - ce->set_mode = timer_ce_set_mode; + ce->set_state_shutdown = timer_ce_shutdown; + ce->set_state_periodic = timer_ce_set_periodic; + ce->tick_resume = timer_ce_set_periodic; ce->cpumask = cpu_possible_mask; ce->shift = 32; ce->mult = div_sc(sparc_config.clock_rate, NSEC_PER_SEC, @@ -183,24 +182,20 @@ static __init int setup_timer_cs(void) } #ifdef CONFIG_SMP -static void percpu_ce_setup(enum clock_event_mode mode, - struct clock_event_device *evt) +static int percpu_ce_shutdown(struct clock_event_device *evt) { int cpu = cpumask_first(evt->cpumask); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - sparc_config.load_profile_irq(cpu, - SBUS_CLOCK_RATE / HZ); - break; - case CLOCK_EVT_MODE_ONESHOT: - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - sparc_config.load_profile_irq(cpu, 0); - break; - default: - break; - } + sparc_config.load_profile_irq(cpu, 0); + return 0; +} + +static int percpu_ce_set_periodic(struct clock_event_device *evt) +{ + int cpu = cpumask_first(evt->cpumask); + + sparc_config.load_profile_irq(cpu, SBUS_CLOCK_RATE / HZ); + return 0; } static int percpu_ce_set_next_event(unsigned long delta, @@ -224,7 +219,9 @@ void register_percpu_ce(int cpu) ce->name = "percpu_ce"; ce->rating = 200; ce->features = features; - ce->set_mode = percpu_ce_setup; + ce->set_state_shutdown = percpu_ce_shutdown; + ce->set_state_periodic = percpu_ce_set_periodic; + ce->set_state_oneshot = percpu_ce_shutdown; ce->set_next_event = percpu_ce_set_next_event; ce->cpumask = cpumask_of(cpu); ce->shift = 32; diff --git a/arch/sparc/kernel/time_64.c b/arch/sparc/kernel/time_64.c index 2e6035c0a8ca..c69b21e51efc 100644 --- a/arch/sparc/kernel/time_64.c +++ b/arch/sparc/kernel/time_64.c @@ -674,32 +674,19 @@ static int sparc64_next_event(unsigned long delta, return tick_ops->add_compare(delta) ? -ETIME : 0; } -static void sparc64_timer_setup(enum clock_event_mode mode, - struct clock_event_device *evt) +static int sparc64_timer_shutdown(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_ONESHOT: - case CLOCK_EVT_MODE_RESUME: - break; - - case CLOCK_EVT_MODE_SHUTDOWN: - tick_ops->disable_irq(); - break; - - case CLOCK_EVT_MODE_PERIODIC: - case CLOCK_EVT_MODE_UNUSED: - WARN_ON(1); - break; - } + tick_ops->disable_irq(); + return 0; } static struct clock_event_device sparc64_clockevent = { - .features = CLOCK_EVT_FEAT_ONESHOT, - .set_mode = sparc64_timer_setup, - .set_next_event = sparc64_next_event, - .rating = 100, - .shift = 30, - .irq = -1, + .features = CLOCK_EVT_FEAT_ONESHOT, + .set_state_shutdown = sparc64_timer_shutdown, + .set_next_event = sparc64_next_event, + .rating = 100, + .shift = 30, + .irq = -1, }; static DEFINE_PER_CPU(struct clock_event_device, sparc64_events); diff --git a/arch/sparc/lib/atomic32.c b/arch/sparc/lib/atomic32.c index 71cd65ab200c..b9d63c0a7aab 100644 --- a/arch/sparc/lib/atomic32.c +++ b/arch/sparc/lib/atomic32.c @@ -27,22 +27,38 @@ static DEFINE_SPINLOCK(dummy); #endif /* SMP */ -#define ATOMIC_OP(op, cop) \ +#define ATOMIC_OP_RETURN(op, c_op) \ int atomic_##op##_return(int i, atomic_t *v) \ { \ int ret; \ unsigned long flags; \ spin_lock_irqsave(ATOMIC_HASH(v), flags); \ \ - ret = (v->counter cop i); \ + ret = (v->counter c_op i); \ \ spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \ return ret; \ } \ EXPORT_SYMBOL(atomic_##op##_return); -ATOMIC_OP(add, +=) +#define ATOMIC_OP(op, c_op) \ +void atomic_##op(int i, atomic_t *v) \ +{ \ + unsigned long flags; \ + spin_lock_irqsave(ATOMIC_HASH(v), flags); \ + \ + v->counter c_op i; \ + \ + spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \ +} \ +EXPORT_SYMBOL(atomic_##op); +ATOMIC_OP_RETURN(add, +=) +ATOMIC_OP(and, &=) +ATOMIC_OP(or, |=) +ATOMIC_OP(xor, ^=) + +#undef ATOMIC_OP_RETURN #undef ATOMIC_OP int atomic_xchg(atomic_t *v, int new) diff --git a/arch/sparc/lib/atomic_64.S b/arch/sparc/lib/atomic_64.S index 05dac43907d1..d6b0363f345b 100644 --- a/arch/sparc/lib/atomic_64.S +++ b/arch/sparc/lib/atomic_64.S @@ -47,6 +47,9 @@ ENDPROC(atomic_##op##_return); ATOMIC_OPS(add) ATOMIC_OPS(sub) +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN @@ -84,6 +87,9 @@ ENDPROC(atomic64_##op##_return); ATOMIC64_OPS(add) ATOMIC64_OPS(sub) +ATOMIC64_OP(and) +ATOMIC64_OP(or) +ATOMIC64_OP(xor) #undef ATOMIC64_OPS #undef ATOMIC64_OP_RETURN diff --git a/arch/sparc/lib/ksyms.c b/arch/sparc/lib/ksyms.c index 8069ce12f20b..8eb454cfe05c 100644 --- a/arch/sparc/lib/ksyms.c +++ b/arch/sparc/lib/ksyms.c @@ -111,6 +111,9 @@ EXPORT_SYMBOL(atomic64_##op##_return); ATOMIC_OPS(add) ATOMIC_OPS(sub) +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN diff --git a/arch/sparc/net/bpf_jit_comp.c b/arch/sparc/net/bpf_jit_comp.c index 7931eeeb649a..f8b9f71b9a2b 100644 --- a/arch/sparc/net/bpf_jit_comp.c +++ b/arch/sparc/net/bpf_jit_comp.c @@ -807,7 +807,7 @@ cond_branch: f_offset = addrs[i + filter[i].jf]; } if (bpf_jit_enable > 1) - bpf_jit_dump(flen, proglen, pass, image); + bpf_jit_dump(flen, proglen, pass + 1, image); if (image) { bpf_flush_icache(image, image + proglen); diff --git a/arch/tile/Kconfig b/arch/tile/Kconfig index 9def1f52d03a..106c21bd7f44 100644 --- a/arch/tile/Kconfig +++ b/arch/tile/Kconfig @@ -32,6 +32,7 @@ config TILE select EDAC_SUPPORT select GENERIC_STRNCPY_FROM_USER select GENERIC_STRNLEN_USER + select HAVE_ARCH_SECCOMP_FILTER # FIXME: investigate whether we need/want these options. # select HAVE_IOREMAP_PROT @@ -204,6 +205,7 @@ source "kernel/Kconfig.hz" config KEXEC bool "kexec system call" + select KEXEC_CORE ---help--- kexec is a system call that implements the ability to shutdown your current kernel, and to start another kernel. It is like a reboot @@ -221,6 +223,22 @@ config COMPAT If enabled, the kernel will support running TILE-Gx binaries that were built with the -m32 option. +config SECCOMP + bool "Enable seccomp to safely compute untrusted bytecode" + depends on PROC_FS + help + This kernel feature is useful for number crunching applications + that may need to compute untrusted bytecode during their + execution. By using pipes or other transports made available to + the process as file descriptors supporting the read/write + syscalls, it's possible to isolate those applications in + their own address space using seccomp. Once seccomp is + enabled via prctl, it cannot be disabled and the task is only + allowed to execute a few safe syscalls defined by each seccomp + mode. + + If unsure, say N. + config SYSVIPC_COMPAT def_bool y depends on COMPAT && SYSVIPC diff --git a/arch/tile/include/asm/Kbuild b/arch/tile/include/asm/Kbuild index d8a843163471..ba35c41c71ff 100644 --- a/arch/tile/include/asm/Kbuild +++ b/arch/tile/include/asm/Kbuild @@ -28,6 +28,7 @@ generic-y += poll.h generic-y += posix_types.h generic-y += preempt.h generic-y += resource.h +generic-y += seccomp.h generic-y += sembuf.h generic-y += serial.h generic-y += shmbuf.h diff --git a/arch/tile/include/asm/atomic_32.h b/arch/tile/include/asm/atomic_32.h index 1b109fad9fff..d320ce253d86 100644 --- a/arch/tile/include/asm/atomic_32.h +++ b/arch/tile/include/asm/atomic_32.h @@ -34,6 +34,19 @@ static inline void atomic_add(int i, atomic_t *v) _atomic_xchg_add(&v->counter, i); } +#define ATOMIC_OP(op) \ +unsigned long _atomic_##op(volatile unsigned long *p, unsigned long mask); \ +static inline void atomic_##op(int i, atomic_t *v) \ +{ \ + _atomic_##op((unsigned long *)&v->counter, i); \ +} + +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) + +#undef ATOMIC_OP + /** * atomic_add_return - add integer and return * @v: pointer of type atomic_t @@ -113,6 +126,17 @@ static inline void atomic64_add(long long i, atomic64_t *v) _atomic64_xchg_add(&v->counter, i); } +#define ATOMIC64_OP(op) \ +long long _atomic64_##op(long long *v, long long n); \ +static inline void atomic64_##op(long long i, atomic64_t *v) \ +{ \ + _atomic64_##op(&v->counter, i); \ +} + +ATOMIC64_OP(and) +ATOMIC64_OP(or) +ATOMIC64_OP(xor) + /** * atomic64_add_return - add integer and return * @v: pointer of type atomic64_t @@ -225,6 +249,7 @@ extern struct __get_user __atomic_xchg_add(volatile int *p, int *lock, int n); extern struct __get_user __atomic_xchg_add_unless(volatile int *p, int *lock, int o, int n); extern struct __get_user __atomic_or(volatile int *p, int *lock, int n); +extern struct __get_user __atomic_and(volatile int *p, int *lock, int n); extern struct __get_user __atomic_andn(volatile int *p, int *lock, int n); extern struct __get_user __atomic_xor(volatile int *p, int *lock, int n); extern long long __atomic64_cmpxchg(volatile long long *p, int *lock, @@ -234,6 +259,9 @@ extern long long __atomic64_xchg_add(volatile long long *p, int *lock, long long n); extern long long __atomic64_xchg_add_unless(volatile long long *p, int *lock, long long o, long long n); +extern long long __atomic64_and(volatile long long *p, int *lock, long long n); +extern long long __atomic64_or(volatile long long *p, int *lock, long long n); +extern long long __atomic64_xor(volatile long long *p, int *lock, long long n); /* Return failure from the atomic wrappers. */ struct __get_user __atomic_bad_address(int __user *addr); diff --git a/arch/tile/include/asm/atomic_64.h b/arch/tile/include/asm/atomic_64.h index 0496970cef82..096a56d6ead4 100644 --- a/arch/tile/include/asm/atomic_64.h +++ b/arch/tile/include/asm/atomic_64.h @@ -58,6 +58,26 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) return oldval; } +static inline void atomic_and(int i, atomic_t *v) +{ + __insn_fetchand4((void *)&v->counter, i); +} + +static inline void atomic_or(int i, atomic_t *v) +{ + __insn_fetchor4((void *)&v->counter, i); +} + +static inline void atomic_xor(int i, atomic_t *v) +{ + int guess, oldval = v->counter; + do { + guess = oldval; + __insn_mtspr(SPR_CMPEXCH_VALUE, guess); + oldval = __insn_cmpexch4(&v->counter, guess ^ i); + } while (guess != oldval); +} + /* Now the true 64-bit operations. */ #define ATOMIC64_INIT(i) { (i) } @@ -91,6 +111,26 @@ static inline long atomic64_add_unless(atomic64_t *v, long a, long u) return oldval != u; } +static inline void atomic64_and(long i, atomic64_t *v) +{ + __insn_fetchand((void *)&v->counter, i); +} + +static inline void atomic64_or(long i, atomic64_t *v) +{ + __insn_fetchor((void *)&v->counter, i); +} + +static inline void atomic64_xor(long i, atomic64_t *v) +{ + long guess, oldval = v->counter; + do { + guess = oldval; + __insn_mtspr(SPR_CMPEXCH_VALUE, guess); + oldval = __insn_cmpexch(&v->counter, guess ^ i); + } while (guess != oldval); +} + #define atomic64_sub_return(i, v) atomic64_add_return(-(i), (v)) #define atomic64_sub(i, v) atomic64_add(-(i), (v)) #define atomic64_inc_return(v) atomic64_add_return(1, (v)) diff --git a/arch/tile/include/asm/dma-mapping.h b/arch/tile/include/asm/dma-mapping.h index 1eae359d8315..96ac6cce4a32 100644 --- a/arch/tile/include/asm/dma-mapping.h +++ b/arch/tile/include/asm/dma-mapping.h @@ -59,8 +59,6 @@ static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr) static inline void dma_mark_clean(void *addr, size_t size) {} -#include - static inline void set_dma_ops(struct device *dev, struct dma_map_ops *ops) { dev->archdata.dma_ops = ops; @@ -74,18 +72,9 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) return addr + size - 1 <= *dev->dma_mask; } -static inline int -dma_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - debug_dma_mapping_error(dev, dma_addr); - return get_dma_ops(dev)->mapping_error(dev, dma_addr); -} +#define HAVE_ARCH_DMA_SET_MASK 1 -static inline int -dma_supported(struct device *dev, u64 mask) -{ - return get_dma_ops(dev)->dma_supported(dev, mask); -} +#include static inline int dma_set_mask(struct device *dev, u64 mask) @@ -116,36 +105,6 @@ dma_set_mask(struct device *dev, u64 mask) return 0; } -static inline void *dma_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) -{ - struct dma_map_ops *dma_ops = get_dma_ops(dev); - void *cpu_addr; - - cpu_addr = dma_ops->alloc(dev, size, dma_handle, flag, attrs); - - debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr); - - return cpu_addr; -} - -static inline void dma_free_attrs(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t dma_handle, - struct dma_attrs *attrs) -{ - struct dma_map_ops *dma_ops = get_dma_ops(dev); - - debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); - - dma_ops->free(dev, size, cpu_addr, dma_handle, attrs); -} - -#define dma_alloc_coherent(d, s, h, f) dma_alloc_attrs(d, s, h, f, NULL) -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_attrs(d, s, h, f, NULL) -#define dma_free_coherent(d, s, v, h) dma_free_attrs(d, s, v, h, NULL) -#define dma_free_noncoherent(d, s, v, h) dma_free_attrs(d, s, v, h, NULL) - /* * dma_alloc_noncoherent() is #defined to return coherent memory, * so there's no need to do any flushing here. diff --git a/arch/tile/include/asm/elf.h b/arch/tile/include/asm/elf.h index 41d9878a9686..c505d77e4d06 100644 --- a/arch/tile/include/asm/elf.h +++ b/arch/tile/include/asm/elf.h @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -30,9 +31,6 @@ typedef unsigned long elf_greg_t; #define ELF_NGREG (sizeof(struct pt_regs) / sizeof(elf_greg_t)) typedef elf_greg_t elf_gregset_t[ELF_NGREG]; -#define EM_TILEPRO 188 -#define EM_TILEGX 191 - /* Provide a nominal data structure. */ #define ELF_NFPREG 0 typedef double elf_fpreg_t; diff --git a/arch/tile/include/asm/syscall.h b/arch/tile/include/asm/syscall.h index 9644b88f133d..373d73064ea1 100644 --- a/arch/tile/include/asm/syscall.h +++ b/arch/tile/include/asm/syscall.h @@ -20,6 +20,8 @@ #include #include +#include +#include #include /* The array of function pointers for syscalls. */ @@ -61,7 +63,15 @@ static inline void syscall_set_return_value(struct task_struct *task, struct pt_regs *regs, int error, long val) { - regs->regs[0] = (long) error ?: val; + if (error) { + /* R0 is the passed-in negative error, R1 is positive. */ + regs->regs[0] = error; + regs->regs[1] = -error; + } else { + /* R1 set to zero to indicate no error. */ + regs->regs[0] = val; + regs->regs[1] = 0; + } } static inline void syscall_get_arguments(struct task_struct *task, @@ -82,4 +92,20 @@ static inline void syscall_set_arguments(struct task_struct *task, memcpy(®s[i], args, n * sizeof(args[0])); } +/* + * We don't care about endianness (__AUDIT_ARCH_LE bit) here because + * tile has the same system calls both on little- and big- endian. + */ +static inline int syscall_get_arch(void) +{ + if (is_compat_task()) + return AUDIT_ARCH_TILEGX32; + +#ifdef CONFIG_TILEGX + return AUDIT_ARCH_TILEGX; +#else + return AUDIT_ARCH_TILEPRO; +#endif +} + #endif /* _ASM_TILE_SYSCALL_H */ diff --git a/arch/tile/include/uapi/arch/opcode_tilegx.h b/arch/tile/include/uapi/arch/opcode_tilegx.h index d76ff2db745e..9e46eaa847d4 100644 --- a/arch/tile/include/uapi/arch/opcode_tilegx.h +++ b/arch/tile/include/uapi/arch/opcode_tilegx.h @@ -830,11 +830,11 @@ enum ADDX_RRR_0_OPCODE_X0 = 2, ADDX_RRR_0_OPCODE_X1 = 2, ADDX_RRR_0_OPCODE_Y0 = 0, - ADDX_SPECIAL_0_OPCODE_Y1 = 0, + ADDX_RRR_0_OPCODE_Y1 = 0, ADD_RRR_0_OPCODE_X0 = 3, ADD_RRR_0_OPCODE_X1 = 3, ADD_RRR_0_OPCODE_Y0 = 1, - ADD_SPECIAL_0_OPCODE_Y1 = 1, + ADD_RRR_0_OPCODE_Y1 = 1, ANDI_IMM8_OPCODE_X0 = 3, ANDI_IMM8_OPCODE_X1 = 3, ANDI_OPCODE_Y0 = 2, @@ -995,6 +995,7 @@ enum LD4U_ADD_IMM8_OPCODE_X1 = 12, LD4U_OPCODE_Y2 = 2, LD4U_UNARY_OPCODE_X1 = 20, + LDNA_ADD_IMM8_OPCODE_X1 = 21, LDNA_UNARY_OPCODE_X1 = 21, LDNT1S_ADD_IMM8_OPCODE_X1 = 13, LDNT1S_UNARY_OPCODE_X1 = 22, @@ -1015,7 +1016,6 @@ enum LD_UNARY_OPCODE_X1 = 29, LNK_UNARY_OPCODE_X1 = 30, LNK_UNARY_OPCODE_Y1 = 14, - LWNA_ADD_IMM8_OPCODE_X1 = 21, MFSPR_IMM8_OPCODE_X1 = 22, MF_UNARY_OPCODE_X1 = 31, MM_BF_OPCODE_X0 = 7, diff --git a/arch/tile/kernel/intvec_32.S b/arch/tile/kernel/intvec_32.S index cdbda45a4e4b..fbbe2ea882ea 100644 --- a/arch/tile/kernel/intvec_32.S +++ b/arch/tile/kernel/intvec_32.S @@ -1224,6 +1224,7 @@ handle_syscall: jal do_syscall_trace_enter } FEEDBACK_REENTER(handle_syscall) + blz r0, .Lsyscall_sigreturn_skip /* * We always reload our registers from the stack at this diff --git a/arch/tile/kernel/intvec_64.S b/arch/tile/kernel/intvec_64.S index 800b91d3f9dc..58964d209d4d 100644 --- a/arch/tile/kernel/intvec_64.S +++ b/arch/tile/kernel/intvec_64.S @@ -1247,6 +1247,7 @@ handle_syscall: jal do_syscall_trace_enter } FEEDBACK_REENTER(handle_syscall) + bltz r0, .Lsyscall_sigreturn_skip /* * We always reload our registers from the stack at this diff --git a/arch/tile/kernel/pci_gx.c b/arch/tile/kernel/pci_gx.c index b1df847d0686..b3f73fd764a3 100644 --- a/arch/tile/kernel/pci_gx.c +++ b/arch/tile/kernel/pci_gx.c @@ -304,11 +304,12 @@ static struct irq_chip tilegx_legacy_irq_chip = { * to Linux which just calls handle_level_irq() after clearing the * MAC INTx Assert status bit associated with this interrupt. */ -static void trio_handle_level_irq(unsigned int irq, struct irq_desc *desc) +static void trio_handle_level_irq(unsigned int __irq, struct irq_desc *desc) { struct pci_controller *controller = irq_desc_get_handler_data(desc); gxio_trio_context_t *trio_context = controller->trio; uint64_t intx = (uint64_t)irq_desc_get_chip_data(desc); + unsigned int irq = irq_desc_get_irq(desc); int mac = controller->mac; unsigned int reg_offset; uint64_t level_mask; @@ -1442,7 +1443,7 @@ static struct pci_ops tile_cfg_ops = { /* MSI support starts here. */ static unsigned int tilegx_msi_startup(struct irq_data *d) { - if (d->msi_desc) + if (irq_data_get_msi_desc(d)) pci_msi_unmask_irq(d); return 0; diff --git a/arch/tile/kernel/ptrace.c b/arch/tile/kernel/ptrace.c index f84eed8243da..bdc126faf741 100644 --- a/arch/tile/kernel/ptrace.c +++ b/arch/tile/kernel/ptrace.c @@ -262,6 +262,9 @@ int do_syscall_trace_enter(struct pt_regs *regs) if (work & _TIF_NOHZ) user_exit(); + if (secure_computing() == -1) + return -1; + if (work & _TIF_SYSCALL_TRACE) { if (tracehook_report_syscall_entry(regs)) regs->regs[TREG_SYSCALL_NR] = -1; diff --git a/arch/tile/kernel/time.c b/arch/tile/kernel/time.c index 00178ecf9aea..178989e6d3e3 100644 --- a/arch/tile/kernel/time.c +++ b/arch/tile/kernel/time.c @@ -140,10 +140,10 @@ static int tile_timer_set_next_event(unsigned long ticks, * Whenever anyone tries to change modes, we just mask interrupts * and wait for the next event to get set. */ -static void tile_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int tile_timer_shutdown(struct clock_event_device *evt) { arch_local_irq_mask_now(INT_TILE_TIMER); + return 0; } /* @@ -157,7 +157,9 @@ static DEFINE_PER_CPU(struct clock_event_device, tile_timer) = { .rating = 100, .irq = -1, .set_next_event = tile_timer_set_next_event, - .set_mode = tile_timer_set_mode, + .set_state_shutdown = tile_timer_shutdown, + .set_state_oneshot = tile_timer_shutdown, + .tick_resume = tile_timer_shutdown, }; void setup_tile_timer(void) diff --git a/arch/tile/kernel/vdso/Makefile b/arch/tile/kernel/vdso/Makefile index a025f63d54cd..c54fff37b5ff 100644 --- a/arch/tile/kernel/vdso/Makefile +++ b/arch/tile/kernel/vdso/Makefile @@ -54,7 +54,7 @@ $(obj)/built-in.o: $(obj)/vdso-syms.o $(obj)/built-in.o: ld_flags += -R $(obj)/vdso-syms.o SYSCFLAGS_vdso.so.dbg = -shared -s -Wl,-soname=linux-vdso.so.1 \ - $(call cc-ldoption, -Wl$(comma)--hash-style=sysv) + $(call cc-ldoption, -Wl$(comma)--hash-style=both) SYSCFLAGS_vdso_syms.o = -r $(obj)/vdso-syms.o: $(src)/vdso.lds $(obj)/vrt_sigreturn.o FORCE $(call if_changed,vdsold) @@ -113,6 +113,6 @@ $(obj)/vrt_sigreturn32.o: $(obj)/vrt_sigreturn.S $(obj)/vdso32.o: $(obj)/vdso32.so SYSCFLAGS_vdso32.so.dbg = -m32 -shared -s -Wl,-soname=linux-vdso32.so.1 \ - $(call cc-ldoption, -Wl$(comma)--hash-style=sysv) + $(call cc-ldoption, -Wl$(comma)--hash-style=both) $(obj)/vdso32.so.dbg: $(src)/vdso.lds $(obj-vdso32) $(call if_changed,vdsold) diff --git a/arch/tile/lib/atomic_32.c b/arch/tile/lib/atomic_32.c index c89b211fd9e7..298df1e9912a 100644 --- a/arch/tile/lib/atomic_32.c +++ b/arch/tile/lib/atomic_32.c @@ -94,6 +94,12 @@ unsigned long _atomic_or(volatile unsigned long *p, unsigned long mask) } EXPORT_SYMBOL(_atomic_or); +unsigned long _atomic_and(volatile unsigned long *p, unsigned long mask) +{ + return __atomic_and((int *)p, __atomic_setup(p), mask).val; +} +EXPORT_SYMBOL(_atomic_and); + unsigned long _atomic_andn(volatile unsigned long *p, unsigned long mask) { return __atomic_andn((int *)p, __atomic_setup(p), mask).val; @@ -136,6 +142,23 @@ long long _atomic64_cmpxchg(long long *v, long long o, long long n) } EXPORT_SYMBOL(_atomic64_cmpxchg); +long long _atomic64_and(long long *v, long long n) +{ + return __atomic64_and(v, __atomic_setup(v), n); +} +EXPORT_SYMBOL(_atomic64_and); + +long long _atomic64_or(long long *v, long long n) +{ + return __atomic64_or(v, __atomic_setup(v), n); +} +EXPORT_SYMBOL(_atomic64_or); + +long long _atomic64_xor(long long *v, long long n) +{ + return __atomic64_xor(v, __atomic_setup(v), n); +} +EXPORT_SYMBOL(_atomic64_xor); /* * If any of the atomic or futex routines hit a bad address (not in diff --git a/arch/tile/lib/atomic_asm_32.S b/arch/tile/lib/atomic_asm_32.S index 6bda3132cd61..f611265633d6 100644 --- a/arch/tile/lib/atomic_asm_32.S +++ b/arch/tile/lib/atomic_asm_32.S @@ -178,6 +178,7 @@ atomic_op _xchg_add, 32, "add r24, r22, r2" atomic_op _xchg_add_unless, 32, \ "sne r26, r22, r2; { bbns r26, 3f; add r24, r22, r3 }" atomic_op _or, 32, "or r24, r22, r2" +atomic_op _and, 32, "and r24, r22, r2" atomic_op _andn, 32, "nor r2, r2, zero; and r24, r22, r2" atomic_op _xor, 32, "xor r24, r22, r2" @@ -191,6 +192,9 @@ atomic_op 64_xchg_add_unless, 64, \ { bbns r26, 3f; add r24, r22, r4 }; \ { bbns r27, 3f; add r25, r23, r5 }; \ slt_u r26, r24, r22; add r25, r25, r26" +atomic_op 64_or, 64, "{ or r24, r22, r2; or r25, r23, r3 }" +atomic_op 64_and, 64, "{ and r24, r22, r2; and r25, r23, r3 }" +atomic_op 64_xor, 64, "{ xor r24, r22, r2; xor r25, r23, r3 }" jrp lr /* happy backtracer */ diff --git a/arch/tile/mm/init.c b/arch/tile/mm/init.c index 5bd252e3fdc5..d4e1fc41d06d 100644 --- a/arch/tile/mm/init.c +++ b/arch/tile/mm/init.c @@ -863,7 +863,7 @@ void __init mem_init(void) * memory to the highmem for now. */ #ifndef CONFIG_NEED_MULTIPLE_NODES -int arch_add_memory(u64 start, u64 size) +int arch_add_memory(u64 start, u64 size, bool for_device) { struct pglist_data *pgdata = &contig_page_data; struct zone *zone = pgdata->node_zones + MAX_NR_ZONES-1; diff --git a/arch/um/kernel/time.c b/arch/um/kernel/time.c index 117568d4f64a..5af441efb377 100644 --- a/arch/um/kernel/time.c +++ b/arch/um/kernel/time.c @@ -22,23 +22,16 @@ void timer_handler(int sig, struct siginfo *unused_si, struct uml_pt_regs *regs) local_irq_restore(flags); } -static void itimer_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int itimer_shutdown(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - set_interval(); - break; + disable_timer(); + return 0; +} - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_ONESHOT: - disable_timer(); - break; - - case CLOCK_EVT_MODE_RESUME: - break; - } +static int itimer_set_periodic(struct clock_event_device *evt) +{ + set_interval(); + return 0; } static int itimer_next_event(unsigned long delta, @@ -48,14 +41,17 @@ static int itimer_next_event(unsigned long delta, } static struct clock_event_device itimer_clockevent = { - .name = "itimer", - .rating = 250, - .cpumask = cpu_all_mask, - .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .set_mode = itimer_set_mode, - .set_next_event = itimer_next_event, - .shift = 32, - .irq = 0, + .name = "itimer", + .rating = 250, + .cpumask = cpu_all_mask, + .features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT, + .set_state_shutdown = itimer_shutdown, + .set_state_periodic = itimer_set_periodic, + .set_state_oneshot = itimer_shutdown, + .set_next_event = itimer_next_event, + .shift = 32, + .irq = 0, }; static irqreturn_t um_timer(int irq, void *dev) diff --git a/arch/unicore32/boot/compressed/misc.c b/arch/unicore32/boot/compressed/misc.c index 176d5bda3559..5c65dfee278c 100644 --- a/arch/unicore32/boot/compressed/misc.c +++ b/arch/unicore32/boot/compressed/misc.c @@ -119,8 +119,8 @@ unsigned long decompress_kernel(unsigned long output_start, output_ptr = get_unaligned_le32(tmp); arch_decomp_puts("Uncompressing Linux..."); - decompress(input_data, input_data_end - input_data, NULL, NULL, - output_data, NULL, error); + __decompress(input_data, input_data_end - input_data, NULL, NULL, + output_data, 0, NULL, error); arch_decomp_puts(" done, booting the kernel.\n"); return output_ptr; } diff --git a/arch/unicore32/include/asm/dma-mapping.h b/arch/unicore32/include/asm/dma-mapping.h index 366460a81796..8140e053ccd3 100644 --- a/arch/unicore32/include/asm/dma-mapping.h +++ b/arch/unicore32/include/asm/dma-mapping.h @@ -18,8 +18,6 @@ #include #include -#include - #include #include @@ -30,26 +28,6 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev) return &swiotlb_dma_map_ops; } -static inline int dma_supported(struct device *dev, u64 mask) -{ - struct dma_map_ops *dma_ops = get_dma_ops(dev); - - if (unlikely(dma_ops == NULL)) - return 0; - - return dma_ops->dma_supported(dev, mask); -} - -static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - struct dma_map_ops *dma_ops = get_dma_ops(dev); - - if (dma_ops->mapping_error) - return dma_ops->mapping_error(dev, dma_addr); - - return 0; -} - #include static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) @@ -72,41 +50,6 @@ static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr) static inline void dma_mark_clean(void *addr, size_t size) {} -static inline int dma_set_mask(struct device *dev, u64 dma_mask) -{ - if (!dev->dma_mask || !dma_supported(dev, dma_mask)) - return -EIO; - - *dev->dma_mask = dma_mask; - - return 0; -} - -#define dma_alloc_coherent(d,s,h,f) dma_alloc_attrs(d,s,h,f,NULL) - -static inline void *dma_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag, - struct dma_attrs *attrs) -{ - struct dma_map_ops *dma_ops = get_dma_ops(dev); - - return dma_ops->alloc(dev, size, dma_handle, flag, attrs); -} - -#define dma_free_coherent(d,s,c,h) dma_free_attrs(d,s,c,h,NULL) - -static inline void dma_free_attrs(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t dma_handle, - struct dma_attrs *attrs) -{ - struct dma_map_ops *dma_ops = get_dma_ops(dev); - - dma_ops->free(dev, size, cpu_addr, dma_handle, attrs); -} - -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) -#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) - static inline void dma_cache_sync(struct device *dev, void *vaddr, size_t size, enum dma_data_direction direction) { diff --git a/arch/unicore32/include/asm/memory.h b/arch/unicore32/include/asm/memory.h index debafc40200a..3bb0a29fd2d7 100644 --- a/arch/unicore32/include/asm/memory.h +++ b/arch/unicore32/include/asm/memory.h @@ -60,12 +60,6 @@ #define __phys_to_virt(x) ((x) - PHYS_OFFSET + PAGE_OFFSET) #endif -/* - * Convert a physical address to a Page Frame Number and back - */ -#define __phys_to_pfn(paddr) ((paddr) >> PAGE_SHIFT) -#define __pfn_to_phys(pfn) ((pfn) << PAGE_SHIFT) - /* * Convert a page to/from a physical address */ diff --git a/arch/unicore32/kernel/irq.c b/arch/unicore32/kernel/irq.c index 0be5ccd7ccd2..c53729d92e8d 100644 --- a/arch/unicore32/kernel/irq.c +++ b/arch/unicore32/kernel/irq.c @@ -112,10 +112,9 @@ static struct irq_chip puv3_low_gpio_chip = { * irq_controller_lock held, and IRQs disabled. Decode the IRQ * and call the handler. */ -static void -puv3_gpio_handler(unsigned int irq, struct irq_desc *desc) +static void puv3_gpio_handler(unsigned int __irq, struct irq_desc *desc) { - unsigned int mask; + unsigned int mask, irq; mask = readl(GPIO_GEDR); do { diff --git a/arch/unicore32/kernel/time.c b/arch/unicore32/kernel/time.c index d3824b2ff644..ac4c5449bb88 100644 --- a/arch/unicore32/kernel/time.c +++ b/arch/unicore32/kernel/time.c @@ -46,29 +46,20 @@ puv3_osmr0_set_next_event(unsigned long delta, struct clock_event_device *c) return (signed)(next - oscr) <= MIN_OSCR_DELTA ? -ETIME : 0; } -static void -puv3_osmr0_set_mode(enum clock_event_mode mode, struct clock_event_device *c) +static int puv3_osmr0_shutdown(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_ONESHOT: - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - writel(readl(OST_OIER) & ~OST_OIER_E0, OST_OIER); - writel(readl(OST_OSSR) & ~OST_OSSR_M0, OST_OSSR); - break; - - case CLOCK_EVT_MODE_RESUME: - case CLOCK_EVT_MODE_PERIODIC: - break; - } + writel(readl(OST_OIER) & ~OST_OIER_E0, OST_OIER); + writel(readl(OST_OSSR) & ~OST_OSSR_M0, OST_OSSR); + return 0; } static struct clock_event_device ckevt_puv3_osmr0 = { - .name = "osmr0", - .features = CLOCK_EVT_FEAT_ONESHOT, - .rating = 200, - .set_next_event = puv3_osmr0_set_next_event, - .set_mode = puv3_osmr0_set_mode, + .name = "osmr0", + .features = CLOCK_EVT_FEAT_ONESHOT, + .rating = 200, + .set_next_event = puv3_osmr0_set_next_event, + .set_state_shutdown = puv3_osmr0_shutdown, + .set_state_oneshot = puv3_osmr0_shutdown, }; static cycle_t puv3_read_oscr(struct clocksource *cs) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 48f7433dac6f..7aef2d52daa0 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -27,7 +27,8 @@ config X86 select ARCH_HAS_ELF_RANDOMIZE select ARCH_HAS_FAST_MULTIPLIER select ARCH_HAS_GCOV_PROFILE_ALL - select ARCH_HAS_PMEM_API + select ARCH_HAS_PMEM_API if X86_64 + select ARCH_HAS_MMIO_FLUSH select ARCH_HAS_SG_CHAIN select ARCH_HAVE_NMI_SAFE_CMPXCHG select ARCH_MIGHT_HAVE_ACPI_PDC if ACPI @@ -41,6 +42,7 @@ config X86 select ARCH_USE_CMPXCHG_LOCKREF if X86_64 select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_QUEUED_SPINLOCKS + select ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH if SMP select ARCH_WANTS_DYNAMIC_TASK_STRUCT select ARCH_WANT_FRAME_POINTERS select ARCH_WANT_IPC_PARSE_VERSION if X86_32 @@ -1449,10 +1451,14 @@ config ILLEGAL_POINTER_VALUE source "mm/Kconfig" +config X86_PMEM_LEGACY_DEVICE + bool + config X86_PMEM_LEGACY - bool "Support non-standard NVDIMMs and ADR protected memory" + tristate "Support non-standard NVDIMMs and ADR protected memory" depends on PHYS_ADDR_T_64BIT depends on BLK_DEV + select X86_PMEM_LEGACY_DEVICE select LIBNVDIMM help Treat memory marked using the non-standard e820 type of 12 as used @@ -1748,6 +1754,7 @@ source kernel/Kconfig.hz config KEXEC bool "kexec system call" + select KEXEC_CORE ---help--- kexec is a system call that implements the ability to shutdown your current kernel, and to start another kernel. It is like a reboot @@ -1764,8 +1771,8 @@ config KEXEC config KEXEC_FILE bool "kexec file based system call" + select KEXEC_CORE select BUILD_BIN2C - depends on KEXEC depends on X86_64 depends on CRYPTO=y depends on CRYPTO_SHA256=y diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c index f63797942bb5..79dac1758e7c 100644 --- a/arch/x86/boot/compressed/misc.c +++ b/arch/x86/boot/compressed/misc.c @@ -448,7 +448,8 @@ asmlinkage __visible void *decompress_kernel(void *rmode, memptr heap, #endif debug_putstr("\nDecompressing Linux... "); - decompress(input_data, input_len, NULL, NULL, output, NULL, error); + __decompress(input_data, input_len, NULL, NULL, output, output_len, + NULL, error); parse_elf(output); /* * 32-bit always performs relocations. 64-bit relocations are only diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S index 16ef02596db2..2d6b309c8e9a 100644 --- a/arch/x86/boot/header.S +++ b/arch/x86/boot/header.S @@ -414,7 +414,7 @@ xloadflags: # define XLF23 0 #endif -#if defined(CONFIG_X86_64) && defined(CONFIG_EFI) && defined(CONFIG_KEXEC) +#if defined(CONFIG_X86_64) && defined(CONFIG_EFI) && defined(CONFIG_KEXEC_CORE) # define XLF4 XLF_EFI_KEXEC #else # define XLF4 0 diff --git a/arch/x86/configs/x86_64_defconfig b/arch/x86/configs/x86_64_defconfig index 962297d244b3..cb5b3ab5beec 100644 --- a/arch/x86/configs/x86_64_defconfig +++ b/arch/x86/configs/x86_64_defconfig @@ -208,7 +208,6 @@ CONFIG_AGP_AMD64=y CONFIG_AGP_INTEL=y CONFIG_DRM=y CONFIG_DRM_I915=y -CONFIG_DRM_I915_KMS=y CONFIG_FB_MODE_HELPERS=y CONFIG_FB_TILEBLITTING=y CONFIG_FB_EFI=y diff --git a/arch/x86/crypto/ghash-clmulni-intel_glue.c b/arch/x86/crypto/ghash-clmulni-intel_glue.c index 64d7cf1b50e1..440df0c7a2ee 100644 --- a/arch/x86/crypto/ghash-clmulni-intel_glue.c +++ b/arch/x86/crypto/ghash-clmulni-intel_glue.c @@ -294,6 +294,7 @@ static struct ahash_alg ghash_async_alg = { .cra_name = "ghash", .cra_driver_name = "ghash-clmulni", .cra_priority = 400, + .cra_ctxsize = sizeof(struct ghash_async_ctx), .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, .cra_blocksize = GHASH_BLOCK_SIZE, .cra_type = &crypto_ahash_type, diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl index 25e3cf1cd8fd..7663c455b9f6 100644 --- a/arch/x86/entry/syscalls/syscall_32.tbl +++ b/arch/x86/entry/syscalls/syscall_32.tbl @@ -380,3 +380,5 @@ 371 i386 recvfrom sys_recvfrom compat_sys_recvfrom 372 i386 recvmsg sys_recvmsg compat_sys_recvmsg 373 i386 shutdown sys_shutdown +374 i386 userfaultfd sys_userfaultfd +375 i386 membarrier sys_membarrier diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index 9ef32d5f1b19..278842fdf1f6 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -329,6 +329,8 @@ 320 common kexec_file_load sys_kexec_file_load 321 common bpf sys_bpf 322 64 execveat stub_execveat +323 common userfaultfd sys_userfaultfd +324 common membarrier sys_membarrier # # x32-specific system call numbers start at 512 to avoid cache impact diff --git a/arch/x86/entry/vsyscall/vsyscall_64.c b/arch/x86/entry/vsyscall/vsyscall_64.c index 26a46f44e298..b160c0c6baed 100644 --- a/arch/x86/entry/vsyscall/vsyscall_64.c +++ b/arch/x86/entry/vsyscall/vsyscall_64.c @@ -277,7 +277,7 @@ static const char *gate_vma_name(struct vm_area_struct *vma) { return "[vsyscall]"; } -static struct vm_operations_struct gate_vma_ops = { +static const struct vm_operations_struct gate_vma_ops = { .name = gate_vma_name, }; static struct vm_area_struct gate_vma = { diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h index e9168955c42f..fb52aa644aab 100644 --- a/arch/x86/include/asm/atomic.h +++ b/arch/x86/include/asm/atomic.h @@ -182,6 +182,21 @@ static inline int atomic_xchg(atomic_t *v, int new) return xchg(&v->counter, new); } +#define ATOMIC_OP(op) \ +static inline void atomic_##op(int i, atomic_t *v) \ +{ \ + asm volatile(LOCK_PREFIX #op"l %1,%0" \ + : "+m" (v->counter) \ + : "ir" (i) \ + : "memory"); \ +} + +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) + +#undef ATOMIC_OP + /** * __atomic_add_unless - add unless the number is already a given value * @v: pointer of type atomic_t @@ -219,16 +234,6 @@ static __always_inline short int atomic_inc_short(short int *v) return *v; } -/* These are x86-specific, used by some header files */ -#define atomic_clear_mask(mask, addr) \ - asm volatile(LOCK_PREFIX "andl %0,%1" \ - : : "r" (~(mask)), "m" (*(addr)) : "memory") - -#define atomic_set_mask(mask, addr) \ - asm volatile(LOCK_PREFIX "orl %0,%1" \ - : : "r" ((unsigned)(mask)), "m" (*(addr)) \ - : "memory") - #ifdef CONFIG_X86_32 # include #else diff --git a/arch/x86/include/asm/atomic64_32.h b/arch/x86/include/asm/atomic64_32.h index b154de75c90c..a11c30b77fb5 100644 --- a/arch/x86/include/asm/atomic64_32.h +++ b/arch/x86/include/asm/atomic64_32.h @@ -313,4 +313,18 @@ static inline long long atomic64_dec_if_positive(atomic64_t *v) #undef alternative_atomic64 #undef __alternative_atomic64 +#define ATOMIC64_OP(op, c_op) \ +static inline void atomic64_##op(long long i, atomic64_t *v) \ +{ \ + long long old, c = 0; \ + while ((old = atomic64_cmpxchg(v, c, c c_op i)) != c) \ + c = old; \ +} + +ATOMIC64_OP(and, &) +ATOMIC64_OP(or, |) +ATOMIC64_OP(xor, ^) + +#undef ATOMIC64_OP + #endif /* _ASM_X86_ATOMIC64_32_H */ diff --git a/arch/x86/include/asm/atomic64_64.h b/arch/x86/include/asm/atomic64_64.h index b965f9e03f2a..50e33eff58de 100644 --- a/arch/x86/include/asm/atomic64_64.h +++ b/arch/x86/include/asm/atomic64_64.h @@ -220,4 +220,19 @@ static inline long atomic64_dec_if_positive(atomic64_t *v) return dec; } +#define ATOMIC64_OP(op) \ +static inline void atomic64_##op(long i, atomic64_t *v) \ +{ \ + asm volatile(LOCK_PREFIX #op"q %1,%0" \ + : "+m" (v->counter) \ + : "er" (i) \ + : "memory"); \ +} + +ATOMIC64_OP(and) +ATOMIC64_OP(or) +ATOMIC64_OP(xor) + +#undef ATOMIC64_OP + #endif /* _ASM_X86_ATOMIC64_64_H */ diff --git a/arch/x86/include/asm/barrier.h b/arch/x86/include/asm/barrier.h index 818cb8788225..0681d2532527 100644 --- a/arch/x86/include/asm/barrier.h +++ b/arch/x86/include/asm/barrier.h @@ -57,12 +57,12 @@ do { \ compiletime_assert_atomic_type(*p); \ smp_mb(); \ - ACCESS_ONCE(*p) = (v); \ + WRITE_ONCE(*p, v); \ } while (0) #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ smp_mb(); \ ___p1; \ @@ -74,12 +74,12 @@ do { \ do { \ compiletime_assert_atomic_type(*p); \ barrier(); \ - ACCESS_ONCE(*p) = (v); \ + WRITE_ONCE(*p, v); \ } while (0) #define smp_load_acquire(p) \ ({ \ - typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ barrier(); \ ___p1; \ diff --git a/arch/x86/include/asm/cacheflush.h b/arch/x86/include/asm/cacheflush.h index 9bf3ea14b9f0..e63aa38e85fb 100644 --- a/arch/x86/include/asm/cacheflush.h +++ b/arch/x86/include/asm/cacheflush.h @@ -89,6 +89,8 @@ int set_pages_rw(struct page *page, int numpages); void clflush_cache_range(void *addr, unsigned int size); +#define mmio_flush_range(addr, size) clflush_cache_range(addr, size) + #ifdef CONFIG_DEBUG_RODATA void mark_rodata_ro(void); extern const int rodata_test_data; @@ -109,75 +111,4 @@ static inline int rodata_test(void) } #endif -#ifdef ARCH_HAS_NOCACHE_UACCESS - -/** - * arch_memcpy_to_pmem - copy data to persistent memory - * @dst: destination buffer for the copy - * @src: source buffer for the copy - * @n: length of the copy in bytes - * - * Copy data to persistent memory media via non-temporal stores so that - * a subsequent arch_wmb_pmem() can flush cpu and memory controller - * write buffers to guarantee durability. - */ -static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src, - size_t n) -{ - int unwritten; - - /* - * We are copying between two kernel buffers, if - * __copy_from_user_inatomic_nocache() returns an error (page - * fault) we would have already reported a general protection fault - * before the WARN+BUG. - */ - unwritten = __copy_from_user_inatomic_nocache((void __force *) dst, - (void __user *) src, n); - if (WARN(unwritten, "%s: fault copying %p <- %p unwritten: %d\n", - __func__, dst, src, unwritten)) - BUG(); -} - -/** - * arch_wmb_pmem - synchronize writes to persistent memory - * - * After a series of arch_memcpy_to_pmem() operations this drains data - * from cpu write buffers and any platform (memory controller) buffers - * to ensure that written data is durable on persistent memory media. - */ -static inline void arch_wmb_pmem(void) -{ - /* - * wmb() to 'sfence' all previous writes such that they are - * architecturally visible to 'pcommit'. Note, that we've - * already arranged for pmem writes to avoid the cache via - * arch_memcpy_to_pmem(). - */ - wmb(); - pcommit_sfence(); -} - -static inline bool __arch_has_wmb_pmem(void) -{ -#ifdef CONFIG_X86_64 - /* - * We require that wmb() be an 'sfence', that is only guaranteed on - * 64-bit builds - */ - return static_cpu_has(X86_FEATURE_PCOMMIT); -#else - return false; -#endif -} -#else /* ARCH_HAS_NOCACHE_UACCESS i.e. ARCH=um */ -extern void arch_memcpy_to_pmem(void __pmem *dst, const void *src, size_t n); -extern void arch_wmb_pmem(void); - -static inline bool __arch_has_wmb_pmem(void) -{ - return false; -} -#endif - #endif /* _ASM_X86_CACHEFLUSH_H */ diff --git a/arch/x86/include/asm/dma-mapping.h b/arch/x86/include/asm/dma-mapping.h index 1f5b7287d1ad..953b7263f844 100644 --- a/arch/x86/include/asm/dma-mapping.h +++ b/arch/x86/include/asm/dma-mapping.h @@ -12,7 +12,6 @@ #include #include #include -#include #include #ifdef CONFIG_ISA @@ -41,24 +40,13 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev) #endif } -#include - -/* Make sure we keep the same behaviour */ -static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - debug_dma_mapping_error(dev, dma_addr); - if (ops->mapping_error) - return ops->mapping_error(dev, dma_addr); - - return (dma_addr == DMA_ERROR_CODE); -} - -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) -#define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) +bool arch_dma_alloc_attrs(struct device **dev, gfp_t *gfp); +#define arch_dma_alloc_attrs arch_dma_alloc_attrs +#define HAVE_ARCH_DMA_SUPPORTED 1 extern int dma_supported(struct device *hwdev, u64 mask); -extern int dma_set_mask(struct device *dev, u64 mask); + +#include extern void *dma_generic_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addr, gfp_t flag, @@ -125,16 +113,4 @@ static inline gfp_t dma_alloc_coherent_gfp_flags(struct device *dev, gfp_t gfp) return gfp; } -#define dma_alloc_coherent(d,s,h,f) dma_alloc_attrs(d,s,h,f,NULL) - -void * -dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, - gfp_t gfp, struct dma_attrs *attrs); - -#define dma_free_coherent(d,s,c,h) dma_free_attrs(d,s,c,h,NULL) - -void dma_free_attrs(struct device *dev, size_t size, - void *vaddr, dma_addr_t bus, - struct dma_attrs *attrs); - #endif diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h index f45acad3c4b6..24938852db30 100644 --- a/arch/x86/include/asm/ftrace.h +++ b/arch/x86/include/asm/ftrace.h @@ -3,9 +3,9 @@ #ifdef CONFIG_FUNCTION_TRACER #ifdef CC_USING_FENTRY -# define MCOUNT_ADDR ((long)(__fentry__)) +# define MCOUNT_ADDR ((unsigned long)(__fentry__)) #else -# define MCOUNT_ADDR ((long)(mcount)) +# define MCOUNT_ADDR ((unsigned long)(mcount)) #endif #define MCOUNT_INSN_SIZE 5 /* sizeof mcount call */ diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h index 6615032e19c8..1e3408e88604 100644 --- a/arch/x86/include/asm/hw_irq.h +++ b/arch/x86/include/asm/hw_irq.h @@ -182,10 +182,10 @@ extern char irq_entries_start[]; #define trace_irq_entries_start irq_entries_start #endif -#define VECTOR_UNDEFINED (-1) -#define VECTOR_RETRIGGERED (-2) +#define VECTOR_UNUSED NULL +#define VECTOR_RETRIGGERED ((void *)~0UL) -typedef int vector_irq_t[NR_VECTORS]; +typedef struct irq_desc* vector_irq_t[NR_VECTORS]; DECLARE_PER_CPU(vector_irq_t, vector_irq); #endif /* !ASSEMBLY_ */ diff --git a/arch/x86/include/asm/io.h b/arch/x86/include/asm/io.h index 7cfc085b6879..de25aad07853 100644 --- a/arch/x86/include/asm/io.h +++ b/arch/x86/include/asm/io.h @@ -250,12 +250,6 @@ static inline void flush_write_buffers(void) #endif } -static inline void __pmem *arch_memremap_pmem(resource_size_t offset, - unsigned long size) -{ - return (void __force __pmem *) ioremap_cache(offset, size); -} - #endif /* __KERNEL__ */ extern void native_io_delay(void); diff --git a/arch/x86/include/asm/irq.h b/arch/x86/include/asm/irq.h index 8008d06581c7..881b4768644a 100644 --- a/arch/x86/include/asm/irq.h +++ b/arch/x86/include/asm/irq.h @@ -36,7 +36,9 @@ extern void kvm_set_posted_intr_wakeup_handler(void (*handler)(void)); extern void (*x86_platform_ipi_callback)(void); extern void native_init_IRQ(void); -extern bool handle_irq(unsigned irq, struct pt_regs *regs); + +struct irq_desc; +extern bool handle_irq(struct irq_desc *desc, struct pt_regs *regs); extern __visible unsigned int do_IRQ(struct pt_regs *regs); diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h index a4c1cf7e93f8..5daeca3d0f9e 100644 --- a/arch/x86/include/asm/jump_label.h +++ b/arch/x86/include/asm/jump_label.h @@ -16,15 +16,32 @@ # define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC #endif -static __always_inline bool arch_static_branch(struct static_key *key) +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { asm_volatile_goto("1:" ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t" ".pushsection __jump_table, \"aw\" \n\t" _ASM_ALIGN "\n\t" - _ASM_PTR "1b, %l[l_yes], %c0 \n\t" + _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t" ".popsection \n\t" - : : "i" (key) : : l_yes); + : : "i" (key), "i" (branch) : : l_yes); + + return false; +l_yes: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm_volatile_goto("1:" + ".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t" + "2:\n\t" + ".pushsection __jump_table, \"aw\" \n\t" + _ASM_ALIGN "\n\t" + _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t" + ".popsection \n\t" + : : "i" (key), "i" (branch) : : l_yes); + return false; l_yes: return true; diff --git a/arch/x86/include/asm/kdebug.h b/arch/x86/include/asm/kdebug.h index 32ce71375b21..b130d59406fb 100644 --- a/arch/x86/include/asm/kdebug.h +++ b/arch/x86/include/asm/kdebug.h @@ -29,7 +29,7 @@ extern void show_trace(struct task_struct *t, struct pt_regs *regs, extern void __show_regs(struct pt_regs *regs, int all); extern unsigned long oops_begin(void); extern void oops_end(unsigned long, struct pt_regs *, int signr); -#ifdef CONFIG_KEXEC +#ifdef CONFIG_KEXEC_CORE extern int in_crash_kexec; #else /* no crash dump is ever in progress if no crash kernel can be kexec'd */ diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index fcd17c1fc0c6..c1c0a1c14344 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -184,6 +184,12 @@ #define MSR_PP1_ENERGY_STATUS 0x00000641 #define MSR_PP1_POLICY 0x00000642 +#define MSR_CONFIG_TDP_NOMINAL 0x00000648 +#define MSR_CONFIG_TDP_LEVEL_1 0x00000649 +#define MSR_CONFIG_TDP_LEVEL_2 0x0000064A +#define MSR_CONFIG_TDP_CONTROL 0x0000064B +#define MSR_TURBO_ACTIVATION_RATIO 0x0000064C + #define MSR_PKG_WEIGHTED_CORE_C0_RES 0x00000658 #define MSR_PKG_ANY_CORE_C0_RES 0x00000659 #define MSR_PKG_ANY_GFXE_C0_RES 0x0000065A diff --git a/arch/x86/include/asm/pmem.h b/arch/x86/include/asm/pmem.h new file mode 100644 index 000000000000..d8ce3ec816ab --- /dev/null +++ b/arch/x86/include/asm/pmem.h @@ -0,0 +1,153 @@ +/* + * Copyright(c) 2015 Intel Corporation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#ifndef __ASM_X86_PMEM_H__ +#define __ASM_X86_PMEM_H__ + +#include +#include +#include +#include + +#ifdef CONFIG_ARCH_HAS_PMEM_API +/** + * arch_memcpy_to_pmem - copy data to persistent memory + * @dst: destination buffer for the copy + * @src: source buffer for the copy + * @n: length of the copy in bytes + * + * Copy data to persistent memory media via non-temporal stores so that + * a subsequent arch_wmb_pmem() can flush cpu and memory controller + * write buffers to guarantee durability. + */ +static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src, + size_t n) +{ + int unwritten; + + /* + * We are copying between two kernel buffers, if + * __copy_from_user_inatomic_nocache() returns an error (page + * fault) we would have already reported a general protection fault + * before the WARN+BUG. + */ + unwritten = __copy_from_user_inatomic_nocache((void __force *) dst, + (void __user *) src, n); + if (WARN(unwritten, "%s: fault copying %p <- %p unwritten: %d\n", + __func__, dst, src, unwritten)) + BUG(); +} + +/** + * arch_wmb_pmem - synchronize writes to persistent memory + * + * After a series of arch_memcpy_to_pmem() operations this drains data + * from cpu write buffers and any platform (memory controller) buffers + * to ensure that written data is durable on persistent memory media. + */ +static inline void arch_wmb_pmem(void) +{ + /* + * wmb() to 'sfence' all previous writes such that they are + * architecturally visible to 'pcommit'. Note, that we've + * already arranged for pmem writes to avoid the cache via + * arch_memcpy_to_pmem(). + */ + wmb(); + pcommit_sfence(); +} + +/** + * __arch_wb_cache_pmem - write back a cache range with CLWB + * @vaddr: virtual start address + * @size: number of bytes to write back + * + * Write back a cache range using the CLWB (cache line write back) + * instruction. This function requires explicit ordering with an + * arch_wmb_pmem() call. This API is internal to the x86 PMEM implementation. + */ +static inline void __arch_wb_cache_pmem(void *vaddr, size_t size) +{ + u16 x86_clflush_size = boot_cpu_data.x86_clflush_size; + unsigned long clflush_mask = x86_clflush_size - 1; + void *vend = vaddr + size; + void *p; + + for (p = (void *)((unsigned long)vaddr & ~clflush_mask); + p < vend; p += x86_clflush_size) + clwb(p); +} + +/* + * copy_from_iter_nocache() on x86 only uses non-temporal stores for iovec + * iterators, so for other types (bvec & kvec) we must do a cache write-back. + */ +static inline bool __iter_needs_pmem_wb(struct iov_iter *i) +{ + return iter_is_iovec(i) == false; +} + +/** + * arch_copy_from_iter_pmem - copy data from an iterator to PMEM + * @addr: PMEM destination address + * @bytes: number of bytes to copy + * @i: iterator with source data + * + * Copy data from the iterator 'i' to the PMEM buffer starting at 'addr'. + * This function requires explicit ordering with an arch_wmb_pmem() call. + */ +static inline size_t arch_copy_from_iter_pmem(void __pmem *addr, size_t bytes, + struct iov_iter *i) +{ + void *vaddr = (void __force *)addr; + size_t len; + + /* TODO: skip the write-back by always using non-temporal stores */ + len = copy_from_iter_nocache(vaddr, bytes, i); + + if (__iter_needs_pmem_wb(i)) + __arch_wb_cache_pmem(vaddr, bytes); + + return len; +} + +/** + * arch_clear_pmem - zero a PMEM memory range + * @addr: virtual start address + * @size: number of bytes to zero + * + * Write zeros into the memory range starting at 'addr' for 'size' bytes. + * This function requires explicit ordering with an arch_wmb_pmem() call. + */ +static inline void arch_clear_pmem(void __pmem *addr, size_t size) +{ + void *vaddr = (void __force *)addr; + + /* TODO: implement the zeroing via non-temporal writes */ + if (size == PAGE_SIZE && ((unsigned long)vaddr & ~PAGE_MASK) == 0) + clear_page(vaddr); + else + memset(vaddr, 0, size); + + __arch_wb_cache_pmem(vaddr, size); +} + +static inline bool __arch_has_wmb_pmem(void) +{ + /* + * We require that wmb() be an 'sfence', that is only guaranteed on + * 64-bit builds + */ + return static_cpu_has(X86_FEATURE_PCOMMIT); +} +#endif /* CONFIG_ARCH_HAS_PMEM_API */ +#endif /* __ASM_X86_PMEM_H__ */ diff --git a/arch/x86/include/asm/qrwlock.h b/arch/x86/include/asm/qrwlock.h index ae0e241e228b..c537cbb038a7 100644 --- a/arch/x86/include/asm/qrwlock.h +++ b/arch/x86/include/asm/qrwlock.h @@ -2,16 +2,6 @@ #define _ASM_X86_QRWLOCK_H #include - -#ifndef CONFIG_X86_PPRO_FENCE -#define queue_write_unlock queue_write_unlock -static inline void queue_write_unlock(struct qrwlock *lock) -{ - barrier(); - ACCESS_ONCE(*(u8 *)&lock->cnts) = 0; -} -#endif - #include #endif /* _ASM_X86_QRWLOCK_H */ diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h index cd791948b286..6df2029405a3 100644 --- a/arch/x86/include/asm/tlbflush.h +++ b/arch/x86/include/asm/tlbflush.h @@ -261,6 +261,12 @@ static inline void reset_lazy_tlbstate(void) #endif /* SMP */ +/* Not inlined due to inc_irq_stat not being defined yet */ +#define flush_tlb_local() { \ + inc_irq_stat(irq_tlb_count); \ + local_flush_tlb(); \ +} + #ifndef CONFIG_PARAVIRT #define flush_tlb_others(mask, mm, start, end) \ native_flush_tlb_others(mask, mm, start, end) diff --git a/arch/x86/include/asm/xen/events.h b/arch/x86/include/asm/xen/events.h index 608a79d5a466..e6911caf5bbf 100644 --- a/arch/x86/include/asm/xen/events.h +++ b/arch/x86/include/asm/xen/events.h @@ -20,4 +20,15 @@ static inline int xen_irqs_disabled(struct pt_regs *regs) /* No need for a barrier -- XCHG is a barrier on x86. */ #define xchg_xen_ulong(ptr, val) xchg((ptr), (val)) +extern int xen_have_vector_callback; + +/* + * Events delivered via platform PCI interrupts are always + * routed to vcpu 0 and hence cannot be rebound. + */ +static inline bool xen_support_evtchn_rebind(void) +{ + return (!xen_hvm_domain() || xen_have_vector_callback); +} + #endif /* _ASM_X86_XEN_EVENTS_H */ diff --git a/arch/x86/include/asm/xen/hypercall.h b/arch/x86/include/asm/xen/hypercall.h index ca08a27b90b3..83aea8055119 100644 --- a/arch/x86/include/asm/xen/hypercall.h +++ b/arch/x86/include/asm/xen/hypercall.h @@ -465,6 +465,12 @@ HYPERVISOR_tmem_op( return _hypercall1(int, tmem_op, op); } +static inline int +HYPERVISOR_xenpmu_op(unsigned int op, void *arg) +{ + return _hypercall2(int, xenpmu_op, op, arg); +} + static inline void MULTI_fpu_taskswitch(struct multicall_entry *mcl, int set) { diff --git a/arch/x86/include/asm/xen/interface.h b/arch/x86/include/asm/xen/interface.h index 3400dbaec3c3..62ca03ef5c65 100644 --- a/arch/x86/include/asm/xen/interface.h +++ b/arch/x86/include/asm/xen/interface.h @@ -3,12 +3,38 @@ * * Guest OS interface to x86 Xen. * - * Copyright (c) 2004, K A Fraser + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Copyright (c) 2004-2006, K A Fraser */ #ifndef _ASM_X86_XEN_INTERFACE_H #define _ASM_X86_XEN_INTERFACE_H +/* + * XEN_GUEST_HANDLE represents a guest pointer, when passed as a field + * in a struct in memory. + * XEN_GUEST_HANDLE_PARAM represent a guest pointer, when passed as an + * hypercall argument. + * XEN_GUEST_HANDLE_PARAM and XEN_GUEST_HANDLE are the same on X86 but + * they might not be on other architectures. + */ #ifdef __XEN__ #define __DEFINE_GUEST_HANDLE(name, type) \ typedef struct { type *p; } __guest_handle_ ## name @@ -88,13 +114,16 @@ DEFINE_GUEST_HANDLE(xen_ulong_t); * start of the GDT because some stupid OSes export hard-coded selector values * in their ABI. These hard-coded values are always near the start of the GDT, * so Xen places itself out of the way, at the far end of the GDT. + * + * NB The LDT is set using the MMUEXT_SET_LDT op of HYPERVISOR_mmuext_op */ #define FIRST_RESERVED_GDT_PAGE 14 #define FIRST_RESERVED_GDT_BYTE (FIRST_RESERVED_GDT_PAGE * 4096) #define FIRST_RESERVED_GDT_ENTRY (FIRST_RESERVED_GDT_BYTE / 8) /* - * Send an array of these to HYPERVISOR_set_trap_table() + * Send an array of these to HYPERVISOR_set_trap_table(). + * Terminate the array with a sentinel entry, with traps[].address==0. * The privilege level specifies which modes may enter a trap via a software * interrupt. On x86/64, since rings 1 and 2 are unavailable, we allocate * privilege levels as follows: @@ -118,10 +147,41 @@ struct trap_info { DEFINE_GUEST_HANDLE_STRUCT(trap_info); struct arch_shared_info { - unsigned long max_pfn; /* max pfn that appears in table */ - /* Frame containing list of mfns containing list of mfns containing p2m. */ - unsigned long pfn_to_mfn_frame_list_list; - unsigned long nmi_reason; + /* + * Number of valid entries in the p2m table(s) anchored at + * pfn_to_mfn_frame_list_list and/or p2m_vaddr. + */ + unsigned long max_pfn; + /* + * Frame containing list of mfns containing list of mfns containing p2m. + * A value of 0 indicates it has not yet been set up, ~0 indicates it + * has been set to invalid e.g. due to the p2m being too large for the + * 3-level p2m tree. In this case the linear mapper p2m list anchored + * at p2m_vaddr is to be used. + */ + xen_pfn_t pfn_to_mfn_frame_list_list; + unsigned long nmi_reason; + /* + * Following three fields are valid if p2m_cr3 contains a value + * different from 0. + * p2m_cr3 is the root of the address space where p2m_vaddr is valid. + * p2m_cr3 is in the same format as a cr3 value in the vcpu register + * state and holds the folded machine frame number (via xen_pfn_to_cr3) + * of a L3 or L4 page table. + * p2m_vaddr holds the virtual address of the linear p2m list. All + * entries in the range [0...max_pfn[ are accessible via this pointer. + * p2m_generation will be incremented by the guest before and after each + * change of the mappings of the p2m list. p2m_generation starts at 0 + * and a value with the least significant bit set indicates that a + * mapping update is in progress. This allows guest external software + * (e.g. in Dom0) to verify that read mappings are consistent and + * whether they have changed since the last check. + * Modifying a p2m element in the linear p2m list is allowed via an + * atomic write only. + */ + unsigned long p2m_cr3; /* cr3 value of the p2m address space */ + unsigned long p2m_vaddr; /* virtual address of the p2m list */ + unsigned long p2m_generation; /* generation count of p2m mapping */ }; #endif /* !__ASSEMBLY__ */ @@ -137,13 +197,31 @@ struct arch_shared_info { /* * The following is all CPU context. Note that the fpu_ctxt block is filled * in by FXSAVE if the CPU has feature FXSR; otherwise FSAVE is used. + * + * Also note that when calling DOMCTL_setvcpucontext and VCPU_initialise + * for HVM and PVH guests, not all information in this structure is updated: + * + * - For HVM guests, the structures read include: fpu_ctxt (if + * VGCT_I387_VALID is set), flags, user_regs, debugreg[*] + * + * - PVH guests are the same as HVM guests, but additionally use ctrlreg[3] to + * set cr3. All other fields not used should be set to 0. */ struct vcpu_guest_context { /* FPU registers come first so they can be aligned for FXSAVE/FXRSTOR. */ struct { char x[512]; } fpu_ctxt; /* User-level FPU registers */ -#define VGCF_I387_VALID (1<<0) -#define VGCF_HVM_GUEST (1<<1) -#define VGCF_IN_KERNEL (1<<2) +#define VGCF_I387_VALID (1<<0) +#define VGCF_IN_KERNEL (1<<2) +#define _VGCF_i387_valid 0 +#define VGCF_i387_valid (1<<_VGCF_i387_valid) +#define _VGCF_in_kernel 2 +#define VGCF_in_kernel (1<<_VGCF_in_kernel) +#define _VGCF_failsafe_disables_events 3 +#define VGCF_failsafe_disables_events (1<<_VGCF_failsafe_disables_events) +#define _VGCF_syscall_disables_events 4 +#define VGCF_syscall_disables_events (1<<_VGCF_syscall_disables_events) +#define _VGCF_online 5 +#define VGCF_online (1<<_VGCF_online) unsigned long flags; /* VGCF_* flags */ struct cpu_user_regs user_regs; /* User-level CPU registers */ struct trap_info trap_ctxt[256]; /* Virtual IDT */ @@ -172,6 +250,129 @@ struct vcpu_guest_context { #endif }; DEFINE_GUEST_HANDLE_STRUCT(vcpu_guest_context); + +/* AMD PMU registers and structures */ +struct xen_pmu_amd_ctxt { + /* + * Offsets to counter and control MSRs (relative to xen_pmu_arch.c.amd). + * For PV(H) guests these fields are RO. + */ + uint32_t counters; + uint32_t ctrls; + + /* Counter MSRs */ +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + uint64_t regs[]; +#elif defined(__GNUC__) + uint64_t regs[0]; +#endif +}; + +/* Intel PMU registers and structures */ +struct xen_pmu_cntr_pair { + uint64_t counter; + uint64_t control; +}; + +struct xen_pmu_intel_ctxt { + /* + * Offsets to fixed and architectural counter MSRs (relative to + * xen_pmu_arch.c.intel). + * For PV(H) guests these fields are RO. + */ + uint32_t fixed_counters; + uint32_t arch_counters; + + /* PMU registers */ + uint64_t global_ctrl; + uint64_t global_ovf_ctrl; + uint64_t global_status; + uint64_t fixed_ctrl; + uint64_t ds_area; + uint64_t pebs_enable; + uint64_t debugctl; + + /* Fixed and architectural counter MSRs */ +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + uint64_t regs[]; +#elif defined(__GNUC__) + uint64_t regs[0]; +#endif +}; + +/* Sampled domain's registers */ +struct xen_pmu_regs { + uint64_t ip; + uint64_t sp; + uint64_t flags; + uint16_t cs; + uint16_t ss; + uint8_t cpl; + uint8_t pad[3]; +}; + +/* PMU flags */ +#define PMU_CACHED (1<<0) /* PMU MSRs are cached in the context */ +#define PMU_SAMPLE_USER (1<<1) /* Sample is from user or kernel mode */ +#define PMU_SAMPLE_REAL (1<<2) /* Sample is from realmode */ +#define PMU_SAMPLE_PV (1<<3) /* Sample from a PV guest */ + +/* + * Architecture-specific information describing state of the processor at + * the time of PMU interrupt. + * Fields of this structure marked as RW for guest should only be written by + * the guest when PMU_CACHED bit in pmu_flags is set (which is done by the + * hypervisor during PMU interrupt). Hypervisor will read updated data in + * XENPMU_flush hypercall and clear PMU_CACHED bit. + */ +struct xen_pmu_arch { + union { + /* + * Processor's registers at the time of interrupt. + * WO for hypervisor, RO for guests. + */ + struct xen_pmu_regs regs; + /* + * Padding for adding new registers to xen_pmu_regs in + * the future + */ +#define XENPMU_REGS_PAD_SZ 64 + uint8_t pad[XENPMU_REGS_PAD_SZ]; + } r; + + /* WO for hypervisor, RO for guest */ + uint64_t pmu_flags; + + /* + * APIC LVTPC register. + * RW for both hypervisor and guest. + * Only APIC_LVT_MASKED bit is loaded by the hypervisor into hardware + * during XENPMU_flush or XENPMU_lvtpc_set. + */ + union { + uint32_t lapic_lvtpc; + uint64_t pad; + } l; + + /* + * Vendor-specific PMU registers. + * RW for both hypervisor and guest (see exceptions above). + * Guest's updates to this field are verified and then loaded by the + * hypervisor into hardware during XENPMU_flush + */ + union { + struct xen_pmu_amd_ctxt amd; + struct xen_pmu_intel_ctxt intel; + + /* + * Padding for contexts (fixed parts only, does not include + * MSR banks that are specified by offsets) + */ +#define XENPMU_CTXT_PAD_SZ 128 + uint8_t pad[XENPMU_CTXT_PAD_SZ]; + } c; +}; + #endif /* !__ASSEMBLY__ */ /* diff --git a/arch/x86/include/asm/xen/page.h b/arch/x86/include/asm/xen/page.h index c44a5d53e464..0679e11d2cf7 100644 --- a/arch/x86/include/asm/xen/page.h +++ b/arch/x86/include/asm/xen/page.h @@ -35,9 +35,7 @@ typedef struct xpaddr { #define FOREIGN_FRAME(m) ((m) | FOREIGN_FRAME_BIT) #define IDENTITY_FRAME(m) ((m) | IDENTITY_FRAME_BIT) -/* Maximum amount of memory we can handle in a domain in pages */ -#define MAX_DOMAIN_PAGES \ - ((unsigned long)((u64)CONFIG_XEN_MAX_DOMAIN_MEMORY * 1024 * 1024 * 1024 / PAGE_SIZE)) +#define P2M_PER_PAGE (PAGE_SIZE / sizeof(unsigned long)) extern unsigned long *machine_to_phys_mapping; extern unsigned long machine_to_phys_nr; @@ -48,8 +46,8 @@ extern unsigned long xen_max_p2m_pfn; extern unsigned long get_phys_to_machine(unsigned long pfn); extern bool set_phys_to_machine(unsigned long pfn, unsigned long mfn); extern bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn); -extern unsigned long set_phys_range_identity(unsigned long pfn_s, - unsigned long pfn_e); +extern unsigned long __init set_phys_range_identity(unsigned long pfn_s, + unsigned long pfn_e); extern int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, struct gnttab_map_grant_ref *kmap_ops, @@ -103,6 +101,11 @@ static inline unsigned long pfn_to_mfn(unsigned long pfn) { unsigned long mfn; + /* + * Some x86 code are still using pfn_to_mfn instead of + * pfn_to_mfn. This will have to be removed when we figured + * out which call. + */ if (xen_feature(XENFEAT_auto_translated_physmap)) return pfn; @@ -149,6 +152,11 @@ static inline unsigned long mfn_to_pfn(unsigned long mfn) { unsigned long pfn; + /* + * Some x86 code are still using mfn_to_pfn instead of + * gfn_to_pfn. This will have to be removed when we figure + * out which call. + */ if (xen_feature(XENFEAT_auto_translated_physmap)) return mfn; @@ -178,6 +186,27 @@ static inline xpaddr_t machine_to_phys(xmaddr_t machine) return XPADDR(PFN_PHYS(mfn_to_pfn(PFN_DOWN(machine.maddr))) | offset); } +/* Pseudo-physical <-> Guest conversion */ +static inline unsigned long pfn_to_gfn(unsigned long pfn) +{ + if (xen_feature(XENFEAT_auto_translated_physmap)) + return pfn; + else + return pfn_to_mfn(pfn); +} + +static inline unsigned long gfn_to_pfn(unsigned long gfn) +{ + if (xen_feature(XENFEAT_auto_translated_physmap)) + return gfn; + else + return mfn_to_pfn(gfn); +} + +/* Pseudo-physical <-> Bus conversion */ +#define pfn_to_bfn(pfn) pfn_to_gfn(pfn) +#define bfn_to_pfn(bfn) gfn_to_pfn(bfn) + /* * We detect special mappings in one of two ways: * 1. If the MFN is an I/O page then Xen will set the m2p entry @@ -198,7 +227,7 @@ static inline xpaddr_t machine_to_phys(xmaddr_t machine) * require. In all the cases we care about, the FOREIGN_FRAME bit is * masked (e.g., pfn_to_mfn()) so behaviour there is correct. */ -static inline unsigned long mfn_to_local_pfn(unsigned long mfn) +static inline unsigned long bfn_to_local_pfn(unsigned long mfn) { unsigned long pfn; @@ -217,6 +246,10 @@ static inline unsigned long mfn_to_local_pfn(unsigned long mfn) #define virt_to_mfn(v) (pfn_to_mfn(virt_to_pfn(v))) #define mfn_to_virt(m) (__va(mfn_to_pfn(m) << PAGE_SHIFT)) +/* VIRT <-> GUEST conversion */ +#define virt_to_gfn(v) (pfn_to_gfn(virt_to_pfn(v))) +#define gfn_to_virt(g) (__va(gfn_to_pfn(g) << PAGE_SHIFT)) + static inline unsigned long pte_mfn(pte_t pte) { return (pte.pte & PTE_PFN_MASK) >> PAGE_SHIFT; @@ -264,7 +297,7 @@ void make_lowmem_page_readwrite(void *vaddr); static inline bool xen_arch_need_swiotlb(struct device *dev, unsigned long pfn, - unsigned long mfn) + unsigned long bfn) { return false; } diff --git a/arch/x86/include/uapi/asm/e820.h b/arch/x86/include/uapi/asm/e820.h index 0f457e6eab18..9dafe59cf6e2 100644 --- a/arch/x86/include/uapi/asm/e820.h +++ b/arch/x86/include/uapi/asm/e820.h @@ -37,7 +37,7 @@ /* * This is a non-standardized way to represent ADR or NVDIMM regions that * persist over a reboot. The kernel will ignore their special capabilities - * unless the CONFIG_X86_PMEM_LEGACY=y option is set. + * unless the CONFIG_X86_PMEM_LEGACY option is set. * * ( Note that older platforms also used 6 for the same type of memory, * but newer versions switched to 12 as 6 was assigned differently. Some diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 3c3622176340..b1b78ffe01d0 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -71,8 +71,8 @@ obj-$(CONFIG_LIVEPATCH) += livepatch.o obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o obj-$(CONFIG_FTRACE_SYSCALLS) += ftrace.o obj-$(CONFIG_X86_TSC) += trace_clock.o -obj-$(CONFIG_KEXEC) += machine_kexec_$(BITS).o -obj-$(CONFIG_KEXEC) += relocate_kernel_$(BITS).o crash.o +obj-$(CONFIG_KEXEC_CORE) += machine_kexec_$(BITS).o +obj-$(CONFIG_KEXEC_CORE) += relocate_kernel_$(BITS).o crash.o obj-$(CONFIG_KEXEC_FILE) += kexec-bzimage64.o obj-$(CONFIG_CRASH_DUMP) += crash_dump_$(BITS).o obj-y += kprobes/ @@ -94,7 +94,7 @@ obj-$(CONFIG_KVM_GUEST) += kvm.o kvmclock.o obj-$(CONFIG_PARAVIRT) += paravirt.o paravirt_patch_$(BITS).o obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= paravirt-spinlocks.o obj-$(CONFIG_PARAVIRT_CLOCK) += pvclock.o -obj-$(CONFIG_X86_PMEM_LEGACY) += pmem.o +obj-$(CONFIG_X86_PMEM_LEGACY_DEVICE) += pmem.o obj-$(CONFIG_PCSPKR_PLATFORM) += pcspeaker.o diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c index 75e8bad53798..ded848c20e05 100644 --- a/arch/x86/kernel/acpi/boot.c +++ b/arch/x86/kernel/acpi/boot.c @@ -445,6 +445,7 @@ static void __init acpi_sci_ioapic_setup(u8 bus_irq, u16 polarity, u16 trigger, polarity = acpi_sci_flags & ACPI_MADT_POLARITY_MASK; mp_override_legacy_irq(bus_irq, polarity, trigger, gsi); + acpi_penalize_sci_irq(bus_irq, trigger, polarity); /* * stash over-ride to indicate we've been here diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 5aba9220a5ac..3ca3e46aa405 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -462,40 +462,40 @@ static int lapic_next_deadline(unsigned long delta, return 0; } -/* - * Setup the lapic timer in periodic or oneshot mode - */ -static void lapic_timer_setup(enum clock_event_mode mode, - struct clock_event_device *evt) +static int lapic_timer_shutdown(struct clock_event_device *evt) { - unsigned long flags; unsigned int v; /* Lapic used as dummy for broadcast ? */ if (evt->features & CLOCK_EVT_FEAT_DUMMY) - return; + return 0; - local_irq_save(flags); + v = apic_read(APIC_LVTT); + v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR); + apic_write(APIC_LVTT, v); + apic_write(APIC_TMICT, 0); + return 0; +} - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - case CLOCK_EVT_MODE_ONESHOT: - __setup_APIC_LVTT(lapic_timer_frequency, - mode != CLOCK_EVT_MODE_PERIODIC, 1); - break; - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - v = apic_read(APIC_LVTT); - v |= (APIC_LVT_MASKED | LOCAL_TIMER_VECTOR); - apic_write(APIC_LVTT, v); - apic_write(APIC_TMICT, 0); - break; - case CLOCK_EVT_MODE_RESUME: - /* Nothing to do here */ - break; - } +static inline int +lapic_timer_set_periodic_oneshot(struct clock_event_device *evt, bool oneshot) +{ + /* Lapic used as dummy for broadcast ? */ + if (evt->features & CLOCK_EVT_FEAT_DUMMY) + return 0; - local_irq_restore(flags); + __setup_APIC_LVTT(lapic_timer_frequency, oneshot, 1); + return 0; +} + +static int lapic_timer_set_periodic(struct clock_event_device *evt) +{ + return lapic_timer_set_periodic_oneshot(evt, false); +} + +static int lapic_timer_set_oneshot(struct clock_event_device *evt) +{ + return lapic_timer_set_periodic_oneshot(evt, true); } /* @@ -513,15 +513,18 @@ static void lapic_timer_broadcast(const struct cpumask *mask) * The local apic timer can be used for any function which is CPU local. */ static struct clock_event_device lapic_clockevent = { - .name = "lapic", - .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT - | CLOCK_EVT_FEAT_C3STOP | CLOCK_EVT_FEAT_DUMMY, - .shift = 32, - .set_mode = lapic_timer_setup, - .set_next_event = lapic_next_event, - .broadcast = lapic_timer_broadcast, - .rating = 100, - .irq = -1, + .name = "lapic", + .features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_C3STOP + | CLOCK_EVT_FEAT_DUMMY, + .shift = 32, + .set_state_shutdown = lapic_timer_shutdown, + .set_state_periodic = lapic_timer_set_periodic, + .set_state_oneshot = lapic_timer_set_oneshot, + .set_next_event = lapic_next_event, + .broadcast = lapic_timer_broadcast, + .rating = 100, + .irq = -1, }; static DEFINE_PER_CPU(struct clock_event_device, lapic_events); @@ -778,7 +781,7 @@ static int __init calibrate_APIC_clock(void) * Setup the apic timer manually */ levt->event_handler = lapic_cal_handler; - lapic_timer_setup(CLOCK_EVT_MODE_PERIODIC, levt); + lapic_timer_set_periodic(levt); lapic_cal_loops = -1; /* Let the interrupts run */ @@ -788,7 +791,8 @@ static int __init calibrate_APIC_clock(void) cpu_relax(); /* Stop the lapic timer */ - lapic_timer_setup(CLOCK_EVT_MODE_SHUTDOWN, levt); + local_irq_disable(); + lapic_timer_shutdown(levt); /* Jiffies delta */ deltaj = lapic_cal_j2 - lapic_cal_j1; @@ -799,8 +803,8 @@ static int __init calibrate_APIC_clock(void) apic_printk(APIC_VERBOSE, "... jiffies result ok\n"); else levt->features |= CLOCK_EVT_FEAT_DUMMY; - } else - local_irq_enable(); + } + local_irq_enable(); if (levt->features & CLOCK_EVT_FEAT_DUMMY) { pr_warning("APIC timer disabled due to verification failure\n"); @@ -878,7 +882,7 @@ static void local_apic_timer_interrupt(void) if (!evt->event_handler) { pr_warning("Spurious LAPIC timer interrupt on cpu %d\n", cpu); /* Switch it off */ - lapic_timer_setup(CLOCK_EVT_MODE_SHUTDOWN, evt); + lapic_timer_shutdown(evt); return; } diff --git a/arch/x86/kernel/apic/hw_nmi.c b/arch/x86/kernel/apic/hw_nmi.c index 6873ab925d00..045e424fb368 100644 --- a/arch/x86/kernel/apic/hw_nmi.c +++ b/arch/x86/kernel/apic/hw_nmi.c @@ -28,146 +28,21 @@ u64 hw_nmi_get_sample_period(int watchdog_thresh) #endif #ifdef arch_trigger_all_cpu_backtrace -/* For reliability, we're prepared to waste bits here. */ -static DECLARE_BITMAP(backtrace_mask, NR_CPUS) __read_mostly; -static cpumask_t printtrace_mask; - -#define NMI_BUF_SIZE 4096 - -struct nmi_seq_buf { - unsigned char buffer[NMI_BUF_SIZE]; - struct seq_buf seq; -}; - -/* Safe printing in NMI context */ -static DEFINE_PER_CPU(struct nmi_seq_buf, nmi_print_seq); - -/* "in progress" flag of arch_trigger_all_cpu_backtrace */ -static unsigned long backtrace_flag; - -static void print_seq_line(struct nmi_seq_buf *s, int start, int end) +static void nmi_raise_cpu_backtrace(cpumask_t *mask) { - const char *buf = s->buffer + start; - - printk("%.*s", (end - start) + 1, buf); + apic->send_IPI_mask(mask, NMI_VECTOR); } void arch_trigger_all_cpu_backtrace(bool include_self) { - struct nmi_seq_buf *s; - int len; - int cpu; - int i; - int this_cpu = get_cpu(); - - if (test_and_set_bit(0, &backtrace_flag)) { - /* - * If there is already a trigger_all_cpu_backtrace() in progress - * (backtrace_flag == 1), don't output double cpu dump infos. - */ - put_cpu(); - return; - } - - cpumask_copy(to_cpumask(backtrace_mask), cpu_online_mask); - if (!include_self) - cpumask_clear_cpu(this_cpu, to_cpumask(backtrace_mask)); - - cpumask_copy(&printtrace_mask, to_cpumask(backtrace_mask)); - /* - * Set up per_cpu seq_buf buffers that the NMIs running on the other - * CPUs will write to. - */ - for_each_cpu(cpu, to_cpumask(backtrace_mask)) { - s = &per_cpu(nmi_print_seq, cpu); - seq_buf_init(&s->seq, s->buffer, NMI_BUF_SIZE); - } - - if (!cpumask_empty(to_cpumask(backtrace_mask))) { - pr_info("sending NMI to %s CPUs:\n", - (include_self ? "all" : "other")); - apic->send_IPI_mask(to_cpumask(backtrace_mask), NMI_VECTOR); - } - - /* Wait for up to 10 seconds for all CPUs to do the backtrace */ - for (i = 0; i < 10 * 1000; i++) { - if (cpumask_empty(to_cpumask(backtrace_mask))) - break; - mdelay(1); - touch_softlockup_watchdog(); - } - - /* - * Now that all the NMIs have triggered, we can dump out their - * back traces safely to the console. - */ - for_each_cpu(cpu, &printtrace_mask) { - int last_i = 0; - - s = &per_cpu(nmi_print_seq, cpu); - len = seq_buf_used(&s->seq); - if (!len) - continue; - - /* Print line by line. */ - for (i = 0; i < len; i++) { - if (s->buffer[i] == '\n') { - print_seq_line(s, last_i, i); - last_i = i + 1; - } - } - /* Check if there was a partial line. */ - if (last_i < len) { - print_seq_line(s, last_i, len - 1); - pr_cont("\n"); - } - } - - clear_bit(0, &backtrace_flag); - smp_mb__after_atomic(); - put_cpu(); -} - -/* - * It is not safe to call printk() directly from NMI handlers. - * It may be fine if the NMI detected a lock up and we have no choice - * but to do so, but doing a NMI on all other CPUs to get a back trace - * can be done with a sysrq-l. We don't want that to lock up, which - * can happen if the NMI interrupts a printk in progress. - * - * Instead, we redirect the vprintk() to this nmi_vprintk() that writes - * the content into a per cpu seq_buf buffer. Then when the NMIs are - * all done, we can safely dump the contents of the seq_buf to a printk() - * from a non NMI context. - */ -static int nmi_vprintk(const char *fmt, va_list args) -{ - struct nmi_seq_buf *s = this_cpu_ptr(&nmi_print_seq); - unsigned int len = seq_buf_used(&s->seq); - - seq_buf_vprintf(&s->seq, fmt, args); - return seq_buf_used(&s->seq) - len; + nmi_trigger_all_cpu_backtrace(include_self, nmi_raise_cpu_backtrace); } static int arch_trigger_all_cpu_backtrace_handler(unsigned int cmd, struct pt_regs *regs) { - int cpu; - - cpu = smp_processor_id(); - - if (cpumask_test_cpu(cpu, to_cpumask(backtrace_mask))) { - printk_func_t printk_func_save = this_cpu_read(printk_func); - - /* Replace printk to write into the NMI seq */ - this_cpu_write(printk_func, nmi_vprintk); - printk(KERN_WARNING "NMI backtrace for cpu %d\n", cpu); - show_regs(regs); - this_cpu_write(printk_func, printk_func_save); - - cpumask_clear_cpu(cpu, to_cpumask(backtrace_mask)); + if (nmi_cpu_backtrace(regs)) return NMI_HANDLED; - } return NMI_DONE; } diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index 206052e55517..38a76f826530 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -2541,7 +2541,7 @@ void __init setup_ioapic_dest(void) * Honour affinities which have been set in early boot */ if (!irqd_can_balance(idata) || irqd_affinity_was_set(idata)) - mask = idata->affinity; + mask = irq_data_get_affinity_mask(idata); else mask = apic->target_cpus(); diff --git a/arch/x86/kernel/apic/msi.c b/arch/x86/kernel/apic/msi.c index 1a9d735e09c6..5f1feb6854af 100644 --- a/arch/x86/kernel/apic/msi.c +++ b/arch/x86/kernel/apic/msi.c @@ -264,7 +264,7 @@ static inline int hpet_dev_id(struct irq_domain *domain) static void hpet_msi_write_msg(struct irq_data *data, struct msi_msg *msg) { - hpet_msi_write(data->handler_data, msg); + hpet_msi_write(irq_data_get_irq_handler_data(data), msg); } static struct irq_chip hpet_msi_controller = { diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c index 2683f36e4e0a..1bbd0fe2c806 100644 --- a/arch/x86/kernel/apic/vector.c +++ b/arch/x86/kernel/apic/vector.c @@ -169,8 +169,7 @@ static int __assign_irq_vector(int irq, struct apic_chip_data *d, goto next; for_each_cpu_and(new_cpu, vector_cpumask, cpu_online_mask) { - if (per_cpu(vector_irq, new_cpu)[vector] > - VECTOR_UNDEFINED) + if (!IS_ERR_OR_NULL(per_cpu(vector_irq, new_cpu)[vector])) goto next; } /* Found one! */ @@ -182,7 +181,7 @@ static int __assign_irq_vector(int irq, struct apic_chip_data *d, cpumask_intersects(d->old_domain, cpu_online_mask); } for_each_cpu_and(new_cpu, vector_cpumask, cpu_online_mask) - per_cpu(vector_irq, new_cpu)[vector] = irq; + per_cpu(vector_irq, new_cpu)[vector] = irq_to_desc(irq); d->cfg.vector = vector; cpumask_copy(d->domain, vector_cpumask); err = 0; @@ -224,15 +223,16 @@ static int assign_irq_vector_policy(int irq, int node, static void clear_irq_vector(int irq, struct apic_chip_data *data) { - int cpu, vector; + struct irq_desc *desc; unsigned long flags; + int cpu, vector; raw_spin_lock_irqsave(&vector_lock, flags); BUG_ON(!data->cfg.vector); vector = data->cfg.vector; for_each_cpu_and(cpu, data->domain, cpu_online_mask) - per_cpu(vector_irq, cpu)[vector] = VECTOR_UNDEFINED; + per_cpu(vector_irq, cpu)[vector] = VECTOR_UNUSED; data->cfg.vector = 0; cpumask_clear(data->domain); @@ -242,12 +242,13 @@ static void clear_irq_vector(int irq, struct apic_chip_data *data) return; } + desc = irq_to_desc(irq); for_each_cpu_and(cpu, data->old_domain, cpu_online_mask) { for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) { - if (per_cpu(vector_irq, cpu)[vector] != irq) + if (per_cpu(vector_irq, cpu)[vector] != desc) continue; - per_cpu(vector_irq, cpu)[vector] = VECTOR_UNDEFINED; + per_cpu(vector_irq, cpu)[vector] = VECTOR_UNUSED; break; } } @@ -296,7 +297,7 @@ static int x86_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq, struct irq_alloc_info *info = arg; struct apic_chip_data *data; struct irq_data *irq_data; - int i, err; + int i, err, node; if (disable_apic) return -ENXIO; @@ -308,12 +309,13 @@ static int x86_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq, for (i = 0; i < nr_irqs; i++) { irq_data = irq_domain_get_irq_data(domain, virq + i); BUG_ON(!irq_data); + node = irq_data_get_node(irq_data); #ifdef CONFIG_X86_IO_APIC if (virq + i < nr_legacy_irqs() && legacy_irq_data[virq + i]) data = legacy_irq_data[virq + i]; else #endif - data = alloc_apic_chip_data(irq_data->node); + data = alloc_apic_chip_data(node); if (!data) { err = -ENOMEM; goto error; @@ -322,8 +324,7 @@ static int x86_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq, irq_data->chip = &lapic_controller; irq_data->chip_data = data; irq_data->hwirq = virq + i; - err = assign_irq_vector_policy(virq + i, irq_data->node, data, - info); + err = assign_irq_vector_policy(virq + i, node, data, info); if (err) goto error; } @@ -403,32 +404,32 @@ int __init arch_early_irq_init(void) return arch_early_ioapic_init(); } +/* Initialize vector_irq on a new cpu */ static void __setup_vector_irq(int cpu) { - /* Initialize vector_irq on a new cpu */ - int irq, vector; struct apic_chip_data *data; + struct irq_desc *desc; + int irq, vector; /* Mark the inuse vectors */ - for_each_active_irq(irq) { - data = apic_chip_data(irq_get_irq_data(irq)); - if (!data) - continue; + for_each_irq_desc(irq, desc) { + struct irq_data *idata = irq_desc_get_irq_data(desc); - if (!cpumask_test_cpu(cpu, data->domain)) + data = apic_chip_data(idata); + if (!data || !cpumask_test_cpu(cpu, data->domain)) continue; vector = data->cfg.vector; - per_cpu(vector_irq, cpu)[vector] = irq; + per_cpu(vector_irq, cpu)[vector] = desc; } /* Mark the free vectors */ for (vector = 0; vector < NR_VECTORS; ++vector) { - irq = per_cpu(vector_irq, cpu)[vector]; - if (irq <= VECTOR_UNDEFINED) + desc = per_cpu(vector_irq, cpu)[vector]; + if (IS_ERR_OR_NULL(desc)) continue; - data = apic_chip_data(irq_get_irq_data(irq)); + data = apic_chip_data(irq_desc_get_irq_data(desc)); if (!cpumask_test_cpu(cpu, data->domain)) - per_cpu(vector_irq, cpu)[vector] = VECTOR_UNDEFINED; + per_cpu(vector_irq, cpu)[vector] = VECTOR_UNUSED; } } @@ -448,7 +449,7 @@ void setup_vector_irq(int cpu) * legacy vector to irq mapping: */ for (irq = 0; irq < nr_legacy_irqs(); irq++) - per_cpu(vector_irq, cpu)[ISA_IRQ_VECTOR(irq)] = irq; + per_cpu(vector_irq, cpu)[ISA_IRQ_VECTOR(irq)] = irq_to_desc(irq); __setup_vector_irq(cpu); } @@ -490,7 +491,8 @@ static int apic_set_affinity(struct irq_data *irq_data, if (err) { struct irq_data *top = irq_get_irq_data(irq); - if (assign_irq_vector(irq, data, top->affinity)) + if (assign_irq_vector(irq, data, + irq_data_get_affinity_mask(top))) pr_err("Failed to recover vector for irq %d\n", irq); return err; } @@ -538,27 +540,30 @@ asmlinkage __visible void smp_irq_move_cleanup_interrupt(void) entering_ack_irq(); + /* Prevent vectors vanishing under us */ + raw_spin_lock(&vector_lock); + me = smp_processor_id(); for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) { - int irq; - unsigned int irr; - struct irq_desc *desc; struct apic_chip_data *data; + struct irq_desc *desc; + unsigned int irr; - irq = __this_cpu_read(vector_irq[vector]); - - if (irq <= VECTOR_UNDEFINED) + retry: + desc = __this_cpu_read(vector_irq[vector]); + if (IS_ERR_OR_NULL(desc)) continue; - desc = irq_to_desc(irq); - if (!desc) - continue; + if (!raw_spin_trylock(&desc->lock)) { + raw_spin_unlock(&vector_lock); + cpu_relax(); + raw_spin_lock(&vector_lock); + goto retry; + } - data = apic_chip_data(&desc->irq_data); + data = apic_chip_data(irq_desc_get_irq_data(desc)); if (!data) - continue; - - raw_spin_lock(&desc->lock); + goto unlock; /* * Check if the irq migration is in progress. If so, we @@ -583,11 +588,13 @@ asmlinkage __visible void smp_irq_move_cleanup_interrupt(void) apic->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR); goto unlock; } - __this_cpu_write(vector_irq[vector], VECTOR_UNDEFINED); + __this_cpu_write(vector_irq[vector], VECTOR_UNUSED); unlock: raw_spin_unlock(&desc->lock); } + raw_spin_unlock(&vector_lock); + exiting_irq(); } diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c index 3f124d553c5a..cd9b6d0b10bf 100644 --- a/arch/x86/kernel/cpu/perf_event_intel.c +++ b/arch/x86/kernel/cpu/perf_event_intel.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include @@ -3627,7 +3627,10 @@ static __init int fixup_ht_bug(void) return 0; } - watchdog_nmi_disable_all(); + if (lockup_detector_suspend() != 0) { + pr_debug("failed to disable PMU erratum BJ122, BV98, HSD29 workaround\n"); + return 0; + } x86_pmu.flags &= ~(PMU_FL_EXCL_CNTRS | PMU_FL_EXCL_ENABLED); @@ -3635,7 +3638,7 @@ static __init int fixup_ht_bug(void) x86_pmu.commit_scheduling = NULL; x86_pmu.stop_scheduling = NULL; - watchdog_nmi_enable_all(); + lockup_detector_resume(); get_online_cpus(); diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c index f75c5908c7a6..88b4da373081 100644 --- a/arch/x86/kernel/hpet.c +++ b/arch/x86/kernel/hpet.c @@ -226,22 +226,7 @@ static void hpet_reserve_platform_timers(unsigned int id) { } */ static unsigned long hpet_freq; -static void hpet_legacy_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt); -static int hpet_legacy_next_event(unsigned long delta, - struct clock_event_device *evt); - -/* - * The hpet clock event device - */ -static struct clock_event_device hpet_clockevent = { - .name = "hpet", - .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .set_mode = hpet_legacy_set_mode, - .set_next_event = hpet_legacy_next_event, - .irq = 0, - .rating = 50, -}; +static struct clock_event_device hpet_clockevent; static void hpet_stop_counter(void) { @@ -306,64 +291,74 @@ static void hpet_legacy_clockevent_register(void) printk(KERN_DEBUG "hpet clockevent registered\n"); } -static void hpet_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt, int timer) +static int hpet_set_periodic(struct clock_event_device *evt, int timer) { unsigned int cfg, cmp, now; uint64_t delta; - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - hpet_stop_counter(); - delta = ((uint64_t)(NSEC_PER_SEC/HZ)) * evt->mult; - delta >>= evt->shift; - now = hpet_readl(HPET_COUNTER); - cmp = now + (unsigned int) delta; - cfg = hpet_readl(HPET_Tn_CFG(timer)); - cfg |= HPET_TN_ENABLE | HPET_TN_PERIODIC | - HPET_TN_SETVAL | HPET_TN_32BIT; - hpet_writel(cfg, HPET_Tn_CFG(timer)); - hpet_writel(cmp, HPET_Tn_CMP(timer)); - udelay(1); - /* - * HPET on AMD 81xx needs a second write (with HPET_TN_SETVAL - * cleared) to T0_CMP to set the period. The HPET_TN_SETVAL - * bit is automatically cleared after the first write. - * (See AMD-8111 HyperTransport I/O Hub Data Sheet, - * Publication # 24674) - */ - hpet_writel((unsigned int) delta, HPET_Tn_CMP(timer)); - hpet_start_counter(); - hpet_print_config(); - break; + hpet_stop_counter(); + delta = ((uint64_t)(NSEC_PER_SEC / HZ)) * evt->mult; + delta >>= evt->shift; + now = hpet_readl(HPET_COUNTER); + cmp = now + (unsigned int)delta; + cfg = hpet_readl(HPET_Tn_CFG(timer)); + cfg |= HPET_TN_ENABLE | HPET_TN_PERIODIC | HPET_TN_SETVAL | + HPET_TN_32BIT; + hpet_writel(cfg, HPET_Tn_CFG(timer)); + hpet_writel(cmp, HPET_Tn_CMP(timer)); + udelay(1); + /* + * HPET on AMD 81xx needs a second write (with HPET_TN_SETVAL + * cleared) to T0_CMP to set the period. The HPET_TN_SETVAL + * bit is automatically cleared after the first write. + * (See AMD-8111 HyperTransport I/O Hub Data Sheet, + * Publication # 24674) + */ + hpet_writel((unsigned int)delta, HPET_Tn_CMP(timer)); + hpet_start_counter(); + hpet_print_config(); - case CLOCK_EVT_MODE_ONESHOT: - cfg = hpet_readl(HPET_Tn_CFG(timer)); - cfg &= ~HPET_TN_PERIODIC; - cfg |= HPET_TN_ENABLE | HPET_TN_32BIT; - hpet_writel(cfg, HPET_Tn_CFG(timer)); - break; + return 0; +} - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - cfg = hpet_readl(HPET_Tn_CFG(timer)); - cfg &= ~HPET_TN_ENABLE; - hpet_writel(cfg, HPET_Tn_CFG(timer)); - break; +static int hpet_set_oneshot(struct clock_event_device *evt, int timer) +{ + unsigned int cfg; - case CLOCK_EVT_MODE_RESUME: - if (timer == 0) { - hpet_enable_legacy_int(); - } else { - struct hpet_dev *hdev = EVT_TO_HPET_DEV(evt); - irq_domain_activate_irq(irq_get_irq_data(hdev->irq)); - disable_irq(hdev->irq); - irq_set_affinity(hdev->irq, cpumask_of(hdev->cpu)); - enable_irq(hdev->irq); - } - hpet_print_config(); - break; + cfg = hpet_readl(HPET_Tn_CFG(timer)); + cfg &= ~HPET_TN_PERIODIC; + cfg |= HPET_TN_ENABLE | HPET_TN_32BIT; + hpet_writel(cfg, HPET_Tn_CFG(timer)); + + return 0; +} + +static int hpet_shutdown(struct clock_event_device *evt, int timer) +{ + unsigned int cfg; + + cfg = hpet_readl(HPET_Tn_CFG(timer)); + cfg &= ~HPET_TN_ENABLE; + hpet_writel(cfg, HPET_Tn_CFG(timer)); + + return 0; +} + +static int hpet_resume(struct clock_event_device *evt, int timer) +{ + if (!timer) { + hpet_enable_legacy_int(); + } else { + struct hpet_dev *hdev = EVT_TO_HPET_DEV(evt); + + irq_domain_activate_irq(irq_get_irq_data(hdev->irq)); + disable_irq(hdev->irq); + irq_set_affinity(hdev->irq, cpumask_of(hdev->cpu)); + enable_irq(hdev->irq); } + hpet_print_config(); + + return 0; } static int hpet_next_event(unsigned long delta, @@ -403,10 +398,24 @@ static int hpet_next_event(unsigned long delta, return res < HPET_MIN_CYCLES ? -ETIME : 0; } -static void hpet_legacy_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int hpet_legacy_shutdown(struct clock_event_device *evt) { - hpet_set_mode(mode, evt, 0); + return hpet_shutdown(evt, 0); +} + +static int hpet_legacy_set_oneshot(struct clock_event_device *evt) +{ + return hpet_set_oneshot(evt, 0); +} + +static int hpet_legacy_set_periodic(struct clock_event_device *evt) +{ + return hpet_set_periodic(evt, 0); +} + +static int hpet_legacy_resume(struct clock_event_device *evt) +{ + return hpet_resume(evt, 0); } static int hpet_legacy_next_event(unsigned long delta, @@ -415,6 +424,22 @@ static int hpet_legacy_next_event(unsigned long delta, return hpet_next_event(delta, evt, 0); } +/* + * The hpet clock event device + */ +static struct clock_event_device hpet_clockevent = { + .name = "hpet", + .features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT, + .set_state_periodic = hpet_legacy_set_periodic, + .set_state_oneshot = hpet_legacy_set_oneshot, + .set_state_shutdown = hpet_legacy_shutdown, + .tick_resume = hpet_legacy_resume, + .set_next_event = hpet_legacy_next_event, + .irq = 0, + .rating = 50, +}; + /* * HPET MSI Support */ @@ -426,7 +451,7 @@ static struct irq_domain *hpet_domain; void hpet_msi_unmask(struct irq_data *data) { - struct hpet_dev *hdev = data->handler_data; + struct hpet_dev *hdev = irq_data_get_irq_handler_data(data); unsigned int cfg; /* unmask it */ @@ -437,7 +462,7 @@ void hpet_msi_unmask(struct irq_data *data) void hpet_msi_mask(struct irq_data *data) { - struct hpet_dev *hdev = data->handler_data; + struct hpet_dev *hdev = irq_data_get_irq_handler_data(data); unsigned int cfg; /* mask it */ @@ -459,11 +484,32 @@ void hpet_msi_read(struct hpet_dev *hdev, struct msi_msg *msg) msg->address_hi = 0; } -static void hpet_msi_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int hpet_msi_shutdown(struct clock_event_device *evt) { struct hpet_dev *hdev = EVT_TO_HPET_DEV(evt); - hpet_set_mode(mode, evt, hdev->num); + + return hpet_shutdown(evt, hdev->num); +} + +static int hpet_msi_set_oneshot(struct clock_event_device *evt) +{ + struct hpet_dev *hdev = EVT_TO_HPET_DEV(evt); + + return hpet_set_oneshot(evt, hdev->num); +} + +static int hpet_msi_set_periodic(struct clock_event_device *evt) +{ + struct hpet_dev *hdev = EVT_TO_HPET_DEV(evt); + + return hpet_set_periodic(evt, hdev->num); +} + +static int hpet_msi_resume(struct clock_event_device *evt) +{ + struct hpet_dev *hdev = EVT_TO_HPET_DEV(evt); + + return hpet_resume(evt, hdev->num); } static int hpet_msi_next_event(unsigned long delta, @@ -523,10 +569,14 @@ static void init_one_hpet_msi_clockevent(struct hpet_dev *hdev, int cpu) evt->rating = 110; evt->features = CLOCK_EVT_FEAT_ONESHOT; - if (hdev->flags & HPET_DEV_PERI_CAP) + if (hdev->flags & HPET_DEV_PERI_CAP) { evt->features |= CLOCK_EVT_FEAT_PERIODIC; + evt->set_state_periodic = hpet_msi_set_periodic; + } - evt->set_mode = hpet_msi_set_mode; + evt->set_state_shutdown = hpet_msi_shutdown; + evt->set_state_oneshot = hpet_msi_set_oneshot; + evt->tick_resume = hpet_msi_resume; evt->set_next_event = hpet_msi_next_event; evt->cpumask = cpumask_of(hdev->cpu); diff --git a/arch/x86/kernel/i8253.c b/arch/x86/kernel/i8253.c index f2b96de3c7c1..efb82f07b29c 100644 --- a/arch/x86/kernel/i8253.c +++ b/arch/x86/kernel/i8253.c @@ -34,7 +34,7 @@ static int __init init_pit_clocksource(void) * - when local APIC timer is active (PIT is switched off) */ if (num_possible_cpus() > 1 || is_hpet_enabled() || - i8253_clockevent.mode != CLOCK_EVT_MODE_PERIODIC) + !clockevent_state_periodic(&i8253_clockevent)) return 0; return clocksource_i8253_init(); diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index ae00b355114d..f8062aaf5df9 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -214,10 +214,9 @@ u64 arch_irq_stat(void) __visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs) { struct pt_regs *old_regs = set_irq_regs(regs); - + struct irq_desc * desc; /* high bit used in ret_from_ code */ unsigned vector = ~regs->orig_ax; - unsigned irq; /* * NB: Unlike exception entries, IRQ entries do not reliably @@ -236,17 +235,17 @@ __visible unsigned int __irq_entry do_IRQ(struct pt_regs *regs) /* entering_irq() tells RCU that we're not quiescent. Check it. */ RCU_LOCKDEP_WARN(!rcu_is_watching(), "IRQ failed to wake up RCU"); - irq = __this_cpu_read(vector_irq[vector]); + desc = __this_cpu_read(vector_irq[vector]); - if (!handle_irq(irq, regs)) { + if (!handle_irq(desc, regs)) { ack_APIC_irq(); - if (irq != VECTOR_RETRIGGERED) { - pr_emerg_ratelimited("%s: %d.%d No irq handler for vector (irq %d)\n", + if (desc != VECTOR_RETRIGGERED) { + pr_emerg_ratelimited("%s: %d.%d No irq handler for vector\n", __func__, smp_processor_id(), - vector, irq); + vector); } else { - __this_cpu_write(vector_irq[vector], VECTOR_UNDEFINED); + __this_cpu_write(vector_irq[vector], VECTOR_UNUSED); } } @@ -348,10 +347,10 @@ static struct cpumask affinity_new, online_new; */ int check_irq_vectors_for_cpu_disable(void) { - int irq, cpu; unsigned int this_cpu, vector, this_count, count; struct irq_desc *desc; struct irq_data *data; + int cpu; this_cpu = smp_processor_id(); cpumask_copy(&online_new, cpu_online_mask); @@ -359,47 +358,43 @@ int check_irq_vectors_for_cpu_disable(void) this_count = 0; for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) { - irq = __this_cpu_read(vector_irq[vector]); - if (irq >= 0) { - desc = irq_to_desc(irq); - if (!desc) - continue; - - /* - * Protect against concurrent action removal, - * affinity changes etc. - */ - raw_spin_lock(&desc->lock); - data = irq_desc_get_irq_data(desc); - cpumask_copy(&affinity_new, data->affinity); - cpumask_clear_cpu(this_cpu, &affinity_new); - - /* Do not count inactive or per-cpu irqs. */ - if (!irq_has_action(irq) || irqd_is_per_cpu(data)) { - raw_spin_unlock(&desc->lock); - continue; - } + desc = __this_cpu_read(vector_irq[vector]); + if (IS_ERR_OR_NULL(desc)) + continue; + /* + * Protect against concurrent action removal, affinity + * changes etc. + */ + raw_spin_lock(&desc->lock); + data = irq_desc_get_irq_data(desc); + cpumask_copy(&affinity_new, + irq_data_get_affinity_mask(data)); + cpumask_clear_cpu(this_cpu, &affinity_new); + /* Do not count inactive or per-cpu irqs. */ + if (!irq_desc_has_action(desc) || irqd_is_per_cpu(data)) { raw_spin_unlock(&desc->lock); - /* - * A single irq may be mapped to multiple - * cpu's vector_irq[] (for example IOAPIC cluster - * mode). In this case we have two - * possibilities: - * - * 1) the resulting affinity mask is empty; that is - * this the down'd cpu is the last cpu in the irq's - * affinity mask, or - * - * 2) the resulting affinity mask is no longer - * a subset of the online cpus but the affinity - * mask is not zero; that is the down'd cpu is the - * last online cpu in a user set affinity mask. - */ - if (cpumask_empty(&affinity_new) || - !cpumask_subset(&affinity_new, &online_new)) - this_count++; + continue; } + + raw_spin_unlock(&desc->lock); + /* + * A single irq may be mapped to multiple cpu's + * vector_irq[] (for example IOAPIC cluster mode). In + * this case we have two possibilities: + * + * 1) the resulting affinity mask is empty; that is + * this the down'd cpu is the last cpu in the irq's + * affinity mask, or + * + * 2) the resulting affinity mask is no longer a + * subset of the online cpus but the affinity mask is + * not zero; that is the down'd cpu is the last online + * cpu in a user set affinity mask. + */ + if (cpumask_empty(&affinity_new) || + !cpumask_subset(&affinity_new, &online_new)) + this_count++; } count = 0; @@ -418,8 +413,8 @@ int check_irq_vectors_for_cpu_disable(void) for (vector = FIRST_EXTERNAL_VECTOR; vector < first_system_vector; vector++) { if (!test_bit(vector, used_vectors) && - per_cpu(vector_irq, cpu)[vector] < 0) - count++; + IS_ERR_OR_NULL(per_cpu(vector_irq, cpu)[vector])) + count++; } } @@ -455,7 +450,7 @@ void fixup_irqs(void) raw_spin_lock(&desc->lock); data = irq_desc_get_irq_data(desc); - affinity = data->affinity; + affinity = irq_data_get_affinity_mask(data); if (!irq_has_action(irq) || irqd_is_per_cpu(data) || cpumask_subset(affinity, cpu_online_mask)) { raw_spin_unlock(&desc->lock); @@ -523,14 +518,13 @@ void fixup_irqs(void) for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) { unsigned int irr; - if (__this_cpu_read(vector_irq[vector]) <= VECTOR_UNDEFINED) + if (IS_ERR_OR_NULL(__this_cpu_read(vector_irq[vector]))) continue; irr = apic_read(APIC_IRR + (vector / 32 * 0x10)); if (irr & (1 << (vector % 32))) { - irq = __this_cpu_read(vector_irq[vector]); + desc = __this_cpu_read(vector_irq[vector]); - desc = irq_to_desc(irq); raw_spin_lock(&desc->lock); data = irq_desc_get_irq_data(desc); chip = irq_data_get_irq_chip(data); @@ -541,7 +535,7 @@ void fixup_irqs(void) raw_spin_unlock(&desc->lock); } if (__this_cpu_read(vector_irq[vector]) != VECTOR_RETRIGGERED) - __this_cpu_write(vector_irq[vector], VECTOR_UNDEFINED); + __this_cpu_write(vector_irq[vector], VECTOR_UNUSED); } } #endif diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c index cd74f5978ab9..c80cf6699678 100644 --- a/arch/x86/kernel/irq_32.c +++ b/arch/x86/kernel/irq_32.c @@ -148,21 +148,21 @@ void do_softirq_own_stack(void) call_on_stack(__do_softirq, isp); } -bool handle_irq(unsigned irq, struct pt_regs *regs) +bool handle_irq(struct irq_desc *desc, struct pt_regs *regs) { - struct irq_desc *desc; + unsigned int irq; int overflow; overflow = check_stack_overflow(); - desc = irq_to_desc(irq); - if (unlikely(!desc)) + if (IS_ERR_OR_NULL(desc)) return false; + irq = irq_desc_get_irq(desc); if (user_mode(regs) || !execute_on_irq_stack(overflow, desc, irq)) { if (unlikely(overflow)) print_stack_overflow(); - desc->handle_irq(irq, desc); + generic_handle_irq_desc(irq, desc); } return true; diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c index bc4604e500a3..ff16ccb918f2 100644 --- a/arch/x86/kernel/irq_64.c +++ b/arch/x86/kernel/irq_64.c @@ -68,16 +68,13 @@ static inline void stack_overflow_check(struct pt_regs *regs) #endif } -bool handle_irq(unsigned irq, struct pt_regs *regs) +bool handle_irq(struct irq_desc *desc, struct pt_regs *regs) { - struct irq_desc *desc; - stack_overflow_check(regs); - desc = irq_to_desc(irq); - if (unlikely(!desc)) + if (unlikely(IS_ERR_OR_NULL(desc))) return false; - generic_handle_irq_desc(irq, desc); + generic_handle_irq_desc(irq_desc_get_irq(desc), desc); return true; } diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c index a3a5e158ed69..1423ab1b0312 100644 --- a/arch/x86/kernel/irqinit.c +++ b/arch/x86/kernel/irqinit.c @@ -52,7 +52,7 @@ static struct irqaction irq2 = { }; DEFINE_PER_CPU(vector_irq_t, vector_irq) = { - [0 ... NR_VECTORS - 1] = VECTOR_UNDEFINED, + [0 ... NR_VECTORS - 1] = VECTOR_UNUSED, }; int vector_used_by_percpu_irq(unsigned int vector) @@ -60,7 +60,7 @@ int vector_used_by_percpu_irq(unsigned int vector) int cpu; for_each_online_cpu(cpu) { - if (per_cpu(vector_irq, cpu)[vector] > VECTOR_UNDEFINED) + if (!IS_ERR_OR_NULL(per_cpu(vector_irq, cpu)[vector])) return 1; } @@ -94,7 +94,7 @@ void __init init_IRQ(void) * irq's migrate etc. */ for (i = 0; i < nr_legacy_irqs(); i++) - per_cpu(vector_irq, 0)[ISA_IRQ_VECTOR(i)] = i; + per_cpu(vector_irq, 0)[ISA_IRQ_VECTOR(i)] = irq_to_desc(i); x86_init.irqs.intr_init(); } diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c index 26d5a55a2736..e565e0e4d216 100644 --- a/arch/x86/kernel/jump_label.c +++ b/arch/x86/kernel/jump_label.c @@ -45,7 +45,7 @@ static void __jump_label_transform(struct jump_entry *entry, const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP }; const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5]; - if (type == JUMP_LABEL_ENABLE) { + if (type == JUMP_LABEL_JMP) { if (init) { /* * Jump label is enabled for the first time. diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c index 961e51e9c6f6..0f8a6bbaaa44 100644 --- a/arch/x86/kernel/kexec-bzimage64.c +++ b/arch/x86/kernel/kexec-bzimage64.c @@ -533,7 +533,9 @@ static int bzImage64_verify_sig(const char *kernel, unsigned long kernel_len) int ret; ret = verify_pefile_signature(kernel, kernel_len, - system_trusted_keyring, &trusted); + system_trusted_keyring, + VERIFYING_KEXEC_PE_SIGNATURE, + &trusted); if (ret < 0) return ret; if (!trusted) diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c index 49487b488061..2c7aafa70702 100644 --- a/arch/x86/kernel/kvmclock.c +++ b/arch/x86/kernel/kvmclock.c @@ -200,7 +200,7 @@ static void kvm_setup_secondary_clock(void) * kind of shutdown from our side, we unregister the clock by writting anything * that does not have the 'enable' bit set in the msr */ -#ifdef CONFIG_KEXEC +#ifdef CONFIG_KEXEC_CORE static void kvm_crash_shutdown(struct pt_regs *regs) { native_write_msr(msr_kvm_system_time, 0, 0); @@ -259,7 +259,7 @@ void __init kvmclock_init(void) x86_platform.save_sched_clock_state = kvm_save_sched_clock_state; x86_platform.restore_sched_clock_state = kvm_restore_sched_clock_state; machine_ops.shutdown = kvm_shutdown; -#ifdef CONFIG_KEXEC +#ifdef CONFIG_KEXEC_CORE machine_ops.crash_shutdown = kvm_crash_shutdown; #endif kvm_get_preset_lpj(); diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c index 353972c1946c..84b8ef82a159 100644 --- a/arch/x86/kernel/pci-dma.c +++ b/arch/x86/kernel/pci-dma.c @@ -58,17 +58,6 @@ EXPORT_SYMBOL(x86_dma_fallback_dev); /* Number of entries preallocated for DMA-API debugging */ #define PREALLOC_DMA_DEBUG_ENTRIES 65536 -int dma_set_mask(struct device *dev, u64 mask) -{ - if (!dev->dma_mask || !dma_supported(dev, mask)) - return -EIO; - - *dev->dma_mask = mask; - - return 0; -} -EXPORT_SYMBOL(dma_set_mask); - void __init pci_iommu_alloc(void) { struct iommu_table_entry *p; @@ -140,50 +129,19 @@ void dma_generic_free_coherent(struct device *dev, size_t size, void *vaddr, free_pages((unsigned long)vaddr, get_order(size)); } -void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, - gfp_t gfp, struct dma_attrs *attrs) +bool arch_dma_alloc_attrs(struct device **dev, gfp_t *gfp) { - struct dma_map_ops *ops = get_dma_ops(dev); - void *memory; + *gfp = dma_alloc_coherent_gfp_flags(*dev, *gfp); + *gfp &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32); - gfp &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32); + if (!*dev) + *dev = &x86_dma_fallback_dev; + if (!is_device_dma_capable(*dev)) + return false; + return true; - if (dma_alloc_from_coherent(dev, size, dma_handle, &memory)) - return memory; - - if (!dev) - dev = &x86_dma_fallback_dev; - - if (!is_device_dma_capable(dev)) - return NULL; - - if (!ops->alloc) - return NULL; - - memory = ops->alloc(dev, size, dma_handle, - dma_alloc_coherent_gfp_flags(dev, gfp), attrs); - debug_dma_alloc_coherent(dev, size, *dma_handle, memory); - - return memory; } -EXPORT_SYMBOL(dma_alloc_attrs); - -void dma_free_attrs(struct device *dev, size_t size, - void *vaddr, dma_addr_t bus, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - - WARN_ON(irqs_disabled()); /* for portability */ - - if (dma_release_from_coherent(dev, get_order(size), vaddr)) - return; - - debug_dma_free_coherent(dev, size, vaddr, bus); - if (ops->free) - ops->free(dev, size, vaddr, bus, attrs); -} -EXPORT_SYMBOL(dma_free_attrs); +EXPORT_SYMBOL(arch_dma_alloc_attrs); /* * See for the iommu kernel diff --git a/arch/x86/kernel/pmem.c b/arch/x86/kernel/pmem.c index 64f90f53bb85..4f00b63d7ff3 100644 --- a/arch/x86/kernel/pmem.c +++ b/arch/x86/kernel/pmem.c @@ -3,80 +3,17 @@ * Copyright (c) 2015, Intel Corporation. */ #include -#include #include -#include - -static void e820_pmem_release(struct device *dev) -{ - struct nvdimm_bus *nvdimm_bus = dev->platform_data; - - if (nvdimm_bus) - nvdimm_bus_unregister(nvdimm_bus); -} - -static struct platform_device e820_pmem = { - .name = "e820_pmem", - .id = -1, - .dev = { - .release = e820_pmem_release, - }, -}; - -static const struct attribute_group *e820_pmem_attribute_groups[] = { - &nvdimm_bus_attribute_group, - NULL, -}; - -static const struct attribute_group *e820_pmem_region_attribute_groups[] = { - &nd_region_attribute_group, - &nd_device_attribute_group, - NULL, -}; static __init int register_e820_pmem(void) { - static struct nvdimm_bus_descriptor nd_desc; - struct device *dev = &e820_pmem.dev; - struct nvdimm_bus *nvdimm_bus; - int rc, i; + struct platform_device *pdev; - rc = platform_device_register(&e820_pmem); - if (rc) - return rc; - - nd_desc.attr_groups = e820_pmem_attribute_groups; - nd_desc.provider_name = "e820"; - nvdimm_bus = nvdimm_bus_register(dev, &nd_desc); - if (!nvdimm_bus) - goto err; - dev->platform_data = nvdimm_bus; - - for (i = 0; i < e820.nr_map; i++) { - struct e820entry *ei = &e820.map[i]; - struct resource res = { - .flags = IORESOURCE_MEM, - .start = ei->addr, - .end = ei->addr + ei->size - 1, - }; - struct nd_region_desc ndr_desc; - - if (ei->type != E820_PRAM) - continue; - - memset(&ndr_desc, 0, sizeof(ndr_desc)); - ndr_desc.res = &res; - ndr_desc.attr_groups = e820_pmem_region_attribute_groups; - ndr_desc.numa_node = NUMA_NO_NODE; - if (!nvdimm_pmem_region_create(nvdimm_bus, &ndr_desc)) - goto err; - } - - return 0; - - err: - dev_err(dev, "failed to register legacy persistent memory ranges\n"); - platform_device_unregister(&e820_pmem); - return -ENXIO; + /* + * See drivers/nvdimm/e820.c for the implementation, this is + * simply here to trigger the module to load on demand. + */ + pdev = platform_device_alloc("e820_pmem", -1); + return platform_device_add(pdev); } device_initcall(register_e820_pmem); diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c index 86db4bcd7ce5..02693dd9a079 100644 --- a/arch/x86/kernel/reboot.c +++ b/arch/x86/kernel/reboot.c @@ -673,7 +673,7 @@ struct machine_ops machine_ops = { .emergency_restart = native_machine_emergency_restart, .restart = native_machine_restart, .halt = native_machine_halt, -#ifdef CONFIG_KEXEC +#ifdef CONFIG_KEXEC_CORE .crash_shutdown = native_machine_crash_shutdown, #endif }; @@ -703,7 +703,7 @@ void machine_halt(void) machine_ops.halt(); } -#ifdef CONFIG_KEXEC +#ifdef CONFIG_KEXEC_CORE void machine_crash_shutdown(struct pt_regs *regs) { machine_ops.crash_shutdown(regs); diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index b143c2d04420..fdb7f2a2d328 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -317,15 +317,12 @@ static u64 __init get_ramdisk_size(void) return ramdisk_size; } -#define MAX_MAP_CHUNK (NR_FIX_BTMAPS << PAGE_SHIFT) static void __init relocate_initrd(void) { /* Assume only end is not page aligned */ u64 ramdisk_image = get_ramdisk_image(); u64 ramdisk_size = get_ramdisk_size(); u64 area_size = PAGE_ALIGN(ramdisk_size); - unsigned long slop, clen, mapaddr; - char *p, *q; /* We need to move the initrd down into directly mapped mem */ relocated_ramdisk = memblock_find_in_range(0, PFN_PHYS(max_pfn_mapped), @@ -343,25 +340,8 @@ static void __init relocate_initrd(void) printk(KERN_INFO "Allocated new RAMDISK: [mem %#010llx-%#010llx]\n", relocated_ramdisk, relocated_ramdisk + ramdisk_size - 1); - q = (char *)initrd_start; + copy_from_early_mem((void *)initrd_start, ramdisk_image, ramdisk_size); - /* Copy the initrd */ - while (ramdisk_size) { - slop = ramdisk_image & ~PAGE_MASK; - clen = ramdisk_size; - if (clen > MAX_MAP_CHUNK-slop) - clen = MAX_MAP_CHUNK-slop; - mapaddr = ramdisk_image & PAGE_MASK; - p = early_memremap(mapaddr, clen+slop); - memcpy(q, p+slop, clen); - early_memunmap(p, clen+slop); - q += clen; - ramdisk_image += clen; - ramdisk_size -= clen; - } - - ramdisk_image = get_ramdisk_image(); - ramdisk_size = get_ramdisk_size(); printk(KERN_INFO "Move RAMDISK from [mem %#010llx-%#010llx] to" " [mem %#010llx-%#010llx]\n", ramdisk_image, ramdisk_image + ramdisk_size - 1, @@ -498,7 +478,7 @@ static void __init memblock_x86_reserve_range_setup_data(void) * --------- Crashkernel reservation ------------------------------ */ -#ifdef CONFIG_KEXEC +#ifdef CONFIG_KEXEC_CORE /* * Keep the crash kernel below this limit. On 32 bits earlier kernels diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 79055cf2c497..c8d52cb4cb6e 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -38,7 +38,7 @@ static int __read_mostly tsc_unstable; erroneous rdtsc usage on !cpu_has_tsc processors */ static int __read_mostly tsc_disabled = -1; -static struct static_key __use_tsc = STATIC_KEY_INIT; +static DEFINE_STATIC_KEY_FALSE(__use_tsc); int tsc_clocksource_reliable; @@ -274,7 +274,12 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu) */ u64 native_sched_clock(void) { - u64 tsc_now; + if (static_branch_likely(&__use_tsc)) { + u64 tsc_now = rdtsc(); + + /* return the value in ns */ + return cycles_2_ns(tsc_now); + } /* * Fall back to jiffies if there's no TSC available: @@ -284,16 +289,9 @@ u64 native_sched_clock(void) * very important for it to be as fast as the platform * can achieve it. ) */ - if (!static_key_false(&__use_tsc)) { - /* No locking but a rare wrong value is not a big deal: */ - return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ); - } - /* read the Time Stamp Counter: */ - tsc_now = rdtsc(); - - /* return the value in ns */ - return cycles_2_ns(tsc_now); + /* No locking but a rare wrong value is not a big deal: */ + return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ); } /* @@ -1212,7 +1210,7 @@ void __init tsc_init(void) /* now allow native_sched_clock() to use rdtsc */ tsc_disabled = 0; - static_key_slow_inc(&__use_tsc); + static_branch_enable(&__use_tsc); if (!no_sched_irq_time) enable_sched_clock_irqtime(); diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index 00bf300fd846..74e4bf11f562 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -364,7 +364,7 @@ INIT_PER_CPU(irq_stack_union); #endif /* CONFIG_X86_32 */ -#ifdef CONFIG_KEXEC +#ifdef CONFIG_KEXEC_CORE #include . = ASSERT(kexec_control_code_size <= KEXEC_CONTROL_CODE_MAX_SIZE, diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index e7a4fde5d631..b372a7557c16 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -650,6 +650,7 @@ static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt, u16 sel; la = seg_base(ctxt, addr.seg) + addr.ea; + *linear = la; *max_size = 0; switch (mode) { case X86EMUL_MODE_PROT64: @@ -693,7 +694,6 @@ static __always_inline int __linearize(struct x86_emulate_ctxt *ctxt, } if (insn_aligned(ctxt, size) && ((la & (size - 1)) != 0)) return emulate_gp(ctxt, 0); - *linear = la; return X86EMUL_CONTINUE; bad: if (addr.seg == VCPU_SREG_SS) diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c index fb16a8ea3dee..69088a1ba509 100644 --- a/arch/x86/kvm/mmu.c +++ b/arch/x86/kvm/mmu.c @@ -3309,13 +3309,14 @@ walk_shadow_page_get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep) walk_shadow_page_lockless_begin(vcpu); - for (shadow_walk_init(&iterator, vcpu, addr), root = iterator.level; + for (shadow_walk_init(&iterator, vcpu, addr), + leaf = root = iterator.level; shadow_walk_okay(&iterator); __shadow_walk_next(&iterator, spte)) { - leaf = iterator.level; spte = mmu_spte_get_lockless(iterator.sptep); sptes[leaf - 1] = spte; + leaf--; if (!is_shadow_present_pte(spte)) break; @@ -3329,7 +3330,7 @@ walk_shadow_page_get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep) if (reserved) { pr_err("%s: detect reserved bits on spte, addr 0x%llx, dump hierarchy:\n", __func__, addr); - while (root >= leaf) { + while (root > leaf) { pr_err("------ spte 0x%llx level %d.\n", sptes[root - 1], root); root--; diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index 4a4eec30cc08..d01986832afc 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c @@ -1264,7 +1264,7 @@ static void vmcs_load(struct vmcs *vmcs) vmcs, phys_addr); } -#ifdef CONFIG_KEXEC +#ifdef CONFIG_KEXEC_CORE /* * This bitmap is used to indicate whether the vmclear * operation is enabled on all cpus. All disabled by @@ -1302,7 +1302,7 @@ static void crash_vmclear_local_loaded_vmcss(void) #else static inline void crash_enable_local_vmclear(int cpu) { } static inline void crash_disable_local_vmclear(int cpu) { } -#endif /* CONFIG_KEXEC */ +#endif /* CONFIG_KEXEC_CORE */ static void __loaded_vmcs_clear(void *arg) { @@ -3150,7 +3150,7 @@ static struct vmcs *alloc_vmcs_cpu(int cpu) struct page *pages; struct vmcs *vmcs; - pages = alloc_pages_exact_node(node, GFP_KERNEL, vmcs_config.order); + pages = __alloc_pages_node(node, GFP_KERNEL, vmcs_config.order); if (!pages) return NULL; vmcs = page_address(pages); @@ -10411,7 +10411,7 @@ static int __init vmx_init(void) if (r) return r; -#ifdef CONFIG_KEXEC +#ifdef CONFIG_KEXEC_CORE rcu_assign_pointer(crash_vmclear_loaded_vmcss, crash_vmclear_local_loaded_vmcss); #endif @@ -10421,7 +10421,7 @@ static int __init vmx_init(void) static void __exit vmx_exit(void) { -#ifdef CONFIG_KEXEC +#ifdef CONFIG_KEXEC_CORE RCU_INIT_POINTER(crash_vmclear_loaded_vmcss, NULL); synchronize_rcu(); #endif diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 1e7e76e14e89..a60bdbccff51 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -5943,6 +5943,7 @@ static void process_smi_save_seg_32(struct kvm_vcpu *vcpu, char *buf, int n) put_smstate(u32, buf, offset, process_smi_get_segment_flags(&seg)); } +#ifdef CONFIG_X86_64 static void process_smi_save_seg_64(struct kvm_vcpu *vcpu, char *buf, int n) { struct kvm_segment seg; @@ -5958,6 +5959,7 @@ static void process_smi_save_seg_64(struct kvm_vcpu *vcpu, char *buf, int n) put_smstate(u32, buf, offset + 4, seg.limit); put_smstate(u64, buf, offset + 8, seg.base); } +#endif static void process_smi_save_state_32(struct kvm_vcpu *vcpu, char *buf) { diff --git a/arch/x86/lguest/boot.c b/arch/x86/lguest/boot.c index 433e5a7dd37f..161804de124a 100644 --- a/arch/x86/lguest/boot.c +++ b/arch/x86/lguest/boot.c @@ -835,16 +835,46 @@ static struct irq_chip lguest_irq_controller = { .irq_unmask = enable_lguest_irq, }; +/* + * Interrupt descriptors are allocated as-needed, but low-numbered ones are + * reserved by the generic x86 code. So we ignore irq_alloc_desc_at if it + * tells us the irq is already used: other errors (ie. ENOMEM) we take + * seriously. + */ +static int lguest_setup_irq(unsigned int irq) +{ + struct irq_desc *desc; + int err; + + /* Returns -ve error or vector number. */ + err = irq_alloc_desc_at(irq, 0); + if (err < 0 && err != -EEXIST) + return err; + + /* + * Tell the Linux infrastructure that the interrupt is + * controlled by our level-based lguest interrupt controller. + */ + irq_set_chip_and_handler_name(irq, &lguest_irq_controller, + handle_level_irq, "level"); + + /* Some systems map "vectors" to interrupts weirdly. Not us! */ + desc = irq_to_desc(irq); + __this_cpu_write(vector_irq[FIRST_EXTERNAL_VECTOR + irq], desc); + return 0; +} + static int lguest_enable_irq(struct pci_dev *dev) { + int err; u8 line = 0; /* We literally use the PCI interrupt line as the irq number. */ pci_read_config_byte(dev, PCI_INTERRUPT_LINE, &line); - irq_set_chip_and_handler_name(line, &lguest_irq_controller, - handle_level_irq, "level"); - dev->irq = line; - return 0; + err = lguest_setup_irq(line); + if (!err) + dev->irq = line; + return err; } /* We don't do hotplug PCI, so this shouldn't be called. */ @@ -855,17 +885,13 @@ static void lguest_disable_irq(struct pci_dev *dev) /* * This sets up the Interrupt Descriptor Table (IDT) entry for each hardware - * interrupt (except 128, which is used for system calls), and then tells the - * Linux infrastructure that each interrupt is controlled by our level-based - * lguest interrupt controller. + * interrupt (except 128, which is used for system calls). */ static void __init lguest_init_IRQ(void) { unsigned int i; for (i = FIRST_EXTERNAL_VECTOR; i < FIRST_SYSTEM_VECTOR; i++) { - /* Some systems map "vectors" to interrupts weirdly. Not us! */ - __this_cpu_write(vector_irq[i], i - FIRST_EXTERNAL_VECTOR); if (i != IA32_SYSCALL_VECTOR) set_intr_gate(i, irq_entries_start + 8 * (i - FIRST_EXTERNAL_VECTOR)); @@ -878,26 +904,6 @@ static void __init lguest_init_IRQ(void) irq_ctx_init(smp_processor_id()); } -/* - * Interrupt descriptors are allocated as-needed, but low-numbered ones are - * reserved by the generic x86 code. So we ignore irq_alloc_desc_at if it - * tells us the irq is already used: other errors (ie. ENOMEM) we take - * seriously. - */ -int lguest_setup_irq(unsigned int irq) -{ - int err; - - /* Returns -ve error or vector number. */ - err = irq_alloc_desc_at(irq, 0); - if (err < 0 && err != -EEXIST) - return err; - - irq_set_chip_and_handler_name(irq, &lguest_irq_controller, - handle_level_irq, "level"); - return 0; -} - /* * Time. * @@ -1028,7 +1034,8 @@ static void lguest_time_irq(unsigned int irq, struct irq_desc *desc) static void lguest_time_init(void) { /* Set up the timer interrupt (0) to go to our simple timer routine */ - lguest_setup_irq(0); + if (lguest_setup_irq(0) != 0) + panic("Could not set up timer irq"); irq_set_handler(0, lguest_time_irq); clocksource_register_hz(&lguest_clock, NSEC_PER_SEC); diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c index 68aec42545c2..7562f42914b4 100644 --- a/arch/x86/mm/init_32.c +++ b/arch/x86/mm/init_32.c @@ -823,11 +823,11 @@ void __init mem_init(void) } #ifdef CONFIG_MEMORY_HOTPLUG -int arch_add_memory(int nid, u64 start, u64 size) +int arch_add_memory(int nid, u64 start, u64 size, bool for_device) { struct pglist_data *pgdata = NODE_DATA(nid); struct zone *zone = pgdata->node_zones + - zone_for_memory(nid, start, size, ZONE_HIGHMEM); + zone_for_memory(nid, start, size, ZONE_HIGHMEM, for_device); unsigned long start_pfn = start >> PAGE_SHIFT; unsigned long nr_pages = size >> PAGE_SHIFT; diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 3fba623e3ba5..30564e2752d3 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -687,11 +687,11 @@ static void update_end_of_memory_vars(u64 start, u64 size) * Memory is added always to NORMAL zone. This means you will never get * additional DMA/DMA32 memory. */ -int arch_add_memory(int nid, u64 start, u64 size) +int arch_add_memory(int nid, u64 start, u64 size, bool for_device) { struct pglist_data *pgdat = NODE_DATA(nid); struct zone *zone = pgdat->node_zones + - zone_for_memory(nid, start, size, ZONE_NORMAL); + zone_for_memory(nid, start, size, ZONE_NORMAL, for_device); unsigned long start_pfn = start >> PAGE_SHIFT; unsigned long nr_pages = size >> PAGE_SHIFT; int ret; diff --git a/arch/x86/mm/mpx.c b/arch/x86/mm/mpx.c index db1b0bc5017c..134948b0926f 100644 --- a/arch/x86/mm/mpx.c +++ b/arch/x86/mm/mpx.c @@ -42,58 +42,21 @@ static inline unsigned long mpx_bt_size_bytes(struct mm_struct *mm) */ static unsigned long mpx_mmap(unsigned long len) { - unsigned long ret; - unsigned long addr, pgoff; struct mm_struct *mm = current->mm; - vm_flags_t vm_flags; - struct vm_area_struct *vma; + unsigned long addr, populate; /* Only bounds table can be allocated here */ if (len != mpx_bt_size_bytes(mm)) return -EINVAL; down_write(&mm->mmap_sem); - - /* Too many mappings? */ - if (mm->map_count > sysctl_max_map_count) { - ret = -ENOMEM; - goto out; - } - - /* Obtain the address to map to. we verify (or select) it and ensure - * that it represents a valid section of the address space. - */ - addr = get_unmapped_area(NULL, 0, len, 0, MAP_ANONYMOUS | MAP_PRIVATE); - if (addr & ~PAGE_MASK) { - ret = addr; - goto out; - } - - vm_flags = VM_READ | VM_WRITE | VM_MPX | - mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; - - /* Set pgoff according to addr for anon_vma */ - pgoff = addr >> PAGE_SHIFT; - - ret = mmap_region(NULL, addr, len, vm_flags, pgoff); - if (IS_ERR_VALUE(ret)) - goto out; - - vma = find_vma(mm, ret); - if (!vma) { - ret = -ENOMEM; - goto out; - } - - if (vm_flags & VM_LOCKED) { - up_write(&mm->mmap_sem); - mm_populate(ret, len); - return ret; - } - -out: + addr = do_mmap(NULL, 0, len, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, VM_MPX, 0, &populate); up_write(&mm->mmap_sem); - return ret; + if (populate) + mm_populate(addr, populate); + + return addr; } enum reg_type { diff --git a/arch/x86/mm/numa.c b/arch/x86/mm/numa.c index 4053bb58bf92..c3b3f653ed0c 100644 --- a/arch/x86/mm/numa.c +++ b/arch/x86/mm/numa.c @@ -246,8 +246,10 @@ int __init numa_cleanup_meminfo(struct numa_meminfo *mi) bi->start = max(bi->start, low); bi->end = min(bi->end, high); - /* and there's no empty block */ - if (bi->start >= bi->end) + /* and there's no empty or non-exist block */ + if (bi->start >= bi->end || + !memblock_overlaps_region(&memblock.memory, + bi->start, bi->end - bi->start)) numa_remove_memblk_from(i--, mi); } diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 90b924acd982..8ddb5d0d66fb 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -140,6 +140,7 @@ void native_flush_tlb_others(const struct cpumask *cpumask, info.flush_end = end; count_vm_tlb_event(NR_TLB_REMOTE_FLUSH); + trace_tlb_flush(TLB_REMOTE_SEND_IPI, end - start); if (is_uv_system()) { unsigned int cpu; diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index be2e7a2b10d7..70efcd0940f9 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -246,7 +246,7 @@ static void emit_prologue(u8 **pprog) * goto out; * if (++tail_call_cnt > MAX_TAIL_CALL_CNT) * goto out; - * prog = array->prog[index]; + * prog = array->ptrs[index]; * if (prog == NULL) * goto out; * goto *(prog->bpf_func + prologue_size); @@ -284,9 +284,9 @@ static void emit_bpf_tail_call(u8 **pprog) EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */ EMIT2_off32(0x89, 0x85, -STACKSIZE + 36); /* mov dword ptr [rbp - 516], eax */ - /* prog = array->prog[index]; */ + /* prog = array->ptrs[index]; */ EMIT4_off32(0x48, 0x8D, 0x84, 0xD6, /* lea rax, [rsi + rdx * 8 + offsetof(...)] */ - offsetof(struct bpf_array, prog)); + offsetof(struct bpf_array, ptrs)); EMIT3(0x48, 0x8B, 0x00); /* mov rax, qword ptr [rax] */ /* if (prog == NULL) @@ -315,6 +315,26 @@ static void emit_bpf_tail_call(u8 **pprog) *pprog = prog; } + +static void emit_load_skb_data_hlen(u8 **pprog) +{ + u8 *prog = *pprog; + int cnt = 0; + + /* r9d = skb->len - skb->data_len (headlen) + * r10 = skb->data + */ + /* mov %r9d, off32(%rdi) */ + EMIT3_off32(0x44, 0x8b, 0x8f, offsetof(struct sk_buff, len)); + + /* sub %r9d, off32(%rdi) */ + EMIT3_off32(0x44, 0x2b, 0x8f, offsetof(struct sk_buff, data_len)); + + /* mov %r10, off32(%rdi) */ + EMIT3_off32(0x4c, 0x8b, 0x97, offsetof(struct sk_buff, data)); + *pprog = prog; +} + static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, int oldproglen, struct jit_context *ctx) { @@ -329,36 +349,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, emit_prologue(&prog); - if (seen_ld_abs) { - /* r9d : skb->len - skb->data_len (headlen) - * r10 : skb->data - */ - if (is_imm8(offsetof(struct sk_buff, len))) - /* mov %r9d, off8(%rdi) */ - EMIT4(0x44, 0x8b, 0x4f, - offsetof(struct sk_buff, len)); - else - /* mov %r9d, off32(%rdi) */ - EMIT3_off32(0x44, 0x8b, 0x8f, - offsetof(struct sk_buff, len)); - - if (is_imm8(offsetof(struct sk_buff, data_len))) - /* sub %r9d, off8(%rdi) */ - EMIT4(0x44, 0x2b, 0x4f, - offsetof(struct sk_buff, data_len)); - else - EMIT3_off32(0x44, 0x2b, 0x8f, - offsetof(struct sk_buff, data_len)); - - if (is_imm8(offsetof(struct sk_buff, data))) - /* mov %r10, off8(%rdi) */ - EMIT4(0x4c, 0x8b, 0x57, - offsetof(struct sk_buff, data)); - else - /* mov %r10, off32(%rdi) */ - EMIT3_off32(0x4c, 0x8b, 0x97, - offsetof(struct sk_buff, data)); - } + if (seen_ld_abs) + emit_load_skb_data_hlen(&prog); for (i = 0; i < insn_cnt; i++, insn++) { const s32 imm32 = insn->imm; @@ -367,6 +359,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 b1 = 0, b2 = 0, b3 = 0; s64 jmp_offset; u8 jmp_cond; + bool reload_skb_data; int ilen; u8 *func; @@ -818,12 +811,18 @@ xadd: if (is_imm8(insn->off)) func = (u8 *) __bpf_call_base + imm32; jmp_offset = func - (image + addrs[i]); if (seen_ld_abs) { - EMIT2(0x41, 0x52); /* push %r10 */ - EMIT2(0x41, 0x51); /* push %r9 */ - /* need to adjust jmp offset, since - * pop %r9, pop %r10 take 4 bytes after call insn - */ - jmp_offset += 4; + reload_skb_data = bpf_helper_changes_skb_data(func); + if (reload_skb_data) { + EMIT1(0x57); /* push %rdi */ + jmp_offset += 22; /* pop, mov, sub, mov */ + } else { + EMIT2(0x41, 0x52); /* push %r10 */ + EMIT2(0x41, 0x51); /* push %r9 */ + /* need to adjust jmp offset, since + * pop %r9, pop %r10 take 4 bytes after call insn + */ + jmp_offset += 4; + } } if (!imm32 || !is_simm32(jmp_offset)) { pr_err("unsupported bpf func %d addr %p image %p\n", @@ -832,8 +831,13 @@ xadd: if (is_imm8(insn->off)) } EMIT1_off32(0xE8, jmp_offset); if (seen_ld_abs) { - EMIT2(0x41, 0x59); /* pop %r9 */ - EMIT2(0x41, 0x5A); /* pop %r10 */ + if (reload_skb_data) { + EMIT1(0x5F); /* pop %rdi */ + emit_load_skb_data_hlen(&prog); + } else { + EMIT2(0x41, 0x59); /* pop %r9 */ + EMIT2(0x41, 0x5A); /* pop %r10 */ + } } break; @@ -1099,7 +1103,7 @@ void bpf_int_jit_compile(struct bpf_prog *prog) } if (bpf_jit_enable > 1) - bpf_jit_dump(prog->len, proglen, 0, image); + bpf_jit_dump(prog->len, proglen, pass + 1, image); if (image) { bpf_flush_icache(header, image + proglen); diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c index d22f4b5bbc04..ff31ab464213 100644 --- a/arch/x86/pci/xen.c +++ b/arch/x86/pci/xen.c @@ -179,7 +179,7 @@ static int xen_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) if (ret) goto error; i = 0; - list_for_each_entry(msidesc, &dev->msi_list, list) { + for_each_pci_msi_entry(msidesc, dev) { irq = xen_bind_pirq_msi_to_irq(dev, msidesc, v[i], (type == PCI_CAP_ID_MSI) ? nvec : 1, (type == PCI_CAP_ID_MSIX) ? @@ -230,7 +230,7 @@ static int xen_hvm_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) if (type == PCI_CAP_ID_MSI && nvec > 1) return 1; - list_for_each_entry(msidesc, &dev->msi_list, list) { + for_each_pci_msi_entry(msidesc, dev) { __pci_read_msi_msg(msidesc, &msg); pirq = MSI_ADDR_EXT_DEST_ID(msg.address_hi) | ((msg.address_lo >> MSI_ADDR_DEST_ID_SHIFT) & 0xff); @@ -274,7 +274,7 @@ static int xen_initdom_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) int ret = 0; struct msi_desc *msidesc; - list_for_each_entry(msidesc, &dev->msi_list, list) { + for_each_pci_msi_entry(msidesc, dev) { struct physdev_map_pirq map_irq; domid_t domid; @@ -386,7 +386,7 @@ static void xen_teardown_msi_irqs(struct pci_dev *dev) { struct msi_desc *msidesc; - msidesc = list_entry(dev->msi_list.next, struct msi_desc, list); + msidesc = first_pci_msi_entry(dev); if (msidesc->msi_attrib.is_msix) xen_pci_frontend_disable_msix(dev); else diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c index e4308fe6afe8..1db84c0758b7 100644 --- a/arch/x86/platform/efi/efi.c +++ b/arch/x86/platform/efi/efi.c @@ -650,7 +650,7 @@ static void __init get_systab_virt_addr(efi_memory_desc_t *md) static void __init save_runtime_map(void) { -#ifdef CONFIG_KEXEC +#ifdef CONFIG_KEXEC_CORE efi_memory_desc_t *md; void *tmp, *p, *q = NULL; int count = 0; @@ -748,7 +748,7 @@ static void * __init efi_map_regions(int *count, int *pg_shift) static void __init kexec_enter_virtual_mode(void) { -#ifdef CONFIG_KEXEC +#ifdef CONFIG_KEXEC_CORE efi_memory_desc_t *md; void *p; diff --git a/arch/x86/platform/uv/uv_irq.c b/arch/x86/platform/uv/uv_irq.c index 8570abe68be1..e1c24631afbb 100644 --- a/arch/x86/platform/uv/uv_irq.c +++ b/arch/x86/platform/uv/uv_irq.c @@ -89,7 +89,7 @@ static int uv_domain_alloc(struct irq_domain *domain, unsigned int virq, return -EINVAL; chip_data = kmalloc_node(sizeof(*chip_data), GFP_KERNEL, - irq_data->node); + irq_data_get_node(irq_data)); if (!chip_data) return -ENOMEM; diff --git a/arch/x86/platform/uv/uv_nmi.c b/arch/x86/platform/uv/uv_nmi.c index 020c101c255f..5c9f63fa6abf 100644 --- a/arch/x86/platform/uv/uv_nmi.c +++ b/arch/x86/platform/uv/uv_nmi.c @@ -492,7 +492,7 @@ static void uv_nmi_touch_watchdogs(void) touch_nmi_watchdog(); } -#if defined(CONFIG_KEXEC) +#if defined(CONFIG_KEXEC_CORE) static atomic_t uv_nmi_kexec_failed; static void uv_nmi_kdump(int cpu, int master, struct pt_regs *regs) { @@ -519,13 +519,13 @@ static void uv_nmi_kdump(int cpu, int master, struct pt_regs *regs) uv_nmi_sync_exit(0); } -#else /* !CONFIG_KEXEC */ +#else /* !CONFIG_KEXEC_CORE */ static inline void uv_nmi_kdump(int cpu, int master, struct pt_regs *regs) { if (master) pr_err("UV: NMI kdump: KEXEC not supported in this kernel\n"); } -#endif /* !CONFIG_KEXEC */ +#endif /* !CONFIG_KEXEC_CORE */ #ifdef CONFIG_KGDB #ifdef CONFIG_KGDB_KDB diff --git a/arch/x86/xen/Kconfig b/arch/x86/xen/Kconfig index 484145368a24..c7b15f3e2cf3 100644 --- a/arch/x86/xen/Kconfig +++ b/arch/x86/xen/Kconfig @@ -7,6 +7,7 @@ config XEN depends on PARAVIRT select PARAVIRT_CLOCK select XEN_HAVE_PVMMU + select XEN_HAVE_VPMU depends on X86_64 || (X86_32 && X86_PAE) depends on X86_LOCAL_APIC && X86_TSC help @@ -23,14 +24,18 @@ config XEN_PVHVM def_bool y depends on XEN && PCI && X86_LOCAL_APIC -config XEN_MAX_DOMAIN_MEMORY - int - default 500 if X86_64 - default 64 if X86_32 - depends on XEN - help - This only affects the sizing of some bss arrays, the unused - portions of which are freed. +config XEN_512GB + bool "Limit Xen pv-domain memory to 512GB" + depends on XEN && X86_64 + default y + help + Limit paravirtualized user domains to 512GB of RAM. + + The Xen tools and crash dump analysis tools might not support + pv-domains with more than 512 GB of RAM. This option controls the + default setting of the kernel to use only up to 512 GB or more. + It is always possible to change the default via specifying the + boot parameter "xen_512gb_limit". config XEN_SAVE_RESTORE bool diff --git a/arch/x86/xen/Makefile b/arch/x86/xen/Makefile index 4b6e29ac0968..e47e52787d32 100644 --- a/arch/x86/xen/Makefile +++ b/arch/x86/xen/Makefile @@ -13,7 +13,7 @@ CFLAGS_mmu.o := $(nostackp) obj-y := enlighten.o setup.o multicalls.o mmu.o irq.o \ time.o xen-asm.o xen-asm_$(BITS).o \ grant-table.o suspend.o platform-pci-unplug.o \ - p2m.o apic.o + p2m.o apic.o pmu.o obj-$(CONFIG_EVENT_TRACING) += trace.o diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c index 70e060ad879a..acda713ab5be 100644 --- a/arch/x86/xen/apic.c +++ b/arch/x86/xen/apic.c @@ -7,6 +7,7 @@ #include #include #include "xen-ops.h" +#include "pmu.h" #include "smp.h" static unsigned int xen_io_apic_read(unsigned apic, unsigned reg) @@ -72,6 +73,11 @@ static u32 xen_apic_read(u32 reg) static void xen_apic_write(u32 reg, u32 val) { + if (reg == APIC_LVTPC) { + (void)pmu_apic_update(reg); + return; + } + /* Warn to see if there's any stray references */ WARN(1,"register: %x, value: %x\n", reg, val); } diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index d9cfa452da9d..30d12afe52ed 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -84,6 +84,7 @@ #include "mmu.h" #include "smp.h" #include "multicalls.h" +#include "pmu.h" EXPORT_SYMBOL_GPL(hypercall_page); @@ -1010,8 +1011,7 @@ static void xen_write_cr0(unsigned long cr0) static void xen_write_cr4(unsigned long cr4) { - cr4 &= ~X86_CR4_PGE; - cr4 &= ~X86_CR4_PSE; + cr4 &= ~(X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PCE); native_write_cr4(cr4); } @@ -1030,6 +1030,9 @@ static u64 xen_read_msr_safe(unsigned int msr, int *err) { u64 val; + if (pmu_msr_read(msr, &val, err)) + return val; + val = native_read_msr_safe(msr, err); switch (msr) { case MSR_IA32_APICBASE: @@ -1076,7 +1079,8 @@ static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high) Xen console noise. */ default: - ret = native_write_msr_safe(msr, low, high); + if (!pmu_msr_write(msr, low, high, &ret)) + ret = native_write_msr_safe(msr, low, high); } return ret; @@ -1215,7 +1219,7 @@ static const struct pv_cpu_ops xen_cpu_ops __initconst = { .read_msr = xen_read_msr_safe, .write_msr = xen_write_msr_safe, - .read_pmc = native_read_pmc, + .read_pmc = xen_read_pmc, .iret = xen_iret, #ifdef CONFIG_X86_64 @@ -1264,6 +1268,10 @@ static const struct pv_apic_ops xen_apic_ops __initconst = { static void xen_reboot(int reason) { struct sched_shutdown r = { .reason = reason }; + int cpu; + + for_each_online_cpu(cpu) + xen_pmu_finish(cpu); if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r)) BUG(); @@ -1607,7 +1615,9 @@ asmlinkage __visible void __init xen_start_kernel(void) early_boot_irqs_disabled = true; xen_raw_console_write("mapping kernel into physical memory\n"); - xen_setup_kernel_pagetable((pgd_t *)xen_start_info->pt_base, xen_start_info->nr_pages); + xen_setup_kernel_pagetable((pgd_t *)xen_start_info->pt_base, + xen_start_info->nr_pages); + xen_reserve_special_pages(); /* * Modify the cache mode translation tables to match Xen's PAT diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c index dd151b2045b0..9c479fe40459 100644 --- a/arch/x86/xen/mmu.c +++ b/arch/x86/xen/mmu.c @@ -116,6 +116,7 @@ static pud_t level3_user_vsyscall[PTRS_PER_PUD] __page_aligned_bss; DEFINE_PER_CPU(unsigned long, xen_cr3); /* cr3 stored as physaddr */ DEFINE_PER_CPU(unsigned long, xen_current_cr3); /* actual vcpu cr3 */ +static phys_addr_t xen_pt_base, xen_pt_size __initdata; /* * Just beyond the highest usermode address. STACK_TOP_MAX has a @@ -1093,6 +1094,16 @@ static void xen_exit_mmap(struct mm_struct *mm) static void xen_post_allocator_init(void); +static void __init pin_pagetable_pfn(unsigned cmd, unsigned long pfn) +{ + struct mmuext_op op; + + op.cmd = cmd; + op.arg1.mfn = pfn_to_mfn(pfn); + if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF)) + BUG(); +} + #ifdef CONFIG_X86_64 static void __init xen_cleanhighmap(unsigned long vaddr, unsigned long vaddr_end) @@ -1114,6 +1125,83 @@ static void __init xen_cleanhighmap(unsigned long vaddr, xen_mc_flush(); } +/* + * Make a page range writeable and free it. + */ +static void __init xen_free_ro_pages(unsigned long paddr, unsigned long size) +{ + void *vaddr = __va(paddr); + void *vaddr_end = vaddr + size; + + for (; vaddr < vaddr_end; vaddr += PAGE_SIZE) + make_lowmem_page_readwrite(vaddr); + + memblock_free(paddr, size); +} + +static void __init xen_cleanmfnmap_free_pgtbl(void *pgtbl, bool unpin) +{ + unsigned long pa = __pa(pgtbl) & PHYSICAL_PAGE_MASK; + + if (unpin) + pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(pa)); + ClearPagePinned(virt_to_page(__va(pa))); + xen_free_ro_pages(pa, PAGE_SIZE); +} + +/* + * Since it is well isolated we can (and since it is perhaps large we should) + * also free the page tables mapping the initial P->M table. + */ +static void __init xen_cleanmfnmap(unsigned long vaddr) +{ + unsigned long va = vaddr & PMD_MASK; + unsigned long pa; + pgd_t *pgd = pgd_offset_k(va); + pud_t *pud_page = pud_offset(pgd, 0); + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + unsigned int i; + bool unpin; + + unpin = (vaddr == 2 * PGDIR_SIZE); + set_pgd(pgd, __pgd(0)); + do { + pud = pud_page + pud_index(va); + if (pud_none(*pud)) { + va += PUD_SIZE; + } else if (pud_large(*pud)) { + pa = pud_val(*pud) & PHYSICAL_PAGE_MASK; + xen_free_ro_pages(pa, PUD_SIZE); + va += PUD_SIZE; + } else { + pmd = pmd_offset(pud, va); + if (pmd_large(*pmd)) { + pa = pmd_val(*pmd) & PHYSICAL_PAGE_MASK; + xen_free_ro_pages(pa, PMD_SIZE); + } else if (!pmd_none(*pmd)) { + pte = pte_offset_kernel(pmd, va); + set_pmd(pmd, __pmd(0)); + for (i = 0; i < PTRS_PER_PTE; ++i) { + if (pte_none(pte[i])) + break; + pa = pte_pfn(pte[i]) << PAGE_SHIFT; + xen_free_ro_pages(pa, PAGE_SIZE); + } + xen_cleanmfnmap_free_pgtbl(pte, unpin); + } + va += PMD_SIZE; + if (pmd_index(va)) + continue; + set_pud(pud, __pud(0)); + xen_cleanmfnmap_free_pgtbl(pmd, unpin); + } + + } while (pud_index(va) || pmd_index(va)); + xen_cleanmfnmap_free_pgtbl(pud_page, unpin); +} + static void __init xen_pagetable_p2m_free(void) { unsigned long size; @@ -1128,18 +1216,31 @@ static void __init xen_pagetable_p2m_free(void) /* using __ka address and sticking INVALID_P2M_ENTRY! */ memset((void *)xen_start_info->mfn_list, 0xff, size); - /* We should be in __ka space. */ - BUG_ON(xen_start_info->mfn_list < __START_KERNEL_map); addr = xen_start_info->mfn_list; - /* We roundup to the PMD, which means that if anybody at this stage is - * using the __ka address of xen_start_info or xen_start_info->shared_info - * they are in going to crash. Fortunatly we have already revectored - * in xen_setup_kernel_pagetable and in xen_setup_shared_info. */ + /* + * We could be in __ka space. + * We roundup to the PMD, which means that if anybody at this stage is + * using the __ka address of xen_start_info or + * xen_start_info->shared_info they are in going to crash. Fortunatly + * we have already revectored in xen_setup_kernel_pagetable and in + * xen_setup_shared_info. + */ size = roundup(size, PMD_SIZE); - xen_cleanhighmap(addr, addr + size); - size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long)); - memblock_free(__pa(xen_start_info->mfn_list), size); + if (addr >= __START_KERNEL_map) { + xen_cleanhighmap(addr, addr + size); + size = PAGE_ALIGN(xen_start_info->nr_pages * + sizeof(unsigned long)); + memblock_free(__pa(addr), size); + } else { + xen_cleanmfnmap(addr); + } +} + +static void __init xen_pagetable_cleanhighmap(void) +{ + unsigned long size; + unsigned long addr; /* At this stage, cleanup_highmap has already cleaned __ka space * from _brk_limit way up to the max_pfn_mapped (which is the end of @@ -1172,6 +1273,8 @@ static void __init xen_pagetable_p2m_setup(void) #ifdef CONFIG_X86_64 xen_pagetable_p2m_free(); + + xen_pagetable_cleanhighmap(); #endif /* And revector! Bye bye old array */ xen_start_info->mfn_list = (unsigned long)xen_p2m_addr; @@ -1461,6 +1564,24 @@ static pte_t __init mask_rw_pte(pte_t *ptep, pte_t pte) #else /* CONFIG_X86_64 */ static pte_t __init mask_rw_pte(pte_t *ptep, pte_t pte) { + unsigned long pfn; + + if (xen_feature(XENFEAT_writable_page_tables) || + xen_feature(XENFEAT_auto_translated_physmap) || + xen_start_info->mfn_list >= __START_KERNEL_map) + return pte; + + /* + * Pages belonging to the initial p2m list mapped outside the default + * address range must be mapped read-only. This region contains the + * page tables for mapping the p2m list, too, and page tables MUST be + * mapped read-only. + */ + pfn = pte_pfn(pte); + if (pfn >= xen_start_info->first_p2m_pfn && + pfn < xen_start_info->first_p2m_pfn + xen_start_info->nr_p2m_frames) + pte = __pte_ma(pte_val_ma(pte) & ~_PAGE_RW); + return pte; } #endif /* CONFIG_X86_64 */ @@ -1489,15 +1610,6 @@ static void __init xen_set_pte_init(pte_t *ptep, pte_t pte) native_set_pte(ptep, pte); } -static void __init pin_pagetable_pfn(unsigned cmd, unsigned long pfn) -{ - struct mmuext_op op; - op.cmd = cmd; - op.arg1.mfn = pfn_to_mfn(pfn); - if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF)) - BUG(); -} - /* Early in boot, while setting up the initial pagetable, assume everything is pinned. */ static void __init xen_alloc_pte_init(struct mm_struct *mm, unsigned long pfn) @@ -1815,7 +1927,10 @@ void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn) * mappings. Considering that on Xen after the kernel mappings we * have the mappings of some pages that don't exist in pfn space, we * set max_pfn_mapped to the last real pfn mapped. */ - max_pfn_mapped = PFN_DOWN(__pa(xen_start_info->mfn_list)); + if (xen_start_info->mfn_list < __START_KERNEL_map) + max_pfn_mapped = xen_start_info->first_p2m_pfn; + else + max_pfn_mapped = PFN_DOWN(__pa(xen_start_info->mfn_list)); pt_base = PFN_DOWN(__pa(xen_start_info->pt_base)); pt_end = pt_base + xen_start_info->nr_pt_frames; @@ -1855,6 +1970,11 @@ void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn) /* Graft it onto L4[511][510] */ copy_page(level2_kernel_pgt, l2); + /* Copy the initial P->M table mappings if necessary. */ + i = pgd_index(xen_start_info->mfn_list); + if (i && i < pgd_index(__START_KERNEL_map)) + init_level4_pgt[i] = ((pgd_t *)xen_start_info->pt_base)[i]; + if (!xen_feature(XENFEAT_auto_translated_physmap)) { /* Make pagetable pieces RO */ set_page_prot(init_level4_pgt, PAGE_KERNEL_RO); @@ -1894,10 +2014,192 @@ void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn) check_pt_base(&pt_base, &pt_end, addr[i]); /* Our (by three pages) smaller Xen pagetable that we are using */ - memblock_reserve(PFN_PHYS(pt_base), (pt_end - pt_base) * PAGE_SIZE); + xen_pt_base = PFN_PHYS(pt_base); + xen_pt_size = (pt_end - pt_base) * PAGE_SIZE; + memblock_reserve(xen_pt_base, xen_pt_size); + /* Revector the xen_start_info */ xen_start_info = (struct start_info *)__va(__pa(xen_start_info)); } + +/* + * Read a value from a physical address. + */ +static unsigned long __init xen_read_phys_ulong(phys_addr_t addr) +{ + unsigned long *vaddr; + unsigned long val; + + vaddr = early_memremap_ro(addr, sizeof(val)); + val = *vaddr; + early_memunmap(vaddr, sizeof(val)); + return val; +} + +/* + * Translate a virtual address to a physical one without relying on mapped + * page tables. + */ +static phys_addr_t __init xen_early_virt_to_phys(unsigned long vaddr) +{ + phys_addr_t pa; + pgd_t pgd; + pud_t pud; + pmd_t pmd; + pte_t pte; + + pa = read_cr3(); + pgd = native_make_pgd(xen_read_phys_ulong(pa + pgd_index(vaddr) * + sizeof(pgd))); + if (!pgd_present(pgd)) + return 0; + + pa = pgd_val(pgd) & PTE_PFN_MASK; + pud = native_make_pud(xen_read_phys_ulong(pa + pud_index(vaddr) * + sizeof(pud))); + if (!pud_present(pud)) + return 0; + pa = pud_pfn(pud) << PAGE_SHIFT; + if (pud_large(pud)) + return pa + (vaddr & ~PUD_MASK); + + pmd = native_make_pmd(xen_read_phys_ulong(pa + pmd_index(vaddr) * + sizeof(pmd))); + if (!pmd_present(pmd)) + return 0; + pa = pmd_pfn(pmd) << PAGE_SHIFT; + if (pmd_large(pmd)) + return pa + (vaddr & ~PMD_MASK); + + pte = native_make_pte(xen_read_phys_ulong(pa + pte_index(vaddr) * + sizeof(pte))); + if (!pte_present(pte)) + return 0; + pa = pte_pfn(pte) << PAGE_SHIFT; + + return pa | (vaddr & ~PAGE_MASK); +} + +/* + * Find a new area for the hypervisor supplied p2m list and relocate the p2m to + * this area. + */ +void __init xen_relocate_p2m(void) +{ + phys_addr_t size, new_area, pt_phys, pmd_phys, pud_phys; + unsigned long p2m_pfn, p2m_pfn_end, n_frames, pfn, pfn_end; + int n_pte, n_pt, n_pmd, n_pud, idx_pte, idx_pt, idx_pmd, idx_pud; + pte_t *pt; + pmd_t *pmd; + pud_t *pud; + pgd_t *pgd; + unsigned long *new_p2m; + + size = PAGE_ALIGN(xen_start_info->nr_pages * sizeof(unsigned long)); + n_pte = roundup(size, PAGE_SIZE) >> PAGE_SHIFT; + n_pt = roundup(size, PMD_SIZE) >> PMD_SHIFT; + n_pmd = roundup(size, PUD_SIZE) >> PUD_SHIFT; + n_pud = roundup(size, PGDIR_SIZE) >> PGDIR_SHIFT; + n_frames = n_pte + n_pt + n_pmd + n_pud; + + new_area = xen_find_free_area(PFN_PHYS(n_frames)); + if (!new_area) { + xen_raw_console_write("Can't find new memory area for p2m needed due to E820 map conflict\n"); + BUG(); + } + + /* + * Setup the page tables for addressing the new p2m list. + * We have asked the hypervisor to map the p2m list at the user address + * PUD_SIZE. It may have done so, or it may have used a kernel space + * address depending on the Xen version. + * To avoid any possible virtual address collision, just use + * 2 * PUD_SIZE for the new area. + */ + pud_phys = new_area; + pmd_phys = pud_phys + PFN_PHYS(n_pud); + pt_phys = pmd_phys + PFN_PHYS(n_pmd); + p2m_pfn = PFN_DOWN(pt_phys) + n_pt; + + pgd = __va(read_cr3()); + new_p2m = (unsigned long *)(2 * PGDIR_SIZE); + for (idx_pud = 0; idx_pud < n_pud; idx_pud++) { + pud = early_memremap(pud_phys, PAGE_SIZE); + clear_page(pud); + for (idx_pmd = 0; idx_pmd < min(n_pmd, PTRS_PER_PUD); + idx_pmd++) { + pmd = early_memremap(pmd_phys, PAGE_SIZE); + clear_page(pmd); + for (idx_pt = 0; idx_pt < min(n_pt, PTRS_PER_PMD); + idx_pt++) { + pt = early_memremap(pt_phys, PAGE_SIZE); + clear_page(pt); + for (idx_pte = 0; + idx_pte < min(n_pte, PTRS_PER_PTE); + idx_pte++) { + set_pte(pt + idx_pte, + pfn_pte(p2m_pfn, PAGE_KERNEL)); + p2m_pfn++; + } + n_pte -= PTRS_PER_PTE; + early_memunmap(pt, PAGE_SIZE); + make_lowmem_page_readonly(__va(pt_phys)); + pin_pagetable_pfn(MMUEXT_PIN_L1_TABLE, + PFN_DOWN(pt_phys)); + set_pmd(pmd + idx_pt, + __pmd(_PAGE_TABLE | pt_phys)); + pt_phys += PAGE_SIZE; + } + n_pt -= PTRS_PER_PMD; + early_memunmap(pmd, PAGE_SIZE); + make_lowmem_page_readonly(__va(pmd_phys)); + pin_pagetable_pfn(MMUEXT_PIN_L2_TABLE, + PFN_DOWN(pmd_phys)); + set_pud(pud + idx_pmd, __pud(_PAGE_TABLE | pmd_phys)); + pmd_phys += PAGE_SIZE; + } + n_pmd -= PTRS_PER_PUD; + early_memunmap(pud, PAGE_SIZE); + make_lowmem_page_readonly(__va(pud_phys)); + pin_pagetable_pfn(MMUEXT_PIN_L3_TABLE, PFN_DOWN(pud_phys)); + set_pgd(pgd + 2 + idx_pud, __pgd(_PAGE_TABLE | pud_phys)); + pud_phys += PAGE_SIZE; + } + + /* Now copy the old p2m info to the new area. */ + memcpy(new_p2m, xen_p2m_addr, size); + xen_p2m_addr = new_p2m; + + /* Release the old p2m list and set new list info. */ + p2m_pfn = PFN_DOWN(xen_early_virt_to_phys(xen_start_info->mfn_list)); + BUG_ON(!p2m_pfn); + p2m_pfn_end = p2m_pfn + PFN_DOWN(size); + + if (xen_start_info->mfn_list < __START_KERNEL_map) { + pfn = xen_start_info->first_p2m_pfn; + pfn_end = xen_start_info->first_p2m_pfn + + xen_start_info->nr_p2m_frames; + set_pgd(pgd + 1, __pgd(0)); + } else { + pfn = p2m_pfn; + pfn_end = p2m_pfn_end; + } + + memblock_free(PFN_PHYS(pfn), PAGE_SIZE * (pfn_end - pfn)); + while (pfn < pfn_end) { + if (pfn == p2m_pfn) { + pfn = p2m_pfn_end; + continue; + } + make_lowmem_page_readwrite(__va(PFN_PHYS(pfn))); + pfn++; + } + + xen_start_info->mfn_list = (unsigned long)xen_p2m_addr; + xen_start_info->first_p2m_pfn = PFN_DOWN(new_area); + xen_start_info->nr_p2m_frames = n_frames; +} + #else /* !CONFIG_X86_64 */ static RESERVE_BRK_ARRAY(pmd_t, initial_kernel_pmd, PTRS_PER_PMD); static RESERVE_BRK_ARRAY(pmd_t, swapper_kernel_pmd, PTRS_PER_PMD); @@ -1938,18 +2240,41 @@ static void __init xen_write_cr3_init(unsigned long cr3) pv_mmu_ops.write_cr3 = &xen_write_cr3; } +/* + * For 32 bit domains xen_start_info->pt_base is the pgd address which might be + * not the first page table in the page table pool. + * Iterate through the initial page tables to find the real page table base. + */ +static phys_addr_t xen_find_pt_base(pmd_t *pmd) +{ + phys_addr_t pt_base, paddr; + unsigned pmdidx; + + pt_base = min(__pa(xen_start_info->pt_base), __pa(pmd)); + + for (pmdidx = 0; pmdidx < PTRS_PER_PMD; pmdidx++) + if (pmd_present(pmd[pmdidx]) && !pmd_large(pmd[pmdidx])) { + paddr = m2p(pmd[pmdidx].pmd); + pt_base = min(pt_base, paddr); + } + + return pt_base; +} + void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn) { pmd_t *kernel_pmd; + kernel_pmd = m2v(pgd[KERNEL_PGD_BOUNDARY].pgd); + + xen_pt_base = xen_find_pt_base(kernel_pmd); + xen_pt_size = xen_start_info->nr_pt_frames * PAGE_SIZE; + initial_kernel_pmd = extend_brk(sizeof(pmd_t) * PTRS_PER_PMD, PAGE_SIZE); - max_pfn_mapped = PFN_DOWN(__pa(xen_start_info->pt_base) + - xen_start_info->nr_pt_frames * PAGE_SIZE + - 512*1024); + max_pfn_mapped = PFN_DOWN(xen_pt_base + xen_pt_size + 512 * 1024); - kernel_pmd = m2v(pgd[KERNEL_PGD_BOUNDARY].pgd); copy_page(initial_kernel_pmd, kernel_pmd); xen_map_identity_early(initial_kernel_pmd, max_pfn); @@ -1968,11 +2293,33 @@ void __init xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn) PFN_DOWN(__pa(initial_page_table))); xen_write_cr3(__pa(initial_page_table)); - memblock_reserve(__pa(xen_start_info->pt_base), - xen_start_info->nr_pt_frames * PAGE_SIZE); + memblock_reserve(xen_pt_base, xen_pt_size); } #endif /* CONFIG_X86_64 */ +void __init xen_reserve_special_pages(void) +{ + phys_addr_t paddr; + + memblock_reserve(__pa(xen_start_info), PAGE_SIZE); + if (xen_start_info->store_mfn) { + paddr = PFN_PHYS(mfn_to_pfn(xen_start_info->store_mfn)); + memblock_reserve(paddr, PAGE_SIZE); + } + if (!xen_initial_domain()) { + paddr = PFN_PHYS(mfn_to_pfn(xen_start_info->console.domU.mfn)); + memblock_reserve(paddr, PAGE_SIZE); + } +} + +void __init xen_pt_check_e820(void) +{ + if (xen_is_e820_reserved(xen_pt_base, xen_pt_size)) { + xen_raw_console_write("Xen hypervisor allocated page table memory conflicts with E820 map\n"); + BUG(); + } +} + static unsigned char dummy_mapping[PAGE_SIZE] __page_aligned_bss; static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot) @@ -2465,9 +2812,9 @@ static int remap_area_mfn_pte_fn(pte_t *ptep, pgtable_t token, return 0; } -static int do_remap_mfn(struct vm_area_struct *vma, +static int do_remap_gfn(struct vm_area_struct *vma, unsigned long addr, - xen_pfn_t *mfn, int nr, + xen_pfn_t *gfn, int nr, int *err_ptr, pgprot_t prot, unsigned domid, struct page **pages) @@ -2483,14 +2830,14 @@ static int do_remap_mfn(struct vm_area_struct *vma, if (xen_feature(XENFEAT_auto_translated_physmap)) { #ifdef CONFIG_XEN_PVH /* We need to update the local page tables and the xen HAP */ - return xen_xlate_remap_gfn_array(vma, addr, mfn, nr, err_ptr, + return xen_xlate_remap_gfn_array(vma, addr, gfn, nr, err_ptr, prot, domid, pages); #else return -EINVAL; #endif } - rmd.mfn = mfn; + rmd.mfn = gfn; rmd.prot = prot; /* We use the err_ptr to indicate if there we are doing a contigious * mapping or a discontigious mapping. */ @@ -2518,8 +2865,8 @@ static int do_remap_mfn(struct vm_area_struct *vma, batch_left, &done, domid); /* - * @err_ptr may be the same buffer as @mfn, so - * only clear it after each chunk of @mfn is + * @err_ptr may be the same buffer as @gfn, so + * only clear it after each chunk of @gfn is * used. */ if (err_ptr) { @@ -2549,19 +2896,19 @@ static int do_remap_mfn(struct vm_area_struct *vma, return err < 0 ? err : mapped; } -int xen_remap_domain_mfn_range(struct vm_area_struct *vma, +int xen_remap_domain_gfn_range(struct vm_area_struct *vma, unsigned long addr, - xen_pfn_t mfn, int nr, + xen_pfn_t gfn, int nr, pgprot_t prot, unsigned domid, struct page **pages) { - return do_remap_mfn(vma, addr, &mfn, nr, NULL, prot, domid, pages); + return do_remap_gfn(vma, addr, &gfn, nr, NULL, prot, domid, pages); } -EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_range); +EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_range); -int xen_remap_domain_mfn_array(struct vm_area_struct *vma, +int xen_remap_domain_gfn_array(struct vm_area_struct *vma, unsigned long addr, - xen_pfn_t *mfn, int nr, + xen_pfn_t *gfn, int nr, int *err_ptr, pgprot_t prot, unsigned domid, struct page **pages) { @@ -2570,13 +2917,13 @@ int xen_remap_domain_mfn_array(struct vm_area_struct *vma, * cause of "wrong memory was mapped in". */ BUG_ON(err_ptr == NULL); - return do_remap_mfn(vma, addr, mfn, nr, err_ptr, prot, domid, pages); + return do_remap_gfn(vma, addr, gfn, nr, err_ptr, prot, domid, pages); } -EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_array); +EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_array); /* Returns: 0 success */ -int xen_unmap_domain_mfn_range(struct vm_area_struct *vma, +int xen_unmap_domain_gfn_range(struct vm_area_struct *vma, int numpgs, struct page **pages) { if (!pages || !xen_feature(XENFEAT_auto_translated_physmap)) @@ -2588,4 +2935,4 @@ int xen_unmap_domain_mfn_range(struct vm_area_struct *vma, return -EINVAL; #endif } -EXPORT_SYMBOL_GPL(xen_unmap_domain_mfn_range); +EXPORT_SYMBOL_GPL(xen_unmap_domain_gfn_range); diff --git a/arch/x86/xen/p2m.c b/arch/x86/xen/p2m.c index 8b7f18e200aa..bfc08b13044b 100644 --- a/arch/x86/xen/p2m.c +++ b/arch/x86/xen/p2m.c @@ -79,10 +79,14 @@ #include #include -#include "p2m.h" #include "multicalls.h" #include "xen-ops.h" +#define P2M_MID_PER_PAGE (PAGE_SIZE / sizeof(unsigned long *)) +#define P2M_TOP_PER_PAGE (PAGE_SIZE / sizeof(unsigned long **)) + +#define MAX_P2M_PFN (P2M_TOP_PER_PAGE * P2M_MID_PER_PAGE * P2M_PER_PAGE) + #define PMDS_PER_MID_PAGE (P2M_MID_PER_PAGE / PTRS_PER_PTE) unsigned long *xen_p2m_addr __read_mostly; @@ -199,7 +203,8 @@ void __ref xen_build_mfn_list_list(void) unsigned int level, topidx, mididx; unsigned long *mid_mfn_p; - if (xen_feature(XENFEAT_auto_translated_physmap)) + if (xen_feature(XENFEAT_auto_translated_physmap) || + xen_start_info->flags & SIF_VIRT_P2M_4TOOLS) return; /* Pre-initialize p2m_top_mfn to be completely missing */ @@ -260,9 +265,16 @@ void xen_setup_mfn_list_list(void) BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info); - HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list = - virt_to_mfn(p2m_top_mfn); + if (xen_start_info->flags & SIF_VIRT_P2M_4TOOLS) + HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list = ~0UL; + else + HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list = + virt_to_mfn(p2m_top_mfn); HYPERVISOR_shared_info->arch.max_pfn = xen_max_p2m_pfn; + HYPERVISOR_shared_info->arch.p2m_generation = 0; + HYPERVISOR_shared_info->arch.p2m_vaddr = (unsigned long)xen_p2m_addr; + HYPERVISOR_shared_info->arch.p2m_cr3 = + xen_pfn_to_cr3(virt_to_mfn(swapper_pg_dir)); } /* Set up p2m_top to point to the domain-builder provided p2m pages */ @@ -478,8 +490,12 @@ static pte_t *alloc_p2m_pmd(unsigned long addr, pte_t *pte_pg) ptechk = lookup_address(vaddr, &level); if (ptechk == pte_pg) { + HYPERVISOR_shared_info->arch.p2m_generation++; + wmb(); /* Tools are synchronizing via p2m_generation. */ set_pmd(pmdp, __pmd(__pa(pte_newpg[i]) | _KERNPG_TABLE)); + wmb(); /* Tools are synchronizing via p2m_generation. */ + HYPERVISOR_shared_info->arch.p2m_generation++; pte_newpg[i] = NULL; } @@ -505,7 +521,7 @@ static pte_t *alloc_p2m_pmd(unsigned long addr, pte_t *pte_pg) */ static bool alloc_p2m(unsigned long pfn) { - unsigned topidx, mididx; + unsigned topidx; unsigned long *top_mfn_p, *mid_mfn; pte_t *ptep, *pte_pg; unsigned int level; @@ -513,9 +529,6 @@ static bool alloc_p2m(unsigned long pfn) unsigned long addr = (unsigned long)(xen_p2m_addr + pfn); unsigned long p2m_pfn; - topidx = p2m_top_index(pfn); - mididx = p2m_mid_index(pfn); - ptep = lookup_address(addr, &level); BUG_ON(!ptep || level != PG_LEVEL_4K); pte_pg = (pte_t *)((unsigned long)ptep & ~(PAGE_SIZE - 1)); @@ -527,7 +540,8 @@ static bool alloc_p2m(unsigned long pfn) return false; } - if (p2m_top_mfn) { + if (p2m_top_mfn && pfn < MAX_P2M_PFN) { + topidx = p2m_top_index(pfn); top_mfn_p = &p2m_top_mfn[topidx]; mid_mfn = ACCESS_ONCE(p2m_top_mfn_p[topidx]); @@ -577,10 +591,14 @@ static bool alloc_p2m(unsigned long pfn) spin_lock_irqsave(&p2m_update_lock, flags); if (pte_pfn(*ptep) == p2m_pfn) { + HYPERVISOR_shared_info->arch.p2m_generation++; + wmb(); /* Tools are synchronizing via p2m_generation. */ set_pte(ptep, pfn_pte(PFN_DOWN(__pa(p2m)), PAGE_KERNEL)); + wmb(); /* Tools are synchronizing via p2m_generation. */ + HYPERVISOR_shared_info->arch.p2m_generation++; if (mid_mfn) - mid_mfn[mididx] = virt_to_mfn(p2m); + mid_mfn[p2m_mid_index(pfn)] = virt_to_mfn(p2m); p2m = NULL; } @@ -630,6 +648,11 @@ bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn) return true; } + /* + * The interface requires atomic updates on p2m elements. + * xen_safe_write_ulong() is using __put_user which does an atomic + * store via asm(). + */ if (likely(!xen_safe_write_ulong(xen_p2m_addr + pfn, mfn))) return true; diff --git a/arch/x86/xen/p2m.h b/arch/x86/xen/p2m.h deleted file mode 100644 index ad8aee24ab72..000000000000 --- a/arch/x86/xen/p2m.h +++ /dev/null @@ -1,15 +0,0 @@ -#ifndef _XEN_P2M_H -#define _XEN_P2M_H - -#define P2M_PER_PAGE (PAGE_SIZE / sizeof(unsigned long)) -#define P2M_MID_PER_PAGE (PAGE_SIZE / sizeof(unsigned long *)) -#define P2M_TOP_PER_PAGE (PAGE_SIZE / sizeof(unsigned long **)) - -#define MAX_P2M_PFN (P2M_TOP_PER_PAGE * P2M_MID_PER_PAGE * P2M_PER_PAGE) - -#define MAX_REMAP_RANGES 10 - -extern unsigned long __init set_phys_range_identity(unsigned long pfn_s, - unsigned long pfn_e); - -#endif /* _XEN_P2M_H */ diff --git a/arch/x86/xen/platform-pci-unplug.c b/arch/x86/xen/platform-pci-unplug.c index a8261716d58d..9586ff32810c 100644 --- a/arch/x86/xen/platform-pci-unplug.c +++ b/arch/x86/xen/platform-pci-unplug.c @@ -68,7 +68,7 @@ static int check_platform_magic(void) return 0; } -bool xen_has_pv_devices() +bool xen_has_pv_devices(void) { if (!xen_domain()) return false; diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c new file mode 100644 index 000000000000..724a08740a04 --- /dev/null +++ b/arch/x86/xen/pmu.c @@ -0,0 +1,570 @@ +#include +#include + +#include +#include +#include +#include +#include + +#include "xen-ops.h" +#include "pmu.h" + +/* x86_pmu.handle_irq definition */ +#include "../kernel/cpu/perf_event.h" + +#define XENPMU_IRQ_PROCESSING 1 +struct xenpmu { + /* Shared page between hypervisor and domain */ + struct xen_pmu_data *xenpmu_data; + + uint8_t flags; +}; +static DEFINE_PER_CPU(struct xenpmu, xenpmu_shared); +#define get_xenpmu_data() (this_cpu_ptr(&xenpmu_shared)->xenpmu_data) +#define get_xenpmu_flags() (this_cpu_ptr(&xenpmu_shared)->flags) + +/* Macro for computing address of a PMU MSR bank */ +#define field_offset(ctxt, field) ((void *)((uintptr_t)ctxt + \ + (uintptr_t)ctxt->field)) + +/* AMD PMU */ +#define F15H_NUM_COUNTERS 6 +#define F10H_NUM_COUNTERS 4 + +static __read_mostly uint32_t amd_counters_base; +static __read_mostly uint32_t amd_ctrls_base; +static __read_mostly int amd_msr_step; +static __read_mostly int k7_counters_mirrored; +static __read_mostly int amd_num_counters; + +/* Intel PMU */ +#define MSR_TYPE_COUNTER 0 +#define MSR_TYPE_CTRL 1 +#define MSR_TYPE_GLOBAL 2 +#define MSR_TYPE_ARCH_COUNTER 3 +#define MSR_TYPE_ARCH_CTRL 4 + +/* Number of general pmu registers (CPUID.EAX[0xa].EAX[8..15]) */ +#define PMU_GENERAL_NR_SHIFT 8 +#define PMU_GENERAL_NR_BITS 8 +#define PMU_GENERAL_NR_MASK (((1 << PMU_GENERAL_NR_BITS) - 1) \ + << PMU_GENERAL_NR_SHIFT) + +/* Number of fixed pmu registers (CPUID.EDX[0xa].EDX[0..4]) */ +#define PMU_FIXED_NR_SHIFT 0 +#define PMU_FIXED_NR_BITS 5 +#define PMU_FIXED_NR_MASK (((1 << PMU_FIXED_NR_BITS) - 1) \ + << PMU_FIXED_NR_SHIFT) + +/* Alias registers (0x4c1) for full-width writes to PMCs */ +#define MSR_PMC_ALIAS_MASK (~(MSR_IA32_PERFCTR0 ^ MSR_IA32_PMC0)) + +#define INTEL_PMC_TYPE_SHIFT 30 + +static __read_mostly int intel_num_arch_counters, intel_num_fixed_counters; + + +static void xen_pmu_arch_init(void) +{ + if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { + + switch (boot_cpu_data.x86) { + case 0x15: + amd_num_counters = F15H_NUM_COUNTERS; + amd_counters_base = MSR_F15H_PERF_CTR; + amd_ctrls_base = MSR_F15H_PERF_CTL; + amd_msr_step = 2; + k7_counters_mirrored = 1; + break; + case 0x10: + case 0x12: + case 0x14: + case 0x16: + default: + amd_num_counters = F10H_NUM_COUNTERS; + amd_counters_base = MSR_K7_PERFCTR0; + amd_ctrls_base = MSR_K7_EVNTSEL0; + amd_msr_step = 1; + k7_counters_mirrored = 0; + break; + } + } else { + uint32_t eax, ebx, ecx, edx; + + cpuid(0xa, &eax, &ebx, &ecx, &edx); + + intel_num_arch_counters = (eax & PMU_GENERAL_NR_MASK) >> + PMU_GENERAL_NR_SHIFT; + intel_num_fixed_counters = (edx & PMU_FIXED_NR_MASK) >> + PMU_FIXED_NR_SHIFT; + } +} + +static inline uint32_t get_fam15h_addr(u32 addr) +{ + switch (addr) { + case MSR_K7_PERFCTR0: + case MSR_K7_PERFCTR1: + case MSR_K7_PERFCTR2: + case MSR_K7_PERFCTR3: + return MSR_F15H_PERF_CTR + (addr - MSR_K7_PERFCTR0); + case MSR_K7_EVNTSEL0: + case MSR_K7_EVNTSEL1: + case MSR_K7_EVNTSEL2: + case MSR_K7_EVNTSEL3: + return MSR_F15H_PERF_CTL + (addr - MSR_K7_EVNTSEL0); + default: + break; + } + + return addr; +} + +static inline bool is_amd_pmu_msr(unsigned int msr) +{ + if ((msr >= MSR_F15H_PERF_CTL && + msr < MSR_F15H_PERF_CTR + (amd_num_counters * 2)) || + (msr >= MSR_K7_EVNTSEL0 && + msr < MSR_K7_PERFCTR0 + amd_num_counters)) + return true; + + return false; +} + +static int is_intel_pmu_msr(u32 msr_index, int *type, int *index) +{ + u32 msr_index_pmc; + + switch (msr_index) { + case MSR_CORE_PERF_FIXED_CTR_CTRL: + case MSR_IA32_DS_AREA: + case MSR_IA32_PEBS_ENABLE: + *type = MSR_TYPE_CTRL; + return true; + + case MSR_CORE_PERF_GLOBAL_CTRL: + case MSR_CORE_PERF_GLOBAL_STATUS: + case MSR_CORE_PERF_GLOBAL_OVF_CTRL: + *type = MSR_TYPE_GLOBAL; + return true; + + default: + + if ((msr_index >= MSR_CORE_PERF_FIXED_CTR0) && + (msr_index < MSR_CORE_PERF_FIXED_CTR0 + + intel_num_fixed_counters)) { + *index = msr_index - MSR_CORE_PERF_FIXED_CTR0; + *type = MSR_TYPE_COUNTER; + return true; + } + + if ((msr_index >= MSR_P6_EVNTSEL0) && + (msr_index < MSR_P6_EVNTSEL0 + intel_num_arch_counters)) { + *index = msr_index - MSR_P6_EVNTSEL0; + *type = MSR_TYPE_ARCH_CTRL; + return true; + } + + msr_index_pmc = msr_index & MSR_PMC_ALIAS_MASK; + if ((msr_index_pmc >= MSR_IA32_PERFCTR0) && + (msr_index_pmc < MSR_IA32_PERFCTR0 + + intel_num_arch_counters)) { + *type = MSR_TYPE_ARCH_COUNTER; + *index = msr_index_pmc - MSR_IA32_PERFCTR0; + return true; + } + return false; + } +} + +static bool xen_intel_pmu_emulate(unsigned int msr, u64 *val, int type, + int index, bool is_read) +{ + uint64_t *reg = NULL; + struct xen_pmu_intel_ctxt *ctxt; + uint64_t *fix_counters; + struct xen_pmu_cntr_pair *arch_cntr_pair; + struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); + uint8_t xenpmu_flags = get_xenpmu_flags(); + + + if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) + return false; + + ctxt = &xenpmu_data->pmu.c.intel; + + switch (msr) { + case MSR_CORE_PERF_GLOBAL_OVF_CTRL: + reg = &ctxt->global_ovf_ctrl; + break; + case MSR_CORE_PERF_GLOBAL_STATUS: + reg = &ctxt->global_status; + break; + case MSR_CORE_PERF_GLOBAL_CTRL: + reg = &ctxt->global_ctrl; + break; + case MSR_CORE_PERF_FIXED_CTR_CTRL: + reg = &ctxt->fixed_ctrl; + break; + default: + switch (type) { + case MSR_TYPE_COUNTER: + fix_counters = field_offset(ctxt, fixed_counters); + reg = &fix_counters[index]; + break; + case MSR_TYPE_ARCH_COUNTER: + arch_cntr_pair = field_offset(ctxt, arch_counters); + reg = &arch_cntr_pair[index].counter; + break; + case MSR_TYPE_ARCH_CTRL: + arch_cntr_pair = field_offset(ctxt, arch_counters); + reg = &arch_cntr_pair[index].control; + break; + default: + return false; + } + } + + if (reg) { + if (is_read) + *val = *reg; + else { + *reg = *val; + + if (msr == MSR_CORE_PERF_GLOBAL_OVF_CTRL) + ctxt->global_status &= (~(*val)); + } + return true; + } + + return false; +} + +static bool xen_amd_pmu_emulate(unsigned int msr, u64 *val, bool is_read) +{ + uint64_t *reg = NULL; + int i, off = 0; + struct xen_pmu_amd_ctxt *ctxt; + uint64_t *counter_regs, *ctrl_regs; + struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); + uint8_t xenpmu_flags = get_xenpmu_flags(); + + if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) + return false; + + if (k7_counters_mirrored && + ((msr >= MSR_K7_EVNTSEL0) && (msr <= MSR_K7_PERFCTR3))) + msr = get_fam15h_addr(msr); + + ctxt = &xenpmu_data->pmu.c.amd; + for (i = 0; i < amd_num_counters; i++) { + if (msr == amd_ctrls_base + off) { + ctrl_regs = field_offset(ctxt, ctrls); + reg = &ctrl_regs[i]; + break; + } else if (msr == amd_counters_base + off) { + counter_regs = field_offset(ctxt, counters); + reg = &counter_regs[i]; + break; + } + off += amd_msr_step; + } + + if (reg) { + if (is_read) + *val = *reg; + else + *reg = *val; + + return true; + } + return false; +} + +bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err) +{ + if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { + if (is_amd_pmu_msr(msr)) { + if (!xen_amd_pmu_emulate(msr, val, 1)) + *val = native_read_msr_safe(msr, err); + return true; + } + } else { + int type, index; + + if (is_intel_pmu_msr(msr, &type, &index)) { + if (!xen_intel_pmu_emulate(msr, val, type, index, 1)) + *val = native_read_msr_safe(msr, err); + return true; + } + } + + return false; +} + +bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err) +{ + uint64_t val = ((uint64_t)high << 32) | low; + + if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) { + if (is_amd_pmu_msr(msr)) { + if (!xen_amd_pmu_emulate(msr, &val, 0)) + *err = native_write_msr_safe(msr, low, high); + return true; + } + } else { + int type, index; + + if (is_intel_pmu_msr(msr, &type, &index)) { + if (!xen_intel_pmu_emulate(msr, &val, type, index, 0)) + *err = native_write_msr_safe(msr, low, high); + return true; + } + } + + return false; +} + +static unsigned long long xen_amd_read_pmc(int counter) +{ + struct xen_pmu_amd_ctxt *ctxt; + uint64_t *counter_regs; + struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); + uint8_t xenpmu_flags = get_xenpmu_flags(); + + if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) { + uint32_t msr; + int err; + + msr = amd_counters_base + (counter * amd_msr_step); + return native_read_msr_safe(msr, &err); + } + + ctxt = &xenpmu_data->pmu.c.amd; + counter_regs = field_offset(ctxt, counters); + return counter_regs[counter]; +} + +static unsigned long long xen_intel_read_pmc(int counter) +{ + struct xen_pmu_intel_ctxt *ctxt; + uint64_t *fixed_counters; + struct xen_pmu_cntr_pair *arch_cntr_pair; + struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); + uint8_t xenpmu_flags = get_xenpmu_flags(); + + if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) { + uint32_t msr; + int err; + + if (counter & (1 << INTEL_PMC_TYPE_SHIFT)) + msr = MSR_CORE_PERF_FIXED_CTR0 + (counter & 0xffff); + else + msr = MSR_IA32_PERFCTR0 + counter; + + return native_read_msr_safe(msr, &err); + } + + ctxt = &xenpmu_data->pmu.c.intel; + if (counter & (1 << INTEL_PMC_TYPE_SHIFT)) { + fixed_counters = field_offset(ctxt, fixed_counters); + return fixed_counters[counter & 0xffff]; + } + + arch_cntr_pair = field_offset(ctxt, arch_counters); + return arch_cntr_pair[counter].counter; +} + +unsigned long long xen_read_pmc(int counter) +{ + if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) + return xen_amd_read_pmc(counter); + else + return xen_intel_read_pmc(counter); +} + +int pmu_apic_update(uint32_t val) +{ + int ret; + struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); + + if (!xenpmu_data) { + pr_warn_once("%s: pmudata not initialized\n", __func__); + return -EINVAL; + } + + xenpmu_data->pmu.l.lapic_lvtpc = val; + + if (get_xenpmu_flags() & XENPMU_IRQ_PROCESSING) + return 0; + + ret = HYPERVISOR_xenpmu_op(XENPMU_lvtpc_set, NULL); + + return ret; +} + +/* perf callbacks */ +static int xen_is_in_guest(void) +{ + const struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); + + if (!xenpmu_data) { + pr_warn_once("%s: pmudata not initialized\n", __func__); + return 0; + } + + if (!xen_initial_domain() || (xenpmu_data->domain_id >= DOMID_SELF)) + return 0; + + return 1; +} + +static int xen_is_user_mode(void) +{ + const struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); + + if (!xenpmu_data) { + pr_warn_once("%s: pmudata not initialized\n", __func__); + return 0; + } + + if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_PV) + return (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_USER); + else + return !!(xenpmu_data->pmu.r.regs.cpl & 3); +} + +static unsigned long xen_get_guest_ip(void) +{ + const struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); + + if (!xenpmu_data) { + pr_warn_once("%s: pmudata not initialized\n", __func__); + return 0; + } + + return xenpmu_data->pmu.r.regs.ip; +} + +static struct perf_guest_info_callbacks xen_guest_cbs = { + .is_in_guest = xen_is_in_guest, + .is_user_mode = xen_is_user_mode, + .get_guest_ip = xen_get_guest_ip, +}; + +/* Convert registers from Xen's format to Linux' */ +static void xen_convert_regs(const struct xen_pmu_regs *xen_regs, + struct pt_regs *regs, uint64_t pmu_flags) +{ + regs->ip = xen_regs->ip; + regs->cs = xen_regs->cs; + regs->sp = xen_regs->sp; + + if (pmu_flags & PMU_SAMPLE_PV) { + if (pmu_flags & PMU_SAMPLE_USER) + regs->cs |= 3; + else + regs->cs &= ~3; + } else { + if (xen_regs->cpl) + regs->cs |= 3; + else + regs->cs &= ~3; + } +} + +irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id) +{ + int err, ret = IRQ_NONE; + struct pt_regs regs; + const struct xen_pmu_data *xenpmu_data = get_xenpmu_data(); + uint8_t xenpmu_flags = get_xenpmu_flags(); + + if (!xenpmu_data) { + pr_warn_once("%s: pmudata not initialized\n", __func__); + return ret; + } + + this_cpu_ptr(&xenpmu_shared)->flags = + xenpmu_flags | XENPMU_IRQ_PROCESSING; + xen_convert_regs(&xenpmu_data->pmu.r.regs, ®s, + xenpmu_data->pmu.pmu_flags); + if (x86_pmu.handle_irq(®s)) + ret = IRQ_HANDLED; + + /* Write out cached context to HW */ + err = HYPERVISOR_xenpmu_op(XENPMU_flush, NULL); + this_cpu_ptr(&xenpmu_shared)->flags = xenpmu_flags; + if (err) { + pr_warn_once("%s: failed hypercall, err: %d\n", __func__, err); + return IRQ_NONE; + } + + return ret; +} + +bool is_xen_pmu(int cpu) +{ + return (get_xenpmu_data() != NULL); +} + +void xen_pmu_init(int cpu) +{ + int err; + struct xen_pmu_params xp; + unsigned long pfn; + struct xen_pmu_data *xenpmu_data; + + BUILD_BUG_ON(sizeof(struct xen_pmu_data) > PAGE_SIZE); + + if (xen_hvm_domain()) + return; + + xenpmu_data = (struct xen_pmu_data *)get_zeroed_page(GFP_KERNEL); + if (!xenpmu_data) { + pr_err("VPMU init: No memory\n"); + return; + } + pfn = virt_to_pfn(xenpmu_data); + + xp.val = pfn_to_mfn(pfn); + xp.vcpu = cpu; + xp.version.maj = XENPMU_VER_MAJ; + xp.version.min = XENPMU_VER_MIN; + err = HYPERVISOR_xenpmu_op(XENPMU_init, &xp); + if (err) + goto fail; + + per_cpu(xenpmu_shared, cpu).xenpmu_data = xenpmu_data; + per_cpu(xenpmu_shared, cpu).flags = 0; + + if (cpu == 0) { + perf_register_guest_info_callbacks(&xen_guest_cbs); + xen_pmu_arch_init(); + } + + return; + +fail: + pr_warn_once("Could not initialize VPMU for cpu %d, error %d\n", + cpu, err); + free_pages((unsigned long)xenpmu_data, 0); +} + +void xen_pmu_finish(int cpu) +{ + struct xen_pmu_params xp; + + if (xen_hvm_domain()) + return; + + xp.vcpu = cpu; + xp.version.maj = XENPMU_VER_MAJ; + xp.version.min = XENPMU_VER_MIN; + + (void)HYPERVISOR_xenpmu_op(XENPMU_finish, &xp); + + free_pages((unsigned long)per_cpu(xenpmu_shared, cpu).xenpmu_data, 0); + per_cpu(xenpmu_shared, cpu).xenpmu_data = NULL; +} diff --git a/arch/x86/xen/pmu.h b/arch/x86/xen/pmu.h new file mode 100644 index 000000000000..af5f0ad94078 --- /dev/null +++ b/arch/x86/xen/pmu.h @@ -0,0 +1,15 @@ +#ifndef __XEN_PMU_H +#define __XEN_PMU_H + +#include + +irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id); +void xen_pmu_init(int cpu); +void xen_pmu_finish(int cpu); +bool is_xen_pmu(int cpu); +bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err); +bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err); +int pmu_apic_update(uint32_t reg); +unsigned long long xen_read_pmc(int counter); + +#endif /* __XEN_PMU_H */ diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c index 55f388ef481a..f5ef6746d47a 100644 --- a/arch/x86/xen/setup.c +++ b/arch/x86/xen/setup.c @@ -27,17 +27,23 @@ #include #include #include +#include #include "xen-ops.h" #include "vdso.h" -#include "p2m.h" #include "mmu.h" +#define GB(x) ((uint64_t)(x) * 1024 * 1024 * 1024) + /* Amount of extra memory space we add to the e820 ranges */ struct xen_memory_region xen_extra_mem[XEN_EXTRA_MEM_MAX_REGIONS] __initdata; /* Number of pages released from the initial allocation. */ unsigned long xen_released_pages; +/* E820 map used during setting up memory. */ +static struct e820entry xen_e820_map[E820MAX] __initdata; +static u32 xen_e820_map_entries __initdata; + /* * Buffer used to remap identity mapped pages. We only need the virtual space. * The physical page behind this address is remapped as needed to different @@ -64,62 +70,89 @@ static unsigned long xen_remap_mfn __initdata = INVALID_P2M_ENTRY; */ #define EXTRA_MEM_RATIO (10) -static void __init xen_add_extra_mem(phys_addr_t start, phys_addr_t size) +static bool xen_512gb_limit __initdata = IS_ENABLED(CONFIG_XEN_512GB); + +static void __init xen_parse_512gb(void) +{ + bool val = false; + char *arg; + + arg = strstr(xen_start_info->cmd_line, "xen_512gb_limit"); + if (!arg) + return; + + arg = strstr(xen_start_info->cmd_line, "xen_512gb_limit="); + if (!arg) + val = true; + else if (strtobool(arg + strlen("xen_512gb_limit="), &val)) + return; + + xen_512gb_limit = val; +} + +static void __init xen_add_extra_mem(unsigned long start_pfn, + unsigned long n_pfns) { int i; + /* + * No need to check for zero size, should happen rarely and will only + * write a new entry regarded to be unused due to zero size. + */ for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) { /* Add new region. */ - if (xen_extra_mem[i].size == 0) { - xen_extra_mem[i].start = start; - xen_extra_mem[i].size = size; + if (xen_extra_mem[i].n_pfns == 0) { + xen_extra_mem[i].start_pfn = start_pfn; + xen_extra_mem[i].n_pfns = n_pfns; break; } /* Append to existing region. */ - if (xen_extra_mem[i].start + xen_extra_mem[i].size == start) { - xen_extra_mem[i].size += size; + if (xen_extra_mem[i].start_pfn + xen_extra_mem[i].n_pfns == + start_pfn) { + xen_extra_mem[i].n_pfns += n_pfns; break; } } if (i == XEN_EXTRA_MEM_MAX_REGIONS) printk(KERN_WARNING "Warning: not enough extra memory regions\n"); - memblock_reserve(start, size); + memblock_reserve(PFN_PHYS(start_pfn), PFN_PHYS(n_pfns)); } -static void __init xen_del_extra_mem(phys_addr_t start, phys_addr_t size) +static void __init xen_del_extra_mem(unsigned long start_pfn, + unsigned long n_pfns) { int i; - phys_addr_t start_r, size_r; + unsigned long start_r, size_r; for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) { - start_r = xen_extra_mem[i].start; - size_r = xen_extra_mem[i].size; + start_r = xen_extra_mem[i].start_pfn; + size_r = xen_extra_mem[i].n_pfns; /* Start of region. */ - if (start_r == start) { - BUG_ON(size > size_r); - xen_extra_mem[i].start += size; - xen_extra_mem[i].size -= size; + if (start_r == start_pfn) { + BUG_ON(n_pfns > size_r); + xen_extra_mem[i].start_pfn += n_pfns; + xen_extra_mem[i].n_pfns -= n_pfns; break; } /* End of region. */ - if (start_r + size_r == start + size) { - BUG_ON(size > size_r); - xen_extra_mem[i].size -= size; + if (start_r + size_r == start_pfn + n_pfns) { + BUG_ON(n_pfns > size_r); + xen_extra_mem[i].n_pfns -= n_pfns; break; } /* Mid of region. */ - if (start > start_r && start < start_r + size_r) { - BUG_ON(start + size > start_r + size_r); - xen_extra_mem[i].size = start - start_r; + if (start_pfn > start_r && start_pfn < start_r + size_r) { + BUG_ON(start_pfn + n_pfns > start_r + size_r); + xen_extra_mem[i].n_pfns = start_pfn - start_r; /* Calling memblock_reserve() again is okay. */ - xen_add_extra_mem(start + size, start_r + size_r - - (start + size)); + xen_add_extra_mem(start_pfn + n_pfns, start_r + size_r - + (start_pfn + n_pfns)); break; } } - memblock_free(start, size); + memblock_free(PFN_PHYS(start_pfn), PFN_PHYS(n_pfns)); } /* @@ -130,11 +163,10 @@ static void __init xen_del_extra_mem(phys_addr_t start, phys_addr_t size) unsigned long __ref xen_chk_extra_mem(unsigned long pfn) { int i; - phys_addr_t addr = PFN_PHYS(pfn); for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) { - if (addr >= xen_extra_mem[i].start && - addr < xen_extra_mem[i].start + xen_extra_mem[i].size) + if (pfn >= xen_extra_mem[i].start_pfn && + pfn < xen_extra_mem[i].start_pfn + xen_extra_mem[i].n_pfns) return INVALID_P2M_ENTRY; } @@ -150,10 +182,10 @@ void __init xen_inv_extra_mem(void) int i; for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) { - if (!xen_extra_mem[i].size) + if (!xen_extra_mem[i].n_pfns) continue; - pfn_s = PFN_DOWN(xen_extra_mem[i].start); - pfn_e = PFN_UP(xen_extra_mem[i].start + xen_extra_mem[i].size); + pfn_s = xen_extra_mem[i].start_pfn; + pfn_e = pfn_s + xen_extra_mem[i].n_pfns; for (pfn = pfn_s; pfn < pfn_e; pfn++) set_phys_to_machine(pfn, INVALID_P2M_ENTRY); } @@ -164,15 +196,13 @@ void __init xen_inv_extra_mem(void) * This function updates min_pfn with the pfn found and returns * the size of that range or zero if not found. */ -static unsigned long __init xen_find_pfn_range( - const struct e820entry *list, size_t map_size, - unsigned long *min_pfn) +static unsigned long __init xen_find_pfn_range(unsigned long *min_pfn) { - const struct e820entry *entry; + const struct e820entry *entry = xen_e820_map; unsigned int i; unsigned long done = 0; - for (i = 0, entry = list; i < map_size; i++, entry++) { + for (i = 0; i < xen_e820_map_entries; i++, entry++) { unsigned long s_pfn; unsigned long e_pfn; @@ -221,7 +251,7 @@ static int __init xen_free_mfn(unsigned long mfn) * as a fallback if the remapping fails. */ static void __init xen_set_identity_and_release_chunk(unsigned long start_pfn, - unsigned long end_pfn, unsigned long nr_pages, unsigned long *released) + unsigned long end_pfn, unsigned long nr_pages) { unsigned long pfn, end; int ret; @@ -241,7 +271,7 @@ static void __init xen_set_identity_and_release_chunk(unsigned long start_pfn, WARN(ret != 1, "Failed to release pfn %lx err=%d\n", pfn, ret); if (ret == 1) { - (*released)++; + xen_released_pages++; if (!__set_phys_to_machine(pfn, INVALID_P2M_ENTRY)) break; } else @@ -356,9 +386,8 @@ static void __init xen_do_set_identity_and_remap_chunk( * to Xen and not remapped. */ static unsigned long __init xen_set_identity_and_remap_chunk( - const struct e820entry *list, size_t map_size, unsigned long start_pfn, - unsigned long end_pfn, unsigned long nr_pages, unsigned long remap_pfn, - unsigned long *released, unsigned long *remapped) + unsigned long start_pfn, unsigned long end_pfn, unsigned long nr_pages, + unsigned long remap_pfn) { unsigned long pfn; unsigned long i = 0; @@ -379,12 +408,11 @@ static unsigned long __init xen_set_identity_and_remap_chunk( if (cur_pfn + size > nr_pages) size = nr_pages - cur_pfn; - remap_range_size = xen_find_pfn_range(list, map_size, - &remap_pfn); + remap_range_size = xen_find_pfn_range(&remap_pfn); if (!remap_range_size) { pr_warning("Unable to find available pfn range, not remapping identity pages\n"); xen_set_identity_and_release_chunk(cur_pfn, - cur_pfn + left, nr_pages, released); + cur_pfn + left, nr_pages); break; } /* Adjust size to fit in current e820 RAM region */ @@ -396,7 +424,6 @@ static unsigned long __init xen_set_identity_and_remap_chunk( /* Update variables to reflect new mappings. */ i += size; remap_pfn += size; - *remapped += size; } /* @@ -411,15 +438,11 @@ static unsigned long __init xen_set_identity_and_remap_chunk( return remap_pfn; } -static void __init xen_set_identity_and_remap( - const struct e820entry *list, size_t map_size, unsigned long nr_pages, - unsigned long *released, unsigned long *remapped) +static void __init xen_set_identity_and_remap(unsigned long nr_pages) { phys_addr_t start = 0; unsigned long last_pfn = nr_pages; - const struct e820entry *entry; - unsigned long num_released = 0; - unsigned long num_remapped = 0; + const struct e820entry *entry = xen_e820_map; int i; /* @@ -433,9 +456,9 @@ static void __init xen_set_identity_and_remap( * example) the DMI tables in a reserved region that begins on * a non-page boundary. */ - for (i = 0, entry = list; i < map_size; i++, entry++) { + for (i = 0; i < xen_e820_map_entries; i++, entry++) { phys_addr_t end = entry->addr + entry->size; - if (entry->type == E820_RAM || i == map_size - 1) { + if (entry->type == E820_RAM || i == xen_e820_map_entries - 1) { unsigned long start_pfn = PFN_DOWN(start); unsigned long end_pfn = PFN_UP(end); @@ -444,17 +467,13 @@ static void __init xen_set_identity_and_remap( if (start_pfn < end_pfn) last_pfn = xen_set_identity_and_remap_chunk( - list, map_size, start_pfn, - end_pfn, nr_pages, last_pfn, - &num_released, &num_remapped); + start_pfn, end_pfn, nr_pages, + last_pfn); start = end; } } - *released = num_released; - *remapped = num_remapped; - - pr_info("Released %ld page(s)\n", num_released); + pr_info("Released %ld page(s)\n", xen_released_pages); } /* @@ -494,7 +513,7 @@ void __init xen_remap_memory(void) } else if (pfn_s + len == xen_remap_buf.target_pfn) { len += xen_remap_buf.size; } else { - xen_del_extra_mem(PFN_PHYS(pfn_s), PFN_PHYS(len)); + xen_del_extra_mem(pfn_s, len); pfn_s = xen_remap_buf.target_pfn; len = xen_remap_buf.size; } @@ -504,19 +523,36 @@ void __init xen_remap_memory(void) } if (pfn_s != ~0UL && len) - xen_del_extra_mem(PFN_PHYS(pfn_s), PFN_PHYS(len)); + xen_del_extra_mem(pfn_s, len); set_pte_mfn(buf, mfn_save, PAGE_KERNEL); pr_info("Remapped %ld page(s)\n", remapped); } +static unsigned long __init xen_get_pages_limit(void) +{ + unsigned long limit; + +#ifdef CONFIG_X86_32 + limit = GB(64) / PAGE_SIZE; +#else + limit = MAXMEM / PAGE_SIZE; + if (!xen_initial_domain() && xen_512gb_limit) + limit = GB(512) / PAGE_SIZE; +#endif + return limit; +} + static unsigned long __init xen_get_max_pages(void) { - unsigned long max_pages = MAX_DOMAIN_PAGES; + unsigned long max_pages, limit; domid_t domid = DOMID_SELF; int ret; + limit = xen_get_pages_limit(); + max_pages = limit; + /* * For the initial domain we use the maximum reservation as * the maximum page. @@ -532,7 +568,7 @@ static unsigned long __init xen_get_max_pages(void) max_pages = ret; } - return min(max_pages, MAX_DOMAIN_PAGES); + return min(max_pages, limit); } static void __init xen_align_and_add_e820_region(phys_addr_t start, @@ -549,39 +585,188 @@ static void __init xen_align_and_add_e820_region(phys_addr_t start, e820_add_region(start, end - start, type); } -static void __init xen_ignore_unusable(struct e820entry *list, size_t map_size) +static void __init xen_ignore_unusable(void) { - struct e820entry *entry; + struct e820entry *entry = xen_e820_map; unsigned int i; - for (i = 0, entry = list; i < map_size; i++, entry++) { + for (i = 0; i < xen_e820_map_entries; i++, entry++) { if (entry->type == E820_UNUSABLE) entry->type = E820_RAM; } } +static unsigned long __init xen_count_remap_pages(unsigned long max_pfn) +{ + unsigned long extra = 0; + unsigned long start_pfn, end_pfn; + const struct e820entry *entry = xen_e820_map; + int i; + + end_pfn = 0; + for (i = 0; i < xen_e820_map_entries; i++, entry++) { + start_pfn = PFN_DOWN(entry->addr); + /* Adjacent regions on non-page boundaries handling! */ + end_pfn = min(end_pfn, start_pfn); + + if (start_pfn >= max_pfn) + return extra + max_pfn - end_pfn; + + /* Add any holes in map to result. */ + extra += start_pfn - end_pfn; + + end_pfn = PFN_UP(entry->addr + entry->size); + end_pfn = min(end_pfn, max_pfn); + + if (entry->type != E820_RAM) + extra += end_pfn - start_pfn; + } + + return extra; +} + +bool __init xen_is_e820_reserved(phys_addr_t start, phys_addr_t size) +{ + struct e820entry *entry; + unsigned mapcnt; + phys_addr_t end; + + if (!size) + return false; + + end = start + size; + entry = xen_e820_map; + + for (mapcnt = 0; mapcnt < xen_e820_map_entries; mapcnt++) { + if (entry->type == E820_RAM && entry->addr <= start && + (entry->addr + entry->size) >= end) + return false; + + entry++; + } + + return true; +} + +/* + * Find a free area in physical memory not yet reserved and compliant with + * E820 map. + * Used to relocate pre-allocated areas like initrd or p2m list which are in + * conflict with the to be used E820 map. + * In case no area is found, return 0. Otherwise return the physical address + * of the area which is already reserved for convenience. + */ +phys_addr_t __init xen_find_free_area(phys_addr_t size) +{ + unsigned mapcnt; + phys_addr_t addr, start; + struct e820entry *entry = xen_e820_map; + + for (mapcnt = 0; mapcnt < xen_e820_map_entries; mapcnt++, entry++) { + if (entry->type != E820_RAM || entry->size < size) + continue; + start = entry->addr; + for (addr = start; addr < start + size; addr += PAGE_SIZE) { + if (!memblock_is_reserved(addr)) + continue; + start = addr + PAGE_SIZE; + if (start + size > entry->addr + entry->size) + break; + } + if (addr >= start + size) { + memblock_reserve(start, size); + return start; + } + } + + return 0; +} + +/* + * Like memcpy, but with physical addresses for dest and src. + */ +static void __init xen_phys_memcpy(phys_addr_t dest, phys_addr_t src, + phys_addr_t n) +{ + phys_addr_t dest_off, src_off, dest_len, src_len, len; + void *from, *to; + + while (n) { + dest_off = dest & ~PAGE_MASK; + src_off = src & ~PAGE_MASK; + dest_len = n; + if (dest_len > (NR_FIX_BTMAPS << PAGE_SHIFT) - dest_off) + dest_len = (NR_FIX_BTMAPS << PAGE_SHIFT) - dest_off; + src_len = n; + if (src_len > (NR_FIX_BTMAPS << PAGE_SHIFT) - src_off) + src_len = (NR_FIX_BTMAPS << PAGE_SHIFT) - src_off; + len = min(dest_len, src_len); + to = early_memremap(dest - dest_off, dest_len + dest_off); + from = early_memremap(src - src_off, src_len + src_off); + memcpy(to, from, len); + early_memunmap(to, dest_len + dest_off); + early_memunmap(from, src_len + src_off); + n -= len; + dest += len; + src += len; + } +} + +/* + * Reserve Xen mfn_list. + */ +static void __init xen_reserve_xen_mfnlist(void) +{ + phys_addr_t start, size; + + if (xen_start_info->mfn_list >= __START_KERNEL_map) { + start = __pa(xen_start_info->mfn_list); + size = PFN_ALIGN(xen_start_info->nr_pages * + sizeof(unsigned long)); + } else { + start = PFN_PHYS(xen_start_info->first_p2m_pfn); + size = PFN_PHYS(xen_start_info->nr_p2m_frames); + } + + if (!xen_is_e820_reserved(start, size)) { + memblock_reserve(start, size); + return; + } + +#ifdef CONFIG_X86_32 + /* + * Relocating the p2m on 32 bit system to an arbitrary virtual address + * is not supported, so just give up. + */ + xen_raw_console_write("Xen hypervisor allocated p2m list conflicts with E820 map\n"); + BUG(); +#else + xen_relocate_p2m(); +#endif +} + /** * machine_specific_memory_setup - Hook for machine specific memory setup. **/ char * __init xen_memory_setup(void) { - static struct e820entry map[E820MAX] __initdata; - - unsigned long max_pfn = xen_start_info->nr_pages; - phys_addr_t mem_end; + unsigned long max_pfn, pfn_s, n_pfns; + phys_addr_t mem_end, addr, size, chunk_size; + u32 type; int rc; struct xen_memory_map memmap; unsigned long max_pages; unsigned long extra_pages = 0; - unsigned long remapped_pages; int i; int op; - max_pfn = min(MAX_DOMAIN_PAGES, max_pfn); + xen_parse_512gb(); + max_pfn = xen_get_pages_limit(); + max_pfn = min(max_pfn, xen_start_info->nr_pages); mem_end = PFN_PHYS(max_pfn); memmap.nr_entries = E820MAX; - set_xen_guest_handle(memmap.buffer, map); + set_xen_guest_handle(memmap.buffer, xen_e820_map); op = xen_initial_domain() ? XENMEM_machine_memory_map : @@ -590,15 +775,16 @@ char * __init xen_memory_setup(void) if (rc == -ENOSYS) { BUG_ON(xen_initial_domain()); memmap.nr_entries = 1; - map[0].addr = 0ULL; - map[0].size = mem_end; + xen_e820_map[0].addr = 0ULL; + xen_e820_map[0].size = mem_end; /* 8MB slack (to balance backend allocations). */ - map[0].size += 8ULL << 20; - map[0].type = E820_RAM; + xen_e820_map[0].size += 8ULL << 20; + xen_e820_map[0].type = E820_RAM; rc = 0; } BUG_ON(rc); BUG_ON(memmap.nr_entries == 0); + xen_e820_map_entries = memmap.nr_entries; /* * Xen won't allow a 1:1 mapping to be created to UNUSABLE @@ -609,25 +795,20 @@ char * __init xen_memory_setup(void) * a patch in the future. */ if (xen_initial_domain()) - xen_ignore_unusable(map, memmap.nr_entries); + xen_ignore_unusable(); /* Make sure the Xen-supplied memory map is well-ordered. */ - sanitize_e820_map(map, memmap.nr_entries, &memmap.nr_entries); + sanitize_e820_map(xen_e820_map, xen_e820_map_entries, + &xen_e820_map_entries); max_pages = xen_get_max_pages(); + + /* How many extra pages do we need due to remapping? */ + max_pages += xen_count_remap_pages(max_pfn); + if (max_pages > max_pfn) extra_pages += max_pages - max_pfn; - /* - * Set identity map on non-RAM pages and prepare remapping the - * underlying RAM. - */ - xen_set_identity_and_remap(map, memmap.nr_entries, max_pfn, - &xen_released_pages, &remapped_pages); - - extra_pages += xen_released_pages; - extra_pages += remapped_pages; - /* * Clamp the amount of extra memory to a EXTRA_MEM_RATIO * factor the base size. On non-highmem systems, the base @@ -635,46 +816,54 @@ char * __init xen_memory_setup(void) * is limited to the max size of lowmem, so that it doesn't * get completely filled. * + * Make sure we have no memory above max_pages, as this area + * isn't handled by the p2m management. + * * In principle there could be a problem in lowmem systems if * the initial memory is also very large with respect to * lowmem, but we won't try to deal with that here. */ - extra_pages = min(EXTRA_MEM_RATIO * min(max_pfn, PFN_DOWN(MAXMEM)), - extra_pages); + extra_pages = min3(EXTRA_MEM_RATIO * min(max_pfn, PFN_DOWN(MAXMEM)), + extra_pages, max_pages - max_pfn); i = 0; - while (i < memmap.nr_entries) { - phys_addr_t addr = map[i].addr; - phys_addr_t size = map[i].size; - u32 type = map[i].type; + addr = xen_e820_map[0].addr; + size = xen_e820_map[0].size; + while (i < xen_e820_map_entries) { + chunk_size = size; + type = xen_e820_map[i].type; if (type == E820_RAM) { if (addr < mem_end) { - size = min(size, mem_end - addr); + chunk_size = min(size, mem_end - addr); } else if (extra_pages) { - size = min(size, PFN_PHYS(extra_pages)); - extra_pages -= PFN_DOWN(size); - xen_add_extra_mem(addr, size); - xen_max_p2m_pfn = PFN_DOWN(addr + size); + chunk_size = min(size, PFN_PHYS(extra_pages)); + pfn_s = PFN_UP(addr); + n_pfns = PFN_DOWN(addr + chunk_size) - pfn_s; + extra_pages -= n_pfns; + xen_add_extra_mem(pfn_s, n_pfns); + xen_max_p2m_pfn = pfn_s + n_pfns; } else type = E820_UNUSABLE; } - xen_align_and_add_e820_region(addr, size, type); + xen_align_and_add_e820_region(addr, chunk_size, type); - map[i].addr += size; - map[i].size -= size; - if (map[i].size == 0) + addr += chunk_size; + size -= chunk_size; + if (size == 0) { i++; + if (i < xen_e820_map_entries) { + addr = xen_e820_map[i].addr; + size = xen_e820_map[i].size; + } + } } /* * Set the rest as identity mapped, in case PCI BARs are * located here. - * - * PFNs above MAX_P2M_PFN are considered identity mapped as - * well. */ - set_phys_range_identity(map[i-1].addr / PAGE_SIZE, ~0ul); + set_phys_range_identity(addr / PAGE_SIZE, ~0ul); /* * In domU, the ISA region is normal, usable memory, but we @@ -684,35 +873,54 @@ char * __init xen_memory_setup(void) e820_add_region(ISA_START_ADDRESS, ISA_END_ADDRESS - ISA_START_ADDRESS, E820_RESERVED); - /* - * Reserve Xen bits: - * - mfn_list - * - xen_start_info - * See comment above "struct start_info" in - * We tried to make the the memblock_reserve more selective so - * that it would be clear what region is reserved. Sadly we ran - * in the problem wherein on a 64-bit hypervisor with a 32-bit - * initial domain, the pt_base has the cr3 value which is not - * neccessarily where the pagetable starts! As Jan put it: " - * Actually, the adjustment turns out to be correct: The page - * tables for a 32-on-64 dom0 get allocated in the order "first L1", - * "first L2", "first L3", so the offset to the page table base is - * indeed 2. When reading xen/include/public/xen.h's comment - * very strictly, this is not a violation (since there nothing is said - * that the first thing in the page table space is pointed to by - * pt_base; I admit that this seems to be implied though, namely - * do I think that it is implied that the page table space is the - * range [pt_base, pt_base + nt_pt_frames), whereas that - * range here indeed is [pt_base - 2, pt_base - 2 + nt_pt_frames), - * which - without a priori knowledge - the kernel would have - * difficulty to figure out)." - so lets just fall back to the - * easy way and reserve the whole region. - */ - memblock_reserve(__pa(xen_start_info->mfn_list), - xen_start_info->pt_base - xen_start_info->mfn_list); - sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map); + /* + * Check whether the kernel itself conflicts with the target E820 map. + * Failing now is better than running into weird problems later due + * to relocating (and even reusing) pages with kernel text or data. + */ + if (xen_is_e820_reserved(__pa_symbol(_text), + __pa_symbol(__bss_stop) - __pa_symbol(_text))) { + xen_raw_console_write("Xen hypervisor allocated kernel memory conflicts with E820 map\n"); + BUG(); + } + + /* + * Check for a conflict of the hypervisor supplied page tables with + * the target E820 map. + */ + xen_pt_check_e820(); + + xen_reserve_xen_mfnlist(); + + /* Check for a conflict of the initrd with the target E820 map. */ + if (xen_is_e820_reserved(boot_params.hdr.ramdisk_image, + boot_params.hdr.ramdisk_size)) { + phys_addr_t new_area, start, size; + + new_area = xen_find_free_area(boot_params.hdr.ramdisk_size); + if (!new_area) { + xen_raw_console_write("Can't find new memory area for initrd needed due to E820 map conflict\n"); + BUG(); + } + + start = boot_params.hdr.ramdisk_image; + size = boot_params.hdr.ramdisk_size; + xen_phys_memcpy(new_area, start, size); + pr_info("initrd moved from [mem %#010llx-%#010llx] to [mem %#010llx-%#010llx]\n", + start, start + size, new_area, new_area + size); + memblock_free(start, size); + boot_params.hdr.ramdisk_image = new_area; + boot_params.ext_ramdisk_image = new_area >> 32; + } + + /* + * Set identity map on non-RAM pages and prepare remapping the + * underlying RAM. + */ + xen_set_identity_and_remap(max_pfn); + return "Xen"; } @@ -721,26 +929,30 @@ char * __init xen_memory_setup(void) */ char * __init xen_auto_xlated_memory_setup(void) { - static struct e820entry map[E820MAX] __initdata; - struct xen_memory_map memmap; int i; int rc; memmap.nr_entries = E820MAX; - set_xen_guest_handle(memmap.buffer, map); + set_xen_guest_handle(memmap.buffer, xen_e820_map); rc = HYPERVISOR_memory_op(XENMEM_memory_map, &memmap); if (rc < 0) panic("No memory map (%d)\n", rc); - sanitize_e820_map(map, ARRAY_SIZE(map), &memmap.nr_entries); + xen_e820_map_entries = memmap.nr_entries; - for (i = 0; i < memmap.nr_entries; i++) - e820_add_region(map[i].addr, map[i].size, map[i].type); + sanitize_e820_map(xen_e820_map, ARRAY_SIZE(xen_e820_map), + &xen_e820_map_entries); - memblock_reserve(__pa(xen_start_info->mfn_list), - xen_start_info->pt_base - xen_start_info->mfn_list); + for (i = 0; i < xen_e820_map_entries; i++) + e820_add_region(xen_e820_map[i].addr, xen_e820_map[i].size, + xen_e820_map[i].type); + + /* Remove p2m info, it is not needed. */ + xen_start_info->mfn_list = 0; + xen_start_info->first_p2m_pfn = 0; + xen_start_info->nr_p2m_frames = 0; return "Xen"; } diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c index 86484384492e..3f4ebf0261f2 100644 --- a/arch/x86/xen/smp.c +++ b/arch/x86/xen/smp.c @@ -26,6 +26,7 @@ #include #include +#include #include #include @@ -38,6 +39,7 @@ #include "xen-ops.h" #include "mmu.h" #include "smp.h" +#include "pmu.h" cpumask_var_t xen_cpu_initialized_map; @@ -50,6 +52,7 @@ static DEFINE_PER_CPU(struct xen_common_irq, xen_callfunc_irq) = { .irq = -1 }; static DEFINE_PER_CPU(struct xen_common_irq, xen_callfuncsingle_irq) = { .irq = -1 }; static DEFINE_PER_CPU(struct xen_common_irq, xen_irq_work) = { .irq = -1 }; static DEFINE_PER_CPU(struct xen_common_irq, xen_debug_irq) = { .irq = -1 }; +static DEFINE_PER_CPU(struct xen_common_irq, xen_pmu_irq) = { .irq = -1 }; static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id); static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id); @@ -148,11 +151,18 @@ static void xen_smp_intr_free(unsigned int cpu) kfree(per_cpu(xen_irq_work, cpu).name); per_cpu(xen_irq_work, cpu).name = NULL; } + + if (per_cpu(xen_pmu_irq, cpu).irq >= 0) { + unbind_from_irqhandler(per_cpu(xen_pmu_irq, cpu).irq, NULL); + per_cpu(xen_pmu_irq, cpu).irq = -1; + kfree(per_cpu(xen_pmu_irq, cpu).name); + per_cpu(xen_pmu_irq, cpu).name = NULL; + } }; static int xen_smp_intr_init(unsigned int cpu) { int rc; - char *resched_name, *callfunc_name, *debug_name; + char *resched_name, *callfunc_name, *debug_name, *pmu_name; resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu); rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR, @@ -218,6 +228,18 @@ static int xen_smp_intr_init(unsigned int cpu) per_cpu(xen_irq_work, cpu).irq = rc; per_cpu(xen_irq_work, cpu).name = callfunc_name; + if (is_xen_pmu(cpu)) { + pmu_name = kasprintf(GFP_KERNEL, "pmu%d", cpu); + rc = bind_virq_to_irqhandler(VIRQ_XENPMU, cpu, + xen_pmu_irq_handler, + IRQF_PERCPU|IRQF_NOBALANCING, + pmu_name, NULL); + if (rc < 0) + goto fail; + per_cpu(xen_pmu_irq, cpu).irq = rc; + per_cpu(xen_pmu_irq, cpu).name = pmu_name; + } + return 0; fail: @@ -335,6 +357,8 @@ static void __init xen_smp_prepare_cpus(unsigned int max_cpus) } set_cpu_sibling_map(0); + xen_pmu_init(0); + if (xen_smp_intr_init(0)) BUG(); @@ -429,7 +453,7 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle) } #endif ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs); - ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(swapper_pg_dir)); + ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_gfn(swapper_pg_dir)); if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt)) BUG(); @@ -462,6 +486,8 @@ static int xen_cpu_up(unsigned int cpu, struct task_struct *idle) if (rc) return rc; + xen_pmu_init(cpu); + rc = xen_smp_intr_init(cpu); if (rc) return rc; @@ -503,6 +529,7 @@ static void xen_cpu_die(unsigned int cpu) xen_smp_intr_free(cpu); xen_uninit_lock_cpu(cpu); xen_teardown_timer(cpu); + xen_pmu_finish(cpu); } } diff --git a/arch/x86/xen/suspend.c b/arch/x86/xen/suspend.c index 53b4c0811f4f..feddabdab448 100644 --- a/arch/x86/xen/suspend.c +++ b/arch/x86/xen/suspend.c @@ -11,6 +11,7 @@ #include "xen-ops.h" #include "mmu.h" +#include "pmu.h" static void xen_pv_pre_suspend(void) { @@ -67,16 +68,26 @@ static void xen_pv_post_suspend(int suspend_cancelled) void xen_arch_pre_suspend(void) { - if (xen_pv_domain()) - xen_pv_pre_suspend(); + int cpu; + + for_each_online_cpu(cpu) + xen_pmu_finish(cpu); + + if (xen_pv_domain()) + xen_pv_pre_suspend(); } void xen_arch_post_suspend(int cancelled) { - if (xen_pv_domain()) - xen_pv_post_suspend(cancelled); - else - xen_hvm_post_suspend(cancelled); + int cpu; + + if (xen_pv_domain()) + xen_pv_post_suspend(cancelled); + else + xen_hvm_post_suspend(cancelled); + + for_each_online_cpu(cpu) + xen_pmu_init(cpu); } static void xen_vcpu_notify_restore(void *data) diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S index 8afdfccf6086..b65f59a358a2 100644 --- a/arch/x86/xen/xen-head.S +++ b/arch/x86/xen/xen-head.S @@ -104,6 +104,8 @@ ENTRY(hypercall_page) ELFNOTE(Xen, XEN_ELFNOTE_VIRT_BASE, _ASM_PTR __PAGE_OFFSET) #else ELFNOTE(Xen, XEN_ELFNOTE_VIRT_BASE, _ASM_PTR __START_KERNEL_map) + /* Map the p2m table to a 512GB-aligned user address. */ + ELFNOTE(Xen, XEN_ELFNOTE_INIT_P2M, .quad PGDIR_SIZE) #endif ELFNOTE(Xen, XEN_ELFNOTE_ENTRY, _ASM_PTR startup_xen) ELFNOTE(Xen, XEN_ELFNOTE_HYPERCALL_PAGE, _ASM_PTR hypercall_page) diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h index 2292721b1d10..1399423f3418 100644 --- a/arch/x86/xen/xen-ops.h +++ b/arch/x86/xen/xen-ops.h @@ -35,13 +35,20 @@ void xen_build_mfn_list_list(void); void xen_setup_machphys_mapping(void); void xen_setup_kernel_pagetable(pgd_t *pgd, unsigned long max_pfn); void xen_reserve_top(void); +void __init xen_reserve_special_pages(void); +void __init xen_pt_check_e820(void); void xen_mm_pin_all(void); void xen_mm_unpin_all(void); +#ifdef CONFIG_X86_64 +void __init xen_relocate_p2m(void); +#endif +bool __init xen_is_e820_reserved(phys_addr_t start, phys_addr_t size); unsigned long __ref xen_chk_extra_mem(unsigned long pfn); void __init xen_inv_extra_mem(void); void __init xen_remap_memory(void); +phys_addr_t __init xen_find_free_area(phys_addr_t size); char * __init xen_memory_setup(void); char * xen_auto_xlated_memory_setup(void); void __init xen_arch_setup(void); diff --git a/arch/xtensa/configs/iss_defconfig b/arch/xtensa/configs/iss_defconfig index e4d193e7a300..f3dfe0d921c2 100644 --- a/arch/xtensa/configs/iss_defconfig +++ b/arch/xtensa/configs/iss_defconfig @@ -616,7 +616,6 @@ CONFIG_SCHED_DEBUG=y # CONFIG_SLUB_DEBUG_ON is not set # CONFIG_SLUB_STATS is not set # CONFIG_DEBUG_RT_MUTEXES is not set -# CONFIG_RT_MUTEX_TESTER is not set # CONFIG_DEBUG_SPINLOCK is not set # CONFIG_DEBUG_MUTEXES is not set # CONFIG_DEBUG_SPINLOCK_SLEEP is not set diff --git a/arch/xtensa/include/asm/atomic.h b/arch/xtensa/include/asm/atomic.h index ebcd1f6fc8cb..93795d047303 100644 --- a/arch/xtensa/include/asm/atomic.h +++ b/arch/xtensa/include/asm/atomic.h @@ -145,6 +145,10 @@ static inline int atomic_##op##_return(int i, atomic_t * v) \ ATOMIC_OPS(add) ATOMIC_OPS(sub) +ATOMIC_OP(and) +ATOMIC_OP(or) +ATOMIC_OP(xor) + #undef ATOMIC_OPS #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -250,75 +254,6 @@ static __inline__ int __atomic_add_unless(atomic_t *v, int a, int u) return c; } - -static inline void atomic_clear_mask(unsigned int mask, atomic_t *v) -{ -#if XCHAL_HAVE_S32C1I - unsigned long tmp; - int result; - - __asm__ __volatile__( - "1: l32i %1, %3, 0\n" - " wsr %1, scompare1\n" - " and %0, %1, %2\n" - " s32c1i %0, %3, 0\n" - " bne %0, %1, 1b\n" - : "=&a" (result), "=&a" (tmp) - : "a" (~mask), "a" (v) - : "memory" - ); -#else - unsigned int all_f = -1; - unsigned int vval; - - __asm__ __volatile__( - " rsil a15,"__stringify(TOPLEVEL)"\n" - " l32i %0, %2, 0\n" - " xor %1, %4, %3\n" - " and %0, %0, %4\n" - " s32i %0, %2, 0\n" - " wsr a15, ps\n" - " rsync\n" - : "=&a" (vval), "=a" (mask) - : "a" (v), "a" (all_f), "1" (mask) - : "a15", "memory" - ); -#endif -} - -static inline void atomic_set_mask(unsigned int mask, atomic_t *v) -{ -#if XCHAL_HAVE_S32C1I - unsigned long tmp; - int result; - - __asm__ __volatile__( - "1: l32i %1, %3, 0\n" - " wsr %1, scompare1\n" - " or %0, %1, %2\n" - " s32c1i %0, %3, 0\n" - " bne %0, %1, 1b\n" - : "=&a" (result), "=&a" (tmp) - : "a" (mask), "a" (v) - : "memory" - ); -#else - unsigned int vval; - - __asm__ __volatile__( - " rsil a15,"__stringify(TOPLEVEL)"\n" - " l32i %0, %2, 0\n" - " or %0, %0, %1\n" - " s32i %0, %2, 0\n" - " wsr a15, ps\n" - " rsync\n" - : "=&a" (vval) - : "a" (mask), "a" (v) - : "a15", "memory" - ); -#endif -} - #endif /* __KERNEL__ */ #endif /* _XTENSA_ATOMIC_H */ diff --git a/arch/xtensa/include/asm/dma-mapping.h b/arch/xtensa/include/asm/dma-mapping.h index f01cb3044e50..4427f38b634e 100644 --- a/arch/xtensa/include/asm/dma-mapping.h +++ b/arch/xtensa/include/asm/dma-mapping.h @@ -32,66 +32,6 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev) #include -#define dma_alloc_noncoherent(d, s, h, f) dma_alloc_attrs(d, s, h, f, NULL) -#define dma_free_noncoherent(d, s, v, h) dma_free_attrs(d, s, v, h, NULL) -#define dma_alloc_coherent(d, s, h, f) dma_alloc_attrs(d, s, h, f, NULL) -#define dma_free_coherent(d, s, c, h) dma_free_attrs(d, s, c, h, NULL) - -static inline void *dma_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp, - struct dma_attrs *attrs) -{ - void *ret; - struct dma_map_ops *ops = get_dma_ops(dev); - - if (dma_alloc_from_coherent(dev, size, dma_handle, &ret)) - return ret; - - ret = ops->alloc(dev, size, dma_handle, gfp, attrs); - debug_dma_alloc_coherent(dev, size, *dma_handle, ret); - - return ret; -} - -static inline void dma_free_attrs(struct device *dev, size_t size, - void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - - if (dma_release_from_coherent(dev, get_order(size), vaddr)) - return; - - ops->free(dev, size, vaddr, dma_handle, attrs); - debug_dma_free_coherent(dev, size, vaddr, dma_handle); -} - -static inline int -dma_mapping_error(struct device *dev, dma_addr_t dma_addr) -{ - struct dma_map_ops *ops = get_dma_ops(dev); - - debug_dma_mapping_error(dev, dma_addr); - return ops->mapping_error(dev, dma_addr); -} - -static inline int -dma_supported(struct device *dev, u64 mask) -{ - return 1; -} - -static inline int -dma_set_mask(struct device *dev, u64 mask) -{ - if(!dev->dma_mask || !dma_supported(dev, mask)) - return -EIO; - - *dev->dma_mask = mask; - - return 0; -} - void dma_cache_sync(struct device *dev, void *vaddr, size_t size, enum dma_data_direction direction); diff --git a/arch/xtensa/include/asm/io.h b/arch/xtensa/include/asm/io.h index c39bb6e61911..867840f5400f 100644 --- a/arch/xtensa/include/asm/io.h +++ b/arch/xtensa/include/asm/io.h @@ -57,6 +57,7 @@ static inline void __iomem *ioremap_cache(unsigned long offset, else BUG(); } +#define ioremap_cache ioremap_cache #define ioremap_wc ioremap_nocache #define ioremap_wt ioremap_nocache diff --git a/arch/xtensa/kernel/irq.c b/arch/xtensa/kernel/irq.c index 6df31cacc4b8..4ac3d23161cf 100644 --- a/arch/xtensa/kernel/irq.c +++ b/arch/xtensa/kernel/irq.c @@ -177,23 +177,25 @@ void migrate_irqs(void) for_each_active_irq(i) { struct irq_data *data = irq_get_irq_data(i); + struct cpumask *mask; unsigned int newcpu; if (irqd_is_per_cpu(data)) continue; - if (!cpumask_test_cpu(cpu, data->affinity)) + mask = irq_data_get_affinity_mask(data); + if (!cpumask_test_cpu(cpu, mask)) continue; - newcpu = cpumask_any_and(data->affinity, cpu_online_mask); + newcpu = cpumask_any_and(mask, cpu_online_mask); if (newcpu >= nr_cpu_ids) { pr_info_ratelimited("IRQ%u no longer affine to CPU%u\n", i, cpu); - cpumask_setall(data->affinity); + cpumask_setall(mask); } - irq_set_affinity(i, data->affinity); + irq_set_affinity(i, mask); } } #endif /* CONFIG_HOTPLUG_CPU */ diff --git a/arch/xtensa/kernel/time.c b/arch/xtensa/kernel/time.c index 2a1823de69cc..b97767dbc7c8 100644 --- a/arch/xtensa/kernel/time.c +++ b/arch/xtensa/kernel/time.c @@ -52,8 +52,6 @@ static struct clocksource ccount_clocksource = { static int ccount_timer_set_next_event(unsigned long delta, struct clock_event_device *dev); -static void ccount_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt); struct ccount_timer { struct clock_event_device evt; int irq_enabled; @@ -77,35 +75,34 @@ static int ccount_timer_set_next_event(unsigned long delta, return ret; } -static void ccount_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +/* + * There is no way to disable the timer interrupt at the device level, + * only at the intenable register itself. Since enable_irq/disable_irq + * calls are nested, we need to make sure that these calls are + * balanced. + */ +static int ccount_timer_shutdown(struct clock_event_device *evt) { struct ccount_timer *timer = container_of(evt, struct ccount_timer, evt); - /* - * There is no way to disable the timer interrupt at the device level, - * only at the intenable register itself. Since enable_irq/disable_irq - * calls are nested, we need to make sure that these calls are - * balanced. - */ - switch (mode) { - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - if (timer->irq_enabled) { - disable_irq(evt->irq); - timer->irq_enabled = 0; - } - break; - case CLOCK_EVT_MODE_RESUME: - case CLOCK_EVT_MODE_ONESHOT: - if (!timer->irq_enabled) { - enable_irq(evt->irq); - timer->irq_enabled = 1; - } - default: - break; + if (timer->irq_enabled) { + disable_irq(evt->irq); + timer->irq_enabled = 0; } + return 0; +} + +static int ccount_timer_set_oneshot(struct clock_event_device *evt) +{ + struct ccount_timer *timer = + container_of(evt, struct ccount_timer, evt); + + if (!timer->irq_enabled) { + enable_irq(evt->irq); + timer->irq_enabled = 1; + } + return 0; } static irqreturn_t timer_interrupt(int irq, void *dev_id); @@ -126,7 +123,9 @@ void local_timer_setup(unsigned cpu) clockevent->features = CLOCK_EVT_FEAT_ONESHOT; clockevent->rating = 300; clockevent->set_next_event = ccount_timer_set_next_event; - clockevent->set_mode = ccount_timer_set_mode; + clockevent->set_state_shutdown = ccount_timer_shutdown; + clockevent->set_state_oneshot = ccount_timer_set_oneshot; + clockevent->tick_resume = ccount_timer_set_oneshot; clockevent->cpumask = cpumask_of(cpu); clockevent->irq = irq_create_mapping(NULL, LINUX_TIMER_INT); if (WARN(!clockevent->irq, "error: can't map timer irq")) diff --git a/arch/xtensa/platforms/iss/simdisk.c b/arch/xtensa/platforms/iss/simdisk.c index 48eebacdf5fe..fa84ca990caa 100644 --- a/arch/xtensa/platforms/iss/simdisk.c +++ b/arch/xtensa/platforms/iss/simdisk.c @@ -101,8 +101,9 @@ static void simdisk_transfer(struct simdisk *dev, unsigned long sector, spin_unlock(&dev->lock); } -static int simdisk_xfer_bio(struct simdisk *dev, struct bio *bio) +static void simdisk_make_request(struct request_queue *q, struct bio *bio) { + struct simdisk *dev = q->queuedata; struct bio_vec bvec; struct bvec_iter iter; sector_t sector = bio->bi_iter.bi_sector; @@ -116,17 +117,10 @@ static int simdisk_xfer_bio(struct simdisk *dev, struct bio *bio) sector += len; __bio_kunmap_atomic(buffer); } - return 0; -} -static void simdisk_make_request(struct request_queue *q, struct bio *bio) -{ - struct simdisk *dev = q->queuedata; - int status = simdisk_xfer_bio(dev, bio); - bio_endio(bio, status); + bio_endio(bio); } - static int simdisk_open(struct block_device *bdev, fmode_t mode) { struct simdisk *dev = bdev->bd_disk->private_data; diff --git a/block/bio-integrity.c b/block/bio-integrity.c index 719b7152aed1..4aecca79374a 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -355,13 +355,12 @@ static void bio_integrity_verify_fn(struct work_struct *work) container_of(work, struct bio_integrity_payload, bip_work); struct bio *bio = bip->bip_bio; struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); - int error; - error = bio_integrity_process(bio, bi->verify_fn); + bio->bi_error = bio_integrity_process(bio, bi->verify_fn); /* Restore original bio completion handler */ bio->bi_end_io = bip->bip_end_io; - bio_endio(bio, error); + bio_endio(bio); } /** @@ -376,7 +375,7 @@ static void bio_integrity_verify_fn(struct work_struct *work) * in process context. This function postpones completion * accordingly. */ -void bio_integrity_endio(struct bio *bio, int error) +void bio_integrity_endio(struct bio *bio) { struct bio_integrity_payload *bip = bio_integrity(bio); @@ -386,9 +385,9 @@ void bio_integrity_endio(struct bio *bio, int error) * integrity metadata. Restore original bio end_io handler * and run it. */ - if (error) { + if (bio->bi_error) { bio->bi_end_io = bip->bip_end_io; - bio_endio(bio, error); + bio_endio(bio); return; } diff --git a/block/bio.c b/block/bio.c index d6e5ba3399f0..ad3f276d74bc 100644 --- a/block/bio.c +++ b/block/bio.c @@ -269,7 +269,6 @@ static void bio_free(struct bio *bio) void bio_init(struct bio *bio) { memset(bio, 0, sizeof(*bio)); - bio->bi_flags = 1 << BIO_UPTODATE; atomic_set(&bio->__bi_remaining, 1); atomic_set(&bio->__bi_cnt, 1); } @@ -292,14 +291,17 @@ void bio_reset(struct bio *bio) __bio_free(bio); memset(bio, 0, BIO_RESET_BYTES); - bio->bi_flags = flags | (1 << BIO_UPTODATE); + bio->bi_flags = flags; atomic_set(&bio->__bi_remaining, 1); } EXPORT_SYMBOL(bio_reset); -static void bio_chain_endio(struct bio *bio, int error) +static void bio_chain_endio(struct bio *bio) { - bio_endio(bio->bi_private, error); + struct bio *parent = bio->bi_private; + + parent->bi_error = bio->bi_error; + bio_endio(parent); bio_put(bio); } @@ -309,7 +311,7 @@ static void bio_chain_endio(struct bio *bio, int error) */ static inline void bio_inc_remaining(struct bio *bio) { - bio->bi_flags |= (1 << BIO_CHAIN); + bio_set_flag(bio, BIO_CHAIN); smp_mb__before_atomic(); atomic_inc(&bio->__bi_remaining); } @@ -493,7 +495,7 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs) if (unlikely(!bvl)) goto err_free; - bio->bi_flags |= 1 << BIO_OWNS_VEC; + bio_set_flag(bio, BIO_OWNS_VEC); } else if (nr_iovecs) { bvl = bio->bi_inline_vecs; } @@ -578,7 +580,7 @@ void __bio_clone_fast(struct bio *bio, struct bio *bio_src) * so we don't set nor calculate new physical/hw segment counts here */ bio->bi_bdev = bio_src->bi_bdev; - bio->bi_flags |= 1 << BIO_CLONED; + bio_set_flag(bio, BIO_CLONED); bio->bi_rw = bio_src->bi_rw; bio->bi_iter = bio_src->bi_iter; bio->bi_io_vec = bio_src->bi_io_vec; @@ -692,31 +694,22 @@ struct bio *bio_clone_bioset(struct bio *bio_src, gfp_t gfp_mask, EXPORT_SYMBOL(bio_clone_bioset); /** - * bio_get_nr_vecs - return approx number of vecs - * @bdev: I/O target + * bio_add_pc_page - attempt to add page to bio + * @q: the target queue + * @bio: destination bio + * @page: page to add + * @len: vec entry length + * @offset: vec entry offset * - * Return the approximate number of pages we can send to this target. - * There's no guarantee that you will be able to fit this number of pages - * into a bio, it does not account for dynamic restrictions that vary - * on offset. + * Attempt to add a page to the bio_vec maplist. This can fail for a + * number of reasons, such as the bio being full or target block device + * limitations. The target block device must allow bio's up to PAGE_SIZE, + * so it is always possible to add a single page to an empty bio. + * + * This should only be used by REQ_PC bios. */ -int bio_get_nr_vecs(struct block_device *bdev) -{ - struct request_queue *q = bdev_get_queue(bdev); - int nr_pages; - - nr_pages = min_t(unsigned, - queue_max_segments(q), - queue_max_sectors(q) / (PAGE_SIZE >> 9) + 1); - - return min_t(unsigned, nr_pages, BIO_MAX_PAGES); - -} -EXPORT_SYMBOL(bio_get_nr_vecs); - -static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page - *page, unsigned int len, unsigned int offset, - unsigned int max_sectors) +int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page + *page, unsigned int len, unsigned int offset) { int retried_segments = 0; struct bio_vec *bvec; @@ -727,7 +720,7 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page if (unlikely(bio_flagged(bio, BIO_CLONED))) return 0; - if (((bio->bi_iter.bi_size + len) >> 9) > max_sectors) + if (((bio->bi_iter.bi_size + len) >> 9) > queue_max_hw_sectors(q)) return 0; /* @@ -740,28 +733,7 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page if (page == prev->bv_page && offset == prev->bv_offset + prev->bv_len) { - unsigned int prev_bv_len = prev->bv_len; prev->bv_len += len; - - if (q->merge_bvec_fn) { - struct bvec_merge_data bvm = { - /* prev_bvec is already charged in - bi_size, discharge it in order to - simulate merging updated prev_bvec - as new bvec. */ - .bi_bdev = bio->bi_bdev, - .bi_sector = bio->bi_iter.bi_sector, - .bi_size = bio->bi_iter.bi_size - - prev_bv_len, - .bi_rw = bio->bi_rw, - }; - - if (q->merge_bvec_fn(q, &bvm, prev) < prev->bv_len) { - prev->bv_len -= len; - return 0; - } - } - bio->bi_iter.bi_size += len; goto done; } @@ -770,8 +742,7 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page * If the queue doesn't support SG gaps and adding this * offset would create a gap, disallow it. */ - if (q->queue_flags & (1 << QUEUE_FLAG_SG_GAPS) && - bvec_gap_to_prev(prev, offset)) + if (bvec_gap_to_prev(q, prev, offset)) return 0; } @@ -804,30 +775,9 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page blk_recount_segments(q, bio); } - /* - * if queue has other restrictions (eg varying max sector size - * depending on offset), it can specify a merge_bvec_fn in the - * queue to get further control - */ - if (q->merge_bvec_fn) { - struct bvec_merge_data bvm = { - .bi_bdev = bio->bi_bdev, - .bi_sector = bio->bi_iter.bi_sector, - .bi_size = bio->bi_iter.bi_size - len, - .bi_rw = bio->bi_rw, - }; - - /* - * merge_bvec_fn() returns number of bytes it can accept - * at this offset - */ - if (q->merge_bvec_fn(q, &bvm, bvec) < bvec->bv_len) - goto failed; - } - /* If we may be able to merge these biovecs, force a recount */ if (bio->bi_vcnt > 1 && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec))) - bio->bi_flags &= ~(1 << BIO_SEG_VALID); + bio_clear_flag(bio, BIO_SEG_VALID); done: return len; @@ -841,28 +791,6 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page blk_recount_segments(q, bio); return 0; } - -/** - * bio_add_pc_page - attempt to add page to bio - * @q: the target queue - * @bio: destination bio - * @page: page to add - * @len: vec entry length - * @offset: vec entry offset - * - * Attempt to add a page to the bio_vec maplist. This can fail for a - * number of reasons, such as the bio being full or target block device - * limitations. The target block device must allow bio's up to PAGE_SIZE, - * so it is always possible to add a single page to an empty bio. - * - * This should only be used by REQ_PC bios. - */ -int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page *page, - unsigned int len, unsigned int offset) -{ - return __bio_add_page(q, bio, page, len, offset, - queue_max_hw_sectors(q)); -} EXPORT_SYMBOL(bio_add_pc_page); /** @@ -872,22 +800,47 @@ EXPORT_SYMBOL(bio_add_pc_page); * @len: vec entry length * @offset: vec entry offset * - * Attempt to add a page to the bio_vec maplist. This can fail for a - * number of reasons, such as the bio being full or target block device - * limitations. The target block device must allow bio's up to PAGE_SIZE, - * so it is always possible to add a single page to an empty bio. + * Attempt to add a page to the bio_vec maplist. This will only fail + * if either bio->bi_vcnt == bio->bi_max_vecs or it's a cloned bio. */ -int bio_add_page(struct bio *bio, struct page *page, unsigned int len, - unsigned int offset) +int bio_add_page(struct bio *bio, struct page *page, + unsigned int len, unsigned int offset) { - struct request_queue *q = bdev_get_queue(bio->bi_bdev); - unsigned int max_sectors; + struct bio_vec *bv; - max_sectors = blk_max_size_offset(q, bio->bi_iter.bi_sector); - if ((max_sectors < (len >> 9)) && !bio->bi_iter.bi_size) - max_sectors = len >> 9; + /* + * cloned bio must not modify vec list + */ + if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) + return 0; - return __bio_add_page(q, bio, page, len, offset, max_sectors); + /* + * For filesystems with a blocksize smaller than the pagesize + * we will often be called with the same page as last time and + * a consecutive offset. Optimize this special case. + */ + if (bio->bi_vcnt > 0) { + bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; + + if (page == bv->bv_page && + offset == bv->bv_offset + bv->bv_len) { + bv->bv_len += len; + goto done; + } + } + + if (bio->bi_vcnt >= bio->bi_max_vecs) + return 0; + + bv = &bio->bi_io_vec[bio->bi_vcnt]; + bv->bv_page = page; + bv->bv_len = len; + bv->bv_offset = offset; + + bio->bi_vcnt++; +done: + bio->bi_iter.bi_size += len; + return len; } EXPORT_SYMBOL(bio_add_page); @@ -896,11 +849,11 @@ struct submit_bio_ret { int error; }; -static void submit_bio_wait_endio(struct bio *bio, int error) +static void submit_bio_wait_endio(struct bio *bio) { struct submit_bio_ret *ret = bio->bi_private; - ret->error = error; + ret->error = bio->bi_error; complete(&ret->event); } @@ -1388,7 +1341,7 @@ struct bio *bio_map_user_iov(struct request_queue *q, if (iter->type & WRITE) bio->bi_rw |= REQ_WRITE; - bio->bi_flags |= (1 << BIO_USER_MAPPED); + bio_set_flag(bio, BIO_USER_MAPPED); /* * subtle -- if __bio_map_user() ended up bouncing a bio, @@ -1445,7 +1398,7 @@ void bio_unmap_user(struct bio *bio) } EXPORT_SYMBOL(bio_unmap_user); -static void bio_map_kern_endio(struct bio *bio, int err) +static void bio_map_kern_endio(struct bio *bio) { bio_put(bio); } @@ -1501,13 +1454,13 @@ struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len, } EXPORT_SYMBOL(bio_map_kern); -static void bio_copy_kern_endio(struct bio *bio, int err) +static void bio_copy_kern_endio(struct bio *bio) { bio_free_pages(bio); bio_put(bio); } -static void bio_copy_kern_endio_read(struct bio *bio, int err) +static void bio_copy_kern_endio_read(struct bio *bio) { char *p = bio->bi_private; struct bio_vec *bvec; @@ -1518,7 +1471,7 @@ static void bio_copy_kern_endio_read(struct bio *bio, int err) p += bvec->bv_len; } - bio_copy_kern_endio(bio, err); + bio_copy_kern_endio(bio); } /** @@ -1768,7 +1721,7 @@ static inline bool bio_remaining_done(struct bio *bio) BUG_ON(atomic_read(&bio->__bi_remaining) <= 0); if (atomic_dec_and_test(&bio->__bi_remaining)) { - clear_bit(BIO_CHAIN, &bio->bi_flags); + bio_clear_flag(bio, BIO_CHAIN); return true; } @@ -1778,25 +1731,15 @@ static inline bool bio_remaining_done(struct bio *bio) /** * bio_endio - end I/O on a bio * @bio: bio - * @error: error, if any * * Description: - * bio_endio() will end I/O on the whole bio. bio_endio() is the - * preferred way to end I/O on a bio, it takes care of clearing - * BIO_UPTODATE on error. @error is 0 on success, and and one of the - * established -Exxxx (-EIO, for instance) error values in case - * something went wrong. No one should call bi_end_io() directly on a - * bio unless they own it and thus know that it has an end_io - * function. + * bio_endio() will end I/O on the whole bio. bio_endio() is the preferred + * way to end I/O on a bio. No one should call bi_end_io() directly on a + * bio unless they own it and thus know that it has an end_io function. **/ -void bio_endio(struct bio *bio, int error) +void bio_endio(struct bio *bio) { while (bio) { - if (error) - clear_bit(BIO_UPTODATE, &bio->bi_flags); - else if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) - error = -EIO; - if (unlikely(!bio_remaining_done(bio))) break; @@ -1810,11 +1753,12 @@ void bio_endio(struct bio *bio, int error) */ if (bio->bi_end_io == bio_chain_endio) { struct bio *parent = bio->bi_private; + parent->bi_error = bio->bi_error; bio_put(bio); bio = parent; } else { if (bio->bi_end_io) - bio->bi_end_io(bio, error); + bio->bi_end_io(bio); bio = NULL; } } @@ -1882,7 +1826,7 @@ void bio_trim(struct bio *bio, int offset, int size) if (offset == 0 && size == bio->bi_iter.bi_size) return; - clear_bit(BIO_SEG_VALID, &bio->bi_flags); + bio_clear_flag(bio, BIO_SEG_VALID); bio_advance(bio, offset << 9); @@ -2046,7 +1990,7 @@ int bio_associate_current(struct bio *bio) get_io_context_active(ioc); bio->bi_ioc = ioc; - bio->bi_css = task_get_css(current, blkio_cgrp_id); + bio->bi_css = task_get_css(current, io_cgrp_id); return 0; } EXPORT_SYMBOL_GPL(bio_associate_current); diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index d6283b3f5db5..ac8370cb2515 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include "blk.h" @@ -68,9 +69,14 @@ static void blkg_free(struct blkcg_gq *blkg) return; for (i = 0; i < BLKCG_MAX_POLS; i++) - kfree(blkg->pd[i]); + if (blkg->pd[i]) + blkcg_policy[i]->pd_free_fn(blkg->pd[i]); - blk_exit_rl(&blkg->rl); + if (blkg->blkcg != &blkcg_root) + blk_exit_rl(&blkg->rl); + + blkg_rwstat_exit(&blkg->stat_ios); + blkg_rwstat_exit(&blkg->stat_bytes); kfree(blkg); } @@ -93,6 +99,10 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q, if (!blkg) return NULL; + if (blkg_rwstat_init(&blkg->stat_bytes, gfp_mask) || + blkg_rwstat_init(&blkg->stat_ios, gfp_mask)) + goto err_free; + blkg->q = q; INIT_LIST_HEAD(&blkg->q_node); blkg->blkcg = blkcg; @@ -113,7 +123,7 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q, continue; /* alloc per-policy data and attach it to blkg */ - pd = kzalloc_node(pol->pd_size, gfp_mask, q->node); + pd = pol->pd_alloc_fn(gfp_mask, q->node); if (!pd) goto err_free; @@ -129,26 +139,11 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q, return NULL; } -/** - * __blkg_lookup - internal version of blkg_lookup() - * @blkcg: blkcg of interest - * @q: request_queue of interest - * @update_hint: whether to update lookup hint with the result or not - * - * This is internal version and shouldn't be used by policy - * implementations. Looks up blkgs for the @blkcg - @q pair regardless of - * @q's bypass state. If @update_hint is %true, the caller should be - * holding @q->queue_lock and lookup hint is updated on success. - */ -struct blkcg_gq *__blkg_lookup(struct blkcg *blkcg, struct request_queue *q, - bool update_hint) +struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg, + struct request_queue *q, bool update_hint) { struct blkcg_gq *blkg; - blkg = rcu_dereference(blkcg->blkg_hint); - if (blkg && blkg->q == q) - return blkg; - /* * Hint didn't match. Look up from the radix tree. Note that the * hint can only be updated under queue_lock as otherwise @blkg @@ -166,29 +161,11 @@ struct blkcg_gq *__blkg_lookup(struct blkcg *blkcg, struct request_queue *q, return NULL; } - -/** - * blkg_lookup - lookup blkg for the specified blkcg - q pair - * @blkcg: blkcg of interest - * @q: request_queue of interest - * - * Lookup blkg for the @blkcg - @q pair. This function should be called - * under RCU read lock and is guaranteed to return %NULL if @q is bypassing - * - see blk_queue_bypass_start() for details. - */ -struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, struct request_queue *q) -{ - WARN_ON_ONCE(!rcu_read_lock_held()); - - if (unlikely(blk_queue_bypass(q))) - return NULL; - return __blkg_lookup(blkcg, q, false); -} -EXPORT_SYMBOL_GPL(blkg_lookup); +EXPORT_SYMBOL_GPL(blkg_lookup_slowpath); /* * If @new_blkg is %NULL, this function tries to allocate a new one as - * necessary using %GFP_ATOMIC. @new_blkg is always consumed on return. + * necessary using %GFP_NOWAIT. @new_blkg is always consumed on return. */ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct request_queue *q, @@ -203,12 +180,12 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, /* blkg holds a reference to blkcg */ if (!css_tryget_online(&blkcg->css)) { - ret = -EINVAL; + ret = -ENODEV; goto err_free_blkg; } wb_congested = wb_congested_get_create(&q->backing_dev_info, - blkcg->css.id, GFP_ATOMIC); + blkcg->css.id, GFP_NOWAIT); if (!wb_congested) { ret = -ENOMEM; goto err_put_css; @@ -216,7 +193,7 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, /* allocate */ if (!new_blkg) { - new_blkg = blkg_alloc(blkcg, q, GFP_ATOMIC); + new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT); if (unlikely(!new_blkg)) { ret = -ENOMEM; goto err_put_congested; @@ -229,7 +206,7 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, if (blkcg_parent(blkcg)) { blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false); if (WARN_ON_ONCE(!blkg->parent)) { - ret = -EINVAL; + ret = -ENODEV; goto err_put_congested; } blkg_get(blkg->parent); @@ -240,7 +217,7 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct blkcg_policy *pol = blkcg_policy[i]; if (blkg->pd[i] && pol->pd_init_fn) - pol->pd_init_fn(blkg); + pol->pd_init_fn(blkg->pd[i]); } /* insert */ @@ -254,7 +231,7 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct blkcg_policy *pol = blkcg_policy[i]; if (blkg->pd[i] && pol->pd_online_fn) - pol->pd_online_fn(blkg); + pol->pd_online_fn(blkg->pd[i]); } } blkg->online = true; @@ -303,7 +280,7 @@ struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, * we shouldn't allow anything to go through for a bypassing queue. */ if (unlikely(blk_queue_bypass(q))) - return ERR_PTR(blk_queue_dying(q) ? -EINVAL : -EBUSY); + return ERR_PTR(blk_queue_dying(q) ? -ENODEV : -EBUSY); blkg = __blkg_lookup(blkcg, q, true); if (blkg) @@ -327,11 +304,11 @@ struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, return blkg; } } -EXPORT_SYMBOL_GPL(blkg_lookup_create); static void blkg_destroy(struct blkcg_gq *blkg) { struct blkcg *blkcg = blkg->blkcg; + struct blkcg_gq *parent = blkg->parent; int i; lockdep_assert_held(blkg->q->queue_lock); @@ -345,8 +322,14 @@ static void blkg_destroy(struct blkcg_gq *blkg) struct blkcg_policy *pol = blkcg_policy[i]; if (blkg->pd[i] && pol->pd_offline_fn) - pol->pd_offline_fn(blkg); + pol->pd_offline_fn(blkg->pd[i]); } + + if (parent) { + blkg_rwstat_add_aux(&parent->stat_bytes, &blkg->stat_bytes); + blkg_rwstat_add_aux(&parent->stat_ios, &blkg->stat_ios); + } + blkg->online = false; radix_tree_delete(&blkcg->blkg_tree, blkg->q->id); @@ -400,15 +383,6 @@ static void blkg_destroy_all(struct request_queue *q) void __blkg_release_rcu(struct rcu_head *rcu_head) { struct blkcg_gq *blkg = container_of(rcu_head, struct blkcg_gq, rcu_head); - int i; - - /* tell policies that this one is being freed */ - for (i = 0; i < BLKCG_MAX_POLS; i++) { - struct blkcg_policy *pol = blkcg_policy[i]; - - if (blkg->pd[i] && pol->pd_exit_fn) - pol->pd_exit_fn(blkg); - } /* release the blkcg and parent blkg refs this blkg has been holding */ css_put(&blkg->blkcg->css); @@ -472,12 +446,14 @@ static int blkcg_reset_stats(struct cgroup_subsys_state *css, * anyway. If you get hit by a race, retry. */ hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) { + blkg_rwstat_reset(&blkg->stat_bytes); + blkg_rwstat_reset(&blkg->stat_ios); + for (i = 0; i < BLKCG_MAX_POLS; i++) { struct blkcg_policy *pol = blkcg_policy[i]; - if (blkcg_policy_enabled(blkg->q, pol) && - pol->pd_reset_stats_fn) - pol->pd_reset_stats_fn(blkg); + if (blkg->pd[i] && pol->pd_reset_stats_fn) + pol->pd_reset_stats_fn(blkg->pd[i]); } } @@ -486,13 +462,14 @@ static int blkcg_reset_stats(struct cgroup_subsys_state *css, return 0; } -static const char *blkg_dev_name(struct blkcg_gq *blkg) +const char *blkg_dev_name(struct blkcg_gq *blkg) { /* some drivers (floppy) instantiate a queue w/o disk registered */ if (blkg->q->backing_dev_info.dev) return dev_name(blkg->q->backing_dev_info.dev); return NULL; } +EXPORT_SYMBOL_GPL(blkg_dev_name); /** * blkcg_print_blkgs - helper for printing per-blkg data @@ -581,9 +558,10 @@ u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, for (i = 0; i < BLKG_RWSTAT_NR; i++) seq_printf(sf, "%s %s %llu\n", dname, rwstr[i], - (unsigned long long)rwstat->cnt[i]); + (unsigned long long)atomic64_read(&rwstat->aux_cnt[i])); - v = rwstat->cnt[BLKG_RWSTAT_READ] + rwstat->cnt[BLKG_RWSTAT_WRITE]; + v = atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_READ]) + + atomic64_read(&rwstat->aux_cnt[BLKG_RWSTAT_WRITE]); seq_printf(sf, "%s Total %llu\n", dname, (unsigned long long)v); return v; } @@ -620,31 +598,122 @@ u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd, } EXPORT_SYMBOL_GPL(blkg_prfill_rwstat); +static u64 blkg_prfill_rwstat_field(struct seq_file *sf, + struct blkg_policy_data *pd, int off) +{ + struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd->blkg + off); + + return __blkg_prfill_rwstat(sf, pd, &rwstat); +} + +/** + * blkg_print_stat_bytes - seq_show callback for blkg->stat_bytes + * @sf: seq_file to print to + * @v: unused + * + * To be used as cftype->seq_show to print blkg->stat_bytes. + * cftype->private must be set to the blkcg_policy. + */ +int blkg_print_stat_bytes(struct seq_file *sf, void *v) +{ + blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), + blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private, + offsetof(struct blkcg_gq, stat_bytes), true); + return 0; +} +EXPORT_SYMBOL_GPL(blkg_print_stat_bytes); + +/** + * blkg_print_stat_bytes - seq_show callback for blkg->stat_ios + * @sf: seq_file to print to + * @v: unused + * + * To be used as cftype->seq_show to print blkg->stat_ios. cftype->private + * must be set to the blkcg_policy. + */ +int blkg_print_stat_ios(struct seq_file *sf, void *v) +{ + blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), + blkg_prfill_rwstat_field, (void *)seq_cft(sf)->private, + offsetof(struct blkcg_gq, stat_ios), true); + return 0; +} +EXPORT_SYMBOL_GPL(blkg_print_stat_ios); + +static u64 blkg_prfill_rwstat_field_recursive(struct seq_file *sf, + struct blkg_policy_data *pd, + int off) +{ + struct blkg_rwstat rwstat = blkg_rwstat_recursive_sum(pd->blkg, + NULL, off); + return __blkg_prfill_rwstat(sf, pd, &rwstat); +} + +/** + * blkg_print_stat_bytes_recursive - recursive version of blkg_print_stat_bytes + * @sf: seq_file to print to + * @v: unused + */ +int blkg_print_stat_bytes_recursive(struct seq_file *sf, void *v) +{ + blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), + blkg_prfill_rwstat_field_recursive, + (void *)seq_cft(sf)->private, + offsetof(struct blkcg_gq, stat_bytes), true); + return 0; +} +EXPORT_SYMBOL_GPL(blkg_print_stat_bytes_recursive); + +/** + * blkg_print_stat_ios_recursive - recursive version of blkg_print_stat_ios + * @sf: seq_file to print to + * @v: unused + */ +int blkg_print_stat_ios_recursive(struct seq_file *sf, void *v) +{ + blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), + blkg_prfill_rwstat_field_recursive, + (void *)seq_cft(sf)->private, + offsetof(struct blkcg_gq, stat_ios), true); + return 0; +} +EXPORT_SYMBOL_GPL(blkg_print_stat_ios_recursive); + /** * blkg_stat_recursive_sum - collect hierarchical blkg_stat - * @pd: policy private data of interest - * @off: offset to the blkg_stat in @pd + * @blkg: blkg of interest + * @pol: blkcg_policy which contains the blkg_stat + * @off: offset to the blkg_stat in blkg_policy_data or @blkg * - * Collect the blkg_stat specified by @off from @pd and all its online - * descendants and return the sum. The caller must be holding the queue - * lock for online tests. + * Collect the blkg_stat specified by @blkg, @pol and @off and all its + * online descendants and their aux counts. The caller must be holding the + * queue lock for online tests. + * + * If @pol is NULL, blkg_stat is at @off bytes into @blkg; otherwise, it is + * at @off bytes into @blkg's blkg_policy_data of the policy. */ -u64 blkg_stat_recursive_sum(struct blkg_policy_data *pd, int off) +u64 blkg_stat_recursive_sum(struct blkcg_gq *blkg, + struct blkcg_policy *pol, int off) { - struct blkcg_policy *pol = blkcg_policy[pd->plid]; struct blkcg_gq *pos_blkg; struct cgroup_subsys_state *pos_css; u64 sum = 0; - lockdep_assert_held(pd->blkg->q->queue_lock); + lockdep_assert_held(blkg->q->queue_lock); rcu_read_lock(); - blkg_for_each_descendant_pre(pos_blkg, pos_css, pd_to_blkg(pd)) { - struct blkg_policy_data *pos_pd = blkg_to_pd(pos_blkg, pol); - struct blkg_stat *stat = (void *)pos_pd + off; + blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) { + struct blkg_stat *stat; - if (pos_blkg->online) - sum += blkg_stat_read(stat); + if (!pos_blkg->online) + continue; + + if (pol) + stat = (void *)blkg_to_pd(pos_blkg, pol) + off; + else + stat = (void *)blkg + off; + + sum += blkg_stat_read(stat) + atomic64_read(&stat->aux_cnt); } rcu_read_unlock(); @@ -654,37 +723,43 @@ EXPORT_SYMBOL_GPL(blkg_stat_recursive_sum); /** * blkg_rwstat_recursive_sum - collect hierarchical blkg_rwstat - * @pd: policy private data of interest - * @off: offset to the blkg_stat in @pd + * @blkg: blkg of interest + * @pol: blkcg_policy which contains the blkg_rwstat + * @off: offset to the blkg_rwstat in blkg_policy_data or @blkg * - * Collect the blkg_rwstat specified by @off from @pd and all its online - * descendants and return the sum. The caller must be holding the queue - * lock for online tests. + * Collect the blkg_rwstat specified by @blkg, @pol and @off and all its + * online descendants and their aux counts. The caller must be holding the + * queue lock for online tests. + * + * If @pol is NULL, blkg_rwstat is at @off bytes into @blkg; otherwise, it + * is at @off bytes into @blkg's blkg_policy_data of the policy. */ -struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkg_policy_data *pd, - int off) +struct blkg_rwstat blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, + struct blkcg_policy *pol, int off) { - struct blkcg_policy *pol = blkcg_policy[pd->plid]; struct blkcg_gq *pos_blkg; struct cgroup_subsys_state *pos_css; struct blkg_rwstat sum = { }; int i; - lockdep_assert_held(pd->blkg->q->queue_lock); + lockdep_assert_held(blkg->q->queue_lock); rcu_read_lock(); - blkg_for_each_descendant_pre(pos_blkg, pos_css, pd_to_blkg(pd)) { - struct blkg_policy_data *pos_pd = blkg_to_pd(pos_blkg, pol); - struct blkg_rwstat *rwstat = (void *)pos_pd + off; - struct blkg_rwstat tmp; + blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) { + struct blkg_rwstat *rwstat; if (!pos_blkg->online) continue; - tmp = blkg_rwstat_read(rwstat); + if (pol) + rwstat = (void *)blkg_to_pd(pos_blkg, pol) + off; + else + rwstat = (void *)pos_blkg + off; for (i = 0; i < BLKG_RWSTAT_NR; i++) - sum.cnt[i] += tmp.cnt[i]; + atomic64_add(atomic64_read(&rwstat->aux_cnt[i]) + + percpu_counter_sum_positive(&rwstat->cpu_cnt[i]), + &sum.aux_cnt[i]); } rcu_read_unlock(); @@ -700,29 +775,34 @@ EXPORT_SYMBOL_GPL(blkg_rwstat_recursive_sum); * @ctx: blkg_conf_ctx to be filled * * Parse per-blkg config update from @input and initialize @ctx with the - * result. @ctx->blkg points to the blkg to be updated and @ctx->v the new - * value. This function returns with RCU read lock and queue lock held and - * must be paired with blkg_conf_finish(). + * result. @ctx->blkg points to the blkg to be updated and @ctx->body the + * part of @input following MAJ:MIN. This function returns with RCU read + * lock and queue lock held and must be paired with blkg_conf_finish(). */ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, - const char *input, struct blkg_conf_ctx *ctx) + char *input, struct blkg_conf_ctx *ctx) __acquires(rcu) __acquires(disk->queue->queue_lock) { struct gendisk *disk; struct blkcg_gq *blkg; unsigned int major, minor; - unsigned long long v; - int part, ret; + int key_len, part, ret; + char *body; - if (sscanf(input, "%u:%u %llu", &major, &minor, &v) != 3) + if (sscanf(input, "%u:%u%n", &major, &minor, &key_len) != 2) return -EINVAL; + body = input + key_len; + if (!isspace(*body)) + return -EINVAL; + body = skip_spaces(body); + disk = get_gendisk(MKDEV(major, minor), &part); if (!disk) - return -EINVAL; + return -ENODEV; if (part) { put_disk(disk); - return -EINVAL; + return -ENODEV; } rcu_read_lock(); @@ -731,7 +811,7 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, if (blkcg_policy_enabled(disk->queue, pol)) blkg = blkg_lookup_create(blkcg, disk->queue); else - blkg = ERR_PTR(-EINVAL); + blkg = ERR_PTR(-EOPNOTSUPP); if (IS_ERR(blkg)) { ret = PTR_ERR(blkg); @@ -753,7 +833,7 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol, ctx->disk = disk; ctx->blkg = blkg; - ctx->v = v; + ctx->body = body; return 0; } EXPORT_SYMBOL_GPL(blkg_conf_prep); @@ -774,7 +854,54 @@ void blkg_conf_finish(struct blkg_conf_ctx *ctx) } EXPORT_SYMBOL_GPL(blkg_conf_finish); +static int blkcg_print_stat(struct seq_file *sf, void *v) +{ + struct blkcg *blkcg = css_to_blkcg(seq_css(sf)); + struct blkcg_gq *blkg; + + rcu_read_lock(); + + hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { + const char *dname; + struct blkg_rwstat rwstat; + u64 rbytes, wbytes, rios, wios; + + dname = blkg_dev_name(blkg); + if (!dname) + continue; + + spin_lock_irq(blkg->q->queue_lock); + + rwstat = blkg_rwstat_recursive_sum(blkg, NULL, + offsetof(struct blkcg_gq, stat_bytes)); + rbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]); + wbytes = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]); + + rwstat = blkg_rwstat_recursive_sum(blkg, NULL, + offsetof(struct blkcg_gq, stat_ios)); + rios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_READ]); + wios = atomic64_read(&rwstat.aux_cnt[BLKG_RWSTAT_WRITE]); + + spin_unlock_irq(blkg->q->queue_lock); + + if (rbytes || wbytes || rios || wios) + seq_printf(sf, "%s rbytes=%llu wbytes=%llu rios=%llu wios=%llu\n", + dname, rbytes, wbytes, rios, wios); + } + + rcu_read_unlock(); + return 0; +} + struct cftype blkcg_files[] = { + { + .name = "stat", + .seq_show = blkcg_print_stat, + }, + { } /* terminate */ +}; + +struct cftype blkcg_legacy_files[] = { { .name = "reset_stats", .write_u64 = blkcg_reset_stats, @@ -822,18 +949,19 @@ static void blkcg_css_offline(struct cgroup_subsys_state *css) static void blkcg_css_free(struct cgroup_subsys_state *css) { struct blkcg *blkcg = css_to_blkcg(css); + int i; mutex_lock(&blkcg_pol_mutex); + list_del(&blkcg->all_blkcgs_node); + + for (i = 0; i < BLKCG_MAX_POLS; i++) + if (blkcg->cpd[i]) + blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]); + mutex_unlock(&blkcg_pol_mutex); - if (blkcg != &blkcg_root) { - int i; - - for (i = 0; i < BLKCG_MAX_POLS; i++) - kfree(blkcg->pd[i]); - kfree(blkcg); - } + kfree(blkcg); } static struct cgroup_subsys_state * @@ -847,13 +975,12 @@ blkcg_css_alloc(struct cgroup_subsys_state *parent_css) if (!parent_css) { blkcg = &blkcg_root; - goto done; - } - - blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL); - if (!blkcg) { - ret = ERR_PTR(-ENOMEM); - goto free_blkcg; + } else { + blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL); + if (!blkcg) { + ret = ERR_PTR(-ENOMEM); + goto free_blkcg; + } } for (i = 0; i < BLKCG_MAX_POLS ; i++) { @@ -866,23 +993,23 @@ blkcg_css_alloc(struct cgroup_subsys_state *parent_css) * check if the policy requires any specific per-cgroup * data: if it does, allocate and initialize it. */ - if (!pol || !pol->cpd_size) + if (!pol || !pol->cpd_alloc_fn) continue; - BUG_ON(blkcg->pd[i]); - cpd = kzalloc(pol->cpd_size, GFP_KERNEL); + cpd = pol->cpd_alloc_fn(GFP_KERNEL); if (!cpd) { ret = ERR_PTR(-ENOMEM); goto free_pd_blkcg; } - blkcg->pd[i] = cpd; + blkcg->cpd[i] = cpd; + cpd->blkcg = blkcg; cpd->plid = i; - pol->cpd_init_fn(blkcg); + if (pol->cpd_init_fn) + pol->cpd_init_fn(cpd); } -done: spin_lock_init(&blkcg->lock); - INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_ATOMIC); + INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT); INIT_HLIST_HEAD(&blkcg->blkg_list); #ifdef CONFIG_CGROUP_WRITEBACK INIT_LIST_HEAD(&blkcg->cgwb_list); @@ -894,7 +1021,8 @@ blkcg_css_alloc(struct cgroup_subsys_state *parent_css) free_pd_blkcg: for (i--; i >= 0; i--) - kfree(blkcg->pd[i]); + if (blkcg->cpd[i]) + blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]); free_blkcg: kfree(blkcg); mutex_unlock(&blkcg_pol_mutex); @@ -938,7 +1066,7 @@ int blkcg_init_queue(struct request_queue *q) radix_tree_preload_end(); if (IS_ERR(blkg)) { - kfree(new_blkg); + blkg_free(new_blkg); return PTR_ERR(blkg); } @@ -1015,12 +1143,35 @@ static int blkcg_can_attach(struct cgroup_subsys_state *css, return ret; } -struct cgroup_subsys blkio_cgrp_subsys = { +static void blkcg_bind(struct cgroup_subsys_state *root_css) +{ + int i; + + mutex_lock(&blkcg_pol_mutex); + + for (i = 0; i < BLKCG_MAX_POLS; i++) { + struct blkcg_policy *pol = blkcg_policy[i]; + struct blkcg *blkcg; + + if (!pol || !pol->cpd_bind_fn) + continue; + + list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) + if (blkcg->cpd[pol->plid]) + pol->cpd_bind_fn(blkcg->cpd[pol->plid]); + } + mutex_unlock(&blkcg_pol_mutex); +} + +struct cgroup_subsys io_cgrp_subsys = { .css_alloc = blkcg_css_alloc, .css_offline = blkcg_css_offline, .css_free = blkcg_css_free, .can_attach = blkcg_can_attach, - .legacy_cftypes = blkcg_files, + .bind = blkcg_bind, + .dfl_cftypes = blkcg_files, + .legacy_cftypes = blkcg_legacy_files, + .legacy_name = "blkio", #ifdef CONFIG_MEMCG /* * This ensures that, if available, memcg is automatically enabled @@ -1030,7 +1181,7 @@ struct cgroup_subsys blkio_cgrp_subsys = { .depends_on = 1 << memory_cgrp_id, #endif }; -EXPORT_SYMBOL_GPL(blkio_cgrp_subsys); +EXPORT_SYMBOL_GPL(io_cgrp_subsys); /** * blkcg_activate_policy - activate a blkcg policy on a request_queue @@ -1051,65 +1202,54 @@ EXPORT_SYMBOL_GPL(blkio_cgrp_subsys); int blkcg_activate_policy(struct request_queue *q, const struct blkcg_policy *pol) { - LIST_HEAD(pds); + struct blkg_policy_data *pd_prealloc = NULL; struct blkcg_gq *blkg; - struct blkg_policy_data *pd, *nd; - int cnt = 0, ret; + int ret; if (blkcg_policy_enabled(q, pol)) return 0; - /* count and allocate policy_data for all existing blkgs */ blk_queue_bypass_start(q); - spin_lock_irq(q->queue_lock); - list_for_each_entry(blkg, &q->blkg_list, q_node) - cnt++; - spin_unlock_irq(q->queue_lock); - - /* allocate per-blkg policy data for all existing blkgs */ - while (cnt--) { - pd = kzalloc_node(pol->pd_size, GFP_KERNEL, q->node); - if (!pd) { +pd_prealloc: + if (!pd_prealloc) { + pd_prealloc = pol->pd_alloc_fn(GFP_KERNEL, q->node); + if (!pd_prealloc) { ret = -ENOMEM; - goto out_free; + goto out_bypass_end; } - list_add_tail(&pd->alloc_node, &pds); } - /* - * Install the allocated pds and cpds. With @q bypassing, no new blkg - * should have been created while the queue lock was dropped. - */ spin_lock_irq(q->queue_lock); list_for_each_entry(blkg, &q->blkg_list, q_node) { - if (WARN_ON(list_empty(&pds))) { - /* umm... this shouldn't happen, just abort */ - ret = -ENOMEM; - goto out_unlock; - } - pd = list_first_entry(&pds, struct blkg_policy_data, alloc_node); - list_del_init(&pd->alloc_node); + struct blkg_policy_data *pd; - /* grab blkcg lock too while installing @pd on @blkg */ - spin_lock(&blkg->blkcg->lock); + if (blkg->pd[pol->plid]) + continue; + + pd = pol->pd_alloc_fn(GFP_NOWAIT, q->node); + if (!pd) + swap(pd, pd_prealloc); + if (!pd) { + spin_unlock_irq(q->queue_lock); + goto pd_prealloc; + } blkg->pd[pol->plid] = pd; pd->blkg = blkg; pd->plid = pol->plid; - pol->pd_init_fn(blkg); - - spin_unlock(&blkg->blkcg->lock); + if (pol->pd_init_fn) + pol->pd_init_fn(pd); } __set_bit(pol->plid, q->blkcg_pols); ret = 0; -out_unlock: + spin_unlock_irq(q->queue_lock); -out_free: +out_bypass_end: blk_queue_bypass_end(q); - list_for_each_entry_safe(pd, nd, &pds, alloc_node) - kfree(pd); + if (pd_prealloc) + pol->pd_free_fn(pd_prealloc); return ret; } EXPORT_SYMBOL_GPL(blkcg_activate_policy); @@ -1139,13 +1279,12 @@ void blkcg_deactivate_policy(struct request_queue *q, /* grab blkcg lock too while removing @pd from @blkg */ spin_lock(&blkg->blkcg->lock); - if (pol->pd_offline_fn) - pol->pd_offline_fn(blkg); - if (pol->pd_exit_fn) - pol->pd_exit_fn(blkg); - - kfree(blkg->pd[pol->plid]); - blkg->pd[pol->plid] = NULL; + if (blkg->pd[pol->plid]) { + if (pol->pd_offline_fn) + pol->pd_offline_fn(blkg->pd[pol->plid]); + pol->pd_free_fn(blkg->pd[pol->plid]); + blkg->pd[pol->plid] = NULL; + } spin_unlock(&blkg->blkcg->lock); } @@ -1167,9 +1306,6 @@ int blkcg_policy_register(struct blkcg_policy *pol) struct blkcg *blkcg; int i, ret; - if (WARN_ON(pol->pd_size < sizeof(struct blkg_policy_data))) - return -EINVAL; - mutex_lock(&blkcg_pol_register_mutex); mutex_lock(&blkcg_pol_mutex); @@ -1186,36 +1322,42 @@ int blkcg_policy_register(struct blkcg_policy *pol) blkcg_policy[pol->plid] = pol; /* allocate and install cpd's */ - if (pol->cpd_size) { + if (pol->cpd_alloc_fn) { list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) { struct blkcg_policy_data *cpd; - cpd = kzalloc(pol->cpd_size, GFP_KERNEL); + cpd = pol->cpd_alloc_fn(GFP_KERNEL); if (!cpd) { mutex_unlock(&blkcg_pol_mutex); goto err_free_cpds; } - blkcg->pd[pol->plid] = cpd; + blkcg->cpd[pol->plid] = cpd; + cpd->blkcg = blkcg; cpd->plid = pol->plid; - pol->cpd_init_fn(blkcg); + pol->cpd_init_fn(cpd); } } mutex_unlock(&blkcg_pol_mutex); /* everything is in place, add intf files for the new policy */ - if (pol->cftypes) - WARN_ON(cgroup_add_legacy_cftypes(&blkio_cgrp_subsys, - pol->cftypes)); + if (pol->dfl_cftypes) + WARN_ON(cgroup_add_dfl_cftypes(&io_cgrp_subsys, + pol->dfl_cftypes)); + if (pol->legacy_cftypes) + WARN_ON(cgroup_add_legacy_cftypes(&io_cgrp_subsys, + pol->legacy_cftypes)); mutex_unlock(&blkcg_pol_register_mutex); return 0; err_free_cpds: - if (pol->cpd_size) { + if (pol->cpd_alloc_fn) { list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) { - kfree(blkcg->pd[pol->plid]); - blkcg->pd[pol->plid] = NULL; + if (blkcg->cpd[pol->plid]) { + pol->cpd_free_fn(blkcg->cpd[pol->plid]); + blkcg->cpd[pol->plid] = NULL; + } } } blkcg_policy[pol->plid] = NULL; @@ -1242,16 +1384,20 @@ void blkcg_policy_unregister(struct blkcg_policy *pol) goto out_unlock; /* kill the intf files first */ - if (pol->cftypes) - cgroup_rm_cftypes(pol->cftypes); + if (pol->dfl_cftypes) + cgroup_rm_cftypes(pol->dfl_cftypes); + if (pol->legacy_cftypes) + cgroup_rm_cftypes(pol->legacy_cftypes); /* remove cpds and unregister */ mutex_lock(&blkcg_pol_mutex); - if (pol->cpd_size) { + if (pol->cpd_alloc_fn) { list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) { - kfree(blkcg->pd[pol->plid]); - blkcg->pd[pol->plid] = NULL; + if (blkcg->cpd[pol->plid]) { + pol->cpd_free_fn(blkcg->cpd[pol->plid]); + blkcg->cpd[pol->plid] = NULL; + } } } blkcg_policy[pol->plid] = NULL; diff --git a/block/blk-core.c b/block/blk-core.c index 627ed0c593fb..2eb722d48773 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -143,18 +143,16 @@ static void req_bio_endio(struct request *rq, struct bio *bio, unsigned int nbytes, int error) { if (error) - clear_bit(BIO_UPTODATE, &bio->bi_flags); - else if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) - error = -EIO; + bio->bi_error = error; if (unlikely(rq->cmd_flags & REQ_QUIET)) - set_bit(BIO_QUIET, &bio->bi_flags); + bio_set_flag(bio, BIO_QUIET); bio_advance(bio, nbytes); /* don't actually finish bio if it's part of flush sequence */ if (bio->bi_iter.bi_size == 0 && !(rq->cmd_flags & REQ_FLUSH_SEQ)) - bio_endio(bio, error); + bio_endio(bio); } void blk_dump_rq_flags(struct request *rq, char *msg) @@ -645,6 +643,10 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) if (q->id < 0) goto fail_q; + q->bio_split = bioset_create(BIO_POOL_SIZE, 0); + if (!q->bio_split) + goto fail_id; + q->backing_dev_info.ra_pages = (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE; q->backing_dev_info.capabilities = BDI_CAP_CGROUP_WRITEBACK; @@ -653,7 +655,7 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) err = bdi_init(&q->backing_dev_info); if (err) - goto fail_id; + goto fail_split; setup_timer(&q->backing_dev_info.laptop_mode_wb_timer, laptop_mode_timer_fn, (unsigned long) q); @@ -695,6 +697,8 @@ struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) fail_bdi: bdi_destroy(&q->backing_dev_info); +fail_split: + bioset_free(q->bio_split); fail_id: ida_simple_remove(&blk_queue_ida, q->id); fail_q: @@ -1612,6 +1616,8 @@ static void blk_queue_bio(struct request_queue *q, struct bio *bio) struct request *req; unsigned int request_count = 0; + blk_queue_split(q, &bio, q->bio_split); + /* * low level driver can indicate that it wants pages above a * certain limit bounced to low memory (ie for highmem, or even @@ -1620,7 +1626,8 @@ static void blk_queue_bio(struct request_queue *q, struct bio *bio) blk_queue_bounce(q, &bio); if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { - bio_endio(bio, -EIO); + bio->bi_error = -EIO; + bio_endio(bio); return; } @@ -1673,7 +1680,8 @@ static void blk_queue_bio(struct request_queue *q, struct bio *bio) */ req = get_request(q, rw_flags, bio, GFP_NOIO); if (IS_ERR(req)) { - bio_endio(bio, PTR_ERR(req)); /* @q is dead */ + bio->bi_error = PTR_ERR(req); + bio_endio(bio); goto out_unlock; } @@ -1832,15 +1840,6 @@ generic_make_request_checks(struct bio *bio) goto end_io; } - if (likely(bio_is_rw(bio) && - nr_sectors > queue_max_hw_sectors(q))) { - printk(KERN_ERR "bio too big device %s (%u > %u)\n", - bdevname(bio->bi_bdev, b), - bio_sectors(bio), - queue_max_hw_sectors(q)); - goto end_io; - } - part = bio->bi_bdev->bd_part; if (should_fail_request(part, bio->bi_iter.bi_size) || should_fail_request(&part_to_disk(part)->part0, @@ -1889,14 +1888,15 @@ generic_make_request_checks(struct bio *bio) */ create_io_context(GFP_ATOMIC, q->node); - if (blk_throtl_bio(q, bio)) - return false; /* throttled, will be resubmitted later */ + if (!blkcg_bio_issue_check(q, bio)) + return false; trace_block_bio_queue(q, bio); return true; end_io: - bio_endio(bio, err); + bio->bi_error = err; + bio_endio(bio); return false; } diff --git a/block/blk-flush.c b/block/blk-flush.c index 20badd7b9d1b..9c423e53324a 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -73,6 +73,7 @@ #include "blk.h" #include "blk-mq.h" +#include "blk-mq-tag.h" /* FLUSH/FUA sequences */ enum { @@ -226,7 +227,12 @@ static void flush_end_io(struct request *flush_rq, int error) struct blk_flush_queue *fq = blk_get_flush_queue(q, flush_rq->mq_ctx); if (q->mq_ops) { + struct blk_mq_hw_ctx *hctx; + + /* release the tag's ownership to the req cloned from */ spin_lock_irqsave(&fq->mq_flush_lock, flags); + hctx = q->mq_ops->map_queue(q, flush_rq->mq_ctx->cpu); + blk_mq_tag_set_rq(hctx, flush_rq->tag, fq->orig_rq); flush_rq->tag = -1; } @@ -308,11 +314,18 @@ static bool blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq) /* * Borrow tag from the first request since they can't - * be in flight at the same time. + * be in flight at the same time. And acquire the tag's + * ownership for flush req. */ if (q->mq_ops) { + struct blk_mq_hw_ctx *hctx; + flush_rq->mq_ctx = first_rq->mq_ctx; flush_rq->tag = first_rq->tag; + fq->orig_rq = first_rq; + + hctx = q->mq_ops->map_queue(q, first_rq->mq_ctx->cpu); + blk_mq_tag_set_rq(hctx, first_rq->tag, flush_rq); } flush_rq->cmd_type = REQ_TYPE_FS; diff --git a/block/blk-lib.c b/block/blk-lib.c index 7688ee3f5d72..bd40292e5009 100644 --- a/block/blk-lib.c +++ b/block/blk-lib.c @@ -11,21 +11,28 @@ struct bio_batch { atomic_t done; - unsigned long flags; + int error; struct completion *wait; }; -static void bio_batch_end_io(struct bio *bio, int err) +static void bio_batch_end_io(struct bio *bio) { struct bio_batch *bb = bio->bi_private; - if (err && (err != -EOPNOTSUPP)) - clear_bit(BIO_UPTODATE, &bb->flags); + if (bio->bi_error && bio->bi_error != -EOPNOTSUPP) + bb->error = bio->bi_error; if (atomic_dec_and_test(&bb->done)) complete(bb->wait); bio_put(bio); } +/* + * Ensure that max discard sectors doesn't overflow bi_size and hopefully + * it is of the proper granularity as long as the granularity is a power + * of two. + */ +#define MAX_BIO_SECTORS ((1U << 31) >> 9) + /** * blkdev_issue_discard - queue a discard * @bdev: blockdev to issue discard for @@ -43,8 +50,6 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, DECLARE_COMPLETION_ONSTACK(wait); struct request_queue *q = bdev_get_queue(bdev); int type = REQ_WRITE | REQ_DISCARD; - unsigned int max_discard_sectors, granularity; - int alignment; struct bio_batch bb; struct bio *bio; int ret = 0; @@ -56,21 +61,6 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, if (!blk_queue_discard(q)) return -EOPNOTSUPP; - /* Zero-sector (unknown) and one-sector granularities are the same. */ - granularity = max(q->limits.discard_granularity >> 9, 1U); - alignment = (bdev_discard_alignment(bdev) >> 9) % granularity; - - /* - * Ensure that max_discard_sectors is of the proper - * granularity, so that requests stay aligned after a split. - */ - max_discard_sectors = min(q->limits.max_discard_sectors, UINT_MAX >> 9); - max_discard_sectors -= max_discard_sectors % granularity; - if (unlikely(!max_discard_sectors)) { - /* Avoid infinite loop below. Being cautious never hurts. */ - return -EOPNOTSUPP; - } - if (flags & BLKDEV_DISCARD_SECURE) { if (!blk_queue_secdiscard(q)) return -EOPNOTSUPP; @@ -78,13 +68,13 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, } atomic_set(&bb.done, 1); - bb.flags = 1 << BIO_UPTODATE; + bb.error = 0; bb.wait = &wait; blk_start_plug(&plug); while (nr_sects) { unsigned int req_sects; - sector_t end_sect, tmp; + sector_t end_sect; bio = bio_alloc(gfp_mask, 1); if (!bio) { @@ -92,21 +82,8 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, break; } - req_sects = min_t(sector_t, nr_sects, max_discard_sectors); - - /* - * If splitting a request, and the next starting sector would be - * misaligned, stop the discard at the previous aligned sector. - */ + req_sects = min_t(sector_t, nr_sects, MAX_BIO_SECTORS); end_sect = sector + req_sects; - tmp = end_sect; - if (req_sects < nr_sects && - sector_div(tmp, granularity) != alignment) { - end_sect = end_sect - alignment; - sector_div(end_sect, granularity); - end_sect = end_sect * granularity + alignment; - req_sects = end_sect - sector; - } bio->bi_iter.bi_sector = sector; bio->bi_end_io = bio_batch_end_io; @@ -134,9 +111,8 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, if (!atomic_dec_and_test(&bb.done)) wait_for_completion_io(&wait); - if (!test_bit(BIO_UPTODATE, &bb.flags)) - ret = -EIO; - + if (bb.error) + return bb.error; return ret; } EXPORT_SYMBOL(blkdev_issue_discard); @@ -166,13 +142,11 @@ int blkdev_issue_write_same(struct block_device *bdev, sector_t sector, if (!q) return -ENXIO; - max_write_same_sectors = q->limits.max_write_same_sectors; - - if (max_write_same_sectors == 0) - return -EOPNOTSUPP; + /* Ensure that max_write_same_sectors doesn't overflow bi_size */ + max_write_same_sectors = UINT_MAX >> 9; atomic_set(&bb.done, 1); - bb.flags = 1 << BIO_UPTODATE; + bb.error = 0; bb.wait = &wait; while (nr_sects) { @@ -208,9 +182,8 @@ int blkdev_issue_write_same(struct block_device *bdev, sector_t sector, if (!atomic_dec_and_test(&bb.done)) wait_for_completion_io(&wait); - if (!test_bit(BIO_UPTODATE, &bb.flags)) - ret = -ENOTSUPP; - + if (bb.error) + return bb.error; return ret; } EXPORT_SYMBOL(blkdev_issue_write_same); @@ -236,7 +209,7 @@ static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, DECLARE_COMPLETION_ONSTACK(wait); atomic_set(&bb.done, 1); - bb.flags = 1 << BIO_UPTODATE; + bb.error = 0; bb.wait = &wait; ret = 0; @@ -270,10 +243,8 @@ static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, if (!atomic_dec_and_test(&bb.done)) wait_for_completion_io(&wait); - if (!test_bit(BIO_UPTODATE, &bb.flags)) - /* One of bios in the batch was completed with error.*/ - ret = -EIO; - + if (bb.error) + return bb.error; return ret; } diff --git a/block/blk-map.c b/block/blk-map.c index da310a105429..233841644c9d 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -94,7 +94,7 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, return PTR_ERR(bio); if (map_data && map_data->null_mapped) - bio->bi_flags |= (1 << BIO_NULL_MAPPED); + bio_set_flag(bio, BIO_NULL_MAPPED); if (bio->bi_iter.bi_size != iter->count) { /* @@ -103,7 +103,7 @@ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, * normal IO completion path */ bio_get(bio); - bio_endio(bio, 0); + bio_endio(bio); __blk_rq_unmap_user(bio); return -EINVAL; } diff --git a/block/blk-merge.c b/block/blk-merge.c index 30a0d9f89017..d088cffb8105 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -9,12 +9,146 @@ #include "blk.h" +static struct bio *blk_bio_discard_split(struct request_queue *q, + struct bio *bio, + struct bio_set *bs) +{ + unsigned int max_discard_sectors, granularity; + int alignment; + sector_t tmp; + unsigned split_sectors; + + /* Zero-sector (unknown) and one-sector granularities are the same. */ + granularity = max(q->limits.discard_granularity >> 9, 1U); + + max_discard_sectors = min(q->limits.max_discard_sectors, UINT_MAX >> 9); + max_discard_sectors -= max_discard_sectors % granularity; + + if (unlikely(!max_discard_sectors)) { + /* XXX: warn */ + return NULL; + } + + if (bio_sectors(bio) <= max_discard_sectors) + return NULL; + + split_sectors = max_discard_sectors; + + /* + * If the next starting sector would be misaligned, stop the discard at + * the previous aligned sector. + */ + alignment = (q->limits.discard_alignment >> 9) % granularity; + + tmp = bio->bi_iter.bi_sector + split_sectors - alignment; + tmp = sector_div(tmp, granularity); + + if (split_sectors > tmp) + split_sectors -= tmp; + + return bio_split(bio, split_sectors, GFP_NOIO, bs); +} + +static struct bio *blk_bio_write_same_split(struct request_queue *q, + struct bio *bio, + struct bio_set *bs) +{ + if (!q->limits.max_write_same_sectors) + return NULL; + + if (bio_sectors(bio) <= q->limits.max_write_same_sectors) + return NULL; + + return bio_split(bio, q->limits.max_write_same_sectors, GFP_NOIO, bs); +} + +static struct bio *blk_bio_segment_split(struct request_queue *q, + struct bio *bio, + struct bio_set *bs) +{ + struct bio *split; + struct bio_vec bv, bvprv; + struct bvec_iter iter; + unsigned seg_size = 0, nsegs = 0, sectors = 0; + int prev = 0; + + bio_for_each_segment(bv, bio, iter) { + sectors += bv.bv_len >> 9; + + if (sectors > queue_max_sectors(q)) + goto split; + + /* + * If the queue doesn't support SG gaps and adding this + * offset would create a gap, disallow it. + */ + if (prev && bvec_gap_to_prev(q, &bvprv, bv.bv_offset)) + goto split; + + if (prev && blk_queue_cluster(q)) { + if (seg_size + bv.bv_len > queue_max_segment_size(q)) + goto new_segment; + if (!BIOVEC_PHYS_MERGEABLE(&bvprv, &bv)) + goto new_segment; + if (!BIOVEC_SEG_BOUNDARY(q, &bvprv, &bv)) + goto new_segment; + + seg_size += bv.bv_len; + bvprv = bv; + prev = 1; + continue; + } +new_segment: + if (nsegs == queue_max_segments(q)) + goto split; + + nsegs++; + bvprv = bv; + prev = 1; + seg_size = bv.bv_len; + } + + return NULL; +split: + split = bio_clone_bioset(bio, GFP_NOIO, bs); + + split->bi_iter.bi_size -= iter.bi_size; + bio->bi_iter = iter; + + if (bio_integrity(bio)) { + bio_integrity_advance(bio, split->bi_iter.bi_size); + bio_integrity_trim(split, 0, bio_sectors(split)); + } + + return split; +} + +void blk_queue_split(struct request_queue *q, struct bio **bio, + struct bio_set *bs) +{ + struct bio *split; + + if ((*bio)->bi_rw & REQ_DISCARD) + split = blk_bio_discard_split(q, *bio, bs); + else if ((*bio)->bi_rw & REQ_WRITE_SAME) + split = blk_bio_write_same_split(q, *bio, bs); + else + split = blk_bio_segment_split(q, *bio, q->bio_split); + + if (split) { + bio_chain(split, *bio); + generic_make_request(*bio); + *bio = split; + } +} +EXPORT_SYMBOL(blk_queue_split); + static unsigned int __blk_recalc_rq_segments(struct request_queue *q, struct bio *bio, bool no_sg_merge) { struct bio_vec bv, bvprv = { NULL }; - int cluster, high, highprv = 1; + int cluster, prev = 0; unsigned int seg_size, nr_phys_segs; struct bio *fbio, *bbio; struct bvec_iter iter; @@ -36,7 +170,6 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q, cluster = blk_queue_cluster(q); seg_size = 0; nr_phys_segs = 0; - high = 0; for_each_bio(bio) { bio_for_each_segment(bv, bio, iter) { /* @@ -46,13 +179,7 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q, if (no_sg_merge) goto new_segment; - /* - * the trick here is making sure that a high page is - * never considered part of another segment, since - * that might change with the bounce page. - */ - high = page_to_pfn(bv.bv_page) > queue_bounce_pfn(q); - if (!high && !highprv && cluster) { + if (prev && cluster) { if (seg_size + bv.bv_len > queue_max_segment_size(q)) goto new_segment; @@ -72,8 +199,8 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q, nr_phys_segs++; bvprv = bv; + prev = 1; seg_size = bv.bv_len; - highprv = high; } bbio = bio; } @@ -116,7 +243,7 @@ void blk_recount_segments(struct request_queue *q, struct bio *bio) bio->bi_next = nxt; } - bio->bi_flags |= (1 << BIO_SEG_VALID); + bio_set_flag(bio, BIO_SEG_VALID); } EXPORT_SYMBOL(blk_recount_segments); @@ -266,7 +393,7 @@ int blk_rq_map_sg(struct request_queue *q, struct request *rq, if (rq->cmd_flags & REQ_WRITE) memset(q->dma_drain_buffer, 0, q->dma_drain_size); - sg->page_link &= ~0x02; + sg_unmark_end(sg); sg = sg_next(sg); sg_set_page(sg, virt_to_page(q->dma_drain_buffer), q->dma_drain_size, @@ -356,12 +483,12 @@ static bool req_no_special_merge(struct request *req) return !q->mq_ops && req->special; } -static int req_gap_to_prev(struct request *req, struct request *next) +static int req_gap_to_prev(struct request *req, struct bio *next) { struct bio *prev = req->biotail; - return bvec_gap_to_prev(&prev->bi_io_vec[prev->bi_vcnt - 1], - next->bio->bi_io_vec[0].bv_offset); + return bvec_gap_to_prev(req->q, &prev->bi_io_vec[prev->bi_vcnt - 1], + next->bi_io_vec[0].bv_offset); } static int ll_merge_requests_fn(struct request_queue *q, struct request *req, @@ -378,8 +505,7 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req, if (req_no_special_merge(req) || req_no_special_merge(next)) return 0; - if (test_bit(QUEUE_FLAG_SG_GAPS, &q->queue_flags) && - req_gap_to_prev(req, next)) + if (req_gap_to_prev(req, next->bio)) return 0; /* @@ -564,8 +690,6 @@ int blk_attempt_req_merge(struct request_queue *q, struct request *rq, bool blk_rq_merge_ok(struct request *rq, struct bio *bio) { - struct request_queue *q = rq->q; - if (!rq_mergeable(rq) || !bio_mergeable(bio)) return false; @@ -590,13 +714,8 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio) return false; /* Only check gaps if the bio carries data */ - if (q->queue_flags & (1 << QUEUE_FLAG_SG_GAPS) && bio_has_data(bio)) { - struct bio_vec *bprev; - - bprev = &rq->biotail->bi_io_vec[rq->biotail->bi_vcnt - 1]; - if (bvec_gap_to_prev(bprev, bio->bi_io_vec[0].bv_offset)) - return false; - } + if (bio_has_data(bio) && req_gap_to_prev(rq, bio)) + return false; return true; } diff --git a/block/blk-mq-sysfs.c b/block/blk-mq-sysfs.c index b79685e06b70..279c5d674edf 100644 --- a/block/blk-mq-sysfs.c +++ b/block/blk-mq-sysfs.c @@ -141,15 +141,26 @@ static ssize_t blk_mq_sysfs_completed_show(struct blk_mq_ctx *ctx, char *page) static ssize_t sysfs_list_show(char *page, struct list_head *list, char *msg) { - char *start_page = page; struct request *rq; + int len = snprintf(page, PAGE_SIZE - 1, "%s:\n", msg); - page += sprintf(page, "%s:\n", msg); + list_for_each_entry(rq, list, queuelist) { + const int rq_len = 2 * sizeof(rq) + 2; - list_for_each_entry(rq, list, queuelist) - page += sprintf(page, "\t%p\n", rq); + /* if the output will be truncated */ + if (PAGE_SIZE - 1 < len + rq_len) { + /* backspacing if it can't hold '\t...\n' */ + if (PAGE_SIZE - 1 < len + 5) + len -= rq_len; + len += snprintf(page + len, PAGE_SIZE - 1 - len, + "\t...\n"); + break; + } + len += snprintf(page + len, PAGE_SIZE - 1 - len, + "\t%p\n", rq); + } - return page - start_page; + return len; } static ssize_t blk_mq_sysfs_rq_list_show(struct blk_mq_ctx *ctx, char *page) diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index 9b6e28830b82..9115c6d59948 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -429,7 +429,7 @@ static void bt_for_each(struct blk_mq_hw_ctx *hctx, for (bit = find_first_bit(&bm->word, bm->depth); bit < bm->depth; bit = find_next_bit(&bm->word, bm->depth, bit + 1)) { - rq = blk_mq_tag_to_rq(hctx->tags, off + bit); + rq = hctx->tags->rqs[off + bit]; if (rq->q == hctx->queue) fn(hctx, rq, data, reserved); } @@ -453,7 +453,7 @@ static void bt_tags_for_each(struct blk_mq_tags *tags, for (bit = find_first_bit(&bm->word, bm->depth); bit < bm->depth; bit = find_next_bit(&bm->word, bm->depth, bit + 1)) { - rq = blk_mq_tag_to_rq(tags, off + bit); + rq = tags->rqs[off + bit]; fn(rq, data, reserved); } diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h index 75893a34237d..9eb2cf4f01cb 100644 --- a/block/blk-mq-tag.h +++ b/block/blk-mq-tag.h @@ -89,4 +89,16 @@ static inline void blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx) __blk_mq_tag_idle(hctx); } +/* + * This helper should only be used for flush request to share tag + * with the request cloned from, and both the two requests can't be + * in flight at the same time. The caller has to make sure the tag + * can't be freed. + */ +static inline void blk_mq_tag_set_rq(struct blk_mq_hw_ctx *hctx, + unsigned int tag, struct request *rq) +{ + hctx->tags->rqs[tag] = rq; +} + #endif diff --git a/block/blk-mq.c b/block/blk-mq.c index 7d842db59699..f2d67b4047a0 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -559,23 +559,9 @@ void blk_mq_abort_requeue_list(struct request_queue *q) } EXPORT_SYMBOL(blk_mq_abort_requeue_list); -static inline bool is_flush_request(struct request *rq, - struct blk_flush_queue *fq, unsigned int tag) -{ - return ((rq->cmd_flags & REQ_FLUSH_SEQ) && - fq->flush_rq->tag == tag); -} - struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag) { - struct request *rq = tags->rqs[tag]; - /* mq_ctx of flush rq is always cloned from the corresponding req */ - struct blk_flush_queue *fq = blk_get_flush_queue(rq->q, rq->mq_ctx); - - if (!is_flush_request(rq, fq, tag)) - return rq; - - return fq->flush_rq; + return tags->rqs[tag]; } EXPORT_SYMBOL(blk_mq_tag_to_rq); @@ -1199,7 +1185,7 @@ static struct request *blk_mq_map_request(struct request_queue *q, struct blk_mq_alloc_data alloc_data; if (unlikely(blk_mq_queue_enter(q, GFP_KERNEL))) { - bio_endio(bio, -EIO); + bio_io_error(bio); return NULL; } @@ -1283,10 +1269,12 @@ static void blk_mq_make_request(struct request_queue *q, struct bio *bio) blk_queue_bounce(q, &bio); if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { - bio_endio(bio, -EIO); + bio_io_error(bio); return; } + blk_queue_split(q, &bio, q->bio_split); + if (!is_flush_fua && !blk_queue_nomerges(q) && blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq)) return; @@ -1368,10 +1356,12 @@ static void blk_sq_make_request(struct request_queue *q, struct bio *bio) blk_queue_bounce(q, &bio); if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) { - bio_endio(bio, -EIO); + bio_io_error(bio); return; } + blk_queue_split(q, &bio, q->bio_split); + if (!is_flush_fua && !blk_queue_nomerges(q) && blk_attempt_plug_merge(q, bio, &request_count, NULL)) return; diff --git a/block/blk-settings.c b/block/blk-settings.c index e0057d035200..7d8f129a1516 100644 --- a/block/blk-settings.c +++ b/block/blk-settings.c @@ -53,28 +53,6 @@ void blk_queue_unprep_rq(struct request_queue *q, unprep_rq_fn *ufn) } EXPORT_SYMBOL(blk_queue_unprep_rq); -/** - * blk_queue_merge_bvec - set a merge_bvec function for queue - * @q: queue - * @mbfn: merge_bvec_fn - * - * Usually queues have static limitations on the max sectors or segments that - * we can put in a request. Stacking drivers may have some settings that - * are dynamic, and thus we have to query the queue whether it is ok to - * add a new bio_vec to a bio at a given offset or not. If the block device - * has such limitations, it needs to register a merge_bvec_fn to control - * the size of bio's sent to it. Note that a block device *must* allow a - * single page to be added to an empty bio. The block device driver may want - * to use the bio_split() function to deal with these bio's. By default - * no merge_bvec_fn is defined for a queue, and only the fixed limits are - * honored. - */ -void blk_queue_merge_bvec(struct request_queue *q, merge_bvec_fn *mbfn) -{ - q->merge_bvec_fn = mbfn; -} -EXPORT_SYMBOL(blk_queue_merge_bvec); - void blk_queue_softirq_done(struct request_queue *q, softirq_done_fn *fn) { q->softirq_done_fn = fn; @@ -111,11 +89,13 @@ void blk_set_default_limits(struct queue_limits *lim) lim->max_segments = BLK_MAX_SEGMENTS; lim->max_integrity_segments = 0; lim->seg_boundary_mask = BLK_SEG_BOUNDARY_MASK; + lim->virt_boundary_mask = 0; lim->max_segment_size = BLK_MAX_SEGMENT_SIZE; lim->max_sectors = lim->max_hw_sectors = BLK_SAFE_MAX_SECTORS; lim->chunk_sectors = 0; lim->max_write_same_sectors = 0; lim->max_discard_sectors = 0; + lim->max_hw_discard_sectors = 0; lim->discard_granularity = 0; lim->discard_alignment = 0; lim->discard_misaligned = 0; @@ -257,7 +237,9 @@ void blk_limits_max_hw_sectors(struct queue_limits *limits, unsigned int max_hw_ __func__, max_hw_sectors); } - limits->max_sectors = limits->max_hw_sectors = max_hw_sectors; + limits->max_hw_sectors = max_hw_sectors; + limits->max_sectors = min_t(unsigned int, max_hw_sectors, + BLK_DEF_MAX_SECTORS); } EXPORT_SYMBOL(blk_limits_max_hw_sectors); @@ -303,6 +285,7 @@ EXPORT_SYMBOL(blk_queue_chunk_sectors); void blk_queue_max_discard_sectors(struct request_queue *q, unsigned int max_discard_sectors) { + q->limits.max_hw_discard_sectors = max_discard_sectors; q->limits.max_discard_sectors = max_discard_sectors; } EXPORT_SYMBOL(blk_queue_max_discard_sectors); @@ -550,6 +533,8 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, t->seg_boundary_mask = min_not_zero(t->seg_boundary_mask, b->seg_boundary_mask); + t->virt_boundary_mask = min_not_zero(t->virt_boundary_mask, + b->virt_boundary_mask); t->max_segments = min_not_zero(t->max_segments, b->max_segments); t->max_integrity_segments = min_not_zero(t->max_integrity_segments, @@ -641,6 +626,8 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, t->max_discard_sectors = min_not_zero(t->max_discard_sectors, b->max_discard_sectors); + t->max_hw_discard_sectors = min_not_zero(t->max_hw_discard_sectors, + b->max_hw_discard_sectors); t->discard_granularity = max(t->discard_granularity, b->discard_granularity); t->discard_alignment = lcm_not_zero(t->discard_alignment, alignment) % @@ -787,6 +774,17 @@ void blk_queue_segment_boundary(struct request_queue *q, unsigned long mask) } EXPORT_SYMBOL(blk_queue_segment_boundary); +/** + * blk_queue_virt_boundary - set boundary rules for bio merging + * @q: the request queue for the device + * @mask: the memory boundary mask + **/ +void blk_queue_virt_boundary(struct request_queue *q, unsigned long mask) +{ + q->limits.virt_boundary_mask = mask; +} +EXPORT_SYMBOL(blk_queue_virt_boundary); + /** * blk_queue_dma_alignment - set dma length and memory alignment * @q: the request queue for the device diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index 6264b382d4d1..3e44a9da2a13 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -145,12 +145,43 @@ static ssize_t queue_discard_granularity_show(struct request_queue *q, char *pag return queue_var_show(q->limits.discard_granularity, page); } +static ssize_t queue_discard_max_hw_show(struct request_queue *q, char *page) +{ + unsigned long long val; + + val = q->limits.max_hw_discard_sectors << 9; + return sprintf(page, "%llu\n", val); +} + static ssize_t queue_discard_max_show(struct request_queue *q, char *page) { return sprintf(page, "%llu\n", (unsigned long long)q->limits.max_discard_sectors << 9); } +static ssize_t queue_discard_max_store(struct request_queue *q, + const char *page, size_t count) +{ + unsigned long max_discard; + ssize_t ret = queue_var_store(&max_discard, page, count); + + if (ret < 0) + return ret; + + if (max_discard & (q->limits.discard_granularity - 1)) + return -EINVAL; + + max_discard >>= 9; + if (max_discard > UINT_MAX) + return -EINVAL; + + if (max_discard > q->limits.max_hw_discard_sectors) + max_discard = q->limits.max_hw_discard_sectors; + + q->limits.max_discard_sectors = max_discard; + return ret; +} + static ssize_t queue_discard_zeroes_data_show(struct request_queue *q, char *page) { return queue_var_show(queue_discard_zeroes_data(q), page); @@ -360,9 +391,15 @@ static struct queue_sysfs_entry queue_discard_granularity_entry = { .show = queue_discard_granularity_show, }; +static struct queue_sysfs_entry queue_discard_max_hw_entry = { + .attr = {.name = "discard_max_hw_bytes", .mode = S_IRUGO }, + .show = queue_discard_max_hw_show, +}; + static struct queue_sysfs_entry queue_discard_max_entry = { - .attr = {.name = "discard_max_bytes", .mode = S_IRUGO }, + .attr = {.name = "discard_max_bytes", .mode = S_IRUGO | S_IWUSR }, .show = queue_discard_max_show, + .store = queue_discard_max_store, }; static struct queue_sysfs_entry queue_discard_zeroes_data_entry = { @@ -421,6 +458,7 @@ static struct attribute *default_attrs[] = { &queue_io_opt_entry.attr, &queue_discard_granularity_entry.attr, &queue_discard_max_entry.attr, + &queue_discard_max_hw_entry.attr, &queue_discard_zeroes_data_entry.attr, &queue_write_same_max_entry.attr, &queue_nonrot_entry.attr, @@ -523,6 +561,9 @@ static void blk_release_queue(struct kobject *kobj) blk_trace_shutdown(q); + if (q->bio_split) + bioset_free(q->bio_split); + ida_simple_remove(&blk_queue_ida, q->id); call_rcu(&q->rcu_head, blk_free_queue_rcu); } diff --git a/block/blk-throttle.c b/block/blk-throttle.c index b23193518ac7..c75a2636dd40 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -83,14 +83,6 @@ enum tg_state_flags { #define rb_entry_tg(node) rb_entry((node), struct throtl_grp, rb_node) -/* Per-cpu group stats */ -struct tg_stats_cpu { - /* total bytes transferred */ - struct blkg_rwstat service_bytes; - /* total IOs serviced, post merge */ - struct blkg_rwstat serviced; -}; - struct throtl_grp { /* must be the first member */ struct blkg_policy_data pd; @@ -141,12 +133,6 @@ struct throtl_grp { /* When did we start a new slice */ unsigned long slice_start[2]; unsigned long slice_end[2]; - - /* Per cpu stats pointer */ - struct tg_stats_cpu __percpu *stats_cpu; - - /* List of tgs waiting for per cpu stats memory to be allocated */ - struct list_head stats_alloc_node; }; struct throtl_data @@ -168,13 +154,6 @@ struct throtl_data struct work_struct dispatch_work; }; -/* list and work item to allocate percpu group stats */ -static DEFINE_SPINLOCK(tg_stats_alloc_lock); -static LIST_HEAD(tg_stats_alloc_list); - -static void tg_stats_alloc_fn(struct work_struct *); -static DECLARE_DELAYED_WORK(tg_stats_alloc_work, tg_stats_alloc_fn); - static void throtl_pending_timer_fn(unsigned long arg); static inline struct throtl_grp *pd_to_tg(struct blkg_policy_data *pd) @@ -192,11 +171,6 @@ static inline struct blkcg_gq *tg_to_blkg(struct throtl_grp *tg) return pd_to_blkg(&tg->pd); } -static inline struct throtl_grp *td_root_tg(struct throtl_data *td) -{ - return blkg_to_tg(td->queue->root_blkg); -} - /** * sq_to_tg - return the throl_grp the specified service queue belongs to * @sq: the throtl_service_queue of interest @@ -256,53 +230,6 @@ static struct throtl_data *sq_to_td(struct throtl_service_queue *sq) } \ } while (0) -static void tg_stats_init(struct tg_stats_cpu *tg_stats) -{ - blkg_rwstat_init(&tg_stats->service_bytes); - blkg_rwstat_init(&tg_stats->serviced); -} - -/* - * Worker for allocating per cpu stat for tgs. This is scheduled on the - * system_wq once there are some groups on the alloc_list waiting for - * allocation. - */ -static void tg_stats_alloc_fn(struct work_struct *work) -{ - static struct tg_stats_cpu *stats_cpu; /* this fn is non-reentrant */ - struct delayed_work *dwork = to_delayed_work(work); - bool empty = false; - -alloc_stats: - if (!stats_cpu) { - int cpu; - - stats_cpu = alloc_percpu(struct tg_stats_cpu); - if (!stats_cpu) { - /* allocation failed, try again after some time */ - schedule_delayed_work(dwork, msecs_to_jiffies(10)); - return; - } - for_each_possible_cpu(cpu) - tg_stats_init(per_cpu_ptr(stats_cpu, cpu)); - } - - spin_lock_irq(&tg_stats_alloc_lock); - - if (!list_empty(&tg_stats_alloc_list)) { - struct throtl_grp *tg = list_first_entry(&tg_stats_alloc_list, - struct throtl_grp, - stats_alloc_node); - swap(tg->stats_cpu, stats_cpu); - list_del_init(&tg->stats_alloc_node); - } - - empty = list_empty(&tg_stats_alloc_list); - spin_unlock_irq(&tg_stats_alloc_lock); - if (!empty) - goto alloc_stats; -} - static void throtl_qnode_init(struct throtl_qnode *qn, struct throtl_grp *tg) { INIT_LIST_HEAD(&qn->node); @@ -387,29 +314,46 @@ static struct bio *throtl_pop_queued(struct list_head *queued, } /* init a service_queue, assumes the caller zeroed it */ -static void throtl_service_queue_init(struct throtl_service_queue *sq, - struct throtl_service_queue *parent_sq) +static void throtl_service_queue_init(struct throtl_service_queue *sq) { INIT_LIST_HEAD(&sq->queued[0]); INIT_LIST_HEAD(&sq->queued[1]); sq->pending_tree = RB_ROOT; - sq->parent_sq = parent_sq; setup_timer(&sq->pending_timer, throtl_pending_timer_fn, (unsigned long)sq); } -static void throtl_service_queue_exit(struct throtl_service_queue *sq) +static struct blkg_policy_data *throtl_pd_alloc(gfp_t gfp, int node) { - del_timer_sync(&sq->pending_timer); + struct throtl_grp *tg; + int rw; + + tg = kzalloc_node(sizeof(*tg), gfp, node); + if (!tg) + return NULL; + + throtl_service_queue_init(&tg->service_queue); + + for (rw = READ; rw <= WRITE; rw++) { + throtl_qnode_init(&tg->qnode_on_self[rw], tg); + throtl_qnode_init(&tg->qnode_on_parent[rw], tg); + } + + RB_CLEAR_NODE(&tg->rb_node); + tg->bps[READ] = -1; + tg->bps[WRITE] = -1; + tg->iops[READ] = -1; + tg->iops[WRITE] = -1; + + return &tg->pd; } -static void throtl_pd_init(struct blkcg_gq *blkg) +static void throtl_pd_init(struct blkg_policy_data *pd) { - struct throtl_grp *tg = blkg_to_tg(blkg); + struct throtl_grp *tg = pd_to_tg(pd); + struct blkcg_gq *blkg = tg_to_blkg(tg); struct throtl_data *td = blkg->q->td; - struct throtl_service_queue *parent_sq; - unsigned long flags; - int rw; + struct throtl_service_queue *sq = &tg->service_queue; /* * If on the default hierarchy, we switch to properly hierarchical @@ -424,35 +368,10 @@ static void throtl_pd_init(struct blkcg_gq *blkg) * Limits of a group don't interact with limits of other groups * regardless of the position of the group in the hierarchy. */ - parent_sq = &td->service_queue; - + sq->parent_sq = &td->service_queue; if (cgroup_on_dfl(blkg->blkcg->css.cgroup) && blkg->parent) - parent_sq = &blkg_to_tg(blkg->parent)->service_queue; - - throtl_service_queue_init(&tg->service_queue, parent_sq); - - for (rw = READ; rw <= WRITE; rw++) { - throtl_qnode_init(&tg->qnode_on_self[rw], tg); - throtl_qnode_init(&tg->qnode_on_parent[rw], tg); - } - - RB_CLEAR_NODE(&tg->rb_node); + sq->parent_sq = &blkg_to_tg(blkg->parent)->service_queue; tg->td = td; - - tg->bps[READ] = -1; - tg->bps[WRITE] = -1; - tg->iops[READ] = -1; - tg->iops[WRITE] = -1; - - /* - * Ugh... We need to perform per-cpu allocation for tg->stats_cpu - * but percpu allocator can't be called from IO path. Queue tg on - * tg_stats_alloc_list and allocate from work item. - */ - spin_lock_irqsave(&tg_stats_alloc_lock, flags); - list_add(&tg->stats_alloc_node, &tg_stats_alloc_list); - schedule_delayed_work(&tg_stats_alloc_work, 0); - spin_unlock_irqrestore(&tg_stats_alloc_lock, flags); } /* @@ -470,83 +389,21 @@ static void tg_update_has_rules(struct throtl_grp *tg) (tg->bps[rw] != -1 || tg->iops[rw] != -1); } -static void throtl_pd_online(struct blkcg_gq *blkg) +static void throtl_pd_online(struct blkg_policy_data *pd) { /* * We don't want new groups to escape the limits of its ancestors. * Update has_rules[] after a new group is brought online. */ - tg_update_has_rules(blkg_to_tg(blkg)); + tg_update_has_rules(pd_to_tg(pd)); } -static void throtl_pd_exit(struct blkcg_gq *blkg) +static void throtl_pd_free(struct blkg_policy_data *pd) { - struct throtl_grp *tg = blkg_to_tg(blkg); - unsigned long flags; + struct throtl_grp *tg = pd_to_tg(pd); - spin_lock_irqsave(&tg_stats_alloc_lock, flags); - list_del_init(&tg->stats_alloc_node); - spin_unlock_irqrestore(&tg_stats_alloc_lock, flags); - - free_percpu(tg->stats_cpu); - - throtl_service_queue_exit(&tg->service_queue); -} - -static void throtl_pd_reset_stats(struct blkcg_gq *blkg) -{ - struct throtl_grp *tg = blkg_to_tg(blkg); - int cpu; - - if (tg->stats_cpu == NULL) - return; - - for_each_possible_cpu(cpu) { - struct tg_stats_cpu *sc = per_cpu_ptr(tg->stats_cpu, cpu); - - blkg_rwstat_reset(&sc->service_bytes); - blkg_rwstat_reset(&sc->serviced); - } -} - -static struct throtl_grp *throtl_lookup_tg(struct throtl_data *td, - struct blkcg *blkcg) -{ - /* - * This is the common case when there are no blkcgs. Avoid lookup - * in this case - */ - if (blkcg == &blkcg_root) - return td_root_tg(td); - - return blkg_to_tg(blkg_lookup(blkcg, td->queue)); -} - -static struct throtl_grp *throtl_lookup_create_tg(struct throtl_data *td, - struct blkcg *blkcg) -{ - struct request_queue *q = td->queue; - struct throtl_grp *tg = NULL; - - /* - * This is the common case when there are no blkcgs. Avoid lookup - * in this case - */ - if (blkcg == &blkcg_root) { - tg = td_root_tg(td); - } else { - struct blkcg_gq *blkg; - - blkg = blkg_lookup_create(blkcg, q); - - /* if %NULL and @q is alive, fall back to root_tg */ - if (!IS_ERR(blkg)) - tg = blkg_to_tg(blkg); - else if (!blk_queue_dying(q)) - tg = td_root_tg(td); - } - - return tg; + del_timer_sync(&tg->service_queue.pending_timer); + kfree(tg); } static struct throtl_grp * @@ -956,32 +813,6 @@ static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio, return 0; } -static void throtl_update_dispatch_stats(struct blkcg_gq *blkg, u64 bytes, - int rw) -{ - struct throtl_grp *tg = blkg_to_tg(blkg); - struct tg_stats_cpu *stats_cpu; - unsigned long flags; - - /* If per cpu stats are not allocated yet, don't do any accounting. */ - if (tg->stats_cpu == NULL) - return; - - /* - * Disabling interrupts to provide mutual exclusion between two - * writes on same cpu. It probably is not needed for 64bit. Not - * optimizing that case yet. - */ - local_irq_save(flags); - - stats_cpu = this_cpu_ptr(tg->stats_cpu); - - blkg_rwstat_add(&stats_cpu->serviced, rw, 1); - blkg_rwstat_add(&stats_cpu->service_bytes, rw, bytes); - - local_irq_restore(flags); -} - static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio) { bool rw = bio_data_dir(bio); @@ -995,17 +826,9 @@ static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio) * more than once as a throttled bio will go through blk-throtl the * second time when it eventually gets issued. Set it when a bio * is being charged to a tg. - * - * Dispatch stats aren't recursive and each @bio should only be - * accounted by the @tg it was originally associated with. Let's - * update the stats when setting REQ_THROTTLED for the first time - * which is guaranteed to be for the @bio's original tg. */ - if (!(bio->bi_rw & REQ_THROTTLED)) { + if (!(bio->bi_rw & REQ_THROTTLED)) bio->bi_rw |= REQ_THROTTLED; - throtl_update_dispatch_stats(tg_to_blkg(tg), - bio->bi_iter.bi_size, bio->bi_rw); - } } /** @@ -1285,34 +1108,6 @@ static void blk_throtl_dispatch_work_fn(struct work_struct *work) } } -static u64 tg_prfill_cpu_rwstat(struct seq_file *sf, - struct blkg_policy_data *pd, int off) -{ - struct throtl_grp *tg = pd_to_tg(pd); - struct blkg_rwstat rwstat = { }, tmp; - int i, cpu; - - if (tg->stats_cpu == NULL) - return 0; - - for_each_possible_cpu(cpu) { - struct tg_stats_cpu *sc = per_cpu_ptr(tg->stats_cpu, cpu); - - tmp = blkg_rwstat_read((void *)sc + off); - for (i = 0; i < BLKG_RWSTAT_NR; i++) - rwstat.cnt[i] += tmp.cnt[i]; - } - - return __blkg_prfill_rwstat(sf, pd, &rwstat); -} - -static int tg_print_cpu_rwstat(struct seq_file *sf, void *v) -{ - blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), tg_prfill_cpu_rwstat, - &blkcg_policy_throtl, seq_cft(sf)->private, true); - return 0; -} - static u64 tg_prfill_conf_u64(struct seq_file *sf, struct blkg_policy_data *pd, int off) { @@ -1349,31 +1144,11 @@ static int tg_print_conf_uint(struct seq_file *sf, void *v) return 0; } -static ssize_t tg_set_conf(struct kernfs_open_file *of, - char *buf, size_t nbytes, loff_t off, bool is_u64) +static void tg_conf_updated(struct throtl_grp *tg) { - struct blkcg *blkcg = css_to_blkcg(of_css(of)); - struct blkg_conf_ctx ctx; - struct throtl_grp *tg; - struct throtl_service_queue *sq; - struct blkcg_gq *blkg; + struct throtl_service_queue *sq = &tg->service_queue; struct cgroup_subsys_state *pos_css; - int ret; - - ret = blkg_conf_prep(blkcg, &blkcg_policy_throtl, buf, &ctx); - if (ret) - return ret; - - tg = blkg_to_tg(ctx.blkg); - sq = &tg->service_queue; - - if (!ctx.v) - ctx.v = -1; - - if (is_u64) - *(u64 *)((void *)tg + of_cft(of)->private) = ctx.v; - else - *(unsigned int *)((void *)tg + of_cft(of)->private) = ctx.v; + struct blkcg_gq *blkg; throtl_log(&tg->service_queue, "limit change rbps=%llu wbps=%llu riops=%u wiops=%u", @@ -1387,7 +1162,7 @@ static ssize_t tg_set_conf(struct kernfs_open_file *of, * restrictions in the whole hierarchy and allows them to bypass * blk-throttle. */ - blkg_for_each_descendant_pre(blkg, pos_css, ctx.blkg) + blkg_for_each_descendant_pre(blkg, pos_css, tg_to_blkg(tg)) tg_update_has_rules(blkg_to_tg(blkg)); /* @@ -1405,9 +1180,39 @@ static ssize_t tg_set_conf(struct kernfs_open_file *of, tg_update_disptime(tg); throtl_schedule_next_dispatch(sq->parent_sq, true); } +} +static ssize_t tg_set_conf(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off, bool is_u64) +{ + struct blkcg *blkcg = css_to_blkcg(of_css(of)); + struct blkg_conf_ctx ctx; + struct throtl_grp *tg; + int ret; + u64 v; + + ret = blkg_conf_prep(blkcg, &blkcg_policy_throtl, buf, &ctx); + if (ret) + return ret; + + ret = -EINVAL; + if (sscanf(ctx.body, "%llu", &v) != 1) + goto out_finish; + if (!v) + v = -1; + + tg = blkg_to_tg(ctx.blkg); + + if (is_u64) + *(u64 *)((void *)tg + of_cft(of)->private) = v; + else + *(unsigned int *)((void *)tg + of_cft(of)->private) = v; + + tg_conf_updated(tg); + ret = 0; +out_finish: blkg_conf_finish(&ctx); - return nbytes; + return ret ?: nbytes; } static ssize_t tg_set_conf_u64(struct kernfs_open_file *of, @@ -1422,7 +1227,7 @@ static ssize_t tg_set_conf_uint(struct kernfs_open_file *of, return tg_set_conf(of, buf, nbytes, off, false); } -static struct cftype throtl_files[] = { +static struct cftype throtl_legacy_files[] = { { .name = "throttle.read_bps_device", .private = offsetof(struct throtl_grp, bps[READ]), @@ -1449,13 +1254,124 @@ static struct cftype throtl_files[] = { }, { .name = "throttle.io_service_bytes", - .private = offsetof(struct tg_stats_cpu, service_bytes), - .seq_show = tg_print_cpu_rwstat, + .private = (unsigned long)&blkcg_policy_throtl, + .seq_show = blkg_print_stat_bytes, }, { .name = "throttle.io_serviced", - .private = offsetof(struct tg_stats_cpu, serviced), - .seq_show = tg_print_cpu_rwstat, + .private = (unsigned long)&blkcg_policy_throtl, + .seq_show = blkg_print_stat_ios, + }, + { } /* terminate */ +}; + +static u64 tg_prfill_max(struct seq_file *sf, struct blkg_policy_data *pd, + int off) +{ + struct throtl_grp *tg = pd_to_tg(pd); + const char *dname = blkg_dev_name(pd->blkg); + char bufs[4][21] = { "max", "max", "max", "max" }; + + if (!dname) + return 0; + if (tg->bps[READ] == -1 && tg->bps[WRITE] == -1 && + tg->iops[READ] == -1 && tg->iops[WRITE] == -1) + return 0; + + if (tg->bps[READ] != -1) + snprintf(bufs[0], sizeof(bufs[0]), "%llu", tg->bps[READ]); + if (tg->bps[WRITE] != -1) + snprintf(bufs[1], sizeof(bufs[1]), "%llu", tg->bps[WRITE]); + if (tg->iops[READ] != -1) + snprintf(bufs[2], sizeof(bufs[2]), "%u", tg->iops[READ]); + if (tg->iops[WRITE] != -1) + snprintf(bufs[3], sizeof(bufs[3]), "%u", tg->iops[WRITE]); + + seq_printf(sf, "%s rbps=%s wbps=%s riops=%s wiops=%s\n", + dname, bufs[0], bufs[1], bufs[2], bufs[3]); + return 0; +} + +static int tg_print_max(struct seq_file *sf, void *v) +{ + blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), tg_prfill_max, + &blkcg_policy_throtl, seq_cft(sf)->private, false); + return 0; +} + +static ssize_t tg_set_max(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + struct blkcg *blkcg = css_to_blkcg(of_css(of)); + struct blkg_conf_ctx ctx; + struct throtl_grp *tg; + u64 v[4]; + int ret; + + ret = blkg_conf_prep(blkcg, &blkcg_policy_throtl, buf, &ctx); + if (ret) + return ret; + + tg = blkg_to_tg(ctx.blkg); + + v[0] = tg->bps[READ]; + v[1] = tg->bps[WRITE]; + v[2] = tg->iops[READ]; + v[3] = tg->iops[WRITE]; + + while (true) { + char tok[27]; /* wiops=18446744073709551616 */ + char *p; + u64 val = -1; + int len; + + if (sscanf(ctx.body, "%26s%n", tok, &len) != 1) + break; + if (tok[0] == '\0') + break; + ctx.body += len; + + ret = -EINVAL; + p = tok; + strsep(&p, "="); + if (!p || (sscanf(p, "%llu", &val) != 1 && strcmp(p, "max"))) + goto out_finish; + + ret = -ERANGE; + if (!val) + goto out_finish; + + ret = -EINVAL; + if (!strcmp(tok, "rbps")) + v[0] = val; + else if (!strcmp(tok, "wbps")) + v[1] = val; + else if (!strcmp(tok, "riops")) + v[2] = min_t(u64, val, UINT_MAX); + else if (!strcmp(tok, "wiops")) + v[3] = min_t(u64, val, UINT_MAX); + else + goto out_finish; + } + + tg->bps[READ] = v[0]; + tg->bps[WRITE] = v[1]; + tg->iops[READ] = v[2]; + tg->iops[WRITE] = v[3]; + + tg_conf_updated(tg); + ret = 0; +out_finish: + blkg_conf_finish(&ctx); + return ret ?: nbytes; +} + +static struct cftype throtl_files[] = { + { + .name = "max", + .flags = CFTYPE_NOT_ON_ROOT, + .seq_show = tg_print_max, + .write = tg_set_max, }, { } /* terminate */ }; @@ -1468,52 +1384,33 @@ static void throtl_shutdown_wq(struct request_queue *q) } static struct blkcg_policy blkcg_policy_throtl = { - .pd_size = sizeof(struct throtl_grp), - .cftypes = throtl_files, + .dfl_cftypes = throtl_files, + .legacy_cftypes = throtl_legacy_files, + .pd_alloc_fn = throtl_pd_alloc, .pd_init_fn = throtl_pd_init, .pd_online_fn = throtl_pd_online, - .pd_exit_fn = throtl_pd_exit, - .pd_reset_stats_fn = throtl_pd_reset_stats, + .pd_free_fn = throtl_pd_free, }; -bool blk_throtl_bio(struct request_queue *q, struct bio *bio) +bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg, + struct bio *bio) { - struct throtl_data *td = q->td; struct throtl_qnode *qn = NULL; - struct throtl_grp *tg; + struct throtl_grp *tg = blkg_to_tg(blkg ?: q->root_blkg); struct throtl_service_queue *sq; bool rw = bio_data_dir(bio); - struct blkcg *blkcg; bool throttled = false; + WARN_ON_ONCE(!rcu_read_lock_held()); + /* see throtl_charge_bio() */ - if (bio->bi_rw & REQ_THROTTLED) + if ((bio->bi_rw & REQ_THROTTLED) || !tg->has_rules[rw]) goto out; - /* - * A throtl_grp pointer retrieved under rcu can be used to access - * basic fields like stats and io rates. If a group has no rules, - * just update the dispatch stats in lockless manner and return. - */ - rcu_read_lock(); - blkcg = bio_blkcg(bio); - tg = throtl_lookup_tg(td, blkcg); - if (tg) { - if (!tg->has_rules[rw]) { - throtl_update_dispatch_stats(tg_to_blkg(tg), - bio->bi_iter.bi_size, bio->bi_rw); - goto out_unlock_rcu; - } - } - - /* - * Either group has not been allocated yet or it is not an unlimited - * IO group - */ spin_lock_irq(q->queue_lock); - tg = throtl_lookup_create_tg(td, blkcg); - if (unlikely(!tg)) + + if (unlikely(blk_queue_bypass(q))) goto out_unlock; sq = &tg->service_queue; @@ -1580,8 +1477,6 @@ bool blk_throtl_bio(struct request_queue *q, struct bio *bio) out_unlock: spin_unlock_irq(q->queue_lock); -out_unlock_rcu: - rcu_read_unlock(); out: /* * As multiple blk-throtls may stack in the same issue path, we @@ -1667,7 +1562,7 @@ int blk_throtl_init(struct request_queue *q) return -ENOMEM; INIT_WORK(&td->dispatch_work, blk_throtl_dispatch_work_fn); - throtl_service_queue_init(&td->service_queue, NULL); + throtl_service_queue_init(&td->service_queue); q->td = td; td->queue = q; diff --git a/block/blk.h b/block/blk.h index 026d9594142b..98614ad37c81 100644 --- a/block/blk.h +++ b/block/blk.h @@ -22,6 +22,12 @@ struct blk_flush_queue { struct list_head flush_queue[2]; struct list_head flush_data_in_flight; struct request *flush_rq; + + /* + * flush_rq shares tag with this rq, both can't be active + * at the same time + */ + struct request *orig_rq; spinlock_t mq_flush_lock; }; @@ -266,15 +272,10 @@ static inline struct io_context *create_io_context(gfp_t gfp_mask, int node) * Internal throttling interface */ #ifdef CONFIG_BLK_DEV_THROTTLING -extern bool blk_throtl_bio(struct request_queue *q, struct bio *bio); extern void blk_throtl_drain(struct request_queue *q); extern int blk_throtl_init(struct request_queue *q); extern void blk_throtl_exit(struct request_queue *q); #else /* CONFIG_BLK_DEV_THROTTLING */ -static inline bool blk_throtl_bio(struct request_queue *q, struct bio *bio) -{ - return false; -} static inline void blk_throtl_drain(struct request_queue *q) { } static inline int blk_throtl_init(struct request_queue *q) { return 0; } static inline void blk_throtl_exit(struct request_queue *q) { } diff --git a/block/bounce.c b/block/bounce.c index b17311227c12..0611aea1cfe9 100644 --- a/block/bounce.c +++ b/block/bounce.c @@ -123,7 +123,7 @@ static void copy_to_high_bio_irq(struct bio *to, struct bio *from) } } -static void bounce_end_io(struct bio *bio, mempool_t *pool, int err) +static void bounce_end_io(struct bio *bio, mempool_t *pool) { struct bio *bio_orig = bio->bi_private; struct bio_vec *bvec, *org_vec; @@ -141,61 +141,44 @@ static void bounce_end_io(struct bio *bio, mempool_t *pool, int err) mempool_free(bvec->bv_page, pool); } - bio_endio(bio_orig, err); + bio_orig->bi_error = bio->bi_error; + bio_endio(bio_orig); bio_put(bio); } -static void bounce_end_io_write(struct bio *bio, int err) +static void bounce_end_io_write(struct bio *bio) { - bounce_end_io(bio, page_pool, err); + bounce_end_io(bio, page_pool); } -static void bounce_end_io_write_isa(struct bio *bio, int err) +static void bounce_end_io_write_isa(struct bio *bio) { - bounce_end_io(bio, isa_page_pool, err); + bounce_end_io(bio, isa_page_pool); } -static void __bounce_end_io_read(struct bio *bio, mempool_t *pool, int err) +static void __bounce_end_io_read(struct bio *bio, mempool_t *pool) { struct bio *bio_orig = bio->bi_private; - if (test_bit(BIO_UPTODATE, &bio->bi_flags)) + if (!bio->bi_error) copy_to_high_bio_irq(bio_orig, bio); - bounce_end_io(bio, pool, err); + bounce_end_io(bio, pool); } -static void bounce_end_io_read(struct bio *bio, int err) +static void bounce_end_io_read(struct bio *bio) { - __bounce_end_io_read(bio, page_pool, err); + __bounce_end_io_read(bio, page_pool); } -static void bounce_end_io_read_isa(struct bio *bio, int err) +static void bounce_end_io_read_isa(struct bio *bio) { - __bounce_end_io_read(bio, isa_page_pool, err); + __bounce_end_io_read(bio, isa_page_pool); } -#ifdef CONFIG_NEED_BOUNCE_POOL -static int must_snapshot_stable_pages(struct request_queue *q, struct bio *bio) -{ - if (bio_data_dir(bio) != WRITE) - return 0; - - if (!bdi_cap_stable_pages_required(&q->backing_dev_info)) - return 0; - - return test_bit(BIO_SNAP_STABLE, &bio->bi_flags); -} -#else -static int must_snapshot_stable_pages(struct request_queue *q, struct bio *bio) -{ - return 0; -} -#endif /* CONFIG_NEED_BOUNCE_POOL */ - static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig, - mempool_t *pool, int force) + mempool_t *pool) { struct bio *bio; int rw = bio_data_dir(*bio_orig); @@ -203,8 +186,6 @@ static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig, struct bvec_iter iter; unsigned i; - if (force) - goto bounce; bio_for_each_segment(from, *bio_orig, iter) if (page_to_pfn(from.bv_page) > queue_bounce_pfn(q)) goto bounce; @@ -216,7 +197,7 @@ static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig, bio_for_each_segment_all(to, bio, i) { struct page *page = to->bv_page; - if (page_to_pfn(page) <= queue_bounce_pfn(q) && !force) + if (page_to_pfn(page) <= queue_bounce_pfn(q)) continue; to->bv_page = mempool_alloc(pool, q->bounce_gfp); @@ -254,7 +235,6 @@ static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig, void blk_queue_bounce(struct request_queue *q, struct bio **bio_orig) { - int must_bounce; mempool_t *pool; /* @@ -263,15 +243,13 @@ void blk_queue_bounce(struct request_queue *q, struct bio **bio_orig) if (!bio_has_data(*bio_orig)) return; - must_bounce = must_snapshot_stable_pages(q, *bio_orig); - /* * for non-isa bounce case, just check if the bounce pfn is equal * to or bigger than the highest pfn in the system -- in that case, * don't waste time iterating over bio segments */ if (!(q->bounce_gfp & GFP_DMA)) { - if (queue_bounce_pfn(q) >= blk_max_pfn && !must_bounce) + if (queue_bounce_pfn(q) >= blk_max_pfn) return; pool = page_pool; } else { @@ -282,7 +260,7 @@ void blk_queue_bounce(struct request_queue *q, struct bio **bio_orig) /* * slow path */ - __blk_queue_bounce(q, bio_orig, pool, must_bounce); + __blk_queue_bounce(q, bio_orig, pool); } EXPORT_SYMBOL(blk_queue_bounce); diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index c62bb2e650b8..04de88463a98 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -68,9 +68,9 @@ static struct kmem_cache *cfq_pool; #define rb_entry_cfqg(node) rb_entry((node), struct cfq_group, rb_node) /* blkio-related constants */ -#define CFQ_WEIGHT_MIN 10 -#define CFQ_WEIGHT_MAX 1000 -#define CFQ_WEIGHT_DEFAULT 500 +#define CFQ_WEIGHT_LEGACY_MIN 10 +#define CFQ_WEIGHT_LEGACY_DFL 500 +#define CFQ_WEIGHT_LEGACY_MAX 1000 struct cfq_ttime { unsigned long last_end_request; @@ -177,10 +177,6 @@ enum wl_type_t { struct cfqg_stats { #ifdef CONFIG_CFQ_GROUP_IOSCHED - /* total bytes transferred */ - struct blkg_rwstat service_bytes; - /* total IOs serviced, post merge */ - struct blkg_rwstat serviced; /* number of ios merged */ struct blkg_rwstat merged; /* total time spent on device in ns, may not be accurate w/ queueing */ @@ -189,8 +185,6 @@ struct cfqg_stats { struct blkg_rwstat wait_time; /* number of IOs queued up */ struct blkg_rwstat queued; - /* total sectors transferred */ - struct blkg_stat sectors; /* total disk time and nr sectors dispatched by this group */ struct blkg_stat time; #ifdef CONFIG_DEBUG_BLK_CGROUP @@ -220,7 +214,7 @@ struct cfqg_stats { /* Per-cgroup data */ struct cfq_group_data { /* must be the first member */ - struct blkcg_policy_data pd; + struct blkcg_policy_data cpd; unsigned int weight; unsigned int leaf_weight; @@ -304,7 +298,11 @@ struct cfq_group { int dispatched; struct cfq_ttime ttime; struct cfqg_stats stats; /* stats for this cfqg */ - struct cfqg_stats dead_stats; /* stats pushed from dead children */ + + /* async queue for each priority case */ + struct cfq_queue *async_cfqq[2][IOPRIO_BE_NR]; + struct cfq_queue *async_idle_cfqq; + }; struct cfq_io_cq { @@ -370,12 +368,6 @@ struct cfq_data { struct cfq_queue *active_queue; struct cfq_io_cq *active_cic; - /* - * async queue for each priority case - */ - struct cfq_queue *async_cfqq[2][IOPRIO_BE_NR]; - struct cfq_queue *async_idle_cfqq; - sector_t last_position; /* @@ -401,6 +393,7 @@ struct cfq_data { }; static struct cfq_group *cfq_get_next_cfqg(struct cfq_data *cfqd); +static void cfq_put_queue(struct cfq_queue *cfqq); static struct cfq_rb_root *st_for(struct cfq_group *cfqg, enum wl_class_t class, @@ -612,7 +605,7 @@ static inline struct cfq_group *pd_to_cfqg(struct blkg_policy_data *pd) static struct cfq_group_data *cpd_to_cfqgd(struct blkcg_policy_data *cpd) { - return cpd ? container_of(cpd, struct cfq_group_data, pd) : NULL; + return cpd ? container_of(cpd, struct cfq_group_data, cpd) : NULL; } static inline struct blkcg_gq *cfqg_to_blkg(struct cfq_group *cfqg) @@ -693,14 +686,6 @@ static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int rw) blkg_rwstat_add(&cfqg->stats.merged, rw, 1); } -static inline void cfqg_stats_update_dispatch(struct cfq_group *cfqg, - uint64_t bytes, int rw) -{ - blkg_stat_add(&cfqg->stats.sectors, bytes >> 9); - blkg_rwstat_add(&cfqg->stats.serviced, rw, 1); - blkg_rwstat_add(&cfqg->stats.service_bytes, rw, bytes); -} - static inline void cfqg_stats_update_completion(struct cfq_group *cfqg, uint64_t start_time, uint64_t io_start_time, int rw) { @@ -718,8 +703,6 @@ static inline void cfqg_stats_update_completion(struct cfq_group *cfqg, static void cfqg_stats_reset(struct cfqg_stats *stats) { /* queued stats shouldn't be cleared */ - blkg_rwstat_reset(&stats->service_bytes); - blkg_rwstat_reset(&stats->serviced); blkg_rwstat_reset(&stats->merged); blkg_rwstat_reset(&stats->service_time); blkg_rwstat_reset(&stats->wait_time); @@ -736,28 +719,26 @@ static void cfqg_stats_reset(struct cfqg_stats *stats) } /* @to += @from */ -static void cfqg_stats_merge(struct cfqg_stats *to, struct cfqg_stats *from) +static void cfqg_stats_add_aux(struct cfqg_stats *to, struct cfqg_stats *from) { /* queued stats shouldn't be cleared */ - blkg_rwstat_merge(&to->service_bytes, &from->service_bytes); - blkg_rwstat_merge(&to->serviced, &from->serviced); - blkg_rwstat_merge(&to->merged, &from->merged); - blkg_rwstat_merge(&to->service_time, &from->service_time); - blkg_rwstat_merge(&to->wait_time, &from->wait_time); - blkg_stat_merge(&from->time, &from->time); + blkg_rwstat_add_aux(&to->merged, &from->merged); + blkg_rwstat_add_aux(&to->service_time, &from->service_time); + blkg_rwstat_add_aux(&to->wait_time, &from->wait_time); + blkg_stat_add_aux(&from->time, &from->time); #ifdef CONFIG_DEBUG_BLK_CGROUP - blkg_stat_merge(&to->unaccounted_time, &from->unaccounted_time); - blkg_stat_merge(&to->avg_queue_size_sum, &from->avg_queue_size_sum); - blkg_stat_merge(&to->avg_queue_size_samples, &from->avg_queue_size_samples); - blkg_stat_merge(&to->dequeue, &from->dequeue); - blkg_stat_merge(&to->group_wait_time, &from->group_wait_time); - blkg_stat_merge(&to->idle_time, &from->idle_time); - blkg_stat_merge(&to->empty_time, &from->empty_time); + blkg_stat_add_aux(&to->unaccounted_time, &from->unaccounted_time); + blkg_stat_add_aux(&to->avg_queue_size_sum, &from->avg_queue_size_sum); + blkg_stat_add_aux(&to->avg_queue_size_samples, &from->avg_queue_size_samples); + blkg_stat_add_aux(&to->dequeue, &from->dequeue); + blkg_stat_add_aux(&to->group_wait_time, &from->group_wait_time); + blkg_stat_add_aux(&to->idle_time, &from->idle_time); + blkg_stat_add_aux(&to->empty_time, &from->empty_time); #endif } /* - * Transfer @cfqg's stats to its parent's dead_stats so that the ancestors' + * Transfer @cfqg's stats to its parent's aux counts so that the ancestors' * recursive stats can still account for the amount used by this cfqg after * it's gone. */ @@ -770,10 +751,8 @@ static void cfqg_stats_xfer_dead(struct cfq_group *cfqg) if (unlikely(!parent)) return; - cfqg_stats_merge(&parent->dead_stats, &cfqg->stats); - cfqg_stats_merge(&parent->dead_stats, &cfqg->dead_stats); + cfqg_stats_add_aux(&parent->stats, &cfqg->stats); cfqg_stats_reset(&cfqg->stats); - cfqg_stats_reset(&cfqg->dead_stats); } #else /* CONFIG_CFQ_GROUP_IOSCHED */ @@ -795,8 +774,6 @@ static inline void cfqg_stats_update_timeslice_used(struct cfq_group *cfqg, unsigned long time, unsigned long unaccounted_time) { } static inline void cfqg_stats_update_io_remove(struct cfq_group *cfqg, int rw) { } static inline void cfqg_stats_update_io_merged(struct cfq_group *cfqg, int rw) { } -static inline void cfqg_stats_update_dispatch(struct cfq_group *cfqg, - uint64_t bytes, int rw) { } static inline void cfqg_stats_update_completion(struct cfq_group *cfqg, uint64_t start_time, uint64_t io_start_time, int rw) { } @@ -883,8 +860,7 @@ static inline int cfqg_busy_async_queues(struct cfq_data *cfqd, static void cfq_dispatch_insert(struct request_queue *, struct request *); static struct cfq_queue *cfq_get_queue(struct cfq_data *cfqd, bool is_sync, - struct cfq_io_cq *cic, struct bio *bio, - gfp_t gfp_mask); + struct cfq_io_cq *cic, struct bio *bio); static inline struct cfq_io_cq *icq_to_cic(struct io_cq *icq) { @@ -1546,130 +1522,171 @@ static void cfq_init_cfqg_base(struct cfq_group *cfqg) } #ifdef CONFIG_CFQ_GROUP_IOSCHED -static void cfqg_stats_init(struct cfqg_stats *stats) +static int __cfq_set_weight(struct cgroup_subsys_state *css, u64 val, + bool on_dfl, bool reset_dev, bool is_leaf_weight); + +static void cfqg_stats_exit(struct cfqg_stats *stats) { - blkg_rwstat_init(&stats->service_bytes); - blkg_rwstat_init(&stats->serviced); - blkg_rwstat_init(&stats->merged); - blkg_rwstat_init(&stats->service_time); - blkg_rwstat_init(&stats->wait_time); - blkg_rwstat_init(&stats->queued); - - blkg_stat_init(&stats->sectors); - blkg_stat_init(&stats->time); - + blkg_rwstat_exit(&stats->merged); + blkg_rwstat_exit(&stats->service_time); + blkg_rwstat_exit(&stats->wait_time); + blkg_rwstat_exit(&stats->queued); + blkg_stat_exit(&stats->time); #ifdef CONFIG_DEBUG_BLK_CGROUP - blkg_stat_init(&stats->unaccounted_time); - blkg_stat_init(&stats->avg_queue_size_sum); - blkg_stat_init(&stats->avg_queue_size_samples); - blkg_stat_init(&stats->dequeue); - blkg_stat_init(&stats->group_wait_time); - blkg_stat_init(&stats->idle_time); - blkg_stat_init(&stats->empty_time); + blkg_stat_exit(&stats->unaccounted_time); + blkg_stat_exit(&stats->avg_queue_size_sum); + blkg_stat_exit(&stats->avg_queue_size_samples); + blkg_stat_exit(&stats->dequeue); + blkg_stat_exit(&stats->group_wait_time); + blkg_stat_exit(&stats->idle_time); + blkg_stat_exit(&stats->empty_time); #endif } -static void cfq_cpd_init(const struct blkcg *blkcg) +static int cfqg_stats_init(struct cfqg_stats *stats, gfp_t gfp) { - struct cfq_group_data *cgd = - cpd_to_cfqgd(blkcg->pd[blkcg_policy_cfq.plid]); + if (blkg_rwstat_init(&stats->merged, gfp) || + blkg_rwstat_init(&stats->service_time, gfp) || + blkg_rwstat_init(&stats->wait_time, gfp) || + blkg_rwstat_init(&stats->queued, gfp) || + blkg_stat_init(&stats->time, gfp)) + goto err; - if (blkcg == &blkcg_root) { - cgd->weight = 2 * CFQ_WEIGHT_DEFAULT; - cgd->leaf_weight = 2 * CFQ_WEIGHT_DEFAULT; - } else { - cgd->weight = CFQ_WEIGHT_DEFAULT; - cgd->leaf_weight = CFQ_WEIGHT_DEFAULT; - } +#ifdef CONFIG_DEBUG_BLK_CGROUP + if (blkg_stat_init(&stats->unaccounted_time, gfp) || + blkg_stat_init(&stats->avg_queue_size_sum, gfp) || + blkg_stat_init(&stats->avg_queue_size_samples, gfp) || + blkg_stat_init(&stats->dequeue, gfp) || + blkg_stat_init(&stats->group_wait_time, gfp) || + blkg_stat_init(&stats->idle_time, gfp) || + blkg_stat_init(&stats->empty_time, gfp)) + goto err; +#endif + return 0; +err: + cfqg_stats_exit(stats); + return -ENOMEM; } -static void cfq_pd_init(struct blkcg_gq *blkg) +static struct blkcg_policy_data *cfq_cpd_alloc(gfp_t gfp) { - struct cfq_group *cfqg = blkg_to_cfqg(blkg); - struct cfq_group_data *cgd = blkcg_to_cfqgd(blkg->blkcg); + struct cfq_group_data *cgd; + + cgd = kzalloc(sizeof(*cgd), GFP_KERNEL); + if (!cgd) + return NULL; + return &cgd->cpd; +} + +static void cfq_cpd_init(struct blkcg_policy_data *cpd) +{ + struct cfq_group_data *cgd = cpd_to_cfqgd(cpd); + unsigned int weight = cgroup_on_dfl(blkcg_root.css.cgroup) ? + CGROUP_WEIGHT_DFL : CFQ_WEIGHT_LEGACY_DFL; + + if (cpd_to_blkcg(cpd) == &blkcg_root) + weight *= 2; + + cgd->weight = weight; + cgd->leaf_weight = weight; +} + +static void cfq_cpd_free(struct blkcg_policy_data *cpd) +{ + kfree(cpd_to_cfqgd(cpd)); +} + +static void cfq_cpd_bind(struct blkcg_policy_data *cpd) +{ + struct blkcg *blkcg = cpd_to_blkcg(cpd); + bool on_dfl = cgroup_on_dfl(blkcg_root.css.cgroup); + unsigned int weight = on_dfl ? CGROUP_WEIGHT_DFL : CFQ_WEIGHT_LEGACY_DFL; + + if (blkcg == &blkcg_root) + weight *= 2; + + WARN_ON_ONCE(__cfq_set_weight(&blkcg->css, weight, on_dfl, true, false)); + WARN_ON_ONCE(__cfq_set_weight(&blkcg->css, weight, on_dfl, true, true)); +} + +static struct blkg_policy_data *cfq_pd_alloc(gfp_t gfp, int node) +{ + struct cfq_group *cfqg; + + cfqg = kzalloc_node(sizeof(*cfqg), gfp, node); + if (!cfqg) + return NULL; cfq_init_cfqg_base(cfqg); - cfqg->weight = cgd->weight; - cfqg->leaf_weight = cgd->leaf_weight; - cfqg_stats_init(&cfqg->stats); - cfqg_stats_init(&cfqg->dead_stats); + if (cfqg_stats_init(&cfqg->stats, gfp)) { + kfree(cfqg); + return NULL; + } + + return &cfqg->pd; } -static void cfq_pd_offline(struct blkcg_gq *blkg) +static void cfq_pd_init(struct blkg_policy_data *pd) { + struct cfq_group *cfqg = pd_to_cfqg(pd); + struct cfq_group_data *cgd = blkcg_to_cfqgd(pd->blkg->blkcg); + + cfqg->weight = cgd->weight; + cfqg->leaf_weight = cgd->leaf_weight; +} + +static void cfq_pd_offline(struct blkg_policy_data *pd) +{ + struct cfq_group *cfqg = pd_to_cfqg(pd); + int i; + + for (i = 0; i < IOPRIO_BE_NR; i++) { + if (cfqg->async_cfqq[0][i]) + cfq_put_queue(cfqg->async_cfqq[0][i]); + if (cfqg->async_cfqq[1][i]) + cfq_put_queue(cfqg->async_cfqq[1][i]); + } + + if (cfqg->async_idle_cfqq) + cfq_put_queue(cfqg->async_idle_cfqq); + /* * @blkg is going offline and will be ignored by * blkg_[rw]stat_recursive_sum(). Transfer stats to the parent so * that they don't get lost. If IOs complete after this point, the * stats for them will be lost. Oh well... */ - cfqg_stats_xfer_dead(blkg_to_cfqg(blkg)); + cfqg_stats_xfer_dead(cfqg); } -/* offset delta from cfqg->stats to cfqg->dead_stats */ -static const int dead_stats_off_delta = offsetof(struct cfq_group, dead_stats) - - offsetof(struct cfq_group, stats); - -/* to be used by recursive prfill, sums live and dead stats recursively */ -static u64 cfqg_stat_pd_recursive_sum(struct blkg_policy_data *pd, int off) +static void cfq_pd_free(struct blkg_policy_data *pd) { - u64 sum = 0; + struct cfq_group *cfqg = pd_to_cfqg(pd); - sum += blkg_stat_recursive_sum(pd, off); - sum += blkg_stat_recursive_sum(pd, off + dead_stats_off_delta); - return sum; + cfqg_stats_exit(&cfqg->stats); + return kfree(cfqg); } -/* to be used by recursive prfill, sums live and dead rwstats recursively */ -static struct blkg_rwstat cfqg_rwstat_pd_recursive_sum(struct blkg_policy_data *pd, - int off) +static void cfq_pd_reset_stats(struct blkg_policy_data *pd) { - struct blkg_rwstat a, b; - - a = blkg_rwstat_recursive_sum(pd, off); - b = blkg_rwstat_recursive_sum(pd, off + dead_stats_off_delta); - blkg_rwstat_merge(&a, &b); - return a; -} - -static void cfq_pd_reset_stats(struct blkcg_gq *blkg) -{ - struct cfq_group *cfqg = blkg_to_cfqg(blkg); + struct cfq_group *cfqg = pd_to_cfqg(pd); cfqg_stats_reset(&cfqg->stats); - cfqg_stats_reset(&cfqg->dead_stats); } -/* - * Search for the cfq group current task belongs to. request_queue lock must - * be held. - */ -static struct cfq_group *cfq_lookup_create_cfqg(struct cfq_data *cfqd, - struct blkcg *blkcg) +static struct cfq_group *cfq_lookup_cfqg(struct cfq_data *cfqd, + struct blkcg *blkcg) { - struct request_queue *q = cfqd->queue; - struct cfq_group *cfqg = NULL; + struct blkcg_gq *blkg; - /* avoid lookup for the common case where there's no blkcg */ - if (blkcg == &blkcg_root) { - cfqg = cfqd->root_group; - } else { - struct blkcg_gq *blkg; - - blkg = blkg_lookup_create(blkcg, q); - if (!IS_ERR(blkg)) - cfqg = blkg_to_cfqg(blkg); - } - - return cfqg; + blkg = blkg_lookup(blkcg, cfqd->queue); + if (likely(blkg)) + return blkg_to_cfqg(blkg); + return NULL; } static void cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg) { - /* Currently, all async queues are mapped to root group */ - if (!cfq_cfqq_sync(cfqq)) - cfqg = cfqq->cfqd->root_group; - cfqq->cfqg = cfqg; /* cfqq reference on cfqg */ cfqg_get(cfqg); @@ -1739,36 +1756,48 @@ static int cfq_print_leaf_weight(struct seq_file *sf, void *v) static ssize_t __cfqg_set_weight_device(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off, - bool is_leaf_weight) + bool on_dfl, bool is_leaf_weight) { + unsigned int min = on_dfl ? CGROUP_WEIGHT_MIN : CFQ_WEIGHT_LEGACY_MIN; + unsigned int max = on_dfl ? CGROUP_WEIGHT_MAX : CFQ_WEIGHT_LEGACY_MAX; struct blkcg *blkcg = css_to_blkcg(of_css(of)); struct blkg_conf_ctx ctx; struct cfq_group *cfqg; struct cfq_group_data *cfqgd; int ret; + u64 v; ret = blkg_conf_prep(blkcg, &blkcg_policy_cfq, buf, &ctx); if (ret) return ret; - ret = -EINVAL; + if (sscanf(ctx.body, "%llu", &v) == 1) { + /* require "default" on dfl */ + ret = -ERANGE; + if (!v && on_dfl) + goto out_finish; + } else if (!strcmp(strim(ctx.body), "default")) { + v = 0; + } else { + ret = -EINVAL; + goto out_finish; + } + cfqg = blkg_to_cfqg(ctx.blkg); cfqgd = blkcg_to_cfqgd(blkcg); - if (!cfqg || !cfqgd) - goto err; - if (!ctx.v || (ctx.v >= CFQ_WEIGHT_MIN && ctx.v <= CFQ_WEIGHT_MAX)) { + ret = -ERANGE; + if (!v || (v >= min && v <= max)) { if (!is_leaf_weight) { - cfqg->dev_weight = ctx.v; - cfqg->new_weight = ctx.v ?: cfqgd->weight; + cfqg->dev_weight = v; + cfqg->new_weight = v ?: cfqgd->weight; } else { - cfqg->dev_leaf_weight = ctx.v; - cfqg->new_leaf_weight = ctx.v ?: cfqgd->leaf_weight; + cfqg->dev_leaf_weight = v; + cfqg->new_leaf_weight = v ?: cfqgd->leaf_weight; } ret = 0; } - -err: +out_finish: blkg_conf_finish(&ctx); return ret ?: nbytes; } @@ -1776,25 +1805,27 @@ static ssize_t __cfqg_set_weight_device(struct kernfs_open_file *of, static ssize_t cfqg_set_weight_device(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { - return __cfqg_set_weight_device(of, buf, nbytes, off, false); + return __cfqg_set_weight_device(of, buf, nbytes, off, false, false); } static ssize_t cfqg_set_leaf_weight_device(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { - return __cfqg_set_weight_device(of, buf, nbytes, off, true); + return __cfqg_set_weight_device(of, buf, nbytes, off, false, true); } -static int __cfq_set_weight(struct cgroup_subsys_state *css, struct cftype *cft, - u64 val, bool is_leaf_weight) +static int __cfq_set_weight(struct cgroup_subsys_state *css, u64 val, + bool on_dfl, bool reset_dev, bool is_leaf_weight) { + unsigned int min = on_dfl ? CGROUP_WEIGHT_MIN : CFQ_WEIGHT_LEGACY_MIN; + unsigned int max = on_dfl ? CGROUP_WEIGHT_MAX : CFQ_WEIGHT_LEGACY_MAX; struct blkcg *blkcg = css_to_blkcg(css); struct blkcg_gq *blkg; struct cfq_group_data *cfqgd; int ret = 0; - if (val < CFQ_WEIGHT_MIN || val > CFQ_WEIGHT_MAX) - return -EINVAL; + if (val < min || val > max) + return -ERANGE; spin_lock_irq(&blkcg->lock); cfqgd = blkcg_to_cfqgd(blkcg); @@ -1815,9 +1846,13 @@ static int __cfq_set_weight(struct cgroup_subsys_state *css, struct cftype *cft, continue; if (!is_leaf_weight) { + if (reset_dev) + cfqg->dev_weight = 0; if (!cfqg->dev_weight) cfqg->new_weight = cfqgd->weight; } else { + if (reset_dev) + cfqg->dev_leaf_weight = 0; if (!cfqg->dev_leaf_weight) cfqg->new_leaf_weight = cfqgd->leaf_weight; } @@ -1831,13 +1866,13 @@ static int __cfq_set_weight(struct cgroup_subsys_state *css, struct cftype *cft, static int cfq_set_weight(struct cgroup_subsys_state *css, struct cftype *cft, u64 val) { - return __cfq_set_weight(css, cft, val, false); + return __cfq_set_weight(css, val, false, false, false); } static int cfq_set_leaf_weight(struct cgroup_subsys_state *css, struct cftype *cft, u64 val) { - return __cfq_set_weight(css, cft, val, true); + return __cfq_set_weight(css, val, false, false, true); } static int cfqg_print_stat(struct seq_file *sf, void *v) @@ -1857,16 +1892,16 @@ static int cfqg_print_rwstat(struct seq_file *sf, void *v) static u64 cfqg_prfill_stat_recursive(struct seq_file *sf, struct blkg_policy_data *pd, int off) { - u64 sum = cfqg_stat_pd_recursive_sum(pd, off); - + u64 sum = blkg_stat_recursive_sum(pd_to_blkg(pd), + &blkcg_policy_cfq, off); return __blkg_prfill_u64(sf, pd, sum); } static u64 cfqg_prfill_rwstat_recursive(struct seq_file *sf, struct blkg_policy_data *pd, int off) { - struct blkg_rwstat sum = cfqg_rwstat_pd_recursive_sum(pd, off); - + struct blkg_rwstat sum = blkg_rwstat_recursive_sum(pd_to_blkg(pd), + &blkcg_policy_cfq, off); return __blkg_prfill_rwstat(sf, pd, &sum); } @@ -1886,6 +1921,40 @@ static int cfqg_print_rwstat_recursive(struct seq_file *sf, void *v) return 0; } +static u64 cfqg_prfill_sectors(struct seq_file *sf, struct blkg_policy_data *pd, + int off) +{ + u64 sum = blkg_rwstat_total(&pd->blkg->stat_bytes); + + return __blkg_prfill_u64(sf, pd, sum >> 9); +} + +static int cfqg_print_stat_sectors(struct seq_file *sf, void *v) +{ + blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), + cfqg_prfill_sectors, &blkcg_policy_cfq, 0, false); + return 0; +} + +static u64 cfqg_prfill_sectors_recursive(struct seq_file *sf, + struct blkg_policy_data *pd, int off) +{ + struct blkg_rwstat tmp = blkg_rwstat_recursive_sum(pd->blkg, NULL, + offsetof(struct blkcg_gq, stat_bytes)); + u64 sum = atomic64_read(&tmp.aux_cnt[BLKG_RWSTAT_READ]) + + atomic64_read(&tmp.aux_cnt[BLKG_RWSTAT_WRITE]); + + return __blkg_prfill_u64(sf, pd, sum >> 9); +} + +static int cfqg_print_stat_sectors_recursive(struct seq_file *sf, void *v) +{ + blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), + cfqg_prfill_sectors_recursive, &blkcg_policy_cfq, 0, + false); + return 0; +} + #ifdef CONFIG_DEBUG_BLK_CGROUP static u64 cfqg_prfill_avg_queue_size(struct seq_file *sf, struct blkg_policy_data *pd, int off) @@ -1912,7 +1981,7 @@ static int cfqg_print_avg_queue_size(struct seq_file *sf, void *v) } #endif /* CONFIG_DEBUG_BLK_CGROUP */ -static struct cftype cfq_blkcg_files[] = { +static struct cftype cfq_blkcg_legacy_files[] = { /* on root, weight is mapped to leaf_weight */ { .name = "weight_device", @@ -1960,18 +2029,17 @@ static struct cftype cfq_blkcg_files[] = { }, { .name = "sectors", - .private = offsetof(struct cfq_group, stats.sectors), - .seq_show = cfqg_print_stat, + .seq_show = cfqg_print_stat_sectors, }, { .name = "io_service_bytes", - .private = offsetof(struct cfq_group, stats.service_bytes), - .seq_show = cfqg_print_rwstat, + .private = (unsigned long)&blkcg_policy_cfq, + .seq_show = blkg_print_stat_bytes, }, { .name = "io_serviced", - .private = offsetof(struct cfq_group, stats.serviced), - .seq_show = cfqg_print_rwstat, + .private = (unsigned long)&blkcg_policy_cfq, + .seq_show = blkg_print_stat_ios, }, { .name = "io_service_time", @@ -2002,18 +2070,17 @@ static struct cftype cfq_blkcg_files[] = { }, { .name = "sectors_recursive", - .private = offsetof(struct cfq_group, stats.sectors), - .seq_show = cfqg_print_stat_recursive, + .seq_show = cfqg_print_stat_sectors_recursive, }, { .name = "io_service_bytes_recursive", - .private = offsetof(struct cfq_group, stats.service_bytes), - .seq_show = cfqg_print_rwstat_recursive, + .private = (unsigned long)&blkcg_policy_cfq, + .seq_show = blkg_print_stat_bytes_recursive, }, { .name = "io_serviced_recursive", - .private = offsetof(struct cfq_group, stats.serviced), - .seq_show = cfqg_print_rwstat_recursive, + .private = (unsigned long)&blkcg_policy_cfq, + .seq_show = blkg_print_stat_ios_recursive, }, { .name = "io_service_time_recursive", @@ -2068,9 +2135,51 @@ static struct cftype cfq_blkcg_files[] = { #endif /* CONFIG_DEBUG_BLK_CGROUP */ { } /* terminate */ }; + +static int cfq_print_weight_on_dfl(struct seq_file *sf, void *v) +{ + struct blkcg *blkcg = css_to_blkcg(seq_css(sf)); + struct cfq_group_data *cgd = blkcg_to_cfqgd(blkcg); + + seq_printf(sf, "default %u\n", cgd->weight); + blkcg_print_blkgs(sf, blkcg, cfqg_prfill_weight_device, + &blkcg_policy_cfq, 0, false); + return 0; +} + +static ssize_t cfq_set_weight_on_dfl(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + char *endp; + int ret; + u64 v; + + buf = strim(buf); + + /* "WEIGHT" or "default WEIGHT" sets the default weight */ + v = simple_strtoull(buf, &endp, 0); + if (*endp == '\0' || sscanf(buf, "default %llu", &v) == 1) { + ret = __cfq_set_weight(of_css(of), v, true, false, false); + return ret ?: nbytes; + } + + /* "MAJ:MIN WEIGHT" */ + return __cfqg_set_weight_device(of, buf, nbytes, off, true, false); +} + +static struct cftype cfq_blkcg_files[] = { + { + .name = "weight", + .flags = CFTYPE_NOT_ON_ROOT, + .seq_show = cfq_print_weight_on_dfl, + .write = cfq_set_weight_on_dfl, + }, + { } /* terminate */ +}; + #else /* GROUP_IOSCHED */ -static struct cfq_group *cfq_lookup_create_cfqg(struct cfq_data *cfqd, - struct blkcg *blkcg) +static struct cfq_group *cfq_lookup_cfqg(struct cfq_data *cfqd, + struct blkcg *blkcg) { return cfqd->root_group; } @@ -2873,7 +2982,6 @@ static void cfq_dispatch_insert(struct request_queue *q, struct request *rq) cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]++; cfqq->nr_sectors += blk_rq_sectors(rq); - cfqg_stats_update_dispatch(cfqq->cfqg, blk_rq_bytes(rq), rq->cmd_flags); } /* @@ -3506,14 +3614,14 @@ static void cfq_exit_icq(struct io_cq *icq) struct cfq_io_cq *cic = icq_to_cic(icq); struct cfq_data *cfqd = cic_to_cfqd(cic); - if (cic->cfqq[BLK_RW_ASYNC]) { - cfq_exit_cfqq(cfqd, cic->cfqq[BLK_RW_ASYNC]); - cic->cfqq[BLK_RW_ASYNC] = NULL; + if (cic_to_cfqq(cic, false)) { + cfq_exit_cfqq(cfqd, cic_to_cfqq(cic, false)); + cic_set_cfqq(cic, NULL, false); } - if (cic->cfqq[BLK_RW_SYNC]) { - cfq_exit_cfqq(cfqd, cic->cfqq[BLK_RW_SYNC]); - cic->cfqq[BLK_RW_SYNC] = NULL; + if (cic_to_cfqq(cic, true)) { + cfq_exit_cfqq(cfqd, cic_to_cfqq(cic, true)); + cic_set_cfqq(cic, NULL, true); } } @@ -3572,18 +3680,14 @@ static void check_ioprio_changed(struct cfq_io_cq *cic, struct bio *bio) if (unlikely(!cfqd) || likely(cic->ioprio == ioprio)) return; - cfqq = cic->cfqq[BLK_RW_ASYNC]; + cfqq = cic_to_cfqq(cic, false); if (cfqq) { - struct cfq_queue *new_cfqq; - new_cfqq = cfq_get_queue(cfqd, BLK_RW_ASYNC, cic, bio, - GFP_ATOMIC); - if (new_cfqq) { - cic->cfqq[BLK_RW_ASYNC] = new_cfqq; - cfq_put_queue(cfqq); - } + cfq_put_queue(cfqq); + cfqq = cfq_get_queue(cfqd, BLK_RW_ASYNC, cic, bio); + cic_set_cfqq(cic, cfqq, false); } - cfqq = cic->cfqq[BLK_RW_SYNC]; + cfqq = cic_to_cfqq(cic, true); if (cfqq) cfq_mark_cfqq_prio_changed(cfqq); @@ -3614,7 +3718,7 @@ static void cfq_init_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq, static void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio) { struct cfq_data *cfqd = cic_to_cfqd(cic); - struct cfq_queue *sync_cfqq; + struct cfq_queue *cfqq; uint64_t serial_nr; rcu_read_lock(); @@ -3628,15 +3732,22 @@ static void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio) if (unlikely(!cfqd) || likely(cic->blkcg_serial_nr == serial_nr)) return; - sync_cfqq = cic_to_cfqq(cic, 1); - if (sync_cfqq) { - /* - * Drop reference to sync queue. A new sync queue will be - * assigned in new group upon arrival of a fresh request. - */ - cfq_log_cfqq(cfqd, sync_cfqq, "changed cgroup"); - cic_set_cfqq(cic, NULL, 1); - cfq_put_queue(sync_cfqq); + /* + * Drop reference to queues. New queues will be assigned in new + * group upon arrival of fresh requests. + */ + cfqq = cic_to_cfqq(cic, false); + if (cfqq) { + cfq_log_cfqq(cfqd, cfqq, "changed cgroup"); + cic_set_cfqq(cic, NULL, false); + cfq_put_queue(cfqq); + } + + cfqq = cic_to_cfqq(cic, true); + if (cfqq) { + cfq_log_cfqq(cfqd, cfqq, "changed cgroup"); + cic_set_cfqq(cic, NULL, true); + cfq_put_queue(cfqq); } cic->blkcg_serial_nr = serial_nr; @@ -3645,81 +3756,19 @@ static void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio) static inline void check_blkcg_changed(struct cfq_io_cq *cic, struct bio *bio) { } #endif /* CONFIG_CFQ_GROUP_IOSCHED */ -static struct cfq_queue * -cfq_find_alloc_queue(struct cfq_data *cfqd, bool is_sync, struct cfq_io_cq *cic, - struct bio *bio, gfp_t gfp_mask) -{ - struct blkcg *blkcg; - struct cfq_queue *cfqq, *new_cfqq = NULL; - struct cfq_group *cfqg; - -retry: - rcu_read_lock(); - - blkcg = bio_blkcg(bio); - cfqg = cfq_lookup_create_cfqg(cfqd, blkcg); - if (!cfqg) { - cfqq = &cfqd->oom_cfqq; - goto out; - } - - cfqq = cic_to_cfqq(cic, is_sync); - - /* - * Always try a new alloc if we fell back to the OOM cfqq - * originally, since it should just be a temporary situation. - */ - if (!cfqq || cfqq == &cfqd->oom_cfqq) { - cfqq = NULL; - if (new_cfqq) { - cfqq = new_cfqq; - new_cfqq = NULL; - } else if (gfp_mask & __GFP_WAIT) { - rcu_read_unlock(); - spin_unlock_irq(cfqd->queue->queue_lock); - new_cfqq = kmem_cache_alloc_node(cfq_pool, - gfp_mask | __GFP_ZERO, - cfqd->queue->node); - spin_lock_irq(cfqd->queue->queue_lock); - if (new_cfqq) - goto retry; - else - return &cfqd->oom_cfqq; - } else { - cfqq = kmem_cache_alloc_node(cfq_pool, - gfp_mask | __GFP_ZERO, - cfqd->queue->node); - } - - if (cfqq) { - cfq_init_cfqq(cfqd, cfqq, current->pid, is_sync); - cfq_init_prio_data(cfqq, cic); - cfq_link_cfqq_cfqg(cfqq, cfqg); - cfq_log_cfqq(cfqd, cfqq, "alloced"); - } else - cfqq = &cfqd->oom_cfqq; - } -out: - if (new_cfqq) - kmem_cache_free(cfq_pool, new_cfqq); - - rcu_read_unlock(); - return cfqq; -} - static struct cfq_queue ** -cfq_async_queue_prio(struct cfq_data *cfqd, int ioprio_class, int ioprio) +cfq_async_queue_prio(struct cfq_group *cfqg, int ioprio_class, int ioprio) { switch (ioprio_class) { case IOPRIO_CLASS_RT: - return &cfqd->async_cfqq[0][ioprio]; + return &cfqg->async_cfqq[0][ioprio]; case IOPRIO_CLASS_NONE: ioprio = IOPRIO_NORM; /* fall through */ case IOPRIO_CLASS_BE: - return &cfqd->async_cfqq[1][ioprio]; + return &cfqg->async_cfqq[1][ioprio]; case IOPRIO_CLASS_IDLE: - return &cfqd->async_idle_cfqq; + return &cfqg->async_idle_cfqq; default: BUG(); } @@ -3727,12 +3776,20 @@ cfq_async_queue_prio(struct cfq_data *cfqd, int ioprio_class, int ioprio) static struct cfq_queue * cfq_get_queue(struct cfq_data *cfqd, bool is_sync, struct cfq_io_cq *cic, - struct bio *bio, gfp_t gfp_mask) + struct bio *bio) { int ioprio_class = IOPRIO_PRIO_CLASS(cic->ioprio); int ioprio = IOPRIO_PRIO_DATA(cic->ioprio); struct cfq_queue **async_cfqq = NULL; - struct cfq_queue *cfqq = NULL; + struct cfq_queue *cfqq; + struct cfq_group *cfqg; + + rcu_read_lock(); + cfqg = cfq_lookup_cfqg(cfqd, bio_blkcg(bio)); + if (!cfqg) { + cfqq = &cfqd->oom_cfqq; + goto out; + } if (!is_sync) { if (!ioprio_valid(cic->ioprio)) { @@ -3740,22 +3797,32 @@ cfq_get_queue(struct cfq_data *cfqd, bool is_sync, struct cfq_io_cq *cic, ioprio = task_nice_ioprio(tsk); ioprio_class = task_nice_ioclass(tsk); } - async_cfqq = cfq_async_queue_prio(cfqd, ioprio_class, ioprio); + async_cfqq = cfq_async_queue_prio(cfqg, ioprio_class, ioprio); cfqq = *async_cfqq; + if (cfqq) + goto out; } - if (!cfqq) - cfqq = cfq_find_alloc_queue(cfqd, is_sync, cic, bio, gfp_mask); + cfqq = kmem_cache_alloc_node(cfq_pool, GFP_NOWAIT | __GFP_ZERO, + cfqd->queue->node); + if (!cfqq) { + cfqq = &cfqd->oom_cfqq; + goto out; + } - /* - * pin the queue now that it's allocated, scheduler exit will prune it - */ - if (!is_sync && !(*async_cfqq)) { + cfq_init_cfqq(cfqd, cfqq, current->pid, is_sync); + cfq_init_prio_data(cfqq, cic); + cfq_link_cfqq_cfqg(cfqq, cfqg); + cfq_log_cfqq(cfqd, cfqq, "alloced"); + + if (async_cfqq) { + /* a new async queue is created, pin and remember */ cfqq->ref++; *async_cfqq = cfqq; } - +out: cfqq->ref++; + rcu_read_unlock(); return cfqq; } @@ -4289,8 +4356,6 @@ cfq_set_request(struct request_queue *q, struct request *rq, struct bio *bio, const bool is_sync = rq_is_sync(rq); struct cfq_queue *cfqq; - might_sleep_if(gfp_mask & __GFP_WAIT); - spin_lock_irq(q->queue_lock); check_ioprio_changed(cic, bio); @@ -4298,7 +4363,9 @@ cfq_set_request(struct request_queue *q, struct request *rq, struct bio *bio, new_queue: cfqq = cic_to_cfqq(cic, is_sync); if (!cfqq || cfqq == &cfqd->oom_cfqq) { - cfqq = cfq_get_queue(cfqd, is_sync, cic, bio, gfp_mask); + if (cfqq) + cfq_put_queue(cfqq); + cfqq = cfq_get_queue(cfqd, is_sync, cic, bio); cic_set_cfqq(cic, cfqq, is_sync); } else { /* @@ -4404,21 +4471,6 @@ static void cfq_shutdown_timer_wq(struct cfq_data *cfqd) cancel_work_sync(&cfqd->unplug_work); } -static void cfq_put_async_queues(struct cfq_data *cfqd) -{ - int i; - - for (i = 0; i < IOPRIO_BE_NR; i++) { - if (cfqd->async_cfqq[0][i]) - cfq_put_queue(cfqd->async_cfqq[0][i]); - if (cfqd->async_cfqq[1][i]) - cfq_put_queue(cfqd->async_cfqq[1][i]); - } - - if (cfqd->async_idle_cfqq) - cfq_put_queue(cfqd->async_idle_cfqq); -} - static void cfq_exit_queue(struct elevator_queue *e) { struct cfq_data *cfqd = e->elevator_data; @@ -4431,8 +4483,6 @@ static void cfq_exit_queue(struct elevator_queue *e) if (cfqd->active_queue) __cfq_slice_expired(cfqd, cfqd->active_queue, 0); - cfq_put_async_queues(cfqd); - spin_unlock_irq(q->queue_lock); cfq_shutdown_timer_wq(cfqd); @@ -4486,9 +4536,9 @@ static int cfq_init_queue(struct request_queue *q, struct elevator_type *e) goto out_free; cfq_init_cfqg_base(cfqd->root_group); + cfqd->root_group->weight = 2 * CFQ_WEIGHT_LEGACY_DFL; + cfqd->root_group->leaf_weight = 2 * CFQ_WEIGHT_LEGACY_DFL; #endif - cfqd->root_group->weight = 2 * CFQ_WEIGHT_DEFAULT; - cfqd->root_group->leaf_weight = 2 * CFQ_WEIGHT_DEFAULT; /* * Not strictly needed (since RB_ROOT just clears the node and we @@ -4499,7 +4549,7 @@ static int cfq_init_queue(struct request_queue *q, struct elevator_type *e) cfqd->prio_trees[i] = RB_ROOT; /* - * Our fallback cfqq if cfq_find_alloc_queue() runs into OOM issues. + * Our fallback cfqq if cfq_get_queue() runs into OOM issues. * Grab a permanent reference to it, so that the normal code flow * will not attempt to free it. oom_cfqq is linked to root_group * but shouldn't hold a reference as it'll never be unlinked. Lose @@ -4683,13 +4733,18 @@ static struct elevator_type iosched_cfq = { #ifdef CONFIG_CFQ_GROUP_IOSCHED static struct blkcg_policy blkcg_policy_cfq = { - .pd_size = sizeof(struct cfq_group), - .cpd_size = sizeof(struct cfq_group_data), - .cftypes = cfq_blkcg_files, + .dfl_cftypes = cfq_blkcg_files, + .legacy_cftypes = cfq_blkcg_legacy_files, + .cpd_alloc_fn = cfq_cpd_alloc, .cpd_init_fn = cfq_cpd_init, + .cpd_free_fn = cfq_cpd_free, + .cpd_bind_fn = cfq_cpd_bind, + + .pd_alloc_fn = cfq_pd_alloc, .pd_init_fn = cfq_pd_init, .pd_offline_fn = cfq_pd_offline, + .pd_free_fn = cfq_pd_free, .pd_reset_stats_fn = cfq_pd_reset_stats, }; #endif diff --git a/block/genhd.c b/block/genhd.c index 59a1395eedac..0c706f33a599 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -1110,8 +1110,7 @@ static void disk_release(struct device *dev) disk_release_events(disk); kfree(disk->random); disk_replace_part_tbl(disk, NULL); - free_part_stats(&disk->part0); - free_part_info(&disk->part0); + hd_free_part(&disk->part0); if (disk->queue) blk_put_queue(disk->queue); kfree(disk); @@ -1285,7 +1284,11 @@ struct gendisk *alloc_disk_node(int minors, int node_id) * converted to make use of bd_mutex and sequence counters. */ seqcount_init(&disk->part0.nr_sects_seq); - hd_ref_init(&disk->part0); + if (hd_ref_init(&disk->part0)) { + hd_free_part(&disk->part0); + kfree(disk); + return NULL; + } disk->minors = minors; rand_initialize_disk(disk); diff --git a/block/partition-generic.c b/block/partition-generic.c index 0d9e5f97f0a8..e7711133284e 100644 --- a/block/partition-generic.c +++ b/block/partition-generic.c @@ -212,8 +212,7 @@ static void part_release(struct device *dev) { struct hd_struct *p = dev_to_part(dev); blk_free_devt(dev->devt); - free_part_stats(p); - free_part_info(p); + hd_free_part(p); kfree(p); } @@ -233,8 +232,9 @@ static void delete_partition_rcu_cb(struct rcu_head *head) put_device(part_to_dev(part)); } -void __delete_partition(struct hd_struct *part) +void __delete_partition(struct percpu_ref *ref) { + struct hd_struct *part = container_of(ref, struct hd_struct, ref); call_rcu(&part->rcu_head, delete_partition_rcu_cb); } @@ -255,7 +255,7 @@ void delete_partition(struct gendisk *disk, int partno) kobject_put(part->holder_dir); device_del(part_to_dev(part)); - hd_struct_put(part); + hd_struct_kill(part); } static ssize_t whole_disk_show(struct device *dev, @@ -356,8 +356,8 @@ struct hd_struct *add_partition(struct gendisk *disk, int partno, if (!dev_get_uevent_suppress(ddev)) kobject_uevent(&pdev->kobj, KOBJ_ADD); - hd_ref_init(p); - return p; + if (!hd_ref_init(p)) + return p; out_free_info: free_part_info(p); diff --git a/certs/Kconfig b/certs/Kconfig new file mode 100644 index 000000000000..b030b9c7ed34 --- /dev/null +++ b/certs/Kconfig @@ -0,0 +1,42 @@ +menu "Certificates for signature checking" + +config MODULE_SIG_KEY + string "File name or PKCS#11 URI of module signing key" + default "certs/signing_key.pem" + depends on MODULE_SIG + help + Provide the file name of a private key/certificate in PEM format, + or a PKCS#11 URI according to RFC7512. The file should contain, or + the URI should identify, both the certificate and its corresponding + private key. + + If this option is unchanged from its default "certs/signing_key.pem", + then the kernel will automatically generate the private key and + certificate as described in Documentation/module-signing.txt + +config SYSTEM_TRUSTED_KEYRING + bool "Provide system-wide ring of trusted keys" + depends on KEYS + help + Provide a system keyring to which trusted keys can be added. Keys in + the keyring are considered to be trusted. Keys may be added at will + by the kernel from compiled-in data and from hardware key stores, but + userspace may only add extra keys if those keys can be verified by + keys already in the keyring. + + Keys in this keyring are used by module signature checking. + +config SYSTEM_TRUSTED_KEYS + string "Additional X.509 keys for default system keyring" + depends on SYSTEM_TRUSTED_KEYRING + help + If set, this option should be the filename of a PEM-formatted file + containing trusted X.509 certificates to be included in the default + system keyring. Any certificate used for module signing is implicitly + also trusted. + + NOTE: If you previously provided keys for the system keyring in the + form of DER-encoded *.x509 files in the top-level build directory, + those are no longer used. You will need to set this option instead. + +endmenu diff --git a/certs/Makefile b/certs/Makefile new file mode 100644 index 000000000000..28ac694dd11a --- /dev/null +++ b/certs/Makefile @@ -0,0 +1,94 @@ +# +# Makefile for the linux kernel signature checking certificates. +# + +obj-$(CONFIG_SYSTEM_TRUSTED_KEYRING) += system_keyring.o system_certificates.o + +ifeq ($(CONFIG_SYSTEM_TRUSTED_KEYRING),y) + +$(eval $(call config_filename,SYSTEM_TRUSTED_KEYS)) + +# GCC doesn't include .incbin files in -MD generated dependencies (PR#66871) +$(obj)/system_certificates.o: $(obj)/x509_certificate_list + +# Cope with signing_key.x509 existing in $(srctree) not $(objtree) +AFLAGS_system_certificates.o := -I$(srctree) + +quiet_cmd_extract_certs = EXTRACT_CERTS $(patsubst "%",%,$(2)) + cmd_extract_certs = scripts/extract-cert $(2) $@ || ( rm $@; exit 1) + +targets += x509_certificate_list +$(obj)/x509_certificate_list: scripts/extract-cert $(SYSTEM_TRUSTED_KEYS_SRCPREFIX)$(SYSTEM_TRUSTED_KEYS_FILENAME) FORCE + $(call if_changed,extract_certs,$(SYSTEM_TRUSTED_KEYS_SRCPREFIX)$(CONFIG_SYSTEM_TRUSTED_KEYS)) +endif + +clean-files := x509_certificate_list .x509.list + +ifeq ($(CONFIG_MODULE_SIG),y) +############################################################################### +# +# If module signing is requested, say by allyesconfig, but a key has not been +# supplied, then one will need to be generated to make sure the build does not +# fail and that the kernel may be used afterwards. +# +############################################################################### +ifndef CONFIG_MODULE_SIG_HASH +$(error Could not determine digest type to use from kernel config) +endif + +# We do it this way rather than having a boolean option for enabling an +# external private key, because 'make randconfig' might enable such a +# boolean option and we unfortunately can't make it depend on !RANDCONFIG. +ifeq ($(CONFIG_MODULE_SIG_KEY),"certs/signing_key.pem") +$(obj)/signing_key.pem: $(obj)/x509.genkey + @echo "###" + @echo "### Now generating an X.509 key pair to be used for signing modules." + @echo "###" + @echo "### If this takes a long time, you might wish to run rngd in the" + @echo "### background to keep the supply of entropy topped up. It" + @echo "### needs to be run as root, and uses a hardware random" + @echo "### number generator if one is available." + @echo "###" + openssl req -new -nodes -utf8 -$(CONFIG_MODULE_SIG_HASH) -days 36500 \ + -batch -x509 -config $(obj)/x509.genkey \ + -outform PEM -out $(obj)/signing_key.pem \ + -keyout $(obj)/signing_key.pem 2>&1 + @echo "###" + @echo "### Key pair generated." + @echo "###" + +$(obj)/x509.genkey: + @echo Generating X.509 key generation config + @echo >$@ "[ req ]" + @echo >>$@ "default_bits = 4096" + @echo >>$@ "distinguished_name = req_distinguished_name" + @echo >>$@ "prompt = no" + @echo >>$@ "string_mask = utf8only" + @echo >>$@ "x509_extensions = myexts" + @echo >>$@ + @echo >>$@ "[ req_distinguished_name ]" + @echo >>$@ "#O = Unspecified company" + @echo >>$@ "CN = Build time autogenerated kernel key" + @echo >>$@ "#emailAddress = unspecified.user@unspecified.company" + @echo >>$@ + @echo >>$@ "[ myexts ]" + @echo >>$@ "basicConstraints=critical,CA:FALSE" + @echo >>$@ "keyUsage=digitalSignature" + @echo >>$@ "subjectKeyIdentifier=hash" + @echo >>$@ "authorityKeyIdentifier=keyid" +endif + +$(eval $(call config_filename,MODULE_SIG_KEY)) + +# If CONFIG_MODULE_SIG_KEY isn't a PKCS#11 URI, depend on it +ifeq ($(patsubst pkcs11:%,%,$(firstword $(MODULE_SIG_KEY_FILENAME))),$(firstword $(MODULE_SIG_KEY_FILENAME))) +X509_DEP := $(MODULE_SIG_KEY_SRCPREFIX)$(MODULE_SIG_KEY_FILENAME) +endif + +# GCC PR#66871 again. +$(obj)/system_certificates.o: $(obj)/signing_key.x509 + +targets += signing_key.x509 +$(obj)/signing_key.x509: scripts/extract-cert $(X509_DEP) FORCE + $(call if_changed,extract_certs,$(MODULE_SIG_KEY_SRCPREFIX)$(CONFIG_MODULE_SIG_KEY)) +endif diff --git a/kernel/system_certificates.S b/certs/system_certificates.S similarity index 80% rename from kernel/system_certificates.S rename to certs/system_certificates.S index 3e9868d47535..9216e8c81764 100644 --- a/kernel/system_certificates.S +++ b/certs/system_certificates.S @@ -7,7 +7,10 @@ .globl VMLINUX_SYMBOL(system_certificate_list) VMLINUX_SYMBOL(system_certificate_list): __cert_list_start: - .incbin "kernel/x509_certificate_list" +#ifdef CONFIG_MODULE_SIG + .incbin "certs/signing_key.x509" +#endif + .incbin "certs/x509_certificate_list" __cert_list_end: .align 8 diff --git a/kernel/system_keyring.c b/certs/system_keyring.c similarity index 68% rename from kernel/system_keyring.c rename to certs/system_keyring.c index 875f64e8935b..2570598b784d 100644 --- a/kernel/system_keyring.c +++ b/certs/system_keyring.c @@ -16,7 +16,7 @@ #include #include #include -#include "module-internal.h" +#include struct key *system_trusted_keyring; EXPORT_SYMBOL_GPL(system_trusted_keyring); @@ -104,3 +104,54 @@ static __init int load_system_certificate_list(void) return 0; } late_initcall(load_system_certificate_list); + +#ifdef CONFIG_SYSTEM_DATA_VERIFICATION + +/** + * Verify a PKCS#7-based signature on system data. + * @data: The data to be verified. + * @len: Size of @data. + * @raw_pkcs7: The PKCS#7 message that is the signature. + * @pkcs7_len: The size of @raw_pkcs7. + * @usage: The use to which the key is being put. + */ +int system_verify_data(const void *data, unsigned long len, + const void *raw_pkcs7, size_t pkcs7_len, + enum key_being_used_for usage) +{ + struct pkcs7_message *pkcs7; + bool trusted; + int ret; + + pkcs7 = pkcs7_parse_message(raw_pkcs7, pkcs7_len); + if (IS_ERR(pkcs7)) + return PTR_ERR(pkcs7); + + /* The data should be detached - so we need to supply it. */ + if (pkcs7_supply_detached_data(pkcs7, data, len) < 0) { + pr_err("PKCS#7 signature with non-detached data\n"); + ret = -EBADMSG; + goto error; + } + + ret = pkcs7_verify(pkcs7, usage); + if (ret < 0) + goto error; + + ret = pkcs7_validate_trust(pkcs7, system_trusted_keyring, &trusted); + if (ret < 0) + goto error; + + if (!trusted) { + pr_err("PKCS#7 signature not signed with a trusted key\n"); + ret = -ENOKEY; + } + +error: + pkcs7_free_message(pkcs7); + pr_devel("<==%s() = %d\n", __func__, ret); + return ret; +} +EXPORT_SYMBOL_GPL(system_verify_data); + +#endif /* CONFIG_SYSTEM_DATA_VERIFICATION */ diff --git a/crypto/Kconfig b/crypto/Kconfig index b582ea7f78d3..48ee3e175dac 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1635,5 +1635,6 @@ config CRYPTO_HASH_INFO source "drivers/crypto/Kconfig" source crypto/asymmetric_keys/Kconfig +source certs/Kconfig endif # if CRYPTO diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c index 945075292bc9..af31a0ee4057 100644 --- a/crypto/algif_skcipher.c +++ b/crypto/algif_skcipher.c @@ -145,7 +145,7 @@ static int skcipher_alloc_sgl(struct sock *sk) sgl->cur = 0; if (sg) - scatterwalk_sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg); + sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg); list_add_tail(&sgl->list, &ctx->tsgl); } diff --git a/crypto/asymmetric_keys/Makefile b/crypto/asymmetric_keys/Makefile index e47fcd9ac5e8..cd1406f9b14a 100644 --- a/crypto/asymmetric_keys/Makefile +++ b/crypto/asymmetric_keys/Makefile @@ -15,15 +15,21 @@ obj-$(CONFIG_PUBLIC_KEY_ALGO_RSA) += rsa.o obj-$(CONFIG_X509_CERTIFICATE_PARSER) += x509_key_parser.o x509_key_parser-y := \ x509-asn1.o \ + x509_akid-asn1.o \ x509_rsakey-asn1.o \ x509_cert_parser.o \ x509_public_key.o -$(obj)/x509_cert_parser.o: $(obj)/x509-asn1.h $(obj)/x509_rsakey-asn1.h +$(obj)/x509_cert_parser.o: \ + $(obj)/x509-asn1.h \ + $(obj)/x509_akid-asn1.h \ + $(obj)/x509_rsakey-asn1.h $(obj)/x509-asn1.o: $(obj)/x509-asn1.c $(obj)/x509-asn1.h +$(obj)/x509_akid-asn1.o: $(obj)/x509_akid-asn1.c $(obj)/x509_akid-asn1.h $(obj)/x509_rsakey-asn1.o: $(obj)/x509_rsakey-asn1.c $(obj)/x509_rsakey-asn1.h clean-files += x509-asn1.c x509-asn1.h +clean-files += x509_akid-asn1.c x509_akid-asn1.h clean-files += x509_rsakey-asn1.c x509_rsakey-asn1.h # diff --git a/crypto/asymmetric_keys/asymmetric_type.c b/crypto/asymmetric_keys/asymmetric_type.c index b0e4ed23d668..1916680ad81b 100644 --- a/crypto/asymmetric_keys/asymmetric_type.c +++ b/crypto/asymmetric_keys/asymmetric_type.c @@ -12,6 +12,7 @@ */ #include #include +#include #include #include #include @@ -20,6 +21,16 @@ MODULE_LICENSE("GPL"); +const char *const key_being_used_for[NR__KEY_BEING_USED_FOR] = { + [VERIFYING_MODULE_SIGNATURE] = "mod sig", + [VERIFYING_FIRMWARE_SIGNATURE] = "firmware sig", + [VERIFYING_KEXEC_PE_SIGNATURE] = "kexec PE sig", + [VERIFYING_KEY_SIGNATURE] = "key sig", + [VERIFYING_KEY_SELF_SIGNATURE] = "key self sig", + [VERIFYING_UNSPECIFIED_SIGNATURE] = "unspec sig", +}; +EXPORT_SYMBOL_GPL(key_being_used_for); + static LIST_HEAD(asymmetric_key_parsers); static DECLARE_RWSEM(asymmetric_key_parsers_sem); diff --git a/crypto/asymmetric_keys/mscode_parser.c b/crypto/asymmetric_keys/mscode_parser.c index 214a992123cd..adcef59eec0b 100644 --- a/crypto/asymmetric_keys/mscode_parser.c +++ b/crypto/asymmetric_keys/mscode_parser.c @@ -97,6 +97,15 @@ int mscode_note_digest_algo(void *context, size_t hdrlen, case OID_sha256: ctx->digest_algo = HASH_ALGO_SHA256; break; + case OID_sha384: + ctx->digest_algo = HASH_ALGO_SHA384; + break; + case OID_sha512: + ctx->digest_algo = HASH_ALGO_SHA512; + break; + case OID_sha224: + ctx->digest_algo = HASH_ALGO_SHA224; + break; case OID__NR: sprint_oid(value, vlen, buffer, sizeof(buffer)); diff --git a/crypto/asymmetric_keys/pkcs7.asn1 b/crypto/asymmetric_keys/pkcs7.asn1 index a5a14ef28c86..1eca740b816a 100644 --- a/crypto/asymmetric_keys/pkcs7.asn1 +++ b/crypto/asymmetric_keys/pkcs7.asn1 @@ -1,14 +1,14 @@ PKCS7ContentInfo ::= SEQUENCE { - contentType ContentType, + contentType ContentType ({ pkcs7_check_content_type }), content [0] EXPLICIT SignedData OPTIONAL } ContentType ::= OBJECT IDENTIFIER ({ pkcs7_note_OID }) SignedData ::= SEQUENCE { - version INTEGER, + version INTEGER ({ pkcs7_note_signeddata_version }), digestAlgorithms DigestAlgorithmIdentifiers, - contentInfo ContentInfo, + contentInfo ContentInfo ({ pkcs7_note_content }), certificates CHOICE { certSet [0] IMPLICIT ExtendedCertificatesAndCertificates, certSequence [2] IMPLICIT Certificates @@ -21,7 +21,7 @@ SignedData ::= SEQUENCE { } ContentInfo ::= SEQUENCE { - contentType ContentType, + contentType ContentType ({ pkcs7_note_OID }), content [0] EXPLICIT Data OPTIONAL } @@ -68,8 +68,8 @@ SignerInfos ::= CHOICE { } SignerInfo ::= SEQUENCE { - version INTEGER, - issuerAndSerialNumber IssuerAndSerialNumber, + version INTEGER ({ pkcs7_note_signerinfo_version }), + sid SignerIdentifier, -- CMS variant, not PKCS#7 digestAlgorithm DigestAlgorithmIdentifier ({ pkcs7_sig_note_digest_algo }), authenticatedAttributes CHOICE { aaSet [0] IMPLICIT SetOfAuthenticatedAttribute @@ -88,6 +88,12 @@ SignerInfo ::= SEQUENCE { } OPTIONAL } ({ pkcs7_note_signed_info }) +SignerIdentifier ::= CHOICE { + -- RFC5652 sec 5.3 + issuerAndSerialNumber IssuerAndSerialNumber, + subjectKeyIdentifier [0] IMPLICIT SubjectKeyIdentifier +} + IssuerAndSerialNumber ::= SEQUENCE { issuer Name ({ pkcs7_sig_note_issuer }), serialNumber CertificateSerialNumber ({ pkcs7_sig_note_serial }) @@ -95,6 +101,8 @@ IssuerAndSerialNumber ::= SEQUENCE { CertificateSerialNumber ::= INTEGER +SubjectKeyIdentifier ::= OCTET STRING ({ pkcs7_sig_note_skid }) + SetOfAuthenticatedAttribute ::= SET OF AuthenticatedAttribute AuthenticatedAttribute ::= SEQUENCE { @@ -103,7 +111,7 @@ AuthenticatedAttribute ::= SEQUENCE { } UnauthenticatedAttribute ::= SEQUENCE { - type OBJECT IDENTIFIER ({ pkcs7_note_OID }), + type OBJECT IDENTIFIER, values SET OF ANY } diff --git a/crypto/asymmetric_keys/pkcs7_key_type.c b/crypto/asymmetric_keys/pkcs7_key_type.c index 3d13b042da73..e2d0edbbc71a 100644 --- a/crypto/asymmetric_keys/pkcs7_key_type.c +++ b/crypto/asymmetric_keys/pkcs7_key_type.c @@ -14,16 +14,26 @@ #include #include #include +#include #include #include #include #include "pkcs7_parser.h" +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("PKCS#7 testing key type"); + +static unsigned pkcs7_usage; +module_param_named(usage, pkcs7_usage, uint, S_IWUSR | S_IRUGO); +MODULE_PARM_DESC(pkcs7_usage, + "Usage to specify when verifying the PKCS#7 message"); + /* * Preparse a PKCS#7 wrapped and validated data blob. */ static int pkcs7_preparse(struct key_preparsed_payload *prep) { + enum key_being_used_for usage = pkcs7_usage; struct pkcs7_message *pkcs7; const void *data, *saved_prep_data; size_t datalen, saved_prep_datalen; @@ -32,6 +42,11 @@ static int pkcs7_preparse(struct key_preparsed_payload *prep) kenter(""); + if (usage >= NR__KEY_BEING_USED_FOR) { + pr_err("Invalid usage type %d\n", usage); + return -EINVAL; + } + saved_prep_data = prep->data; saved_prep_datalen = prep->datalen; pkcs7 = pkcs7_parse_message(saved_prep_data, saved_prep_datalen); @@ -40,7 +55,7 @@ static int pkcs7_preparse(struct key_preparsed_payload *prep) goto error; } - ret = pkcs7_verify(pkcs7); + ret = pkcs7_verify(pkcs7, usage); if (ret < 0) goto error_free; diff --git a/crypto/asymmetric_keys/pkcs7_parser.c b/crypto/asymmetric_keys/pkcs7_parser.c index 3bd5a1e4c493..758acabf2d81 100644 --- a/crypto/asymmetric_keys/pkcs7_parser.c +++ b/crypto/asymmetric_keys/pkcs7_parser.c @@ -33,6 +33,9 @@ struct pkcs7_parse_context { unsigned raw_serial_size; unsigned raw_issuer_size; const void *raw_issuer; + const void *raw_skid; + unsigned raw_skid_size; + bool expect_skid; }; /* @@ -78,6 +81,30 @@ void pkcs7_free_message(struct pkcs7_message *pkcs7) } EXPORT_SYMBOL_GPL(pkcs7_free_message); +/* + * Check authenticatedAttributes are provided or not provided consistently. + */ +static int pkcs7_check_authattrs(struct pkcs7_message *msg) +{ + struct pkcs7_signed_info *sinfo; + bool want; + + sinfo = msg->signed_infos; + if (sinfo->authattrs) { + want = true; + msg->have_authattrs = true; + } + + for (sinfo = sinfo->next; sinfo; sinfo = sinfo->next) + if (!!sinfo->authattrs != want) + goto inconsistent; + return 0; + +inconsistent: + pr_warn("Inconsistently supplied authAttrs\n"); + return -EINVAL; +} + /** * pkcs7_parse_message - Parse a PKCS#7 message * @data: The raw binary ASN.1 encoded message to be parsed @@ -110,6 +137,10 @@ struct pkcs7_message *pkcs7_parse_message(const void *data, size_t datalen) goto out; } + ret = pkcs7_check_authattrs(ctx->msg); + if (ret < 0) + goto out; + msg = ctx->msg; ctx->msg = NULL; @@ -198,6 +229,14 @@ int pkcs7_sig_note_digest_algo(void *context, size_t hdrlen, case OID_sha256: ctx->sinfo->sig.pkey_hash_algo = HASH_ALGO_SHA256; break; + case OID_sha384: + ctx->sinfo->sig.pkey_hash_algo = HASH_ALGO_SHA384; + break; + case OID_sha512: + ctx->sinfo->sig.pkey_hash_algo = HASH_ALGO_SHA512; + break; + case OID_sha224: + ctx->sinfo->sig.pkey_hash_algo = HASH_ALGO_SHA224; default: printk("Unsupported digest algo: %u\n", ctx->last_oid); return -ENOPKG; @@ -225,6 +264,100 @@ int pkcs7_sig_note_pkey_algo(void *context, size_t hdrlen, return 0; } +/* + * We only support signed data [RFC2315 sec 9]. + */ +int pkcs7_check_content_type(void *context, size_t hdrlen, + unsigned char tag, + const void *value, size_t vlen) +{ + struct pkcs7_parse_context *ctx = context; + + if (ctx->last_oid != OID_signed_data) { + pr_warn("Only support pkcs7_signedData type\n"); + return -EINVAL; + } + + return 0; +} + +/* + * Note the SignedData version + */ +int pkcs7_note_signeddata_version(void *context, size_t hdrlen, + unsigned char tag, + const void *value, size_t vlen) +{ + struct pkcs7_parse_context *ctx = context; + unsigned version; + + if (vlen != 1) + goto unsupported; + + ctx->msg->version = version = *(const u8 *)value; + switch (version) { + case 1: + /* PKCS#7 SignedData [RFC2315 sec 9.1] + * CMS ver 1 SignedData [RFC5652 sec 5.1] + */ + break; + case 3: + /* CMS ver 3 SignedData [RFC2315 sec 5.1] */ + break; + default: + goto unsupported; + } + + return 0; + +unsupported: + pr_warn("Unsupported SignedData version\n"); + return -EINVAL; +} + +/* + * Note the SignerInfo version + */ +int pkcs7_note_signerinfo_version(void *context, size_t hdrlen, + unsigned char tag, + const void *value, size_t vlen) +{ + struct pkcs7_parse_context *ctx = context; + unsigned version; + + if (vlen != 1) + goto unsupported; + + version = *(const u8 *)value; + switch (version) { + case 1: + /* PKCS#7 SignerInfo [RFC2315 sec 9.2] + * CMS ver 1 SignerInfo [RFC5652 sec 5.3] + */ + if (ctx->msg->version != 1) + goto version_mismatch; + ctx->expect_skid = false; + break; + case 3: + /* CMS ver 3 SignerInfo [RFC2315 sec 5.3] */ + if (ctx->msg->version == 1) + goto version_mismatch; + ctx->expect_skid = true; + break; + default: + goto unsupported; + } + + return 0; + +unsupported: + pr_warn("Unsupported SignerInfo version\n"); + return -EINVAL; +version_mismatch: + pr_warn("SignedData-SignerInfo version mismatch\n"); + return -EBADMSG; +} + /* * Extract a certificate and store it in the context. */ @@ -283,6 +416,25 @@ int pkcs7_note_certificate_list(void *context, size_t hdrlen, return 0; } +/* + * Note the content type. + */ +int pkcs7_note_content(void *context, size_t hdrlen, + unsigned char tag, + const void *value, size_t vlen) +{ + struct pkcs7_parse_context *ctx = context; + + if (ctx->last_oid != OID_data && + ctx->last_oid != OID_msIndirectData) { + pr_warn("Unsupported data type %d\n", ctx->last_oid); + return -EINVAL; + } + + ctx->msg->data_type = ctx->last_oid; + return 0; +} + /* * Extract the data from the message and store that and its content type OID in * the context. @@ -298,45 +450,119 @@ int pkcs7_note_data(void *context, size_t hdrlen, ctx->msg->data = value; ctx->msg->data_len = vlen; ctx->msg->data_hdrlen = hdrlen; - ctx->msg->data_type = ctx->last_oid; return 0; } /* - * Parse authenticated attributes + * Parse authenticated attributes. */ int pkcs7_sig_note_authenticated_attr(void *context, size_t hdrlen, unsigned char tag, const void *value, size_t vlen) { struct pkcs7_parse_context *ctx = context; + struct pkcs7_signed_info *sinfo = ctx->sinfo; + enum OID content_type; pr_devel("AuthAttr: %02x %zu [%*ph]\n", tag, vlen, (unsigned)vlen, value); switch (ctx->last_oid) { + case OID_contentType: + if (__test_and_set_bit(sinfo_has_content_type, &sinfo->aa_set)) + goto repeated; + content_type = look_up_OID(value, vlen); + if (content_type != ctx->msg->data_type) { + pr_warn("Mismatch between global data type (%d) and sinfo %u (%d)\n", + ctx->msg->data_type, sinfo->index, + content_type); + return -EBADMSG; + } + return 0; + + case OID_signingTime: + if (__test_and_set_bit(sinfo_has_signing_time, &sinfo->aa_set)) + goto repeated; + /* Should we check that the signing time is consistent + * with the signer's X.509 cert? + */ + return x509_decode_time(&sinfo->signing_time, + hdrlen, tag, value, vlen); + case OID_messageDigest: + if (__test_and_set_bit(sinfo_has_message_digest, &sinfo->aa_set)) + goto repeated; if (tag != ASN1_OTS) return -EBADMSG; - ctx->sinfo->msgdigest = value; - ctx->sinfo->msgdigest_len = vlen; + sinfo->msgdigest = value; + sinfo->msgdigest_len = vlen; + return 0; + + case OID_smimeCapabilites: + if (__test_and_set_bit(sinfo_has_smime_caps, &sinfo->aa_set)) + goto repeated; + if (ctx->msg->data_type != OID_msIndirectData) { + pr_warn("S/MIME Caps only allowed with Authenticode\n"); + return -EKEYREJECTED; + } + return 0; + + /* Microsoft SpOpusInfo seems to be contain cont[0] 16-bit BE + * char URLs and cont[1] 8-bit char URLs. + * + * Microsoft StatementType seems to contain a list of OIDs that + * are also used as extendedKeyUsage types in X.509 certs. + */ + case OID_msSpOpusInfo: + if (__test_and_set_bit(sinfo_has_ms_opus_info, &sinfo->aa_set)) + goto repeated; + goto authenticode_check; + case OID_msStatementType: + if (__test_and_set_bit(sinfo_has_ms_statement_type, &sinfo->aa_set)) + goto repeated; + authenticode_check: + if (ctx->msg->data_type != OID_msIndirectData) { + pr_warn("Authenticode AuthAttrs only allowed with Authenticode\n"); + return -EKEYREJECTED; + } + /* I'm not sure how to validate these */ return 0; default: return 0; } + +repeated: + /* We permit max one item per AuthenticatedAttribute and no repeats */ + pr_warn("Repeated/multivalue AuthAttrs not permitted\n"); + return -EKEYREJECTED; } /* - * Note the set of auth attributes for digestion purposes [RFC2315 9.3] + * Note the set of auth attributes for digestion purposes [RFC2315 sec 9.3] */ int pkcs7_sig_note_set_of_authattrs(void *context, size_t hdrlen, unsigned char tag, const void *value, size_t vlen) { struct pkcs7_parse_context *ctx = context; + struct pkcs7_signed_info *sinfo = ctx->sinfo; + + if (!test_bit(sinfo_has_content_type, &sinfo->aa_set) || + !test_bit(sinfo_has_message_digest, &sinfo->aa_set) || + (ctx->msg->data_type == OID_msIndirectData && + !test_bit(sinfo_has_ms_opus_info, &sinfo->aa_set))) { + pr_warn("Missing required AuthAttr\n"); + return -EBADMSG; + } + + if (ctx->msg->data_type != OID_msIndirectData && + test_bit(sinfo_has_ms_opus_info, &sinfo->aa_set)) { + pr_warn("Unexpected Authenticode AuthAttr\n"); + return -EBADMSG; + } /* We need to switch the 'CONT 0' to a 'SET OF' when we digest */ - ctx->sinfo->authattrs = value - (hdrlen - 1); - ctx->sinfo->authattrs_len = vlen + (hdrlen - 1); + sinfo->authattrs = value - (hdrlen - 1); + sinfo->authattrs_len = vlen + (hdrlen - 1); return 0; } @@ -366,6 +592,22 @@ int pkcs7_sig_note_issuer(void *context, size_t hdrlen, return 0; } +/* + * Note the issuing cert's subjectKeyIdentifier + */ +int pkcs7_sig_note_skid(void *context, size_t hdrlen, + unsigned char tag, + const void *value, size_t vlen) +{ + struct pkcs7_parse_context *ctx = context; + + pr_devel("SKID: %02x %zu [%*ph]\n", tag, vlen, (unsigned)vlen, value); + + ctx->raw_skid = value; + ctx->raw_skid_size = vlen; + return 0; +} + /* * Note the signature data */ @@ -398,14 +640,27 @@ int pkcs7_note_signed_info(void *context, size_t hdrlen, struct pkcs7_signed_info *sinfo = ctx->sinfo; struct asymmetric_key_id *kid; + if (ctx->msg->data_type == OID_msIndirectData && !sinfo->authattrs) { + pr_warn("Authenticode requires AuthAttrs\n"); + return -EBADMSG; + } + /* Generate cert issuer + serial number key ID */ - kid = asymmetric_key_generate_id(ctx->raw_serial, - ctx->raw_serial_size, - ctx->raw_issuer, - ctx->raw_issuer_size); + if (!ctx->expect_skid) { + kid = asymmetric_key_generate_id(ctx->raw_serial, + ctx->raw_serial_size, + ctx->raw_issuer, + ctx->raw_issuer_size); + } else { + kid = asymmetric_key_generate_id(ctx->raw_skid, + ctx->raw_skid_size, + "", 0); + } if (IS_ERR(kid)) return PTR_ERR(kid); + pr_devel("SINFO KID: %u [%*phN]\n", kid->len, kid->len, kid->data); + sinfo->signing_cert_id = kid; sinfo->index = ++ctx->sinfo_index; *ctx->ppsinfo = sinfo; diff --git a/crypto/asymmetric_keys/pkcs7_parser.h b/crypto/asymmetric_keys/pkcs7_parser.h index efc7dc9b8f9c..a66b19ebcf47 100644 --- a/crypto/asymmetric_keys/pkcs7_parser.h +++ b/crypto/asymmetric_keys/pkcs7_parser.h @@ -21,9 +21,9 @@ struct pkcs7_signed_info { struct pkcs7_signed_info *next; struct x509_certificate *signer; /* Signing certificate (in msg->certs) */ - unsigned index; - bool trusted; - bool unsupported_crypto; /* T if not usable due to missing crypto */ + unsigned index; + bool trusted; + bool unsupported_crypto; /* T if not usable due to missing crypto */ /* Message digest - the digest of the Content Data (or NULL) */ const void *msgdigest; @@ -32,8 +32,18 @@ struct pkcs7_signed_info { /* Authenticated Attribute data (or NULL) */ unsigned authattrs_len; const void *authattrs; + unsigned long aa_set; +#define sinfo_has_content_type 0 +#define sinfo_has_signing_time 1 +#define sinfo_has_message_digest 2 +#define sinfo_has_smime_caps 3 +#define sinfo_has_ms_opus_info 4 +#define sinfo_has_ms_statement_type 5 + time64_t signing_time; - /* Issuing cert serial number and issuer's name */ + /* Issuing cert serial number and issuer's name [PKCS#7 or CMS ver 1] + * or issuing cert's SKID [CMS ver 3]. + */ struct asymmetric_key_id *signing_cert_id; /* Message signature. @@ -50,6 +60,8 @@ struct pkcs7_message { struct x509_certificate *certs; /* Certificate list */ struct x509_certificate *crl; /* Revocation list */ struct pkcs7_signed_info *signed_infos; + u8 version; /* Version of cert (1 -> PKCS#7 or CMS; 3 -> CMS) */ + bool have_authattrs; /* T if have authattrs */ /* Content Data (or NULL) */ enum OID data_type; /* Type of Data */ diff --git a/crypto/asymmetric_keys/pkcs7_trust.c b/crypto/asymmetric_keys/pkcs7_trust.c index 1d29376072da..90d6d47965b0 100644 --- a/crypto/asymmetric_keys/pkcs7_trust.c +++ b/crypto/asymmetric_keys/pkcs7_trust.c @@ -54,7 +54,8 @@ static int pkcs7_validate_trust_one(struct pkcs7_message *pkcs7, /* Look to see if this certificate is present in the trusted * keys. */ - key = x509_request_asymmetric_key(trust_keyring, x509->id, + key = x509_request_asymmetric_key(trust_keyring, + x509->id, x509->skid, false); if (!IS_ERR(key)) { /* One of the X.509 certificates in the PKCS#7 message @@ -85,8 +86,10 @@ static int pkcs7_validate_trust_one(struct pkcs7_message *pkcs7, /* No match - see if the root certificate has a signer amongst the * trusted keys. */ - if (last && last->authority) { - key = x509_request_asymmetric_key(trust_keyring, last->authority, + if (last && (last->akid_id || last->akid_skid)) { + key = x509_request_asymmetric_key(trust_keyring, + last->akid_id, + last->akid_skid, false); if (!IS_ERR(key)) { x509 = last; @@ -103,6 +106,7 @@ static int pkcs7_validate_trust_one(struct pkcs7_message *pkcs7, */ key = x509_request_asymmetric_key(trust_keyring, sinfo->signing_cert_id, + NULL, false); if (!IS_ERR(key)) { pr_devel("sinfo %u: Direct signer is key %x\n", diff --git a/crypto/asymmetric_keys/pkcs7_verify.c b/crypto/asymmetric_keys/pkcs7_verify.c index cd455450b069..d20c0b4b880e 100644 --- a/crypto/asymmetric_keys/pkcs7_verify.c +++ b/crypto/asymmetric_keys/pkcs7_verify.c @@ -70,9 +70,15 @@ static int pkcs7_digest(struct pkcs7_message *pkcs7, * message digest attribute amongst them which corresponds to the * digest we just calculated. */ - if (sinfo->msgdigest) { + if (sinfo->authattrs) { u8 tag; + if (!sinfo->msgdigest) { + pr_warn("Sig %u: No messageDigest\n", sinfo->index); + ret = -EKEYREJECTED; + goto error; + } + if (sinfo->msgdigest_len != sinfo->sig.digest_size) { pr_debug("Sig %u: Invalid digest size (%u)\n", sinfo->index, sinfo->msgdigest_len); @@ -170,6 +176,7 @@ static int pkcs7_verify_sig_chain(struct pkcs7_message *pkcs7, struct pkcs7_signed_info *sinfo) { struct x509_certificate *x509 = sinfo->signer, *p; + struct asymmetric_key_id *auth; int ret; kenter(""); @@ -187,11 +194,14 @@ static int pkcs7_verify_sig_chain(struct pkcs7_message *pkcs7, goto maybe_missing_crypto_in_x509; pr_debug("- issuer %s\n", x509->issuer); - if (x509->authority) - pr_debug("- authkeyid %*phN\n", - x509->authority->len, x509->authority->data); + if (x509->akid_id) + pr_debug("- authkeyid.id %*phN\n", + x509->akid_id->len, x509->akid_id->data); + if (x509->akid_skid) + pr_debug("- authkeyid.skid %*phN\n", + x509->akid_skid->len, x509->akid_skid->data); - if (!x509->authority || + if ((!x509->akid_id && !x509->akid_skid) || strcmp(x509->subject, x509->issuer) == 0) { /* If there's no authority certificate specified, then * the certificate must be self-signed and is the root @@ -215,21 +225,42 @@ static int pkcs7_verify_sig_chain(struct pkcs7_message *pkcs7, /* Look through the X.509 certificates in the PKCS#7 message's * list to see if the next one is there. */ - pr_debug("- want %*phN\n", - x509->authority->len, x509->authority->data); - for (p = pkcs7->certs; p; p = p->next) { - if (!p->skid) - continue; - pr_debug("- cmp [%u] %*phN\n", - p->index, p->skid->len, p->skid->data); - if (asymmetric_key_id_same(p->skid, x509->authority)) - goto found_issuer; + auth = x509->akid_id; + if (auth) { + pr_debug("- want %*phN\n", auth->len, auth->data); + for (p = pkcs7->certs; p; p = p->next) { + pr_debug("- cmp [%u] %*phN\n", + p->index, p->id->len, p->id->data); + if (asymmetric_key_id_same(p->id, auth)) + goto found_issuer_check_skid; + } + } else { + auth = x509->akid_skid; + pr_debug("- want %*phN\n", auth->len, auth->data); + for (p = pkcs7->certs; p; p = p->next) { + if (!p->skid) + continue; + pr_debug("- cmp [%u] %*phN\n", + p->index, p->skid->len, p->skid->data); + if (asymmetric_key_id_same(p->skid, auth)) + goto found_issuer; + } } /* We didn't find the root of this chain */ pr_debug("- top\n"); return 0; + found_issuer_check_skid: + /* We matched issuer + serialNumber, but if there's an + * authKeyId.keyId, that must match the CA subjKeyId also. + */ + if (x509->akid_skid && + !asymmetric_key_id_same(p->skid, x509->akid_skid)) { + pr_warn("Sig %u: X.509 chain contains auth-skid nonmatch (%u->%u)\n", + sinfo->index, x509->index, p->index); + return -EKEYREJECTED; + } found_issuer: pr_debug("- subject %s\n", p->subject); if (p->seen) { @@ -289,6 +320,18 @@ static int pkcs7_verify_one(struct pkcs7_message *pkcs7, pr_devel("Using X.509[%u] for sig %u\n", sinfo->signer->index, sinfo->index); + /* Check that the PKCS#7 signing time is valid according to the X.509 + * certificate. We can't, however, check against the system clock + * since that may not have been set yet and may be wrong. + */ + if (test_bit(sinfo_has_signing_time, &sinfo->aa_set)) { + if (sinfo->signing_time < sinfo->signer->valid_from || + sinfo->signing_time > sinfo->signer->valid_to) { + pr_warn("Message signed outside of X.509 validity window\n"); + return -EKEYREJECTED; + } + } + /* Verify the PKCS#7 binary against the key */ ret = public_key_verify_signature(sinfo->signer->pub, &sinfo->sig); if (ret < 0) @@ -303,6 +346,7 @@ static int pkcs7_verify_one(struct pkcs7_message *pkcs7, /** * pkcs7_verify - Verify a PKCS#7 message * @pkcs7: The PKCS#7 message to be verified + * @usage: The use to which the key is being put * * Verify a PKCS#7 message is internally consistent - that is, the data digest * matches the digest in the AuthAttrs and any signature in the message or one @@ -314,6 +358,9 @@ static int pkcs7_verify_one(struct pkcs7_message *pkcs7, * * Returns, in order of descending priority: * + * (*) -EKEYREJECTED if a key was selected that had a usage restriction at + * odds with the specified usage, or: + * * (*) -EKEYREJECTED if a signature failed to match for which we found an * appropriate X.509 certificate, or: * @@ -325,7 +372,8 @@ static int pkcs7_verify_one(struct pkcs7_message *pkcs7, * (*) 0 if all the signature chains that don't incur -ENOPKG can be verified * (note that a signature chain may be of zero length), or: */ -int pkcs7_verify(struct pkcs7_message *pkcs7) +int pkcs7_verify(struct pkcs7_message *pkcs7, + enum key_being_used_for usage) { struct pkcs7_signed_info *sinfo; struct x509_certificate *x509; @@ -334,12 +382,48 @@ int pkcs7_verify(struct pkcs7_message *pkcs7) kenter(""); + switch (usage) { + case VERIFYING_MODULE_SIGNATURE: + if (pkcs7->data_type != OID_data) { + pr_warn("Invalid module sig (not pkcs7-data)\n"); + return -EKEYREJECTED; + } + if (pkcs7->have_authattrs) { + pr_warn("Invalid module sig (has authattrs)\n"); + return -EKEYREJECTED; + } + break; + case VERIFYING_FIRMWARE_SIGNATURE: + if (pkcs7->data_type != OID_data) { + pr_warn("Invalid firmware sig (not pkcs7-data)\n"); + return -EKEYREJECTED; + } + if (!pkcs7->have_authattrs) { + pr_warn("Invalid firmware sig (missing authattrs)\n"); + return -EKEYREJECTED; + } + break; + case VERIFYING_KEXEC_PE_SIGNATURE: + if (pkcs7->data_type != OID_msIndirectData) { + pr_warn("Invalid kexec sig (not Authenticode)\n"); + return -EKEYREJECTED; + } + /* Authattr presence checked in parser */ + break; + case VERIFYING_UNSPECIFIED_SIGNATURE: + if (pkcs7->data_type != OID_data) { + pr_warn("Invalid unspecified sig (not pkcs7-data)\n"); + return -EKEYREJECTED; + } + break; + default: + return -EINVAL; + } + for (n = 0, x509 = pkcs7->certs; x509; x509 = x509->next, n++) { ret = x509_get_sig_params(x509); if (ret < 0) return ret; - pr_debug("X.509[%u] %*phN\n", - n, x509->authority->len, x509->authority->data); } for (sinfo = pkcs7->signed_infos; sinfo; sinfo = sinfo->next) { @@ -359,3 +443,28 @@ int pkcs7_verify(struct pkcs7_message *pkcs7) return enopkg; } EXPORT_SYMBOL_GPL(pkcs7_verify); + +/** + * pkcs7_supply_detached_data - Supply the data needed to verify a PKCS#7 message + * @pkcs7: The PKCS#7 message + * @data: The data to be verified + * @datalen: The amount of data + * + * Supply the detached data needed to verify a PKCS#7 message. Note that no + * attempt to retain/pin the data is made. That is left to the caller. The + * data will not be modified by pkcs7_verify() and will not be freed when the + * PKCS#7 message is freed. + * + * Returns -EINVAL if data is already supplied in the message, 0 otherwise. + */ +int pkcs7_supply_detached_data(struct pkcs7_message *pkcs7, + const void *data, size_t datalen) +{ + if (pkcs7->data) { + pr_debug("Data already supplied\n"); + return -EINVAL; + } + pkcs7->data = data; + pkcs7->data_len = datalen; + return 0; +} diff --git a/crypto/asymmetric_keys/public_key.c b/crypto/asymmetric_keys/public_key.c index 2f6e4fb1a1ea..81efccbe22d5 100644 --- a/crypto/asymmetric_keys/public_key.c +++ b/crypto/asymmetric_keys/public_key.c @@ -39,6 +39,7 @@ EXPORT_SYMBOL_GPL(pkey_algo); const char *const pkey_id_type_name[PKEY_ID_TYPE__LAST] = { [PKEY_ID_PGP] = "PGP", [PKEY_ID_X509] = "X509", + [PKEY_ID_PKCS7] = "PKCS#7", }; EXPORT_SYMBOL_GPL(pkey_id_type_name); diff --git a/crypto/asymmetric_keys/verify_pefile.c b/crypto/asymmetric_keys/verify_pefile.c index 2421f46184ce..897b734dabf9 100644 --- a/crypto/asymmetric_keys/verify_pefile.c +++ b/crypto/asymmetric_keys/verify_pefile.c @@ -393,6 +393,7 @@ static int pefile_digest_pe(const void *pebuf, unsigned int pelen, * @pebuf: Buffer containing the PE binary image * @pelen: Length of the binary image * @trust_keyring: Signing certificates to use as starting points + * @usage: The use to which the key is being put. * @_trusted: Set to true if trustworth, false otherwise * * Validate that the certificate chain inside the PKCS#7 message inside the PE @@ -417,7 +418,9 @@ static int pefile_digest_pe(const void *pebuf, unsigned int pelen, * May also return -ENOMEM. */ int verify_pefile_signature(const void *pebuf, unsigned pelen, - struct key *trusted_keyring, bool *_trusted) + struct key *trusted_keyring, + enum key_being_used_for usage, + bool *_trusted) { struct pkcs7_message *pkcs7; struct pefile_context ctx; @@ -462,7 +465,7 @@ int verify_pefile_signature(const void *pebuf, unsigned pelen, if (ret < 0) goto error; - ret = pkcs7_verify(pkcs7); + ret = pkcs7_verify(pkcs7, usage); if (ret < 0) goto error; diff --git a/crypto/asymmetric_keys/x509_akid.asn1 b/crypto/asymmetric_keys/x509_akid.asn1 new file mode 100644 index 000000000000..1a33231a75a8 --- /dev/null +++ b/crypto/asymmetric_keys/x509_akid.asn1 @@ -0,0 +1,35 @@ +-- X.509 AuthorityKeyIdentifier +-- rfc5280 section 4.2.1.1 + +AuthorityKeyIdentifier ::= SEQUENCE { + keyIdentifier [0] IMPLICIT KeyIdentifier OPTIONAL, + authorityCertIssuer [1] IMPLICIT GeneralNames OPTIONAL, + authorityCertSerialNumber [2] IMPLICIT CertificateSerialNumber OPTIONAL + } + +KeyIdentifier ::= OCTET STRING ({ x509_akid_note_kid }) + +CertificateSerialNumber ::= INTEGER ({ x509_akid_note_serial }) + +GeneralNames ::= SEQUENCE OF GeneralName + +GeneralName ::= CHOICE { + otherName [0] ANY, + rfc822Name [1] IA5String, + dNSName [2] IA5String, + x400Address [3] ANY, + directoryName [4] Name ({ x509_akid_note_name }), + ediPartyName [5] ANY, + uniformResourceIdentifier [6] IA5String, + iPAddress [7] OCTET STRING, + registeredID [8] OBJECT IDENTIFIER + } + +Name ::= SEQUENCE OF RelativeDistinguishedName + +RelativeDistinguishedName ::= SET OF AttributeValueAssertion + +AttributeValueAssertion ::= SEQUENCE { + attributeType OBJECT IDENTIFIER ({ x509_note_OID }), + attributeValue ANY ({ x509_extract_name_segment }) + } diff --git a/crypto/asymmetric_keys/x509_cert_parser.c b/crypto/asymmetric_keys/x509_cert_parser.c index a668d90302d3..af71878dc15b 100644 --- a/crypto/asymmetric_keys/x509_cert_parser.c +++ b/crypto/asymmetric_keys/x509_cert_parser.c @@ -18,6 +18,7 @@ #include "public_key.h" #include "x509_parser.h" #include "x509-asn1.h" +#include "x509_akid-asn1.h" #include "x509_rsakey-asn1.h" struct x509_parse_context { @@ -35,6 +36,10 @@ struct x509_parse_context { u16 o_offset; /* Offset of organizationName (O) */ u16 cn_offset; /* Offset of commonName (CN) */ u16 email_offset; /* Offset of emailAddress */ + unsigned raw_akid_size; + const void *raw_akid; /* Raw authorityKeyId in ASN.1 */ + const void *akid_raw_issuer; /* Raw directoryName in authorityKeyId */ + unsigned akid_raw_issuer_size; }; /* @@ -48,7 +53,8 @@ void x509_free_certificate(struct x509_certificate *cert) kfree(cert->subject); kfree(cert->id); kfree(cert->skid); - kfree(cert->authority); + kfree(cert->akid_id); + kfree(cert->akid_skid); kfree(cert->sig.digest); mpi_free(cert->sig.rsa.s); kfree(cert); @@ -85,6 +91,18 @@ struct x509_certificate *x509_cert_parse(const void *data, size_t datalen) if (ret < 0) goto error_decode; + /* Decode the AuthorityKeyIdentifier */ + if (ctx->raw_akid) { + pr_devel("AKID: %u %*phN\n", + ctx->raw_akid_size, ctx->raw_akid_size, ctx->raw_akid); + ret = asn1_ber_decoder(&x509_akid_decoder, ctx, + ctx->raw_akid, ctx->raw_akid_size); + if (ret < 0) { + pr_warn("Couldn't decode AuthKeyIdentifier\n"); + goto error_decode; + } + } + /* Decode the public key */ ret = asn1_ber_decoder(&x509_rsakey_decoder, ctx, ctx->key, ctx->key_size); @@ -422,7 +440,6 @@ int x509_process_extension(void *context, size_t hdrlen, struct x509_parse_context *ctx = context; struct asymmetric_key_id *kid; const unsigned char *v = value; - int i; pr_debug("Extension: %u\n", ctx->last_oid); @@ -437,9 +454,7 @@ int x509_process_extension(void *context, size_t hdrlen, ctx->cert->raw_skid_size = vlen; ctx->cert->raw_skid = v; - kid = asymmetric_key_generate_id(ctx->cert->raw_subject, - ctx->cert->raw_subject_size, - v, vlen); + kid = asymmetric_key_generate_id(v, vlen, "", 0); if (IS_ERR(kid)) return PTR_ERR(kid); ctx->cert->skid = kid; @@ -449,117 +464,113 @@ int x509_process_extension(void *context, size_t hdrlen, if (ctx->last_oid == OID_authorityKeyIdentifier) { /* Get hold of the CA key fingerprint */ - if (ctx->cert->authority || vlen < 5) - return -EBADMSG; - - /* Authority Key Identifier must be a Constructed SEQUENCE */ - if (v[0] != (ASN1_SEQ | (ASN1_CONS << 5))) - return -EBADMSG; - - /* Authority Key Identifier is not indefinite length */ - if (unlikely(vlen == ASN1_INDEFINITE_LENGTH)) - return -EBADMSG; - - if (vlen < ASN1_INDEFINITE_LENGTH) { - /* Short Form length */ - if (v[1] != vlen - 2 || - v[2] != SEQ_TAG_KEYID || - v[3] > vlen - 4) - return -EBADMSG; - - vlen = v[3]; - v += 4; - } else { - /* Long Form length */ - size_t seq_len = 0; - size_t sub = v[1] - ASN1_INDEFINITE_LENGTH; - - if (sub > 2) - return -EBADMSG; - - /* calculate the length from subsequent octets */ - v += 2; - for (i = 0; i < sub; i++) { - seq_len <<= 8; - seq_len |= v[i]; - } - - if (seq_len != vlen - 2 - sub || - v[sub] != SEQ_TAG_KEYID || - v[sub + 1] > vlen - 4 - sub) - return -EBADMSG; - - vlen = v[sub + 1]; - v += (sub + 2); - } - - kid = asymmetric_key_generate_id(ctx->cert->raw_issuer, - ctx->cert->raw_issuer_size, - v, vlen); - if (IS_ERR(kid)) - return PTR_ERR(kid); - pr_debug("authkeyid %*phN\n", kid->len, kid->data); - ctx->cert->authority = kid; + ctx->raw_akid = v; + ctx->raw_akid_size = vlen; return 0; } return 0; } -/* - * Record a certificate time. +/** + * x509_decode_time - Decode an X.509 time ASN.1 object + * @_t: The time to fill in + * @hdrlen: The length of the object header + * @tag: The object tag + * @value: The object value + * @vlen: The size of the object value + * + * Decode an ASN.1 universal time or generalised time field into a struct the + * kernel can handle and check it for validity. The time is decoded thus: + * + * [RFC5280 §4.1.2.5] + * CAs conforming to this profile MUST always encode certificate validity + * dates through the year 2049 as UTCTime; certificate validity dates in + * 2050 or later MUST be encoded as GeneralizedTime. Conforming + * applications MUST be able to process validity dates that are encoded in + * either UTCTime or GeneralizedTime. */ -static int x509_note_time(struct tm *tm, size_t hdrlen, - unsigned char tag, - const unsigned char *value, size_t vlen) +int x509_decode_time(time64_t *_t, size_t hdrlen, + unsigned char tag, + const unsigned char *value, size_t vlen) { + static const unsigned char month_lengths[] = { 31, 29, 31, 30, 31, 30, + 31, 31, 30, 31, 30, 31 }; const unsigned char *p = value; + unsigned year, mon, day, hour, min, sec, mon_len; -#define dec2bin(X) ((X) - '0') +#define dec2bin(X) ({ unsigned char x = (X) - '0'; if (x > 9) goto invalid_time; x; }) #define DD2bin(P) ({ unsigned x = dec2bin(P[0]) * 10 + dec2bin(P[1]); P += 2; x; }) if (tag == ASN1_UNITIM) { /* UTCTime: YYMMDDHHMMSSZ */ if (vlen != 13) goto unsupported_time; - tm->tm_year = DD2bin(p); - if (tm->tm_year >= 50) - tm->tm_year += 1900; + year = DD2bin(p); + if (year >= 50) + year += 1900; else - tm->tm_year += 2000; + year += 2000; } else if (tag == ASN1_GENTIM) { /* GenTime: YYYYMMDDHHMMSSZ */ if (vlen != 15) goto unsupported_time; - tm->tm_year = DD2bin(p) * 100 + DD2bin(p); + year = DD2bin(p) * 100 + DD2bin(p); + if (year >= 1950 && year <= 2049) + goto invalid_time; } else { goto unsupported_time; } - tm->tm_year -= 1900; - tm->tm_mon = DD2bin(p) - 1; - tm->tm_mday = DD2bin(p); - tm->tm_hour = DD2bin(p); - tm->tm_min = DD2bin(p); - tm->tm_sec = DD2bin(p); + mon = DD2bin(p); + day = DD2bin(p); + hour = DD2bin(p); + min = DD2bin(p); + sec = DD2bin(p); if (*p != 'Z') goto unsupported_time; + mon_len = month_lengths[mon]; + if (mon == 2) { + if (year % 4 == 0) { + mon_len = 29; + if (year % 100 == 0) { + year /= 100; + if (year % 4 != 0) + mon_len = 28; + } + } + } + + if (year < 1970 || + mon < 1 || mon > 12 || + day < 1 || day > mon_len || + hour < 0 || hour > 23 || + min < 0 || min > 59 || + sec < 0 || sec > 59) + goto invalid_time; + + *_t = mktime64(year, mon, day, hour, min, sec); return 0; unsupported_time: - pr_debug("Got unsupported time [tag %02x]: '%*.*s'\n", - tag, (int)vlen, (int)vlen, value); + pr_debug("Got unsupported time [tag %02x]: '%*phN'\n", + tag, (int)vlen, value); + return -EBADMSG; +invalid_time: + pr_debug("Got invalid time [tag %02x]: '%*phN'\n", + tag, (int)vlen, value); return -EBADMSG; } +EXPORT_SYMBOL_GPL(x509_decode_time); int x509_note_not_before(void *context, size_t hdrlen, unsigned char tag, const void *value, size_t vlen) { struct x509_parse_context *ctx = context; - return x509_note_time(&ctx->cert->valid_from, hdrlen, tag, value, vlen); + return x509_decode_time(&ctx->cert->valid_from, hdrlen, tag, value, vlen); } int x509_note_not_after(void *context, size_t hdrlen, @@ -567,5 +578,71 @@ int x509_note_not_after(void *context, size_t hdrlen, const void *value, size_t vlen) { struct x509_parse_context *ctx = context; - return x509_note_time(&ctx->cert->valid_to, hdrlen, tag, value, vlen); + return x509_decode_time(&ctx->cert->valid_to, hdrlen, tag, value, vlen); +} + +/* + * Note a key identifier-based AuthorityKeyIdentifier + */ +int x509_akid_note_kid(void *context, size_t hdrlen, + unsigned char tag, + const void *value, size_t vlen) +{ + struct x509_parse_context *ctx = context; + struct asymmetric_key_id *kid; + + pr_debug("AKID: keyid: %*phN\n", (int)vlen, value); + + if (ctx->cert->akid_skid) + return 0; + + kid = asymmetric_key_generate_id(value, vlen, "", 0); + if (IS_ERR(kid)) + return PTR_ERR(kid); + pr_debug("authkeyid %*phN\n", kid->len, kid->data); + ctx->cert->akid_skid = kid; + return 0; +} + +/* + * Note a directoryName in an AuthorityKeyIdentifier + */ +int x509_akid_note_name(void *context, size_t hdrlen, + unsigned char tag, + const void *value, size_t vlen) +{ + struct x509_parse_context *ctx = context; + + pr_debug("AKID: name: %*phN\n", (int)vlen, value); + + ctx->akid_raw_issuer = value; + ctx->akid_raw_issuer_size = vlen; + return 0; +} + +/* + * Note a serial number in an AuthorityKeyIdentifier + */ +int x509_akid_note_serial(void *context, size_t hdrlen, + unsigned char tag, + const void *value, size_t vlen) +{ + struct x509_parse_context *ctx = context; + struct asymmetric_key_id *kid; + + pr_debug("AKID: serial: %*phN\n", (int)vlen, value); + + if (!ctx->akid_raw_issuer || ctx->cert->akid_id) + return 0; + + kid = asymmetric_key_generate_id(value, + vlen, + ctx->akid_raw_issuer, + ctx->akid_raw_issuer_size); + if (IS_ERR(kid)) + return PTR_ERR(kid); + + pr_debug("authkeyid %*phN\n", kid->len, kid->data); + ctx->cert->akid_id = kid; + return 0; } diff --git a/crypto/asymmetric_keys/x509_parser.h b/crypto/asymmetric_keys/x509_parser.h index 3dfe6b5d6f0b..1de01eaec884 100644 --- a/crypto/asymmetric_keys/x509_parser.h +++ b/crypto/asymmetric_keys/x509_parser.h @@ -19,11 +19,12 @@ struct x509_certificate { struct public_key_signature sig; /* Signature parameters */ char *issuer; /* Name of certificate issuer */ char *subject; /* Name of certificate subject */ - struct asymmetric_key_id *id; /* Serial number + issuer */ + struct asymmetric_key_id *id; /* Issuer + Serial number */ struct asymmetric_key_id *skid; /* Subject + subjectKeyId (optional) */ - struct asymmetric_key_id *authority; /* Authority key identifier (optional) */ - struct tm valid_from; - struct tm valid_to; + struct asymmetric_key_id *akid_id; /* CA AuthKeyId matching ->id (optional) */ + struct asymmetric_key_id *akid_skid; /* CA AuthKeyId matching ->skid (optional) */ + time64_t valid_from; + time64_t valid_to; const void *tbs; /* Signed data */ unsigned tbs_size; /* Size of signed data */ unsigned raw_sig_size; /* Size of sigature */ @@ -48,6 +49,9 @@ struct x509_certificate { */ extern void x509_free_certificate(struct x509_certificate *cert); extern struct x509_certificate *x509_cert_parse(const void *data, size_t datalen); +extern int x509_decode_time(time64_t *_t, size_t hdrlen, + unsigned char tag, + const unsigned char *value, size_t vlen); /* * x509_public_key.c diff --git a/crypto/asymmetric_keys/x509_public_key.c b/crypto/asymmetric_keys/x509_public_key.c index 24f17e6c5904..6d88dd15c98d 100644 --- a/crypto/asymmetric_keys/x509_public_key.c +++ b/crypto/asymmetric_keys/x509_public_key.c @@ -65,23 +65,37 @@ __setup("ca_keys=", ca_keys_setup); /** * x509_request_asymmetric_key - Request a key by X.509 certificate params. * @keyring: The keys to search. - * @kid: The key ID. + * @id: The issuer & serialNumber to look for or NULL. + * @skid: The subjectKeyIdentifier to look for or NULL. * @partial: Use partial match if true, exact if false. * - * Find a key in the given keyring by subject name and key ID. These might, - * for instance, be the issuer name and the authority key ID of an X.509 - * certificate that needs to be verified. + * Find a key in the given keyring by identifier. The preferred identifier is + * the issuer + serialNumber and the fallback identifier is the + * subjectKeyIdentifier. If both are given, the lookup is by the former, but + * the latter must also match. */ struct key *x509_request_asymmetric_key(struct key *keyring, - const struct asymmetric_key_id *kid, + const struct asymmetric_key_id *id, + const struct asymmetric_key_id *skid, bool partial) { - key_ref_t key; - char *id, *p; + struct key *key; + key_ref_t ref; + const char *lookup; + char *req, *p; + int len; + if (id) { + lookup = id->data; + len = id->len; + } else { + lookup = skid->data; + len = skid->len; + } + /* Construct an identifier "id:". */ - p = id = kmalloc(2 + 1 + kid->len * 2 + 1, GFP_KERNEL); - if (!id) + p = req = kmalloc(2 + 1 + len * 2 + 1, GFP_KERNEL); + if (!req) return ERR_PTR(-ENOMEM); if (partial) { @@ -92,32 +106,48 @@ struct key *x509_request_asymmetric_key(struct key *keyring, *p++ = 'x'; } *p++ = ':'; - p = bin2hex(p, kid->data, kid->len); + p = bin2hex(p, lookup, len); *p = 0; - pr_debug("Look up: \"%s\"\n", id); + pr_debug("Look up: \"%s\"\n", req); - key = keyring_search(make_key_ref(keyring, 1), - &key_type_asymmetric, id); - if (IS_ERR(key)) - pr_debug("Request for key '%s' err %ld\n", id, PTR_ERR(key)); - kfree(id); + ref = keyring_search(make_key_ref(keyring, 1), + &key_type_asymmetric, req); + if (IS_ERR(ref)) + pr_debug("Request for key '%s' err %ld\n", req, PTR_ERR(ref)); + kfree(req); - if (IS_ERR(key)) { - switch (PTR_ERR(key)) { + if (IS_ERR(ref)) { + switch (PTR_ERR(ref)) { /* Hide some search errors */ case -EACCES: case -ENOTDIR: case -EAGAIN: return ERR_PTR(-ENOKEY); default: - return ERR_CAST(key); + return ERR_CAST(ref); } } - pr_devel("<==%s() = 0 [%x]\n", __func__, - key_serial(key_ref_to_ptr(key))); - return key_ref_to_ptr(key); + key = key_ref_to_ptr(ref); + if (id && skid) { + const struct asymmetric_key_ids *kids = asymmetric_key_ids(key); + if (!kids->id[1]) { + pr_debug("issuer+serial match, but expected SKID missing\n"); + goto reject; + } + if (!asymmetric_key_id_same(skid, kids->id[1])) { + pr_debug("issuer+serial match, but SKID does not\n"); + goto reject; + } + } + + pr_devel("<==%s() = 0 [%x]\n", __func__, key_serial(key)); + return key; + +reject: + key_put(key); + return ERR_PTR(-EKEYREJECTED); } EXPORT_SYMBOL_GPL(x509_request_asymmetric_key); @@ -227,10 +257,11 @@ static int x509_validate_trust(struct x509_certificate *cert, if (!trust_keyring) return -EOPNOTSUPP; - if (ca_keyid && !asymmetric_key_id_partial(cert->authority, ca_keyid)) + if (ca_keyid && !asymmetric_key_id_partial(cert->akid_skid, ca_keyid)) return -EPERM; - key = x509_request_asymmetric_key(trust_keyring, cert->authority, + key = x509_request_asymmetric_key(trust_keyring, + cert->akid_id, cert->akid_skid, false); if (!IS_ERR(key)) { if (!use_builtin_keys @@ -271,14 +302,7 @@ static int x509_key_preparse(struct key_preparsed_payload *prep) } pr_devel("Cert Key Algo: %s\n", pkey_algo_name[cert->pub->pkey_algo]); - pr_devel("Cert Valid From: %04ld-%02d-%02d %02d:%02d:%02d\n", - cert->valid_from.tm_year + 1900, cert->valid_from.tm_mon + 1, - cert->valid_from.tm_mday, cert->valid_from.tm_hour, - cert->valid_from.tm_min, cert->valid_from.tm_sec); - pr_devel("Cert Valid To: %04ld-%02d-%02d %02d:%02d:%02d\n", - cert->valid_to.tm_year + 1900, cert->valid_to.tm_mon + 1, - cert->valid_to.tm_mday, cert->valid_to.tm_hour, - cert->valid_to.tm_min, cert->valid_to.tm_sec); + pr_devel("Cert Valid period: %lld-%lld\n", cert->valid_from, cert->valid_to); pr_devel("Cert Signature: %s + %s\n", pkey_algo_name[cert->sig.pkey_algo], hash_algo_name[cert->sig.pkey_hash_algo]); @@ -287,8 +311,9 @@ static int x509_key_preparse(struct key_preparsed_payload *prep) cert->pub->id_type = PKEY_ID_X509; /* Check the signature on the key if it appears to be self-signed */ - if (!cert->authority || - asymmetric_key_id_same(cert->skid, cert->authority)) { + if ((!cert->akid_skid && !cert->akid_id) || + asymmetric_key_id_same(cert->skid, cert->akid_skid) || + asymmetric_key_id_same(cert->id, cert->akid_id)) { ret = x509_check_signature(cert->pub, cert); /* self-signed */ if (ret < 0) goto error_free_cert; diff --git a/crypto/gcm.c b/crypto/gcm.c index ddb4f29b2fe6..bec329b3de8d 100644 --- a/crypto/gcm.c +++ b/crypto/gcm.c @@ -206,14 +206,14 @@ static void crypto_gcm_init_common(struct aead_request *req) sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag)); sg = scatterwalk_ffwd(pctx->src + 1, req->src, req->assoclen); if (sg != pctx->src + 1) - scatterwalk_sg_chain(pctx->src, 2, sg); + sg_chain(pctx->src, 2, sg); if (req->src != req->dst) { sg_init_table(pctx->dst, 3); sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag)); sg = scatterwalk_ffwd(pctx->dst + 1, req->dst, req->assoclen); if (sg != pctx->dst + 1) - scatterwalk_sg_chain(pctx->dst, 2, sg); + sg_chain(pctx->dst, 2, sg); } } diff --git a/drivers/Kconfig b/drivers/Kconfig index 4e2e6aaf0b88..46b4a8e0f859 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -176,6 +176,8 @@ source "drivers/powercap/Kconfig" source "drivers/mcb/Kconfig" +source "drivers/perf/Kconfig" + source "drivers/ras/Kconfig" source "drivers/thunderbolt/Kconfig" diff --git a/drivers/Makefile b/drivers/Makefile index 4c270f5414f0..b250b36b54f2 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -11,7 +11,7 @@ obj-y += bus/ obj-$(CONFIG_GENERIC_PHY) += phy/ # GPIO must come after pinctrl as gpios may need to mux pins etc -obj-y += pinctrl/ +obj-$(CONFIG_PINCTRL) += pinctrl/ obj-y += gpio/ obj-y += pwm/ obj-$(CONFIG_PCI) += pci/ @@ -161,6 +161,7 @@ obj-$(CONFIG_NTB) += ntb/ obj-$(CONFIG_FMC) += fmc/ obj-$(CONFIG_POWERCAP) += powercap/ obj-$(CONFIG_MCB) += mcb/ +obj-$(CONFIG_PERF_EVENTS) += perf/ obj-$(CONFIG_RAS) += ras/ obj-$(CONFIG_THUNDERBOLT) += thunderbolt/ obj-$(CONFIG_CORESIGHT) += hwtracing/coresight/ diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig index 114cf48085ab..5d1015c26ff4 100644 --- a/drivers/acpi/Kconfig +++ b/drivers/acpi/Kconfig @@ -189,17 +189,24 @@ config ACPI_DOCK This driver supports ACPI-controlled docking stations and removable drive bays such as the IBM Ultrabay and the Dell Module Bay. +config ACPI_CPU_FREQ_PSS + bool + select THERMAL + +config ACPI_PROCESSOR_IDLE + bool + select CPU_IDLE + config ACPI_PROCESSOR tristate "Processor" - select THERMAL - select CPU_IDLE depends on X86 || IA64 + select ACPI_PROCESSOR_IDLE + select ACPI_CPU_FREQ_PSS default y help - This driver installs ACPI as the idle handler for Linux and uses - ACPI C2 and C3 processor states to save power on systems that - support it. It is required by several flavors of cpufreq - performance-state drivers. + This driver adds support for the ACPI Processor package. It is required + by several flavors of cpufreq performance-state, thermal, throttling and + idle drivers. To compile this driver as a module, choose M here: the module will be called processor. @@ -410,6 +417,7 @@ config ACPI_NFIT tristate "ACPI NVDIMM Firmware Interface Table (NFIT)" depends on PHYS_ADDR_T_64BIT depends on BLK_DEV + depends on ARCH_HAS_MMIO_FLUSH select LIBNVDIMM help Infrastructure to probe ACPI 6 compliant platforms for diff --git a/drivers/acpi/Makefile b/drivers/acpi/Makefile index 8321430d7f24..b5e7cd8a9c71 100644 --- a/drivers/acpi/Makefile +++ b/drivers/acpi/Makefile @@ -24,7 +24,7 @@ acpi-y += nvs.o # Power management related files acpi-y += wakeup.o acpi-$(CONFIG_ACPI_SYSTEM_POWER_STATES_SUPPORT) += sleep.o -acpi-y += device_pm.o +acpi-y += device_sysfs.o device_pm.o acpi-$(CONFIG_ACPI_SLEEP) += proc.o @@ -80,8 +80,10 @@ obj-$(CONFIG_ACPI_CUSTOM_METHOD)+= custom_method.o obj-$(CONFIG_ACPI_BGRT) += bgrt.o # processor has its own "processor." module_param namespace -processor-y := processor_driver.o processor_throttling.o -processor-y += processor_idle.o processor_thermal.o +processor-y := processor_driver.o +processor-$(CONFIG_ACPI_PROCESSOR_IDLE) += processor_idle.o +processor-$(CONFIG_ACPI_CPU_FREQ_PSS) += processor_throttling.o \ + processor_thermal.o processor-$(CONFIG_CPU_FREQ) += processor_perflib.o obj-$(CONFIG_ACPI_PROCESSOR_AGGREGATOR) += acpi_pad.o diff --git a/drivers/acpi/ac.c b/drivers/acpi/ac.c index 9b5354a2cd08..f71b756b05c4 100644 --- a/drivers/acpi/ac.c +++ b/drivers/acpi/ac.c @@ -16,10 +16,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ diff --git a/drivers/acpi/acpi_ipmi.c b/drivers/acpi/acpi_ipmi.c index ac0f52f6df2b..f77956c3fd45 100644 --- a/drivers/acpi/acpi_ipmi.c +++ b/drivers/acpi/acpi_ipmi.c @@ -17,10 +17,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ diff --git a/drivers/acpi/acpi_lpss.c b/drivers/acpi/acpi_lpss.c index 502454c24e69..f51bd0d0bc17 100644 --- a/drivers/acpi/acpi_lpss.c +++ b/drivers/acpi/acpi_lpss.c @@ -59,6 +59,7 @@ ACPI_MODULE_NAME("acpi_lpss"); #define LPSS_CLK_DIVIDER BIT(2) #define LPSS_LTR BIT(3) #define LPSS_SAVE_CTX BIT(4) +#define LPSS_NO_D3_DELAY BIT(5) struct lpss_private_data; @@ -155,6 +156,10 @@ static const struct lpss_device_desc byt_pwm_dev_desc = { .flags = LPSS_SAVE_CTX, }; +static const struct lpss_device_desc bsw_pwm_dev_desc = { + .flags = LPSS_SAVE_CTX | LPSS_NO_D3_DELAY, +}; + static const struct lpss_device_desc byt_uart_dev_desc = { .flags = LPSS_CLK | LPSS_CLK_GATE | LPSS_CLK_DIVIDER | LPSS_SAVE_CTX, .clk_con_id = "baudclk", @@ -162,6 +167,14 @@ static const struct lpss_device_desc byt_uart_dev_desc = { .setup = lpss_uart_setup, }; +static const struct lpss_device_desc bsw_uart_dev_desc = { + .flags = LPSS_CLK | LPSS_CLK_GATE | LPSS_CLK_DIVIDER | LPSS_SAVE_CTX + | LPSS_NO_D3_DELAY, + .clk_con_id = "baudclk", + .prv_offset = 0x800, + .setup = lpss_uart_setup, +}; + static const struct lpss_device_desc byt_spi_dev_desc = { .flags = LPSS_CLK | LPSS_CLK_GATE | LPSS_CLK_DIVIDER | LPSS_SAVE_CTX, .prv_offset = 0x400, @@ -177,8 +190,15 @@ static const struct lpss_device_desc byt_i2c_dev_desc = { .setup = byt_i2c_setup, }; +static const struct lpss_device_desc bsw_i2c_dev_desc = { + .flags = LPSS_CLK | LPSS_SAVE_CTX | LPSS_NO_D3_DELAY, + .prv_offset = 0x800, + .setup = byt_i2c_setup, +}; + static struct lpss_device_desc bsw_spi_dev_desc = { - .flags = LPSS_CLK | LPSS_CLK_GATE | LPSS_CLK_DIVIDER | LPSS_SAVE_CTX, + .flags = LPSS_CLK | LPSS_CLK_GATE | LPSS_CLK_DIVIDER | LPSS_SAVE_CTX + | LPSS_NO_D3_DELAY, .prv_offset = 0x400, .setup = lpss_deassert_reset, }; @@ -213,11 +233,12 @@ static const struct acpi_device_id acpi_lpss_device_ids[] = { { "INT33FC", }, /* Braswell LPSS devices */ - { "80862288", LPSS_ADDR(byt_pwm_dev_desc) }, - { "8086228A", LPSS_ADDR(byt_uart_dev_desc) }, + { "80862288", LPSS_ADDR(bsw_pwm_dev_desc) }, + { "8086228A", LPSS_ADDR(bsw_uart_dev_desc) }, { "8086228E", LPSS_ADDR(bsw_spi_dev_desc) }, - { "808622C1", LPSS_ADDR(byt_i2c_dev_desc) }, + { "808622C1", LPSS_ADDR(bsw_i2c_dev_desc) }, + /* Broadwell LPSS devices */ { "INT3430", LPSS_ADDR(lpt_dev_desc) }, { "INT3431", LPSS_ADDR(lpt_dev_desc) }, { "INT3432", LPSS_ADDR(lpt_i2c_dev_desc) }, @@ -557,9 +578,14 @@ static void acpi_lpss_restore_ctx(struct device *dev, * The following delay is needed or the subsequent write operations may * fail. The LPSS devices are actually PCI devices and the PCI spec * expects 10ms delay before the device can be accessed after D3 to D0 - * transition. + * transition. However some platforms like BSW does not need this delay. */ - msleep(10); + unsigned int delay = 10; /* default 10ms delay */ + + if (pdata->dev_desc->flags & LPSS_NO_D3_DELAY) + delay = 0; + + msleep(delay); for (i = 0; i < LPSS_PRV_REG_COUNT; i++) { unsigned long offset = i * sizeof(u32); diff --git a/drivers/acpi/acpi_memhotplug.c b/drivers/acpi/acpi_memhotplug.c index ee28f4d15625..6b0d3ef7309c 100644 --- a/drivers/acpi/acpi_memhotplug.c +++ b/drivers/acpi/acpi_memhotplug.c @@ -16,11 +16,6 @@ * NON INFRINGEMENT. See the GNU General Public License for more * details. * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - * - * * ACPI based HotPlug driver that supports Memory Hotplug * This driver fields notifications from firmware for memory add * and remove operations and alerts the VM of the affected memory diff --git a/drivers/acpi/acpi_pad.c b/drivers/acpi/acpi_pad.c index 00b39802d7ec..ae307ff36acb 100644 --- a/drivers/acpi/acpi_pad.c +++ b/drivers/acpi/acpi_pad.c @@ -12,10 +12,6 @@ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - * */ #include diff --git a/drivers/acpi/acpi_pnp.c b/drivers/acpi/acpi_pnp.c index fb765524cc3d..c58940b231d6 100644 --- a/drivers/acpi/acpi_pnp.c +++ b/drivers/acpi/acpi_pnp.c @@ -19,8 +19,6 @@ static const struct acpi_device_id acpi_pnp_device_ids[] = { {"PNP0600"}, /* Generic ESDI/IDE/ATA compatible hard disk controller */ /* floppy */ {"PNP0700"}, - /* ipmi_si */ - {"IPI0001"}, /* tpm_inf_pnp */ {"IFX0101"}, /* Infineon TPMs */ {"IFX0102"}, /* Infineon TPMs */ diff --git a/drivers/acpi/acpi_processor.c b/drivers/acpi/acpi_processor.c index 92a5f738e370..985b8a83184e 100644 --- a/drivers/acpi/acpi_processor.c +++ b/drivers/acpi/acpi_processor.c @@ -485,7 +485,7 @@ static const struct acpi_device_id processor_device_ids[] = { { } }; -static struct acpi_scan_handler __refdata processor_handler = { +static struct acpi_scan_handler processor_handler = { .ids = processor_device_ids, .attach = acpi_processor_add, #ifdef CONFIG_ACPI_HOTPLUG_CPU diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c index 8c2fe2f2f9fd..5778e8e4313a 100644 --- a/drivers/acpi/acpi_video.c +++ b/drivers/acpi/acpi_video.c @@ -17,10 +17,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ diff --git a/drivers/acpi/acpica/Makefile b/drivers/acpi/acpica/Makefile index c1a963581dc0..fedcc16b56cc 100644 --- a/drivers/acpi/acpica/Makefile +++ b/drivers/acpi/acpica/Makefile @@ -11,6 +11,7 @@ obj-y += acpi.o acpi-y := \ dsargs.o \ dscontrol.o \ + dsdebug.o \ dsfield.o \ dsinit.o \ dsmethod.o \ @@ -164,6 +165,7 @@ acpi-y += \ utmath.o \ utmisc.o \ utmutex.o \ + utnonansi.o \ utobject.o \ utosi.o \ utownerid.o \ diff --git a/drivers/acpi/acpica/acdebug.h b/drivers/acpi/acpica/acdebug.h index 43685dd36c77..eb2e926d8218 100644 --- a/drivers/acpi/acpica/acdebug.h +++ b/drivers/acpi/acpica/acdebug.h @@ -67,9 +67,6 @@ struct acpi_db_execute_walk { }; #define PARAM_LIST(pl) pl -#define DBTEST_OUTPUT_LEVEL(lvl) if (acpi_gbl_db_opt_verbose) -#define VERBOSE_PRINT(fp) DBTEST_OUTPUT_LEVEL(lvl) {\ - acpi_os_printf PARAM_LIST(fp);} #define EX_NO_SINGLE_STEP 1 #define EX_SINGLE_STEP 2 @@ -77,10 +74,6 @@ struct acpi_db_execute_walk { /* * dbxface - external debugger interfaces */ -acpi_status acpi_db_initialize(void); - -void acpi_db_terminate(void); - acpi_status acpi_db_single_step(struct acpi_walk_state *walk_state, union acpi_parse_object *op, u32 op_type); @@ -102,6 +95,8 @@ void acpi_db_display_interfaces(char *action_arg, char *interface_name_arg); acpi_status acpi_db_sleep(char *object_arg); +void acpi_db_trace(char *enable_arg, char *method_arg, char *once_arg); + void acpi_db_display_locks(void); void acpi_db_display_resources(char *object_arg); @@ -261,6 +256,23 @@ acpi_status acpi_db_user_commands(char prompt, union acpi_parse_object *op); char *acpi_db_get_next_token(char *string, char **next, acpi_object_type * return_type); +/* + * dbobject + */ +void acpi_db_decode_internal_object(union acpi_operand_object *obj_desc); + +void +acpi_db_display_internal_object(union acpi_operand_object *obj_desc, + struct acpi_walk_state *walk_state); + +void acpi_db_decode_arguments(struct acpi_walk_state *walk_state); + +void acpi_db_decode_locals(struct acpi_walk_state *walk_state); + +void +acpi_db_dump_method_info(acpi_status status, + struct acpi_walk_state *walk_state); + /* * dbstats - Generation and display of ACPI table statistics */ diff --git a/drivers/acpi/acpica/acdispat.h b/drivers/acpi/acpica/acdispat.h index 408f04bcaab4..7094dc89eb81 100644 --- a/drivers/acpi/acpica/acdispat.h +++ b/drivers/acpi/acpica/acdispat.h @@ -354,4 +354,12 @@ acpi_status acpi_ds_result_push(union acpi_operand_object *object, struct acpi_walk_state *walk_state); +/* + * dsdebug - parser debugging routines + */ +void +acpi_ds_dump_method_stack(acpi_status status, + struct acpi_walk_state *walk_state, + union acpi_parse_object *op); + #endif /* _ACDISPAT_H_ */ diff --git a/drivers/acpi/acpica/acglobal.h b/drivers/acpi/acpica/acglobal.h index 53f96a370762..09f37b516808 100644 --- a/drivers/acpi/acpica/acglobal.h +++ b/drivers/acpi/acpica/acglobal.h @@ -58,11 +58,12 @@ ACPI_GLOBAL(struct acpi_table_list, acpi_gbl_root_table_list); ACPI_GLOBAL(struct acpi_table_header *, acpi_gbl_DSDT); ACPI_GLOBAL(struct acpi_table_header, acpi_gbl_original_dsdt_header); +ACPI_INIT_GLOBAL(u32, acpi_gbl_dsdt_index, ACPI_INVALID_TABLE_INDEX); +ACPI_INIT_GLOBAL(u32, acpi_gbl_facs_index, ACPI_INVALID_TABLE_INDEX); +ACPI_INIT_GLOBAL(u32, acpi_gbl_xfacs_index, ACPI_INVALID_TABLE_INDEX); #if (!ACPI_REDUCED_HARDWARE) ACPI_GLOBAL(struct acpi_table_facs *, acpi_gbl_FACS); -ACPI_GLOBAL(struct acpi_table_facs *, acpi_gbl_facs32); -ACPI_GLOBAL(struct acpi_table_facs *, acpi_gbl_facs64); #endif /* !ACPI_REDUCED_HARDWARE */ @@ -235,6 +236,10 @@ ACPI_INIT_GLOBAL(u32, acpi_gbl_nesting_level, 0); ACPI_GLOBAL(struct acpi_thread_state *, acpi_gbl_current_walk_list); +/* Maximum number of While() loop iterations before forced abort */ + +ACPI_GLOBAL(u16, acpi_gbl_max_loop_iterations); + /* Control method single step flag */ ACPI_GLOBAL(u8, acpi_gbl_cm_single_step); @@ -290,8 +295,6 @@ ACPI_GLOBAL(u32, acpi_fixed_event_count[ACPI_NUM_FIXED_EVENTS]); ACPI_GLOBAL(u32, acpi_gbl_original_dbg_level); ACPI_GLOBAL(u32, acpi_gbl_original_dbg_layer); -ACPI_GLOBAL(u32, acpi_gbl_trace_dbg_level); -ACPI_GLOBAL(u32, acpi_gbl_trace_dbg_layer); /***************************************************************************** * @@ -309,9 +312,10 @@ ACPI_INIT_GLOBAL(u8, acpi_gbl_no_resource_disassembly, FALSE); ACPI_INIT_GLOBAL(u8, acpi_gbl_ignore_noop_operator, FALSE); ACPI_INIT_GLOBAL(u8, acpi_gbl_cstyle_disassembly, TRUE); ACPI_INIT_GLOBAL(u8, acpi_gbl_force_aml_disassembly, FALSE); +ACPI_INIT_GLOBAL(u8, acpi_gbl_dm_opt_verbose, TRUE); -ACPI_GLOBAL(u8, acpi_gbl_db_opt_disasm); -ACPI_GLOBAL(u8, acpi_gbl_db_opt_verbose); +ACPI_GLOBAL(u8, acpi_gbl_dm_opt_disasm); +ACPI_GLOBAL(u8, acpi_gbl_dm_opt_listing); ACPI_GLOBAL(u8, acpi_gbl_num_external_methods); ACPI_GLOBAL(u32, acpi_gbl_resolved_external_methods); ACPI_GLOBAL(struct acpi_external_list *, acpi_gbl_external_list); @@ -346,8 +350,8 @@ ACPI_GLOBAL(char, acpi_gbl_db_debug_filename[ACPI_DB_LINE_BUFFER_SIZE]); /* * Statistic globals */ -ACPI_GLOBAL(u16, acpi_gbl_obj_type_count[ACPI_TYPE_NS_NODE_MAX + 1]); -ACPI_GLOBAL(u16, acpi_gbl_node_type_count[ACPI_TYPE_NS_NODE_MAX + 1]); +ACPI_GLOBAL(u16, acpi_gbl_obj_type_count[ACPI_TOTAL_TYPES]); +ACPI_GLOBAL(u16, acpi_gbl_node_type_count[ACPI_TOTAL_TYPES]); ACPI_GLOBAL(u16, acpi_gbl_obj_type_count_misc); ACPI_GLOBAL(u16, acpi_gbl_node_type_count_misc); ACPI_GLOBAL(u32, acpi_gbl_num_nodes); diff --git a/drivers/acpi/acpica/acinterp.h b/drivers/acpi/acpica/acinterp.h index 7ac98000b46b..e820ed8f173f 100644 --- a/drivers/acpi/acpica/acinterp.h +++ b/drivers/acpi/acpica/acinterp.h @@ -131,6 +131,28 @@ void acpi_ex_do_debug_object(union acpi_operand_object *source_desc, u32 level, u32 index); +void +acpi_ex_start_trace_method(struct acpi_namespace_node *method_node, + union acpi_operand_object *obj_desc, + struct acpi_walk_state *walk_state); + +void +acpi_ex_stop_trace_method(struct acpi_namespace_node *method_node, + union acpi_operand_object *obj_desc, + struct acpi_walk_state *walk_state); + +void +acpi_ex_start_trace_opcode(union acpi_parse_object *op, + struct acpi_walk_state *walk_state); + +void +acpi_ex_stop_trace_opcode(union acpi_parse_object *op, + struct acpi_walk_state *walk_state); + +void +acpi_ex_trace_point(acpi_trace_event_type type, + u8 begin, u8 *aml, char *pathname); + /* * exfield - ACPI AML (p-code) execution - field manipulation */ diff --git a/drivers/acpi/acpica/aclocal.h b/drivers/acpi/acpica/aclocal.h index bc600969c6a1..6f708267ad8c 100644 --- a/drivers/acpi/acpica/aclocal.h +++ b/drivers/acpi/acpica/aclocal.h @@ -174,8 +174,12 @@ struct acpi_namespace_node { */ #ifdef ACPI_LARGE_NAMESPACE_NODE union acpi_parse_object *op; + void *method_locals; + void *method_args; u32 value; u32 length; + u8 arg_count; + #endif }; @@ -209,11 +213,9 @@ struct acpi_table_list { #define ACPI_ROOT_ORIGIN_ALLOCATED (1) #define ACPI_ROOT_ALLOW_RESIZE (2) -/* Predefined (fixed) table indexes */ +/* Predefined table indexes */ -#define ACPI_TABLE_INDEX_DSDT (0) -#define ACPI_TABLE_INDEX_FACS (1) -#define ACPI_TABLE_INDEX_X_FACS (2) +#define ACPI_INVALID_TABLE_INDEX (0xFFFFFFFF) struct acpi_find_context { char *search_for; @@ -404,6 +406,13 @@ struct acpi_simple_repair_info { #define ACPI_NUM_RTYPES 5 /* Number of actual object types */ +/* Info for running the _REG methods */ + +struct acpi_reg_walk_info { + acpi_adr_space_type space_id; + u32 reg_run_count; +}; + /***************************************************************************** * * Event typedefs and structs @@ -715,7 +724,7 @@ union acpi_parse_value { union acpi_parse_object *arg; /* arguments and contained ops */ }; -#ifdef ACPI_DISASSEMBLER +#if defined(ACPI_DISASSEMBLER) || defined(ACPI_DEBUG_OUTPUT) #define ACPI_DISASM_ONLY_MEMBERS(a) a; #else #define ACPI_DISASM_ONLY_MEMBERS(a) @@ -726,7 +735,7 @@ union acpi_parse_value { u8 descriptor_type; /* To differentiate various internal objs */\ u8 flags; /* Type of Op */\ u16 aml_opcode; /* AML opcode */\ - u32 aml_offset; /* Offset of declaration in AML */\ + u8 *aml; /* Address of declaration in AML */\ union acpi_parse_object *next; /* Next op */\ struct acpi_namespace_node *node; /* For use by interpreter */\ union acpi_parse_value value; /* Value or args associated with the opcode */\ @@ -1103,6 +1112,9 @@ struct acpi_db_method_info { * Index of current thread inside all them created. */ char init_args; +#ifdef ACPI_DEBUGGER + acpi_object_type arg_types[4]; +#endif char *arguments[4]; char num_threads_str[11]; char id_of_thread_str[11]; @@ -1119,6 +1131,10 @@ struct acpi_integrity_info { #define ACPI_DB_CONSOLE_OUTPUT 0x02 #define ACPI_DB_DUPLICATE_OUTPUT 0x03 +struct acpi_object_info { + u32 types[ACPI_TOTAL_TYPES]; +}; + /***************************************************************************** * * Debug diff --git a/drivers/acpi/acpica/acmacros.h b/drivers/acpi/acpica/acmacros.h index c240bdf824f2..e85366ceb15a 100644 --- a/drivers/acpi/acpica/acmacros.h +++ b/drivers/acpi/acpica/acmacros.h @@ -220,6 +220,15 @@ #define ACPI_MUL_32(a) _ACPI_MUL(a, 5) #define ACPI_MOD_32(a) _ACPI_MOD(a, 32) +/* Test for ASCII character */ + +#define ACPI_IS_ASCII(c) ((c) < 0x80) + +/* Signed integers */ + +#define ACPI_SIGN_POSITIVE 0 +#define ACPI_SIGN_NEGATIVE 1 + /* * Rounding macros (Power of two boundaries only) */ diff --git a/drivers/acpi/acpica/acnamesp.h b/drivers/acpi/acpica/acnamesp.h index 0dd088290d80..ea0d9076d408 100644 --- a/drivers/acpi/acpica/acnamesp.h +++ b/drivers/acpi/acpica/acnamesp.h @@ -272,17 +272,20 @@ acpi_ns_check_package(struct acpi_evaluate_info *info, */ u32 acpi_ns_opens_scope(acpi_object_type type); -acpi_status -acpi_ns_build_external_path(struct acpi_namespace_node *node, - acpi_size size, char *name_buffer); - char *acpi_ns_get_external_pathname(struct acpi_namespace_node *node); +u32 +acpi_ns_build_normalized_path(struct acpi_namespace_node *node, + char *full_path, u32 path_size, u8 no_trailing); + +char *acpi_ns_get_normalized_pathname(struct acpi_namespace_node *node, + u8 no_trailing); + char *acpi_ns_name_of_current_scope(struct acpi_walk_state *walk_state); acpi_status acpi_ns_handle_to_pathname(acpi_handle target_handle, - struct acpi_buffer *buffer); + struct acpi_buffer *buffer, u8 no_trailing); u8 acpi_ns_pattern_match(struct acpi_namespace_node *obj_node, char *search_for); diff --git a/drivers/acpi/acpica/acobject.h b/drivers/acpi/acpica/acobject.h index c81d98d09cac..0bd02c4a5f75 100644 --- a/drivers/acpi/acpica/acobject.h +++ b/drivers/acpi/acpica/acobject.h @@ -176,6 +176,7 @@ struct acpi_object_method { u8 param_count; u8 sync_level; union acpi_operand_object *mutex; + union acpi_operand_object *node; u8 *aml_start; union { acpi_internal_method implementation; diff --git a/drivers/acpi/acpica/acparser.h b/drivers/acpi/acpica/acparser.h index 0cdd2fce493a..6021ccfb0b1c 100644 --- a/drivers/acpi/acpica/acparser.h +++ b/drivers/acpi/acpica/acparser.h @@ -225,11 +225,11 @@ void acpi_ps_delete_parse_tree(union acpi_parse_object *root); /* * psutils - parser utilities */ -union acpi_parse_object *acpi_ps_create_scope_op(void); +union acpi_parse_object *acpi_ps_create_scope_op(u8 *aml); void acpi_ps_init_op(union acpi_parse_object *op, u16 opcode); -union acpi_parse_object *acpi_ps_alloc_op(u16 opcode); +union acpi_parse_object *acpi_ps_alloc_op(u16 opcode, u8 *aml); void acpi_ps_free_op(union acpi_parse_object *op); diff --git a/drivers/acpi/acpica/acstruct.h b/drivers/acpi/acpica/acstruct.h index 44997ca02ae2..f9992dced1f9 100644 --- a/drivers/acpi/acpica/acstruct.h +++ b/drivers/acpi/acpica/acstruct.h @@ -85,7 +85,7 @@ struct acpi_walk_state { u8 namespace_override; /* Override existing objects */ u8 result_size; /* Total elements for the result stack */ u8 result_count; /* Current number of occupied elements of result stack */ - u32 aml_offset; + u8 *aml; u32 arg_types; u32 method_breakpoint; /* For single stepping */ u32 user_breakpoint; /* User AML breakpoint */ diff --git a/drivers/acpi/acpica/actables.h b/drivers/acpi/acpica/actables.h index 7e0b6f1bec9c..f7731f260c31 100644 --- a/drivers/acpi/acpica/actables.h +++ b/drivers/acpi/acpica/actables.h @@ -154,14 +154,20 @@ void acpi_tb_check_dsdt_header(void); struct acpi_table_header *acpi_tb_copy_dsdt(u32 table_index); void -acpi_tb_install_table_with_override(u32 table_index, - struct acpi_table_desc *new_table_desc, - u8 override); +acpi_tb_install_table_with_override(struct acpi_table_desc *new_table_desc, + u8 override, u32 *table_index); acpi_status acpi_tb_install_fixed_table(acpi_physical_address address, - char *signature, u32 table_index); + char *signature, u32 *table_index); acpi_status acpi_tb_parse_root_table(acpi_physical_address rsdp_address); +u8 acpi_is_valid_signature(char *signature); + +/* + * tbxfload + */ +acpi_status acpi_tb_load_namespace(void); + #endif /* __ACTABLES_H__ */ diff --git a/drivers/acpi/acpica/acutils.h b/drivers/acpi/acpica/acutils.h index 6de0d3573037..fb2aa5066f3f 100644 --- a/drivers/acpi/acpica/acutils.h +++ b/drivers/acpi/acpica/acutils.h @@ -166,6 +166,17 @@ struct acpi_pkg_info { #define DB_DWORD_DISPLAY 4 #define DB_QWORD_DISPLAY 8 +/* + * utnonansi - Non-ANSI C library functions + */ +void acpi_ut_strupr(char *src_string); + +void acpi_ut_strlwr(char *src_string); + +int acpi_ut_stricmp(char *string1, char *string2); + +acpi_status acpi_ut_strtoul64(char *string, u32 base, u64 *ret_integer); + /* * utglobal - Global data structures and procedures */ @@ -205,8 +216,6 @@ acpi_status acpi_ut_hardware_initialize(void); void acpi_ut_subsystem_shutdown(void); -#define ACPI_IS_ASCII(c) ((c) < 0x80) - /* * utcopy - Object construction and conversion interfaces */ @@ -508,7 +517,7 @@ const struct acpi_exception_info *acpi_ut_validate_exception(acpi_status u8 acpi_ut_is_pci_root_bridge(char *id); -#if (defined ACPI_ASL_COMPILER || defined ACPI_EXEC_APP) +#if (defined ACPI_ASL_COMPILER || defined ACPI_EXEC_APP || defined ACPI_NAMES_APP) u8 acpi_ut_is_aml_table(struct acpi_table_header *table); #endif @@ -567,16 +576,6 @@ acpi_ut_get_resource_end_tag(union acpi_operand_object *obj_desc, u8 **end_tag); /* * utstring - String and character utilities */ -void acpi_ut_strupr(char *src_string); - -#ifdef ACPI_ASL_COMPILER -void acpi_ut_strlwr(char *src_string); - -int acpi_ut_stricmp(char *string1, char *string2); -#endif - -acpi_status acpi_ut_strtoul64(char *string, u32 base, u64 *ret_integer); - void acpi_ut_print_string(char *string, u16 max_length); #if defined ACPI_ASL_COMPILER || defined ACPI_EXEC_APP diff --git a/drivers/acpi/acpica/dsargs.c b/drivers/acpi/acpica/dsargs.c index 3e6989738e85..e2ab59e39162 100644 --- a/drivers/acpi/acpica/dsargs.c +++ b/drivers/acpi/acpica/dsargs.c @@ -86,7 +86,7 @@ acpi_ds_execute_arguments(struct acpi_namespace_node *node, /* Allocate a new parser op to be the root of the parsed tree */ - op = acpi_ps_alloc_op(AML_INT_EVAL_SUBTREE_OP); + op = acpi_ps_alloc_op(AML_INT_EVAL_SUBTREE_OP, aml_start); if (!op) { return_ACPI_STATUS(AE_NO_MEMORY); } @@ -129,7 +129,7 @@ acpi_ds_execute_arguments(struct acpi_namespace_node *node, /* Evaluate the deferred arguments */ - op = acpi_ps_alloc_op(AML_INT_EVAL_SUBTREE_OP); + op = acpi_ps_alloc_op(AML_INT_EVAL_SUBTREE_OP, aml_start); if (!op) { return_ACPI_STATUS(AE_NO_MEMORY); } diff --git a/drivers/acpi/acpica/dscontrol.c b/drivers/acpi/acpica/dscontrol.c index 39da9da62bbf..435fc16e2f83 100644 --- a/drivers/acpi/acpica/dscontrol.c +++ b/drivers/acpi/acpica/dscontrol.c @@ -212,7 +212,7 @@ acpi_ds_exec_end_control_op(struct acpi_walk_state * walk_state, */ control_state->control.loop_count++; if (control_state->control.loop_count > - ACPI_MAX_LOOP_ITERATIONS) { + acpi_gbl_max_loop_iterations) { status = AE_AML_INFINITE_LOOP; break; } diff --git a/drivers/acpi/acpica/dsdebug.c b/drivers/acpi/acpica/dsdebug.c new file mode 100644 index 000000000000..309556efc553 --- /dev/null +++ b/drivers/acpi/acpica/dsdebug.c @@ -0,0 +1,231 @@ +/****************************************************************************** + * + * Module Name: dsdebug - Parser/Interpreter interface - debugging + * + *****************************************************************************/ + +/* + * Copyright (C) 2000 - 2015, Intel Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions, and the following disclaimer, + * without modification. + * 2. Redistributions in binary form must reproduce at minimum a disclaimer + * substantially similar to the "NO WARRANTY" disclaimer below + * ("Disclaimer") and any redistribution must be conditioned upon + * including a substantially similar Disclaimer requirement for further + * binary redistribution. + * 3. Neither the names of the above-listed copyright holders nor the names + * of any contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * Alternatively, this software may be distributed under the terms of the + * GNU General Public License ("GPL") version 2 as published by the Free + * Software Foundation. + * + * NO WARRANTY + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGES. + */ + +#include +#include "accommon.h" +#include "acdispat.h" +#include "acnamesp.h" +#ifdef ACPI_DISASSEMBLER +#include "acdisasm.h" +#endif +#include "acinterp.h" + +#define _COMPONENT ACPI_DISPATCHER +ACPI_MODULE_NAME("dsdebug") + +#if defined(ACPI_DEBUG_OUTPUT) || defined(ACPI_DEBUGGER) +/* Local prototypes */ +static void +acpi_ds_print_node_pathname(struct acpi_namespace_node *node, + const char *message); + +/******************************************************************************* + * + * FUNCTION: acpi_ds_print_node_pathname + * + * PARAMETERS: node - Object + * message - Prefix message + * + * DESCRIPTION: Print an object's full namespace pathname + * Manages allocation/freeing of a pathname buffer + * + ******************************************************************************/ + +static void +acpi_ds_print_node_pathname(struct acpi_namespace_node *node, + const char *message) +{ + struct acpi_buffer buffer; + acpi_status status; + + ACPI_FUNCTION_TRACE(ds_print_node_pathname); + + if (!node) { + ACPI_DEBUG_PRINT_RAW((ACPI_DB_DISPATCH, "[NULL NAME]")); + return_VOID; + } + + /* Convert handle to full pathname and print it (with supplied message) */ + + buffer.length = ACPI_ALLOCATE_LOCAL_BUFFER; + + status = acpi_ns_handle_to_pathname(node, &buffer, TRUE); + if (ACPI_SUCCESS(status)) { + if (message) { + ACPI_DEBUG_PRINT_RAW((ACPI_DB_DISPATCH, "%s ", + message)); + } + + ACPI_DEBUG_PRINT_RAW((ACPI_DB_DISPATCH, "[%s] (Node %p)", + (char *)buffer.pointer, node)); + ACPI_FREE(buffer.pointer); + } + + return_VOID; +} + +/******************************************************************************* + * + * FUNCTION: acpi_ds_dump_method_stack + * + * PARAMETERS: status - Method execution status + * walk_state - Current state of the parse tree walk + * op - Executing parse op + * + * RETURN: None + * + * DESCRIPTION: Called when a method has been aborted because of an error. + * Dumps the method execution stack. + * + ******************************************************************************/ + +void +acpi_ds_dump_method_stack(acpi_status status, + struct acpi_walk_state *walk_state, + union acpi_parse_object *op) +{ + union acpi_parse_object *next; + struct acpi_thread_state *thread; + struct acpi_walk_state *next_walk_state; + struct acpi_namespace_node *previous_method = NULL; + union acpi_operand_object *method_desc; + + ACPI_FUNCTION_TRACE(ds_dump_method_stack); + + /* Ignore control codes, they are not errors */ + + if ((status & AE_CODE_MASK) == AE_CODE_CONTROL) { + return_VOID; + } + + /* We may be executing a deferred opcode */ + + if (walk_state->deferred_node) { + ACPI_DEBUG_PRINT((ACPI_DB_DISPATCH, + "Executing subtree for Buffer/Package/Region\n")); + return_VOID; + } + + /* + * If there is no Thread, we are not actually executing a method. + * This can happen when the iASL compiler calls the interpreter + * to perform constant folding. + */ + thread = walk_state->thread; + if (!thread) { + return_VOID; + } + + /* Display exception and method name */ + + ACPI_DEBUG_PRINT((ACPI_DB_DISPATCH, + "\n**** Exception %s during execution of method ", + acpi_format_exception(status))); + acpi_ds_print_node_pathname(walk_state->method_node, NULL); + + /* Display stack of executing methods */ + + ACPI_DEBUG_PRINT_RAW((ACPI_DB_DISPATCH, + "\n\nMethod Execution Stack:\n")); + next_walk_state = thread->walk_state_list; + + /* Walk list of linked walk states */ + + while (next_walk_state) { + method_desc = next_walk_state->method_desc; + if (method_desc) { + acpi_ex_stop_trace_method((struct acpi_namespace_node *) + method_desc->method.node, + method_desc, walk_state); + } + + ACPI_DEBUG_PRINT((ACPI_DB_DISPATCH, + " Method [%4.4s] executing: ", + acpi_ut_get_node_name(next_walk_state-> + method_node))); + + /* First method is the currently executing method */ + + if (next_walk_state == walk_state) { + if (op) { + + /* Display currently executing ASL statement */ + + next = op->common.next; + op->common.next = NULL; + +#ifdef ACPI_DISASSEMBLER + acpi_dm_disassemble(next_walk_state, op, + ACPI_UINT32_MAX); +#endif + op->common.next = next; + } + } else { + /* + * This method has called another method + * NOTE: the method call parse subtree is already deleted at this + * point, so we cannot disassemble the method invocation. + */ + ACPI_DEBUG_PRINT_RAW((ACPI_DB_DISPATCH, + "Call to method ")); + acpi_ds_print_node_pathname(previous_method, NULL); + } + + previous_method = next_walk_state->method_node; + next_walk_state = next_walk_state->next; + ACPI_DEBUG_PRINT_RAW((ACPI_DB_DISPATCH, "\n")); + } + + return_VOID; +} + +#else +void +acpi_ds_dump_method_stack(acpi_status status, + struct acpi_walk_state *walk_state, + union acpi_parse_object *op) +{ + return; +} + +#endif diff --git a/drivers/acpi/acpica/dsinit.c b/drivers/acpi/acpica/dsinit.c index 95779e8ec3bb..920f1b199bc6 100644 --- a/drivers/acpi/acpica/dsinit.c +++ b/drivers/acpi/acpica/dsinit.c @@ -237,12 +237,22 @@ acpi_ds_initialize_objects(u32 table_index, return_ACPI_STATUS(status); } + /* DSDT is always the first AML table */ + + if (ACPI_COMPARE_NAME(table->signature, ACPI_SIG_DSDT)) { + ACPI_DEBUG_PRINT_RAW((ACPI_DB_INIT, + "\nInitializing Namespace objects:\n")); + } + + /* Summary of objects initialized */ + ACPI_DEBUG_PRINT_RAW((ACPI_DB_INIT, - "Table [%4.4s] (id %4.4X) - %4u Objects with %3u Devices, " - "%3u Regions, %3u Methods (%u/%u/%u Serial/Non/Cvt)\n", - table->signature, owner_id, info.object_count, - info.device_count, info.op_region_count, - info.method_count, info.serial_method_count, + "Table [%4.4s:%8.8s] (id %.2X) - %4u Objects with %3u Devices, " + "%3u Regions, %4u Methods (%u/%u/%u Serial/Non/Cvt)\n", + table->signature, table->oem_table_id, owner_id, + info.object_count, info.device_count, + info.op_region_count, info.method_count, + info.serial_method_count, info.non_serial_method_count, info.serialized_method_count)); diff --git a/drivers/acpi/acpica/dsmethod.c b/drivers/acpi/acpica/dsmethod.c index 85bb951430d9..bc32f3194afe 100644 --- a/drivers/acpi/acpica/dsmethod.c +++ b/drivers/acpi/acpica/dsmethod.c @@ -46,11 +46,9 @@ #include "acdispat.h" #include "acinterp.h" #include "acnamesp.h" -#ifdef ACPI_DISASSEMBLER -#include "acdisasm.h" -#endif #include "acparser.h" #include "amlcode.h" +#include "acdebug.h" #define _COMPONENT ACPI_DISPATCHER ACPI_MODULE_NAME("dsmethod") @@ -103,7 +101,7 @@ acpi_ds_auto_serialize_method(struct acpi_namespace_node *node, /* Create/Init a root op for the method parse tree */ - op = acpi_ps_alloc_op(AML_METHOD_OP); + op = acpi_ps_alloc_op(AML_METHOD_OP, obj_desc->method.aml_start); if (!op) { return_ACPI_STATUS(AE_NO_MEMORY); } @@ -205,7 +203,7 @@ acpi_ds_detect_named_opcodes(struct acpi_walk_state *walk_state, * RETURN: Status * * DESCRIPTION: Called on method error. Invoke the global exception handler if - * present, dump the method data if the disassembler is configured + * present, dump the method data if the debugger is configured * * Note: Allows the exception handler to change the status code * @@ -214,6 +212,8 @@ acpi_ds_detect_named_opcodes(struct acpi_walk_state *walk_state, acpi_status acpi_ds_method_error(acpi_status status, struct acpi_walk_state * walk_state) { + u32 aml_offset; + ACPI_FUNCTION_ENTRY(); /* Ignore AE_OK and control exception codes */ @@ -234,26 +234,30 @@ acpi_ds_method_error(acpi_status status, struct acpi_walk_state * walk_state) * Handler can map the exception code to anything it wants, including * AE_OK, in which case the executing method will not be aborted. */ + aml_offset = (u32)ACPI_PTR_DIFF(walk_state->aml, + walk_state->parser_state. + aml_start); + status = acpi_gbl_exception_handler(status, walk_state->method_node ? walk_state->method_node-> name.integer : 0, walk_state->opcode, - walk_state->aml_offset, - NULL); + aml_offset, NULL); acpi_ex_enter_interpreter(); } acpi_ds_clear_implicit_return(walk_state); -#ifdef ACPI_DISASSEMBLER if (ACPI_FAILURE(status)) { + acpi_ds_dump_method_stack(status, walk_state, walk_state->op); - /* Display method locals/args if disassembler is present */ + /* Display method locals/args if debugger is present */ - acpi_dm_dump_method_info(status, walk_state, walk_state->op); - } +#ifdef ACPI_DEBUGGER + acpi_db_dump_method_info(status, walk_state); #endif + } return (status); } @@ -328,6 +332,8 @@ acpi_ds_begin_method_execution(struct acpi_namespace_node *method_node, return_ACPI_STATUS(AE_NULL_ENTRY); } + acpi_ex_start_trace_method(method_node, obj_desc, walk_state); + /* Prevent wraparound of thread count */ if (obj_desc->method.thread_count == ACPI_UINT8_MAX) { @@ -574,9 +580,7 @@ acpi_ds_call_control_method(struct acpi_thread_state *thread, /* On error, we must terminate the method properly */ acpi_ds_terminate_control_method(obj_desc, next_walk_state); - if (next_walk_state) { - acpi_ds_delete_walk_state(next_walk_state); - } + acpi_ds_delete_walk_state(next_walk_state); return_ACPI_STATUS(status); } @@ -826,5 +830,8 @@ acpi_ds_terminate_control_method(union acpi_operand_object *method_desc, } } + acpi_ex_stop_trace_method((struct acpi_namespace_node *)method_desc-> + method.node, method_desc, walk_state); + return_VOID; } diff --git a/drivers/acpi/acpica/dsopcode.c b/drivers/acpi/acpica/dsopcode.c index ea0cc4e08f80..81d7b9863e32 100644 --- a/drivers/acpi/acpica/dsopcode.c +++ b/drivers/acpi/acpica/dsopcode.c @@ -480,8 +480,8 @@ acpi_ds_eval_table_region_operands(struct acpi_walk_state *walk_state, union acpi_operand_object **operand; struct acpi_namespace_node *node; union acpi_parse_object *next_op; - u32 table_index; struct acpi_table_header *table; + u32 table_index; ACPI_FUNCTION_TRACE_PTR(ds_eval_table_region_operands, op); @@ -504,6 +504,8 @@ acpi_ds_eval_table_region_operands(struct acpi_walk_state *walk_state, return_ACPI_STATUS(status); } + operand = &walk_state->operands[0]; + /* * Resolve the Signature string, oem_id string, * and oem_table_id string operands @@ -511,32 +513,34 @@ acpi_ds_eval_table_region_operands(struct acpi_walk_state *walk_state, status = acpi_ex_resolve_operands(op->common.aml_opcode, ACPI_WALK_OPERANDS, walk_state); if (ACPI_FAILURE(status)) { - return_ACPI_STATUS(status); + goto cleanup; } - operand = &walk_state->operands[0]; - /* Find the ACPI table */ status = acpi_tb_find_table(operand[0]->string.pointer, operand[1]->string.pointer, operand[2]->string.pointer, &table_index); if (ACPI_FAILURE(status)) { - return_ACPI_STATUS(status); + if (status == AE_NOT_FOUND) { + ACPI_ERROR((AE_INFO, + "ACPI Table [%4.4s] OEM:(%s, %s) not found in RSDT/XSDT", + operand[0]->string.pointer, + operand[1]->string.pointer, + operand[2]->string.pointer)); + } + goto cleanup; } - acpi_ut_remove_reference(operand[0]); - acpi_ut_remove_reference(operand[1]); - acpi_ut_remove_reference(operand[2]); - status = acpi_get_table_by_index(table_index, &table); if (ACPI_FAILURE(status)) { - return_ACPI_STATUS(status); + goto cleanup; } obj_desc = acpi_ns_get_attached_object(node); if (!obj_desc) { - return_ACPI_STATUS(AE_NOT_EXIST); + status = AE_NOT_EXIST; + goto cleanup; } obj_desc->region.address = ACPI_PTR_TO_PHYSADDR(table); @@ -551,6 +555,11 @@ acpi_ds_eval_table_region_operands(struct acpi_walk_state *walk_state, obj_desc->region.flags |= AOPOBJ_DATA_VALID; +cleanup: + acpi_ut_remove_reference(operand[0]); + acpi_ut_remove_reference(operand[1]); + acpi_ut_remove_reference(operand[2]); + return_ACPI_STATUS(status); } diff --git a/drivers/acpi/acpica/dswload.c b/drivers/acpi/acpica/dswload.c index 845ff44919c3..097188a6b1c1 100644 --- a/drivers/acpi/acpica/dswload.c +++ b/drivers/acpi/acpica/dswload.c @@ -388,7 +388,7 @@ acpi_ds_load1_begin_op(struct acpi_walk_state * walk_state, /* Create a new op */ - op = acpi_ps_alloc_op(walk_state->opcode); + op = acpi_ps_alloc_op(walk_state->opcode, walk_state->aml); if (!op) { return_ACPI_STATUS(AE_NO_MEMORY); } diff --git a/drivers/acpi/acpica/dswload2.c b/drivers/acpi/acpica/dswload2.c index fcaa30c611fb..e2c08cd79aca 100644 --- a/drivers/acpi/acpica/dswload2.c +++ b/drivers/acpi/acpica/dswload2.c @@ -335,7 +335,7 @@ acpi_ds_load2_begin_op(struct acpi_walk_state *walk_state, /* Create a new op */ - op = acpi_ps_alloc_op(walk_state->opcode); + op = acpi_ps_alloc_op(walk_state->opcode, walk_state->aml); if (!op) { return_ACPI_STATUS(AE_NO_MEMORY); } diff --git a/drivers/acpi/acpica/evregion.c b/drivers/acpi/acpica/evregion.c index 2ba28a63fb68..5ee79a16fe33 100644 --- a/drivers/acpi/acpica/evregion.c +++ b/drivers/acpi/acpica/evregion.c @@ -626,9 +626,17 @@ acpi_ev_execute_reg_methods(struct acpi_namespace_node *node, acpi_adr_space_type space_id) { acpi_status status; + struct acpi_reg_walk_info info; ACPI_FUNCTION_TRACE(ev_execute_reg_methods); + info.space_id = space_id; + info.reg_run_count = 0; + + ACPI_DEBUG_PRINT_RAW((ACPI_DB_NAMES, + " Running _REG methods for SpaceId %s\n", + acpi_ut_get_region_name(info.space_id))); + /* * Run all _REG methods for all Operation Regions for this space ID. This * is a separate walk in order to handle any interdependencies between @@ -637,7 +645,7 @@ acpi_ev_execute_reg_methods(struct acpi_namespace_node *node, */ status = acpi_ns_walk_namespace(ACPI_TYPE_ANY, node, ACPI_UINT32_MAX, ACPI_NS_WALK_UNLOCK, acpi_ev_reg_run, - NULL, &space_id, NULL); + NULL, &info, NULL); /* Special case for EC: handle "orphan" _REG methods with no region */ @@ -645,6 +653,11 @@ acpi_ev_execute_reg_methods(struct acpi_namespace_node *node, acpi_ev_orphan_ec_reg_method(node); } + ACPI_DEBUG_PRINT_RAW((ACPI_DB_NAMES, + " Executed %u _REG methods for SpaceId %s\n", + info.reg_run_count, + acpi_ut_get_region_name(info.space_id))); + return_ACPI_STATUS(status); } @@ -664,10 +677,10 @@ acpi_ev_reg_run(acpi_handle obj_handle, { union acpi_operand_object *obj_desc; struct acpi_namespace_node *node; - acpi_adr_space_type space_id; acpi_status status; + struct acpi_reg_walk_info *info; - space_id = *ACPI_CAST_PTR(acpi_adr_space_type, context); + info = ACPI_CAST_PTR(struct acpi_reg_walk_info, context); /* Convert and validate the device handle */ @@ -696,13 +709,14 @@ acpi_ev_reg_run(acpi_handle obj_handle, /* Object is a Region */ - if (obj_desc->region.space_id != space_id) { + if (obj_desc->region.space_id != info->space_id) { /* This region is for a different address space, just ignore it */ return (AE_OK); } + info->reg_run_count++; status = acpi_ev_execute_reg_method(obj_desc, ACPI_REG_CONNECT); return (status); } diff --git a/drivers/acpi/acpica/exconfig.c b/drivers/acpi/acpica/exconfig.c index 24a4c5c2b124..b540913c11ac 100644 --- a/drivers/acpi/acpica/exconfig.c +++ b/drivers/acpi/acpica/exconfig.c @@ -162,14 +162,6 @@ acpi_ex_load_table_op(struct acpi_walk_state *walk_state, ACPI_FUNCTION_TRACE(ex_load_table_op); - /* Validate lengths for the Signature, oem_id, and oem_table_id strings */ - - if ((operand[0]->string.length > ACPI_NAME_SIZE) || - (operand[1]->string.length > ACPI_OEM_ID_SIZE) || - (operand[2]->string.length > ACPI_OEM_TABLE_ID_SIZE)) { - return_ACPI_STATUS(AE_AML_STRING_LIMIT); - } - /* Find the ACPI table in the RSDT/XSDT */ status = acpi_tb_find_table(operand[0]->string.pointer, diff --git a/drivers/acpi/acpica/excreate.c b/drivers/acpi/acpica/excreate.c index aaeea4840aaa..ccb7219bdcee 100644 --- a/drivers/acpi/acpica/excreate.c +++ b/drivers/acpi/acpica/excreate.c @@ -486,6 +486,7 @@ acpi_ex_create_method(u8 * aml_start, obj_desc->method.aml_start = aml_start; obj_desc->method.aml_length = aml_length; + obj_desc->method.node = operand[0]; /* * Disassemble the method flags. Split off the arg_count, Serialized diff --git a/drivers/acpi/acpica/exdebug.c b/drivers/acpi/acpica/exdebug.c index 815442bbd051..de92458236f5 100644 --- a/drivers/acpi/acpica/exdebug.c +++ b/drivers/acpi/acpica/exdebug.c @@ -43,11 +43,21 @@ #include #include "accommon.h" +#include "acnamesp.h" #include "acinterp.h" +#include "acparser.h" #define _COMPONENT ACPI_EXECUTER ACPI_MODULE_NAME("exdebug") +static union acpi_operand_object *acpi_gbl_trace_method_object = NULL; + +/* Local prototypes */ + +#ifdef ACPI_DEBUG_OUTPUT +static const char *acpi_ex_get_trace_event_name(acpi_trace_event_type type); +#endif + #ifndef ACPI_NO_ERROR_MESSAGES /******************************************************************************* * @@ -70,6 +80,7 @@ ACPI_MODULE_NAME("exdebug") * enabled if necessary. * ******************************************************************************/ + void acpi_ex_do_debug_object(union acpi_operand_object *source_desc, u32 level, u32 index) @@ -308,3 +319,316 @@ acpi_ex_do_debug_object(union acpi_operand_object *source_desc, return_VOID; } #endif + +/******************************************************************************* + * + * FUNCTION: acpi_ex_interpreter_trace_enabled + * + * PARAMETERS: name - Whether method name should be matched, + * this should be checked before starting + * the tracer + * + * RETURN: TRUE if interpreter trace is enabled. + * + * DESCRIPTION: Check whether interpreter trace is enabled + * + ******************************************************************************/ + +static u8 acpi_ex_interpreter_trace_enabled(char *name) +{ + + /* Check if tracing is enabled */ + + if (!(acpi_gbl_trace_flags & ACPI_TRACE_ENABLED)) { + return (FALSE); + } + + /* + * Check if tracing is filtered: + * + * 1. If the tracer is started, acpi_gbl_trace_method_object should have + * been filled by the trace starter + * 2. If the tracer is not started, acpi_gbl_trace_method_name should be + * matched if it is specified + * 3. If the tracer is oneshot style, acpi_gbl_trace_method_name should + * not be cleared by the trace stopper during the first match + */ + if (acpi_gbl_trace_method_object) { + return (TRUE); + } + if (name && + (acpi_gbl_trace_method_name && + strcmp(acpi_gbl_trace_method_name, name))) { + return (FALSE); + } + if ((acpi_gbl_trace_flags & ACPI_TRACE_ONESHOT) && + !acpi_gbl_trace_method_name) { + return (FALSE); + } + + return (TRUE); +} + +/******************************************************************************* + * + * FUNCTION: acpi_ex_get_trace_event_name + * + * PARAMETERS: type - Trace event type + * + * RETURN: Trace event name. + * + * DESCRIPTION: Used to obtain the full trace event name. + * + ******************************************************************************/ + +#ifdef ACPI_DEBUG_OUTPUT + +static const char *acpi_ex_get_trace_event_name(acpi_trace_event_type type) +{ + switch (type) { + case ACPI_TRACE_AML_METHOD: + + return "Method"; + + case ACPI_TRACE_AML_OPCODE: + + return "Opcode"; + + case ACPI_TRACE_AML_REGION: + + return "Region"; + + default: + + return ""; + } +} + +#endif + +/******************************************************************************* + * + * FUNCTION: acpi_ex_trace_point + * + * PARAMETERS: type - Trace event type + * begin - TRUE if before execution + * aml - Executed AML address + * pathname - Object path + * + * RETURN: None + * + * DESCRIPTION: Internal interpreter execution trace. + * + ******************************************************************************/ + +void +acpi_ex_trace_point(acpi_trace_event_type type, + u8 begin, u8 *aml, char *pathname) +{ + + ACPI_FUNCTION_NAME(ex_trace_point); + + if (pathname) { + ACPI_DEBUG_PRINT((ACPI_DB_TRACE_POINT, + "%s %s [0x%p:%s] execution.\n", + acpi_ex_get_trace_event_name(type), + begin ? "Begin" : "End", aml, pathname)); + } else { + ACPI_DEBUG_PRINT((ACPI_DB_TRACE_POINT, + "%s %s [0x%p] execution.\n", + acpi_ex_get_trace_event_name(type), + begin ? "Begin" : "End", aml)); + } +} + +/******************************************************************************* + * + * FUNCTION: acpi_ex_start_trace_method + * + * PARAMETERS: method_node - Node of the method + * obj_desc - The method object + * walk_state - current state, NULL if not yet executing + * a method. + * + * RETURN: None + * + * DESCRIPTION: Start control method execution trace + * + ******************************************************************************/ + +void +acpi_ex_start_trace_method(struct acpi_namespace_node *method_node, + union acpi_operand_object *obj_desc, + struct acpi_walk_state *walk_state) +{ + acpi_status status; + char *pathname = NULL; + u8 enabled = FALSE; + + ACPI_FUNCTION_NAME(ex_start_trace_method); + + if (method_node) { + pathname = acpi_ns_get_normalized_pathname(method_node, TRUE); + } + + status = acpi_ut_acquire_mutex(ACPI_MTX_NAMESPACE); + if (ACPI_FAILURE(status)) { + goto exit; + } + + enabled = acpi_ex_interpreter_trace_enabled(pathname); + if (enabled && !acpi_gbl_trace_method_object) { + acpi_gbl_trace_method_object = obj_desc; + acpi_gbl_original_dbg_level = acpi_dbg_level; + acpi_gbl_original_dbg_layer = acpi_dbg_layer; + acpi_dbg_level = ACPI_TRACE_LEVEL_ALL; + acpi_dbg_layer = ACPI_TRACE_LAYER_ALL; + + if (acpi_gbl_trace_dbg_level) { + acpi_dbg_level = acpi_gbl_trace_dbg_level; + } + if (acpi_gbl_trace_dbg_layer) { + acpi_dbg_layer = acpi_gbl_trace_dbg_layer; + } + } + (void)acpi_ut_release_mutex(ACPI_MTX_NAMESPACE); + +exit: + if (enabled) { + ACPI_TRACE_POINT(ACPI_TRACE_AML_METHOD, TRUE, + obj_desc ? obj_desc->method.aml_start : NULL, + pathname); + } + if (pathname) { + ACPI_FREE(pathname); + } +} + +/******************************************************************************* + * + * FUNCTION: acpi_ex_stop_trace_method + * + * PARAMETERS: method_node - Node of the method + * obj_desc - The method object + * walk_state - current state, NULL if not yet executing + * a method. + * + * RETURN: None + * + * DESCRIPTION: Stop control method execution trace + * + ******************************************************************************/ + +void +acpi_ex_stop_trace_method(struct acpi_namespace_node *method_node, + union acpi_operand_object *obj_desc, + struct acpi_walk_state *walk_state) +{ + acpi_status status; + char *pathname = NULL; + u8 enabled; + + ACPI_FUNCTION_NAME(ex_stop_trace_method); + + if (method_node) { + pathname = acpi_ns_get_normalized_pathname(method_node, TRUE); + } + + status = acpi_ut_acquire_mutex(ACPI_MTX_NAMESPACE); + if (ACPI_FAILURE(status)) { + goto exit_path; + } + + enabled = acpi_ex_interpreter_trace_enabled(NULL); + + (void)acpi_ut_release_mutex(ACPI_MTX_NAMESPACE); + + if (enabled) { + ACPI_TRACE_POINT(ACPI_TRACE_AML_METHOD, FALSE, + obj_desc ? obj_desc->method.aml_start : NULL, + pathname); + } + + status = acpi_ut_acquire_mutex(ACPI_MTX_NAMESPACE); + if (ACPI_FAILURE(status)) { + goto exit_path; + } + + /* Check whether the tracer should be stopped */ + + if (acpi_gbl_trace_method_object == obj_desc) { + + /* Disable further tracing if type is one-shot */ + + if (acpi_gbl_trace_flags & ACPI_TRACE_ONESHOT) { + acpi_gbl_trace_method_name = NULL; + } + + acpi_dbg_level = acpi_gbl_original_dbg_level; + acpi_dbg_layer = acpi_gbl_original_dbg_layer; + acpi_gbl_trace_method_object = NULL; + } + + (void)acpi_ut_release_mutex(ACPI_MTX_NAMESPACE); + +exit_path: + if (pathname) { + ACPI_FREE(pathname); + } +} + +/******************************************************************************* + * + * FUNCTION: acpi_ex_start_trace_opcode + * + * PARAMETERS: op - The parser opcode object + * walk_state - current state, NULL if not yet executing + * a method. + * + * RETURN: None + * + * DESCRIPTION: Start opcode execution trace + * + ******************************************************************************/ + +void +acpi_ex_start_trace_opcode(union acpi_parse_object *op, + struct acpi_walk_state *walk_state) +{ + + ACPI_FUNCTION_NAME(ex_start_trace_opcode); + + if (acpi_ex_interpreter_trace_enabled(NULL) && + (acpi_gbl_trace_flags & ACPI_TRACE_OPCODE)) { + ACPI_TRACE_POINT(ACPI_TRACE_AML_OPCODE, TRUE, + op->common.aml, op->common.aml_op_name); + } +} + +/******************************************************************************* + * + * FUNCTION: acpi_ex_stop_trace_opcode + * + * PARAMETERS: op - The parser opcode object + * walk_state - current state, NULL if not yet executing + * a method. + * + * RETURN: None + * + * DESCRIPTION: Stop opcode execution trace + * + ******************************************************************************/ + +void +acpi_ex_stop_trace_opcode(union acpi_parse_object *op, + struct acpi_walk_state *walk_state) +{ + + ACPI_FUNCTION_NAME(ex_stop_trace_opcode); + + if (acpi_ex_interpreter_trace_enabled(NULL) && + (acpi_gbl_trace_flags & ACPI_TRACE_OPCODE)) { + ACPI_TRACE_POINT(ACPI_TRACE_AML_OPCODE, FALSE, + op->common.aml, op->common.aml_op_name); + } +} diff --git a/drivers/acpi/acpica/exdump.c b/drivers/acpi/acpica/exdump.c index 401e7edcd419..d836f888bb16 100644 --- a/drivers/acpi/acpica/exdump.c +++ b/drivers/acpi/acpica/exdump.c @@ -995,9 +995,8 @@ static void acpi_ex_dump_reference_obj(union acpi_operand_object *obj_desc) if (obj_desc->reference.class == ACPI_REFCLASS_NAME) { acpi_os_printf(" %p ", obj_desc->reference.node); - status = - acpi_ns_handle_to_pathname(obj_desc->reference.node, - &ret_buf); + status = acpi_ns_handle_to_pathname(obj_desc->reference.node, + &ret_buf, TRUE); if (ACPI_FAILURE(status)) { acpi_os_printf(" Could not convert name to pathname\n"); } else { diff --git a/drivers/acpi/acpica/exresnte.c b/drivers/acpi/acpica/exresnte.c index c7e3b929aa85..1b372ef69308 100644 --- a/drivers/acpi/acpica/exresnte.c +++ b/drivers/acpi/acpica/exresnte.c @@ -126,7 +126,7 @@ acpi_ex_resolve_node_to_value(struct acpi_namespace_node **object_ptr, if (!source_desc) { ACPI_ERROR((AE_INFO, "No object attached to node [%4.4s] %p", node->name.ascii, node)); - return_ACPI_STATUS(AE_AML_NO_OPERAND); + return_ACPI_STATUS(AE_AML_UNINITIALIZED_NODE); } /* diff --git a/drivers/acpi/acpica/exresolv.c b/drivers/acpi/acpica/exresolv.c index b6b7f3af29e4..7b109128b035 100644 --- a/drivers/acpi/acpica/exresolv.c +++ b/drivers/acpi/acpica/exresolv.c @@ -337,8 +337,9 @@ acpi_ex_resolve_multiple(struct acpi_walk_state *walk_state, acpi_object_type * return_type, union acpi_operand_object **return_desc) { - union acpi_operand_object *obj_desc = (void *)operand; - struct acpi_namespace_node *node; + union acpi_operand_object *obj_desc = ACPI_CAST_PTR(void, operand); + struct acpi_namespace_node *node = + ACPI_CAST_PTR(struct acpi_namespace_node, operand); acpi_object_type type; acpi_status status; @@ -355,9 +356,7 @@ acpi_ex_resolve_multiple(struct acpi_walk_state *walk_state, case ACPI_DESC_TYPE_NAMED: type = ((struct acpi_namespace_node *)obj_desc)->type; - obj_desc = - acpi_ns_get_attached_object((struct acpi_namespace_node *) - obj_desc); + obj_desc = acpi_ns_get_attached_object(node); /* If we had an Alias node, use the attached object for type info */ @@ -368,6 +367,13 @@ acpi_ex_resolve_multiple(struct acpi_walk_state *walk_state, acpi_namespace_node *) obj_desc); } + + if (!obj_desc) { + ACPI_ERROR((AE_INFO, + "[%4.4s] Node is unresolved or uninitialized", + acpi_ut_get_node_name(node))); + return_ACPI_STATUS(AE_AML_UNINITIALIZED_NODE); + } break; default: diff --git a/drivers/acpi/acpica/hwxfsleep.c b/drivers/acpi/acpica/hwxfsleep.c index 52dfd0d050fa..d62a61612b3f 100644 --- a/drivers/acpi/acpica/hwxfsleep.c +++ b/drivers/acpi/acpica/hwxfsleep.c @@ -160,19 +160,8 @@ acpi_set_firmware_waking_vectors(acpi_physical_address physical_address, ACPI_FUNCTION_TRACE(acpi_set_firmware_waking_vectors); - /* If Hardware Reduced flag is set, there is no FACS */ - - if (acpi_gbl_reduced_hardware) { - return_ACPI_STATUS (AE_OK); - } - - if (acpi_gbl_facs32) { - (void)acpi_hw_set_firmware_waking_vectors(acpi_gbl_facs32, - physical_address, - physical_address64); - } - if (acpi_gbl_facs64) { - (void)acpi_hw_set_firmware_waking_vectors(acpi_gbl_facs64, + if (acpi_gbl_FACS) { + (void)acpi_hw_set_firmware_waking_vectors(acpi_gbl_FACS, physical_address, physical_address64); } diff --git a/drivers/acpi/acpica/nseval.c b/drivers/acpi/acpica/nseval.c index 80670cb32b5a..7eba578d36f3 100644 --- a/drivers/acpi/acpica/nseval.c +++ b/drivers/acpi/acpica/nseval.c @@ -274,6 +274,7 @@ acpi_status acpi_ns_evaluate(struct acpi_evaluate_info *info) acpi_ex_exit_interpreter(); if (ACPI_FAILURE(status)) { + info->return_object = NULL; goto cleanup; } @@ -464,7 +465,8 @@ acpi_ns_exec_module_code(union acpi_operand_object *method_obj, status = acpi_ns_evaluate(info); - ACPI_DEBUG_PRINT((ACPI_DB_INIT, "Executed module-level code at %p\n", + ACPI_DEBUG_PRINT((ACPI_DB_INIT_NAMES, + "Executed module-level code at %p\n", method_obj->method.aml_start)); /* Delete a possible implicit return value (in slack mode) */ diff --git a/drivers/acpi/acpica/nsload.c b/drivers/acpi/acpica/nsload.c index bd6cd4a81316..14ab83668207 100644 --- a/drivers/acpi/acpica/nsload.c +++ b/drivers/acpi/acpica/nsload.c @@ -111,7 +111,21 @@ acpi_ns_load_table(u32 table_index, struct acpi_namespace_node *node) if (ACPI_SUCCESS(status)) { acpi_tb_set_table_loaded_flag(table_index, TRUE); } else { - (void)acpi_tb_release_owner_id(table_index); + /* + * On error, delete any namespace objects created by this table. + * We cannot initialize these objects, so delete them. There are + * a couple of expecially bad cases: + * AE_ALREADY_EXISTS - namespace collision. + * AE_NOT_FOUND - the target of a Scope operator does not + * exist. This target of Scope must already exist in the + * namespace, as per the ACPI specification. + */ + (void)acpi_ut_release_mutex(ACPI_MTX_NAMESPACE); + acpi_ns_delete_namespace_by_owner(acpi_gbl_root_table_list. + tables[table_index].owner_id); + acpi_tb_release_owner_id(table_index); + + return_ACPI_STATUS(status); } unlock: diff --git a/drivers/acpi/acpica/nsnames.c b/drivers/acpi/acpica/nsnames.c index d293d9748036..8934b4eddb73 100644 --- a/drivers/acpi/acpica/nsnames.c +++ b/drivers/acpi/acpica/nsnames.c @@ -49,73 +49,6 @@ #define _COMPONENT ACPI_NAMESPACE ACPI_MODULE_NAME("nsnames") -/******************************************************************************* - * - * FUNCTION: acpi_ns_build_external_path - * - * PARAMETERS: node - NS node whose pathname is needed - * size - Size of the pathname - * *name_buffer - Where to return the pathname - * - * RETURN: Status - * Places the pathname into the name_buffer, in external format - * (name segments separated by path separators) - * - * DESCRIPTION: Generate a full pathaname - * - ******************************************************************************/ -acpi_status -acpi_ns_build_external_path(struct acpi_namespace_node *node, - acpi_size size, char *name_buffer) -{ - acpi_size index; - struct acpi_namespace_node *parent_node; - - ACPI_FUNCTION_ENTRY(); - - /* Special case for root */ - - index = size - 1; - if (index < ACPI_NAME_SIZE) { - name_buffer[0] = AML_ROOT_PREFIX; - name_buffer[1] = 0; - return (AE_OK); - } - - /* Store terminator byte, then build name backwards */ - - parent_node = node; - name_buffer[index] = 0; - - while ((index > ACPI_NAME_SIZE) && (parent_node != acpi_gbl_root_node)) { - index -= ACPI_NAME_SIZE; - - /* Put the name into the buffer */ - - ACPI_MOVE_32_TO_32((name_buffer + index), &parent_node->name); - parent_node = parent_node->parent; - - /* Prefix name with the path separator */ - - index--; - name_buffer[index] = ACPI_PATH_SEPARATOR; - } - - /* Overwrite final separator with the root prefix character */ - - name_buffer[index] = AML_ROOT_PREFIX; - - if (index != 0) { - ACPI_ERROR((AE_INFO, - "Could not construct external pathname; index=%u, size=%u, Path=%s", - (u32) index, (u32) size, &name_buffer[size])); - - return (AE_BAD_PARAMETER); - } - - return (AE_OK); -} - /******************************************************************************* * * FUNCTION: acpi_ns_get_external_pathname @@ -130,37 +63,13 @@ acpi_ns_build_external_path(struct acpi_namespace_node *node, * for error and debug statements. * ******************************************************************************/ - char *acpi_ns_get_external_pathname(struct acpi_namespace_node *node) { - acpi_status status; char *name_buffer; - acpi_size size; ACPI_FUNCTION_TRACE_PTR(ns_get_external_pathname, node); - /* Calculate required buffer size based on depth below root */ - - size = acpi_ns_get_pathname_length(node); - if (!size) { - return_PTR(NULL); - } - - /* Allocate a buffer to be returned to caller */ - - name_buffer = ACPI_ALLOCATE_ZEROED(size); - if (!name_buffer) { - ACPI_ERROR((AE_INFO, "Could not allocate %u bytes", (u32)size)); - return_PTR(NULL); - } - - /* Build the path in the allocated buffer */ - - status = acpi_ns_build_external_path(node, size, name_buffer); - if (ACPI_FAILURE(status)) { - ACPI_FREE(name_buffer); - return_PTR(NULL); - } + name_buffer = acpi_ns_get_normalized_pathname(node, FALSE); return_PTR(name_buffer); } @@ -180,33 +89,12 @@ char *acpi_ns_get_external_pathname(struct acpi_namespace_node *node) acpi_size acpi_ns_get_pathname_length(struct acpi_namespace_node *node) { acpi_size size; - struct acpi_namespace_node *next_node; ACPI_FUNCTION_ENTRY(); - /* - * Compute length of pathname as 5 * number of name segments. - * Go back up the parent tree to the root - */ - size = 0; - next_node = node; + size = acpi_ns_build_normalized_path(node, NULL, 0, FALSE); - while (next_node && (next_node != acpi_gbl_root_node)) { - if (ACPI_GET_DESCRIPTOR_TYPE(next_node) != ACPI_DESC_TYPE_NAMED) { - ACPI_ERROR((AE_INFO, - "Invalid Namespace Node (%p) while traversing namespace", - next_node)); - return (0); - } - size += ACPI_PATH_SEGMENT_LENGTH; - next_node = next_node->parent; - } - - if (!size) { - size = 1; /* Root node case */ - } - - return (size + 1); /* +1 for null string terminator */ + return (size); } /******************************************************************************* @@ -216,6 +104,8 @@ acpi_size acpi_ns_get_pathname_length(struct acpi_namespace_node *node) * PARAMETERS: target_handle - Handle of named object whose name is * to be found * buffer - Where the pathname is returned + * no_trailing - Remove trailing '_' for each name + * segment * * RETURN: Status, Buffer is filled with pathname if status is AE_OK * @@ -225,7 +115,7 @@ acpi_size acpi_ns_get_pathname_length(struct acpi_namespace_node *node) acpi_status acpi_ns_handle_to_pathname(acpi_handle target_handle, - struct acpi_buffer * buffer) + struct acpi_buffer * buffer, u8 no_trailing) { acpi_status status; struct acpi_namespace_node *node; @@ -240,7 +130,8 @@ acpi_ns_handle_to_pathname(acpi_handle target_handle, /* Determine size required for the caller buffer */ - required_size = acpi_ns_get_pathname_length(node); + required_size = + acpi_ns_build_normalized_path(node, NULL, 0, no_trailing); if (!required_size) { return_ACPI_STATUS(AE_BAD_PARAMETER); } @@ -254,8 +145,8 @@ acpi_ns_handle_to_pathname(acpi_handle target_handle, /* Build the path in the caller buffer */ - status = - acpi_ns_build_external_path(node, required_size, buffer->pointer); + (void)acpi_ns_build_normalized_path(node, buffer->pointer, + required_size, no_trailing); if (ACPI_FAILURE(status)) { return_ACPI_STATUS(status); } @@ -264,3 +155,149 @@ acpi_ns_handle_to_pathname(acpi_handle target_handle, (char *)buffer->pointer, (u32) required_size)); return_ACPI_STATUS(AE_OK); } + +/******************************************************************************* + * + * FUNCTION: acpi_ns_build_normalized_path + * + * PARAMETERS: node - Namespace node + * full_path - Where the path name is returned + * path_size - Size of returned path name buffer + * no_trailing - Remove trailing '_' from each name segment + * + * RETURN: Return 1 if the AML path is empty, otherwise returning (length + * of pathname + 1) which means the 'FullPath' contains a trailing + * null. + * + * DESCRIPTION: Build and return a full namespace pathname. + * Note that if the size of 'FullPath' isn't large enough to + * contain the namespace node's path name, the actual required + * buffer length is returned, and it should be greater than + * 'PathSize'. So callers are able to check the returning value + * to determine the buffer size of 'FullPath'. + * + ******************************************************************************/ + +u32 +acpi_ns_build_normalized_path(struct acpi_namespace_node *node, + char *full_path, u32 path_size, u8 no_trailing) +{ + u32 length = 0, i; + char name[ACPI_NAME_SIZE]; + u8 do_no_trailing; + char c, *left, *right; + struct acpi_namespace_node *next_node; + + ACPI_FUNCTION_TRACE_PTR(ns_build_normalized_path, node); + +#define ACPI_PATH_PUT8(path, size, byte, length) \ + do { \ + if ((length) < (size)) \ + { \ + (path)[(length)] = (byte); \ + } \ + (length)++; \ + } while (0) + + /* + * Make sure the path_size is correct, so that we don't need to + * validate both full_path and path_size. + */ + if (!full_path) { + path_size = 0; + } + + if (!node) { + goto build_trailing_null; + } + + next_node = node; + while (next_node && next_node != acpi_gbl_root_node) { + if (next_node != node) { + ACPI_PATH_PUT8(full_path, path_size, + AML_DUAL_NAME_PREFIX, length); + } + ACPI_MOVE_32_TO_32(name, &next_node->name); + do_no_trailing = no_trailing; + for (i = 0; i < 4; i++) { + c = name[4 - i - 1]; + if (do_no_trailing && c != '_') { + do_no_trailing = FALSE; + } + if (!do_no_trailing) { + ACPI_PATH_PUT8(full_path, path_size, c, length); + } + } + next_node = next_node->parent; + } + ACPI_PATH_PUT8(full_path, path_size, AML_ROOT_PREFIX, length); + + /* Reverse the path string */ + + if (length <= path_size) { + left = full_path; + right = full_path + length - 1; + while (left < right) { + c = *left; + *left++ = *right; + *right-- = c; + } + } + + /* Append the trailing null */ + +build_trailing_null: + ACPI_PATH_PUT8(full_path, path_size, '\0', length); + +#undef ACPI_PATH_PUT8 + + return_UINT32(length); +} + +/******************************************************************************* + * + * FUNCTION: acpi_ns_get_normalized_pathname + * + * PARAMETERS: node - Namespace node whose pathname is needed + * no_trailing - Remove trailing '_' from each name segment + * + * RETURN: Pointer to storage containing the fully qualified name of + * the node, In external format (name segments separated by path + * separators.) + * + * DESCRIPTION: Used to obtain the full pathname to a namespace node, usually + * for error and debug statements. All trailing '_' will be + * removed from the full pathname if 'NoTrailing' is specified.. + * + ******************************************************************************/ + +char *acpi_ns_get_normalized_pathname(struct acpi_namespace_node *node, + u8 no_trailing) +{ + char *name_buffer; + acpi_size size; + + ACPI_FUNCTION_TRACE_PTR(ns_get_normalized_pathname, node); + + /* Calculate required buffer size based on depth below root */ + + size = acpi_ns_build_normalized_path(node, NULL, 0, no_trailing); + if (!size) { + return_PTR(NULL); + } + + /* Allocate a buffer to be returned to caller */ + + name_buffer = ACPI_ALLOCATE_ZEROED(size); + if (!name_buffer) { + ACPI_ERROR((AE_INFO, "Could not allocate %u bytes", (u32)size)); + return_PTR(NULL); + } + + /* Build the path in the allocated buffer */ + + (void)acpi_ns_build_normalized_path(node, name_buffer, size, + no_trailing); + + return_PTR(name_buffer); +} diff --git a/drivers/acpi/acpica/nsparse.c b/drivers/acpi/acpica/nsparse.c index 57a4cfe547e4..3736d43b18b9 100644 --- a/drivers/acpi/acpica/nsparse.c +++ b/drivers/acpi/acpica/nsparse.c @@ -70,7 +70,7 @@ acpi_ns_one_complete_parse(u32 pass_number, { union acpi_parse_object *parse_root; acpi_status status; - u32 aml_length; + u32 aml_length; u8 *aml_start; struct acpi_walk_state *walk_state; struct acpi_table_header *table; @@ -78,6 +78,20 @@ acpi_ns_one_complete_parse(u32 pass_number, ACPI_FUNCTION_TRACE(ns_one_complete_parse); + status = acpi_get_table_by_index(table_index, &table); + if (ACPI_FAILURE(status)) { + return_ACPI_STATUS(status); + } + + /* Table must consist of at least a complete header */ + + if (table->length < sizeof(struct acpi_table_header)) { + return_ACPI_STATUS(AE_BAD_HEADER); + } + + aml_start = (u8 *)table + sizeof(struct acpi_table_header); + aml_length = table->length - sizeof(struct acpi_table_header); + status = acpi_tb_get_owner_id(table_index, &owner_id); if (ACPI_FAILURE(status)) { return_ACPI_STATUS(status); @@ -85,7 +99,7 @@ acpi_ns_one_complete_parse(u32 pass_number, /* Create and init a Root Node */ - parse_root = acpi_ps_create_scope_op(); + parse_root = acpi_ps_create_scope_op(aml_start); if (!parse_root) { return_ACPI_STATUS(AE_NO_MEMORY); } @@ -98,23 +112,12 @@ acpi_ns_one_complete_parse(u32 pass_number, return_ACPI_STATUS(AE_NO_MEMORY); } - status = acpi_get_table_by_index(table_index, &table); + status = acpi_ds_init_aml_walk(walk_state, parse_root, NULL, + aml_start, aml_length, NULL, + (u8)pass_number); if (ACPI_FAILURE(status)) { acpi_ds_delete_walk_state(walk_state); - acpi_ps_free_op(parse_root); - return_ACPI_STATUS(status); - } - - /* Table must consist of at least a complete header */ - - if (table->length < sizeof(struct acpi_table_header)) { - status = AE_BAD_HEADER; - } else { - aml_start = (u8 *) table + sizeof(struct acpi_table_header); - aml_length = table->length - sizeof(struct acpi_table_header); - status = acpi_ds_init_aml_walk(walk_state, parse_root, NULL, - aml_start, aml_length, NULL, - (u8) pass_number); + goto cleanup; } /* Found OSDT table, enable the namespace override feature */ @@ -124,11 +127,6 @@ acpi_ns_one_complete_parse(u32 pass_number, walk_state->namespace_override = TRUE; } - if (ACPI_FAILURE(status)) { - acpi_ds_delete_walk_state(walk_state); - goto cleanup; - } - /* start_node is the default location to load the table */ if (start_node && start_node != acpi_gbl_root_node) { diff --git a/drivers/acpi/acpica/nsutils.c b/drivers/acpi/acpica/nsutils.c index 8d8104b8bd28..de325ae04ce1 100644 --- a/drivers/acpi/acpica/nsutils.c +++ b/drivers/acpi/acpica/nsutils.c @@ -83,7 +83,7 @@ acpi_ns_print_node_pathname(struct acpi_namespace_node *node, buffer.length = ACPI_ALLOCATE_LOCAL_BUFFER; - status = acpi_ns_handle_to_pathname(node, &buffer); + status = acpi_ns_handle_to_pathname(node, &buffer, TRUE); if (ACPI_SUCCESS(status)) { if (message) { acpi_os_printf("%s ", message); @@ -596,6 +596,23 @@ void acpi_ns_terminate(void) ACPI_FUNCTION_TRACE(ns_terminate); +#ifdef ACPI_EXEC_APP + { + union acpi_operand_object *prev; + union acpi_operand_object *next; + + /* Delete any module-level code blocks */ + + next = acpi_gbl_module_code_list; + while (next) { + prev = next; + next = next->method.mutex; + prev->method.mutex = NULL; /* Clear the Mutex (cheated) field */ + acpi_ut_remove_reference(prev); + } + } +#endif + /* * Free the entire namespace -- all nodes and all objects * attached to the nodes diff --git a/drivers/acpi/acpica/nsxfname.c b/drivers/acpi/acpica/nsxfname.c index 9ff643b9553f..4b4d2f43d406 100644 --- a/drivers/acpi/acpica/nsxfname.c +++ b/drivers/acpi/acpica/nsxfname.c @@ -172,11 +172,15 @@ acpi_get_name(acpi_handle handle, u32 name_type, struct acpi_buffer * buffer) return (status); } - if (name_type == ACPI_FULL_PATHNAME) { + if (name_type == ACPI_FULL_PATHNAME || + name_type == ACPI_FULL_PATHNAME_NO_TRAILING) { /* Get the full pathname (From the namespace root) */ - status = acpi_ns_handle_to_pathname(handle, buffer); + status = acpi_ns_handle_to_pathname(handle, buffer, + name_type == + ACPI_FULL_PATHNAME ? FALSE : + TRUE); return (status); } diff --git a/drivers/acpi/acpica/psargs.c b/drivers/acpi/acpica/psargs.c index 6d038770577b..29d8b7b01dca 100644 --- a/drivers/acpi/acpica/psargs.c +++ b/drivers/acpi/acpica/psargs.c @@ -287,7 +287,7 @@ acpi_ps_get_next_namepath(struct acpi_walk_state *walk_state, "Control Method - %p Desc %p Path=%p\n", node, method_desc, path)); - name_op = acpi_ps_alloc_op(AML_INT_NAMEPATH_OP); + name_op = acpi_ps_alloc_op(AML_INT_NAMEPATH_OP, start); if (!name_op) { return_ACPI_STATUS(AE_NO_MEMORY); } @@ -484,7 +484,7 @@ acpi_ps_get_next_simple_arg(struct acpi_parse_state *parser_state, static union acpi_parse_object *acpi_ps_get_next_field(struct acpi_parse_state *parser_state) { - u32 aml_offset; + u8 *aml; union acpi_parse_object *field; union acpi_parse_object *arg = NULL; u16 opcode; @@ -498,8 +498,7 @@ static union acpi_parse_object *acpi_ps_get_next_field(struct acpi_parse_state ACPI_FUNCTION_TRACE(ps_get_next_field); - aml_offset = - (u32)ACPI_PTR_DIFF(parser_state->aml, parser_state->aml_start); + aml = parser_state->aml; /* Determine field type */ @@ -536,13 +535,11 @@ static union acpi_parse_object *acpi_ps_get_next_field(struct acpi_parse_state /* Allocate a new field op */ - field = acpi_ps_alloc_op(opcode); + field = acpi_ps_alloc_op(opcode, aml); if (!field) { return_PTR(NULL); } - field->common.aml_offset = aml_offset; - /* Decode the field type */ switch (opcode) { @@ -604,6 +601,7 @@ static union acpi_parse_object *acpi_ps_get_next_field(struct acpi_parse_state * Argument for Connection operator can be either a Buffer * (resource descriptor), or a name_string. */ + aml = parser_state->aml; if (ACPI_GET8(parser_state->aml) == AML_BUFFER_OP) { parser_state->aml++; @@ -616,7 +614,8 @@ static union acpi_parse_object *acpi_ps_get_next_field(struct acpi_parse_state /* Non-empty list */ - arg = acpi_ps_alloc_op(AML_INT_BYTELIST_OP); + arg = + acpi_ps_alloc_op(AML_INT_BYTELIST_OP, aml); if (!arg) { acpi_ps_free_op(field); return_PTR(NULL); @@ -665,7 +664,7 @@ static union acpi_parse_object *acpi_ps_get_next_field(struct acpi_parse_state parser_state->aml = pkg_end; } else { - arg = acpi_ps_alloc_op(AML_INT_NAMEPATH_OP); + arg = acpi_ps_alloc_op(AML_INT_NAMEPATH_OP, aml); if (!arg) { acpi_ps_free_op(field); return_PTR(NULL); @@ -730,7 +729,7 @@ acpi_ps_get_next_arg(struct acpi_walk_state *walk_state, /* Constants, strings, and namestrings are all the same size */ - arg = acpi_ps_alloc_op(AML_BYTE_OP); + arg = acpi_ps_alloc_op(AML_BYTE_OP, parser_state->aml); if (!arg) { return_ACPI_STATUS(AE_NO_MEMORY); } @@ -777,7 +776,8 @@ acpi_ps_get_next_arg(struct acpi_walk_state *walk_state, /* Non-empty list */ - arg = acpi_ps_alloc_op(AML_INT_BYTELIST_OP); + arg = acpi_ps_alloc_op(AML_INT_BYTELIST_OP, + parser_state->aml); if (!arg) { return_ACPI_STATUS(AE_NO_MEMORY); } @@ -807,7 +807,9 @@ acpi_ps_get_next_arg(struct acpi_walk_state *walk_state, /* null_name or name_string */ - arg = acpi_ps_alloc_op(AML_INT_NAMEPATH_OP); + arg = + acpi_ps_alloc_op(AML_INT_NAMEPATH_OP, + parser_state->aml); if (!arg) { return_ACPI_STATUS(AE_NO_MEMORY); } diff --git a/drivers/acpi/acpica/psloop.c b/drivers/acpi/acpica/psloop.c index 90437227d790..03ac8c9a67ab 100644 --- a/drivers/acpi/acpica/psloop.c +++ b/drivers/acpi/acpica/psloop.c @@ -51,6 +51,7 @@ #include #include "accommon.h" +#include "acinterp.h" #include "acparser.h" #include "acdispat.h" #include "amlcode.h" @@ -125,10 +126,7 @@ acpi_ps_get_arguments(struct acpi_walk_state *walk_state, */ while (GET_CURRENT_ARG_TYPE(walk_state->arg_types) && !walk_state->arg_count) { - walk_state->aml_offset = - (u32) ACPI_PTR_DIFF(walk_state->parser_state.aml, - walk_state->parser_state. - aml_start); + walk_state->aml = walk_state->parser_state.aml; status = acpi_ps_get_next_arg(walk_state, @@ -140,7 +138,6 @@ acpi_ps_get_arguments(struct acpi_walk_state *walk_state, } if (arg) { - arg->common.aml_offset = walk_state->aml_offset; acpi_ps_append_arg(op, arg); } @@ -324,6 +321,8 @@ acpi_ps_link_module_code(union acpi_parse_object *parent_op, union acpi_operand_object *method_obj; struct acpi_namespace_node *parent_node; + ACPI_FUNCTION_TRACE(ps_link_module_code); + /* Get the tail of the list */ prev = next = acpi_gbl_module_code_list; @@ -343,9 +342,13 @@ acpi_ps_link_module_code(union acpi_parse_object *parent_op, method_obj = acpi_ut_create_internal_object(ACPI_TYPE_METHOD); if (!method_obj) { - return; + return_VOID; } + ACPI_DEBUG_PRINT((ACPI_DB_PARSE, + "Create/Link new code block: %p\n", + method_obj)); + if (parent_op->common.node) { parent_node = parent_op->common.node; } else { @@ -370,8 +373,14 @@ acpi_ps_link_module_code(union acpi_parse_object *parent_op, prev->method.mutex = method_obj; } } else { + ACPI_DEBUG_PRINT((ACPI_DB_PARSE, + "Appending to existing code block: %p\n", + prev)); + prev->method.aml_length += aml_length; } + + return_VOID; } /******************************************************************************* @@ -494,16 +503,7 @@ acpi_status acpi_ps_parse_loop(struct acpi_walk_state *walk_state) continue; } - op->common.aml_offset = walk_state->aml_offset; - - if (walk_state->op_info) { - ACPI_DEBUG_PRINT((ACPI_DB_PARSE, - "Opcode %4.4X [%s] Op %p Aml %p AmlOffset %5.5X\n", - (u32) op->common.aml_opcode, - walk_state->op_info->name, op, - parser_state->aml, - op->common.aml_offset)); - } + acpi_ex_start_trace_opcode(op, walk_state); } /* diff --git a/drivers/acpi/acpica/psobject.c b/drivers/acpi/acpica/psobject.c index 2f5ddd806c58..e54bc2aa7a88 100644 --- a/drivers/acpi/acpica/psobject.c +++ b/drivers/acpi/acpica/psobject.c @@ -66,12 +66,11 @@ static acpi_status acpi_ps_get_aml_opcode(struct acpi_walk_state *walk_state); static acpi_status acpi_ps_get_aml_opcode(struct acpi_walk_state *walk_state) { + u32 aml_offset; ACPI_FUNCTION_TRACE_PTR(ps_get_aml_opcode, walk_state); - walk_state->aml_offset = - (u32)ACPI_PTR_DIFF(walk_state->parser_state.aml, - walk_state->parser_state.aml_start); + walk_state->aml = walk_state->parser_state.aml; walk_state->opcode = acpi_ps_peek_opcode(&(walk_state->parser_state)); /* @@ -98,10 +97,14 @@ static acpi_status acpi_ps_get_aml_opcode(struct acpi_walk_state *walk_state) /* The opcode is unrecognized. Complain and skip unknown opcodes */ if (walk_state->pass_number == 2) { + aml_offset = (u32)ACPI_PTR_DIFF(walk_state->aml, + walk_state-> + parser_state.aml_start); + ACPI_ERROR((AE_INFO, "Unknown opcode 0x%.2X at table offset 0x%.4X, ignoring", walk_state->opcode, - (u32)(walk_state->aml_offset + + (u32)(aml_offset + sizeof(struct acpi_table_header)))); ACPI_DUMP_BUFFER((walk_state->parser_state.aml - 16), @@ -115,14 +118,14 @@ static acpi_status acpi_ps_get_aml_opcode(struct acpi_walk_state *walk_state) acpi_os_printf ("/*\nError: Unknown opcode 0x%.2X at table offset 0x%.4X, context:\n", walk_state->opcode, - (u32)(walk_state->aml_offset + + (u32)(aml_offset + sizeof(struct acpi_table_header))); /* Dump the context surrounding the invalid opcode */ acpi_ut_dump_buffer(((u8 *)walk_state->parser_state. aml - 16), 48, DB_BYTE_DISPLAY, - (walk_state->aml_offset + + (aml_offset + sizeof(struct acpi_table_header) - 16)); acpi_os_printf(" */\n"); @@ -294,7 +297,7 @@ acpi_ps_create_op(struct acpi_walk_state *walk_state, /* Create Op structure and append to parent's argument list */ walk_state->op_info = acpi_ps_get_opcode_info(walk_state->opcode); - op = acpi_ps_alloc_op(walk_state->opcode); + op = acpi_ps_alloc_op(walk_state->opcode, aml_op_start); if (!op) { return_ACPI_STATUS(AE_NO_MEMORY); } diff --git a/drivers/acpi/acpica/psparse.c b/drivers/acpi/acpica/psparse.c index a555f7f7b9a2..98001d7f6f80 100644 --- a/drivers/acpi/acpica/psparse.c +++ b/drivers/acpi/acpica/psparse.c @@ -147,6 +147,8 @@ acpi_ps_complete_this_op(struct acpi_walk_state * walk_state, return_ACPI_STATUS(AE_OK); /* OK for now */ } + acpi_ex_stop_trace_opcode(op, walk_state); + /* Delete this op and the subtree below it if asked to */ if (((walk_state->parse_flags & ACPI_PARSE_TREE_MASK) != @@ -185,7 +187,8 @@ acpi_ps_complete_this_op(struct acpi_walk_state * walk_state, * op must be replaced by a placeholder return op */ replacement_op = - acpi_ps_alloc_op(AML_INT_RETURN_VALUE_OP); + acpi_ps_alloc_op(AML_INT_RETURN_VALUE_OP, + op->common.aml); if (!replacement_op) { status = AE_NO_MEMORY; } @@ -209,7 +212,8 @@ acpi_ps_complete_this_op(struct acpi_walk_state * walk_state, || (op->common.parent->common.aml_opcode == AML_VAR_PACKAGE_OP)) { replacement_op = - acpi_ps_alloc_op(AML_INT_RETURN_VALUE_OP); + acpi_ps_alloc_op(AML_INT_RETURN_VALUE_OP, + op->common.aml); if (!replacement_op) { status = AE_NO_MEMORY; } @@ -224,7 +228,8 @@ acpi_ps_complete_this_op(struct acpi_walk_state * walk_state, AML_VAR_PACKAGE_OP)) { replacement_op = acpi_ps_alloc_op(op->common. - aml_opcode); + aml_opcode, + op->common.aml); if (!replacement_op) { status = AE_NO_MEMORY; } else { @@ -240,7 +245,8 @@ acpi_ps_complete_this_op(struct acpi_walk_state * walk_state, default: replacement_op = - acpi_ps_alloc_op(AML_INT_RETURN_VALUE_OP); + acpi_ps_alloc_op(AML_INT_RETURN_VALUE_OP, + op->common.aml); if (!replacement_op) { status = AE_NO_MEMORY; } diff --git a/drivers/acpi/acpica/psutils.c b/drivers/acpi/acpica/psutils.c index 32440912023a..183cc1efbc51 100644 --- a/drivers/acpi/acpica/psutils.c +++ b/drivers/acpi/acpica/psutils.c @@ -60,11 +60,11 @@ ACPI_MODULE_NAME("psutils") * DESCRIPTION: Create a Scope and associated namepath op with the root name * ******************************************************************************/ -union acpi_parse_object *acpi_ps_create_scope_op(void) +union acpi_parse_object *acpi_ps_create_scope_op(u8 *aml) { union acpi_parse_object *scope_op; - scope_op = acpi_ps_alloc_op(AML_SCOPE_OP); + scope_op = acpi_ps_alloc_op(AML_SCOPE_OP, aml); if (!scope_op) { return (NULL); } @@ -103,6 +103,7 @@ void acpi_ps_init_op(union acpi_parse_object *op, u16 opcode) * FUNCTION: acpi_ps_alloc_op * * PARAMETERS: opcode - Opcode that will be stored in the new Op + * aml - Address of the opcode * * RETURN: Pointer to the new Op, null on failure * @@ -112,7 +113,7 @@ void acpi_ps_init_op(union acpi_parse_object *op, u16 opcode) * ******************************************************************************/ -union acpi_parse_object *acpi_ps_alloc_op(u16 opcode) +union acpi_parse_object *acpi_ps_alloc_op(u16 opcode, u8 *aml) { union acpi_parse_object *op; const struct acpi_opcode_info *op_info; @@ -149,6 +150,7 @@ union acpi_parse_object *acpi_ps_alloc_op(u16 opcode) if (op) { acpi_ps_init_op(op, opcode); + op->common.aml = aml; op->common.flags = flags; } diff --git a/drivers/acpi/acpica/psxface.c b/drivers/acpi/acpica/psxface.c index 841a5ea06094..4254805dd319 100644 --- a/drivers/acpi/acpica/psxface.c +++ b/drivers/acpi/acpica/psxface.c @@ -47,15 +47,12 @@ #include "acdispat.h" #include "acinterp.h" #include "actables.h" +#include "acnamesp.h" #define _COMPONENT ACPI_PARSER ACPI_MODULE_NAME("psxface") /* Local Prototypes */ -static void acpi_ps_start_trace(struct acpi_evaluate_info *info); - -static void acpi_ps_stop_trace(struct acpi_evaluate_info *info); - static void acpi_ps_update_parameter_list(struct acpi_evaluate_info *info, u16 action); @@ -76,7 +73,7 @@ acpi_ps_update_parameter_list(struct acpi_evaluate_info *info, u16 action); ******************************************************************************/ acpi_status -acpi_debug_trace(char *name, u32 debug_level, u32 debug_layer, u32 flags) +acpi_debug_trace(const char *name, u32 debug_level, u32 debug_layer, u32 flags) { acpi_status status; @@ -85,108 +82,14 @@ acpi_debug_trace(char *name, u32 debug_level, u32 debug_layer, u32 flags) return (status); } - /* TBDs: Validate name, allow full path or just nameseg */ - - acpi_gbl_trace_method_name = *ACPI_CAST_PTR(u32, name); + acpi_gbl_trace_method_name = name; acpi_gbl_trace_flags = flags; - - if (debug_level) { - acpi_gbl_trace_dbg_level = debug_level; - } - if (debug_layer) { - acpi_gbl_trace_dbg_layer = debug_layer; - } + acpi_gbl_trace_dbg_level = debug_level; + acpi_gbl_trace_dbg_layer = debug_layer; + status = AE_OK; (void)acpi_ut_release_mutex(ACPI_MTX_NAMESPACE); - return (AE_OK); -} - -/******************************************************************************* - * - * FUNCTION: acpi_ps_start_trace - * - * PARAMETERS: info - Method info struct - * - * RETURN: None - * - * DESCRIPTION: Start control method execution trace - * - ******************************************************************************/ - -static void acpi_ps_start_trace(struct acpi_evaluate_info *info) -{ - acpi_status status; - - ACPI_FUNCTION_ENTRY(); - - status = acpi_ut_acquire_mutex(ACPI_MTX_NAMESPACE); - if (ACPI_FAILURE(status)) { - return; - } - - if ((!acpi_gbl_trace_method_name) || - (acpi_gbl_trace_method_name != info->node->name.integer)) { - goto exit; - } - - acpi_gbl_original_dbg_level = acpi_dbg_level; - acpi_gbl_original_dbg_layer = acpi_dbg_layer; - - acpi_dbg_level = 0x00FFFFFF; - acpi_dbg_layer = ACPI_UINT32_MAX; - - if (acpi_gbl_trace_dbg_level) { - acpi_dbg_level = acpi_gbl_trace_dbg_level; - } - if (acpi_gbl_trace_dbg_layer) { - acpi_dbg_layer = acpi_gbl_trace_dbg_layer; - } - -exit: - (void)acpi_ut_release_mutex(ACPI_MTX_NAMESPACE); -} - -/******************************************************************************* - * - * FUNCTION: acpi_ps_stop_trace - * - * PARAMETERS: info - Method info struct - * - * RETURN: None - * - * DESCRIPTION: Stop control method execution trace - * - ******************************************************************************/ - -static void acpi_ps_stop_trace(struct acpi_evaluate_info *info) -{ - acpi_status status; - - ACPI_FUNCTION_ENTRY(); - - status = acpi_ut_acquire_mutex(ACPI_MTX_NAMESPACE); - if (ACPI_FAILURE(status)) { - return; - } - - if ((!acpi_gbl_trace_method_name) || - (acpi_gbl_trace_method_name != info->node->name.integer)) { - goto exit; - } - - /* Disable further tracing if type is one-shot */ - - if (acpi_gbl_trace_flags & 1) { - acpi_gbl_trace_method_name = 0; - acpi_gbl_trace_dbg_level = 0; - acpi_gbl_trace_dbg_layer = 0; - } - - acpi_dbg_level = acpi_gbl_original_dbg_level; - acpi_dbg_layer = acpi_gbl_original_dbg_layer; - -exit: - (void)acpi_ut_release_mutex(ACPI_MTX_NAMESPACE); + return (status); } /******************************************************************************* @@ -212,7 +115,7 @@ static void acpi_ps_stop_trace(struct acpi_evaluate_info *info) * ******************************************************************************/ -acpi_status acpi_ps_execute_method(struct acpi_evaluate_info *info) +acpi_status acpi_ps_execute_method(struct acpi_evaluate_info * info) { acpi_status status; union acpi_parse_object *op; @@ -243,10 +146,6 @@ acpi_status acpi_ps_execute_method(struct acpi_evaluate_info *info) */ acpi_ps_update_parameter_list(info, REF_INCREMENT); - /* Begin tracing if requested */ - - acpi_ps_start_trace(info); - /* * Execute the method. Performs parse simultaneously */ @@ -256,7 +155,7 @@ acpi_status acpi_ps_execute_method(struct acpi_evaluate_info *info) /* Create and init a Root Node */ - op = acpi_ps_create_scope_op(); + op = acpi_ps_create_scope_op(info->obj_desc->method.aml_start); if (!op) { status = AE_NO_MEMORY; goto cleanup; @@ -326,10 +225,6 @@ acpi_status acpi_ps_execute_method(struct acpi_evaluate_info *info) cleanup: acpi_ps_delete_parse_tree(op); - /* End optional tracing */ - - acpi_ps_stop_trace(info); - /* Take away the extra reference that we gave the parameters above */ acpi_ps_update_parameter_list(info, REF_DECREMENT); diff --git a/drivers/acpi/acpica/rscreate.c b/drivers/acpi/acpica/rscreate.c index 3fa829e96c2a..a5344428f3ae 100644 --- a/drivers/acpi/acpica/rscreate.c +++ b/drivers/acpi/acpica/rscreate.c @@ -348,7 +348,8 @@ acpi_rs_create_pci_routing_table(union acpi_operand_object *package_object, status = acpi_ns_handle_to_pathname((acpi_handle) node, - &path_buffer); + &path_buffer, + FALSE); /* +1 to include null terminator */ diff --git a/drivers/acpi/acpica/tbfadt.c b/drivers/acpi/acpica/tbfadt.c index 6253001b6375..455a0700db39 100644 --- a/drivers/acpi/acpica/tbfadt.c +++ b/drivers/acpi/acpica/tbfadt.c @@ -345,7 +345,7 @@ void acpi_tb_parse_fadt(u32 table_index) /* Obtain the DSDT and FACS tables via their addresses within the FADT */ acpi_tb_install_fixed_table((acpi_physical_address) acpi_gbl_FADT.Xdsdt, - ACPI_SIG_DSDT, ACPI_TABLE_INDEX_DSDT); + ACPI_SIG_DSDT, &acpi_gbl_dsdt_index); /* If Hardware Reduced flag is set, there is no FACS */ @@ -354,13 +354,13 @@ void acpi_tb_parse_fadt(u32 table_index) acpi_tb_install_fixed_table((acpi_physical_address) acpi_gbl_FADT.facs, ACPI_SIG_FACS, - ACPI_TABLE_INDEX_FACS); + &acpi_gbl_facs_index); } if (acpi_gbl_FADT.Xfacs) { acpi_tb_install_fixed_table((acpi_physical_address) acpi_gbl_FADT.Xfacs, ACPI_SIG_FACS, - ACPI_TABLE_INDEX_X_FACS); + &acpi_gbl_xfacs_index); } } } diff --git a/drivers/acpi/acpica/tbfind.c b/drivers/acpi/acpica/tbfind.c index 119c84ad9833..405529d49a1a 100644 --- a/drivers/acpi/acpica/tbfind.c +++ b/drivers/acpi/acpica/tbfind.c @@ -68,12 +68,25 @@ acpi_status acpi_tb_find_table(char *signature, char *oem_id, char *oem_table_id, u32 *table_index) { - u32 i; acpi_status status; struct acpi_table_header header; + u32 i; ACPI_FUNCTION_TRACE(tb_find_table); + /* Validate the input table signature */ + + if (!acpi_is_valid_signature(signature)) { + return_ACPI_STATUS(AE_BAD_SIGNATURE); + } + + /* Don't allow the OEM strings to be too long */ + + if ((strlen(oem_id) > ACPI_OEM_ID_SIZE) || + (strlen(oem_table_id) > ACPI_OEM_TABLE_ID_SIZE)) { + return_ACPI_STATUS(AE_AML_STRING_LIMIT); + } + /* Normalize the input strings */ memset(&header, 0, sizeof(struct acpi_table_header)); diff --git a/drivers/acpi/acpica/tbinstal.c b/drivers/acpi/acpica/tbinstal.c index 15ea98e0068d..6319b42420c6 100644 --- a/drivers/acpi/acpica/tbinstal.c +++ b/drivers/acpi/acpica/tbinstal.c @@ -100,9 +100,9 @@ acpi_tb_compare_tables(struct acpi_table_desc *table_desc, u32 table_index) * * FUNCTION: acpi_tb_install_table_with_override * - * PARAMETERS: table_index - Index into root table array - * new_table_desc - New table descriptor to install + * PARAMETERS: new_table_desc - New table descriptor to install * override - Whether override should be performed + * table_index - Where the table index is returned * * RETURN: None * @@ -114,12 +114,14 @@ acpi_tb_compare_tables(struct acpi_table_desc *table_desc, u32 table_index) ******************************************************************************/ void -acpi_tb_install_table_with_override(u32 table_index, - struct acpi_table_desc *new_table_desc, - u8 override) +acpi_tb_install_table_with_override(struct acpi_table_desc *new_table_desc, + u8 override, u32 *table_index) { + u32 i; + acpi_status status; - if (table_index >= acpi_gbl_root_table_list.current_table_count) { + status = acpi_tb_get_next_table_descriptor(&i, NULL); + if (ACPI_FAILURE(status)) { return; } @@ -134,8 +136,7 @@ acpi_tb_install_table_with_override(u32 table_index, acpi_tb_override_table(new_table_desc); } - acpi_tb_init_table_descriptor(&acpi_gbl_root_table_list. - tables[table_index], + acpi_tb_init_table_descriptor(&acpi_gbl_root_table_list.tables[i], new_table_desc->address, new_table_desc->flags, new_table_desc->pointer); @@ -143,9 +144,13 @@ acpi_tb_install_table_with_override(u32 table_index, acpi_tb_print_table_header(new_table_desc->address, new_table_desc->pointer); + /* This synchronizes acpi_gbl_dsdt_index */ + + *table_index = i; + /* Set the global integer width (based upon revision of the DSDT) */ - if (table_index == ACPI_TABLE_INDEX_DSDT) { + if (i == acpi_gbl_dsdt_index) { acpi_ut_set_integer_width(new_table_desc->pointer->revision); } } @@ -157,7 +162,7 @@ acpi_tb_install_table_with_override(u32 table_index, * PARAMETERS: address - Physical address of DSDT or FACS * signature - Table signature, NULL if no need to * match - * table_index - Index into root table array + * table_index - Where the table index is returned * * RETURN: Status * @@ -168,7 +173,7 @@ acpi_tb_install_table_with_override(u32 table_index, acpi_status acpi_tb_install_fixed_table(acpi_physical_address address, - char *signature, u32 table_index) + char *signature, u32 *table_index) { struct acpi_table_desc new_table_desc; acpi_status status; @@ -200,7 +205,9 @@ acpi_tb_install_fixed_table(acpi_physical_address address, goto release_and_exit; } - acpi_tb_install_table_with_override(table_index, &new_table_desc, TRUE); + /* Add the table to the global root table list */ + + acpi_tb_install_table_with_override(&new_table_desc, TRUE, table_index); release_and_exit: @@ -355,13 +362,8 @@ acpi_tb_install_standard_table(acpi_physical_address address, /* Add the table to the global root table list */ - status = acpi_tb_get_next_table_descriptor(&i, NULL); - if (ACPI_FAILURE(status)) { - goto release_and_exit; - } - - *table_index = i; - acpi_tb_install_table_with_override(i, &new_table_desc, override); + acpi_tb_install_table_with_override(&new_table_desc, override, + table_index); release_and_exit: diff --git a/drivers/acpi/acpica/tbutils.c b/drivers/acpi/acpica/tbutils.c index 568ac0e4a3c6..4337990127cc 100644 --- a/drivers/acpi/acpica/tbutils.c +++ b/drivers/acpi/acpica/tbutils.c @@ -68,28 +68,27 @@ acpi_tb_get_root_table_entry(u8 *table_entry, u32 table_entry_size); acpi_status acpi_tb_initialize_facs(void) { + struct acpi_table_facs *facs; /* If Hardware Reduced flag is set, there is no FACS */ if (acpi_gbl_reduced_hardware) { acpi_gbl_FACS = NULL; return (AE_OK); - } - - (void)acpi_get_table_by_index(ACPI_TABLE_INDEX_FACS, - ACPI_CAST_INDIRECT_PTR(struct - acpi_table_header, - &acpi_gbl_facs32)); - (void)acpi_get_table_by_index(ACPI_TABLE_INDEX_X_FACS, - ACPI_CAST_INDIRECT_PTR(struct - acpi_table_header, - &acpi_gbl_facs64)); - - if (acpi_gbl_facs64 - && (!acpi_gbl_facs32 || !acpi_gbl_use32_bit_facs_addresses)) { - acpi_gbl_FACS = acpi_gbl_facs64; - } else if (acpi_gbl_facs32) { - acpi_gbl_FACS = acpi_gbl_facs32; + } else if (acpi_gbl_FADT.Xfacs && + (!acpi_gbl_FADT.facs + || !acpi_gbl_use32_bit_facs_addresses)) { + (void)acpi_get_table_by_index(acpi_gbl_xfacs_index, + ACPI_CAST_INDIRECT_PTR(struct + acpi_table_header, + &facs)); + acpi_gbl_FACS = facs; + } else if (acpi_gbl_FADT.facs) { + (void)acpi_get_table_by_index(acpi_gbl_facs_index, + ACPI_CAST_INDIRECT_PTR(struct + acpi_table_header, + &facs)); + acpi_gbl_FACS = facs; } /* If there is no FACS, just continue. There was already an error msg */ @@ -192,7 +191,7 @@ struct acpi_table_header *acpi_tb_copy_dsdt(u32 table_index) acpi_tb_uninstall_table(table_desc); acpi_tb_init_table_descriptor(&acpi_gbl_root_table_list. - tables[ACPI_TABLE_INDEX_DSDT], + tables[acpi_gbl_dsdt_index], ACPI_PTR_TO_PHYSADDR(new_table), ACPI_TABLE_ORIGIN_INTERNAL_VIRTUAL, new_table); @@ -369,13 +368,6 @@ acpi_status __init acpi_tb_parse_root_table(acpi_physical_address rsdp_address) table_entry_size); table_entry = ACPI_ADD_PTR(u8, table, sizeof(struct acpi_table_header)); - /* - * First three entries in the table array are reserved for the DSDT - * and 32bit/64bit FACS, which are not actually present in the - * RSDT/XSDT - they come from the FADT - */ - acpi_gbl_root_table_list.current_table_count = 3; - /* Initialize the root table array from the RSDT/XSDT */ for (i = 0; i < table_count; i++) { @@ -412,3 +404,36 @@ acpi_status __init acpi_tb_parse_root_table(acpi_physical_address rsdp_address) return_ACPI_STATUS(AE_OK); } + +/******************************************************************************* + * + * FUNCTION: acpi_is_valid_signature + * + * PARAMETERS: signature - Sig string to be validated + * + * RETURN: TRUE if signature is correct length and has valid characters + * + * DESCRIPTION: Validate an ACPI table signature. + * + ******************************************************************************/ + +u8 acpi_is_valid_signature(char *signature) +{ + u32 i; + + /* Validate the signature length */ + + if (strlen(signature) != ACPI_NAME_SIZE) { + return (FALSE); + } + + /* Validate each character in the signature */ + + for (i = 0; i < ACPI_NAME_SIZE; i++) { + if (!acpi_ut_valid_acpi_char(signature[i], i)) { + return (FALSE); + } + } + + return (TRUE); +} diff --git a/drivers/acpi/acpica/tbxfload.c b/drivers/acpi/acpica/tbxfload.c index 9682d40ca6ff..55ee14ca9418 100644 --- a/drivers/acpi/acpica/tbxfload.c +++ b/drivers/acpi/acpica/tbxfload.c @@ -51,9 +51,6 @@ #define _COMPONENT ACPI_TABLES ACPI_MODULE_NAME("tbxfload") -/* Local prototypes */ -static acpi_status acpi_tb_load_namespace(void); - /******************************************************************************* * * FUNCTION: acpi_load_tables @@ -65,7 +62,6 @@ static acpi_status acpi_tb_load_namespace(void); * DESCRIPTION: Load the ACPI tables from the RSDT/XSDT * ******************************************************************************/ - acpi_status __init acpi_load_tables(void) { acpi_status status; @@ -75,6 +71,13 @@ acpi_status __init acpi_load_tables(void) /* Load the namespace from the tables */ status = acpi_tb_load_namespace(); + + /* Don't let single failures abort the load */ + + if (status == AE_CTRL_TERMINATE) { + status = AE_OK; + } + if (ACPI_FAILURE(status)) { ACPI_EXCEPTION((AE_INFO, status, "While loading namespace from ACPI tables")); @@ -97,11 +100,14 @@ ACPI_EXPORT_SYMBOL_INIT(acpi_load_tables) * the RSDT/XSDT. * ******************************************************************************/ -static acpi_status acpi_tb_load_namespace(void) +acpi_status acpi_tb_load_namespace(void) { acpi_status status; u32 i; struct acpi_table_header *new_dsdt; + struct acpi_table_desc *table; + u32 tables_loaded = 0; + u32 tables_failed = 0; ACPI_FUNCTION_TRACE(tb_load_namespace); @@ -111,15 +117,11 @@ static acpi_status acpi_tb_load_namespace(void) * Load the namespace. The DSDT is required, but any SSDT and * PSDT tables are optional. Verify the DSDT. */ + table = &acpi_gbl_root_table_list.tables[acpi_gbl_dsdt_index]; + if (!acpi_gbl_root_table_list.current_table_count || - !ACPI_COMPARE_NAME(& - (acpi_gbl_root_table_list. - tables[ACPI_TABLE_INDEX_DSDT].signature), - ACPI_SIG_DSDT) - || - ACPI_FAILURE(acpi_tb_validate_table - (&acpi_gbl_root_table_list. - tables[ACPI_TABLE_INDEX_DSDT]))) { + !ACPI_COMPARE_NAME(table->signature.ascii, ACPI_SIG_DSDT) || + ACPI_FAILURE(acpi_tb_validate_table(table))) { status = AE_NO_ACPI_TABLES; goto unlock_and_exit; } @@ -130,8 +132,7 @@ static acpi_status acpi_tb_load_namespace(void) * array can change dynamically as tables are loaded at run-time. Note: * .Pointer field is not validated until after call to acpi_tb_validate_table. */ - acpi_gbl_DSDT = - acpi_gbl_root_table_list.tables[ACPI_TABLE_INDEX_DSDT].pointer; + acpi_gbl_DSDT = table->pointer; /* * Optionally copy the entire DSDT to local memory (instead of simply @@ -140,7 +141,7 @@ static acpi_status acpi_tb_load_namespace(void) * the DSDT. */ if (acpi_gbl_copy_dsdt_locally) { - new_dsdt = acpi_tb_copy_dsdt(ACPI_TABLE_INDEX_DSDT); + new_dsdt = acpi_tb_copy_dsdt(acpi_gbl_dsdt_index); if (new_dsdt) { acpi_gbl_DSDT = new_dsdt; } @@ -157,41 +158,65 @@ static acpi_status acpi_tb_load_namespace(void) /* Load and parse tables */ - status = acpi_ns_load_table(ACPI_TABLE_INDEX_DSDT, acpi_gbl_root_node); + status = acpi_ns_load_table(acpi_gbl_dsdt_index, acpi_gbl_root_node); if (ACPI_FAILURE(status)) { - return_ACPI_STATUS(status); + ACPI_EXCEPTION((AE_INFO, status, "[DSDT] table load failed")); + tables_failed++; + } else { + tables_loaded++; } /* Load any SSDT or PSDT tables. Note: Loop leaves tables locked */ (void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES); for (i = 0; i < acpi_gbl_root_table_list.current_table_count; ++i) { + table = &acpi_gbl_root_table_list.tables[i]; + if (!acpi_gbl_root_table_list.tables[i].address || - (!ACPI_COMPARE_NAME - (&(acpi_gbl_root_table_list.tables[i].signature), - ACPI_SIG_SSDT) - && - !ACPI_COMPARE_NAME(& - (acpi_gbl_root_table_list.tables[i]. - signature), ACPI_SIG_PSDT) - && - !ACPI_COMPARE_NAME(& - (acpi_gbl_root_table_list.tables[i]. - signature), ACPI_SIG_OSDT)) - || - ACPI_FAILURE(acpi_tb_validate_table - (&acpi_gbl_root_table_list.tables[i]))) { + (!ACPI_COMPARE_NAME(table->signature.ascii, ACPI_SIG_SSDT) + && !ACPI_COMPARE_NAME(table->signature.ascii, + ACPI_SIG_PSDT) + && !ACPI_COMPARE_NAME(table->signature.ascii, + ACPI_SIG_OSDT)) + || ACPI_FAILURE(acpi_tb_validate_table(table))) { continue; } /* Ignore errors while loading tables, get as many as possible */ (void)acpi_ut_release_mutex(ACPI_MTX_TABLES); - (void)acpi_ns_load_table(i, acpi_gbl_root_node); + status = acpi_ns_load_table(i, acpi_gbl_root_node); + if (ACPI_FAILURE(status)) { + ACPI_EXCEPTION((AE_INFO, status, + "(%4.4s:%8.8s) while loading table", + table->signature.ascii, + table->pointer->oem_table_id)); + tables_failed++; + + ACPI_DEBUG_PRINT_RAW((ACPI_DB_INIT, + "Table [%4.4s:%8.8s] (id FF) - Table namespace load failed\n\n", + table->signature.ascii, + table->pointer->oem_table_id)); + } else { + tables_loaded++; + } + (void)acpi_ut_acquire_mutex(ACPI_MTX_TABLES); } - ACPI_INFO((AE_INFO, "All ACPI Tables successfully acquired")); + if (!tables_failed) { + ACPI_INFO((AE_INFO, + "%u ACPI AML tables successfully acquired and loaded", + tables_loaded)); + } else { + ACPI_ERROR((AE_INFO, + "%u table load failures, %u successful", + tables_failed, tables_loaded)); + + /* Indicate at least one failure */ + + status = AE_CTRL_TERMINATE; + } unlock_and_exit: (void)acpi_ut_release_mutex(ACPI_MTX_TABLES); diff --git a/drivers/acpi/acpica/utdebug.c b/drivers/acpi/acpica/utdebug.c index cd02693841db..4146229103c8 100644 --- a/drivers/acpi/acpica/utdebug.c +++ b/drivers/acpi/acpica/utdebug.c @@ -45,6 +45,7 @@ #include #include "accommon.h" +#include "acinterp.h" #define _COMPONENT ACPI_UTILITIES ACPI_MODULE_NAME("utdebug") @@ -560,8 +561,37 @@ acpi_ut_ptr_exit(u32 line_number, } } -#endif +/******************************************************************************* + * + * FUNCTION: acpi_trace_point + * + * PARAMETERS: type - Trace event type + * begin - TRUE if before execution + * aml - Executed AML address + * pathname - Object path + * pointer - Pointer to the related object + * + * RETURN: None + * + * DESCRIPTION: Interpreter execution trace. + * + ******************************************************************************/ +void +acpi_trace_point(acpi_trace_event_type type, u8 begin, u8 *aml, char *pathname) +{ + + ACPI_FUNCTION_ENTRY(); + + acpi_ex_trace_point(type, begin, aml, pathname); + +#ifdef ACPI_USE_SYSTEM_TRACER + acpi_os_trace_point(type, begin, aml, pathname); +#endif +} + +ACPI_EXPORT_SYMBOL(acpi_trace_point) +#endif #ifdef ACPI_APPLICATION /******************************************************************************* * @@ -575,7 +605,6 @@ acpi_ut_ptr_exit(u32 line_number, * DESCRIPTION: Print error message to the console, used by applications. * ******************************************************************************/ - void ACPI_INTERNAL_VAR_XFACE acpi_log_error(const char *format, ...) { va_list args; diff --git a/drivers/acpi/acpica/utdelete.c b/drivers/acpi/acpica/utdelete.c index 71fce389fd48..1638312e3d8f 100644 --- a/drivers/acpi/acpica/utdelete.c +++ b/drivers/acpi/acpica/utdelete.c @@ -209,6 +209,9 @@ static void acpi_ut_delete_internal_obj(union acpi_operand_object *object) acpi_ut_delete_object_desc(object->method.mutex); object->method.mutex = NULL; } + if (object->method.node) { + object->method.node = NULL; + } break; case ACPI_TYPE_REGION: diff --git a/drivers/acpi/acpica/utfileio.c b/drivers/acpi/acpica/utfileio.c index 857af824337b..75a94f52b4be 100644 --- a/drivers/acpi/acpica/utfileio.c +++ b/drivers/acpi/acpica/utfileio.c @@ -312,7 +312,7 @@ acpi_ut_read_table_from_file(char *filename, struct acpi_table_header ** table) /* Get the entire file */ fprintf(stderr, - "Reading ACPI table from file %10s - Length %.8u (0x%06X)\n", + "Reading ACPI table from file %12s - Length %.8u (0x%06X)\n", filename, file_size, file_size); status = acpi_ut_read_table(file, table, &table_length); diff --git a/drivers/acpi/acpica/utinit.c b/drivers/acpi/acpica/utinit.c index e402e07b4846..28ab3a1d5ec1 100644 --- a/drivers/acpi/acpica/utinit.c +++ b/drivers/acpi/acpica/utinit.c @@ -204,11 +204,10 @@ acpi_status acpi_ut_init_globals(void) acpi_gbl_acpi_hardware_present = TRUE; acpi_gbl_last_owner_id_index = 0; acpi_gbl_next_owner_id_offset = 0; - acpi_gbl_trace_dbg_level = 0; - acpi_gbl_trace_dbg_layer = 0; acpi_gbl_debugger_configuration = DEBUGGER_THREADING; acpi_gbl_osi_mutex = NULL; acpi_gbl_reg_methods_executed = FALSE; + acpi_gbl_max_loop_iterations = 0xFFFF; /* Hardware oriented */ diff --git a/drivers/acpi/acpica/utmisc.c b/drivers/acpi/acpica/utmisc.c index 71b66537f826..bd4443bdcbad 100644 --- a/drivers/acpi/acpica/utmisc.c +++ b/drivers/acpi/acpica/utmisc.c @@ -75,7 +75,7 @@ u8 acpi_ut_is_pci_root_bridge(char *id) return (FALSE); } -#if (defined ACPI_ASL_COMPILER || defined ACPI_EXEC_APP) +#if (defined ACPI_ASL_COMPILER || defined ACPI_EXEC_APP || defined ACPI_NAMES_APP) /******************************************************************************* * * FUNCTION: acpi_ut_is_aml_table @@ -376,7 +376,7 @@ acpi_ut_display_init_pathname(u8 type, /* Get the full pathname to the node */ buffer.length = ACPI_ALLOCATE_LOCAL_BUFFER; - status = acpi_ns_handle_to_pathname(obj_handle, &buffer); + status = acpi_ns_handle_to_pathname(obj_handle, &buffer, TRUE); if (ACPI_FAILURE(status)) { return; } diff --git a/drivers/acpi/acpica/utnonansi.c b/drivers/acpi/acpica/utnonansi.c new file mode 100644 index 000000000000..1d5f6b17b766 --- /dev/null +++ b/drivers/acpi/acpica/utnonansi.c @@ -0,0 +1,380 @@ +/******************************************************************************* + * + * Module Name: utnonansi - Non-ansi C library functions + * + ******************************************************************************/ + +/* + * Copyright (C) 2000 - 2015, Intel Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions, and the following disclaimer, + * without modification. + * 2. Redistributions in binary form must reproduce at minimum a disclaimer + * substantially similar to the "NO WARRANTY" disclaimer below + * ("Disclaimer") and any redistribution must be conditioned upon + * including a substantially similar Disclaimer requirement for further + * binary redistribution. + * 3. Neither the names of the above-listed copyright holders nor the names + * of any contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * Alternatively, this software may be distributed under the terms of the + * GNU General Public License ("GPL") version 2 as published by the Free + * Software Foundation. + * + * NO WARRANTY + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGES. + */ + +#include +#include "accommon.h" + +#define _COMPONENT ACPI_UTILITIES +ACPI_MODULE_NAME("utnonansi") + +/* + * Non-ANSI C library functions - strlwr, strupr, stricmp, and a 64-bit + * version of strtoul. + */ +/******************************************************************************* + * + * FUNCTION: acpi_ut_strlwr (strlwr) + * + * PARAMETERS: src_string - The source string to convert + * + * RETURN: None + * + * DESCRIPTION: Convert a string to lowercase + * + ******************************************************************************/ +void acpi_ut_strlwr(char *src_string) +{ + char *string; + + ACPI_FUNCTION_ENTRY(); + + if (!src_string) { + return; + } + + /* Walk entire string, lowercasing the letters */ + + for (string = src_string; *string; string++) { + *string = (char)tolower((int)*string); + } +} + +/******************************************************************************* + * + * FUNCTION: acpi_ut_strupr (strupr) + * + * PARAMETERS: src_string - The source string to convert + * + * RETURN: None + * + * DESCRIPTION: Convert a string to uppercase + * + ******************************************************************************/ + +void acpi_ut_strupr(char *src_string) +{ + char *string; + + ACPI_FUNCTION_ENTRY(); + + if (!src_string) { + return; + } + + /* Walk entire string, uppercasing the letters */ + + for (string = src_string; *string; string++) { + *string = (char)toupper((int)*string); + } +} + +/****************************************************************************** + * + * FUNCTION: acpi_ut_stricmp (stricmp) + * + * PARAMETERS: string1 - first string to compare + * string2 - second string to compare + * + * RETURN: int that signifies string relationship. Zero means strings + * are equal. + * + * DESCRIPTION: Case-insensitive string compare. Implementation of the + * non-ANSI stricmp function. + * + ******************************************************************************/ + +int acpi_ut_stricmp(char *string1, char *string2) +{ + int c1; + int c2; + + do { + c1 = tolower((int)*string1); + c2 = tolower((int)*string2); + + string1++; + string2++; + } + while ((c1 == c2) && (c1)); + + return (c1 - c2); +} + +/******************************************************************************* + * + * FUNCTION: acpi_ut_strtoul64 + * + * PARAMETERS: string - Null terminated string + * base - Radix of the string: 16 or ACPI_ANY_BASE; + * ACPI_ANY_BASE means 'in behalf of to_integer' + * ret_integer - Where the converted integer is returned + * + * RETURN: Status and Converted value + * + * DESCRIPTION: Convert a string into an unsigned value. Performs either a + * 32-bit or 64-bit conversion, depending on the current mode + * of the interpreter. + * + * NOTE: Does not support Octal strings, not needed. + * + ******************************************************************************/ + +acpi_status acpi_ut_strtoul64(char *string, u32 base, u64 *ret_integer) +{ + u32 this_digit = 0; + u64 return_value = 0; + u64 quotient; + u64 dividend; + u32 to_integer_op = (base == ACPI_ANY_BASE); + u32 mode32 = (acpi_gbl_integer_byte_width == 4); + u8 valid_digits = 0; + u8 sign_of0x = 0; + u8 term = 0; + + ACPI_FUNCTION_TRACE_STR(ut_stroul64, string); + + switch (base) { + case ACPI_ANY_BASE: + case 16: + + break; + + default: + + /* Invalid Base */ + + return_ACPI_STATUS(AE_BAD_PARAMETER); + } + + if (!string) { + goto error_exit; + } + + /* Skip over any white space in the buffer */ + + while ((*string) && (isspace((int)*string) || *string == '\t')) { + string++; + } + + if (to_integer_op) { + /* + * Base equal to ACPI_ANY_BASE means 'ToInteger operation case'. + * We need to determine if it is decimal or hexadecimal. + */ + if ((*string == '0') && (tolower((int)*(string + 1)) == 'x')) { + sign_of0x = 1; + base = 16; + + /* Skip over the leading '0x' */ + string += 2; + } else { + base = 10; + } + } + + /* Any string left? Check that '0x' is not followed by white space. */ + + if (!(*string) || isspace((int)*string) || *string == '\t') { + if (to_integer_op) { + goto error_exit; + } else { + goto all_done; + } + } + + /* + * Perform a 32-bit or 64-bit conversion, depending upon the current + * execution mode of the interpreter + */ + dividend = (mode32) ? ACPI_UINT32_MAX : ACPI_UINT64_MAX; + + /* Main loop: convert the string to a 32- or 64-bit integer */ + + while (*string) { + if (isdigit((int)*string)) { + + /* Convert ASCII 0-9 to Decimal value */ + + this_digit = ((u8)*string) - '0'; + } else if (base == 10) { + + /* Digit is out of range; possible in to_integer case only */ + + term = 1; + } else { + this_digit = (u8)toupper((int)*string); + if (isxdigit((int)this_digit)) { + + /* Convert ASCII Hex char to value */ + + this_digit = this_digit - 'A' + 10; + } else { + term = 1; + } + } + + if (term) { + if (to_integer_op) { + goto error_exit; + } else { + break; + } + } else if ((valid_digits == 0) && (this_digit == 0) + && !sign_of0x) { + + /* Skip zeros */ + string++; + continue; + } + + valid_digits++; + + if (sign_of0x + && ((valid_digits > 16) + || ((valid_digits > 8) && mode32))) { + /* + * This is to_integer operation case. + * No any restrictions for string-to-integer conversion, + * see ACPI spec. + */ + goto error_exit; + } + + /* Divide the digit into the correct position */ + + (void)acpi_ut_short_divide((dividend - (u64)this_digit), + base, "ient, NULL); + + if (return_value > quotient) { + if (to_integer_op) { + goto error_exit; + } else { + break; + } + } + + return_value *= base; + return_value += this_digit; + string++; + } + + /* All done, normal exit */ + +all_done: + + ACPI_DEBUG_PRINT((ACPI_DB_EXEC, "Converted value: %8.8X%8.8X\n", + ACPI_FORMAT_UINT64(return_value))); + + *ret_integer = return_value; + return_ACPI_STATUS(AE_OK); + +error_exit: + /* Base was set/validated above */ + + if (base == 10) { + return_ACPI_STATUS(AE_BAD_DECIMAL_CONSTANT); + } else { + return_ACPI_STATUS(AE_BAD_HEX_CONSTANT); + } +} + +#if defined (ACPI_DEBUGGER) || defined (ACPI_APPLICATION) +/******************************************************************************* + * + * FUNCTION: acpi_ut_safe_strcpy, acpi_ut_safe_strcat, acpi_ut_safe_strncat + * + * PARAMETERS: Adds a "DestSize" parameter to each of the standard string + * functions. This is the size of the Destination buffer. + * + * RETURN: TRUE if the operation would overflow the destination buffer. + * + * DESCRIPTION: Safe versions of standard Clib string functions. Ensure that + * the result of the operation will not overflow the output string + * buffer. + * + * NOTE: These functions are typically only helpful for processing + * user input and command lines. For most ACPICA code, the + * required buffer length is precisely calculated before buffer + * allocation, so the use of these functions is unnecessary. + * + ******************************************************************************/ + +u8 acpi_ut_safe_strcpy(char *dest, acpi_size dest_size, char *source) +{ + + if (strlen(source) >= dest_size) { + return (TRUE); + } + + strcpy(dest, source); + return (FALSE); +} + +u8 acpi_ut_safe_strcat(char *dest, acpi_size dest_size, char *source) +{ + + if ((strlen(dest) + strlen(source)) >= dest_size) { + return (TRUE); + } + + strcat(dest, source); + return (FALSE); +} + +u8 +acpi_ut_safe_strncat(char *dest, + acpi_size dest_size, + char *source, acpi_size max_transfer_length) +{ + acpi_size actual_transfer_length; + + actual_transfer_length = ACPI_MIN(max_transfer_length, strlen(source)); + + if ((strlen(dest) + actual_transfer_length) >= dest_size) { + return (TRUE); + } + + strncat(dest, source, max_transfer_length); + return (FALSE); +} +#endif diff --git a/drivers/acpi/acpica/utstring.c b/drivers/acpi/acpica/utstring.c index 8f3c883dfe0e..4ddd105d9741 100644 --- a/drivers/acpi/acpica/utstring.c +++ b/drivers/acpi/acpica/utstring.c @@ -48,286 +48,6 @@ #define _COMPONENT ACPI_UTILITIES ACPI_MODULE_NAME("utstring") -/* - * Non-ANSI C library functions - strlwr, strupr, stricmp, and a 64-bit - * version of strtoul. - */ -#ifdef ACPI_ASL_COMPILER -/******************************************************************************* - * - * FUNCTION: acpi_ut_strlwr (strlwr) - * - * PARAMETERS: src_string - The source string to convert - * - * RETURN: None - * - * DESCRIPTION: Convert string to lowercase - * - * NOTE: This is not a POSIX function, so it appears here, not in utclib.c - * - ******************************************************************************/ -void acpi_ut_strlwr(char *src_string) -{ - char *string; - - ACPI_FUNCTION_ENTRY(); - - if (!src_string) { - return; - } - - /* Walk entire string, lowercasing the letters */ - - for (string = src_string; *string; string++) { - *string = (char)tolower((int)*string); - } - - return; -} - -/****************************************************************************** - * - * FUNCTION: acpi_ut_stricmp (stricmp) - * - * PARAMETERS: string1 - first string to compare - * string2 - second string to compare - * - * RETURN: int that signifies string relationship. Zero means strings - * are equal. - * - * DESCRIPTION: Implementation of the non-ANSI stricmp function (compare - * strings with no case sensitivity) - * - ******************************************************************************/ - -int acpi_ut_stricmp(char *string1, char *string2) -{ - int c1; - int c2; - - do { - c1 = tolower((int)*string1); - c2 = tolower((int)*string2); - - string1++; - string2++; - } - while ((c1 == c2) && (c1)); - - return (c1 - c2); -} -#endif - -/******************************************************************************* - * - * FUNCTION: acpi_ut_strupr (strupr) - * - * PARAMETERS: src_string - The source string to convert - * - * RETURN: None - * - * DESCRIPTION: Convert string to uppercase - * - * NOTE: This is not a POSIX function, so it appears here, not in utclib.c - * - ******************************************************************************/ - -void acpi_ut_strupr(char *src_string) -{ - char *string; - - ACPI_FUNCTION_ENTRY(); - - if (!src_string) { - return; - } - - /* Walk entire string, uppercasing the letters */ - - for (string = src_string; *string; string++) { - *string = (char)toupper((int)*string); - } - - return; -} - -/******************************************************************************* - * - * FUNCTION: acpi_ut_strtoul64 - * - * PARAMETERS: string - Null terminated string - * base - Radix of the string: 16 or ACPI_ANY_BASE; - * ACPI_ANY_BASE means 'in behalf of to_integer' - * ret_integer - Where the converted integer is returned - * - * RETURN: Status and Converted value - * - * DESCRIPTION: Convert a string into an unsigned value. Performs either a - * 32-bit or 64-bit conversion, depending on the current mode - * of the interpreter. - * NOTE: Does not support Octal strings, not needed. - * - ******************************************************************************/ - -acpi_status acpi_ut_strtoul64(char *string, u32 base, u64 *ret_integer) -{ - u32 this_digit = 0; - u64 return_value = 0; - u64 quotient; - u64 dividend; - u32 to_integer_op = (base == ACPI_ANY_BASE); - u32 mode32 = (acpi_gbl_integer_byte_width == 4); - u8 valid_digits = 0; - u8 sign_of0x = 0; - u8 term = 0; - - ACPI_FUNCTION_TRACE_STR(ut_stroul64, string); - - switch (base) { - case ACPI_ANY_BASE: - case 16: - - break; - - default: - - /* Invalid Base */ - - return_ACPI_STATUS(AE_BAD_PARAMETER); - } - - if (!string) { - goto error_exit; - } - - /* Skip over any white space in the buffer */ - - while ((*string) && (isspace((int)*string) || *string == '\t')) { - string++; - } - - if (to_integer_op) { - /* - * Base equal to ACPI_ANY_BASE means 'ToInteger operation case'. - * We need to determine if it is decimal or hexadecimal. - */ - if ((*string == '0') && (tolower((int)*(string + 1)) == 'x')) { - sign_of0x = 1; - base = 16; - - /* Skip over the leading '0x' */ - string += 2; - } else { - base = 10; - } - } - - /* Any string left? Check that '0x' is not followed by white space. */ - - if (!(*string) || isspace((int)*string) || *string == '\t') { - if (to_integer_op) { - goto error_exit; - } else { - goto all_done; - } - } - - /* - * Perform a 32-bit or 64-bit conversion, depending upon the current - * execution mode of the interpreter - */ - dividend = (mode32) ? ACPI_UINT32_MAX : ACPI_UINT64_MAX; - - /* Main loop: convert the string to a 32- or 64-bit integer */ - - while (*string) { - if (isdigit((int)*string)) { - - /* Convert ASCII 0-9 to Decimal value */ - - this_digit = ((u8)*string) - '0'; - } else if (base == 10) { - - /* Digit is out of range; possible in to_integer case only */ - - term = 1; - } else { - this_digit = (u8)toupper((int)*string); - if (isxdigit((int)this_digit)) { - - /* Convert ASCII Hex char to value */ - - this_digit = this_digit - 'A' + 10; - } else { - term = 1; - } - } - - if (term) { - if (to_integer_op) { - goto error_exit; - } else { - break; - } - } else if ((valid_digits == 0) && (this_digit == 0) - && !sign_of0x) { - - /* Skip zeros */ - string++; - continue; - } - - valid_digits++; - - if (sign_of0x - && ((valid_digits > 16) - || ((valid_digits > 8) && mode32))) { - /* - * This is to_integer operation case. - * No any restrictions for string-to-integer conversion, - * see ACPI spec. - */ - goto error_exit; - } - - /* Divide the digit into the correct position */ - - (void)acpi_ut_short_divide((dividend - (u64)this_digit), - base, "ient, NULL); - - if (return_value > quotient) { - if (to_integer_op) { - goto error_exit; - } else { - break; - } - } - - return_value *= base; - return_value += this_digit; - string++; - } - - /* All done, normal exit */ - -all_done: - - ACPI_DEBUG_PRINT((ACPI_DB_EXEC, "Converted value: %8.8X%8.8X\n", - ACPI_FORMAT_UINT64(return_value))); - - *ret_integer = return_value; - return_ACPI_STATUS(AE_OK); - -error_exit: - /* Base was set/validated above */ - - if (base == 10) { - return_ACPI_STATUS(AE_BAD_DECIMAL_CONSTANT); - } else { - return_ACPI_STATUS(AE_BAD_HEX_CONSTANT); - } -} - /******************************************************************************* * * FUNCTION: acpi_ut_print_string @@ -342,7 +62,6 @@ acpi_status acpi_ut_strtoul64(char *string, u32 base, u64 *ret_integer) * sequences. * ******************************************************************************/ - void acpi_ut_print_string(char *string, u16 max_length) { u32 i; @@ -584,64 +303,3 @@ void ut_convert_backslashes(char *pathname) } } #endif - -#if defined (ACPI_DEBUGGER) || defined (ACPI_APPLICATION) -/******************************************************************************* - * - * FUNCTION: acpi_ut_safe_strcpy, acpi_ut_safe_strcat, acpi_ut_safe_strncat - * - * PARAMETERS: Adds a "DestSize" parameter to each of the standard string - * functions. This is the size of the Destination buffer. - * - * RETURN: TRUE if the operation would overflow the destination buffer. - * - * DESCRIPTION: Safe versions of standard Clib string functions. Ensure that - * the result of the operation will not overflow the output string - * buffer. - * - * NOTE: These functions are typically only helpful for processing - * user input and command lines. For most ACPICA code, the - * required buffer length is precisely calculated before buffer - * allocation, so the use of these functions is unnecessary. - * - ******************************************************************************/ - -u8 acpi_ut_safe_strcpy(char *dest, acpi_size dest_size, char *source) -{ - - if (strlen(source) >= dest_size) { - return (TRUE); - } - - strcpy(dest, source); - return (FALSE); -} - -u8 acpi_ut_safe_strcat(char *dest, acpi_size dest_size, char *source) -{ - - if ((strlen(dest) + strlen(source)) >= dest_size) { - return (TRUE); - } - - strcat(dest, source); - return (FALSE); -} - -u8 -acpi_ut_safe_strncat(char *dest, - acpi_size dest_size, - char *source, acpi_size max_transfer_length) -{ - acpi_size actual_transfer_length; - - actual_transfer_length = ACPI_MIN(max_transfer_length, strlen(source)); - - if ((strlen(dest) + actual_transfer_length) >= dest_size) { - return (TRUE); - } - - strncat(dest, source, max_transfer_length); - return (FALSE); -} -#endif diff --git a/drivers/acpi/acpica/utxface.c b/drivers/acpi/acpica/utxface.c index 51cf52d52243..4f332815db00 100644 --- a/drivers/acpi/acpica/utxface.c +++ b/drivers/acpi/acpica/utxface.c @@ -92,13 +92,6 @@ acpi_status __init acpi_terminate(void) acpi_ut_mutex_terminate(); -#ifdef ACPI_DEBUGGER - - /* Shut down the debugger */ - - acpi_db_terminate(); -#endif - /* Now we can shutdown the OS-dependent layer */ status = acpi_os_terminate(); @@ -517,7 +510,8 @@ acpi_decode_pld_buffer(u8 *in_buffer, /* Parameter validation */ - if (!in_buffer || !return_buffer || (length < 16)) { + if (!in_buffer || !return_buffer + || (length < ACPI_PLD_REV1_BUFFER_SIZE)) { return (AE_BAD_PARAMETER); } @@ -567,7 +561,7 @@ acpi_decode_pld_buffer(u8 *in_buffer, pld_info->rotation = ACPI_PLD_GET_ROTATION(&dword); pld_info->order = ACPI_PLD_GET_ORDER(&dword); - if (length >= ACPI_PLD_BUFFER_SIZE) { + if (length >= ACPI_PLD_REV2_BUFFER_SIZE) { /* Fifth 32-bit DWord (Revision 2 of _PLD) */ diff --git a/drivers/acpi/acpica/utxfinit.c b/drivers/acpi/acpica/utxfinit.c index 42a32a66ef22..a7137ec28447 100644 --- a/drivers/acpi/acpica/utxfinit.c +++ b/drivers/acpi/acpica/utxfinit.c @@ -124,17 +124,6 @@ acpi_status __init acpi_initialize_subsystem(void) return_ACPI_STATUS(status); } - /* If configured, initialize the AML debugger */ - -#ifdef ACPI_DEBUGGER - status = acpi_db_initialize(); - if (ACPI_FAILURE(status)) { - ACPI_EXCEPTION((AE_INFO, status, - "During Debugger initialization")); - return_ACPI_STATUS(status); - } -#endif - return_ACPI_STATUS(AE_OK); } diff --git a/drivers/acpi/apei/apei-base.c b/drivers/acpi/apei/apei-base.c index a85ac07f3da3..a2c8d7adb6eb 100644 --- a/drivers/acpi/apei/apei-base.c +++ b/drivers/acpi/apei/apei-base.c @@ -24,10 +24,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include diff --git a/drivers/acpi/apei/einj.c b/drivers/acpi/apei/einj.c index a095d4f858da..0431883653be 100644 --- a/drivers/acpi/apei/einj.c +++ b/drivers/acpi/apei/einj.c @@ -18,10 +18,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include diff --git a/drivers/acpi/apei/erst-dbg.c b/drivers/acpi/apei/erst-dbg.c index 04ab5c9d3ced..6330f557a2c8 100644 --- a/drivers/acpi/apei/erst-dbg.c +++ b/drivers/acpi/apei/erst-dbg.c @@ -17,10 +17,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include diff --git a/drivers/acpi/apei/erst.c b/drivers/acpi/apei/erst.c index 3670bbab57a3..6682c5daf742 100644 --- a/drivers/acpi/apei/erst.c +++ b/drivers/acpi/apei/erst.c @@ -18,10 +18,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index 2bfd53cbfe80..23981ac1c6c2 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -23,10 +23,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include diff --git a/drivers/acpi/apei/hest.c b/drivers/acpi/apei/hest.c index 06e9b411a0a2..20b3fcf4007c 100644 --- a/drivers/acpi/apei/hest.c +++ b/drivers/acpi/apei/hest.c @@ -21,10 +21,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c index b3628cc01a53..b719ab3090bb 100644 --- a/drivers/acpi/battery.c +++ b/drivers/acpi/battery.c @@ -18,10 +18,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ diff --git a/drivers/acpi/blacklist.c b/drivers/acpi/blacklist.c index 278dc4be992a..96809cd99ace 100644 --- a/drivers/acpi/blacklist.c +++ b/drivers/acpi/blacklist.c @@ -20,10 +20,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c index 513e7230e3d0..46506e7687cd 100644 --- a/drivers/acpi/bus.c +++ b/drivers/acpi/bus.c @@ -15,10 +15,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ @@ -423,6 +419,406 @@ static void acpi_bus_notify(acpi_handle handle, u32 type, void *data) acpi_evaluate_ost(handle, type, ost_code, NULL); } +static void acpi_device_notify(acpi_handle handle, u32 event, void *data) +{ + struct acpi_device *device = data; + + device->driver->ops.notify(device, event); +} + +static void acpi_device_notify_fixed(void *data) +{ + struct acpi_device *device = data; + + /* Fixed hardware devices have no handles */ + acpi_device_notify(NULL, ACPI_FIXED_HARDWARE_EVENT, device); +} + +static u32 acpi_device_fixed_event(void *data) +{ + acpi_os_execute(OSL_NOTIFY_HANDLER, acpi_device_notify_fixed, data); + return ACPI_INTERRUPT_HANDLED; +} + +static int acpi_device_install_notify_handler(struct acpi_device *device) +{ + acpi_status status; + + if (device->device_type == ACPI_BUS_TYPE_POWER_BUTTON) + status = + acpi_install_fixed_event_handler(ACPI_EVENT_POWER_BUTTON, + acpi_device_fixed_event, + device); + else if (device->device_type == ACPI_BUS_TYPE_SLEEP_BUTTON) + status = + acpi_install_fixed_event_handler(ACPI_EVENT_SLEEP_BUTTON, + acpi_device_fixed_event, + device); + else + status = acpi_install_notify_handler(device->handle, + ACPI_DEVICE_NOTIFY, + acpi_device_notify, + device); + + if (ACPI_FAILURE(status)) + return -EINVAL; + return 0; +} + +static void acpi_device_remove_notify_handler(struct acpi_device *device) +{ + if (device->device_type == ACPI_BUS_TYPE_POWER_BUTTON) + acpi_remove_fixed_event_handler(ACPI_EVENT_POWER_BUTTON, + acpi_device_fixed_event); + else if (device->device_type == ACPI_BUS_TYPE_SLEEP_BUTTON) + acpi_remove_fixed_event_handler(ACPI_EVENT_SLEEP_BUTTON, + acpi_device_fixed_event); + else + acpi_remove_notify_handler(device->handle, ACPI_DEVICE_NOTIFY, + acpi_device_notify); +} + +/* -------------------------------------------------------------------------- + Device Matching + -------------------------------------------------------------------------- */ + +static struct acpi_device *acpi_primary_dev_companion(struct acpi_device *adev, + const struct device *dev) +{ + struct mutex *physical_node_lock = &adev->physical_node_lock; + + mutex_lock(physical_node_lock); + if (list_empty(&adev->physical_node_list)) { + adev = NULL; + } else { + const struct acpi_device_physical_node *node; + + node = list_first_entry(&adev->physical_node_list, + struct acpi_device_physical_node, node); + if (node->dev != dev) + adev = NULL; + } + mutex_unlock(physical_node_lock); + return adev; +} + +/** + * acpi_device_is_first_physical_node - Is given dev first physical node + * @adev: ACPI companion device + * @dev: Physical device to check + * + * Function checks if given @dev is the first physical devices attached to + * the ACPI companion device. This distinction is needed in some cases + * where the same companion device is shared between many physical devices. + * + * Note that the caller have to provide valid @adev pointer. + */ +bool acpi_device_is_first_physical_node(struct acpi_device *adev, + const struct device *dev) +{ + return !!acpi_primary_dev_companion(adev, dev); +} + +/* + * acpi_companion_match() - Can we match via ACPI companion device + * @dev: Device in question + * + * Check if the given device has an ACPI companion and if that companion has + * a valid list of PNP IDs, and if the device is the first (primary) physical + * device associated with it. Return the companion pointer if that's the case + * or NULL otherwise. + * + * If multiple physical devices are attached to a single ACPI companion, we need + * to be careful. The usage scenario for this kind of relationship is that all + * of the physical devices in question use resources provided by the ACPI + * companion. A typical case is an MFD device where all the sub-devices share + * the parent's ACPI companion. In such cases we can only allow the primary + * (first) physical device to be matched with the help of the companion's PNP + * IDs. + * + * Additional physical devices sharing the ACPI companion can still use + * resources available from it but they will be matched normally using functions + * provided by their bus types (and analogously for their modalias). + */ +struct acpi_device *acpi_companion_match(const struct device *dev) +{ + struct acpi_device *adev; + + adev = ACPI_COMPANION(dev); + if (!adev) + return NULL; + + if (list_empty(&adev->pnp.ids)) + return NULL; + + return acpi_primary_dev_companion(adev, dev); +} + +/** + * acpi_of_match_device - Match device object using the "compatible" property. + * @adev: ACPI device object to match. + * @of_match_table: List of device IDs to match against. + * + * If @dev has an ACPI companion which has ACPI_DT_NAMESPACE_HID in its list of + * identifiers and a _DSD object with the "compatible" property, use that + * property to match against the given list of identifiers. + */ +static bool acpi_of_match_device(struct acpi_device *adev, + const struct of_device_id *of_match_table) +{ + const union acpi_object *of_compatible, *obj; + int i, nval; + + if (!adev) + return false; + + of_compatible = adev->data.of_compatible; + if (!of_match_table || !of_compatible) + return false; + + if (of_compatible->type == ACPI_TYPE_PACKAGE) { + nval = of_compatible->package.count; + obj = of_compatible->package.elements; + } else { /* Must be ACPI_TYPE_STRING. */ + nval = 1; + obj = of_compatible; + } + /* Now we can look for the driver DT compatible strings */ + for (i = 0; i < nval; i++, obj++) { + const struct of_device_id *id; + + for (id = of_match_table; id->compatible[0]; id++) + if (!strcasecmp(obj->string.pointer, id->compatible)) + return true; + } + + return false; +} + +static bool __acpi_match_device_cls(const struct acpi_device_id *id, + struct acpi_hardware_id *hwid) +{ + int i, msk, byte_shift; + char buf[3]; + + if (!id->cls) + return false; + + /* Apply class-code bitmask, before checking each class-code byte */ + for (i = 1; i <= 3; i++) { + byte_shift = 8 * (3 - i); + msk = (id->cls_msk >> byte_shift) & 0xFF; + if (!msk) + continue; + + sprintf(buf, "%02x", (id->cls >> byte_shift) & msk); + if (strncmp(buf, &hwid->id[(i - 1) * 2], 2)) + return false; + } + return true; +} + +static const struct acpi_device_id *__acpi_match_device( + struct acpi_device *device, + const struct acpi_device_id *ids, + const struct of_device_id *of_ids) +{ + const struct acpi_device_id *id; + struct acpi_hardware_id *hwid; + + /* + * If the device is not present, it is unnecessary to load device + * driver for it. + */ + if (!device || !device->status.present) + return NULL; + + list_for_each_entry(hwid, &device->pnp.ids, list) { + /* First, check the ACPI/PNP IDs provided by the caller. */ + for (id = ids; id->id[0] || id->cls; id++) { + if (id->id[0] && !strcmp((char *) id->id, hwid->id)) + return id; + else if (id->cls && __acpi_match_device_cls(id, hwid)) + return id; + } + + /* + * Next, check ACPI_DT_NAMESPACE_HID and try to match the + * "compatible" property if found. + * + * The id returned by the below is not valid, but the only + * caller passing non-NULL of_ids here is only interested in + * whether or not the return value is NULL. + */ + if (!strcmp(ACPI_DT_NAMESPACE_HID, hwid->id) + && acpi_of_match_device(device, of_ids)) + return id; + } + return NULL; +} + +/** + * acpi_match_device - Match a struct device against a given list of ACPI IDs + * @ids: Array of struct acpi_device_id object to match against. + * @dev: The device structure to match. + * + * Check if @dev has a valid ACPI handle and if there is a struct acpi_device + * object for that handle and use that object to match against a given list of + * device IDs. + * + * Return a pointer to the first matching ID on success or %NULL on failure. + */ +const struct acpi_device_id *acpi_match_device(const struct acpi_device_id *ids, + const struct device *dev) +{ + return __acpi_match_device(acpi_companion_match(dev), ids, NULL); +} +EXPORT_SYMBOL_GPL(acpi_match_device); + +int acpi_match_device_ids(struct acpi_device *device, + const struct acpi_device_id *ids) +{ + return __acpi_match_device(device, ids, NULL) ? 0 : -ENOENT; +} +EXPORT_SYMBOL(acpi_match_device_ids); + +bool acpi_driver_match_device(struct device *dev, + const struct device_driver *drv) +{ + if (!drv->acpi_match_table) + return acpi_of_match_device(ACPI_COMPANION(dev), + drv->of_match_table); + + return !!__acpi_match_device(acpi_companion_match(dev), + drv->acpi_match_table, drv->of_match_table); +} +EXPORT_SYMBOL_GPL(acpi_driver_match_device); + +/* -------------------------------------------------------------------------- + ACPI Driver Management + -------------------------------------------------------------------------- */ + +/** + * acpi_bus_register_driver - register a driver with the ACPI bus + * @driver: driver being registered + * + * Registers a driver with the ACPI bus. Searches the namespace for all + * devices that match the driver's criteria and binds. Returns zero for + * success or a negative error status for failure. + */ +int acpi_bus_register_driver(struct acpi_driver *driver) +{ + int ret; + + if (acpi_disabled) + return -ENODEV; + driver->drv.name = driver->name; + driver->drv.bus = &acpi_bus_type; + driver->drv.owner = driver->owner; + + ret = driver_register(&driver->drv); + return ret; +} + +EXPORT_SYMBOL(acpi_bus_register_driver); + +/** + * acpi_bus_unregister_driver - unregisters a driver with the ACPI bus + * @driver: driver to unregister + * + * Unregisters a driver with the ACPI bus. Searches the namespace for all + * devices that match the driver's criteria and unbinds. + */ +void acpi_bus_unregister_driver(struct acpi_driver *driver) +{ + driver_unregister(&driver->drv); +} + +EXPORT_SYMBOL(acpi_bus_unregister_driver); + +/* -------------------------------------------------------------------------- + ACPI Bus operations + -------------------------------------------------------------------------- */ + +static int acpi_bus_match(struct device *dev, struct device_driver *drv) +{ + struct acpi_device *acpi_dev = to_acpi_device(dev); + struct acpi_driver *acpi_drv = to_acpi_driver(drv); + + return acpi_dev->flags.match_driver + && !acpi_match_device_ids(acpi_dev, acpi_drv->ids); +} + +static int acpi_device_uevent(struct device *dev, struct kobj_uevent_env *env) +{ + return __acpi_device_uevent_modalias(to_acpi_device(dev), env); +} + +static int acpi_device_probe(struct device *dev) +{ + struct acpi_device *acpi_dev = to_acpi_device(dev); + struct acpi_driver *acpi_drv = to_acpi_driver(dev->driver); + int ret; + + if (acpi_dev->handler && !acpi_is_pnp_device(acpi_dev)) + return -EINVAL; + + if (!acpi_drv->ops.add) + return -ENOSYS; + + ret = acpi_drv->ops.add(acpi_dev); + if (ret) + return ret; + + acpi_dev->driver = acpi_drv; + ACPI_DEBUG_PRINT((ACPI_DB_INFO, + "Driver [%s] successfully bound to device [%s]\n", + acpi_drv->name, acpi_dev->pnp.bus_id)); + + if (acpi_drv->ops.notify) { + ret = acpi_device_install_notify_handler(acpi_dev); + if (ret) { + if (acpi_drv->ops.remove) + acpi_drv->ops.remove(acpi_dev); + + acpi_dev->driver = NULL; + acpi_dev->driver_data = NULL; + return ret; + } + } + + ACPI_DEBUG_PRINT((ACPI_DB_INFO, "Found driver [%s] for device [%s]\n", + acpi_drv->name, acpi_dev->pnp.bus_id)); + get_device(dev); + return 0; +} + +static int acpi_device_remove(struct device * dev) +{ + struct acpi_device *acpi_dev = to_acpi_device(dev); + struct acpi_driver *acpi_drv = acpi_dev->driver; + + if (acpi_drv) { + if (acpi_drv->ops.notify) + acpi_device_remove_notify_handler(acpi_dev); + if (acpi_drv->ops.remove) + acpi_drv->ops.remove(acpi_dev); + } + acpi_dev->driver = NULL; + acpi_dev->driver_data = NULL; + + put_device(dev); + return 0; +} + +struct bus_type acpi_bus_type = { + .name = "acpi", + .match = acpi_bus_match, + .probe = acpi_device_probe, + .remove = acpi_device_remove, + .uevent = acpi_device_uevent, +}; + /* -------------------------------------------------------------------------- Initialization/Cleanup -------------------------------------------------------------------------- */ @@ -661,7 +1057,9 @@ static int __init acpi_bus_init(void) */ acpi_root_dir = proc_mkdir(ACPI_BUS_FILE_ROOT, NULL); - return 0; + result = bus_register(&acpi_bus_type); + if (!result) + return 0; /* Mimic structured exception handling */ error1: diff --git a/drivers/acpi/button.c b/drivers/acpi/button.c index 6d5d1832a588..5c3b0918d5fd 100644 --- a/drivers/acpi/button.c +++ b/drivers/acpi/button.c @@ -16,10 +16,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ diff --git a/drivers/acpi/cm_sbs.c b/drivers/acpi/cm_sbs.c index 6c9ee68e46fb..d0918d421f90 100644 --- a/drivers/acpi/cm_sbs.c +++ b/drivers/acpi/cm_sbs.c @@ -11,10 +11,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ diff --git a/drivers/acpi/container.c b/drivers/acpi/container.c index c8ead9f97375..12c240903c18 100644 --- a/drivers/acpi/container.c +++ b/drivers/acpi/container.c @@ -20,10 +20,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ #include diff --git a/drivers/acpi/debugfs.c b/drivers/acpi/debugfs.c index 6b1919f6bd82..68bb305b977f 100644 --- a/drivers/acpi/debugfs.c +++ b/drivers/acpi/debugfs.c @@ -7,6 +7,8 @@ #include #include +#include "internal.h" + #define _COMPONENT ACPI_SYSTEM_COMPONENT ACPI_MODULE_NAME("debugfs"); diff --git a/drivers/acpi/device_pm.c b/drivers/acpi/device_pm.c index 88dbbb115285..4806b7f856c4 100644 --- a/drivers/acpi/device_pm.c +++ b/drivers/acpi/device_pm.c @@ -15,10 +15,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ @@ -1123,6 +1119,14 @@ int acpi_dev_pm_attach(struct device *dev, bool power_on) if (dev->pm_domain) return -EEXIST; + /* + * Only attach the power domain to the first device if the + * companion is shared by multiple. This is to prevent doing power + * management twice. + */ + if (!acpi_device_is_first_physical_node(adev, dev)) + return -EBUSY; + acpi_add_pm_notifier(adev, dev, acpi_pm_notify_work_func); dev->pm_domain = &acpi_general_pm_domain; if (power_on) { diff --git a/drivers/acpi/device_sysfs.c b/drivers/acpi/device_sysfs.c new file mode 100644 index 000000000000..4ab4582e586b --- /dev/null +++ b/drivers/acpi/device_sysfs.c @@ -0,0 +1,521 @@ +/* + * drivers/acpi/device_sysfs.c - ACPI device sysfs attributes and modalias. + * + * Copyright (C) 2015, Intel Corp. + * Author: Mika Westerberg + * Author: Rafael J. Wysocki + * + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + */ + +#include +#include +#include +#include + +#include "internal.h" + +/** + * create_pnp_modalias - Create hid/cid(s) string for modalias and uevent + * @acpi_dev: ACPI device object. + * @modalias: Buffer to print into. + * @size: Size of the buffer. + * + * Creates hid/cid(s) string needed for modalias and uevent + * e.g. on a device with hid:IBM0001 and cid:ACPI0001 you get: + * char *modalias: "acpi:IBM0001:ACPI0001" + * Return: 0: no _HID and no _CID + * -EINVAL: output error + * -ENOMEM: output is truncated +*/ +static int create_pnp_modalias(struct acpi_device *acpi_dev, char *modalias, + int size) +{ + int len; + int count; + struct acpi_hardware_id *id; + + /* + * Since we skip ACPI_DT_NAMESPACE_HID from the modalias below, 0 should + * be returned if ACPI_DT_NAMESPACE_HID is the only ACPI/PNP ID in the + * device's list. + */ + count = 0; + list_for_each_entry(id, &acpi_dev->pnp.ids, list) + if (strcmp(id->id, ACPI_DT_NAMESPACE_HID)) + count++; + + if (!count) + return 0; + + len = snprintf(modalias, size, "acpi:"); + if (len <= 0) + return len; + + size -= len; + + list_for_each_entry(id, &acpi_dev->pnp.ids, list) { + if (!strcmp(id->id, ACPI_DT_NAMESPACE_HID)) + continue; + + count = snprintf(&modalias[len], size, "%s:", id->id); + if (count < 0) + return -EINVAL; + + if (count >= size) + return -ENOMEM; + + len += count; + size -= count; + } + modalias[len] = '\0'; + return len; +} + +/** + * create_of_modalias - Creates DT compatible string for modalias and uevent + * @acpi_dev: ACPI device object. + * @modalias: Buffer to print into. + * @size: Size of the buffer. + * + * Expose DT compatible modalias as of:NnameTCcompatible. This function should + * only be called for devices having ACPI_DT_NAMESPACE_HID in their list of + * ACPI/PNP IDs. + */ +static int create_of_modalias(struct acpi_device *acpi_dev, char *modalias, + int size) +{ + struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER }; + const union acpi_object *of_compatible, *obj; + int len, count; + int i, nval; + char *c; + + acpi_get_name(acpi_dev->handle, ACPI_SINGLE_NAME, &buf); + /* DT strings are all in lower case */ + for (c = buf.pointer; *c != '\0'; c++) + *c = tolower(*c); + + len = snprintf(modalias, size, "of:N%sT", (char *)buf.pointer); + ACPI_FREE(buf.pointer); + + if (len <= 0) + return len; + + of_compatible = acpi_dev->data.of_compatible; + if (of_compatible->type == ACPI_TYPE_PACKAGE) { + nval = of_compatible->package.count; + obj = of_compatible->package.elements; + } else { /* Must be ACPI_TYPE_STRING. */ + nval = 1; + obj = of_compatible; + } + for (i = 0; i < nval; i++, obj++) { + count = snprintf(&modalias[len], size, "C%s", + obj->string.pointer); + if (count < 0) + return -EINVAL; + + if (count >= size) + return -ENOMEM; + + len += count; + size -= count; + } + modalias[len] = '\0'; + return len; +} + +int __acpi_device_uevent_modalias(struct acpi_device *adev, + struct kobj_uevent_env *env) +{ + int len; + + if (!adev) + return -ENODEV; + + if (list_empty(&adev->pnp.ids)) + return 0; + + if (add_uevent_var(env, "MODALIAS=")) + return -ENOMEM; + + len = create_pnp_modalias(adev, &env->buf[env->buflen - 1], + sizeof(env->buf) - env->buflen); + if (len < 0) + return len; + + env->buflen += len; + if (!adev->data.of_compatible) + return 0; + + if (len > 0 && add_uevent_var(env, "MODALIAS=")) + return -ENOMEM; + + len = create_of_modalias(adev, &env->buf[env->buflen - 1], + sizeof(env->buf) - env->buflen); + if (len < 0) + return len; + + env->buflen += len; + + return 0; +} + +/** + * acpi_device_uevent_modalias - uevent modalias for ACPI-enumerated devices. + * + * Create the uevent modalias field for ACPI-enumerated devices. + * + * Because other buses do not support ACPI HIDs & CIDs, e.g. for a device with + * hid:IBM0001 and cid:ACPI0001 you get: "acpi:IBM0001:ACPI0001". + */ +int acpi_device_uevent_modalias(struct device *dev, struct kobj_uevent_env *env) +{ + return __acpi_device_uevent_modalias(acpi_companion_match(dev), env); +} +EXPORT_SYMBOL_GPL(acpi_device_uevent_modalias); + +static int __acpi_device_modalias(struct acpi_device *adev, char *buf, int size) +{ + int len, count; + + if (!adev) + return -ENODEV; + + if (list_empty(&adev->pnp.ids)) + return 0; + + len = create_pnp_modalias(adev, buf, size - 1); + if (len < 0) { + return len; + } else if (len > 0) { + buf[len++] = '\n'; + size -= len; + } + if (!adev->data.of_compatible) + return len; + + count = create_of_modalias(adev, buf + len, size - 1); + if (count < 0) { + return count; + } else if (count > 0) { + len += count; + buf[len++] = '\n'; + } + + return len; +} + +/** + * acpi_device_modalias - modalias sysfs attribute for ACPI-enumerated devices. + * + * Create the modalias sysfs attribute for ACPI-enumerated devices. + * + * Because other buses do not support ACPI HIDs & CIDs, e.g. for a device with + * hid:IBM0001 and cid:ACPI0001 you get: "acpi:IBM0001:ACPI0001". + */ +int acpi_device_modalias(struct device *dev, char *buf, int size) +{ + return __acpi_device_modalias(acpi_companion_match(dev), buf, size); +} +EXPORT_SYMBOL_GPL(acpi_device_modalias); + +static ssize_t +acpi_device_modalias_show(struct device *dev, struct device_attribute *attr, char *buf) { + return __acpi_device_modalias(to_acpi_device(dev), buf, 1024); +} +static DEVICE_ATTR(modalias, 0444, acpi_device_modalias_show, NULL); + +static ssize_t real_power_state_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct acpi_device *adev = to_acpi_device(dev); + int state; + int ret; + + ret = acpi_device_get_power(adev, &state); + if (ret) + return ret; + + return sprintf(buf, "%s\n", acpi_power_state_string(state)); +} + +static DEVICE_ATTR(real_power_state, 0444, real_power_state_show, NULL); + +static ssize_t power_state_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct acpi_device *adev = to_acpi_device(dev); + + return sprintf(buf, "%s\n", acpi_power_state_string(adev->power.state)); +} + +static DEVICE_ATTR(power_state, 0444, power_state_show, NULL); + +static ssize_t +acpi_eject_store(struct device *d, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct acpi_device *acpi_device = to_acpi_device(d); + acpi_object_type not_used; + acpi_status status; + + if (!count || buf[0] != '1') + return -EINVAL; + + if ((!acpi_device->handler || !acpi_device->handler->hotplug.enabled) + && !acpi_device->driver) + return -ENODEV; + + status = acpi_get_type(acpi_device->handle, ¬_used); + if (ACPI_FAILURE(status) || !acpi_device->flags.ejectable) + return -ENODEV; + + get_device(&acpi_device->dev); + status = acpi_hotplug_schedule(acpi_device, ACPI_OST_EC_OSPM_EJECT); + if (ACPI_SUCCESS(status)) + return count; + + put_device(&acpi_device->dev); + acpi_evaluate_ost(acpi_device->handle, ACPI_OST_EC_OSPM_EJECT, + ACPI_OST_SC_NON_SPECIFIC_FAILURE, NULL); + return status == AE_NO_MEMORY ? -ENOMEM : -EAGAIN; +} + +static DEVICE_ATTR(eject, 0200, NULL, acpi_eject_store); + +static ssize_t +acpi_device_hid_show(struct device *dev, struct device_attribute *attr, char *buf) { + struct acpi_device *acpi_dev = to_acpi_device(dev); + + return sprintf(buf, "%s\n", acpi_device_hid(acpi_dev)); +} +static DEVICE_ATTR(hid, 0444, acpi_device_hid_show, NULL); + +static ssize_t acpi_device_uid_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct acpi_device *acpi_dev = to_acpi_device(dev); + + return sprintf(buf, "%s\n", acpi_dev->pnp.unique_id); +} +static DEVICE_ATTR(uid, 0444, acpi_device_uid_show, NULL); + +static ssize_t acpi_device_adr_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct acpi_device *acpi_dev = to_acpi_device(dev); + + return sprintf(buf, "0x%08x\n", + (unsigned int)(acpi_dev->pnp.bus_address)); +} +static DEVICE_ATTR(adr, 0444, acpi_device_adr_show, NULL); + +static ssize_t +acpi_device_path_show(struct device *dev, struct device_attribute *attr, char *buf) { + struct acpi_device *acpi_dev = to_acpi_device(dev); + struct acpi_buffer path = {ACPI_ALLOCATE_BUFFER, NULL}; + int result; + + result = acpi_get_name(acpi_dev->handle, ACPI_FULL_PATHNAME, &path); + if (result) + goto end; + + result = sprintf(buf, "%s\n", (char*)path.pointer); + kfree(path.pointer); +end: + return result; +} +static DEVICE_ATTR(path, 0444, acpi_device_path_show, NULL); + +/* sysfs file that shows description text from the ACPI _STR method */ +static ssize_t description_show(struct device *dev, + struct device_attribute *attr, + char *buf) { + struct acpi_device *acpi_dev = to_acpi_device(dev); + int result; + + if (acpi_dev->pnp.str_obj == NULL) + return 0; + + /* + * The _STR object contains a Unicode identifier for a device. + * We need to convert to utf-8 so it can be displayed. + */ + result = utf16s_to_utf8s( + (wchar_t *)acpi_dev->pnp.str_obj->buffer.pointer, + acpi_dev->pnp.str_obj->buffer.length, + UTF16_LITTLE_ENDIAN, buf, + PAGE_SIZE); + + buf[result++] = '\n'; + + return result; +} +static DEVICE_ATTR(description, 0444, description_show, NULL); + +static ssize_t +acpi_device_sun_show(struct device *dev, struct device_attribute *attr, + char *buf) { + struct acpi_device *acpi_dev = to_acpi_device(dev); + acpi_status status; + unsigned long long sun; + + status = acpi_evaluate_integer(acpi_dev->handle, "_SUN", NULL, &sun); + if (ACPI_FAILURE(status)) + return -ENODEV; + + return sprintf(buf, "%llu\n", sun); +} +static DEVICE_ATTR(sun, 0444, acpi_device_sun_show, NULL); + +static ssize_t status_show(struct device *dev, struct device_attribute *attr, + char *buf) { + struct acpi_device *acpi_dev = to_acpi_device(dev); + acpi_status status; + unsigned long long sta; + + status = acpi_evaluate_integer(acpi_dev->handle, "_STA", NULL, &sta); + if (ACPI_FAILURE(status)) + return -ENODEV; + + return sprintf(buf, "%llu\n", sta); +} +static DEVICE_ATTR_RO(status); + +/** + * acpi_device_setup_files - Create sysfs attributes of an ACPI device. + * @dev: ACPI device object. + */ +int acpi_device_setup_files(struct acpi_device *dev) +{ + struct acpi_buffer buffer = {ACPI_ALLOCATE_BUFFER, NULL}; + acpi_status status; + int result = 0; + + /* + * Devices gotten from FADT don't have a "path" attribute + */ + if (dev->handle) { + result = device_create_file(&dev->dev, &dev_attr_path); + if (result) + goto end; + } + + if (!list_empty(&dev->pnp.ids)) { + result = device_create_file(&dev->dev, &dev_attr_hid); + if (result) + goto end; + + result = device_create_file(&dev->dev, &dev_attr_modalias); + if (result) + goto end; + } + + /* + * If device has _STR, 'description' file is created + */ + if (acpi_has_method(dev->handle, "_STR")) { + status = acpi_evaluate_object(dev->handle, "_STR", + NULL, &buffer); + if (ACPI_FAILURE(status)) + buffer.pointer = NULL; + dev->pnp.str_obj = buffer.pointer; + result = device_create_file(&dev->dev, &dev_attr_description); + if (result) + goto end; + } + + if (dev->pnp.type.bus_address) + result = device_create_file(&dev->dev, &dev_attr_adr); + if (dev->pnp.unique_id) + result = device_create_file(&dev->dev, &dev_attr_uid); + + if (acpi_has_method(dev->handle, "_SUN")) { + result = device_create_file(&dev->dev, &dev_attr_sun); + if (result) + goto end; + } + + if (acpi_has_method(dev->handle, "_STA")) { + result = device_create_file(&dev->dev, &dev_attr_status); + if (result) + goto end; + } + + /* + * If device has _EJ0, 'eject' file is created that is used to trigger + * hot-removal function from userland. + */ + if (acpi_has_method(dev->handle, "_EJ0")) { + result = device_create_file(&dev->dev, &dev_attr_eject); + if (result) + return result; + } + + if (dev->flags.power_manageable) { + result = device_create_file(&dev->dev, &dev_attr_power_state); + if (result) + return result; + + if (dev->power.flags.power_resources) + result = device_create_file(&dev->dev, + &dev_attr_real_power_state); + } + +end: + return result; +} + +/** + * acpi_device_remove_files - Remove sysfs attributes of an ACPI device. + * @dev: ACPI device object. + */ +void acpi_device_remove_files(struct acpi_device *dev) +{ + if (dev->flags.power_manageable) { + device_remove_file(&dev->dev, &dev_attr_power_state); + if (dev->power.flags.power_resources) + device_remove_file(&dev->dev, + &dev_attr_real_power_state); + } + + /* + * If device has _STR, remove 'description' file + */ + if (acpi_has_method(dev->handle, "_STR")) { + kfree(dev->pnp.str_obj); + device_remove_file(&dev->dev, &dev_attr_description); + } + /* + * If device has _EJ0, remove 'eject' file. + */ + if (acpi_has_method(dev->handle, "_EJ0")) + device_remove_file(&dev->dev, &dev_attr_eject); + + if (acpi_has_method(dev->handle, "_SUN")) + device_remove_file(&dev->dev, &dev_attr_sun); + + if (dev->pnp.unique_id) + device_remove_file(&dev->dev, &dev_attr_uid); + if (dev->pnp.type.bus_address) + device_remove_file(&dev->dev, &dev_attr_adr); + device_remove_file(&dev->dev, &dev_attr_modalias); + device_remove_file(&dev->dev, &dev_attr_hid); + if (acpi_has_method(dev->handle, "_STA")) + device_remove_file(&dev->dev, &dev_attr_status); + if (dev->handle) + device_remove_file(&dev->dev, &dev_attr_path); +} diff --git a/drivers/acpi/dock.c b/drivers/acpi/dock.c index a688aa243f6c..e8e128dede29 100644 --- a/drivers/acpi/dock.c +++ b/drivers/acpi/dock.c @@ -17,10 +17,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c index 9d4761d2f6b7..2614a839c60d 100644 --- a/drivers/acpi/ec.c +++ b/drivers/acpi/ec.c @@ -22,10 +22,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ @@ -165,8 +161,16 @@ struct transaction { u8 flags; }; +struct acpi_ec_query { + struct transaction transaction; + struct work_struct work; + struct acpi_ec_query_handler *handler; +}; + static int acpi_ec_query(struct acpi_ec *ec, u8 *data); static void advance_transaction(struct acpi_ec *ec); +static void acpi_ec_event_handler(struct work_struct *work); +static void acpi_ec_event_processor(struct work_struct *work); struct acpi_ec *boot_ec, *first_ec; EXPORT_SYMBOL(first_ec); @@ -978,60 +982,90 @@ void acpi_ec_remove_query_handler(struct acpi_ec *ec, u8 query_bit) } EXPORT_SYMBOL_GPL(acpi_ec_remove_query_handler); -static void acpi_ec_run(void *cxt) +static struct acpi_ec_query *acpi_ec_create_query(u8 *pval) { - struct acpi_ec_query_handler *handler = cxt; + struct acpi_ec_query *q; + struct transaction *t; + + q = kzalloc(sizeof (struct acpi_ec_query), GFP_KERNEL); + if (!q) + return NULL; + INIT_WORK(&q->work, acpi_ec_event_processor); + t = &q->transaction; + t->command = ACPI_EC_COMMAND_QUERY; + t->rdata = pval; + t->rlen = 1; + return q; +} + +static void acpi_ec_delete_query(struct acpi_ec_query *q) +{ + if (q) { + if (q->handler) + acpi_ec_put_query_handler(q->handler); + kfree(q); + } +} + +static void acpi_ec_event_processor(struct work_struct *work) +{ + struct acpi_ec_query *q = container_of(work, struct acpi_ec_query, work); + struct acpi_ec_query_handler *handler = q->handler; - if (!handler) - return; ec_dbg_evt("Query(0x%02x) started", handler->query_bit); if (handler->func) handler->func(handler->data); else if (handler->handle) acpi_evaluate_object(handler->handle, NULL, NULL, NULL); ec_dbg_evt("Query(0x%02x) stopped", handler->query_bit); - acpi_ec_put_query_handler(handler); + acpi_ec_delete_query(q); } static int acpi_ec_query(struct acpi_ec *ec, u8 *data) { u8 value = 0; int result; - acpi_status status; struct acpi_ec_query_handler *handler; - struct transaction t = {.command = ACPI_EC_COMMAND_QUERY, - .wdata = NULL, .rdata = &value, - .wlen = 0, .rlen = 1}; + struct acpi_ec_query *q; + + q = acpi_ec_create_query(&value); + if (!q) + return -ENOMEM; /* * Query the EC to find out which _Qxx method we need to evaluate. * Note that successful completion of the query causes the ACPI_EC_SCI * bit to be cleared (and thus clearing the interrupt source). */ - result = acpi_ec_transaction(ec, &t); - if (result) - return result; - if (data) - *data = value; + result = acpi_ec_transaction(ec, &q->transaction); if (!value) - return -ENODATA; + result = -ENODATA; + if (result) + goto err_exit; mutex_lock(&ec->mutex); list_for_each_entry(handler, &ec->list, node) { if (value == handler->query_bit) { - /* have custom handler for this bit */ - handler = acpi_ec_get_query_handler(handler); + q->handler = acpi_ec_get_query_handler(handler); ec_dbg_evt("Query(0x%02x) scheduled", - handler->query_bit); - status = acpi_os_execute((handler->func) ? - OSL_NOTIFY_HANDLER : OSL_GPE_HANDLER, - acpi_ec_run, handler); - if (ACPI_FAILURE(status)) + q->handler->query_bit); + /* + * It is reported that _Qxx are evaluated in a + * parallel way on Windows: + * https://bugzilla.kernel.org/show_bug.cgi?id=94411 + */ + if (!schedule_work(&q->work)) result = -EBUSY; break; } } mutex_unlock(&ec->mutex); + +err_exit: + if (result && q) + acpi_ec_delete_query(q); + if (data) + *data = value; return result; } diff --git a/drivers/acpi/fan.c b/drivers/acpi/fan.c index bea0bbaafa97..e297a480e135 100644 --- a/drivers/acpi/fan.c +++ b/drivers/acpi/fan.c @@ -16,10 +16,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ diff --git a/drivers/acpi/hed.c b/drivers/acpi/hed.c index a322710b5ba4..5c67a6d8f803 100644 --- a/drivers/acpi/hed.c +++ b/drivers/acpi/hed.c @@ -15,10 +15,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include diff --git a/drivers/acpi/internal.h b/drivers/acpi/internal.h index 4683a96932b9..9e426210c2a8 100644 --- a/drivers/acpi/internal.h +++ b/drivers/acpi/internal.h @@ -13,9 +13,6 @@ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. */ #ifndef _ACPI_INTERNAL_H_ @@ -70,7 +67,7 @@ void acpi_scan_hotplug_enabled(struct acpi_hotplug_profile *hotplug, bool val); #ifdef CONFIG_DEBUG_FS extern struct dentry *acpi_debugfs_dir; -int acpi_debugfs_init(void); +void acpi_debugfs_init(void); #else static inline void acpi_debugfs_init(void) { return; } #endif @@ -93,10 +90,21 @@ int acpi_device_add(struct acpi_device *device, void (*release)(struct device *)); void acpi_init_device_object(struct acpi_device *device, acpi_handle handle, int type, unsigned long long sta); +int acpi_device_setup_files(struct acpi_device *dev); +void acpi_device_remove_files(struct acpi_device *dev); void acpi_device_add_finalize(struct acpi_device *device); void acpi_free_pnp_ids(struct acpi_device_pnp *pnp); bool acpi_device_is_present(struct acpi_device *adev); bool acpi_device_is_battery(struct acpi_device *adev); +bool acpi_device_is_first_physical_node(struct acpi_device *adev, + const struct device *dev); + +/* -------------------------------------------------------------------------- + Device Matching and Notification + -------------------------------------------------------------------------- */ +struct acpi_device *acpi_companion_match(const struct device *dev); +int __acpi_device_uevent_modalias(struct acpi_device *adev, + struct kobj_uevent_env *env); /* -------------------------------------------------------------------------- Power Resource diff --git a/drivers/acpi/nfit.c b/drivers/acpi/nfit.c index cf0fd96a7602..c1b8d03e262e 100644 --- a/drivers/acpi/nfit.c +++ b/drivers/acpi/nfit.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "nfit.h" /* @@ -764,9 +765,7 @@ static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc, struct acpi_device *adev, *adev_dimm; struct device *dev = acpi_desc->dev; const u8 *uuid = to_nfit_uuid(NFIT_DEV_DIMM); - unsigned long long sta; - int i, rc = -ENODEV; - acpi_status status; + int i; nfit_mem->dsm_mask = acpi_desc->dimm_dsm_force_en; adev = to_acpi_dev(acpi_desc); @@ -781,25 +780,11 @@ static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc, return force_enable_dimms ? 0 : -ENODEV; } - status = acpi_evaluate_integer(adev_dimm->handle, "_STA", NULL, &sta); - if (status == AE_NOT_FOUND) { - dev_dbg(dev, "%s missing _STA, assuming enabled...\n", - dev_name(&adev_dimm->dev)); - rc = 0; - } else if (ACPI_FAILURE(status)) - dev_err(dev, "%s failed to retrieve_STA, disabling...\n", - dev_name(&adev_dimm->dev)); - else if ((sta & ACPI_STA_DEVICE_ENABLED) == 0) - dev_info(dev, "%s disabled by firmware\n", - dev_name(&adev_dimm->dev)); - else - rc = 0; - for (i = ND_CMD_SMART; i <= ND_CMD_VENDOR; i++) if (acpi_check_dsm(adev_dimm->handle, uuid, 1, 1ULL << i)) set_bit(i, &nfit_mem->dsm_mask); - return force_enable_dimms ? 0 : rc; + return 0; } static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc) @@ -868,6 +853,7 @@ static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc) struct acpi_device *adev; int i; + nd_desc->dsm_mask = acpi_desc->bus_dsm_force_en; adev = to_acpi_dev(acpi_desc); if (!adev) return; @@ -1032,7 +1018,7 @@ static u32 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw) if (mmio->num_lines) offset = to_interleave_offset(offset, mmio); - return readl(mmio->base + offset); + return readl(mmio->addr.base + offset); } static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw, @@ -1057,11 +1043,11 @@ static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw, if (mmio->num_lines) offset = to_interleave_offset(offset, mmio); - writeq(cmd, mmio->base + offset); + writeq(cmd, mmio->addr.base + offset); wmb_blk(nfit_blk); if (nfit_blk->dimm_flags & ND_BLK_DCR_LATCH) - readq(mmio->base + offset); + readq(mmio->addr.base + offset); } static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk, @@ -1093,11 +1079,16 @@ static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk, } if (rw) - memcpy_to_pmem(mmio->aperture + offset, + memcpy_to_pmem(mmio->addr.aperture + offset, iobuf + copied, c); - else + else { + if (nfit_blk->dimm_flags & ND_BLK_READ_FLUSH) + mmio_flush_range((void __force *) + mmio->addr.aperture + offset, c); + memcpy_from_pmem(iobuf + copied, - mmio->aperture + offset, c); + mmio->addr.aperture + offset, c); + } copied += c; len -= c; @@ -1144,7 +1135,10 @@ static void nfit_spa_mapping_release(struct kref *kref) WARN_ON(!mutex_is_locked(&acpi_desc->spa_map_mutex)); dev_dbg(acpi_desc->dev, "%s: SPA%d\n", __func__, spa->range_index); - iounmap(spa_map->iomem); + if (spa_map->type == SPA_MAP_APERTURE) + memunmap((void __force *)spa_map->addr.aperture); + else + iounmap(spa_map->addr.base); release_mem_region(spa->address, spa->length); list_del(&spa_map->list); kfree(spa_map); @@ -1190,7 +1184,7 @@ static void __iomem *__nfit_spa_map(struct acpi_nfit_desc *acpi_desc, spa_map = find_spa_mapping(acpi_desc, spa); if (spa_map) { kref_get(&spa_map->kref); - return spa_map->iomem; + return spa_map->addr.base; } spa_map = kzalloc(sizeof(*spa_map), GFP_KERNEL); @@ -1206,20 +1200,19 @@ static void __iomem *__nfit_spa_map(struct acpi_nfit_desc *acpi_desc, if (!res) goto err_mem; - if (type == SPA_MAP_APERTURE) { - /* - * TODO: memremap_pmem() support, but that requires cache - * flushing when the aperture is moved. - */ - spa_map->iomem = ioremap_wc(start, n); - } else - spa_map->iomem = ioremap_nocache(start, n); + spa_map->type = type; + if (type == SPA_MAP_APERTURE) + spa_map->addr.aperture = (void __pmem *)memremap(start, n, + ARCH_MEMREMAP_PMEM); + else + spa_map->addr.base = ioremap_nocache(start, n); - if (!spa_map->iomem) + + if (!spa_map->addr.base) goto err_map; list_add_tail(&spa_map->list, &acpi_desc->spa_maps); - return spa_map->iomem; + return spa_map->addr.base; err_map: release_mem_region(start, n); @@ -1282,7 +1275,7 @@ static int acpi_nfit_blk_get_flags(struct nvdimm_bus_descriptor *nd_desc, nfit_blk->dimm_flags = flags.flags; else if (rc == -ENOTTY) { /* fall back to a conservative default */ - nfit_blk->dimm_flags = ND_BLK_DCR_LATCH; + nfit_blk->dimm_flags = ND_BLK_DCR_LATCH | ND_BLK_READ_FLUSH; rc = 0; } else rc = -ENXIO; @@ -1322,9 +1315,9 @@ static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus, /* map block aperture memory */ nfit_blk->bdw_offset = nfit_mem->bdw->offset; mmio = &nfit_blk->mmio[BDW]; - mmio->base = nfit_spa_map(acpi_desc, nfit_mem->spa_bdw, + mmio->addr.base = nfit_spa_map(acpi_desc, nfit_mem->spa_bdw, SPA_MAP_APERTURE); - if (!mmio->base) { + if (!mmio->addr.base) { dev_dbg(dev, "%s: %s failed to map bdw\n", __func__, nvdimm_name(nvdimm)); return -ENOMEM; @@ -1345,9 +1338,9 @@ static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus, nfit_blk->cmd_offset = nfit_mem->dcr->command_offset; nfit_blk->stat_offset = nfit_mem->dcr->status_offset; mmio = &nfit_blk->mmio[DCR]; - mmio->base = nfit_spa_map(acpi_desc, nfit_mem->spa_dcr, + mmio->addr.base = nfit_spa_map(acpi_desc, nfit_mem->spa_dcr, SPA_MAP_CONTROL); - if (!mmio->base) { + if (!mmio->addr.base) { dev_dbg(dev, "%s: %s failed to map dcr\n", __func__, nvdimm_name(nvdimm)); return -ENOMEM; @@ -1379,7 +1372,7 @@ static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus, return -ENOMEM; } - if (!arch_has_pmem_api() && !nfit_blk->nvdimm_flush) + if (!arch_has_wmb_pmem() && !nfit_blk->nvdimm_flush) dev_warn(dev, "unable to guarantee persistence of writes\n"); if (mmio->line_size == 0) @@ -1414,7 +1407,7 @@ static void acpi_nfit_blk_region_disable(struct nvdimm_bus *nvdimm_bus, for (i = 0; i < 2; i++) { struct nfit_blk_mmio *mmio = &nfit_blk->mmio[i]; - if (mmio->base) + if (mmio->addr.base) nfit_spa_unmap(acpi_desc, mmio->spa); } nd_blk_region_set_provider_data(ndbr, NULL); diff --git a/drivers/acpi/nfit.h b/drivers/acpi/nfit.h index 79b6d83875c1..7e740156b9c2 100644 --- a/drivers/acpi/nfit.h +++ b/drivers/acpi/nfit.h @@ -41,6 +41,7 @@ enum nfit_uuids { }; enum { + ND_BLK_READ_FLUSH = 1, ND_BLK_DCR_LATCH = 2, }; @@ -107,6 +108,7 @@ struct acpi_nfit_desc { struct nvdimm_bus *nvdimm_bus; struct device *dev; unsigned long dimm_dsm_force_en; + unsigned long bus_dsm_force_en; int (*blk_do_io)(struct nd_blk_region *ndbr, resource_size_t dpa, void *iobuf, u64 len, int rw); }; @@ -116,12 +118,16 @@ enum nd_blk_mmio_selector { DCR, }; +struct nd_blk_addr { + union { + void __iomem *base; + void __pmem *aperture; + }; +}; + struct nfit_blk { struct nfit_blk_mmio { - union { - void __iomem *base; - void __pmem *aperture; - }; + struct nd_blk_addr addr; u64 size; u64 base_offset; u32 line_size; @@ -148,7 +154,8 @@ struct nfit_spa_mapping { struct acpi_nfit_system_address *spa; struct list_head list; struct kref kref; - void __iomem *iomem; + enum spa_map_type type; + struct nd_blk_addr addr; }; static inline struct nfit_spa_mapping *to_spa_map(struct kref *kref) diff --git a/drivers/acpi/numa.c b/drivers/acpi/numa.c index acaa3b4ea504..72b6e9ef0ae9 100644 --- a/drivers/acpi/numa.c +++ b/drivers/acpi/numa.c @@ -15,10 +15,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * */ diff --git a/drivers/acpi/osl.c b/drivers/acpi/osl.c index 3b8963f21b36..739a4a6b3b9b 100644 --- a/drivers/acpi/osl.c +++ b/drivers/acpi/osl.c @@ -19,10 +19,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * */ @@ -47,6 +43,7 @@ #include #include +#include #include "internal.h" @@ -83,6 +80,7 @@ static void *acpi_irq_context; static struct workqueue_struct *kacpid_wq; static struct workqueue_struct *kacpi_notify_wq; static struct workqueue_struct *kacpi_hotplug_wq; +static bool acpi_os_initialized; /* * This list of permanent mappings is for memory that may be accessed from @@ -947,21 +945,6 @@ acpi_status acpi_os_write_port(acpi_io_address port, u32 value, u32 width) EXPORT_SYMBOL(acpi_os_write_port); -#ifdef readq -static inline u64 read64(const volatile void __iomem *addr) -{ - return readq(addr); -} -#else -static inline u64 read64(const volatile void __iomem *addr) -{ - u64 l, h; - l = readl(addr); - h = readl(addr+4); - return l | (h << 32); -} -#endif - acpi_status acpi_os_read_memory(acpi_physical_address phys_addr, u64 *value, u32 width) { @@ -994,7 +977,7 @@ acpi_os_read_memory(acpi_physical_address phys_addr, u64 *value, u32 width) *(u32 *) value = readl(virt_addr); break; case 64: - *(u64 *) value = read64(virt_addr); + *(u64 *) value = readq(virt_addr); break; default: BUG(); @@ -1008,19 +991,6 @@ acpi_os_read_memory(acpi_physical_address phys_addr, u64 *value, u32 width) return AE_OK; } -#ifdef writeq -static inline void write64(u64 val, volatile void __iomem *addr) -{ - writeq(val, addr); -} -#else -static inline void write64(u64 val, volatile void __iomem *addr) -{ - writel(val, addr); - writel(val>>32, addr+4); -} -#endif - acpi_status acpi_os_write_memory(acpi_physical_address phys_addr, u64 value, u32 width) { @@ -1049,7 +1019,7 @@ acpi_os_write_memory(acpi_physical_address phys_addr, u64 value, u32 width) writel(value, virt_addr); break; case 64: - write64(value, virt_addr); + writeq(value, virt_addr); break; default: BUG(); @@ -1316,6 +1286,9 @@ acpi_status acpi_os_wait_semaphore(acpi_handle handle, u32 units, u16 timeout) long jiffies; int ret = 0; + if (!acpi_os_initialized) + return AE_OK; + if (!sem || (units < 1)) return AE_BAD_PARAMETER; @@ -1355,6 +1328,9 @@ acpi_status acpi_os_signal_semaphore(acpi_handle handle, u32 units) { struct semaphore *sem = (struct semaphore *)handle; + if (!acpi_os_initialized) + return AE_OK; + if (!sem || (units < 1)) return AE_BAD_PARAMETER; @@ -1863,6 +1839,7 @@ acpi_status __init acpi_os_initialize(void) rv = acpi_os_map_generic_address(&acpi_gbl_FADT.reset_register); pr_debug(PREFIX "%s: map reset_reg status %d\n", __func__, rv); } + acpi_os_initialized = true; return AE_OK; } diff --git a/drivers/acpi/pci_irq.c b/drivers/acpi/pci_irq.c index afa16c557c17..6da0f9beab19 100644 --- a/drivers/acpi/pci_irq.c +++ b/drivers/acpi/pci_irq.c @@ -19,10 +19,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ diff --git a/drivers/acpi/pci_link.c b/drivers/acpi/pci_link.c index cfd7581cc19f..3b4ea98e3ea0 100644 --- a/drivers/acpi/pci_link.c +++ b/drivers/acpi/pci_link.c @@ -17,10 +17,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * * TBD: @@ -825,6 +821,22 @@ void acpi_penalize_isa_irq(int irq, int active) } } +/* + * Penalize IRQ used by ACPI SCI. If ACPI SCI pin attributes conflict with + * PCI IRQ attributes, mark ACPI SCI as ISA_ALWAYS so it won't be use for + * PCI IRQs. + */ +void acpi_penalize_sci_irq(int irq, int trigger, int polarity) +{ + if (irq >= 0 && irq < ARRAY_SIZE(acpi_irq_penalty)) { + if (trigger != ACPI_MADT_TRIGGER_LEVEL || + polarity != ACPI_MADT_POLARITY_ACTIVE_LOW) + acpi_irq_penalty[irq] += PIRQ_PENALTY_ISA_ALWAYS; + else + acpi_irq_penalty[irq] += PIRQ_PENALTY_PCI_USING; + } +} + /* * Over-ride default table to reserve additional IRQs for use by ISA * e.g. acpi_irq_isa=5 diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c index 1b5569c092c6..393706a5261b 100644 --- a/drivers/acpi/pci_root.c +++ b/drivers/acpi/pci_root.c @@ -16,10 +16,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ diff --git a/drivers/acpi/pci_slot.c b/drivers/acpi/pci_slot.c index 139d9e479370..7188e53b6b7c 100644 --- a/drivers/acpi/pci_slot.c +++ b/drivers/acpi/pci_slot.c @@ -20,10 +20,6 @@ * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. */ #include diff --git a/drivers/acpi/power.c b/drivers/acpi/power.c index 93eac53b5110..fcd4ce6f78d5 100644 --- a/drivers/acpi/power.c +++ b/drivers/acpi/power.c @@ -1,8 +1,10 @@ /* - * acpi_power.c - ACPI Bus Power Management ($Revision: 39 $) + * drivers/acpi/power.c - ACPI Power Resources management. * - * Copyright (C) 2001, 2002 Andy Grover - * Copyright (C) 2001, 2002 Paul Diefenbaugh + * Copyright (C) 2001 - 2015 Intel Corp. + * Author: Andy Grover + * Author: Paul Diefenbaugh + * Author: Rafael J. Wysocki * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * @@ -16,10 +18,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ @@ -27,10 +25,11 @@ * ACPI power-managed devices may be controlled in two ways: * 1. via "Device Specific (D-State) Control" * 2. via "Power Resource Control". - * This module is used to manage devices relying on Power Resource Control. + * The code below deals with ACPI Power Resources control. * - * An ACPI "power resource object" describes a software controllable power - * plane, clock plane, or other resource used by a power managed device. + * An ACPI "power resource object" represents a software controllable power + * plane, clock plane, or other resource depended on by a device. + * * A device may rely on multiple power resources, and a power resource * may be shared by multiple devices. */ diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c index d9f71581b79b..51e658f21e95 100644 --- a/drivers/acpi/processor_driver.c +++ b/drivers/acpi/processor_driver.c @@ -21,10 +21,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ @@ -159,38 +155,28 @@ static int acpi_cpu_soft_notify(struct notifier_block *nfb, return NOTIFY_OK; } -static struct notifier_block __refdata acpi_cpu_notifier = { +static struct notifier_block acpi_cpu_notifier = { .notifier_call = acpi_cpu_soft_notify, }; -static int __acpi_processor_start(struct acpi_device *device) +#ifdef CONFIG_ACPI_CPU_FREQ_PSS +static int acpi_pss_perf_init(struct acpi_processor *pr, + struct acpi_device *device) { - struct acpi_processor *pr = acpi_driver_data(device); - acpi_status status; int result = 0; - if (!pr) - return -ENODEV; - - if (pr->flags.need_hotplug_init) - return 0; - -#ifdef CONFIG_CPU_FREQ acpi_processor_ppc_has_changed(pr, 0); -#endif + acpi_processor_get_throttling_info(pr); if (pr->flags.throttling) pr->flags.limit = 1; - if (!cpuidle_get_driver() || cpuidle_get_driver() == &acpi_idle_driver) - acpi_processor_power_init(pr); - pr->cdev = thermal_cooling_device_register("Processor", device, &processor_cooling_ops); if (IS_ERR(pr->cdev)) { result = PTR_ERR(pr->cdev); - goto err_power_exit; + return result; } dev_dbg(&device->dev, "registered as cooling_device%d\n", @@ -204,6 +190,7 @@ static int __acpi_processor_start(struct acpi_device *device) "Failed to create sysfs link 'thermal_cooling'\n"); goto err_thermal_unregister; } + result = sysfs_create_link(&pr->cdev->device.kobj, &device->dev.kobj, "device"); @@ -213,17 +200,61 @@ static int __acpi_processor_start(struct acpi_device *device) goto err_remove_sysfs_thermal; } - status = acpi_install_notify_handler(device->handle, ACPI_DEVICE_NOTIFY, - acpi_processor_notify, device); - if (ACPI_SUCCESS(status)) - return 0; - sysfs_remove_link(&pr->cdev->device.kobj, "device"); err_remove_sysfs_thermal: sysfs_remove_link(&device->dev.kobj, "thermal_cooling"); err_thermal_unregister: thermal_cooling_device_unregister(pr->cdev); - err_power_exit: + + return result; +} + +static void acpi_pss_perf_exit(struct acpi_processor *pr, + struct acpi_device *device) +{ + if (pr->cdev) { + sysfs_remove_link(&device->dev.kobj, "thermal_cooling"); + sysfs_remove_link(&pr->cdev->device.kobj, "device"); + thermal_cooling_device_unregister(pr->cdev); + pr->cdev = NULL; + } +} +#else +static inline int acpi_pss_perf_init(struct acpi_processor *pr, + struct acpi_device *device) +{ + return 0; +} + +static inline void acpi_pss_perf_exit(struct acpi_processor *pr, + struct acpi_device *device) {} +#endif /* CONFIG_ACPI_CPU_FREQ_PSS */ + +static int __acpi_processor_start(struct acpi_device *device) +{ + struct acpi_processor *pr = acpi_driver_data(device); + acpi_status status; + int result = 0; + + if (!pr) + return -ENODEV; + + if (pr->flags.need_hotplug_init) + return 0; + + if (!cpuidle_get_driver() || cpuidle_get_driver() == &acpi_idle_driver) + acpi_processor_power_init(pr); + + result = acpi_pss_perf_init(pr, device); + if (result) + goto err_power_exit; + + status = acpi_install_notify_handler(device->handle, ACPI_DEVICE_NOTIFY, + acpi_processor_notify, device); + if (ACPI_SUCCESS(status)) + return 0; + +err_power_exit: acpi_processor_power_exit(pr); return result; } @@ -252,15 +283,10 @@ static int acpi_processor_stop(struct device *dev) pr = acpi_driver_data(device); if (!pr) return 0; - acpi_processor_power_exit(pr); - if (pr->cdev) { - sysfs_remove_link(&device->dev.kobj, "thermal_cooling"); - sysfs_remove_link(&pr->cdev->device.kobj, "device"); - thermal_cooling_device_unregister(pr->cdev); - pr->cdev = NULL; - } + acpi_pss_perf_exit(pr, device); + return 0; } diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index d540f42c9232..175c86bee3a9 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -21,10 +21,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ diff --git a/drivers/acpi/processor_perflib.c b/drivers/acpi/processor_perflib.c index cfc8aba72f86..bb01dea39fdc 100644 --- a/drivers/acpi/processor_perflib.c +++ b/drivers/acpi/processor_perflib.c @@ -20,10 +20,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * */ #include @@ -87,7 +83,7 @@ static int acpi_processor_ppc_notifier(struct notifier_block *nb, if (ignore_ppc) return 0; - if (event != CPUFREQ_INCOMPATIBLE) + if (event != CPUFREQ_ADJUST) return 0; mutex_lock(&performance_mutex); @@ -784,9 +780,7 @@ acpi_processor_register_performance(struct acpi_processor_performance EXPORT_SYMBOL(acpi_processor_register_performance); -void -acpi_processor_unregister_performance(struct acpi_processor_performance - *performance, unsigned int cpu) +void acpi_processor_unregister_performance(unsigned int cpu) { struct acpi_processor *pr; diff --git a/drivers/acpi/processor_thermal.c b/drivers/acpi/processor_thermal.c index e003663b2f8e..1fed84a092c2 100644 --- a/drivers/acpi/processor_thermal.c +++ b/drivers/acpi/processor_thermal.c @@ -19,10 +19,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ diff --git a/drivers/acpi/processor_throttling.c b/drivers/acpi/processor_throttling.c index 84243c32e29c..f170d746336d 100644 --- a/drivers/acpi/processor_throttling.c +++ b/drivers/acpi/processor_throttling.c @@ -19,10 +19,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ diff --git a/drivers/acpi/property.c b/drivers/acpi/property.c index 7836e2e980f4..6d99450549c5 100644 --- a/drivers/acpi/property.c +++ b/drivers/acpi/property.c @@ -528,13 +528,14 @@ int acpi_dev_prop_read(struct acpi_device *adev, const char *propname, if (!val) return obj->package.count; - else if (nval <= 0) - return -EINVAL; if (nval > obj->package.count) return -EOVERFLOW; + else if (nval <= 0) + return -EINVAL; items = obj->package.elements; + switch (proptype) { case DEV_PROP_U8: ret = acpi_copy_property_array_u8(items, (u8 *)val, nval); diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c index f1c966e05078..15d22db05054 100644 --- a/drivers/acpi/resource.c +++ b/drivers/acpi/resource.c @@ -15,10 +15,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ diff --git a/drivers/acpi/sbs.c b/drivers/acpi/sbs.c index 01504c819e8f..cb3dedb1beae 100644 --- a/drivers/acpi/sbs.c +++ b/drivers/acpi/sbs.c @@ -17,10 +17,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index ec256352f423..01136b879038 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -115,264 +115,6 @@ int acpi_scan_add_handler_with_hotplug(struct acpi_scan_handler *handler, return 0; } -/** - * create_pnp_modalias - Create hid/cid(s) string for modalias and uevent - * @acpi_dev: ACPI device object. - * @modalias: Buffer to print into. - * @size: Size of the buffer. - * - * Creates hid/cid(s) string needed for modalias and uevent - * e.g. on a device with hid:IBM0001 and cid:ACPI0001 you get: - * char *modalias: "acpi:IBM0001:ACPI0001" - * Return: 0: no _HID and no _CID - * -EINVAL: output error - * -ENOMEM: output is truncated -*/ -static int create_pnp_modalias(struct acpi_device *acpi_dev, char *modalias, - int size) -{ - int len; - int count; - struct acpi_hardware_id *id; - - /* - * Since we skip ACPI_DT_NAMESPACE_HID from the modalias below, 0 should - * be returned if ACPI_DT_NAMESPACE_HID is the only ACPI/PNP ID in the - * device's list. - */ - count = 0; - list_for_each_entry(id, &acpi_dev->pnp.ids, list) - if (strcmp(id->id, ACPI_DT_NAMESPACE_HID)) - count++; - - if (!count) - return 0; - - len = snprintf(modalias, size, "acpi:"); - if (len <= 0) - return len; - - size -= len; - - list_for_each_entry(id, &acpi_dev->pnp.ids, list) { - if (!strcmp(id->id, ACPI_DT_NAMESPACE_HID)) - continue; - - count = snprintf(&modalias[len], size, "%s:", id->id); - if (count < 0) - return -EINVAL; - - if (count >= size) - return -ENOMEM; - - len += count; - size -= count; - } - modalias[len] = '\0'; - return len; -} - -/** - * create_of_modalias - Creates DT compatible string for modalias and uevent - * @acpi_dev: ACPI device object. - * @modalias: Buffer to print into. - * @size: Size of the buffer. - * - * Expose DT compatible modalias as of:NnameTCcompatible. This function should - * only be called for devices having ACPI_DT_NAMESPACE_HID in their list of - * ACPI/PNP IDs. - */ -static int create_of_modalias(struct acpi_device *acpi_dev, char *modalias, - int size) -{ - struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER }; - const union acpi_object *of_compatible, *obj; - int len, count; - int i, nval; - char *c; - - acpi_get_name(acpi_dev->handle, ACPI_SINGLE_NAME, &buf); - /* DT strings are all in lower case */ - for (c = buf.pointer; *c != '\0'; c++) - *c = tolower(*c); - - len = snprintf(modalias, size, "of:N%sT", (char *)buf.pointer); - ACPI_FREE(buf.pointer); - - if (len <= 0) - return len; - - of_compatible = acpi_dev->data.of_compatible; - if (of_compatible->type == ACPI_TYPE_PACKAGE) { - nval = of_compatible->package.count; - obj = of_compatible->package.elements; - } else { /* Must be ACPI_TYPE_STRING. */ - nval = 1; - obj = of_compatible; - } - for (i = 0; i < nval; i++, obj++) { - count = snprintf(&modalias[len], size, "C%s", - obj->string.pointer); - if (count < 0) - return -EINVAL; - - if (count >= size) - return -ENOMEM; - - len += count; - size -= count; - } - modalias[len] = '\0'; - return len; -} - -/* - * acpi_companion_match() - Can we match via ACPI companion device - * @dev: Device in question - * - * Check if the given device has an ACPI companion and if that companion has - * a valid list of PNP IDs, and if the device is the first (primary) physical - * device associated with it. Return the companion pointer if that's the case - * or NULL otherwise. - * - * If multiple physical devices are attached to a single ACPI companion, we need - * to be careful. The usage scenario for this kind of relationship is that all - * of the physical devices in question use resources provided by the ACPI - * companion. A typical case is an MFD device where all the sub-devices share - * the parent's ACPI companion. In such cases we can only allow the primary - * (first) physical device to be matched with the help of the companion's PNP - * IDs. - * - * Additional physical devices sharing the ACPI companion can still use - * resources available from it but they will be matched normally using functions - * provided by their bus types (and analogously for their modalias). - */ -static struct acpi_device *acpi_companion_match(const struct device *dev) -{ - struct acpi_device *adev; - struct mutex *physical_node_lock; - - adev = ACPI_COMPANION(dev); - if (!adev) - return NULL; - - if (list_empty(&adev->pnp.ids)) - return NULL; - - physical_node_lock = &adev->physical_node_lock; - mutex_lock(physical_node_lock); - if (list_empty(&adev->physical_node_list)) { - adev = NULL; - } else { - const struct acpi_device_physical_node *node; - - node = list_first_entry(&adev->physical_node_list, - struct acpi_device_physical_node, node); - if (node->dev != dev) - adev = NULL; - } - mutex_unlock(physical_node_lock); - - return adev; -} - -static int __acpi_device_uevent_modalias(struct acpi_device *adev, - struct kobj_uevent_env *env) -{ - int len; - - if (!adev) - return -ENODEV; - - if (list_empty(&adev->pnp.ids)) - return 0; - - if (add_uevent_var(env, "MODALIAS=")) - return -ENOMEM; - - len = create_pnp_modalias(adev, &env->buf[env->buflen - 1], - sizeof(env->buf) - env->buflen); - if (len < 0) - return len; - - env->buflen += len; - if (!adev->data.of_compatible) - return 0; - - if (len > 0 && add_uevent_var(env, "MODALIAS=")) - return -ENOMEM; - - len = create_of_modalias(adev, &env->buf[env->buflen - 1], - sizeof(env->buf) - env->buflen); - if (len < 0) - return len; - - env->buflen += len; - - return 0; -} - -/* - * Creates uevent modalias field for ACPI enumerated devices. - * Because the other buses does not support ACPI HIDs & CIDs. - * e.g. for a device with hid:IBM0001 and cid:ACPI0001 you get: - * "acpi:IBM0001:ACPI0001" - */ -int acpi_device_uevent_modalias(struct device *dev, struct kobj_uevent_env *env) -{ - return __acpi_device_uevent_modalias(acpi_companion_match(dev), env); -} -EXPORT_SYMBOL_GPL(acpi_device_uevent_modalias); - -static int __acpi_device_modalias(struct acpi_device *adev, char *buf, int size) -{ - int len, count; - - if (!adev) - return -ENODEV; - - if (list_empty(&adev->pnp.ids)) - return 0; - - len = create_pnp_modalias(adev, buf, size - 1); - if (len < 0) { - return len; - } else if (len > 0) { - buf[len++] = '\n'; - size -= len; - } - if (!adev->data.of_compatible) - return len; - - count = create_of_modalias(adev, buf + len, size - 1); - if (count < 0) { - return count; - } else if (count > 0) { - len += count; - buf[len++] = '\n'; - } - - return len; -} - -/* - * Creates modalias sysfs attribute for ACPI enumerated devices. - * Because the other buses does not support ACPI HIDs & CIDs. - * e.g. for a device with hid:IBM0001 and cid:ACPI0001 you get: - * "acpi:IBM0001:ACPI0001" - */ -int acpi_device_modalias(struct device *dev, char *buf, int size) -{ - return __acpi_device_modalias(acpi_companion_match(dev), buf, size); -} -EXPORT_SYMBOL_GPL(acpi_device_modalias); - -static ssize_t -acpi_device_modalias_show(struct device *dev, struct device_attribute *attr, char *buf) { - return __acpi_device_modalias(to_acpi_device(dev), buf, 1024); -} -static DEVICE_ATTR(modalias, 0444, acpi_device_modalias_show, NULL); - bool acpi_scan_is_offline(struct acpi_device *adev, bool uevent) { struct acpi_device_physical_node *pn; @@ -701,423 +443,6 @@ void acpi_device_hotplug(struct acpi_device *adev, u32 src) unlock_device_hotplug(); } -static ssize_t real_power_state_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct acpi_device *adev = to_acpi_device(dev); - int state; - int ret; - - ret = acpi_device_get_power(adev, &state); - if (ret) - return ret; - - return sprintf(buf, "%s\n", acpi_power_state_string(state)); -} - -static DEVICE_ATTR(real_power_state, 0444, real_power_state_show, NULL); - -static ssize_t power_state_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct acpi_device *adev = to_acpi_device(dev); - - return sprintf(buf, "%s\n", acpi_power_state_string(adev->power.state)); -} - -static DEVICE_ATTR(power_state, 0444, power_state_show, NULL); - -static ssize_t -acpi_eject_store(struct device *d, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct acpi_device *acpi_device = to_acpi_device(d); - acpi_object_type not_used; - acpi_status status; - - if (!count || buf[0] != '1') - return -EINVAL; - - if ((!acpi_device->handler || !acpi_device->handler->hotplug.enabled) - && !acpi_device->driver) - return -ENODEV; - - status = acpi_get_type(acpi_device->handle, ¬_used); - if (ACPI_FAILURE(status) || !acpi_device->flags.ejectable) - return -ENODEV; - - get_device(&acpi_device->dev); - status = acpi_hotplug_schedule(acpi_device, ACPI_OST_EC_OSPM_EJECT); - if (ACPI_SUCCESS(status)) - return count; - - put_device(&acpi_device->dev); - acpi_evaluate_ost(acpi_device->handle, ACPI_OST_EC_OSPM_EJECT, - ACPI_OST_SC_NON_SPECIFIC_FAILURE, NULL); - return status == AE_NO_MEMORY ? -ENOMEM : -EAGAIN; -} - -static DEVICE_ATTR(eject, 0200, NULL, acpi_eject_store); - -static ssize_t -acpi_device_hid_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct acpi_device *acpi_dev = to_acpi_device(dev); - - return sprintf(buf, "%s\n", acpi_device_hid(acpi_dev)); -} -static DEVICE_ATTR(hid, 0444, acpi_device_hid_show, NULL); - -static ssize_t acpi_device_uid_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct acpi_device *acpi_dev = to_acpi_device(dev); - - return sprintf(buf, "%s\n", acpi_dev->pnp.unique_id); -} -static DEVICE_ATTR(uid, 0444, acpi_device_uid_show, NULL); - -static ssize_t acpi_device_adr_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct acpi_device *acpi_dev = to_acpi_device(dev); - - return sprintf(buf, "0x%08x\n", - (unsigned int)(acpi_dev->pnp.bus_address)); -} -static DEVICE_ATTR(adr, 0444, acpi_device_adr_show, NULL); - -static ssize_t -acpi_device_path_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct acpi_device *acpi_dev = to_acpi_device(dev); - struct acpi_buffer path = {ACPI_ALLOCATE_BUFFER, NULL}; - int result; - - result = acpi_get_name(acpi_dev->handle, ACPI_FULL_PATHNAME, &path); - if (result) - goto end; - - result = sprintf(buf, "%s\n", (char*)path.pointer); - kfree(path.pointer); -end: - return result; -} -static DEVICE_ATTR(path, 0444, acpi_device_path_show, NULL); - -/* sysfs file that shows description text from the ACPI _STR method */ -static ssize_t description_show(struct device *dev, - struct device_attribute *attr, - char *buf) { - struct acpi_device *acpi_dev = to_acpi_device(dev); - int result; - - if (acpi_dev->pnp.str_obj == NULL) - return 0; - - /* - * The _STR object contains a Unicode identifier for a device. - * We need to convert to utf-8 so it can be displayed. - */ - result = utf16s_to_utf8s( - (wchar_t *)acpi_dev->pnp.str_obj->buffer.pointer, - acpi_dev->pnp.str_obj->buffer.length, - UTF16_LITTLE_ENDIAN, buf, - PAGE_SIZE); - - buf[result++] = '\n'; - - return result; -} -static DEVICE_ATTR(description, 0444, description_show, NULL); - -static ssize_t -acpi_device_sun_show(struct device *dev, struct device_attribute *attr, - char *buf) { - struct acpi_device *acpi_dev = to_acpi_device(dev); - acpi_status status; - unsigned long long sun; - - status = acpi_evaluate_integer(acpi_dev->handle, "_SUN", NULL, &sun); - if (ACPI_FAILURE(status)) - return -ENODEV; - - return sprintf(buf, "%llu\n", sun); -} -static DEVICE_ATTR(sun, 0444, acpi_device_sun_show, NULL); - -static ssize_t status_show(struct device *dev, struct device_attribute *attr, - char *buf) { - struct acpi_device *acpi_dev = to_acpi_device(dev); - acpi_status status; - unsigned long long sta; - - status = acpi_evaluate_integer(acpi_dev->handle, "_STA", NULL, &sta); - if (ACPI_FAILURE(status)) - return -ENODEV; - - return sprintf(buf, "%llu\n", sta); -} -static DEVICE_ATTR_RO(status); - -static int acpi_device_setup_files(struct acpi_device *dev) -{ - struct acpi_buffer buffer = {ACPI_ALLOCATE_BUFFER, NULL}; - acpi_status status; - int result = 0; - - /* - * Devices gotten from FADT don't have a "path" attribute - */ - if (dev->handle) { - result = device_create_file(&dev->dev, &dev_attr_path); - if (result) - goto end; - } - - if (!list_empty(&dev->pnp.ids)) { - result = device_create_file(&dev->dev, &dev_attr_hid); - if (result) - goto end; - - result = device_create_file(&dev->dev, &dev_attr_modalias); - if (result) - goto end; - } - - /* - * If device has _STR, 'description' file is created - */ - if (acpi_has_method(dev->handle, "_STR")) { - status = acpi_evaluate_object(dev->handle, "_STR", - NULL, &buffer); - if (ACPI_FAILURE(status)) - buffer.pointer = NULL; - dev->pnp.str_obj = buffer.pointer; - result = device_create_file(&dev->dev, &dev_attr_description); - if (result) - goto end; - } - - if (dev->pnp.type.bus_address) - result = device_create_file(&dev->dev, &dev_attr_adr); - if (dev->pnp.unique_id) - result = device_create_file(&dev->dev, &dev_attr_uid); - - if (acpi_has_method(dev->handle, "_SUN")) { - result = device_create_file(&dev->dev, &dev_attr_sun); - if (result) - goto end; - } - - if (acpi_has_method(dev->handle, "_STA")) { - result = device_create_file(&dev->dev, &dev_attr_status); - if (result) - goto end; - } - - /* - * If device has _EJ0, 'eject' file is created that is used to trigger - * hot-removal function from userland. - */ - if (acpi_has_method(dev->handle, "_EJ0")) { - result = device_create_file(&dev->dev, &dev_attr_eject); - if (result) - return result; - } - - if (dev->flags.power_manageable) { - result = device_create_file(&dev->dev, &dev_attr_power_state); - if (result) - return result; - - if (dev->power.flags.power_resources) - result = device_create_file(&dev->dev, - &dev_attr_real_power_state); - } - -end: - return result; -} - -static void acpi_device_remove_files(struct acpi_device *dev) -{ - if (dev->flags.power_manageable) { - device_remove_file(&dev->dev, &dev_attr_power_state); - if (dev->power.flags.power_resources) - device_remove_file(&dev->dev, - &dev_attr_real_power_state); - } - - /* - * If device has _STR, remove 'description' file - */ - if (acpi_has_method(dev->handle, "_STR")) { - kfree(dev->pnp.str_obj); - device_remove_file(&dev->dev, &dev_attr_description); - } - /* - * If device has _EJ0, remove 'eject' file. - */ - if (acpi_has_method(dev->handle, "_EJ0")) - device_remove_file(&dev->dev, &dev_attr_eject); - - if (acpi_has_method(dev->handle, "_SUN")) - device_remove_file(&dev->dev, &dev_attr_sun); - - if (dev->pnp.unique_id) - device_remove_file(&dev->dev, &dev_attr_uid); - if (dev->pnp.type.bus_address) - device_remove_file(&dev->dev, &dev_attr_adr); - device_remove_file(&dev->dev, &dev_attr_modalias); - device_remove_file(&dev->dev, &dev_attr_hid); - if (acpi_has_method(dev->handle, "_STA")) - device_remove_file(&dev->dev, &dev_attr_status); - if (dev->handle) - device_remove_file(&dev->dev, &dev_attr_path); -} -/* -------------------------------------------------------------------------- - ACPI Bus operations - -------------------------------------------------------------------------- */ - -/** - * acpi_of_match_device - Match device object using the "compatible" property. - * @adev: ACPI device object to match. - * @of_match_table: List of device IDs to match against. - * - * If @dev has an ACPI companion which has ACPI_DT_NAMESPACE_HID in its list of - * identifiers and a _DSD object with the "compatible" property, use that - * property to match against the given list of identifiers. - */ -static bool acpi_of_match_device(struct acpi_device *adev, - const struct of_device_id *of_match_table) -{ - const union acpi_object *of_compatible, *obj; - int i, nval; - - if (!adev) - return false; - - of_compatible = adev->data.of_compatible; - if (!of_match_table || !of_compatible) - return false; - - if (of_compatible->type == ACPI_TYPE_PACKAGE) { - nval = of_compatible->package.count; - obj = of_compatible->package.elements; - } else { /* Must be ACPI_TYPE_STRING. */ - nval = 1; - obj = of_compatible; - } - /* Now we can look for the driver DT compatible strings */ - for (i = 0; i < nval; i++, obj++) { - const struct of_device_id *id; - - for (id = of_match_table; id->compatible[0]; id++) - if (!strcasecmp(obj->string.pointer, id->compatible)) - return true; - } - - return false; -} - -static bool __acpi_match_device_cls(const struct acpi_device_id *id, - struct acpi_hardware_id *hwid) -{ - int i, msk, byte_shift; - char buf[3]; - - if (!id->cls) - return false; - - /* Apply class-code bitmask, before checking each class-code byte */ - for (i = 1; i <= 3; i++) { - byte_shift = 8 * (3 - i); - msk = (id->cls_msk >> byte_shift) & 0xFF; - if (!msk) - continue; - - sprintf(buf, "%02x", (id->cls >> byte_shift) & msk); - if (strncmp(buf, &hwid->id[(i - 1) * 2], 2)) - return false; - } - return true; -} - -static const struct acpi_device_id *__acpi_match_device( - struct acpi_device *device, - const struct acpi_device_id *ids, - const struct of_device_id *of_ids) -{ - const struct acpi_device_id *id; - struct acpi_hardware_id *hwid; - - /* - * If the device is not present, it is unnecessary to load device - * driver for it. - */ - if (!device || !device->status.present) - return NULL; - - list_for_each_entry(hwid, &device->pnp.ids, list) { - /* First, check the ACPI/PNP IDs provided by the caller. */ - for (id = ids; id->id[0] || id->cls; id++) { - if (id->id[0] && !strcmp((char *) id->id, hwid->id)) - return id; - else if (id->cls && __acpi_match_device_cls(id, hwid)) - return id; - } - - /* - * Next, check ACPI_DT_NAMESPACE_HID and try to match the - * "compatible" property if found. - * - * The id returned by the below is not valid, but the only - * caller passing non-NULL of_ids here is only interested in - * whether or not the return value is NULL. - */ - if (!strcmp(ACPI_DT_NAMESPACE_HID, hwid->id) - && acpi_of_match_device(device, of_ids)) - return id; - } - return NULL; -} - -/** - * acpi_match_device - Match a struct device against a given list of ACPI IDs - * @ids: Array of struct acpi_device_id object to match against. - * @dev: The device structure to match. - * - * Check if @dev has a valid ACPI handle and if there is a struct acpi_device - * object for that handle and use that object to match against a given list of - * device IDs. - * - * Return a pointer to the first matching ID on success or %NULL on failure. - */ -const struct acpi_device_id *acpi_match_device(const struct acpi_device_id *ids, - const struct device *dev) -{ - return __acpi_match_device(acpi_companion_match(dev), ids, NULL); -} -EXPORT_SYMBOL_GPL(acpi_match_device); - -int acpi_match_device_ids(struct acpi_device *device, - const struct acpi_device_id *ids) -{ - return __acpi_match_device(device, ids, NULL) ? 0 : -ENOENT; -} -EXPORT_SYMBOL(acpi_match_device_ids); - -bool acpi_driver_match_device(struct device *dev, - const struct device_driver *drv) -{ - if (!drv->acpi_match_table) - return acpi_of_match_device(ACPI_COMPANION(dev), - drv->of_match_table); - - return !!__acpi_match_device(acpi_companion_match(dev), - drv->acpi_match_table, drv->of_match_table); -} -EXPORT_SYMBOL_GPL(acpi_driver_match_device); - static void acpi_free_power_resources_lists(struct acpi_device *device) { int i; @@ -1144,144 +469,6 @@ static void acpi_device_release(struct device *dev) kfree(acpi_dev); } -static int acpi_bus_match(struct device *dev, struct device_driver *drv) -{ - struct acpi_device *acpi_dev = to_acpi_device(dev); - struct acpi_driver *acpi_drv = to_acpi_driver(drv); - - return acpi_dev->flags.match_driver - && !acpi_match_device_ids(acpi_dev, acpi_drv->ids); -} - -static int acpi_device_uevent(struct device *dev, struct kobj_uevent_env *env) -{ - return __acpi_device_uevent_modalias(to_acpi_device(dev), env); -} - -static void acpi_device_notify(acpi_handle handle, u32 event, void *data) -{ - struct acpi_device *device = data; - - device->driver->ops.notify(device, event); -} - -static void acpi_device_notify_fixed(void *data) -{ - struct acpi_device *device = data; - - /* Fixed hardware devices have no handles */ - acpi_device_notify(NULL, ACPI_FIXED_HARDWARE_EVENT, device); -} - -static u32 acpi_device_fixed_event(void *data) -{ - acpi_os_execute(OSL_NOTIFY_HANDLER, acpi_device_notify_fixed, data); - return ACPI_INTERRUPT_HANDLED; -} - -static int acpi_device_install_notify_handler(struct acpi_device *device) -{ - acpi_status status; - - if (device->device_type == ACPI_BUS_TYPE_POWER_BUTTON) - status = - acpi_install_fixed_event_handler(ACPI_EVENT_POWER_BUTTON, - acpi_device_fixed_event, - device); - else if (device->device_type == ACPI_BUS_TYPE_SLEEP_BUTTON) - status = - acpi_install_fixed_event_handler(ACPI_EVENT_SLEEP_BUTTON, - acpi_device_fixed_event, - device); - else - status = acpi_install_notify_handler(device->handle, - ACPI_DEVICE_NOTIFY, - acpi_device_notify, - device); - - if (ACPI_FAILURE(status)) - return -EINVAL; - return 0; -} - -static void acpi_device_remove_notify_handler(struct acpi_device *device) -{ - if (device->device_type == ACPI_BUS_TYPE_POWER_BUTTON) - acpi_remove_fixed_event_handler(ACPI_EVENT_POWER_BUTTON, - acpi_device_fixed_event); - else if (device->device_type == ACPI_BUS_TYPE_SLEEP_BUTTON) - acpi_remove_fixed_event_handler(ACPI_EVENT_SLEEP_BUTTON, - acpi_device_fixed_event); - else - acpi_remove_notify_handler(device->handle, ACPI_DEVICE_NOTIFY, - acpi_device_notify); -} - -static int acpi_device_probe(struct device *dev) -{ - struct acpi_device *acpi_dev = to_acpi_device(dev); - struct acpi_driver *acpi_drv = to_acpi_driver(dev->driver); - int ret; - - if (acpi_dev->handler && !acpi_is_pnp_device(acpi_dev)) - return -EINVAL; - - if (!acpi_drv->ops.add) - return -ENOSYS; - - ret = acpi_drv->ops.add(acpi_dev); - if (ret) - return ret; - - acpi_dev->driver = acpi_drv; - ACPI_DEBUG_PRINT((ACPI_DB_INFO, - "Driver [%s] successfully bound to device [%s]\n", - acpi_drv->name, acpi_dev->pnp.bus_id)); - - if (acpi_drv->ops.notify) { - ret = acpi_device_install_notify_handler(acpi_dev); - if (ret) { - if (acpi_drv->ops.remove) - acpi_drv->ops.remove(acpi_dev); - - acpi_dev->driver = NULL; - acpi_dev->driver_data = NULL; - return ret; - } - } - - ACPI_DEBUG_PRINT((ACPI_DB_INFO, "Found driver [%s] for device [%s]\n", - acpi_drv->name, acpi_dev->pnp.bus_id)); - get_device(dev); - return 0; -} - -static int acpi_device_remove(struct device * dev) -{ - struct acpi_device *acpi_dev = to_acpi_device(dev); - struct acpi_driver *acpi_drv = acpi_dev->driver; - - if (acpi_drv) { - if (acpi_drv->ops.notify) - acpi_device_remove_notify_handler(acpi_dev); - if (acpi_drv->ops.remove) - acpi_drv->ops.remove(acpi_dev); - } - acpi_dev->driver = NULL; - acpi_dev->driver_data = NULL; - - put_device(dev); - return 0; -} - -struct bus_type acpi_bus_type = { - .name = "acpi", - .match = acpi_bus_match, - .probe = acpi_device_probe, - .remove = acpi_device_remove, - .uevent = acpi_device_uevent, -}; - static void acpi_device_del(struct acpi_device *device) { mutex_lock(&acpi_device_lock); @@ -1528,47 +715,6 @@ struct acpi_device *acpi_get_next_child(struct device *dev, return next == head ? NULL : list_entry(next, struct acpi_device, node); } -/* -------------------------------------------------------------------------- - Driver Management - -------------------------------------------------------------------------- */ -/** - * acpi_bus_register_driver - register a driver with the ACPI bus - * @driver: driver being registered - * - * Registers a driver with the ACPI bus. Searches the namespace for all - * devices that match the driver's criteria and binds. Returns zero for - * success or a negative error status for failure. - */ -int acpi_bus_register_driver(struct acpi_driver *driver) -{ - int ret; - - if (acpi_disabled) - return -ENODEV; - driver->drv.name = driver->name; - driver->drv.bus = &acpi_bus_type; - driver->drv.owner = driver->owner; - - ret = driver_register(&driver->drv); - return ret; -} - -EXPORT_SYMBOL(acpi_bus_register_driver); - -/** - * acpi_bus_unregister_driver - unregisters a driver with the ACPI bus - * @driver: driver to unregister - * - * Unregisters a driver with the ACPI bus. Searches the namespace for all - * devices that match the driver's criteria and unbinds. - */ -void acpi_bus_unregister_driver(struct acpi_driver *driver) -{ - driver_unregister(&driver->drv); -} - -EXPORT_SYMBOL(acpi_bus_unregister_driver); - /* -------------------------------------------------------------------------- Device Enumeration -------------------------------------------------------------------------- */ @@ -2744,12 +1890,6 @@ int __init acpi_scan_init(void) { int result; - result = bus_register(&acpi_bus_type); - if (result) { - /* We don't want to quit even if we failed to add suspend/resume */ - printk(KERN_ERR PREFIX "Could not register bus type\n"); - } - acpi_pci_root_init(); acpi_pci_link_init(); acpi_processor_init(); diff --git a/drivers/acpi/sysfs.c b/drivers/acpi/sysfs.c index 0876d77b3206..40a42655227c 100644 --- a/drivers/acpi/sysfs.c +++ b/drivers/acpi/sysfs.c @@ -69,6 +69,8 @@ static const struct acpi_dlevel acpi_debug_levels[] = { ACPI_DEBUG_INIT(ACPI_LV_INIT), ACPI_DEBUG_INIT(ACPI_LV_DEBUG_OBJECT), ACPI_DEBUG_INIT(ACPI_LV_INFO), + ACPI_DEBUG_INIT(ACPI_LV_REPAIR), + ACPI_DEBUG_INIT(ACPI_LV_TRACE_POINT), ACPI_DEBUG_INIT(ACPI_LV_INIT_NAMES), ACPI_DEBUG_INIT(ACPI_LV_PARSE), @@ -162,55 +164,116 @@ static const struct kernel_param_ops param_ops_debug_level = { module_param_cb(debug_layer, ¶m_ops_debug_layer, &acpi_dbg_layer, 0644); module_param_cb(debug_level, ¶m_ops_debug_level, &acpi_dbg_level, 0644); -static char trace_method_name[6]; -module_param_string(trace_method_name, trace_method_name, 6, 0644); -static unsigned int trace_debug_layer; -module_param(trace_debug_layer, uint, 0644); -static unsigned int trace_debug_level; -module_param(trace_debug_level, uint, 0644); +static char trace_method_name[1024]; + +int param_set_trace_method_name(const char *val, const struct kernel_param *kp) +{ + u32 saved_flags = 0; + bool is_abs_path = true; + + if (*val != '\\') + is_abs_path = false; + + if ((is_abs_path && strlen(val) > 1023) || + (!is_abs_path && strlen(val) > 1022)) { + pr_err("%s: string parameter too long\n", kp->name); + return -ENOSPC; + } + + /* + * It's not safe to update acpi_gbl_trace_method_name without + * having the tracer stopped, so we save the original tracer + * state and disable it. + */ + saved_flags = acpi_gbl_trace_flags; + (void)acpi_debug_trace(NULL, + acpi_gbl_trace_dbg_level, + acpi_gbl_trace_dbg_layer, + 0); + + /* This is a hack. We can't kmalloc in early boot. */ + if (is_abs_path) + strcpy(trace_method_name, val); + else { + trace_method_name[0] = '\\'; + strcpy(trace_method_name+1, val); + } + + /* Restore the original tracer state */ + (void)acpi_debug_trace(trace_method_name, + acpi_gbl_trace_dbg_level, + acpi_gbl_trace_dbg_layer, + saved_flags); + + return 0; +} + +static int param_get_trace_method_name(char *buffer, const struct kernel_param *kp) +{ + return scnprintf(buffer, PAGE_SIZE, "%s", acpi_gbl_trace_method_name); +} + +static const struct kernel_param_ops param_ops_trace_method = { + .set = param_set_trace_method_name, + .get = param_get_trace_method_name, +}; + +static const struct kernel_param_ops param_ops_trace_attrib = { + .set = param_set_uint, + .get = param_get_uint, +}; + +module_param_cb(trace_method_name, ¶m_ops_trace_method, &trace_method_name, 0644); +module_param_cb(trace_debug_layer, ¶m_ops_trace_attrib, &acpi_gbl_trace_dbg_layer, 0644); +module_param_cb(trace_debug_level, ¶m_ops_trace_attrib, &acpi_gbl_trace_dbg_level, 0644); static int param_set_trace_state(const char *val, struct kernel_param *kp) { - int result = 0; + acpi_status status; + const char *method = trace_method_name; + u32 flags = 0; - if (!strncmp(val, "enable", sizeof("enable") - 1)) { - result = acpi_debug_trace(trace_method_name, trace_debug_level, - trace_debug_layer, 0); - if (result) - result = -EBUSY; - goto exit; - } +/* So "xxx-once" comparison should go prior than "xxx" comparison */ +#define acpi_compare_param(val, key) \ + strncmp((val), (key), sizeof(key) - 1) - if (!strncmp(val, "disable", sizeof("disable") - 1)) { - int name = 0; - result = acpi_debug_trace((char *)&name, trace_debug_level, - trace_debug_layer, 0); - if (result) - result = -EBUSY; - goto exit; - } + if (!acpi_compare_param(val, "enable")) { + method = NULL; + flags = ACPI_TRACE_ENABLED; + } else if (!acpi_compare_param(val, "disable")) + method = NULL; + else if (!acpi_compare_param(val, "method-once")) + flags = ACPI_TRACE_ENABLED | ACPI_TRACE_ONESHOT; + else if (!acpi_compare_param(val, "method")) + flags = ACPI_TRACE_ENABLED; + else if (!acpi_compare_param(val, "opcode-once")) + flags = ACPI_TRACE_ENABLED | ACPI_TRACE_ONESHOT | ACPI_TRACE_OPCODE; + else if (!acpi_compare_param(val, "opcode")) + flags = ACPI_TRACE_ENABLED | ACPI_TRACE_OPCODE; + else + return -EINVAL; - if (!strncmp(val, "1", 1)) { - result = acpi_debug_trace(trace_method_name, trace_debug_level, - trace_debug_layer, 1); - if (result) - result = -EBUSY; - goto exit; - } + status = acpi_debug_trace(method, + acpi_gbl_trace_dbg_level, + acpi_gbl_trace_dbg_layer, + flags); + if (ACPI_FAILURE(status)) + return -EBUSY; - result = -EINVAL; -exit: - return result; + return 0; } static int param_get_trace_state(char *buffer, struct kernel_param *kp) { - if (!acpi_gbl_trace_method_name) + if (!(acpi_gbl_trace_flags & ACPI_TRACE_ENABLED)) return sprintf(buffer, "disable"); else { - if (acpi_gbl_trace_flags & 1) - return sprintf(buffer, "1"); - else + if (acpi_gbl_trace_method_name) { + if (acpi_gbl_trace_flags & ACPI_TRACE_ONESHOT) + return sprintf(buffer, "method-once"); + else + return sprintf(buffer, "method"); + } else return sprintf(buffer, "enable"); } return 0; diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c index 2e19189da0ee..17a6fa01a338 100644 --- a/drivers/acpi/tables.c +++ b/drivers/acpi/tables.c @@ -15,10 +15,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * */ diff --git a/drivers/acpi/thermal.c b/drivers/acpi/thermal.c index 6d4e44ea74ac..30d8518b25fb 100644 --- a/drivers/acpi/thermal.c +++ b/drivers/acpi/thermal.c @@ -16,10 +16,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * * This driver fully implements the ACPI thermal policy as described in the @@ -529,8 +525,7 @@ static void acpi_thermal_check(void *data) /* sys I/F for generic thermal sysfs support */ -static int thermal_get_temp(struct thermal_zone_device *thermal, - unsigned long *temp) +static int thermal_get_temp(struct thermal_zone_device *thermal, int *temp) { struct acpi_thermal *tz = thermal->devdata; int result; @@ -637,7 +632,7 @@ static int thermal_get_trip_type(struct thermal_zone_device *thermal, } static int thermal_get_trip_temp(struct thermal_zone_device *thermal, - int trip, unsigned long *temp) + int trip, int *temp) { struct acpi_thermal *tz = thermal->devdata; int i; @@ -690,7 +685,8 @@ static int thermal_get_trip_temp(struct thermal_zone_device *thermal, } static int thermal_get_crit_temp(struct thermal_zone_device *thermal, - unsigned long *temperature) { + int *temperature) +{ struct acpi_thermal *tz = thermal->devdata; if (tz->trips.critical.flags.valid) { @@ -713,8 +709,8 @@ static int thermal_get_trend(struct thermal_zone_device *thermal, return -EINVAL; if (type == THERMAL_TRIP_ACTIVE) { - unsigned long trip_temp; - unsigned long temp = DECI_KELVIN_TO_MILLICELSIUS_WITH_OFFSET( + int trip_temp; + int temp = DECI_KELVIN_TO_MILLICELSIUS_WITH_OFFSET( tz->temperature, tz->kelvin_offset); if (thermal_get_trip_temp(thermal, trip, &trip_temp)) return -EINVAL; diff --git a/drivers/acpi/utils.c b/drivers/acpi/utils.c index 67c548ad3764..475c9079bf85 100644 --- a/drivers/acpi/utils.c +++ b/drivers/acpi/utils.c @@ -16,10 +16,6 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - * * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ diff --git a/drivers/android/binder.c b/drivers/android/binder.c index 6607f3c6ace1..a39e85f9efa9 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -2834,7 +2834,7 @@ static int binder_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) return VM_FAULT_SIGBUS; } -static struct vm_operations_struct binder_vm_ops = { +static const struct vm_operations_struct binder_vm_ops = { .open = binder_vma_open, .close = binder_vma_close, .fault = binder_vm_fault, diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 790e0deb278e..b79cb10e289e 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -4753,6 +4753,7 @@ void swap_buf_le16(u16 *buf, unsigned int buf_words) /** * ata_qc_new_init - Request an available ATA command, and initialize it * @dev: Device from whom we request an available command structure + * @tag: tag * * LOCKING: * None. diff --git a/drivers/ata/pata_arasan_cf.c b/drivers/ata/pata_arasan_cf.c index 5d9ee99c2148..80fe0f6fed29 100644 --- a/drivers/ata/pata_arasan_cf.c +++ b/drivers/ata/pata_arasan_cf.c @@ -834,7 +834,7 @@ static int arasan_cf_probe(struct platform_device *pdev) return -ENOMEM; } - acdev->clk = clk_get(&pdev->dev, NULL); + acdev->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(acdev->clk)) { dev_warn(&pdev->dev, "Clock not found\n"); return PTR_ERR(acdev->clk); @@ -843,9 +843,8 @@ static int arasan_cf_probe(struct platform_device *pdev) /* allocate host */ host = ata_host_alloc(&pdev->dev, 1); if (!host) { - ret = -ENOMEM; dev_warn(&pdev->dev, "alloc host fail\n"); - goto free_clk; + return -ENOMEM; } ap = host->ports[0]; @@ -894,7 +893,7 @@ static int arasan_cf_probe(struct platform_device *pdev) ret = cf_init(acdev); if (ret) - goto free_clk; + return ret; cf_card_detect(acdev, 0); @@ -904,8 +903,7 @@ static int arasan_cf_probe(struct platform_device *pdev) return 0; cf_exit(acdev); -free_clk: - clk_put(acdev->clk); + return ret; } @@ -916,7 +914,6 @@ static int arasan_cf_remove(struct platform_device *pdev) ata_host_detach(host); cf_exit(acdev); - clk_put(acdev->clk); return 0; } diff --git a/drivers/ata/pata_rb532_cf.c b/drivers/ata/pata_rb532_cf.c index 6d08446b877c..12fe0f3bb7e9 100644 --- a/drivers/ata/pata_rb532_cf.c +++ b/drivers/ata/pata_rb532_cf.c @@ -27,12 +27,11 @@ #include #include #include +#include #include #include -#include - #define DRV_NAME "pata-rb532-cf" #define DRV_VERSION "0.1.0" #define DRV_DESC "PATA driver for RouterBOARD 532 Compact Flash" diff --git a/drivers/ata/sata_rcar.c b/drivers/ata/sata_rcar.c index d49a5193b7de..8804127b108c 100644 --- a/drivers/ata/sata_rcar.c +++ b/drivers/ata/sata_rcar.c @@ -861,10 +861,6 @@ MODULE_DEVICE_TABLE(of, sata_rcar_match); static const struct platform_device_id sata_rcar_id_table[] = { { "sata_rcar", RCAR_GEN1_SATA }, /* Deprecated by "sata-r8a7779" */ { "sata-r8a7779", RCAR_GEN1_SATA }, - { "sata-r8a7790", RCAR_GEN2_SATA }, - { "sata-r8a7790-es1", RCAR_R8A7790_ES1_SATA }, - { "sata-r8a7791", RCAR_GEN2_SATA }, - { "sata-r8a7793", RCAR_GEN2_SATA }, { }, }; MODULE_DEVICE_TABLE(platform, sata_rcar_id_table); diff --git a/drivers/base/Makefile b/drivers/base/Makefile index 527d291706e8..6b2a84e7f2be 100644 --- a/drivers/base/Makefile +++ b/drivers/base/Makefile @@ -22,6 +22,7 @@ obj-$(CONFIG_REGMAP) += regmap/ obj-$(CONFIG_SOC_BUS) += soc.o obj-$(CONFIG_PINCTRL) += pinctrl.o obj-$(CONFIG_DEV_COREDUMP) += devcoredump.o +obj-$(CONFIG_GENERIC_MSI_IRQ_DOMAIN) += platform-msi.o ccflags-$(CONFIG_DEBUG_DRIVER) := -DDEBUG diff --git a/drivers/base/core.c b/drivers/base/core.c index fc5a558f62f9..334ec7ef1960 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -708,6 +708,9 @@ void device_initialize(struct device *dev) INIT_LIST_HEAD(&dev->devres_head); device_pm_init(dev); set_dev_node(dev, -1); +#ifdef CONFIG_GENERIC_MSI_IRQ + INIT_LIST_HEAD(&dev->msi_list); +#endif } EXPORT_SYMBOL_GPL(device_initialize); @@ -1298,6 +1301,19 @@ void device_unregister(struct device *dev) } EXPORT_SYMBOL_GPL(device_unregister); +static struct device *prev_device(struct klist_iter *i) +{ + struct klist_node *n = klist_prev(i); + struct device *dev = NULL; + struct device_private *p; + + if (n) { + p = to_device_private_parent(n); + dev = p->device; + } + return dev; +} + static struct device *next_device(struct klist_iter *i) { struct klist_node *n = klist_next(i); @@ -1386,6 +1402,36 @@ int device_for_each_child(struct device *parent, void *data, } EXPORT_SYMBOL_GPL(device_for_each_child); +/** + * device_for_each_child_reverse - device child iterator in reversed order. + * @parent: parent struct device. + * @fn: function to be called for each device. + * @data: data for the callback. + * + * Iterate over @parent's child devices, and call @fn for each, + * passing it @data. + * + * We check the return of @fn each time. If it returns anything + * other than 0, we break out and return that value. + */ +int device_for_each_child_reverse(struct device *parent, void *data, + int (*fn)(struct device *dev, void *data)) +{ + struct klist_iter i; + struct device *child; + int error = 0; + + if (!parent->p) + return 0; + + klist_iter_init(&parent->p->klist_children, &i); + while ((child = prev_device(&i)) && !error) + error = fn(child, data); + klist_iter_exit(&i); + return error; +} +EXPORT_SYMBOL_GPL(device_for_each_child_reverse); + /** * device_find_child - device iterator for locating a particular device. * @parent: parent struct device diff --git a/drivers/base/dd.c b/drivers/base/dd.c index cc2b1d4801fd..be0eb4639128 100644 --- a/drivers/base/dd.c +++ b/drivers/base/dd.c @@ -407,6 +407,8 @@ EXPORT_SYMBOL_GPL(wait_for_device_probe); * * This function must be called with @dev lock held. When called for a * USB interface, @dev->parent lock must be held as well. + * + * If the device has a parent, runtime-resume the parent before driver probing. */ int driver_probe_device(struct device_driver *drv, struct device *dev) { @@ -418,10 +420,16 @@ int driver_probe_device(struct device_driver *drv, struct device *dev) pr_debug("bus: '%s': %s: matched device %s with driver %s\n", drv->bus->name, __func__, dev_name(dev), drv->name); + if (dev->parent) + pm_runtime_get_sync(dev->parent); + pm_runtime_barrier(dev); ret = really_probe(dev, drv); pm_request_idle(dev); + if (dev->parent) + pm_runtime_put(dev->parent); + return ret; } @@ -515,11 +523,17 @@ static void __device_attach_async_helper(void *_dev, async_cookie_t cookie) device_lock(dev); + if (dev->parent) + pm_runtime_get_sync(dev->parent); + bus_for_each_drv(dev->bus, NULL, &data, __device_attach_driver); dev_dbg(dev, "async probe completed\n"); pm_request_idle(dev); + if (dev->parent) + pm_runtime_put(dev->parent); + device_unlock(dev); put_device(dev); @@ -549,6 +563,9 @@ static int __device_attach(struct device *dev, bool allow_async) .want_async = false, }; + if (dev->parent) + pm_runtime_get_sync(dev->parent); + ret = bus_for_each_drv(dev->bus, NULL, &data, __device_attach_driver); if (!ret && allow_async && data.have_async) { @@ -565,6 +582,9 @@ static int __device_attach(struct device *dev, bool allow_async) } else { pm_request_idle(dev); } + + if (dev->parent) + pm_runtime_put(dev->parent); } out_unlock: device_unlock(dev); diff --git a/drivers/base/node.c b/drivers/base/node.c index 31df474d72f4..560751bad294 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c @@ -392,6 +392,16 @@ int register_mem_sect_under_node(struct memory_block *mem_blk, int nid) for (pfn = sect_start_pfn; pfn <= sect_end_pfn; pfn++) { int page_nid; + /* + * memory block could have several absent sections from start. + * skip pfn range from absent section + */ + if (!pfn_present(pfn)) { + pfn = round_down(pfn + PAGES_PER_SECTION, + PAGES_PER_SECTION) - 1; + continue; + } + page_nid = get_nid_for_pfn(pfn); if (page_nid < 0) continue; diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c new file mode 100644 index 000000000000..1857a5dd0816 --- /dev/null +++ b/drivers/base/platform-msi.c @@ -0,0 +1,282 @@ +/* + * MSI framework for platform devices + * + * Copyright (C) 2015 ARM Limited, All Rights Reserved. + * Author: Marc Zyngier + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include + +#define DEV_ID_SHIFT 24 + +/* + * Internal data structure containing a (made up, but unique) devid + * and the callback to write the MSI message. + */ +struct platform_msi_priv_data { + irq_write_msi_msg_t write_msg; + int devid; +}; + +/* The devid allocator */ +static DEFINE_IDA(platform_msi_devid_ida); + +#ifdef GENERIC_MSI_DOMAIN_OPS +/* + * Convert an msi_desc to a globaly unique identifier (per-device + * devid + msi_desc position in the msi_list). + */ +static irq_hw_number_t platform_msi_calc_hwirq(struct msi_desc *desc) +{ + u32 devid; + + devid = desc->platform.msi_priv_data->devid; + + return (devid << (32 - DEV_ID_SHIFT)) | desc->platform.msi_index; +} + +static void platform_msi_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc) +{ + arg->desc = desc; + arg->hwirq = platform_msi_calc_hwirq(desc); +} + +static int platform_msi_init(struct irq_domain *domain, + struct msi_domain_info *info, + unsigned int virq, irq_hw_number_t hwirq, + msi_alloc_info_t *arg) +{ + struct irq_data *data; + + irq_domain_set_hwirq_and_chip(domain, virq, hwirq, + info->chip, info->chip_data); + + /* + * Save the MSI descriptor in handler_data so that the + * irq_write_msi_msg callback can retrieve it (and the + * associated device). + */ + data = irq_domain_get_irq_data(domain, virq); + data->handler_data = arg->desc; + + return 0; +} +#else +#define platform_msi_set_desc NULL +#define platform_msi_init NULL +#endif + +static void platform_msi_update_dom_ops(struct msi_domain_info *info) +{ + struct msi_domain_ops *ops = info->ops; + + BUG_ON(!ops); + + if (ops->msi_init == NULL) + ops->msi_init = platform_msi_init; + if (ops->set_desc == NULL) + ops->set_desc = platform_msi_set_desc; +} + +static void platform_msi_write_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct msi_desc *desc = irq_data_get_irq_handler_data(data); + struct platform_msi_priv_data *priv_data; + + priv_data = desc->platform.msi_priv_data; + + priv_data->write_msg(desc, msg); +} + +static void platform_msi_update_chip_ops(struct msi_domain_info *info) +{ + struct irq_chip *chip = info->chip; + + BUG_ON(!chip); + if (!chip->irq_mask) + chip->irq_mask = irq_chip_mask_parent; + if (!chip->irq_unmask) + chip->irq_unmask = irq_chip_unmask_parent; + if (!chip->irq_eoi) + chip->irq_eoi = irq_chip_eoi_parent; + if (!chip->irq_set_affinity) + chip->irq_set_affinity = msi_domain_set_affinity; + if (!chip->irq_write_msi_msg) + chip->irq_write_msi_msg = platform_msi_write_msg; +} + +static void platform_msi_free_descs(struct device *dev) +{ + struct msi_desc *desc, *tmp; + + list_for_each_entry_safe(desc, tmp, dev_to_msi_list(dev), list) { + list_del(&desc->list); + free_msi_entry(desc); + } +} + +static int platform_msi_alloc_descs(struct device *dev, int nvec, + struct platform_msi_priv_data *data) + +{ + int i; + + for (i = 0; i < nvec; i++) { + struct msi_desc *desc; + + desc = alloc_msi_entry(dev); + if (!desc) + break; + + desc->platform.msi_priv_data = data; + desc->platform.msi_index = i; + desc->nvec_used = 1; + + list_add_tail(&desc->list, dev_to_msi_list(dev)); + } + + if (i != nvec) { + /* Clean up the mess */ + platform_msi_free_descs(dev); + + return -ENOMEM; + } + + return 0; +} + +/** + * platform_msi_create_irq_domain - Create a platform MSI interrupt domain + * @np: Optional device-tree node of the interrupt controller + * @info: MSI domain info + * @parent: Parent irq domain + * + * Updates the domain and chip ops and creates a platform MSI + * interrupt domain. + * + * Returns: + * A domain pointer or NULL in case of failure. + */ +struct irq_domain *platform_msi_create_irq_domain(struct device_node *np, + struct msi_domain_info *info, + struct irq_domain *parent) +{ + struct irq_domain *domain; + + if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS) + platform_msi_update_dom_ops(info); + if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) + platform_msi_update_chip_ops(info); + + domain = msi_create_irq_domain(np, info, parent); + if (domain) + domain->bus_token = DOMAIN_BUS_PLATFORM_MSI; + + return domain; +} + +/** + * platform_msi_domain_alloc_irqs - Allocate MSI interrupts for @dev + * @dev: The device for which to allocate interrupts + * @nvec: The number of interrupts to allocate + * @write_msi_msg: Callback to write an interrupt message for @dev + * + * Returns: + * Zero for success, or an error code in case of failure + */ +int platform_msi_domain_alloc_irqs(struct device *dev, unsigned int nvec, + irq_write_msi_msg_t write_msi_msg) +{ + struct platform_msi_priv_data *priv_data; + int err; + + /* + * Limit the number of interrupts to 256 per device. Should we + * need to bump this up, DEV_ID_SHIFT should be adjusted + * accordingly (which would impact the max number of MSI + * capable devices). + */ + if (!dev->msi_domain || !write_msi_msg || !nvec || + nvec > (1 << (32 - DEV_ID_SHIFT))) + return -EINVAL; + + if (dev->msi_domain->bus_token != DOMAIN_BUS_PLATFORM_MSI) { + dev_err(dev, "Incompatible msi_domain, giving up\n"); + return -EINVAL; + } + + /* Already had a helping of MSI? Greed... */ + if (!list_empty(dev_to_msi_list(dev))) + return -EBUSY; + + priv_data = kzalloc(sizeof(*priv_data), GFP_KERNEL); + if (!priv_data) + return -ENOMEM; + + priv_data->devid = ida_simple_get(&platform_msi_devid_ida, + 0, 1 << DEV_ID_SHIFT, GFP_KERNEL); + if (priv_data->devid < 0) { + err = priv_data->devid; + goto out_free_data; + } + + priv_data->write_msg = write_msi_msg; + + err = platform_msi_alloc_descs(dev, nvec, priv_data); + if (err) + goto out_free_id; + + err = msi_domain_alloc_irqs(dev->msi_domain, dev, nvec); + if (err) + goto out_free_desc; + + return 0; + +out_free_desc: + platform_msi_free_descs(dev); +out_free_id: + ida_simple_remove(&platform_msi_devid_ida, priv_data->devid); +out_free_data: + kfree(priv_data); + + return err; +} + +/** + * platform_msi_domain_free_irqs - Free MSI interrupts for @dev + * @dev: The device for which to free interrupts + */ +void platform_msi_domain_free_irqs(struct device *dev) +{ + struct msi_desc *desc; + + desc = first_msi_entry(dev); + if (desc) { + struct platform_msi_priv_data *data; + + data = desc->platform.msi_priv_data; + + ida_simple_remove(&platform_msi_devid_ida, data->devid); + kfree(data); + } + + msi_domain_free_irqs(dev->msi_domain, dev); + platform_msi_free_descs(dev); +} diff --git a/drivers/base/platform.c b/drivers/base/platform.c index 063f0ab15259..f80aaaf9f610 100644 --- a/drivers/base/platform.c +++ b/drivers/base/platform.c @@ -375,9 +375,7 @@ int platform_device_add(struct platform_device *pdev) while (--i >= 0) { struct resource *r = &pdev->resource[i]; - unsigned long type = resource_type(r); - - if (type == IORESOURCE_MEM || type == IORESOURCE_IO) + if (r->parent) release_resource(r); } @@ -408,9 +406,7 @@ void platform_device_del(struct platform_device *pdev) for (i = 0; i < pdev->num_resources; i++) { struct resource *r = &pdev->resource[i]; - unsigned long type = resource_type(r); - - if (type == IORESOURCE_MEM || type == IORESOURCE_IO) + if (r->parent) release_resource(r); } } diff --git a/drivers/base/power/clock_ops.c b/drivers/base/power/clock_ops.c index acef9f9f759a..652b5a367c1f 100644 --- a/drivers/base/power/clock_ops.c +++ b/drivers/base/power/clock_ops.c @@ -38,7 +38,7 @@ struct pm_clock_entry { * @dev: The device for the given clock * @ce: PM clock entry corresponding to the clock. */ -static inline int __pm_clk_enable(struct device *dev, struct pm_clock_entry *ce) +static inline void __pm_clk_enable(struct device *dev, struct pm_clock_entry *ce) { int ret; @@ -50,8 +50,6 @@ static inline int __pm_clk_enable(struct device *dev, struct pm_clock_entry *ce) dev_err(dev, "%s: failed to enable clk %p, error %d\n", __func__, ce->clk, ret); } - - return ret; } /** diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index 0ee43c1056e0..16550c63d611 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -114,8 +114,12 @@ static int genpd_stop_dev(struct generic_pm_domain *genpd, struct device *dev) stop_latency_ns, "stop"); } -static int genpd_start_dev(struct generic_pm_domain *genpd, struct device *dev) +static int genpd_start_dev(struct generic_pm_domain *genpd, struct device *dev, + bool timed) { + if (!timed) + return GENPD_DEV_CALLBACK(genpd, int, start, dev); + return GENPD_DEV_TIMED_CALLBACK(genpd, int, start, dev, start_latency_ns, "start"); } @@ -136,41 +140,6 @@ static void genpd_sd_counter_inc(struct generic_pm_domain *genpd) smp_mb__after_atomic(); } -static void genpd_acquire_lock(struct generic_pm_domain *genpd) -{ - DEFINE_WAIT(wait); - - mutex_lock(&genpd->lock); - /* - * Wait for the domain to transition into either the active, - * or the power off state. - */ - for (;;) { - prepare_to_wait(&genpd->status_wait_queue, &wait, - TASK_UNINTERRUPTIBLE); - if (genpd->status == GPD_STATE_ACTIVE - || genpd->status == GPD_STATE_POWER_OFF) - break; - mutex_unlock(&genpd->lock); - - schedule(); - - mutex_lock(&genpd->lock); - } - finish_wait(&genpd->status_wait_queue, &wait); -} - -static void genpd_release_lock(struct generic_pm_domain *genpd) -{ - mutex_unlock(&genpd->lock); -} - -static void genpd_set_active(struct generic_pm_domain *genpd) -{ - if (genpd->resume_count == 0) - genpd->status = GPD_STATE_ACTIVE; -} - static void genpd_recalc_cpu_exit_latency(struct generic_pm_domain *genpd) { s64 usecs64; @@ -243,6 +212,18 @@ static int genpd_power_off(struct generic_pm_domain *genpd, bool timed) return ret; } +/** + * genpd_queue_power_off_work - Queue up the execution of pm_genpd_poweroff(). + * @genpd: PM domait to power off. + * + * Queue up the execution of pm_genpd_poweroff() unless it's already been done + * before. + */ +static void genpd_queue_power_off_work(struct generic_pm_domain *genpd) +{ + queue_work(pm_wq, &genpd->power_off_work); +} + /** * __pm_genpd_poweron - Restore power to a given PM domain and its masters. * @genpd: PM domain to power up. @@ -251,35 +232,14 @@ static int genpd_power_off(struct generic_pm_domain *genpd, bool timed) * resume a device belonging to it. */ static int __pm_genpd_poweron(struct generic_pm_domain *genpd) - __releases(&genpd->lock) __acquires(&genpd->lock) { struct gpd_link *link; - DEFINE_WAIT(wait); int ret = 0; - /* If the domain's master is being waited for, we have to wait too. */ - for (;;) { - prepare_to_wait(&genpd->status_wait_queue, &wait, - TASK_UNINTERRUPTIBLE); - if (genpd->status != GPD_STATE_WAIT_MASTER) - break; - mutex_unlock(&genpd->lock); - - schedule(); - - mutex_lock(&genpd->lock); - } - finish_wait(&genpd->status_wait_queue, &wait); - if (genpd->status == GPD_STATE_ACTIVE || (genpd->prepared_count > 0 && genpd->suspend_power_off)) return 0; - if (genpd->status != GPD_STATE_POWER_OFF) { - genpd_set_active(genpd); - return 0; - } - if (genpd->cpuidle_data) { cpuidle_pause_and_lock(); genpd->cpuidle_data->idle_state->disabled = true; @@ -294,20 +254,8 @@ static int __pm_genpd_poweron(struct generic_pm_domain *genpd) */ list_for_each_entry(link, &genpd->slave_links, slave_node) { genpd_sd_counter_inc(link->master); - genpd->status = GPD_STATE_WAIT_MASTER; - - mutex_unlock(&genpd->lock); ret = pm_genpd_poweron(link->master); - - mutex_lock(&genpd->lock); - - /* - * The "wait for parent" status is guaranteed not to change - * while the master is powering on. - */ - genpd->status = GPD_STATE_POWER_OFF; - wake_up_all(&genpd->status_wait_queue); if (ret) { genpd_sd_counter_dec(link->master); goto err; @@ -319,13 +267,16 @@ static int __pm_genpd_poweron(struct generic_pm_domain *genpd) goto err; out: - genpd_set_active(genpd); - + genpd->status = GPD_STATE_ACTIVE; return 0; err: - list_for_each_entry_continue_reverse(link, &genpd->slave_links, slave_node) + list_for_each_entry_continue_reverse(link, + &genpd->slave_links, + slave_node) { genpd_sd_counter_dec(link->master); + genpd_queue_power_off_work(link->master); + } return ret; } @@ -356,20 +307,18 @@ int pm_genpd_name_poweron(const char *domain_name) return genpd ? pm_genpd_poweron(genpd) : -EINVAL; } -static int genpd_start_dev_no_timing(struct generic_pm_domain *genpd, - struct device *dev) -{ - return GENPD_DEV_CALLBACK(genpd, int, start, dev); -} - static int genpd_save_dev(struct generic_pm_domain *genpd, struct device *dev) { return GENPD_DEV_TIMED_CALLBACK(genpd, int, save_state, dev, save_state_latency_ns, "state save"); } -static int genpd_restore_dev(struct generic_pm_domain *genpd, struct device *dev) +static int genpd_restore_dev(struct generic_pm_domain *genpd, + struct device *dev, bool timed) { + if (!timed) + return GENPD_DEV_CALLBACK(genpd, int, restore_state, dev); + return GENPD_DEV_TIMED_CALLBACK(genpd, int, restore_state, dev, restore_state_latency_ns, "state restore"); @@ -415,134 +364,31 @@ static int genpd_dev_pm_qos_notifier(struct notifier_block *nb, return NOTIFY_DONE; } -/** - * __pm_genpd_save_device - Save the pre-suspend state of a device. - * @pdd: Domain data of the device to save the state of. - * @genpd: PM domain the device belongs to. - */ -static int __pm_genpd_save_device(struct pm_domain_data *pdd, - struct generic_pm_domain *genpd) - __releases(&genpd->lock) __acquires(&genpd->lock) -{ - struct generic_pm_domain_data *gpd_data = to_gpd_data(pdd); - struct device *dev = pdd->dev; - int ret = 0; - - if (gpd_data->need_restore > 0) - return 0; - - /* - * If the value of the need_restore flag is still unknown at this point, - * we trust that pm_genpd_poweroff() has verified that the device is - * already runtime PM suspended. - */ - if (gpd_data->need_restore < 0) { - gpd_data->need_restore = 1; - return 0; - } - - mutex_unlock(&genpd->lock); - - genpd_start_dev(genpd, dev); - ret = genpd_save_dev(genpd, dev); - genpd_stop_dev(genpd, dev); - - mutex_lock(&genpd->lock); - - if (!ret) - gpd_data->need_restore = 1; - - return ret; -} - -/** - * __pm_genpd_restore_device - Restore the pre-suspend state of a device. - * @pdd: Domain data of the device to restore the state of. - * @genpd: PM domain the device belongs to. - */ -static void __pm_genpd_restore_device(struct pm_domain_data *pdd, - struct generic_pm_domain *genpd) - __releases(&genpd->lock) __acquires(&genpd->lock) -{ - struct generic_pm_domain_data *gpd_data = to_gpd_data(pdd); - struct device *dev = pdd->dev; - int need_restore = gpd_data->need_restore; - - gpd_data->need_restore = 0; - mutex_unlock(&genpd->lock); - - genpd_start_dev(genpd, dev); - - /* - * Call genpd_restore_dev() for recently added devices too (need_restore - * is negative then). - */ - if (need_restore) - genpd_restore_dev(genpd, dev); - - mutex_lock(&genpd->lock); -} - -/** - * genpd_abort_poweroff - Check if a PM domain power off should be aborted. - * @genpd: PM domain to check. - * - * Return true if a PM domain's status changed to GPD_STATE_ACTIVE during - * a "power off" operation, which means that a "power on" has occured in the - * meantime, or if its resume_count field is different from zero, which means - * that one of its devices has been resumed in the meantime. - */ -static bool genpd_abort_poweroff(struct generic_pm_domain *genpd) -{ - return genpd->status == GPD_STATE_WAIT_MASTER - || genpd->status == GPD_STATE_ACTIVE || genpd->resume_count > 0; -} - -/** - * genpd_queue_power_off_work - Queue up the execution of pm_genpd_poweroff(). - * @genpd: PM domait to power off. - * - * Queue up the execution of pm_genpd_poweroff() unless it's already been done - * before. - */ -static void genpd_queue_power_off_work(struct generic_pm_domain *genpd) -{ - queue_work(pm_wq, &genpd->power_off_work); -} - /** * pm_genpd_poweroff - Remove power from a given PM domain. * @genpd: PM domain to power down. * * If all of the @genpd's devices have been suspended and all of its subdomains - * have been powered down, run the runtime suspend callbacks provided by all of - * the @genpd's devices' drivers and remove power from @genpd. + * have been powered down, remove power from @genpd. */ static int pm_genpd_poweroff(struct generic_pm_domain *genpd) - __releases(&genpd->lock) __acquires(&genpd->lock) { struct pm_domain_data *pdd; struct gpd_link *link; - unsigned int not_suspended; - int ret = 0; + unsigned int not_suspended = 0; - start: /* * Do not try to power off the domain in the following situations: * (1) The domain is already in the "power off" state. - * (2) The domain is waiting for its master to power up. - * (3) One of the domain's devices is being resumed right now. - * (4) System suspend is in progress. + * (2) System suspend is in progress. */ if (genpd->status == GPD_STATE_POWER_OFF - || genpd->status == GPD_STATE_WAIT_MASTER - || genpd->resume_count > 0 || genpd->prepared_count > 0) + || genpd->prepared_count > 0) return 0; if (atomic_read(&genpd->sd_count) > 0) return -EBUSY; - not_suspended = 0; list_for_each_entry(pdd, &genpd->dev_list, list_node) { enum pm_qos_flags_status stat; @@ -560,41 +406,11 @@ static int pm_genpd_poweroff(struct generic_pm_domain *genpd) if (not_suspended > genpd->in_progress) return -EBUSY; - if (genpd->poweroff_task) { - /* - * Another instance of pm_genpd_poweroff() is executing - * callbacks, so tell it to start over and return. - */ - genpd->status = GPD_STATE_REPEAT; - return 0; - } - if (genpd->gov && genpd->gov->power_down_ok) { if (!genpd->gov->power_down_ok(&genpd->domain)) return -EAGAIN; } - genpd->status = GPD_STATE_BUSY; - genpd->poweroff_task = current; - - list_for_each_entry_reverse(pdd, &genpd->dev_list, list_node) { - ret = atomic_read(&genpd->sd_count) == 0 ? - __pm_genpd_save_device(pdd, genpd) : -EBUSY; - - if (genpd_abort_poweroff(genpd)) - goto out; - - if (ret) { - genpd_set_active(genpd); - goto out; - } - - if (genpd->status == GPD_STATE_REPEAT) { - genpd->poweroff_task = NULL; - goto start; - } - } - if (genpd->cpuidle_data) { /* * If cpuidle_data is set, cpuidle should turn the domain off @@ -607,14 +423,14 @@ static int pm_genpd_poweroff(struct generic_pm_domain *genpd) cpuidle_pause_and_lock(); genpd->cpuidle_data->idle_state->disabled = false; cpuidle_resume_and_unlock(); - goto out; + return 0; } if (genpd->power_off) { - if (atomic_read(&genpd->sd_count) > 0) { - ret = -EBUSY; - goto out; - } + int ret; + + if (atomic_read(&genpd->sd_count) > 0) + return -EBUSY; /* * If sd_count > 0 at this point, one of the subdomains hasn't @@ -625,10 +441,8 @@ static int pm_genpd_poweroff(struct generic_pm_domain *genpd) * happen very often). */ ret = genpd_power_off(genpd, true); - if (ret == -EBUSY) { - genpd_set_active(genpd); - goto out; - } + if (ret) + return ret; } genpd->status = GPD_STATE_POWER_OFF; @@ -638,10 +452,7 @@ static int pm_genpd_poweroff(struct generic_pm_domain *genpd) genpd_queue_power_off_work(link->master); } - out: - genpd->poweroff_task = NULL; - wake_up_all(&genpd->status_wait_queue); - return ret; + return 0; } /** @@ -654,9 +465,9 @@ static void genpd_power_off_work_fn(struct work_struct *work) genpd = container_of(work, struct generic_pm_domain, power_off_work); - genpd_acquire_lock(genpd); + mutex_lock(&genpd->lock); pm_genpd_poweroff(genpd); - genpd_release_lock(genpd); + mutex_unlock(&genpd->lock); } /** @@ -670,7 +481,6 @@ static void genpd_power_off_work_fn(struct work_struct *work) static int pm_genpd_runtime_suspend(struct device *dev) { struct generic_pm_domain *genpd; - struct generic_pm_domain_data *gpd_data; bool (*stop_ok)(struct device *__dev); int ret; @@ -684,10 +494,16 @@ static int pm_genpd_runtime_suspend(struct device *dev) if (stop_ok && !stop_ok(dev)) return -EBUSY; - ret = genpd_stop_dev(genpd, dev); + ret = genpd_save_dev(genpd, dev); if (ret) return ret; + ret = genpd_stop_dev(genpd, dev); + if (ret) { + genpd_restore_dev(genpd, dev, true); + return ret; + } + /* * If power.irq_safe is set, this routine will be run with interrupts * off, so it can't use mutexes. @@ -696,16 +512,6 @@ static int pm_genpd_runtime_suspend(struct device *dev) return 0; mutex_lock(&genpd->lock); - - /* - * If we have an unknown state of the need_restore flag, it means none - * of the runtime PM callbacks has been invoked yet. Let's update the - * flag to reflect that the current state is active. - */ - gpd_data = to_gpd_data(dev->power.subsys_data->domain_data); - if (gpd_data->need_restore < 0) - gpd_data->need_restore = 0; - genpd->in_progress++; pm_genpd_poweroff(genpd); genpd->in_progress--; @@ -725,8 +531,8 @@ static int pm_genpd_runtime_suspend(struct device *dev) static int pm_genpd_runtime_resume(struct device *dev) { struct generic_pm_domain *genpd; - DEFINE_WAIT(wait); int ret; + bool timed = true; dev_dbg(dev, "%s()\n", __func__); @@ -735,40 +541,22 @@ static int pm_genpd_runtime_resume(struct device *dev) return -EINVAL; /* If power.irq_safe, the PM domain is never powered off. */ - if (dev->power.irq_safe) - return genpd_start_dev_no_timing(genpd, dev); + if (dev->power.irq_safe) { + timed = false; + goto out; + } mutex_lock(&genpd->lock); ret = __pm_genpd_poweron(genpd); - if (ret) { - mutex_unlock(&genpd->lock); - return ret; - } - genpd->status = GPD_STATE_BUSY; - genpd->resume_count++; - for (;;) { - prepare_to_wait(&genpd->status_wait_queue, &wait, - TASK_UNINTERRUPTIBLE); - /* - * If current is the powering off task, we have been called - * reentrantly from one of the device callbacks, so we should - * not wait. - */ - if (!genpd->poweroff_task || genpd->poweroff_task == current) - break; - mutex_unlock(&genpd->lock); - - schedule(); - - mutex_lock(&genpd->lock); - } - finish_wait(&genpd->status_wait_queue, &wait); - __pm_genpd_restore_device(dev->power.subsys_data->domain_data, genpd); - genpd->resume_count--; - genpd_set_active(genpd); - wake_up_all(&genpd->status_wait_queue); mutex_unlock(&genpd->lock); + if (ret) + return ret; + + out: + genpd_start_dev(genpd, dev, timed); + genpd_restore_dev(genpd, dev, timed); + return 0; } @@ -883,7 +671,7 @@ static void pm_genpd_sync_poweron(struct generic_pm_domain *genpd, { struct gpd_link *link; - if (genpd->status != GPD_STATE_POWER_OFF) + if (genpd->status == GPD_STATE_ACTIVE) return; list_for_each_entry(link, &genpd->slave_links, slave_node) { @@ -960,14 +748,14 @@ static int pm_genpd_prepare(struct device *dev) if (resume_needed(dev, genpd)) pm_runtime_resume(dev); - genpd_acquire_lock(genpd); + mutex_lock(&genpd->lock); if (genpd->prepared_count++ == 0) { genpd->suspended_count = 0; genpd->suspend_power_off = genpd->status == GPD_STATE_POWER_OFF; } - genpd_release_lock(genpd); + mutex_unlock(&genpd->lock); if (genpd->suspend_power_off) { pm_runtime_put_noidle(dev); @@ -1102,7 +890,7 @@ static int pm_genpd_resume_noirq(struct device *dev) pm_genpd_sync_poweron(genpd, true); genpd->suspended_count--; - return genpd_start_dev(genpd, dev); + return genpd_start_dev(genpd, dev, true); } /** @@ -1230,7 +1018,7 @@ static int pm_genpd_thaw_noirq(struct device *dev) if (IS_ERR(genpd)) return -EINVAL; - return genpd->suspend_power_off ? 0 : genpd_start_dev(genpd, dev); + return genpd->suspend_power_off ? 0 : genpd_start_dev(genpd, dev, true); } /** @@ -1324,7 +1112,7 @@ static int pm_genpd_restore_noirq(struct device *dev) pm_genpd_sync_poweron(genpd, true); - return genpd_start_dev(genpd, dev); + return genpd_start_dev(genpd, dev, true); } /** @@ -1440,7 +1228,6 @@ static struct generic_pm_domain_data *genpd_alloc_dev_data(struct device *dev, gpd_data->td = *td; gpd_data->base.dev = dev; - gpd_data->need_restore = -1; gpd_data->td.constraint_changed = true; gpd_data->td.effective_constraint_ns = -1; gpd_data->nb.notifier_call = genpd_dev_pm_qos_notifier; @@ -1502,7 +1289,7 @@ int __pm_genpd_add_device(struct generic_pm_domain *genpd, struct device *dev, if (IS_ERR(gpd_data)) return PTR_ERR(gpd_data); - genpd_acquire_lock(genpd); + mutex_lock(&genpd->lock); if (genpd->prepared_count > 0) { ret = -EAGAIN; @@ -1519,7 +1306,7 @@ int __pm_genpd_add_device(struct generic_pm_domain *genpd, struct device *dev, list_add_tail(&gpd_data->base.list_node, &genpd->dev_list); out: - genpd_release_lock(genpd); + mutex_unlock(&genpd->lock); if (ret) genpd_free_dev_data(dev, gpd_data); @@ -1563,7 +1350,7 @@ int pm_genpd_remove_device(struct generic_pm_domain *genpd, gpd_data = to_gpd_data(pdd); dev_pm_qos_remove_notifier(dev, &gpd_data->nb); - genpd_acquire_lock(genpd); + mutex_lock(&genpd->lock); if (genpd->prepared_count > 0) { ret = -EAGAIN; @@ -1578,14 +1365,14 @@ int pm_genpd_remove_device(struct generic_pm_domain *genpd, list_del_init(&pdd->list_node); - genpd_release_lock(genpd); + mutex_unlock(&genpd->lock); genpd_free_dev_data(dev, gpd_data); return 0; out: - genpd_release_lock(genpd); + mutex_unlock(&genpd->lock); dev_pm_qos_add_notifier(dev, &gpd_data->nb); return ret; @@ -1606,17 +1393,9 @@ int pm_genpd_add_subdomain(struct generic_pm_domain *genpd, || genpd == subdomain) return -EINVAL; - start: - genpd_acquire_lock(genpd); + mutex_lock(&genpd->lock); mutex_lock_nested(&subdomain->lock, SINGLE_DEPTH_NESTING); - if (subdomain->status != GPD_STATE_POWER_OFF - && subdomain->status != GPD_STATE_ACTIVE) { - mutex_unlock(&subdomain->lock); - genpd_release_lock(genpd); - goto start; - } - if (genpd->status == GPD_STATE_POWER_OFF && subdomain->status != GPD_STATE_POWER_OFF) { ret = -EINVAL; @@ -1644,7 +1423,7 @@ int pm_genpd_add_subdomain(struct generic_pm_domain *genpd, out: mutex_unlock(&subdomain->lock); - genpd_release_lock(genpd); + mutex_unlock(&genpd->lock); return ret; } @@ -1692,8 +1471,14 @@ int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd, if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(subdomain)) return -EINVAL; - start: - genpd_acquire_lock(genpd); + mutex_lock(&genpd->lock); + + if (!list_empty(&subdomain->slave_links) || subdomain->device_count) { + pr_warn("%s: unable to remove subdomain %s\n", genpd->name, + subdomain->name); + ret = -EBUSY; + goto out; + } list_for_each_entry(link, &genpd->master_links, master_node) { if (link->slave != subdomain) @@ -1701,13 +1486,6 @@ int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd, mutex_lock_nested(&subdomain->lock, SINGLE_DEPTH_NESTING); - if (subdomain->status != GPD_STATE_POWER_OFF - && subdomain->status != GPD_STATE_ACTIVE) { - mutex_unlock(&subdomain->lock); - genpd_release_lock(genpd); - goto start; - } - list_del(&link->master_node); list_del(&link->slave_node); kfree(link); @@ -1720,7 +1498,8 @@ int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd, break; } - genpd_release_lock(genpd); +out: + mutex_unlock(&genpd->lock); return ret; } @@ -1744,7 +1523,7 @@ int pm_genpd_attach_cpuidle(struct generic_pm_domain *genpd, int state) if (IS_ERR_OR_NULL(genpd) || state < 0) return -EINVAL; - genpd_acquire_lock(genpd); + mutex_lock(&genpd->lock); if (genpd->cpuidle_data) { ret = -EEXIST; @@ -1775,7 +1554,7 @@ int pm_genpd_attach_cpuidle(struct generic_pm_domain *genpd, int state) genpd_recalc_cpu_exit_latency(genpd); out: - genpd_release_lock(genpd); + mutex_unlock(&genpd->lock); return ret; err: @@ -1812,7 +1591,7 @@ int pm_genpd_detach_cpuidle(struct generic_pm_domain *genpd) if (IS_ERR_OR_NULL(genpd)) return -EINVAL; - genpd_acquire_lock(genpd); + mutex_lock(&genpd->lock); cpuidle_data = genpd->cpuidle_data; if (!cpuidle_data) { @@ -1830,7 +1609,7 @@ int pm_genpd_detach_cpuidle(struct generic_pm_domain *genpd) kfree(cpuidle_data); out: - genpd_release_lock(genpd); + mutex_unlock(&genpd->lock); return ret; } @@ -1912,9 +1691,6 @@ void pm_genpd_init(struct generic_pm_domain *genpd, genpd->in_progress = 0; atomic_set(&genpd->sd_count, 0); genpd->status = is_off ? GPD_STATE_POWER_OFF : GPD_STATE_ACTIVE; - init_waitqueue_head(&genpd->status_wait_queue); - genpd->poweroff_task = NULL; - genpd->resume_count = 0; genpd->device_count = 0; genpd->max_off_time_ns = -1; genpd->max_off_time_changed = true; @@ -1952,6 +1728,7 @@ void pm_genpd_init(struct generic_pm_domain *genpd, list_add(&genpd->gpd_list_node, &gpd_list); mutex_unlock(&gpd_list_lock); } +EXPORT_SYMBOL_GPL(pm_genpd_init); #ifdef CONFIG_PM_GENERIC_DOMAINS_OF /* @@ -2125,7 +1902,7 @@ EXPORT_SYMBOL_GPL(of_genpd_get_from_provider); /** * genpd_dev_pm_detach - Detach a device from its PM domain. - * @dev: Device to attach. + * @dev: Device to detach. * @power_off: Currently not used * * Try to locate a corresponding generic PM domain, which the device was @@ -2183,7 +1960,10 @@ static void genpd_dev_pm_sync(struct device *dev) * Both generic and legacy Samsung-specific DT bindings are supported to keep * backwards compatibility with existing DTBs. * - * Returns 0 on successfully attached PM domain or negative error code. + * Returns 0 on successfully attached PM domain or negative error code. Note + * that if a power-domain exists for the device, but it cannot be found or + * turned on, then return -EPROBE_DEFER to ensure that the device is not + * probed and to re-try again later. */ int genpd_dev_pm_attach(struct device *dev) { @@ -2220,7 +2000,7 @@ int genpd_dev_pm_attach(struct device *dev) dev_dbg(dev, "%s() failed to find PM domain: %ld\n", __func__, PTR_ERR(pd)); of_node_put(dev->of_node); - return PTR_ERR(pd); + return -EPROBE_DEFER; } dev_dbg(dev, "adding to PM domain %s\n", pd->name); @@ -2238,14 +2018,15 @@ int genpd_dev_pm_attach(struct device *dev) dev_err(dev, "failed to add to PM domain %s: %d", pd->name, ret); of_node_put(dev->of_node); - return ret; + goto out; } dev->pm_domain->detach = genpd_dev_pm_detach; dev->pm_domain->sync = genpd_dev_pm_sync; - pm_genpd_poweron(pd); + ret = pm_genpd_poweron(pd); - return 0; +out: + return ret ? -EPROBE_DEFER : 0; } EXPORT_SYMBOL_GPL(genpd_dev_pm_attach); #endif /* CONFIG_PM_GENERIC_DOMAINS_OF */ @@ -2293,9 +2074,6 @@ static int pm_genpd_summary_one(struct seq_file *s, { static const char * const status_lookup[] = { [GPD_STATE_ACTIVE] = "on", - [GPD_STATE_WAIT_MASTER] = "wait-master", - [GPD_STATE_BUSY] = "busy", - [GPD_STATE_REPEAT] = "off-in-progress", [GPD_STATE_POWER_OFF] = "off" }; struct pm_domain_data *pm_data; @@ -2309,7 +2087,7 @@ static int pm_genpd_summary_one(struct seq_file *s, if (WARN_ON(genpd->status >= ARRAY_SIZE(status_lookup))) goto exit; - seq_printf(s, "%-30s %-15s ", genpd->name, status_lookup[genpd->status]); + seq_printf(s, "%-30s %-15s ", genpd->name, status_lookup[genpd->status]); /* * Modifications on the list require holding locks on both @@ -2344,8 +2122,8 @@ static int pm_genpd_summary_show(struct seq_file *s, void *data) struct generic_pm_domain *genpd; int ret = 0; - seq_puts(s, " domain status slaves\n"); - seq_puts(s, " /device runtime status\n"); + seq_puts(s, "domain status slaves\n"); + seq_puts(s, " /device runtime status\n"); seq_puts(s, "----------------------------------------------------------------------\n"); ret = mutex_lock_interruptible(&gpd_list_lock); diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index 30b7bbfdc558..1710c26ba097 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -1377,7 +1377,7 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async) if (dev->power.direct_complete) { if (pm_runtime_status_suspended(dev)) { pm_runtime_disable(dev); - if (pm_runtime_suspended_if_enabled(dev)) + if (pm_runtime_status_suspended(dev)) goto Complete; pm_runtime_enable(dev); diff --git a/drivers/base/power/opp.c b/drivers/base/power/opp.c index 3b188f20b43f..28cd75c535b0 100644 --- a/drivers/base/power/opp.c +++ b/drivers/base/power/opp.c @@ -11,6 +11,7 @@ * published by the Free Software Foundation. */ +#include #include #include #include @@ -51,10 +52,17 @@ * order. * @dynamic: not-created from static DT entries. * @available: true/false - marks if this OPP as available or not + * @turbo: true if turbo (boost) OPP * @rate: Frequency in hertz - * @u_volt: Nominal voltage in microvolts corresponding to this OPP + * @u_volt: Target voltage in microvolts corresponding to this OPP + * @u_volt_min: Minimum voltage in microvolts corresponding to this OPP + * @u_volt_max: Maximum voltage in microvolts corresponding to this OPP + * @u_amp: Maximum current drawn by the device in microamperes + * @clock_latency_ns: Latency (in nanoseconds) of switching to this OPP's + * frequency from any other OPP's frequency. * @dev_opp: points back to the device_opp struct this opp belongs to * @rcu_head: RCU callback head used for deferred freeing + * @np: OPP's device node. * * This structure stores the OPP information for a given device. */ @@ -63,11 +71,34 @@ struct dev_pm_opp { bool available; bool dynamic; + bool turbo; unsigned long rate; + unsigned long u_volt; + unsigned long u_volt_min; + unsigned long u_volt_max; + unsigned long u_amp; + unsigned long clock_latency_ns; struct device_opp *dev_opp; struct rcu_head rcu_head; + + struct device_node *np; +}; + +/** + * struct device_list_opp - devices managed by 'struct device_opp' + * @node: list node + * @dev: device to which the struct object belongs + * @rcu_head: RCU callback head used for deferred freeing + * + * This is an internal data structure maintaining the list of devices that are + * managed by 'struct device_opp'. + */ +struct device_list_opp { + struct list_head node; + const struct device *dev; + struct rcu_head rcu_head; }; /** @@ -77,10 +108,12 @@ struct dev_pm_opp { * list. * RCU usage: nodes are not modified in the list of device_opp, * however addition is possible and is secured by dev_opp_list_lock - * @dev: device pointer * @srcu_head: notifier head to notify the OPP availability changes. * @rcu_head: RCU callback head used for deferred freeing + * @dev_list: list of devices that share these OPPs * @opp_list: list of opps + * @np: struct device_node pointer for opp's DT node. + * @shared_opp: OPP is shared between multiple devices. * * This is an internal data structure maintaining the link to opps attached to * a device. This structure is not meant to be shared to users as it is @@ -93,10 +126,15 @@ struct dev_pm_opp { struct device_opp { struct list_head node; - struct device *dev; struct srcu_notifier_head srcu_head; struct rcu_head rcu_head; + struct list_head dev_list; struct list_head opp_list; + + struct device_node *np; + unsigned long clock_latency_ns_max; + bool shared_opp; + struct dev_pm_opp *suspend_opp; }; /* @@ -116,6 +154,38 @@ do { \ "dev_opp_list_lock protection"); \ } while (0) +static struct device_list_opp *_find_list_dev(const struct device *dev, + struct device_opp *dev_opp) +{ + struct device_list_opp *list_dev; + + list_for_each_entry(list_dev, &dev_opp->dev_list, node) + if (list_dev->dev == dev) + return list_dev; + + return NULL; +} + +static struct device_opp *_managed_opp(const struct device_node *np) +{ + struct device_opp *dev_opp; + + list_for_each_entry_rcu(dev_opp, &dev_opp_list, node) { + if (dev_opp->np == np) { + /* + * Multiple devices can point to the same OPP table and + * so will have same node-pointer, np. + * + * But the OPPs will be considered as shared only if the + * OPP table contains a "opp-shared" property. + */ + return dev_opp->shared_opp ? dev_opp : NULL; + } + } + + return NULL; +} + /** * _find_device_opp() - find device_opp struct using device pointer * @dev: device pointer used to lookup device OPPs @@ -132,21 +202,18 @@ do { \ */ static struct device_opp *_find_device_opp(struct device *dev) { - struct device_opp *tmp_dev_opp, *dev_opp = ERR_PTR(-ENODEV); + struct device_opp *dev_opp; - if (unlikely(IS_ERR_OR_NULL(dev))) { + if (IS_ERR_OR_NULL(dev)) { pr_err("%s: Invalid parameters\n", __func__); return ERR_PTR(-EINVAL); } - list_for_each_entry_rcu(tmp_dev_opp, &dev_opp_list, node) { - if (tmp_dev_opp->dev == dev) { - dev_opp = tmp_dev_opp; - break; - } - } + list_for_each_entry_rcu(dev_opp, &dev_opp_list, node) + if (_find_list_dev(dev, dev_opp)) + return dev_opp; - return dev_opp; + return ERR_PTR(-ENODEV); } /** @@ -172,7 +239,7 @@ unsigned long dev_pm_opp_get_voltage(struct dev_pm_opp *opp) opp_rcu_lockdep_assert(); tmp_opp = rcu_dereference(opp); - if (unlikely(IS_ERR_OR_NULL(tmp_opp)) || !tmp_opp->available) + if (IS_ERR_OR_NULL(tmp_opp) || !tmp_opp->available) pr_err("%s: Invalid parameters\n", __func__); else v = tmp_opp->u_volt; @@ -204,7 +271,7 @@ unsigned long dev_pm_opp_get_freq(struct dev_pm_opp *opp) opp_rcu_lockdep_assert(); tmp_opp = rcu_dereference(opp); - if (unlikely(IS_ERR_OR_NULL(tmp_opp)) || !tmp_opp->available) + if (IS_ERR_OR_NULL(tmp_opp) || !tmp_opp->available) pr_err("%s: Invalid parameters\n", __func__); else f = tmp_opp->rate; @@ -213,6 +280,94 @@ unsigned long dev_pm_opp_get_freq(struct dev_pm_opp *opp) } EXPORT_SYMBOL_GPL(dev_pm_opp_get_freq); +/** + * dev_pm_opp_is_turbo() - Returns if opp is turbo OPP or not + * @opp: opp for which turbo mode is being verified + * + * Turbo OPPs are not for normal use, and can be enabled (under certain + * conditions) for short duration of times to finish high throughput work + * quickly. Running on them for longer times may overheat the chip. + * + * Return: true if opp is turbo opp, else false. + * + * Locking: This function must be called under rcu_read_lock(). opp is a rcu + * protected pointer. This means that opp which could have been fetched by + * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are + * under RCU lock. The pointer returned by the opp_find_freq family must be + * used in the same section as the usage of this function with the pointer + * prior to unlocking with rcu_read_unlock() to maintain the integrity of the + * pointer. + */ +bool dev_pm_opp_is_turbo(struct dev_pm_opp *opp) +{ + struct dev_pm_opp *tmp_opp; + + opp_rcu_lockdep_assert(); + + tmp_opp = rcu_dereference(opp); + if (IS_ERR_OR_NULL(tmp_opp) || !tmp_opp->available) { + pr_err("%s: Invalid parameters\n", __func__); + return false; + } + + return tmp_opp->turbo; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_is_turbo); + +/** + * dev_pm_opp_get_max_clock_latency() - Get max clock latency in nanoseconds + * @dev: device for which we do this operation + * + * Return: This function returns the max clock latency in nanoseconds. + * + * Locking: This function takes rcu_read_lock(). + */ +unsigned long dev_pm_opp_get_max_clock_latency(struct device *dev) +{ + struct device_opp *dev_opp; + unsigned long clock_latency_ns; + + rcu_read_lock(); + + dev_opp = _find_device_opp(dev); + if (IS_ERR(dev_opp)) + clock_latency_ns = 0; + else + clock_latency_ns = dev_opp->clock_latency_ns_max; + + rcu_read_unlock(); + return clock_latency_ns; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_get_max_clock_latency); + +/** + * dev_pm_opp_get_suspend_opp() - Get suspend opp + * @dev: device for which we do this operation + * + * Return: This function returns pointer to the suspend opp if it is + * defined and available, otherwise it returns NULL. + * + * Locking: This function must be called under rcu_read_lock(). opp is a rcu + * protected pointer. The reason for the same is that the opp pointer which is + * returned will remain valid for use with opp_get_{voltage, freq} only while + * under the locked area. The pointer returned must be used prior to unlocking + * with rcu_read_unlock() to maintain the integrity of the pointer. + */ +struct dev_pm_opp *dev_pm_opp_get_suspend_opp(struct device *dev) +{ + struct device_opp *dev_opp; + + opp_rcu_lockdep_assert(); + + dev_opp = _find_device_opp(dev); + if (IS_ERR(dev_opp) || !dev_opp->suspend_opp || + !dev_opp->suspend_opp->available) + return NULL; + + return dev_opp->suspend_opp; +} +EXPORT_SYMBOL_GPL(dev_pm_opp_get_suspend_opp); + /** * dev_pm_opp_get_opp_count() - Get number of opps available in the opp list * @dev: device for which we do this operation @@ -407,18 +562,57 @@ struct dev_pm_opp *dev_pm_opp_find_freq_floor(struct device *dev, } EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_floor); +/* List-dev Helpers */ +static void _kfree_list_dev_rcu(struct rcu_head *head) +{ + struct device_list_opp *list_dev; + + list_dev = container_of(head, struct device_list_opp, rcu_head); + kfree_rcu(list_dev, rcu_head); +} + +static void _remove_list_dev(struct device_list_opp *list_dev, + struct device_opp *dev_opp) +{ + list_del(&list_dev->node); + call_srcu(&dev_opp->srcu_head.srcu, &list_dev->rcu_head, + _kfree_list_dev_rcu); +} + +static struct device_list_opp *_add_list_dev(const struct device *dev, + struct device_opp *dev_opp) +{ + struct device_list_opp *list_dev; + + list_dev = kzalloc(sizeof(*list_dev), GFP_KERNEL); + if (!list_dev) + return NULL; + + /* Initialize list-dev */ + list_dev->dev = dev; + list_add_rcu(&list_dev->node, &dev_opp->dev_list); + + return list_dev; +} + /** - * _add_device_opp() - Allocate a new device OPP table + * _add_device_opp() - Find device OPP table or allocate a new one * @dev: device for which we do this operation * - * New device node which uses OPPs - used when multiple devices with OPP tables - * are maintained. + * It tries to find an existing table first, if it couldn't find one, it + * allocates a new OPP table and returns that. * * Return: valid device_opp pointer if success, else NULL. */ static struct device_opp *_add_device_opp(struct device *dev) { struct device_opp *dev_opp; + struct device_list_opp *list_dev; + + /* Check for existing list for 'dev' first */ + dev_opp = _find_device_opp(dev); + if (!IS_ERR(dev_opp)) + return dev_opp; /* * Allocate a new device OPP table. In the infrequent case where a new @@ -428,7 +622,14 @@ static struct device_opp *_add_device_opp(struct device *dev) if (!dev_opp) return NULL; - dev_opp->dev = dev; + INIT_LIST_HEAD(&dev_opp->dev_list); + + list_dev = _add_list_dev(dev, dev_opp); + if (!list_dev) { + kfree(dev_opp); + return NULL; + } + srcu_init_notifier_head(&dev_opp->srcu_head); INIT_LIST_HEAD(&dev_opp->opp_list); @@ -437,149 +638,6 @@ static struct device_opp *_add_device_opp(struct device *dev) return dev_opp; } -/** - * _opp_add_dynamic() - Allocate a dynamic OPP. - * @dev: device for which we do this operation - * @freq: Frequency in Hz for this OPP - * @u_volt: Voltage in uVolts for this OPP - * @dynamic: Dynamically added OPPs. - * - * This function adds an opp definition to the opp list and returns status. - * The opp is made available by default and it can be controlled using - * dev_pm_opp_enable/disable functions and may be removed by dev_pm_opp_remove. - * - * NOTE: "dynamic" parameter impacts OPPs added by the of_init_opp_table and - * freed by of_free_opp_table. - * - * Locking: The internal device_opp and opp structures are RCU protected. - * Hence this function internally uses RCU updater strategy with mutex locks - * to keep the integrity of the internal data structures. Callers should ensure - * that this function is *NOT* called under RCU protection or in contexts where - * mutex cannot be locked. - * - * Return: - * 0 On success OR - * Duplicate OPPs (both freq and volt are same) and opp->available - * -EEXIST Freq are same and volt are different OR - * Duplicate OPPs (both freq and volt are same) and !opp->available - * -ENOMEM Memory allocation failure - */ -static int _opp_add_dynamic(struct device *dev, unsigned long freq, - long u_volt, bool dynamic) -{ - struct device_opp *dev_opp = NULL; - struct dev_pm_opp *opp, *new_opp; - struct list_head *head; - int ret; - - /* allocate new OPP node */ - new_opp = kzalloc(sizeof(*new_opp), GFP_KERNEL); - if (!new_opp) - return -ENOMEM; - - /* Hold our list modification lock here */ - mutex_lock(&dev_opp_list_lock); - - /* populate the opp table */ - new_opp->rate = freq; - new_opp->u_volt = u_volt; - new_opp->available = true; - new_opp->dynamic = dynamic; - - /* Check for existing list for 'dev' */ - dev_opp = _find_device_opp(dev); - if (IS_ERR(dev_opp)) { - dev_opp = _add_device_opp(dev); - if (!dev_opp) { - ret = -ENOMEM; - goto free_opp; - } - - head = &dev_opp->opp_list; - goto list_add; - } - - /* - * Insert new OPP in order of increasing frequency - * and discard if already present - */ - head = &dev_opp->opp_list; - list_for_each_entry_rcu(opp, &dev_opp->opp_list, node) { - if (new_opp->rate <= opp->rate) - break; - else - head = &opp->node; - } - - /* Duplicate OPPs ? */ - if (new_opp->rate == opp->rate) { - ret = opp->available && new_opp->u_volt == opp->u_volt ? - 0 : -EEXIST; - - dev_warn(dev, "%s: duplicate OPPs detected. Existing: freq: %lu, volt: %lu, enabled: %d. New: freq: %lu, volt: %lu, enabled: %d\n", - __func__, opp->rate, opp->u_volt, opp->available, - new_opp->rate, new_opp->u_volt, new_opp->available); - goto free_opp; - } - -list_add: - new_opp->dev_opp = dev_opp; - list_add_rcu(&new_opp->node, head); - mutex_unlock(&dev_opp_list_lock); - - /* - * Notify the changes in the availability of the operable - * frequency/voltage list. - */ - srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_ADD, new_opp); - return 0; - -free_opp: - mutex_unlock(&dev_opp_list_lock); - kfree(new_opp); - return ret; -} - -/** - * dev_pm_opp_add() - Add an OPP table from a table definitions - * @dev: device for which we do this operation - * @freq: Frequency in Hz for this OPP - * @u_volt: Voltage in uVolts for this OPP - * - * This function adds an opp definition to the opp list and returns status. - * The opp is made available by default and it can be controlled using - * dev_pm_opp_enable/disable functions. - * - * Locking: The internal device_opp and opp structures are RCU protected. - * Hence this function internally uses RCU updater strategy with mutex locks - * to keep the integrity of the internal data structures. Callers should ensure - * that this function is *NOT* called under RCU protection or in contexts where - * mutex cannot be locked. - * - * Return: - * 0 On success OR - * Duplicate OPPs (both freq and volt are same) and opp->available - * -EEXIST Freq are same and volt are different OR - * Duplicate OPPs (both freq and volt are same) and !opp->available - * -ENOMEM Memory allocation failure - */ -int dev_pm_opp_add(struct device *dev, unsigned long freq, unsigned long u_volt) -{ - return _opp_add_dynamic(dev, freq, u_volt, true); -} -EXPORT_SYMBOL_GPL(dev_pm_opp_add); - -/** - * _kfree_opp_rcu() - Free OPP RCU handler - * @head: RCU head - */ -static void _kfree_opp_rcu(struct rcu_head *head) -{ - struct dev_pm_opp *opp = container_of(head, struct dev_pm_opp, rcu_head); - - kfree_rcu(opp, rcu_head); -} - /** * _kfree_device_rcu() - Free device_opp RCU handler * @head: RCU head @@ -591,10 +649,48 @@ static void _kfree_device_rcu(struct rcu_head *head) kfree_rcu(device_opp, rcu_head); } +/** + * _remove_device_opp() - Removes a device OPP table + * @dev_opp: device OPP table to be removed. + * + * Removes/frees device OPP table it it doesn't contain any OPPs. + */ +static void _remove_device_opp(struct device_opp *dev_opp) +{ + struct device_list_opp *list_dev; + + if (!list_empty(&dev_opp->opp_list)) + return; + + list_dev = list_first_entry(&dev_opp->dev_list, struct device_list_opp, + node); + + _remove_list_dev(list_dev, dev_opp); + + /* dev_list must be empty now */ + WARN_ON(!list_empty(&dev_opp->dev_list)); + + list_del_rcu(&dev_opp->node); + call_srcu(&dev_opp->srcu_head.srcu, &dev_opp->rcu_head, + _kfree_device_rcu); +} + +/** + * _kfree_opp_rcu() - Free OPP RCU handler + * @head: RCU head + */ +static void _kfree_opp_rcu(struct rcu_head *head) +{ + struct dev_pm_opp *opp = container_of(head, struct dev_pm_opp, rcu_head); + + kfree_rcu(opp, rcu_head); +} + /** * _opp_remove() - Remove an OPP from a table definition * @dev_opp: points back to the device_opp struct this opp belongs to * @opp: pointer to the OPP to remove + * @notify: OPP_EVENT_REMOVE notification should be sent or not * * This function removes an opp definition from the opp list. * @@ -603,21 +699,18 @@ static void _kfree_device_rcu(struct rcu_head *head) * strategy. */ static void _opp_remove(struct device_opp *dev_opp, - struct dev_pm_opp *opp) + struct dev_pm_opp *opp, bool notify) { /* * Notify the changes in the availability of the operable * frequency/voltage list. */ - srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_REMOVE, opp); + if (notify) + srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_REMOVE, opp); list_del_rcu(&opp->node); call_srcu(&dev_opp->srcu_head.srcu, &opp->rcu_head, _kfree_opp_rcu); - if (list_empty(&dev_opp->opp_list)) { - list_del_rcu(&dev_opp->node); - call_srcu(&dev_opp->srcu_head.srcu, &dev_opp->rcu_head, - _kfree_device_rcu); - } + _remove_device_opp(dev_opp); } /** @@ -659,12 +752,307 @@ void dev_pm_opp_remove(struct device *dev, unsigned long freq) goto unlock; } - _opp_remove(dev_opp, opp); + _opp_remove(dev_opp, opp, true); unlock: mutex_unlock(&dev_opp_list_lock); } EXPORT_SYMBOL_GPL(dev_pm_opp_remove); +static struct dev_pm_opp *_allocate_opp(struct device *dev, + struct device_opp **dev_opp) +{ + struct dev_pm_opp *opp; + + /* allocate new OPP node */ + opp = kzalloc(sizeof(*opp), GFP_KERNEL); + if (!opp) + return NULL; + + INIT_LIST_HEAD(&opp->node); + + *dev_opp = _add_device_opp(dev); + if (!*dev_opp) { + kfree(opp); + return NULL; + } + + return opp; +} + +static int _opp_add(struct device *dev, struct dev_pm_opp *new_opp, + struct device_opp *dev_opp) +{ + struct dev_pm_opp *opp; + struct list_head *head = &dev_opp->opp_list; + + /* + * Insert new OPP in order of increasing frequency and discard if + * already present. + * + * Need to use &dev_opp->opp_list in the condition part of the 'for' + * loop, don't replace it with head otherwise it will become an infinite + * loop. + */ + list_for_each_entry_rcu(opp, &dev_opp->opp_list, node) { + if (new_opp->rate > opp->rate) { + head = &opp->node; + continue; + } + + if (new_opp->rate < opp->rate) + break; + + /* Duplicate OPPs */ + dev_warn(dev, "%s: duplicate OPPs detected. Existing: freq: %lu, volt: %lu, enabled: %d. New: freq: %lu, volt: %lu, enabled: %d\n", + __func__, opp->rate, opp->u_volt, opp->available, + new_opp->rate, new_opp->u_volt, new_opp->available); + + return opp->available && new_opp->u_volt == opp->u_volt ? + 0 : -EEXIST; + } + + new_opp->dev_opp = dev_opp; + list_add_rcu(&new_opp->node, head); + + return 0; +} + +/** + * _opp_add_dynamic() - Allocate a dynamic OPP. + * @dev: device for which we do this operation + * @freq: Frequency in Hz for this OPP + * @u_volt: Voltage in uVolts for this OPP + * @dynamic: Dynamically added OPPs. + * + * This function adds an opp definition to the opp list and returns status. + * The opp is made available by default and it can be controlled using + * dev_pm_opp_enable/disable functions and may be removed by dev_pm_opp_remove. + * + * NOTE: "dynamic" parameter impacts OPPs added by the of_init_opp_table and + * freed by of_free_opp_table. + * + * Locking: The internal device_opp and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + * + * Return: + * 0 On success OR + * Duplicate OPPs (both freq and volt are same) and opp->available + * -EEXIST Freq are same and volt are different OR + * Duplicate OPPs (both freq and volt are same) and !opp->available + * -ENOMEM Memory allocation failure + */ +static int _opp_add_dynamic(struct device *dev, unsigned long freq, + long u_volt, bool dynamic) +{ + struct device_opp *dev_opp; + struct dev_pm_opp *new_opp; + int ret; + + /* Hold our list modification lock here */ + mutex_lock(&dev_opp_list_lock); + + new_opp = _allocate_opp(dev, &dev_opp); + if (!new_opp) { + ret = -ENOMEM; + goto unlock; + } + + /* populate the opp table */ + new_opp->rate = freq; + new_opp->u_volt = u_volt; + new_opp->available = true; + new_opp->dynamic = dynamic; + + ret = _opp_add(dev, new_opp, dev_opp); + if (ret) + goto free_opp; + + mutex_unlock(&dev_opp_list_lock); + + /* + * Notify the changes in the availability of the operable + * frequency/voltage list. + */ + srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_ADD, new_opp); + return 0; + +free_opp: + _opp_remove(dev_opp, new_opp, false); +unlock: + mutex_unlock(&dev_opp_list_lock); + return ret; +} + +/* TODO: Support multiple regulators */ +static int opp_get_microvolt(struct dev_pm_opp *opp, struct device *dev) +{ + u32 microvolt[3] = {0}; + int count, ret; + + count = of_property_count_u32_elems(opp->np, "opp-microvolt"); + if (!count) + return 0; + + /* There can be one or three elements here */ + if (count != 1 && count != 3) { + dev_err(dev, "%s: Invalid number of elements in opp-microvolt property (%d)\n", + __func__, count); + return -EINVAL; + } + + ret = of_property_read_u32_array(opp->np, "opp-microvolt", microvolt, + count); + if (ret) { + dev_err(dev, "%s: error parsing opp-microvolt: %d\n", __func__, + ret); + return -EINVAL; + } + + opp->u_volt = microvolt[0]; + opp->u_volt_min = microvolt[1]; + opp->u_volt_max = microvolt[2]; + + return 0; +} + +/** + * _opp_add_static_v2() - Allocate static OPPs (As per 'v2' DT bindings) + * @dev: device for which we do this operation + * @np: device node + * + * This function adds an opp definition to the opp list and returns status. The + * opp can be controlled using dev_pm_opp_enable/disable functions and may be + * removed by dev_pm_opp_remove. + * + * Locking: The internal device_opp and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + * + * Return: + * 0 On success OR + * Duplicate OPPs (both freq and volt are same) and opp->available + * -EEXIST Freq are same and volt are different OR + * Duplicate OPPs (both freq and volt are same) and !opp->available + * -ENOMEM Memory allocation failure + * -EINVAL Failed parsing the OPP node + */ +static int _opp_add_static_v2(struct device *dev, struct device_node *np) +{ + struct device_opp *dev_opp; + struct dev_pm_opp *new_opp; + u64 rate; + u32 val; + int ret; + + /* Hold our list modification lock here */ + mutex_lock(&dev_opp_list_lock); + + new_opp = _allocate_opp(dev, &dev_opp); + if (!new_opp) { + ret = -ENOMEM; + goto unlock; + } + + ret = of_property_read_u64(np, "opp-hz", &rate); + if (ret < 0) { + dev_err(dev, "%s: opp-hz not found\n", __func__); + goto free_opp; + } + + /* + * Rate is defined as an unsigned long in clk API, and so casting + * explicitly to its type. Must be fixed once rate is 64 bit + * guaranteed in clk API. + */ + new_opp->rate = (unsigned long)rate; + new_opp->turbo = of_property_read_bool(np, "turbo-mode"); + + new_opp->np = np; + new_opp->dynamic = false; + new_opp->available = true; + + if (!of_property_read_u32(np, "clock-latency-ns", &val)) + new_opp->clock_latency_ns = val; + + ret = opp_get_microvolt(new_opp, dev); + if (ret) + goto free_opp; + + if (!of_property_read_u32(new_opp->np, "opp-microamp", &val)) + new_opp->u_amp = val; + + ret = _opp_add(dev, new_opp, dev_opp); + if (ret) + goto free_opp; + + /* OPP to select on device suspend */ + if (of_property_read_bool(np, "opp-suspend")) { + if (dev_opp->suspend_opp) + dev_warn(dev, "%s: Multiple suspend OPPs found (%lu %lu)\n", + __func__, dev_opp->suspend_opp->rate, + new_opp->rate); + else + dev_opp->suspend_opp = new_opp; + } + + if (new_opp->clock_latency_ns > dev_opp->clock_latency_ns_max) + dev_opp->clock_latency_ns_max = new_opp->clock_latency_ns; + + mutex_unlock(&dev_opp_list_lock); + + pr_debug("%s: turbo:%d rate:%lu uv:%lu uvmin:%lu uvmax:%lu latency:%lu\n", + __func__, new_opp->turbo, new_opp->rate, new_opp->u_volt, + new_opp->u_volt_min, new_opp->u_volt_max, + new_opp->clock_latency_ns); + + /* + * Notify the changes in the availability of the operable + * frequency/voltage list. + */ + srcu_notifier_call_chain(&dev_opp->srcu_head, OPP_EVENT_ADD, new_opp); + return 0; + +free_opp: + _opp_remove(dev_opp, new_opp, false); +unlock: + mutex_unlock(&dev_opp_list_lock); + return ret; +} + +/** + * dev_pm_opp_add() - Add an OPP table from a table definitions + * @dev: device for which we do this operation + * @freq: Frequency in Hz for this OPP + * @u_volt: Voltage in uVolts for this OPP + * + * This function adds an opp definition to the opp list and returns status. + * The opp is made available by default and it can be controlled using + * dev_pm_opp_enable/disable functions. + * + * Locking: The internal device_opp and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + * + * Return: + * 0 On success OR + * Duplicate OPPs (both freq and volt are same) and opp->available + * -EEXIST Freq are same and volt are different OR + * Duplicate OPPs (both freq and volt are same) and !opp->available + * -ENOMEM Memory allocation failure + */ +int dev_pm_opp_add(struct device *dev, unsigned long freq, unsigned long u_volt) +{ + return _opp_add_dynamic(dev, freq, u_volt, true); +} +EXPORT_SYMBOL_GPL(dev_pm_opp_add); + /** * _opp_set_availability() - helper to set the availability of an opp * @dev: device for which we do this operation @@ -825,28 +1213,179 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_get_notifier); #ifdef CONFIG_OF /** - * of_init_opp_table() - Initialize opp table from device tree + * of_free_opp_table() - Free OPP table entries created from static DT entries * @dev: device pointer used to lookup device OPPs. * - * Register the initial OPP table with the OPP library for given device. + * Free OPPs created using static entries present in DT. * * Locking: The internal device_opp and opp structures are RCU protected. * Hence this function indirectly uses RCU updater strategy with mutex locks * to keep the integrity of the internal data structures. Callers should ensure * that this function is *NOT* called under RCU protection or in contexts where * mutex cannot be locked. - * - * Return: - * 0 On success OR - * Duplicate OPPs (both freq and volt are same) and opp->available - * -EEXIST Freq are same and volt are different OR - * Duplicate OPPs (both freq and volt are same) and !opp->available - * -ENOMEM Memory allocation failure - * -ENODEV when 'operating-points' property is not found or is invalid data - * in device node. - * -ENODATA when empty 'operating-points' property is found */ -int of_init_opp_table(struct device *dev) +void of_free_opp_table(struct device *dev) +{ + struct device_opp *dev_opp; + struct dev_pm_opp *opp, *tmp; + + /* Hold our list modification lock here */ + mutex_lock(&dev_opp_list_lock); + + /* Check for existing list for 'dev' */ + dev_opp = _find_device_opp(dev); + if (IS_ERR(dev_opp)) { + int error = PTR_ERR(dev_opp); + + if (error != -ENODEV) + WARN(1, "%s: dev_opp: %d\n", + IS_ERR_OR_NULL(dev) ? + "Invalid device" : dev_name(dev), + error); + goto unlock; + } + + /* Find if dev_opp manages a single device */ + if (list_is_singular(&dev_opp->dev_list)) { + /* Free static OPPs */ + list_for_each_entry_safe(opp, tmp, &dev_opp->opp_list, node) { + if (!opp->dynamic) + _opp_remove(dev_opp, opp, true); + } + } else { + _remove_list_dev(_find_list_dev(dev, dev_opp), dev_opp); + } + +unlock: + mutex_unlock(&dev_opp_list_lock); +} +EXPORT_SYMBOL_GPL(of_free_opp_table); + +void of_cpumask_free_opp_table(cpumask_var_t cpumask) +{ + struct device *cpu_dev; + int cpu; + + WARN_ON(cpumask_empty(cpumask)); + + for_each_cpu(cpu, cpumask) { + cpu_dev = get_cpu_device(cpu); + if (!cpu_dev) { + pr_err("%s: failed to get cpu%d device\n", __func__, + cpu); + continue; + } + + of_free_opp_table(cpu_dev); + } +} +EXPORT_SYMBOL_GPL(of_cpumask_free_opp_table); + +/* Returns opp descriptor node from its phandle. Caller must do of_node_put() */ +static struct device_node * +_of_get_opp_desc_node_from_prop(struct device *dev, const struct property *prop) +{ + struct device_node *opp_np; + + opp_np = of_find_node_by_phandle(be32_to_cpup(prop->value)); + if (!opp_np) { + dev_err(dev, "%s: Prop: %s contains invalid opp desc phandle\n", + __func__, prop->name); + return ERR_PTR(-EINVAL); + } + + return opp_np; +} + +/* Returns opp descriptor node for a device. Caller must do of_node_put() */ +static struct device_node *_of_get_opp_desc_node(struct device *dev) +{ + const struct property *prop; + + prop = of_find_property(dev->of_node, "operating-points-v2", NULL); + if (!prop) + return ERR_PTR(-ENODEV); + if (!prop->value) + return ERR_PTR(-ENODATA); + + /* + * TODO: Support for multiple OPP tables. + * + * There should be only ONE phandle present in "operating-points-v2" + * property. + */ + if (prop->length != sizeof(__be32)) { + dev_err(dev, "%s: Invalid opp desc phandle\n", __func__); + return ERR_PTR(-EINVAL); + } + + return _of_get_opp_desc_node_from_prop(dev, prop); +} + +/* Initializes OPP tables based on new bindings */ +static int _of_init_opp_table_v2(struct device *dev, + const struct property *prop) +{ + struct device_node *opp_np, *np; + struct device_opp *dev_opp; + int ret = 0, count = 0; + + if (!prop->value) + return -ENODATA; + + /* Get opp node */ + opp_np = _of_get_opp_desc_node_from_prop(dev, prop); + if (IS_ERR(opp_np)) + return PTR_ERR(opp_np); + + dev_opp = _managed_opp(opp_np); + if (dev_opp) { + /* OPPs are already managed */ + if (!_add_list_dev(dev, dev_opp)) + ret = -ENOMEM; + goto put_opp_np; + } + + /* We have opp-list node now, iterate over it and add OPPs */ + for_each_available_child_of_node(opp_np, np) { + count++; + + ret = _opp_add_static_v2(dev, np); + if (ret) { + dev_err(dev, "%s: Failed to add OPP, %d\n", __func__, + ret); + goto free_table; + } + } + + /* There should be one of more OPP defined */ + if (WARN_ON(!count)) { + ret = -ENOENT; + goto put_opp_np; + } + + dev_opp = _find_device_opp(dev); + if (WARN_ON(IS_ERR(dev_opp))) { + ret = PTR_ERR(dev_opp); + goto free_table; + } + + dev_opp->np = opp_np; + dev_opp->shared_opp = of_property_read_bool(opp_np, "opp-shared"); + + of_node_put(opp_np); + return 0; + +free_table: + of_free_opp_table(dev); +put_opp_np: + of_node_put(opp_np); + + return ret; +} + +/* Initializes OPP tables based on old-deprecated bindings */ +static int _of_init_opp_table_v1(struct device *dev) { const struct property *prop; const __be32 *val; @@ -881,47 +1420,177 @@ int of_init_opp_table(struct device *dev) return 0; } -EXPORT_SYMBOL_GPL(of_init_opp_table); /** - * of_free_opp_table() - Free OPP table entries created from static DT entries + * of_init_opp_table() - Initialize opp table from device tree * @dev: device pointer used to lookup device OPPs. * - * Free OPPs created using static entries present in DT. + * Register the initial OPP table with the OPP library for given device. * * Locking: The internal device_opp and opp structures are RCU protected. * Hence this function indirectly uses RCU updater strategy with mutex locks * to keep the integrity of the internal data structures. Callers should ensure * that this function is *NOT* called under RCU protection or in contexts where * mutex cannot be locked. + * + * Return: + * 0 On success OR + * Duplicate OPPs (both freq and volt are same) and opp->available + * -EEXIST Freq are same and volt are different OR + * Duplicate OPPs (both freq and volt are same) and !opp->available + * -ENOMEM Memory allocation failure + * -ENODEV when 'operating-points' property is not found or is invalid data + * in device node. + * -ENODATA when empty 'operating-points' property is found + * -EINVAL when invalid entries are found in opp-v2 table */ -void of_free_opp_table(struct device *dev) +int of_init_opp_table(struct device *dev) { - struct device_opp *dev_opp; - struct dev_pm_opp *opp, *tmp; + const struct property *prop; - /* Check for existing list for 'dev' */ - dev_opp = _find_device_opp(dev); - if (IS_ERR(dev_opp)) { - int error = PTR_ERR(dev_opp); - if (error != -ENODEV) - WARN(1, "%s: dev_opp: %d\n", - IS_ERR_OR_NULL(dev) ? - "Invalid device" : dev_name(dev), - error); - return; + /* + * OPPs have two version of bindings now. The older one is deprecated, + * try for the new binding first. + */ + prop = of_find_property(dev->of_node, "operating-points-v2", NULL); + if (!prop) { + /* + * Try old-deprecated bindings for backward compatibility with + * older dtbs. + */ + return _of_init_opp_table_v1(dev); } - /* Hold our list modification lock here */ - mutex_lock(&dev_opp_list_lock); - - /* Free static OPPs */ - list_for_each_entry_safe(opp, tmp, &dev_opp->opp_list, node) { - if (!opp->dynamic) - _opp_remove(dev_opp, opp); - } - - mutex_unlock(&dev_opp_list_lock); + return _of_init_opp_table_v2(dev, prop); } -EXPORT_SYMBOL_GPL(of_free_opp_table); +EXPORT_SYMBOL_GPL(of_init_opp_table); + +int of_cpumask_init_opp_table(cpumask_var_t cpumask) +{ + struct device *cpu_dev; + int cpu, ret = 0; + + WARN_ON(cpumask_empty(cpumask)); + + for_each_cpu(cpu, cpumask) { + cpu_dev = get_cpu_device(cpu); + if (!cpu_dev) { + pr_err("%s: failed to get cpu%d device\n", __func__, + cpu); + continue; + } + + ret = of_init_opp_table(cpu_dev); + if (ret) { + pr_err("%s: couldn't find opp table for cpu:%d, %d\n", + __func__, cpu, ret); + + /* Free all other OPPs */ + of_cpumask_free_opp_table(cpumask); + break; + } + } + + return ret; +} +EXPORT_SYMBOL_GPL(of_cpumask_init_opp_table); + +/* Required only for V1 bindings, as v2 can manage it from DT itself */ +int set_cpus_sharing_opps(struct device *cpu_dev, cpumask_var_t cpumask) +{ + struct device_list_opp *list_dev; + struct device_opp *dev_opp; + struct device *dev; + int cpu, ret = 0; + + rcu_read_lock(); + + dev_opp = _find_device_opp(cpu_dev); + if (IS_ERR(dev_opp)) { + ret = -EINVAL; + goto out_rcu_read_unlock; + } + + for_each_cpu(cpu, cpumask) { + if (cpu == cpu_dev->id) + continue; + + dev = get_cpu_device(cpu); + if (!dev) { + dev_err(cpu_dev, "%s: failed to get cpu%d device\n", + __func__, cpu); + continue; + } + + list_dev = _add_list_dev(dev, dev_opp); + if (!list_dev) { + dev_err(dev, "%s: failed to add list-dev for cpu%d device\n", + __func__, cpu); + continue; + } + } +out_rcu_read_unlock: + rcu_read_unlock(); + + return 0; +} +EXPORT_SYMBOL_GPL(set_cpus_sharing_opps); + +/* + * Works only for OPP v2 bindings. + * + * cpumask should be already set to mask of cpu_dev->id. + * Returns -ENOENT if operating-points-v2 bindings aren't supported. + */ +int of_get_cpus_sharing_opps(struct device *cpu_dev, cpumask_var_t cpumask) +{ + struct device_node *np, *tmp_np; + struct device *tcpu_dev; + int cpu, ret = 0; + + /* Get OPP descriptor node */ + np = _of_get_opp_desc_node(cpu_dev); + if (IS_ERR(np)) { + dev_dbg(cpu_dev, "%s: Couldn't find opp node: %ld\n", __func__, + PTR_ERR(np)); + return -ENOENT; + } + + /* OPPs are shared ? */ + if (!of_property_read_bool(np, "opp-shared")) + goto put_cpu_node; + + for_each_possible_cpu(cpu) { + if (cpu == cpu_dev->id) + continue; + + tcpu_dev = get_cpu_device(cpu); + if (!tcpu_dev) { + dev_err(cpu_dev, "%s: failed to get cpu%d device\n", + __func__, cpu); + ret = -ENODEV; + goto put_cpu_node; + } + + /* Get OPP descriptor node */ + tmp_np = _of_get_opp_desc_node(tcpu_dev); + if (IS_ERR(tmp_np)) { + dev_err(tcpu_dev, "%s: Couldn't find opp node: %ld\n", + __func__, PTR_ERR(tmp_np)); + ret = PTR_ERR(tmp_np); + goto put_cpu_node; + } + + /* CPUs are sharing opp node */ + if (np == tmp_np) + cpumask_set_cpu(cpu, cpumask); + + of_node_put(tmp_np); + } + +put_cpu_node: + of_node_put(np); + return ret; +} +EXPORT_SYMBOL_GPL(of_get_cpus_sharing_opps); #endif diff --git a/drivers/base/power/power.h b/drivers/base/power/power.h index f1a5d95e7b20..998fa6b23084 100644 --- a/drivers/base/power/power.h +++ b/drivers/base/power/power.h @@ -73,6 +73,8 @@ extern int pm_qos_sysfs_add_resume_latency(struct device *dev); extern void pm_qos_sysfs_remove_resume_latency(struct device *dev); extern int pm_qos_sysfs_add_flags(struct device *dev); extern void pm_qos_sysfs_remove_flags(struct device *dev); +extern int pm_qos_sysfs_add_latency_tolerance(struct device *dev); +extern void pm_qos_sysfs_remove_latency_tolerance(struct device *dev); #else /* CONFIG_PM */ diff --git a/drivers/base/power/qos.c b/drivers/base/power/qos.c index e56d538d039e..7f3646e459cb 100644 --- a/drivers/base/power/qos.c +++ b/drivers/base/power/qos.c @@ -883,3 +883,40 @@ int dev_pm_qos_update_user_latency_tolerance(struct device *dev, s32 val) mutex_unlock(&dev_pm_qos_mtx); return ret; } + +/** + * dev_pm_qos_expose_latency_tolerance - Expose latency tolerance to userspace + * @dev: Device whose latency tolerance to expose + */ +int dev_pm_qos_expose_latency_tolerance(struct device *dev) +{ + int ret; + + if (!dev->power.set_latency_tolerance) + return -EINVAL; + + mutex_lock(&dev_pm_qos_sysfs_mtx); + ret = pm_qos_sysfs_add_latency_tolerance(dev); + mutex_unlock(&dev_pm_qos_sysfs_mtx); + + return ret; +} +EXPORT_SYMBOL_GPL(dev_pm_qos_expose_latency_tolerance); + +/** + * dev_pm_qos_hide_latency_tolerance - Hide latency tolerance from userspace + * @dev: Device whose latency tolerance to hide + */ +void dev_pm_qos_hide_latency_tolerance(struct device *dev) +{ + mutex_lock(&dev_pm_qos_sysfs_mtx); + pm_qos_sysfs_remove_latency_tolerance(dev); + mutex_unlock(&dev_pm_qos_sysfs_mtx); + + /* Remove the request from user space now */ + pm_runtime_get_sync(dev); + dev_pm_qos_update_user_latency_tolerance(dev, + PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT); + pm_runtime_put(dev); +} +EXPORT_SYMBOL_GPL(dev_pm_qos_hide_latency_tolerance); diff --git a/drivers/base/power/sysfs.c b/drivers/base/power/sysfs.c index d2be3f9c211c..a7b46798c81d 100644 --- a/drivers/base/power/sysfs.c +++ b/drivers/base/power/sysfs.c @@ -738,6 +738,17 @@ void pm_qos_sysfs_remove_flags(struct device *dev) sysfs_unmerge_group(&dev->kobj, &pm_qos_flags_attr_group); } +int pm_qos_sysfs_add_latency_tolerance(struct device *dev) +{ + return sysfs_merge_group(&dev->kobj, + &pm_qos_latency_tolerance_attr_group); +} + +void pm_qos_sysfs_remove_latency_tolerance(struct device *dev) +{ + sysfs_unmerge_group(&dev->kobj, &pm_qos_latency_tolerance_attr_group); +} + void rpm_sysfs_remove(struct device *dev) { sysfs_unmerge_group(&dev->kobj, &pm_runtime_attr_group); diff --git a/drivers/base/property.c b/drivers/base/property.c index f3f6d167f3f1..2d75366c61e0 100644 --- a/drivers/base/property.c +++ b/drivers/base/property.c @@ -16,6 +16,8 @@ #include #include #include +#include +#include /** * device_add_property_set - Add a collection of properties to a device object. @@ -27,9 +29,10 @@ */ void device_add_property_set(struct device *dev, struct property_set *pset) { - if (pset) - pset->fwnode.type = FWNODE_PDATA; + if (!pset) + return; + pset->fwnode.type = FWNODE_PDATA; set_secondary_fwnode(dev, &pset->fwnode); } EXPORT_SYMBOL_GPL(device_add_property_set); @@ -153,6 +156,7 @@ EXPORT_SYMBOL_GPL(fwnode_property_present); * %-ENODATA if the property does not have a value, * %-EPROTO if the property is not an array of numbers, * %-EOVERFLOW if the size of the property is not as expected. + * %-ENXIO if no suitable firmware interface is present. */ int device_property_read_u8_array(struct device *dev, const char *propname, u8 *val, size_t nval) @@ -177,6 +181,7 @@ EXPORT_SYMBOL_GPL(device_property_read_u8_array); * %-ENODATA if the property does not have a value, * %-EPROTO if the property is not an array of numbers, * %-EOVERFLOW if the size of the property is not as expected. + * %-ENXIO if no suitable firmware interface is present. */ int device_property_read_u16_array(struct device *dev, const char *propname, u16 *val, size_t nval) @@ -201,6 +206,7 @@ EXPORT_SYMBOL_GPL(device_property_read_u16_array); * %-ENODATA if the property does not have a value, * %-EPROTO if the property is not an array of numbers, * %-EOVERFLOW if the size of the property is not as expected. + * %-ENXIO if no suitable firmware interface is present. */ int device_property_read_u32_array(struct device *dev, const char *propname, u32 *val, size_t nval) @@ -225,6 +231,7 @@ EXPORT_SYMBOL_GPL(device_property_read_u32_array); * %-ENODATA if the property does not have a value, * %-EPROTO if the property is not an array of numbers, * %-EOVERFLOW if the size of the property is not as expected. + * %-ENXIO if no suitable firmware interface is present. */ int device_property_read_u64_array(struct device *dev, const char *propname, u64 *val, size_t nval) @@ -249,6 +256,7 @@ EXPORT_SYMBOL_GPL(device_property_read_u64_array); * %-ENODATA if the property does not have a value, * %-EPROTO or %-EILSEQ if the property is not an array of strings, * %-EOVERFLOW if the size of the property is not as expected. + * %-ENXIO if no suitable firmware interface is present. */ int device_property_read_string_array(struct device *dev, const char *propname, const char **val, size_t nval) @@ -270,6 +278,7 @@ EXPORT_SYMBOL_GPL(device_property_read_string_array); * %-EINVAL if given arguments are not valid, * %-ENODATA if the property does not have a value, * %-EPROTO or %-EILSEQ if the property type is not a string. + * %-ENXIO if no suitable firmware interface is present. */ int device_property_read_string(struct device *dev, const char *propname, const char **val) @@ -291,9 +300,11 @@ EXPORT_SYMBOL_GPL(device_property_read_string); else if (is_acpi_node(_fwnode_)) \ _ret_ = acpi_dev_prop_read(to_acpi_node(_fwnode_), _propname_, \ _proptype_, _val_, _nval_); \ - else \ + else if (is_pset(_fwnode_)) \ _ret_ = pset_prop_read_array(to_pset(_fwnode_), _propname_, \ _proptype_, _val_, _nval_); \ + else \ + _ret_ = -ENXIO; \ _ret_; \ }) @@ -431,9 +442,10 @@ int fwnode_property_read_string_array(struct fwnode_handle *fwnode, else if (is_acpi_node(fwnode)) return acpi_dev_prop_read(to_acpi_node(fwnode), propname, DEV_PROP_STRING, val, nval); - - return pset_prop_read_array(to_pset(fwnode), propname, - DEV_PROP_STRING, val, nval); + else if (is_pset(fwnode)) + return pset_prop_read_array(to_pset(fwnode), propname, + DEV_PROP_STRING, val, nval); + return -ENXIO; } EXPORT_SYMBOL_GPL(fwnode_property_read_string_array); @@ -461,7 +473,8 @@ int fwnode_property_read_string(struct fwnode_handle *fwnode, return acpi_dev_prop_read(to_acpi_node(fwnode), propname, DEV_PROP_STRING, val, 1); - return -ENXIO; + return pset_prop_read_array(to_pset(fwnode), propname, + DEV_PROP_STRING, val, 1); } EXPORT_SYMBOL_GPL(fwnode_property_read_string); @@ -533,3 +546,81 @@ bool device_dma_is_coherent(struct device *dev) return coherent; } EXPORT_SYMBOL_GPL(device_dma_is_coherent); + +/** + * device_get_phy_mode - Get phy mode for given device + * @dev: Pointer to the given device + * + * The function gets phy interface string from property 'phy-mode' or + * 'phy-connection-type', and return its index in phy_modes table, or errno in + * error case. + */ +int device_get_phy_mode(struct device *dev) +{ + const char *pm; + int err, i; + + err = device_property_read_string(dev, "phy-mode", &pm); + if (err < 0) + err = device_property_read_string(dev, + "phy-connection-type", &pm); + if (err < 0) + return err; + + for (i = 0; i < PHY_INTERFACE_MODE_MAX; i++) + if (!strcasecmp(pm, phy_modes(i))) + return i; + + return -ENODEV; +} +EXPORT_SYMBOL_GPL(device_get_phy_mode); + +static void *device_get_mac_addr(struct device *dev, + const char *name, char *addr, + int alen) +{ + int ret = device_property_read_u8_array(dev, name, addr, alen); + + if (ret == 0 && alen == ETH_ALEN && is_valid_ether_addr(addr)) + return addr; + return NULL; +} + +/** + * device_get_mac_address - Get the MAC for a given device + * @dev: Pointer to the device + * @addr: Address of buffer to store the MAC in + * @alen: Length of the buffer pointed to by addr, should be ETH_ALEN + * + * Search the firmware node for the best MAC address to use. 'mac-address' is + * checked first, because that is supposed to contain to "most recent" MAC + * address. If that isn't set, then 'local-mac-address' is checked next, + * because that is the default address. If that isn't set, then the obsolete + * 'address' is checked, just in case we're using an old device tree. + * + * Note that the 'address' property is supposed to contain a virtual address of + * the register set, but some DTS files have redefined that property to be the + * MAC address. + * + * All-zero MAC addresses are rejected, because those could be properties that + * exist in the firmware tables, but were not updated by the firmware. For + * example, the DTS could define 'mac-address' and 'local-mac-address', with + * zero MAC addresses. Some older U-Boots only initialized 'local-mac-address'. + * In this case, the real MAC is in 'local-mac-address', and 'mac-address' + * exists but is all zeros. +*/ +void *device_get_mac_address(struct device *dev, char *addr, int alen) +{ + char *res; + + res = device_get_mac_addr(dev, "mac-address", addr, alen); + if (res) + return res; + + res = device_get_mac_addr(dev, "local-mac-address", addr, alen); + if (res) + return res; + + return device_get_mac_addr(dev, "address", addr, alen); +} +EXPORT_SYMBOL(device_get_mac_address); diff --git a/drivers/base/regmap/internal.h b/drivers/base/regmap/internal.h index b2b2849fc6d3..cc557886ab23 100644 --- a/drivers/base/regmap/internal.h +++ b/drivers/base/regmap/internal.h @@ -136,14 +136,20 @@ struct regmap { /* if set, the HW registers are known to match map->reg_defaults */ bool no_sync_defaults; - struct reg_default *patch; + struct reg_sequence *patch; int patch_regs; - /* if set, converts bulk rw to single rw */ - bool use_single_rw; + /* if set, converts bulk read to single read */ + bool use_single_read; + /* if set, converts bulk read to single read */ + bool use_single_write; /* if set, the device supports multi write mode */ bool can_multi_write; + /* if set, raw reads/writes are limited to this size */ + size_t max_raw_read; + size_t max_raw_write; + struct rb_root range_tree; void *selector_work_buf; /* Scratch buffer used for selector */ }; diff --git a/drivers/base/regmap/regcache.c b/drivers/base/regmap/regcache.c index b9862d741a56..6f8a13ec32a4 100644 --- a/drivers/base/regmap/regcache.c +++ b/drivers/base/regmap/regcache.c @@ -729,7 +729,7 @@ int regcache_sync_block(struct regmap *map, void *block, unsigned int block_base, unsigned int start, unsigned int end) { - if (regmap_can_raw_write(map) && !map->use_single_rw) + if (regmap_can_raw_write(map) && !map->use_single_write) return regcache_sync_block_raw(map, block, cache_present, block_base, start, end); else diff --git a/drivers/base/regmap/regmap-ac97.c b/drivers/base/regmap/regmap-ac97.c index 8d304e2a943d..c03ebfd4c731 100644 --- a/drivers/base/regmap/regmap-ac97.c +++ b/drivers/base/regmap/regmap-ac97.c @@ -78,37 +78,24 @@ static const struct regmap_bus ac97_regmap_bus = { .reg_read = regmap_ac97_reg_read, }; -/** - * regmap_init_ac97(): Initialise AC'97 register map - * - * @ac97: Device that will be interacted with - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer to - * a struct regmap. - */ -struct regmap *regmap_init_ac97(struct snd_ac97 *ac97, - const struct regmap_config *config) +struct regmap *__regmap_init_ac97(struct snd_ac97 *ac97, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) { - return regmap_init(&ac97->dev, &ac97_regmap_bus, ac97, config); + return __regmap_init(&ac97->dev, &ac97_regmap_bus, ac97, config, + lock_key, lock_name); } -EXPORT_SYMBOL_GPL(regmap_init_ac97); +EXPORT_SYMBOL_GPL(__regmap_init_ac97); -/** - * devm_regmap_init_ac97(): Initialise AC'97 register map - * - * @ac97: Device that will be interacted with - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer - * to a struct regmap. The regmap will be automatically freed by the - * device management code. - */ -struct regmap *devm_regmap_init_ac97(struct snd_ac97 *ac97, - const struct regmap_config *config) +struct regmap *__devm_regmap_init_ac97(struct snd_ac97 *ac97, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) { - return devm_regmap_init(&ac97->dev, &ac97_regmap_bus, ac97, config); + return __devm_regmap_init(&ac97->dev, &ac97_regmap_bus, ac97, config, + lock_key, lock_name); } -EXPORT_SYMBOL_GPL(devm_regmap_init_ac97); +EXPORT_SYMBOL_GPL(__devm_regmap_init_ac97); MODULE_LICENSE("GPL v2"); diff --git a/drivers/base/regmap/regmap-debugfs.c b/drivers/base/regmap/regmap-debugfs.c index 5799a0b9e6cc..f42f2bac6466 100644 --- a/drivers/base/regmap/regmap-debugfs.c +++ b/drivers/base/regmap/regmap-debugfs.c @@ -469,6 +469,87 @@ static const struct file_operations regmap_access_fops = { .llseek = default_llseek, }; +static ssize_t regmap_cache_only_write_file(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct regmap *map = container_of(file->private_data, + struct regmap, cache_only); + ssize_t result; + bool was_enabled, require_sync = false; + int err; + + map->lock(map->lock_arg); + + was_enabled = map->cache_only; + + result = debugfs_write_file_bool(file, user_buf, count, ppos); + if (result < 0) { + map->unlock(map->lock_arg); + return result; + } + + if (map->cache_only && !was_enabled) { + dev_warn(map->dev, "debugfs cache_only=Y forced\n"); + add_taint(TAINT_USER, LOCKDEP_STILL_OK); + } else if (!map->cache_only && was_enabled) { + dev_warn(map->dev, "debugfs cache_only=N forced: syncing cache\n"); + require_sync = true; + } + + map->unlock(map->lock_arg); + + if (require_sync) { + err = regcache_sync(map); + if (err) + dev_err(map->dev, "Failed to sync cache %d\n", err); + } + + return result; +} + +static const struct file_operations regmap_cache_only_fops = { + .open = simple_open, + .read = debugfs_read_file_bool, + .write = regmap_cache_only_write_file, +}; + +static ssize_t regmap_cache_bypass_write_file(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct regmap *map = container_of(file->private_data, + struct regmap, cache_bypass); + ssize_t result; + bool was_enabled; + + map->lock(map->lock_arg); + + was_enabled = map->cache_bypass; + + result = debugfs_write_file_bool(file, user_buf, count, ppos); + if (result < 0) + goto out; + + if (map->cache_bypass && !was_enabled) { + dev_warn(map->dev, "debugfs cache_bypass=Y forced\n"); + add_taint(TAINT_USER, LOCKDEP_STILL_OK); + } else if (!map->cache_bypass && was_enabled) { + dev_warn(map->dev, "debugfs cache_bypass=N forced\n"); + } + +out: + map->unlock(map->lock_arg); + + return result; +} + +static const struct file_operations regmap_cache_bypass_fops = { + .open = simple_open, + .read = debugfs_read_file_bool, + .write = regmap_cache_bypass_write_file, +}; + void regmap_debugfs_init(struct regmap *map, const char *name) { struct rb_node *next; @@ -518,10 +599,11 @@ void regmap_debugfs_init(struct regmap *map, const char *name) if (map->max_register || regmap_readable(map, 0)) { umode_t registers_mode; - if (IS_ENABLED(REGMAP_ALLOW_WRITE_DEBUGFS)) - registers_mode = 0600; - else - registers_mode = 0400; +#if defined(REGMAP_ALLOW_WRITE_DEBUGFS) + registers_mode = 0600; +#else + registers_mode = 0400; +#endif debugfs_create_file("registers", registers_mode, map->debugfs, map, ®map_map_fops); @@ -530,12 +612,13 @@ void regmap_debugfs_init(struct regmap *map, const char *name) } if (map->cache_type) { - debugfs_create_bool("cache_only", 0400, map->debugfs, - &map->cache_only); + debugfs_create_file("cache_only", 0600, map->debugfs, + &map->cache_only, ®map_cache_only_fops); debugfs_create_bool("cache_dirty", 0400, map->debugfs, &map->cache_dirty); - debugfs_create_bool("cache_bypass", 0400, map->debugfs, - &map->cache_bypass); + debugfs_create_file("cache_bypass", 0600, map->debugfs, + &map->cache_bypass, + ®map_cache_bypass_fops); } next = rb_first(&map->range_tree); diff --git a/drivers/base/regmap/regmap-i2c.c b/drivers/base/regmap/regmap-i2c.c index 4b76e33110a2..1a8ec3b2b601 100644 --- a/drivers/base/regmap/regmap-i2c.c +++ b/drivers/base/regmap/regmap-i2c.c @@ -209,11 +209,60 @@ static struct regmap_bus regmap_i2c = { .val_format_endian_default = REGMAP_ENDIAN_BIG, }; +static int regmap_i2c_smbus_i2c_write(void *context, const void *data, + size_t count) +{ + struct device *dev = context; + struct i2c_client *i2c = to_i2c_client(dev); + + if (count < 1) + return -EINVAL; + if (count >= I2C_SMBUS_BLOCK_MAX) + return -E2BIG; + + --count; + return i2c_smbus_write_i2c_block_data(i2c, ((u8 *)data)[0], count, + ((u8 *)data + 1)); +} + +static int regmap_i2c_smbus_i2c_read(void *context, const void *reg, + size_t reg_size, void *val, + size_t val_size) +{ + struct device *dev = context; + struct i2c_client *i2c = to_i2c_client(dev); + int ret; + + if (reg_size != 1 || val_size < 1) + return -EINVAL; + if (val_size >= I2C_SMBUS_BLOCK_MAX) + return -E2BIG; + + ret = i2c_smbus_read_i2c_block_data(i2c, ((u8 *)reg)[0], val_size, val); + if (ret == val_size) + return 0; + else if (ret < 0) + return ret; + else + return -EIO; +} + +static struct regmap_bus regmap_i2c_smbus_i2c_block = { + .write = regmap_i2c_smbus_i2c_write, + .read = regmap_i2c_smbus_i2c_read, + .max_raw_read = I2C_SMBUS_BLOCK_MAX, + .max_raw_write = I2C_SMBUS_BLOCK_MAX, +}; + static const struct regmap_bus *regmap_get_i2c_bus(struct i2c_client *i2c, const struct regmap_config *config) { if (i2c_check_functionality(i2c->adapter, I2C_FUNC_I2C)) return ®map_i2c; + else if (config->reg_bits == 8 && + i2c_check_functionality(i2c->adapter, + I2C_FUNC_SMBUS_I2C_BLOCK)) + return ®map_i2c_smbus_i2c_block; else if (config->val_bits == 16 && config->reg_bits == 8 && i2c_check_functionality(i2c->adapter, I2C_FUNC_SMBUS_WORD_DATA)) @@ -233,47 +282,34 @@ static const struct regmap_bus *regmap_get_i2c_bus(struct i2c_client *i2c, return ERR_PTR(-ENOTSUPP); } -/** - * regmap_init_i2c(): Initialise register map - * - * @i2c: Device that will be interacted with - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer to - * a struct regmap. - */ -struct regmap *regmap_init_i2c(struct i2c_client *i2c, - const struct regmap_config *config) +struct regmap *__regmap_init_i2c(struct i2c_client *i2c, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) { const struct regmap_bus *bus = regmap_get_i2c_bus(i2c, config); if (IS_ERR(bus)) return ERR_CAST(bus); - return regmap_init(&i2c->dev, bus, &i2c->dev, config); + return __regmap_init(&i2c->dev, bus, &i2c->dev, config, + lock_key, lock_name); } -EXPORT_SYMBOL_GPL(regmap_init_i2c); +EXPORT_SYMBOL_GPL(__regmap_init_i2c); -/** - * devm_regmap_init_i2c(): Initialise managed register map - * - * @i2c: Device that will be interacted with - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer - * to a struct regmap. The regmap will be automatically freed by the - * device management code. - */ -struct regmap *devm_regmap_init_i2c(struct i2c_client *i2c, - const struct regmap_config *config) +struct regmap *__devm_regmap_init_i2c(struct i2c_client *i2c, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) { const struct regmap_bus *bus = regmap_get_i2c_bus(i2c, config); if (IS_ERR(bus)) return ERR_CAST(bus); - return devm_regmap_init(&i2c->dev, bus, &i2c->dev, config); + return __devm_regmap_init(&i2c->dev, bus, &i2c->dev, config, + lock_key, lock_name); } -EXPORT_SYMBOL_GPL(devm_regmap_init_i2c); +EXPORT_SYMBOL_GPL(__devm_regmap_init_i2c); MODULE_LICENSE("GPL"); diff --git a/drivers/base/regmap/regmap-irq.c b/drivers/base/regmap/regmap-irq.c index 2597600a5d26..38d1f72d869c 100644 --- a/drivers/base/regmap/regmap-irq.c +++ b/drivers/base/regmap/regmap-irq.c @@ -209,7 +209,7 @@ static irqreturn_t regmap_irq_thread(int irq, void *d) * Read in the statuses, using a single bulk read if possible * in order to reduce the I/O overheads. */ - if (!map->use_single_rw && map->reg_stride == 1 && + if (!map->use_single_read && map->reg_stride == 1 && data->irq_reg_stride == 1) { u8 *buf8 = data->status_reg_buf; u16 *buf16 = data->status_reg_buf; @@ -398,7 +398,7 @@ int regmap_add_irq_chip(struct regmap *map, int irq, int irq_flags, else d->irq_reg_stride = 1; - if (!map->use_single_rw && map->reg_stride == 1 && + if (!map->use_single_read && map->reg_stride == 1 && d->irq_reg_stride == 1) { d->status_reg_buf = kmalloc(map->format.val_bytes * chip->num_regs, GFP_KERNEL); diff --git a/drivers/base/regmap/regmap-mmio.c b/drivers/base/regmap/regmap-mmio.c index 04a329a377e9..426a57e41ac7 100644 --- a/drivers/base/regmap/regmap-mmio.c +++ b/drivers/base/regmap/regmap-mmio.c @@ -296,20 +296,11 @@ static struct regmap_mmio_context *regmap_mmio_gen_context(struct device *dev, return ERR_PTR(ret); } -/** - * regmap_init_mmio_clk(): Initialise register map with register clock - * - * @dev: Device that will be interacted with - * @clk_id: register clock consumer ID - * @regs: Pointer to memory-mapped IO region - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer to - * a struct regmap. - */ -struct regmap *regmap_init_mmio_clk(struct device *dev, const char *clk_id, - void __iomem *regs, - const struct regmap_config *config) +struct regmap *__regmap_init_mmio_clk(struct device *dev, const char *clk_id, + void __iomem *regs, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) { struct regmap_mmio_context *ctx; @@ -317,25 +308,17 @@ struct regmap *regmap_init_mmio_clk(struct device *dev, const char *clk_id, if (IS_ERR(ctx)) return ERR_CAST(ctx); - return regmap_init(dev, ®map_mmio, ctx, config); + return __regmap_init(dev, ®map_mmio, ctx, config, + lock_key, lock_name); } -EXPORT_SYMBOL_GPL(regmap_init_mmio_clk); +EXPORT_SYMBOL_GPL(__regmap_init_mmio_clk); -/** - * devm_regmap_init_mmio_clk(): Initialise managed register map with clock - * - * @dev: Device that will be interacted with - * @clk_id: register clock consumer ID - * @regs: Pointer to memory-mapped IO region - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer - * to a struct regmap. The regmap will be automatically freed by the - * device management code. - */ -struct regmap *devm_regmap_init_mmio_clk(struct device *dev, const char *clk_id, - void __iomem *regs, - const struct regmap_config *config) +struct regmap *__devm_regmap_init_mmio_clk(struct device *dev, + const char *clk_id, + void __iomem *regs, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) { struct regmap_mmio_context *ctx; @@ -343,8 +326,9 @@ struct regmap *devm_regmap_init_mmio_clk(struct device *dev, const char *clk_id, if (IS_ERR(ctx)) return ERR_CAST(ctx); - return devm_regmap_init(dev, ®map_mmio, ctx, config); + return __devm_regmap_init(dev, ®map_mmio, ctx, config, + lock_key, lock_name); } -EXPORT_SYMBOL_GPL(devm_regmap_init_mmio_clk); +EXPORT_SYMBOL_GPL(__devm_regmap_init_mmio_clk); MODULE_LICENSE("GPL v2"); diff --git a/drivers/base/regmap/regmap-spi.c b/drivers/base/regmap/regmap-spi.c index 53d1148e80a0..edd9a839d004 100644 --- a/drivers/base/regmap/regmap-spi.c +++ b/drivers/base/regmap/regmap-spi.c @@ -113,37 +113,24 @@ static struct regmap_bus regmap_spi = { .val_format_endian_default = REGMAP_ENDIAN_BIG, }; -/** - * regmap_init_spi(): Initialise register map - * - * @spi: Device that will be interacted with - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer to - * a struct regmap. - */ -struct regmap *regmap_init_spi(struct spi_device *spi, - const struct regmap_config *config) +struct regmap *__regmap_init_spi(struct spi_device *spi, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) { - return regmap_init(&spi->dev, ®map_spi, &spi->dev, config); + return __regmap_init(&spi->dev, ®map_spi, &spi->dev, config, + lock_key, lock_name); } -EXPORT_SYMBOL_GPL(regmap_init_spi); +EXPORT_SYMBOL_GPL(__regmap_init_spi); -/** - * devm_regmap_init_spi(): Initialise register map - * - * @spi: Device that will be interacted with - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer - * to a struct regmap. The map will be automatically freed by the - * device management code. - */ -struct regmap *devm_regmap_init_spi(struct spi_device *spi, - const struct regmap_config *config) +struct regmap *__devm_regmap_init_spi(struct spi_device *spi, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) { - return devm_regmap_init(&spi->dev, ®map_spi, &spi->dev, config); + return __devm_regmap_init(&spi->dev, ®map_spi, &spi->dev, config, + lock_key, lock_name); } -EXPORT_SYMBOL_GPL(devm_regmap_init_spi); +EXPORT_SYMBOL_GPL(__devm_regmap_init_spi); MODULE_LICENSE("GPL"); diff --git a/drivers/base/regmap/regmap-spmi.c b/drivers/base/regmap/regmap-spmi.c index d7026dc33388..7e58f6560399 100644 --- a/drivers/base/regmap/regmap-spmi.c +++ b/drivers/base/regmap/regmap-spmi.c @@ -91,36 +91,25 @@ static struct regmap_bus regmap_spmi_base = { .val_format_endian_default = REGMAP_ENDIAN_NATIVE, }; -/** - * regmap_init_spmi_base(): Create regmap for the Base register space - * @sdev: SPMI device that will be interacted with - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer to - * a struct regmap. - */ -struct regmap *regmap_init_spmi_base(struct spmi_device *sdev, - const struct regmap_config *config) +struct regmap *__regmap_init_spmi_base(struct spmi_device *sdev, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) { - return regmap_init(&sdev->dev, ®map_spmi_base, sdev, config); + return __regmap_init(&sdev->dev, ®map_spmi_base, sdev, config, + lock_key, lock_name); } -EXPORT_SYMBOL_GPL(regmap_init_spmi_base); +EXPORT_SYMBOL_GPL(__regmap_init_spmi_base); -/** - * devm_regmap_init_spmi_base(): Create managed regmap for Base register space - * @sdev: SPMI device that will be interacted with - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer - * to a struct regmap. The regmap will be automatically freed by the - * device management code. - */ -struct regmap *devm_regmap_init_spmi_base(struct spmi_device *sdev, - const struct regmap_config *config) +struct regmap *__devm_regmap_init_spmi_base(struct spmi_device *sdev, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) { - return devm_regmap_init(&sdev->dev, ®map_spmi_base, sdev, config); + return __devm_regmap_init(&sdev->dev, ®map_spmi_base, sdev, config, + lock_key, lock_name); } -EXPORT_SYMBOL_GPL(devm_regmap_init_spmi_base); +EXPORT_SYMBOL_GPL(__devm_regmap_init_spmi_base); static int regmap_spmi_ext_read(void *context, const void *reg, size_t reg_size, @@ -222,35 +211,24 @@ static struct regmap_bus regmap_spmi_ext = { .val_format_endian_default = REGMAP_ENDIAN_NATIVE, }; -/** - * regmap_init_spmi_ext(): Create regmap for Ext register space - * @sdev: Device that will be interacted with - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer to - * a struct regmap. - */ -struct regmap *regmap_init_spmi_ext(struct spmi_device *sdev, - const struct regmap_config *config) +struct regmap *__regmap_init_spmi_ext(struct spmi_device *sdev, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) { - return regmap_init(&sdev->dev, ®map_spmi_ext, sdev, config); + return __regmap_init(&sdev->dev, ®map_spmi_ext, sdev, config, + lock_key, lock_name); } -EXPORT_SYMBOL_GPL(regmap_init_spmi_ext); +EXPORT_SYMBOL_GPL(__regmap_init_spmi_ext); -/** - * devm_regmap_init_spmi_ext(): Create managed regmap for Ext register space - * @sdev: SPMI device that will be interacted with - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer - * to a struct regmap. The regmap will be automatically freed by the - * device management code. - */ -struct regmap *devm_regmap_init_spmi_ext(struct spmi_device *sdev, - const struct regmap_config *config) +struct regmap *__devm_regmap_init_spmi_ext(struct spmi_device *sdev, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) { - return devm_regmap_init(&sdev->dev, ®map_spmi_ext, sdev, config); + return __devm_regmap_init(&sdev->dev, ®map_spmi_ext, sdev, config, + lock_key, lock_name); } -EXPORT_SYMBOL_GPL(devm_regmap_init_spmi_ext); +EXPORT_SYMBOL_GPL(__devm_regmap_init_spmi_ext); MODULE_LICENSE("GPL"); diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c index 7111d04f2621..afaf56200674 100644 --- a/drivers/base/regmap/regmap.c +++ b/drivers/base/regmap/regmap.c @@ -18,6 +18,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include "trace.h" @@ -34,7 +35,7 @@ static int _regmap_update_bits(struct regmap *map, unsigned int reg, unsigned int mask, unsigned int val, - bool *change); + bool *change, bool force_write); static int _regmap_bus_reg_read(void *context, unsigned int reg, unsigned int *val); @@ -93,6 +94,9 @@ bool regmap_writeable(struct regmap *map, unsigned int reg) bool regmap_readable(struct regmap *map, unsigned int reg) { + if (!map->reg_read) + return false; + if (map->max_register && reg > map->max_register) return false; @@ -515,22 +519,12 @@ enum regmap_endian regmap_get_val_endian(struct device *dev, } EXPORT_SYMBOL_GPL(regmap_get_val_endian); -/** - * regmap_init(): Initialise register map - * - * @dev: Device that will be interacted with - * @bus: Bus-specific callbacks to use with device - * @bus_context: Data passed to bus-specific callbacks - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer to - * a struct regmap. This function should generally not be called - * directly, it should be called by bus-specific init functions. - */ -struct regmap *regmap_init(struct device *dev, - const struct regmap_bus *bus, - void *bus_context, - const struct regmap_config *config) +struct regmap *__regmap_init(struct device *dev, + const struct regmap_bus *bus, + void *bus_context, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) { struct regmap *map; int ret = -EINVAL; @@ -556,10 +550,14 @@ struct regmap *regmap_init(struct device *dev, spin_lock_init(&map->spinlock); map->lock = regmap_lock_spinlock; map->unlock = regmap_unlock_spinlock; + lockdep_set_class_and_name(&map->spinlock, + lock_key, lock_name); } else { mutex_init(&map->mutex); map->lock = regmap_lock_mutex; map->unlock = regmap_unlock_mutex; + lockdep_set_class_and_name(&map->mutex, + lock_key, lock_name); } map->lock_arg = map; } @@ -573,8 +571,13 @@ struct regmap *regmap_init(struct device *dev, map->reg_stride = config->reg_stride; else map->reg_stride = 1; - map->use_single_rw = config->use_single_rw; - map->can_multi_write = config->can_multi_write; + map->use_single_read = config->use_single_rw || !bus || !bus->read; + map->use_single_write = config->use_single_rw || !bus || !bus->write; + map->can_multi_write = config->can_multi_write && bus && bus->write; + if (bus) { + map->max_raw_read = bus->max_raw_read; + map->max_raw_write = bus->max_raw_write; + } map->dev = dev; map->bus = bus; map->bus_context = bus_context; @@ -763,7 +766,7 @@ struct regmap *regmap_init(struct device *dev, if ((reg_endian != REGMAP_ENDIAN_BIG) || (val_endian != REGMAP_ENDIAN_BIG)) goto err_map; - map->use_single_rw = true; + map->use_single_write = true; } if (!map->format.format_write && @@ -899,30 +902,19 @@ struct regmap *regmap_init(struct device *dev, err: return ERR_PTR(ret); } -EXPORT_SYMBOL_GPL(regmap_init); +EXPORT_SYMBOL_GPL(__regmap_init); static void devm_regmap_release(struct device *dev, void *res) { regmap_exit(*(struct regmap **)res); } -/** - * devm_regmap_init(): Initialise managed register map - * - * @dev: Device that will be interacted with - * @bus: Bus-specific callbacks to use with device - * @bus_context: Data passed to bus-specific callbacks - * @config: Configuration for register map - * - * The return value will be an ERR_PTR() on error or a valid pointer - * to a struct regmap. This function should generally not be called - * directly, it should be called by bus-specific init functions. The - * map will be automatically freed by the device management code. - */ -struct regmap *devm_regmap_init(struct device *dev, - const struct regmap_bus *bus, - void *bus_context, - const struct regmap_config *config) +struct regmap *__devm_regmap_init(struct device *dev, + const struct regmap_bus *bus, + void *bus_context, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name) { struct regmap **ptr, *regmap; @@ -930,7 +922,8 @@ struct regmap *devm_regmap_init(struct device *dev, if (!ptr) return ERR_PTR(-ENOMEM); - regmap = regmap_init(dev, bus, bus_context, config); + regmap = __regmap_init(dev, bus, bus_context, config, + lock_key, lock_name); if (!IS_ERR(regmap)) { *ptr = regmap; devres_add(dev, ptr); @@ -940,7 +933,7 @@ struct regmap *devm_regmap_init(struct device *dev, return regmap; } -EXPORT_SYMBOL_GPL(devm_regmap_init); +EXPORT_SYMBOL_GPL(__devm_regmap_init); static void regmap_field_init(struct regmap_field *rm_field, struct regmap *regmap, struct reg_field reg_field) @@ -1178,7 +1171,7 @@ static int _regmap_select_page(struct regmap *map, unsigned int *reg, ret = _regmap_update_bits(map, range->selector_reg, range->selector_mask, win_page << range->selector_shift, - &page_chg); + &page_chg, false); map->work_buf = orig_work_buf; @@ -1382,10 +1375,33 @@ int _regmap_raw_write(struct regmap *map, unsigned int reg, */ bool regmap_can_raw_write(struct regmap *map) { - return map->bus && map->format.format_val && map->format.format_reg; + return map->bus && map->bus->write && map->format.format_val && + map->format.format_reg; } EXPORT_SYMBOL_GPL(regmap_can_raw_write); +/** + * regmap_get_raw_read_max - Get the maximum size we can read + * + * @map: Map to check. + */ +size_t regmap_get_raw_read_max(struct regmap *map) +{ + return map->max_raw_read; +} +EXPORT_SYMBOL_GPL(regmap_get_raw_read_max); + +/** + * regmap_get_raw_write_max - Get the maximum size we can read + * + * @map: Map to check. + */ +size_t regmap_get_raw_write_max(struct regmap *map) +{ + return map->max_raw_write; +} +EXPORT_SYMBOL_GPL(regmap_get_raw_write_max); + static int _regmap_bus_formatted_write(void *context, unsigned int reg, unsigned int val) { @@ -1555,6 +1571,8 @@ int regmap_raw_write(struct regmap *map, unsigned int reg, return -EINVAL; if (val_len % map->format.val_bytes) return -EINVAL; + if (map->max_raw_write && map->max_raw_write > val_len) + return -E2BIG; map->lock(map->lock_arg); @@ -1624,6 +1642,18 @@ int regmap_fields_write(struct regmap_field *field, unsigned int id, } EXPORT_SYMBOL_GPL(regmap_fields_write); +int regmap_fields_force_write(struct regmap_field *field, unsigned int id, + unsigned int val) +{ + if (id >= field->id_size) + return -EINVAL; + + return regmap_write_bits(field->regmap, + field->reg + (field->id_offset * id), + field->mask, val << field->shift); +} +EXPORT_SYMBOL_GPL(regmap_fields_force_write); + /** * regmap_fields_update_bits(): Perform a read/modify/write cycle * on the register field @@ -1669,6 +1699,7 @@ int regmap_bulk_write(struct regmap *map, unsigned int reg, const void *val, { int ret = 0, i; size_t val_bytes = map->format.val_bytes; + size_t total_size = val_bytes * val_count; if (map->bus && !map->format.parse_inplace) return -EINVAL; @@ -1677,9 +1708,15 @@ int regmap_bulk_write(struct regmap *map, unsigned int reg, const void *val, /* * Some devices don't support bulk write, for - * them we have a series of single write operations. + * them we have a series of single write operations in the first two if + * blocks. + * + * The first if block is used for memory mapped io. It does not allow + * val_bytes of 3 for example. + * The second one is used for busses which do not have this limitation + * and can write arbitrary value lengths. */ - if (!map->bus || map->use_single_rw) { + if (!map->bus) { map->lock(map->lock_arg); for (i = 0; i < val_count; i++) { unsigned int ival; @@ -1711,6 +1748,38 @@ int regmap_bulk_write(struct regmap *map, unsigned int reg, const void *val, } out: map->unlock(map->lock_arg); + } else if (map->use_single_write || + (map->max_raw_write && map->max_raw_write < total_size)) { + int chunk_stride = map->reg_stride; + size_t chunk_size = val_bytes; + size_t chunk_count = val_count; + + if (!map->use_single_write) { + chunk_size = map->max_raw_write; + if (chunk_size % val_bytes) + chunk_size -= chunk_size % val_bytes; + chunk_count = total_size / chunk_size; + chunk_stride *= chunk_size / val_bytes; + } + + map->lock(map->lock_arg); + /* Write as many bytes as possible with chunk_size */ + for (i = 0; i < chunk_count; i++) { + ret = _regmap_raw_write(map, + reg + (i * chunk_stride), + val + (i * chunk_size), + chunk_size); + if (ret) + break; + } + + /* Write remaining bytes */ + if (!ret && chunk_size * i < total_size) { + ret = _regmap_raw_write(map, reg + (i * chunk_stride), + val + (i * chunk_size), + total_size - i * chunk_size); + } + map->unlock(map->lock_arg); } else { void *wval; @@ -1740,10 +1809,10 @@ EXPORT_SYMBOL_GPL(regmap_bulk_write); * * the (register,newvalue) pairs in regs have not been formatted, but * they are all in the same page and have been changed to being page - * relative. The page register has been written if that was neccessary. + * relative. The page register has been written if that was necessary. */ static int _regmap_raw_multi_reg_write(struct regmap *map, - const struct reg_default *regs, + const struct reg_sequence *regs, size_t num_regs) { int ret; @@ -1768,8 +1837,8 @@ static int _regmap_raw_multi_reg_write(struct regmap *map, u8 = buf; for (i = 0; i < num_regs; i++) { - int reg = regs[i].reg; - int val = regs[i].def; + unsigned int reg = regs[i].reg; + unsigned int val = regs[i].def; trace_regmap_hw_write_start(map, reg, 1); map->format.format_reg(u8, reg, map->reg_shift); u8 += reg_bytes + pad_bytes; @@ -1800,17 +1869,19 @@ static unsigned int _regmap_register_page(struct regmap *map, } static int _regmap_range_multi_paged_reg_write(struct regmap *map, - struct reg_default *regs, + struct reg_sequence *regs, size_t num_regs) { int ret; int i, n; - struct reg_default *base; + struct reg_sequence *base; unsigned int this_page = 0; + unsigned int page_change = 0; /* * the set of registers are not neccessarily in order, but * since the order of write must be preserved this algorithm - * chops the set each time the page changes + * chops the set each time the page changes. This also applies + * if there is a delay required at any point in the sequence. */ base = regs; for (i = 0, n = 0; i < num_regs; i++, n++) { @@ -1826,16 +1897,48 @@ static int _regmap_range_multi_paged_reg_write(struct regmap *map, this_page = win_page; if (win_page != this_page) { this_page = win_page; + page_change = 1; + } + } + + /* If we have both a page change and a delay make sure to + * write the regs and apply the delay before we change the + * page. + */ + + if (page_change || regs[i].delay_us) { + + /* For situations where the first write requires + * a delay we need to make sure we don't call + * raw_multi_reg_write with n=0 + * This can't occur with page breaks as we + * never write on the first iteration + */ + if (regs[i].delay_us && i == 0) + n = 1; + ret = _regmap_raw_multi_reg_write(map, base, n); if (ret != 0) return ret; + + if (regs[i].delay_us) + udelay(regs[i].delay_us); + base += n; n = 0; - } - ret = _regmap_select_page(map, &base[n].reg, range, 1); - if (ret != 0) - return ret; + + if (page_change) { + ret = _regmap_select_page(map, + &base[n].reg, + range, 1); + if (ret != 0) + return ret; + + page_change = 0; + } + } + } if (n > 0) return _regmap_raw_multi_reg_write(map, base, n); @@ -1843,7 +1946,7 @@ static int _regmap_range_multi_paged_reg_write(struct regmap *map, } static int _regmap_multi_reg_write(struct regmap *map, - const struct reg_default *regs, + const struct reg_sequence *regs, size_t num_regs) { int i; @@ -1854,6 +1957,9 @@ static int _regmap_multi_reg_write(struct regmap *map, ret = _regmap_write(map, regs[i].reg, regs[i].def); if (ret != 0) return ret; + + if (regs[i].delay_us) + udelay(regs[i].delay_us); } return 0; } @@ -1893,10 +1999,14 @@ static int _regmap_multi_reg_write(struct regmap *map, for (i = 0; i < num_regs; i++) { unsigned int reg = regs[i].reg; struct regmap_range_node *range; + + /* Coalesce all the writes between a page break or a delay + * in a sequence + */ range = _regmap_range_lookup(map, reg); - if (range) { - size_t len = sizeof(struct reg_default)*num_regs; - struct reg_default *base = kmemdup(regs, len, + if (range || regs[i].delay_us) { + size_t len = sizeof(struct reg_sequence)*num_regs; + struct reg_sequence *base = kmemdup(regs, len, GFP_KERNEL); if (!base) return -ENOMEM; @@ -1929,7 +2039,7 @@ static int _regmap_multi_reg_write(struct regmap *map, * A value of zero will be returned on success, a negative errno will be * returned in error cases. */ -int regmap_multi_reg_write(struct regmap *map, const struct reg_default *regs, +int regmap_multi_reg_write(struct regmap *map, const struct reg_sequence *regs, int num_regs) { int ret; @@ -1962,7 +2072,7 @@ EXPORT_SYMBOL_GPL(regmap_multi_reg_write); * be returned in error cases. */ int regmap_multi_reg_write_bypassed(struct regmap *map, - const struct reg_default *regs, + const struct reg_sequence *regs, int num_regs) { int ret; @@ -2050,7 +2160,7 @@ static int _regmap_raw_read(struct regmap *map, unsigned int reg, void *val, /* * Some buses or devices flag reads by setting the high bits in the - * register addresss; since it's always the high bits for all + * register address; since it's always the high bits for all * current formats we can do this here rather than in * formatting. This may break if we get interesting formats. */ @@ -2097,8 +2207,6 @@ static int _regmap_read(struct regmap *map, unsigned int reg, int ret; void *context = _regmap_map_get_context(map); - WARN_ON(!map->reg_read); - if (!map->cache_bypass) { ret = regcache_read(map, reg, val); if (ret == 0) @@ -2179,11 +2287,22 @@ int regmap_raw_read(struct regmap *map, unsigned int reg, void *val, return -EINVAL; if (reg % map->reg_stride) return -EINVAL; + if (val_count == 0) + return -EINVAL; map->lock(map->lock_arg); if (regmap_volatile_range(map, reg, val_count) || map->cache_bypass || map->cache_type == REGCACHE_NONE) { + if (!map->bus->read) { + ret = -ENOTSUPP; + goto out; + } + if (map->max_raw_read && map->max_raw_read < val_len) { + ret = -E2BIG; + goto out; + } + /* Physical block read if there's no cache involved */ ret = _regmap_raw_read(map, reg, val, val_len); @@ -2292,20 +2411,51 @@ int regmap_bulk_read(struct regmap *map, unsigned int reg, void *val, * Some devices does not support bulk read, for * them we have a series of single read operations. */ - if (map->use_single_rw) { - for (i = 0; i < val_count; i++) { - ret = regmap_raw_read(map, - reg + (i * map->reg_stride), - val + (i * val_bytes), - val_bytes); - if (ret != 0) - return ret; - } - } else { + size_t total_size = val_bytes * val_count; + + if (!map->use_single_read && + (!map->max_raw_read || map->max_raw_read > total_size)) { ret = regmap_raw_read(map, reg, val, val_bytes * val_count); if (ret != 0) return ret; + } else { + /* + * Some devices do not support bulk read or do not + * support large bulk reads, for them we have a series + * of read operations. + */ + int chunk_stride = map->reg_stride; + size_t chunk_size = val_bytes; + size_t chunk_count = val_count; + + if (!map->use_single_read) { + chunk_size = map->max_raw_read; + if (chunk_size % val_bytes) + chunk_size -= chunk_size % val_bytes; + chunk_count = total_size / chunk_size; + chunk_stride *= chunk_size / val_bytes; + } + + /* Read bytes that fit into a multiple of chunk_size */ + for (i = 0; i < chunk_count; i++) { + ret = regmap_raw_read(map, + reg + (i * chunk_stride), + val + (i * chunk_size), + chunk_size); + if (ret != 0) + return ret; + } + + /* Read remaining bytes */ + if (chunk_size * i < total_size) { + ret = regmap_raw_read(map, + reg + (i * chunk_stride), + val + (i * chunk_size), + total_size - i * chunk_size); + if (ret != 0) + return ret; + } } for (i = 0; i < val_count * val_bytes; i += val_bytes) @@ -2317,7 +2467,34 @@ int regmap_bulk_read(struct regmap *map, unsigned int reg, void *val, &ival); if (ret != 0) return ret; - map->format.format_val(val + (i * val_bytes), ival, 0); + + if (map->format.format_val) { + map->format.format_val(val + (i * val_bytes), ival, 0); + } else { + /* Devices providing read and write + * operations can use the bulk I/O + * functions if they define a val_bytes, + * we assume that the values are native + * endian. + */ + u32 *u32 = val; + u16 *u16 = val; + u8 *u8 = val; + + switch (map->format.val_bytes) { + case 4: + u32[i] = ival; + break; + case 2: + u16[i] = ival; + break; + case 1: + u8[i] = ival; + break; + default: + return -EINVAL; + } + } } } @@ -2327,7 +2504,7 @@ EXPORT_SYMBOL_GPL(regmap_bulk_read); static int _regmap_update_bits(struct regmap *map, unsigned int reg, unsigned int mask, unsigned int val, - bool *change) + bool *change, bool force_write) { int ret; unsigned int tmp, orig; @@ -2339,7 +2516,7 @@ static int _regmap_update_bits(struct regmap *map, unsigned int reg, tmp = orig & ~mask; tmp |= val & mask; - if (tmp != orig) { + if (force_write || (tmp != orig)) { ret = _regmap_write(map, reg, tmp); if (change) *change = true; @@ -2367,13 +2544,36 @@ int regmap_update_bits(struct regmap *map, unsigned int reg, int ret; map->lock(map->lock_arg); - ret = _regmap_update_bits(map, reg, mask, val, NULL); + ret = _regmap_update_bits(map, reg, mask, val, NULL, false); map->unlock(map->lock_arg); return ret; } EXPORT_SYMBOL_GPL(regmap_update_bits); +/** + * regmap_write_bits: Perform a read/modify/write cycle on the register map + * + * @map: Register map to update + * @reg: Register to update + * @mask: Bitmask to change + * @val: New value for bitmask + * + * Returns zero for success, a negative number on error. + */ +int regmap_write_bits(struct regmap *map, unsigned int reg, + unsigned int mask, unsigned int val) +{ + int ret; + + map->lock(map->lock_arg); + ret = _regmap_update_bits(map, reg, mask, val, NULL, true); + map->unlock(map->lock_arg); + + return ret; +} +EXPORT_SYMBOL_GPL(regmap_write_bits); + /** * regmap_update_bits_async: Perform a read/modify/write cycle on the register * map asynchronously @@ -2398,7 +2598,7 @@ int regmap_update_bits_async(struct regmap *map, unsigned int reg, map->async = true; - ret = _regmap_update_bits(map, reg, mask, val, NULL); + ret = _regmap_update_bits(map, reg, mask, val, NULL, false); map->async = false; @@ -2427,7 +2627,7 @@ int regmap_update_bits_check(struct regmap *map, unsigned int reg, int ret; map->lock(map->lock_arg); - ret = _regmap_update_bits(map, reg, mask, val, change); + ret = _regmap_update_bits(map, reg, mask, val, change, false); map->unlock(map->lock_arg); return ret; } @@ -2460,7 +2660,7 @@ int regmap_update_bits_check_async(struct regmap *map, unsigned int reg, map->async = true; - ret = _regmap_update_bits(map, reg, mask, val, change); + ret = _regmap_update_bits(map, reg, mask, val, change, false); map->async = false; @@ -2552,10 +2752,10 @@ EXPORT_SYMBOL_GPL(regmap_async_complete); * The caller must ensure that this function cannot be called * concurrently with either itself or regcache_sync(). */ -int regmap_register_patch(struct regmap *map, const struct reg_default *regs, +int regmap_register_patch(struct regmap *map, const struct reg_sequence *regs, int num_regs) { - struct reg_default *p; + struct reg_sequence *p; int ret; bool bypass; @@ -2564,7 +2764,7 @@ int regmap_register_patch(struct regmap *map, const struct reg_default *regs, return 0; p = krealloc(map->patch, - sizeof(struct reg_default) * (map->patch_regs + num_regs), + sizeof(struct reg_sequence) * (map->patch_regs + num_regs), GFP_KERNEL); if (p) { memcpy(p + map->patch_regs, regs, num_regs * sizeof(*regs)); diff --git a/drivers/bcma/Kconfig b/drivers/bcma/Kconfig index be5fffb6da24..023d448ed3fa 100644 --- a/drivers/bcma/Kconfig +++ b/drivers/bcma/Kconfig @@ -92,7 +92,7 @@ config BCMA_DRIVER_GMAC_CMN config BCMA_DRIVER_GPIO bool "BCMA GPIO driver" depends on BCMA && GPIOLIB - select IRQ_DOMAIN if BCMA_HOST_SOC + select GPIOLIB_IRQCHIP if BCMA_HOST_SOC help Driver to provide access to the GPIO pins of the bcma bus. diff --git a/drivers/bcma/bcma_private.h b/drivers/bcma/bcma_private.h index 15f2b2e242ea..38f156745d53 100644 --- a/drivers/bcma/bcma_private.h +++ b/drivers/bcma/bcma_private.h @@ -34,6 +34,7 @@ int __init bcma_bus_early_register(struct bcma_bus *bus); int bcma_bus_suspend(struct bcma_bus *bus); int bcma_bus_resume(struct bcma_bus *bus); #endif +struct device *bcma_bus_get_host_dev(struct bcma_bus *bus); /* scan.c */ void bcma_detect_chip(struct bcma_bus *bus); diff --git a/drivers/bcma/driver_gpio.c b/drivers/bcma/driver_gpio.c index 5f6018e7cd4c..504899a72966 100644 --- a/drivers/bcma/driver_gpio.c +++ b/drivers/bcma/driver_gpio.c @@ -8,10 +8,8 @@ * Licensed under the GNU/GPL. See COPYING for details. */ -#include -#include +#include #include -#include #include #include @@ -79,19 +77,11 @@ static void bcma_gpio_free(struct gpio_chip *chip, unsigned gpio) } #if IS_BUILTIN(CONFIG_BCM47XX) || IS_BUILTIN(CONFIG_ARCH_BCM_5301X) -static int bcma_gpio_to_irq(struct gpio_chip *chip, unsigned gpio) -{ - struct bcma_drv_cc *cc = bcma_gpio_get_cc(chip); - - if (cc->core->bus->hosttype == BCMA_HOSTTYPE_SOC) - return irq_find_mapping(cc->irq_domain, gpio); - else - return -EINVAL; -} static void bcma_gpio_irq_unmask(struct irq_data *d) { - struct bcma_drv_cc *cc = irq_data_get_irq_chip_data(d); + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct bcma_drv_cc *cc = bcma_gpio_get_cc(gc); int gpio = irqd_to_hwirq(d); u32 val = bcma_chipco_gpio_in(cc, BIT(gpio)); @@ -101,7 +91,8 @@ static void bcma_gpio_irq_unmask(struct irq_data *d) static void bcma_gpio_irq_mask(struct irq_data *d) { - struct bcma_drv_cc *cc = irq_data_get_irq_chip_data(d); + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct bcma_drv_cc *cc = bcma_gpio_get_cc(gc); int gpio = irqd_to_hwirq(d); bcma_chipco_gpio_intmask(cc, BIT(gpio), 0); @@ -116,6 +107,7 @@ static struct irq_chip bcma_gpio_irq_chip = { static irqreturn_t bcma_gpio_irq_handler(int irq, void *dev_id) { struct bcma_drv_cc *cc = dev_id; + struct gpio_chip *gc = &cc->gpio; u32 val = bcma_cc_read32(cc, BCMA_CC_GPIOIN); u32 mask = bcma_cc_read32(cc, BCMA_CC_GPIOIRQ); u32 pol = bcma_cc_read32(cc, BCMA_CC_GPIOPOL); @@ -125,81 +117,58 @@ static irqreturn_t bcma_gpio_irq_handler(int irq, void *dev_id) if (!irqs) return IRQ_NONE; - for_each_set_bit(gpio, &irqs, cc->gpio.ngpio) - generic_handle_irq(bcma_gpio_to_irq(&cc->gpio, gpio)); + for_each_set_bit(gpio, &irqs, gc->ngpio) + generic_handle_irq(irq_find_mapping(gc->irqdomain, gpio)); bcma_chipco_gpio_polarity(cc, irqs, val & irqs); return IRQ_HANDLED; } -static int bcma_gpio_irq_domain_init(struct bcma_drv_cc *cc) +static int bcma_gpio_irq_init(struct bcma_drv_cc *cc) { struct gpio_chip *chip = &cc->gpio; - int gpio, hwirq, err; + int hwirq, err; if (cc->core->bus->hosttype != BCMA_HOSTTYPE_SOC) return 0; - cc->irq_domain = irq_domain_add_linear(NULL, chip->ngpio, - &irq_domain_simple_ops, cc); - if (!cc->irq_domain) { - err = -ENODEV; - goto err_irq_domain; - } - for (gpio = 0; gpio < chip->ngpio; gpio++) { - int irq = irq_create_mapping(cc->irq_domain, gpio); - - irq_set_chip_data(irq, cc); - irq_set_chip_and_handler(irq, &bcma_gpio_irq_chip, - handle_simple_irq); - } - hwirq = bcma_core_irq(cc->core, 0); err = request_irq(hwirq, bcma_gpio_irq_handler, IRQF_SHARED, "gpio", cc); if (err) - goto err_req_irq; + return err; bcma_chipco_gpio_intmask(cc, ~0, 0); bcma_cc_set32(cc, BCMA_CC_IRQMASK, BCMA_CC_IRQ_GPIO); - return 0; - -err_req_irq: - for (gpio = 0; gpio < chip->ngpio; gpio++) { - int irq = irq_find_mapping(cc->irq_domain, gpio); - - irq_dispose_mapping(irq); + err = gpiochip_irqchip_add(chip, + &bcma_gpio_irq_chip, + 0, + handle_simple_irq, + IRQ_TYPE_NONE); + if (err) { + free_irq(hwirq, cc); + return err; } - irq_domain_remove(cc->irq_domain); -err_irq_domain: - return err; + + return 0; } -static void bcma_gpio_irq_domain_exit(struct bcma_drv_cc *cc) +static void bcma_gpio_irq_exit(struct bcma_drv_cc *cc) { - struct gpio_chip *chip = &cc->gpio; - int gpio; - if (cc->core->bus->hosttype != BCMA_HOSTTYPE_SOC) return; bcma_cc_mask32(cc, BCMA_CC_IRQMASK, ~BCMA_CC_IRQ_GPIO); free_irq(bcma_core_irq(cc->core, 0), cc); - for (gpio = 0; gpio < chip->ngpio; gpio++) { - int irq = irq_find_mapping(cc->irq_domain, gpio); - - irq_dispose_mapping(irq); - } - irq_domain_remove(cc->irq_domain); } #else -static int bcma_gpio_irq_domain_init(struct bcma_drv_cc *cc) +static int bcma_gpio_irq_init(struct bcma_drv_cc *cc) { return 0; } -static void bcma_gpio_irq_domain_exit(struct bcma_drv_cc *cc) +static void bcma_gpio_irq_exit(struct bcma_drv_cc *cc) { } #endif @@ -218,9 +187,8 @@ int bcma_gpio_init(struct bcma_drv_cc *cc) chip->set = bcma_gpio_set_value; chip->direction_input = bcma_gpio_direction_input; chip->direction_output = bcma_gpio_direction_output; -#if IS_BUILTIN(CONFIG_BCM47XX) || IS_BUILTIN(CONFIG_ARCH_BCM_5301X) - chip->to_irq = bcma_gpio_to_irq; -#endif + chip->owner = THIS_MODULE; + chip->dev = bcma_bus_get_host_dev(bus); #if IS_BUILTIN(CONFIG_OF) if (cc->core->bus->hosttype == BCMA_HOSTTYPE_SOC) chip->of_node = cc->core->dev.of_node; @@ -248,13 +216,13 @@ int bcma_gpio_init(struct bcma_drv_cc *cc) else chip->base = -1; - err = bcma_gpio_irq_domain_init(cc); + err = gpiochip_add(chip); if (err) return err; - err = gpiochip_add(chip); + err = bcma_gpio_irq_init(cc); if (err) { - bcma_gpio_irq_domain_exit(cc); + gpiochip_remove(chip); return err; } @@ -263,7 +231,7 @@ int bcma_gpio_init(struct bcma_drv_cc *cc) int bcma_gpio_unregister(struct bcma_drv_cc *cc) { - bcma_gpio_irq_domain_exit(cc); + bcma_gpio_irq_exit(cc); gpiochip_remove(&cc->gpio); return 0; } diff --git a/drivers/bcma/main.c b/drivers/bcma/main.c index 9635f1033ce5..24882c18fcbe 100644 --- a/drivers/bcma/main.c +++ b/drivers/bcma/main.c @@ -7,11 +7,14 @@ #include "bcma_private.h" #include +#include #include +#include #include #include #include #include +#include MODULE_DESCRIPTION("Broadcom's specific AMBA driver"); MODULE_LICENSE("GPL"); @@ -268,6 +271,28 @@ void bcma_prepare_core(struct bcma_bus *bus, struct bcma_device *core) } } +struct device *bcma_bus_get_host_dev(struct bcma_bus *bus) +{ + switch (bus->hosttype) { + case BCMA_HOSTTYPE_PCI: + if (bus->host_pci) + return &bus->host_pci->dev; + else + return NULL; + case BCMA_HOSTTYPE_SOC: + if (bus->host_pdev) + return &bus->host_pdev->dev; + else + return NULL; + case BCMA_HOSTTYPE_SDIO: + if (bus->host_sdio) + return &bus->host_sdio->dev; + else + return NULL; + } + return NULL; +} + void bcma_init_bus(struct bcma_bus *bus) { mutex_lock(&bcma_buses_mutex); @@ -387,6 +412,7 @@ int bcma_bus_register(struct bcma_bus *bus) { int err; struct bcma_device *core; + struct device *dev; /* Scan for devices (cores) */ err = bcma_bus_scan(bus); @@ -409,6 +435,16 @@ int bcma_bus_register(struct bcma_bus *bus) bcma_core_pci_early_init(&bus->drv_pci[0]); } + dev = bcma_bus_get_host_dev(bus); + /* TODO: remove check for IS_BUILTIN(CONFIG_BCMA) check when + * of_default_bus_match_table is exported or in some other way + * accessible. This is just a temporary workaround. + */ + if (IS_BUILTIN(CONFIG_BCMA) && dev) { + of_platform_populate(dev->of_node, of_default_bus_match_table, + NULL, dev); + } + /* Cores providing flash access go before SPROM init */ list_for_each_entry(core, &bus->cores, list) { if (bcma_is_core_needed_early(core->id.id)) diff --git a/drivers/block/aoe/aoeblk.c b/drivers/block/aoe/aoeblk.c index 46c282fff104..dd73e1ff1759 100644 --- a/drivers/block/aoe/aoeblk.c +++ b/drivers/block/aoe/aoeblk.c @@ -395,7 +395,7 @@ aoeblk_gdalloc(void *vp) WARN_ON(d->flags & DEVFL_TKILL); WARN_ON(d->gd); WARN_ON(d->flags & DEVFL_UP); - blk_queue_max_hw_sectors(q, 1024); + blk_queue_max_hw_sectors(q, BLK_DEF_MAX_SECTORS); q->backing_dev_info.name = "aoe"; q->backing_dev_info.ra_pages = READ_AHEAD / PAGE_CACHE_SIZE; d->bufpool = mp; diff --git a/drivers/block/aoe/aoecmd.c b/drivers/block/aoe/aoecmd.c index 422b7d84f686..ad80c85e0857 100644 --- a/drivers/block/aoe/aoecmd.c +++ b/drivers/block/aoe/aoecmd.c @@ -1110,7 +1110,7 @@ aoe_end_request(struct aoedev *d, struct request *rq, int fastfail) d->ip.rq = NULL; do { bio = rq->bio; - bok = !fastfail && test_bit(BIO_UPTODATE, &bio->bi_flags); + bok = !fastfail && !bio->bi_error; } while (__blk_end_request(rq, bok ? 0 : -EIO, bio->bi_iter.bi_size)); /* cf. http://lkml.org/lkml/2006/10/31/28 */ @@ -1172,7 +1172,7 @@ ktiocomplete(struct frame *f) ahout->cmdstat, ahin->cmdstat, d->aoemajor, d->aoeminor); noskb: if (buf) - clear_bit(BIO_UPTODATE, &buf->bio->bi_flags); + buf->bio->bi_error = -EIO; goto out; } @@ -1185,7 +1185,7 @@ noskb: if (buf) "aoe: runt data size in read from", (long) d->aoemajor, d->aoeminor, skb->len, n); - clear_bit(BIO_UPTODATE, &buf->bio->bi_flags); + buf->bio->bi_error = -EIO; break; } if (n > f->iter.bi_size) { @@ -1193,7 +1193,7 @@ noskb: if (buf) "aoe: too-large data size in read from", (long) d->aoemajor, d->aoeminor, n, f->iter.bi_size); - clear_bit(BIO_UPTODATE, &buf->bio->bi_flags); + buf->bio->bi_error = -EIO; break; } bvcpy(skb, f->buf->bio, f->iter, n); @@ -1695,7 +1695,7 @@ aoe_failbuf(struct aoedev *d, struct buf *buf) if (buf == NULL) return; buf->iter.bi_size = 0; - clear_bit(BIO_UPTODATE, &buf->bio->bi_flags); + buf->bio->bi_error = -EIO; if (buf->nframesout == 0) aoe_end_buf(d, buf); } diff --git a/drivers/block/aoe/aoedev.c b/drivers/block/aoe/aoedev.c index e774c50b6842..ffd1947500c6 100644 --- a/drivers/block/aoe/aoedev.c +++ b/drivers/block/aoe/aoedev.c @@ -170,7 +170,7 @@ aoe_failip(struct aoedev *d) if (rq == NULL) return; while ((bio = d->ip.nxbio)) { - clear_bit(BIO_UPTODATE, &bio->bi_flags); + bio->bi_error = -EIO; d->ip.nxbio = bio->bi_next; n = (unsigned long) rq->special; rq->special = (void *) --n; diff --git a/drivers/block/brd.c b/drivers/block/brd.c index 64ab4951e9d6..b9794aeeb878 100644 --- a/drivers/block/brd.c +++ b/drivers/block/brd.c @@ -331,14 +331,12 @@ static void brd_make_request(struct request_queue *q, struct bio *bio) struct bio_vec bvec; sector_t sector; struct bvec_iter iter; - int err = -EIO; sector = bio->bi_iter.bi_sector; if (bio_end_sector(bio) > get_capacity(bdev->bd_disk)) - goto out; + goto io_error; if (unlikely(bio->bi_rw & REQ_DISCARD)) { - err = 0; discard_from_brd(brd, sector, bio->bi_iter.bi_size); goto out; } @@ -349,15 +347,20 @@ static void brd_make_request(struct request_queue *q, struct bio *bio) bio_for_each_segment(bvec, bio, iter) { unsigned int len = bvec.bv_len; + int err; + err = brd_do_bvec(brd, bvec.bv_page, len, bvec.bv_offset, rw, sector); if (err) - break; + goto io_error; sector += len >> SECTOR_SHIFT; } out: - bio_endio(bio, err); + bio_endio(bio); + return; +io_error: + bio_io_error(bio); } static int brd_rw_page(struct block_device *bdev, sector_t sector, @@ -371,7 +374,7 @@ static int brd_rw_page(struct block_device *bdev, sector_t sector, #ifdef CONFIG_BLK_DEV_RAM_DAX static long brd_direct_access(struct block_device *bdev, sector_t sector, - void **kaddr, unsigned long *pfn, long size) + void __pmem **kaddr, unsigned long *pfn) { struct brd_device *brd = bdev->bd_disk->private_data; struct page *page; @@ -381,13 +384,9 @@ static long brd_direct_access(struct block_device *bdev, sector_t sector, page = brd_insert_page(brd, sector); if (!page) return -ENOSPC; - *kaddr = page_address(page); + *kaddr = (void __pmem *)page_address(page); *pfn = page_to_pfn(page); - /* - * TODO: If size > PAGE_SIZE, we could look to see if the next page in - * the file happens to be mapped to the next page of physical RAM. - */ return PAGE_SIZE; } #else @@ -500,7 +499,7 @@ static struct brd_device *brd_alloc(int i) blk_queue_physical_block_size(brd->brd_queue, PAGE_SIZE); brd->brd_queue->limits.discard_granularity = PAGE_SIZE; - brd->brd_queue->limits.max_discard_sectors = UINT_MAX; + blk_queue_max_discard_sectors(brd->brd_queue, UINT_MAX); brd->brd_queue->limits.discard_zeroes_data = 1; queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, brd->brd_queue); diff --git a/drivers/block/drbd/drbd_actlog.c b/drivers/block/drbd/drbd_actlog.c index 1318e3217cb0..b3868e7a1ffd 100644 --- a/drivers/block/drbd/drbd_actlog.c +++ b/drivers/block/drbd/drbd_actlog.c @@ -175,11 +175,11 @@ static int _drbd_md_sync_page_io(struct drbd_device *device, atomic_inc(&device->md_io.in_use); /* drbd_md_put_buffer() is in the completion handler */ device->md_io.submit_jif = jiffies; if (drbd_insert_fault(device, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) - bio_endio(bio, -EIO); + bio_io_error(bio); else submit_bio(rw, bio); wait_until_done_or_force_detached(device, bdev, &device->md_io.done); - if (bio_flagged(bio, BIO_UPTODATE)) + if (!bio->bi_error) err = device->md_io.error; out: diff --git a/drivers/block/drbd/drbd_bitmap.c b/drivers/block/drbd/drbd_bitmap.c index 434c77dcc99e..e5e0f19ceda0 100644 --- a/drivers/block/drbd/drbd_bitmap.c +++ b/drivers/block/drbd/drbd_bitmap.c @@ -941,36 +941,27 @@ static void drbd_bm_aio_ctx_destroy(struct kref *kref) } /* bv_page may be a copy, or may be the original */ -static void drbd_bm_endio(struct bio *bio, int error) +static void drbd_bm_endio(struct bio *bio) { struct drbd_bm_aio_ctx *ctx = bio->bi_private; struct drbd_device *device = ctx->device; struct drbd_bitmap *b = device->bitmap; unsigned int idx = bm_page_to_idx(bio->bi_io_vec[0].bv_page); - int uptodate = bio_flagged(bio, BIO_UPTODATE); - - - /* strange behavior of some lower level drivers... - * fail the request by clearing the uptodate flag, - * but do not return any error?! - * do we want to WARN() on this? */ - if (!error && !uptodate) - error = -EIO; if ((ctx->flags & BM_AIO_COPY_PAGES) == 0 && !bm_test_page_unchanged(b->bm_pages[idx])) drbd_warn(device, "bitmap page idx %u changed during IO!\n", idx); - if (error) { + if (bio->bi_error) { /* ctx error will hold the completed-last non-zero error code, * in case error codes differ. */ - ctx->error = error; + ctx->error = bio->bi_error; bm_set_page_io_err(b->bm_pages[idx]); /* Not identical to on disk version of it. * Is BM_PAGE_IO_ERROR enough? */ if (__ratelimit(&drbd_ratelimit_state)) drbd_err(device, "IO ERROR %d on bitmap page idx %u\n", - error, idx); + bio->bi_error, idx); } else { bm_clear_page_io_err(b->bm_pages[idx]); dynamic_drbd_dbg(device, "bitmap page idx %u completed\n", idx); @@ -1031,7 +1022,7 @@ static void bm_page_io_async(struct drbd_bm_aio_ctx *ctx, int page_nr) __must_ho if (drbd_insert_fault(device, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) { bio->bi_rw |= rw; - bio_endio(bio, -EIO); + bio_io_error(bio); } else { submit_bio(rw, bio); /* this should not count as user activity and cause the diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h index efd19c2da9c2..015c6e91b756 100644 --- a/drivers/block/drbd/drbd_int.h +++ b/drivers/block/drbd/drbd_int.h @@ -1450,7 +1450,6 @@ extern void do_submit(struct work_struct *ws); extern void __drbd_make_request(struct drbd_device *, struct bio *, unsigned long); extern void drbd_make_request(struct request_queue *q, struct bio *bio); extern int drbd_read_remote(struct drbd_device *device, struct drbd_request *req); -extern int drbd_merge_bvec(struct request_queue *q, struct bvec_merge_data *bvm, struct bio_vec *bvec); extern int is_valid_ar_handle(struct drbd_request *, sector_t); @@ -1481,9 +1480,9 @@ extern int drbd_khelper(struct drbd_device *device, char *cmd); /* drbd_worker.c */ /* bi_end_io handlers */ -extern void drbd_md_endio(struct bio *bio, int error); -extern void drbd_peer_request_endio(struct bio *bio, int error); -extern void drbd_request_endio(struct bio *bio, int error); +extern void drbd_md_endio(struct bio *bio); +extern void drbd_peer_request_endio(struct bio *bio); +extern void drbd_request_endio(struct bio *bio); extern int drbd_worker(struct drbd_thread *thi); enum drbd_ret_code drbd_resync_after_valid(struct drbd_device *device, int o_minor); void drbd_resync_after_changed(struct drbd_device *device); @@ -1604,12 +1603,13 @@ static inline void drbd_generic_make_request(struct drbd_device *device, __release(local); if (!bio->bi_bdev) { drbd_err(device, "drbd_generic_make_request: bio->bi_bdev == NULL\n"); - bio_endio(bio, -ENODEV); + bio->bi_error = -ENODEV; + bio_endio(bio); return; } if (drbd_insert_fault(device, fault_type)) - bio_endio(bio, -EIO); + bio_io_error(bio); else generic_make_request(bio); } diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c index a1518539b858..74d97f4bac34 100644 --- a/drivers/block/drbd/drbd_main.c +++ b/drivers/block/drbd/drbd_main.c @@ -2774,7 +2774,6 @@ enum drbd_ret_code drbd_create_device(struct drbd_config_context *adm_ctx, unsig This triggers a max_bio_size message upon first attach or connect */ blk_queue_max_hw_sectors(q, DRBD_MAX_BIO_SIZE_SAFE >> 8); blk_queue_bounce_limit(q, BLK_BOUNCE_ANY); - blk_queue_merge_bvec(q, drbd_merge_bvec); q->queue_lock = &resource->req_lock; device->md_io.page = alloc_page(GFP_KERNEL); diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c index 74df8cfad414..e80cbefbc2b5 100644 --- a/drivers/block/drbd/drbd_nl.c +++ b/drivers/block/drbd/drbd_nl.c @@ -1156,14 +1156,14 @@ static void drbd_setup_queue_param(struct drbd_device *device, struct drbd_backi /* For now, don't allow more than one activity log extent worth of data * to be discarded in one go. We may need to rework drbd_al_begin_io() * to allow for even larger discard ranges */ - q->limits.max_discard_sectors = DRBD_MAX_DISCARD_SECTORS; + blk_queue_max_discard_sectors(q, DRBD_MAX_DISCARD_SECTORS); queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); /* REALLY? Is stacking secdiscard "legal"? */ if (blk_queue_secdiscard(b)) queue_flag_set_unlocked(QUEUE_FLAG_SECDISCARD, q); } else { - q->limits.max_discard_sectors = 0; + blk_queue_max_discard_sectors(q, 0); queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q); queue_flag_clear_unlocked(QUEUE_FLAG_SECDISCARD, q); } diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c index 3907202fb9d9..211592682169 100644 --- a/drivers/block/drbd/drbd_req.c +++ b/drivers/block/drbd/drbd_req.c @@ -201,7 +201,8 @@ void start_new_tl_epoch(struct drbd_connection *connection) void complete_master_bio(struct drbd_device *device, struct bio_and_error *m) { - bio_endio(m->bio, m->error); + m->bio->bi_error = m->error; + bio_endio(m->bio); dec_ap_bio(device); } @@ -1153,12 +1154,12 @@ drbd_submit_req_private_bio(struct drbd_request *req) rw == WRITE ? DRBD_FAULT_DT_WR : rw == READ ? DRBD_FAULT_DT_RD : DRBD_FAULT_DT_RA)) - bio_endio(bio, -EIO); + bio_io_error(bio); else generic_make_request(bio); put_ldev(device); } else - bio_endio(bio, -EIO); + bio_io_error(bio); } static void drbd_queue_write(struct drbd_device *device, struct drbd_request *req) @@ -1191,7 +1192,8 @@ drbd_request_prepare(struct drbd_device *device, struct bio *bio, unsigned long /* only pass the error to the upper layers. * if user cannot handle io errors, that's not our business. */ drbd_err(device, "could not kmalloc() req\n"); - bio_endio(bio, -ENOMEM); + bio->bi_error = -ENOMEM; + bio_endio(bio); return ERR_PTR(-ENOMEM); } req->start_jif = start_jif; @@ -1497,6 +1499,8 @@ void drbd_make_request(struct request_queue *q, struct bio *bio) struct drbd_device *device = (struct drbd_device *) q->queuedata; unsigned long start_jif; + blk_queue_split(q, &bio, q->bio_split); + start_jif = jiffies; /* @@ -1508,41 +1512,6 @@ void drbd_make_request(struct request_queue *q, struct bio *bio) __drbd_make_request(device, bio, start_jif); } -/* This is called by bio_add_page(). - * - * q->max_hw_sectors and other global limits are already enforced there. - * - * We need to call down to our lower level device, - * in case it has special restrictions. - * - * We also may need to enforce configured max-bio-bvecs limits. - * - * As long as the BIO is empty we have to allow at least one bvec, - * regardless of size and offset, so no need to ask lower levels. - */ -int drbd_merge_bvec(struct request_queue *q, struct bvec_merge_data *bvm, struct bio_vec *bvec) -{ - struct drbd_device *device = (struct drbd_device *) q->queuedata; - unsigned int bio_size = bvm->bi_size; - int limit = DRBD_MAX_BIO_SIZE; - int backing_limit; - - if (bio_size && get_ldev(device)) { - unsigned int max_hw_sectors = queue_max_hw_sectors(q); - struct request_queue * const b = - device->ldev->backing_bdev->bd_disk->queue; - if (b->merge_bvec_fn) { - bvm->bi_bdev = device->ldev->backing_bdev; - backing_limit = b->merge_bvec_fn(b, bvm, bvec); - limit = min(limit, backing_limit); - } - put_ldev(device); - if ((limit >> 9) > max_hw_sectors) - limit = max_hw_sectors << 9; - } - return limit; -} - void request_timer_fn(unsigned long data) { struct drbd_device *device = (struct drbd_device *) data; diff --git a/drivers/block/drbd/drbd_worker.c b/drivers/block/drbd/drbd_worker.c index d0fae55d871d..5578c1477ba6 100644 --- a/drivers/block/drbd/drbd_worker.c +++ b/drivers/block/drbd/drbd_worker.c @@ -65,12 +65,12 @@ rwlock_t global_state_lock; /* used for synchronous meta data and bitmap IO * submitted by drbd_md_sync_page_io() */ -void drbd_md_endio(struct bio *bio, int error) +void drbd_md_endio(struct bio *bio) { struct drbd_device *device; device = bio->bi_private; - device->md_io.error = error; + device->md_io.error = bio->bi_error; /* We grabbed an extra reference in _drbd_md_sync_page_io() to be able * to timeout on the lower level device, and eventually detach from it. @@ -170,31 +170,20 @@ void drbd_endio_write_sec_final(struct drbd_peer_request *peer_req) __releases(l /* writes on behalf of the partner, or resync writes, * "submitted" by the receiver. */ -void drbd_peer_request_endio(struct bio *bio, int error) +void drbd_peer_request_endio(struct bio *bio) { struct drbd_peer_request *peer_req = bio->bi_private; struct drbd_device *device = peer_req->peer_device->device; - int uptodate = bio_flagged(bio, BIO_UPTODATE); int is_write = bio_data_dir(bio) == WRITE; int is_discard = !!(bio->bi_rw & REQ_DISCARD); - if (error && __ratelimit(&drbd_ratelimit_state)) + if (bio->bi_error && __ratelimit(&drbd_ratelimit_state)) drbd_warn(device, "%s: error=%d s=%llus\n", is_write ? (is_discard ? "discard" : "write") - : "read", error, + : "read", bio->bi_error, (unsigned long long)peer_req->i.sector); - if (!error && !uptodate) { - if (__ratelimit(&drbd_ratelimit_state)) - drbd_warn(device, "%s: setting error to -EIO s=%llus\n", - is_write ? "write" : "read", - (unsigned long long)peer_req->i.sector); - /* strange behavior of some lower level drivers... - * fail the request by clearing the uptodate flag, - * but do not return any error?! */ - error = -EIO; - } - if (error) + if (bio->bi_error) set_bit(__EE_WAS_ERROR, &peer_req->flags); bio_put(bio); /* no need for the bio anymore */ @@ -208,24 +197,13 @@ void drbd_peer_request_endio(struct bio *bio, int error) /* read, readA or write requests on R_PRIMARY coming from drbd_make_request */ -void drbd_request_endio(struct bio *bio, int error) +void drbd_request_endio(struct bio *bio) { unsigned long flags; struct drbd_request *req = bio->bi_private; struct drbd_device *device = req->device; struct bio_and_error m; enum drbd_req_event what; - int uptodate = bio_flagged(bio, BIO_UPTODATE); - - if (!error && !uptodate) { - drbd_warn(device, "p %s: setting error to -EIO\n", - bio_data_dir(bio) == WRITE ? "write" : "read"); - /* strange behavior of some lower level drivers... - * fail the request by clearing the uptodate flag, - * but do not return any error?! */ - error = -EIO; - } - /* If this request was aborted locally before, * but now was completed "successfully", @@ -259,14 +237,14 @@ void drbd_request_endio(struct bio *bio, int error) if (__ratelimit(&drbd_ratelimit_state)) drbd_emerg(device, "delayed completion of aborted local request; disk-timeout may be too aggressive\n"); - if (!error) + if (!bio->bi_error) panic("possible random memory corruption caused by delayed completion of aborted local request\n"); } /* to avoid recursion in __req_mod */ - if (unlikely(error)) { + if (unlikely(bio->bi_error)) { if (bio->bi_rw & REQ_DISCARD) - what = (error == -EOPNOTSUPP) + what = (bio->bi_error == -EOPNOTSUPP) ? DISCARD_COMPLETED_NOTSUPP : DISCARD_COMPLETED_WITH_ERROR; else @@ -279,7 +257,7 @@ void drbd_request_endio(struct bio *bio, int error) what = COMPLETED_OK; bio_put(req->private_bio); - req->private_bio = ERR_PTR(error); + req->private_bio = ERR_PTR(bio->bi_error); /* not req_mod(), we need irqsave here! */ spin_lock_irqsave(&device->resource->req_lock, flags); diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index a08cda955285..331363e7de0f 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -3771,13 +3771,14 @@ struct rb0_cbdata { struct completion complete; }; -static void floppy_rb0_cb(struct bio *bio, int err) +static void floppy_rb0_cb(struct bio *bio) { struct rb0_cbdata *cbdata = (struct rb0_cbdata *)bio->bi_private; int drive = cbdata->drive; - if (err) { - pr_info("floppy: error %d while reading block 0\n", err); + if (bio->bi_error) { + pr_info("floppy: error %d while reading block 0\n", + bio->bi_error); set_bit(FD_OPEN_SHOULD_FAIL_BIT, &UDRS->flags); } complete(&cbdata->complete); diff --git a/drivers/block/loop.c b/drivers/block/loop.c index f7a4c9d7f721..f9889b6bc02c 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -675,7 +675,7 @@ static void loop_config_discard(struct loop_device *lo) lo->lo_encrypt_key_size) { q->limits.discard_granularity = 0; q->limits.discard_alignment = 0; - q->limits.max_discard_sectors = 0; + blk_queue_max_discard_sectors(q, 0); q->limits.discard_zeroes_data = 0; queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q); return; @@ -683,7 +683,7 @@ static void loop_config_discard(struct loop_device *lo) q->limits.discard_granularity = inode->i_sb->s_blocksize; q->limits.discard_alignment = 0; - q->limits.max_discard_sectors = UINT_MAX >> 9; + blk_queue_max_discard_sectors(q, UINT_MAX >> 9); q->limits.discard_zeroes_data = 1; queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); } diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 0e385d8e9b86..293495a75d3d 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -40,8 +41,7 @@ #include struct nbd_device { - int flags; - int harderror; /* Code of hard error */ + u32 flags; struct socket * sock; /* If == NULL, device is not ready, yet */ int magic; @@ -56,11 +56,24 @@ struct nbd_device { struct gendisk *disk; int blksize; loff_t bytesize; - pid_t pid; /* pid of nbd-client, if attached */ int xmit_timeout; - int disconnect; /* a disconnect has been requested by user */ + bool disconnect; /* a disconnect has been requested by user */ + + struct timer_list timeout_timer; + struct task_struct *task_recv; + struct task_struct *task_send; + +#if IS_ENABLED(CONFIG_DEBUG_FS) + struct dentry *dbg_dir; +#endif }; +#if IS_ENABLED(CONFIG_DEBUG_FS) +static struct dentry *nbd_dbg_dir; +#endif + +#define nbd_name(nbd) ((nbd)->disk->disk_name) + #define NBD_MAGIC 0x68797548 static unsigned int nbds_max = 16; @@ -113,26 +126,36 @@ static void nbd_end_request(struct nbd_device *nbd, struct request *req) /* * Forcibly shutdown the socket causing all listeners to error */ -static void sock_shutdown(struct nbd_device *nbd, int lock) +static void sock_shutdown(struct nbd_device *nbd) { - if (lock) - mutex_lock(&nbd->tx_lock); - if (nbd->sock) { - dev_warn(disk_to_dev(nbd->disk), "shutting down socket\n"); - kernel_sock_shutdown(nbd->sock, SHUT_RDWR); - nbd->sock = NULL; - } - if (lock) - mutex_unlock(&nbd->tx_lock); + if (!nbd->sock) + return; + + dev_warn(disk_to_dev(nbd->disk), "shutting down socket\n"); + kernel_sock_shutdown(nbd->sock, SHUT_RDWR); + nbd->sock = NULL; + del_timer_sync(&nbd->timeout_timer); } static void nbd_xmit_timeout(unsigned long arg) { - struct task_struct *task = (struct task_struct *)arg; + struct nbd_device *nbd = (struct nbd_device *)arg; + struct task_struct *task; - printk(KERN_WARNING "nbd: killing hung xmit (%s, pid: %d)\n", - task->comm, task->pid); - force_sig(SIGKILL, task); + if (list_empty(&nbd->queue_head)) + return; + + nbd->disconnect = true; + + task = READ_ONCE(nbd->task_recv); + if (task) + force_sig(SIGKILL, task); + + task = READ_ONCE(nbd->task_send); + if (task) + force_sig(SIGKILL, nbd->task_send); + + dev_err(nbd_to_dev(nbd), "Connection timed out, killed receiver and sender, shutting down connection\n"); } /* @@ -171,33 +194,12 @@ static int sock_xmit(struct nbd_device *nbd, int send, void *buf, int size, msg.msg_controllen = 0; msg.msg_flags = msg_flags | MSG_NOSIGNAL; - if (send) { - struct timer_list ti; - - if (nbd->xmit_timeout) { - init_timer(&ti); - ti.function = nbd_xmit_timeout; - ti.data = (unsigned long)current; - ti.expires = jiffies + nbd->xmit_timeout; - add_timer(&ti); - } + if (send) result = kernel_sendmsg(sock, &msg, &iov, 1, size); - if (nbd->xmit_timeout) - del_timer_sync(&ti); - } else + else result = kernel_recvmsg(sock, &msg, &iov, 1, size, msg.msg_flags); - if (signal_pending(current)) { - siginfo_t info; - printk(KERN_WARNING "nbd (pid %d: %s) got signal %d\n", - task_pid_nr(current), current->comm, - dequeue_signal_lock(current, ¤t->blocked, &info)); - result = -EINTR; - sock_shutdown(nbd, !send); - break; - } - if (result <= 0) { if (result == 0) result = -EPIPE; /* short read */ @@ -210,6 +212,9 @@ static int sock_xmit(struct nbd_device *nbd, int send, void *buf, int size, sigprocmask(SIG_SETMASK, &oldset, NULL); tsk_restore_flags(current, pflags, PF_MEMALLOC); + if (!send && nbd->xmit_timeout) + mod_timer(&nbd->timeout_timer, jiffies + nbd->xmit_timeout); + return result; } @@ -333,26 +338,24 @@ static struct request *nbd_read_stat(struct nbd_device *nbd) if (result <= 0) { dev_err(disk_to_dev(nbd->disk), "Receive control failed (result %d)\n", result); - goto harderror; + return ERR_PTR(result); } if (ntohl(reply.magic) != NBD_REPLY_MAGIC) { dev_err(disk_to_dev(nbd->disk), "Wrong magic (0x%lx)\n", (unsigned long)ntohl(reply.magic)); - result = -EPROTO; - goto harderror; + return ERR_PTR(-EPROTO); } req = nbd_find_request(nbd, *(struct request **)reply.handle); if (IS_ERR(req)) { result = PTR_ERR(req); if (result != -ENOENT) - goto harderror; + return ERR_PTR(result); dev_err(disk_to_dev(nbd->disk), "Unexpected reply (%p)\n", reply.handle); - result = -EBADR; - goto harderror; + return ERR_PTR(-EBADR); } if (ntohl(reply.error)) { @@ -380,18 +383,15 @@ static struct request *nbd_read_stat(struct nbd_device *nbd) } } return req; -harderror: - nbd->harderror = result; - return NULL; } static ssize_t pid_show(struct device *dev, struct device_attribute *attr, char *buf) { struct gendisk *disk = dev_to_disk(dev); + struct nbd_device *nbd = (struct nbd_device *)disk->private_data; - return sprintf(buf, "%ld\n", - (long) ((struct nbd_device *)disk->private_data)->pid); + return sprintf(buf, "%d\n", task_pid_nr(nbd->task_recv)); } static struct device_attribute pid_attr = { @@ -399,7 +399,7 @@ static struct device_attribute pid_attr = { .show = pid_show, }; -static int nbd_do_it(struct nbd_device *nbd) +static int nbd_thread_recv(struct nbd_device *nbd) { struct request *req; int ret; @@ -407,20 +407,43 @@ static int nbd_do_it(struct nbd_device *nbd) BUG_ON(nbd->magic != NBD_MAGIC); sk_set_memalloc(nbd->sock->sk); - nbd->pid = task_pid_nr(current); + + nbd->task_recv = current; + ret = device_create_file(disk_to_dev(nbd->disk), &pid_attr); if (ret) { dev_err(disk_to_dev(nbd->disk), "device_create_file failed!\n"); - nbd->pid = 0; + nbd->task_recv = NULL; return ret; } - while ((req = nbd_read_stat(nbd)) != NULL) + while (1) { + req = nbd_read_stat(nbd); + if (IS_ERR(req)) { + ret = PTR_ERR(req); + break; + } + nbd_end_request(nbd, req); + } device_remove_file(disk_to_dev(nbd->disk), &pid_attr); - nbd->pid = 0; - return 0; + + nbd->task_recv = NULL; + + if (signal_pending(current)) { + siginfo_t info; + + ret = dequeue_signal_lock(current, ¤t->blocked, &info); + dev_warn(nbd_to_dev(nbd), "pid %d, %s, got signal %d\n", + task_pid_nr(current), current->comm, ret); + mutex_lock(&nbd->tx_lock); + sock_shutdown(nbd); + mutex_unlock(&nbd->tx_lock); + ret = -ETIMEDOUT; + } + + return ret; } static void nbd_clear_que(struct nbd_device *nbd) @@ -455,6 +478,7 @@ static void nbd_clear_que(struct nbd_device *nbd) req->errors++; nbd_end_request(nbd, req); } + dev_dbg(disk_to_dev(nbd->disk), "queue cleared\n"); } @@ -482,6 +506,9 @@ static void nbd_handle_req(struct nbd_device *nbd, struct request *req) nbd->active_req = req; + if (nbd->xmit_timeout && list_empty_careful(&nbd->queue_head)) + mod_timer(&nbd->timeout_timer, jiffies + nbd->xmit_timeout); + if (nbd_send_req(nbd, req) != 0) { dev_err(disk_to_dev(nbd->disk), "Request send failed\n"); req->errors++; @@ -503,11 +530,13 @@ static void nbd_handle_req(struct nbd_device *nbd, struct request *req) nbd_end_request(nbd, req); } -static int nbd_thread(void *data) +static int nbd_thread_send(void *data) { struct nbd_device *nbd = data; struct request *req; + nbd->task_send = current; + set_user_nice(current, MIN_NICE); while (!kthread_should_stop() || !list_empty(&nbd->waiting_queue)) { /* wait for something to do */ @@ -515,6 +544,20 @@ static int nbd_thread(void *data) kthread_should_stop() || !list_empty(&nbd->waiting_queue)); + if (signal_pending(current)) { + siginfo_t info; + int ret; + + ret = dequeue_signal_lock(current, ¤t->blocked, + &info); + dev_warn(nbd_to_dev(nbd), "pid %d, %s, got signal %d\n", + task_pid_nr(current), current->comm, ret); + mutex_lock(&nbd->tx_lock); + sock_shutdown(nbd); + mutex_unlock(&nbd->tx_lock); + break; + } + /* extract request */ if (list_empty(&nbd->waiting_queue)) continue; @@ -528,6 +571,9 @@ static int nbd_thread(void *data) /* handle request */ nbd_handle_req(nbd, req); } + + nbd->task_send = NULL; + return 0; } @@ -538,7 +584,7 @@ static int nbd_thread(void *data) * { printk( "Warning: Ignoring result!\n"); nbd_end_request( req ); } */ -static void do_nbd_request(struct request_queue *q) +static void nbd_request_handler(struct request_queue *q) __releases(q->queue_lock) __acquires(q->queue_lock) { struct request *req; @@ -574,6 +620,9 @@ static void do_nbd_request(struct request_queue *q) } } +static int nbd_dev_dbg_init(struct nbd_device *nbd); +static void nbd_dev_dbg_close(struct nbd_device *nbd); + /* Must be called with tx_lock held */ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd, @@ -597,7 +646,7 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd, if (!nbd->sock) return -EINVAL; - nbd->disconnect = 1; + nbd->disconnect = true; nbd_send_req(nbd, &sreq); return 0; @@ -625,7 +674,7 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd, nbd->sock = sock; if (max_part > 0) bdev->bd_invalidated = 1; - nbd->disconnect = 0; /* we're connected now */ + nbd->disconnect = false; /* we're connected now */ return 0; } return -EINVAL; @@ -648,6 +697,12 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd, case NBD_SET_TIMEOUT: nbd->xmit_timeout = arg * HZ; + if (arg) + mod_timer(&nbd->timeout_timer, + jiffies + nbd->xmit_timeout); + else + del_timer_sync(&nbd->timeout_timer); + return 0; case NBD_SET_FLAGS: @@ -666,7 +721,7 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd, struct socket *sock; int error; - if (nbd->pid) + if (nbd->task_recv) return -EBUSY; if (!nbd->sock) return -EINVAL; @@ -683,24 +738,24 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd, else blk_queue_flush(nbd->disk->queue, 0); - thread = kthread_run(nbd_thread, nbd, "%s", - nbd->disk->disk_name); + thread = kthread_run(nbd_thread_send, nbd, "%s", + nbd_name(nbd)); if (IS_ERR(thread)) { mutex_lock(&nbd->tx_lock); return PTR_ERR(thread); } - error = nbd_do_it(nbd); + nbd_dev_dbg_init(nbd); + error = nbd_thread_recv(nbd); + nbd_dev_dbg_close(nbd); kthread_stop(thread); mutex_lock(&nbd->tx_lock); - if (error) - return error; - sock_shutdown(nbd, 0); + + sock_shutdown(nbd); sock = nbd->sock; nbd->sock = NULL; nbd_clear_que(nbd); - dev_warn(disk_to_dev(nbd->disk), "queue cleared\n"); kill_bdev(bdev); queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, nbd->disk->queue); set_device_ro(bdev, false); @@ -714,7 +769,7 @@ static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd, blkdev_reread_part(bdev); if (nbd->disconnect) /* user requested, ignore socket errors */ return 0; - return nbd->harderror; + return error; } case NBD_CLEAR_QUE: @@ -758,6 +813,161 @@ static const struct block_device_operations nbd_fops = .ioctl = nbd_ioctl, }; +#if IS_ENABLED(CONFIG_DEBUG_FS) + +static int nbd_dbg_tasks_show(struct seq_file *s, void *unused) +{ + struct nbd_device *nbd = s->private; + + if (nbd->task_recv) + seq_printf(s, "recv: %d\n", task_pid_nr(nbd->task_recv)); + if (nbd->task_send) + seq_printf(s, "send: %d\n", task_pid_nr(nbd->task_send)); + + return 0; +} + +static int nbd_dbg_tasks_open(struct inode *inode, struct file *file) +{ + return single_open(file, nbd_dbg_tasks_show, inode->i_private); +} + +static const struct file_operations nbd_dbg_tasks_ops = { + .open = nbd_dbg_tasks_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int nbd_dbg_flags_show(struct seq_file *s, void *unused) +{ + struct nbd_device *nbd = s->private; + u32 flags = nbd->flags; + + seq_printf(s, "Hex: 0x%08x\n\n", flags); + + seq_puts(s, "Known flags:\n"); + + if (flags & NBD_FLAG_HAS_FLAGS) + seq_puts(s, "NBD_FLAG_HAS_FLAGS\n"); + if (flags & NBD_FLAG_READ_ONLY) + seq_puts(s, "NBD_FLAG_READ_ONLY\n"); + if (flags & NBD_FLAG_SEND_FLUSH) + seq_puts(s, "NBD_FLAG_SEND_FLUSH\n"); + if (flags & NBD_FLAG_SEND_TRIM) + seq_puts(s, "NBD_FLAG_SEND_TRIM\n"); + + return 0; +} + +static int nbd_dbg_flags_open(struct inode *inode, struct file *file) +{ + return single_open(file, nbd_dbg_flags_show, inode->i_private); +} + +static const struct file_operations nbd_dbg_flags_ops = { + .open = nbd_dbg_flags_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int nbd_dev_dbg_init(struct nbd_device *nbd) +{ + struct dentry *dir; + struct dentry *f; + + dir = debugfs_create_dir(nbd_name(nbd), nbd_dbg_dir); + if (IS_ERR_OR_NULL(dir)) { + dev_err(nbd_to_dev(nbd), "Failed to create debugfs dir for '%s' (%ld)\n", + nbd_name(nbd), PTR_ERR(dir)); + return PTR_ERR(dir); + } + nbd->dbg_dir = dir; + + f = debugfs_create_file("tasks", 0444, dir, nbd, &nbd_dbg_tasks_ops); + if (IS_ERR_OR_NULL(f)) { + dev_err(nbd_to_dev(nbd), "Failed to create debugfs file 'tasks', %ld\n", + PTR_ERR(f)); + return PTR_ERR(f); + } + + f = debugfs_create_u64("size_bytes", 0444, dir, &nbd->bytesize); + if (IS_ERR_OR_NULL(f)) { + dev_err(nbd_to_dev(nbd), "Failed to create debugfs file 'size_bytes', %ld\n", + PTR_ERR(f)); + return PTR_ERR(f); + } + + f = debugfs_create_u32("timeout", 0444, dir, &nbd->xmit_timeout); + if (IS_ERR_OR_NULL(f)) { + dev_err(nbd_to_dev(nbd), "Failed to create debugfs file 'timeout', %ld\n", + PTR_ERR(f)); + return PTR_ERR(f); + } + + f = debugfs_create_u32("blocksize", 0444, dir, &nbd->blksize); + if (IS_ERR_OR_NULL(f)) { + dev_err(nbd_to_dev(nbd), "Failed to create debugfs file 'blocksize', %ld\n", + PTR_ERR(f)); + return PTR_ERR(f); + } + + f = debugfs_create_file("flags", 0444, dir, &nbd, &nbd_dbg_flags_ops); + if (IS_ERR_OR_NULL(f)) { + dev_err(nbd_to_dev(nbd), "Failed to create debugfs file 'flags', %ld\n", + PTR_ERR(f)); + return PTR_ERR(f); + } + + return 0; +} + +static void nbd_dev_dbg_close(struct nbd_device *nbd) +{ + debugfs_remove_recursive(nbd->dbg_dir); +} + +static int nbd_dbg_init(void) +{ + struct dentry *dbg_dir; + + dbg_dir = debugfs_create_dir("nbd", NULL); + if (IS_ERR(dbg_dir)) + return PTR_ERR(dbg_dir); + + nbd_dbg_dir = dbg_dir; + + return 0; +} + +static void nbd_dbg_close(void) +{ + debugfs_remove_recursive(nbd_dbg_dir); +} + +#else /* IS_ENABLED(CONFIG_DEBUG_FS) */ + +static int nbd_dev_dbg_init(struct nbd_device *nbd) +{ + return 0; +} + +static void nbd_dev_dbg_close(struct nbd_device *nbd) +{ +} + +static int nbd_dbg_init(void) +{ + return 0; +} + +static void nbd_dbg_close(void) +{ +} + +#endif + /* * And here should be modules and kernel interface * (Just smiley confuses emacs :-) @@ -811,7 +1021,7 @@ static int __init nbd_init(void) * every gendisk to have its very own request_queue struct. * These structs are big so we dynamically allocate them. */ - disk->queue = blk_init_queue(do_nbd_request, &nbd_lock); + disk->queue = blk_init_queue(nbd_request_handler, &nbd_lock); if (!disk->queue) { put_disk(disk); goto out; @@ -822,7 +1032,7 @@ static int __init nbd_init(void) queue_flag_set_unlocked(QUEUE_FLAG_NONROT, disk->queue); queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, disk->queue); disk->queue->limits.discard_granularity = 512; - disk->queue->limits.max_discard_sectors = UINT_MAX; + blk_queue_max_discard_sectors(disk->queue, UINT_MAX); disk->queue->limits.discard_zeroes_data = 0; blk_queue_max_hw_sectors(disk->queue, 65536); disk->queue->limits.max_sectors = 256; @@ -835,6 +1045,8 @@ static int __init nbd_init(void) printk(KERN_INFO "nbd: registered device at major %d\n", NBD_MAJOR); + nbd_dbg_init(); + for (i = 0; i < nbds_max; i++) { struct gendisk *disk = nbd_dev[i].disk; nbd_dev[i].magic = NBD_MAGIC; @@ -842,6 +1054,9 @@ static int __init nbd_init(void) spin_lock_init(&nbd_dev[i].queue_lock); INIT_LIST_HEAD(&nbd_dev[i].queue_head); mutex_init(&nbd_dev[i].tx_lock); + init_timer(&nbd_dev[i].timeout_timer); + nbd_dev[i].timeout_timer.function = nbd_xmit_timeout; + nbd_dev[i].timeout_timer.data = (unsigned long)&nbd_dev[i]; init_waitqueue_head(&nbd_dev[i].active_wq); init_waitqueue_head(&nbd_dev[i].waiting_wq); nbd_dev[i].blksize = 1024; @@ -868,6 +1083,9 @@ static int __init nbd_init(void) static void __exit nbd_cleanup(void) { int i; + + nbd_dbg_close(); + for (i = 0; i < nbds_max; i++) { struct gendisk *disk = nbd_dev[i].disk; nbd_dev[i].magic = 0; diff --git a/drivers/block/null_blk.c b/drivers/block/null_blk.c index 3177b245d2bd..17269a3b85f2 100644 --- a/drivers/block/null_blk.c +++ b/drivers/block/null_blk.c @@ -222,7 +222,7 @@ static void end_cmd(struct nullb_cmd *cmd) blk_end_request_all(cmd->rq, 0); break; case NULL_Q_BIO: - bio_endio(cmd->bio, 0); + bio_endio(cmd->bio); break; } diff --git a/drivers/block/nvme-core.c b/drivers/block/nvme-core.c index 7920c2741b47..b97fc3fe0916 100644 --- a/drivers/block/nvme-core.c +++ b/drivers/block/nvme-core.c @@ -72,6 +72,10 @@ module_param(nvme_char_major, int, 0); static int use_threaded_interrupts; module_param(use_threaded_interrupts, int, 0); +static bool use_cmb_sqes = true; +module_param(use_cmb_sqes, bool, 0644); +MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes"); + static DEFINE_SPINLOCK(dev_list_lock); static LIST_HEAD(dev_list); static struct task_struct *nvme_thread; @@ -103,6 +107,7 @@ struct nvme_queue { char irqname[24]; /* nvme4294967295-65535\0 */ spinlock_t q_lock; struct nvme_command *sq_cmds; + struct nvme_command __iomem *sq_cmds_io; volatile struct nvme_completion *cqes; struct blk_mq_tags **tags; dma_addr_t sq_dma_addr; @@ -379,27 +384,28 @@ static void *nvme_finish_cmd(struct nvme_queue *nvmeq, int tag, * * Safe to use from interrupt context */ -static int __nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd) +static void __nvme_submit_cmd(struct nvme_queue *nvmeq, + struct nvme_command *cmd) { u16 tail = nvmeq->sq_tail; - memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd)); + if (nvmeq->sq_cmds_io) + memcpy_toio(&nvmeq->sq_cmds_io[tail], cmd, sizeof(*cmd)); + else + memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd)); + if (++tail == nvmeq->q_depth) tail = 0; writel(tail, nvmeq->q_db); nvmeq->sq_tail = tail; - - return 0; } -static int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd) +static void nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd) { unsigned long flags; - int ret; spin_lock_irqsave(&nvmeq->q_lock, flags); - ret = __nvme_submit_cmd(nvmeq, cmd); + __nvme_submit_cmd(nvmeq, cmd); spin_unlock_irqrestore(&nvmeq->q_lock, flags); - return ret; } static __le64 **iod_list(struct nvme_iod *iod) @@ -730,18 +736,16 @@ static int nvme_setup_prps(struct nvme_dev *dev, struct nvme_iod *iod, static void nvme_submit_priv(struct nvme_queue *nvmeq, struct request *req, struct nvme_iod *iod) { - struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; + struct nvme_command cmnd; - memcpy(cmnd, req->cmd, sizeof(struct nvme_command)); - cmnd->rw.command_id = req->tag; + memcpy(&cmnd, req->cmd, sizeof(cmnd)); + cmnd.rw.command_id = req->tag; if (req->nr_phys_segments) { - cmnd->rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg)); - cmnd->rw.prp2 = cpu_to_le64(iod->first_dma); + cmnd.rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg)); + cmnd.rw.prp2 = cpu_to_le64(iod->first_dma); } - if (++nvmeq->sq_tail == nvmeq->q_depth) - nvmeq->sq_tail = 0; - writel(nvmeq->sq_tail, nvmeq->q_db); + __nvme_submit_cmd(nvmeq, &cmnd); } /* @@ -754,45 +758,41 @@ static void nvme_submit_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns, { struct nvme_dsm_range *range = (struct nvme_dsm_range *)iod_list(iod)[0]; - struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; + struct nvme_command cmnd; range->cattr = cpu_to_le32(0); range->nlb = cpu_to_le32(blk_rq_bytes(req) >> ns->lba_shift); range->slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req))); - memset(cmnd, 0, sizeof(*cmnd)); - cmnd->dsm.opcode = nvme_cmd_dsm; - cmnd->dsm.command_id = req->tag; - cmnd->dsm.nsid = cpu_to_le32(ns->ns_id); - cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma); - cmnd->dsm.nr = 0; - cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD); + memset(&cmnd, 0, sizeof(cmnd)); + cmnd.dsm.opcode = nvme_cmd_dsm; + cmnd.dsm.command_id = req->tag; + cmnd.dsm.nsid = cpu_to_le32(ns->ns_id); + cmnd.dsm.prp1 = cpu_to_le64(iod->first_dma); + cmnd.dsm.nr = 0; + cmnd.dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD); - if (++nvmeq->sq_tail == nvmeq->q_depth) - nvmeq->sq_tail = 0; - writel(nvmeq->sq_tail, nvmeq->q_db); + __nvme_submit_cmd(nvmeq, &cmnd); } static void nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns, int cmdid) { - struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; + struct nvme_command cmnd; - memset(cmnd, 0, sizeof(*cmnd)); - cmnd->common.opcode = nvme_cmd_flush; - cmnd->common.command_id = cmdid; - cmnd->common.nsid = cpu_to_le32(ns->ns_id); + memset(&cmnd, 0, sizeof(cmnd)); + cmnd.common.opcode = nvme_cmd_flush; + cmnd.common.command_id = cmdid; + cmnd.common.nsid = cpu_to_le32(ns->ns_id); - if (++nvmeq->sq_tail == nvmeq->q_depth) - nvmeq->sq_tail = 0; - writel(nvmeq->sq_tail, nvmeq->q_db); + __nvme_submit_cmd(nvmeq, &cmnd); } static int nvme_submit_iod(struct nvme_queue *nvmeq, struct nvme_iod *iod, struct nvme_ns *ns) { struct request *req = iod_get_private(iod); - struct nvme_command *cmnd; + struct nvme_command cmnd; u16 control = 0; u32 dsmgmt = 0; @@ -804,19 +804,16 @@ static int nvme_submit_iod(struct nvme_queue *nvmeq, struct nvme_iod *iod, if (req->cmd_flags & REQ_RAHEAD) dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH; - cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail]; - memset(cmnd, 0, sizeof(*cmnd)); + memset(&cmnd, 0, sizeof(cmnd)); + cmnd.rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read); + cmnd.rw.command_id = req->tag; + cmnd.rw.nsid = cpu_to_le32(ns->ns_id); + cmnd.rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg)); + cmnd.rw.prp2 = cpu_to_le64(iod->first_dma); + cmnd.rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req))); + cmnd.rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1); - cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read); - cmnd->rw.command_id = req->tag; - cmnd->rw.nsid = cpu_to_le32(ns->ns_id); - cmnd->rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg)); - cmnd->rw.prp2 = cpu_to_le64(iod->first_dma); - cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req))); - cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1); - - if (blk_integrity_rq(req)) { - cmnd->rw.metadata = cpu_to_le64(sg_dma_address(iod->meta_sg)); + if (ns->ms) { switch (ns->pi_type) { case NVME_NS_DPS_PI_TYPE3: control |= NVME_RW_PRINFO_PRCHK_GUARD; @@ -825,19 +822,21 @@ static int nvme_submit_iod(struct nvme_queue *nvmeq, struct nvme_iod *iod, case NVME_NS_DPS_PI_TYPE2: control |= NVME_RW_PRINFO_PRCHK_GUARD | NVME_RW_PRINFO_PRCHK_REF; - cmnd->rw.reftag = cpu_to_le32( + cmnd.rw.reftag = cpu_to_le32( nvme_block_nr(ns, blk_rq_pos(req))); break; } - } else if (ns->ms) - control |= NVME_RW_PRINFO_PRACT; + if (blk_integrity_rq(req)) + cmnd.rw.metadata = + cpu_to_le64(sg_dma_address(iod->meta_sg)); + else + control |= NVME_RW_PRINFO_PRACT; + } - cmnd->rw.control = cpu_to_le16(control); - cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt); + cmnd.rw.control = cpu_to_le16(control); + cmnd.rw.dsmgmt = cpu_to_le32(dsmgmt); - if (++nvmeq->sq_tail == nvmeq->q_depth) - nvmeq->sq_tail = 0; - writel(nvmeq->sq_tail, nvmeq->q_db); + __nvme_submit_cmd(nvmeq, &cmnd); return 0; } @@ -1080,7 +1079,8 @@ static int nvme_submit_async_admin_req(struct nvme_dev *dev) c.common.command_id = req->tag; blk_mq_free_request(req); - return __nvme_submit_cmd(nvmeq, &c); + __nvme_submit_cmd(nvmeq, &c); + return 0; } static int nvme_submit_admin_async_cmd(struct nvme_dev *dev, @@ -1103,7 +1103,8 @@ static int nvme_submit_admin_async_cmd(struct nvme_dev *dev, cmd->common.command_id = req->tag; - return nvme_submit_cmd(nvmeq, cmd); + nvme_submit_cmd(nvmeq, cmd); + return 0; } static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id) @@ -1315,12 +1316,7 @@ static void nvme_abort_req(struct request *req) dev_warn(nvmeq->q_dmadev, "Aborting I/O %d QID %d\n", req->tag, nvmeq->qid); - if (nvme_submit_cmd(dev->queues[0], &cmd) < 0) { - dev_warn(nvmeq->q_dmadev, - "Could not abort I/O %d QID %d", - req->tag, nvmeq->qid); - blk_mq_free_request(abort_req); - } + nvme_submit_cmd(dev->queues[0], &cmd); } static void nvme_cancel_queue_ios(struct request *req, void *data, bool reserved) @@ -1374,7 +1370,8 @@ static void nvme_free_queue(struct nvme_queue *nvmeq) { dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth), (void *)nvmeq->cqes, nvmeq->cq_dma_addr); - dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth), + if (nvmeq->sq_cmds) + dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth), nvmeq->sq_cmds, nvmeq->sq_dma_addr); kfree(nvmeq); } @@ -1447,6 +1444,47 @@ static void nvme_disable_queue(struct nvme_dev *dev, int qid) spin_unlock_irq(&nvmeq->q_lock); } +static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues, + int entry_size) +{ + int q_depth = dev->q_depth; + unsigned q_size_aligned = roundup(q_depth * entry_size, dev->page_size); + + if (q_size_aligned * nr_io_queues > dev->cmb_size) { + u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues); + mem_per_q = round_down(mem_per_q, dev->page_size); + q_depth = div_u64(mem_per_q, entry_size); + + /* + * Ensure the reduced q_depth is above some threshold where it + * would be better to map queues in system memory with the + * original depth + */ + if (q_depth < 64) + return -ENOMEM; + } + + return q_depth; +} + +static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq, + int qid, int depth) +{ + if (qid && dev->cmb && use_cmb_sqes && NVME_CMB_SQS(dev->cmbsz)) { + unsigned offset = (qid - 1) * + roundup(SQ_SIZE(depth), dev->page_size); + nvmeq->sq_dma_addr = dev->cmb_dma_addr + offset; + nvmeq->sq_cmds_io = dev->cmb + offset; + } else { + nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(depth), + &nvmeq->sq_dma_addr, GFP_KERNEL); + if (!nvmeq->sq_cmds) + return -ENOMEM; + } + + return 0; +} + static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth) { @@ -1459,9 +1497,7 @@ static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid, if (!nvmeq->cqes) goto free_nvmeq; - nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(depth), - &nvmeq->sq_dma_addr, GFP_KERNEL); - if (!nvmeq->sq_cmds) + if (nvme_alloc_sq_cmds(dev, nvmeq, qid, depth)) goto free_cqdma; nvmeq->q_dmadev = dev->dev; @@ -1696,6 +1732,12 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev) page_shift = dev_page_max; } + dev->subsystem = readl(&dev->bar->vs) >= NVME_VS(1, 1) ? + NVME_CAP_NSSRC(cap) : 0; + + if (dev->subsystem && (readl(&dev->bar->csts) & NVME_CSTS_NSSRO)) + writel(NVME_CSTS_NSSRO, &dev->bar->csts); + result = nvme_disable_ctrl(dev, cap); if (result < 0) return result; @@ -1856,6 +1898,15 @@ static int nvme_user_cmd(struct nvme_dev *dev, struct nvme_ns *ns, return status; } +static int nvme_subsys_reset(struct nvme_dev *dev) +{ + if (!dev->subsystem) + return -ENOTTY; + + writel(0x4E564D65, &dev->bar->nssr); /* "NVMe" */ + return 0; +} + static int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg) { @@ -1935,7 +1986,7 @@ static void nvme_config_discard(struct nvme_ns *ns) ns->queue->limits.discard_zeroes_data = 0; ns->queue->limits.discard_alignment = logical_block_size; ns->queue->limits.discard_granularity = logical_block_size; - ns->queue->limits.max_discard_sectors = 0xffffffff; + blk_queue_max_discard_sectors(ns->queue, 0xffffffff); queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue); } @@ -1989,7 +2040,7 @@ static int nvme_revalidate_disk(struct gendisk *disk) !ns->ext) nvme_init_integrity(ns); - if (ns->ms && !blk_get_integrity(disk)) + if (ns->ms && !(ns->ms == 8 && ns->pi_type) && !blk_get_integrity(disk)) set_capacity(disk, 0); else set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9)); @@ -2020,7 +2071,10 @@ static int nvme_kthread(void *data) spin_lock(&dev_list_lock); list_for_each_entry_safe(dev, next, &dev_list, node) { int i; - if (readl(&dev->bar->csts) & NVME_CSTS_CFS) { + u32 csts = readl(&dev->bar->csts); + + if ((dev->subsystem && (csts & NVME_CSTS_NSSRO)) || + csts & NVME_CSTS_CFS) { if (work_busy(&dev->reset_work)) continue; list_del_init(&dev->node); @@ -2067,7 +2121,6 @@ static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid) goto out_free_ns; queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue); queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue); - queue_flag_set_unlocked(QUEUE_FLAG_SG_GAPS, ns->queue); ns->dev = dev; ns->queue->queuedata = ns; @@ -2081,12 +2134,16 @@ static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid) list_add_tail(&ns->list, &dev->namespaces); blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift); - if (dev->max_hw_sectors) + if (dev->max_hw_sectors) { blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors); + blk_queue_max_segments(ns->queue, + ((dev->max_hw_sectors << 9) / dev->page_size) + 1); + } if (dev->stripe_size) blk_queue_chunk_sectors(ns->queue, dev->stripe_size >> 9); if (dev->vwc & NVME_CTRL_VWC_PRESENT) blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA); + blk_queue_virt_boundary(ns->queue, dev->page_size - 1); disk->major = nvme_major; disk->first_minor = 0; @@ -2159,6 +2216,58 @@ static int set_queue_count(struct nvme_dev *dev, int count) return min(result & 0xffff, result >> 16) + 1; } +static void __iomem *nvme_map_cmb(struct nvme_dev *dev) +{ + u64 szu, size, offset; + u32 cmbloc; + resource_size_t bar_size; + struct pci_dev *pdev = to_pci_dev(dev->dev); + void __iomem *cmb; + dma_addr_t dma_addr; + + if (!use_cmb_sqes) + return NULL; + + dev->cmbsz = readl(&dev->bar->cmbsz); + if (!(NVME_CMB_SZ(dev->cmbsz))) + return NULL; + + cmbloc = readl(&dev->bar->cmbloc); + + szu = (u64)1 << (12 + 4 * NVME_CMB_SZU(dev->cmbsz)); + size = szu * NVME_CMB_SZ(dev->cmbsz); + offset = szu * NVME_CMB_OFST(cmbloc); + bar_size = pci_resource_len(pdev, NVME_CMB_BIR(cmbloc)); + + if (offset > bar_size) + return NULL; + + /* + * Controllers may support a CMB size larger than their BAR, + * for example, due to being behind a bridge. Reduce the CMB to + * the reported size of the BAR + */ + if (size > bar_size - offset) + size = bar_size - offset; + + dma_addr = pci_resource_start(pdev, NVME_CMB_BIR(cmbloc)) + offset; + cmb = ioremap_wc(dma_addr, size); + if (!cmb) + return NULL; + + dev->cmb_dma_addr = dma_addr; + dev->cmb_size = size; + return cmb; +} + +static inline void nvme_release_cmb(struct nvme_dev *dev) +{ + if (dev->cmb) { + iounmap(dev->cmb); + dev->cmb = NULL; + } +} + static size_t db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues) { return 4096 + ((nr_io_queues + 1) * 8 * dev->db_stride); @@ -2177,6 +2286,15 @@ static int nvme_setup_io_queues(struct nvme_dev *dev) if (result < nr_io_queues) nr_io_queues = result; + if (dev->cmb && NVME_CMB_SQS(dev->cmbsz)) { + result = nvme_cmb_qdepth(dev, nr_io_queues, + sizeof(struct nvme_command)); + if (result > 0) + dev->q_depth = result; + else + nvme_release_cmb(dev); + } + size = db_bar_size(dev, nr_io_queues); if (size > 8192) { iounmap(dev->bar); @@ -2344,7 +2462,6 @@ static int nvme_dev_add(struct nvme_dev *dev) { struct pci_dev *pdev = to_pci_dev(dev->dev); int res; - unsigned nn; struct nvme_id_ctrl *ctrl; int shift = NVME_CAP_MPSMIN(readq(&dev->bar->cap)) + 12; @@ -2354,7 +2471,6 @@ static int nvme_dev_add(struct nvme_dev *dev) return -EIO; } - nn = le32_to_cpup(&ctrl->nn); dev->oncs = le16_to_cpup(&ctrl->oncs); dev->abort_limit = ctrl->acl + 1; dev->vwc = ctrl->vwc; @@ -2440,6 +2556,8 @@ static int nvme_dev_map(struct nvme_dev *dev) dev->q_depth = min_t(int, NVME_CAP_MQES(cap) + 1, NVME_Q_DEPTH); dev->db_stride = 1 << NVME_CAP_STRIDE(cap); dev->dbs = ((void __iomem *)dev->bar) + 4096; + if (readl(&dev->bar->vs) >= NVME_VS(1, 2)) + dev->cmb = nvme_map_cmb(dev); return 0; @@ -2820,6 +2938,8 @@ static long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg) case NVME_IOCTL_RESET: dev_warn(dev->dev, "resetting controller\n"); return nvme_reset(dev); + case NVME_IOCTL_SUBSYS_RESET: + return nvme_subsys_reset(dev); default: return -ENOTTY; } @@ -3145,6 +3265,7 @@ static void nvme_remove(struct pci_dev *pdev) nvme_dev_remove_admin(dev); device_destroy(nvme_class, MKDEV(nvme_char_major, dev->instance)); nvme_free_queues(dev, 0); + nvme_release_cmb(dev); nvme_release_prp_pools(dev); kref_put(&dev->kref, nvme_free_dev); } diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index 4c20c228184c..7be2375db7f2 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c @@ -977,7 +977,7 @@ static void pkt_make_local_copy(struct packet_data *pkt, struct bio_vec *bvec) } } -static void pkt_end_io_read(struct bio *bio, int err) +static void pkt_end_io_read(struct bio *bio) { struct packet_data *pkt = bio->bi_private; struct pktcdvd_device *pd = pkt->pd; @@ -985,9 +985,9 @@ static void pkt_end_io_read(struct bio *bio, int err) pkt_dbg(2, pd, "bio=%p sec0=%llx sec=%llx err=%d\n", bio, (unsigned long long)pkt->sector, - (unsigned long long)bio->bi_iter.bi_sector, err); + (unsigned long long)bio->bi_iter.bi_sector, bio->bi_error); - if (err) + if (bio->bi_error) atomic_inc(&pkt->io_errors); if (atomic_dec_and_test(&pkt->io_wait)) { atomic_inc(&pkt->run_sm); @@ -996,13 +996,13 @@ static void pkt_end_io_read(struct bio *bio, int err) pkt_bio_finished(pd); } -static void pkt_end_io_packet_write(struct bio *bio, int err) +static void pkt_end_io_packet_write(struct bio *bio) { struct packet_data *pkt = bio->bi_private; struct pktcdvd_device *pd = pkt->pd; BUG_ON(!pd); - pkt_dbg(2, pd, "id=%d, err=%d\n", pkt->id, err); + pkt_dbg(2, pd, "id=%d, err=%d\n", pkt->id, bio->bi_error); pd->stats.pkt_ended++; @@ -1340,22 +1340,22 @@ static void pkt_start_write(struct pktcdvd_device *pd, struct packet_data *pkt) pkt_queue_bio(pd, pkt->w_bio); } -static void pkt_finish_packet(struct packet_data *pkt, int uptodate) +static void pkt_finish_packet(struct packet_data *pkt, int error) { struct bio *bio; - if (!uptodate) + if (error) pkt->cache_valid = 0; /* Finish all bios corresponding to this packet */ - while ((bio = bio_list_pop(&pkt->orig_bios))) - bio_endio(bio, uptodate ? 0 : -EIO); + while ((bio = bio_list_pop(&pkt->orig_bios))) { + bio->bi_error = error; + bio_endio(bio); + } } static void pkt_run_state_machine(struct pktcdvd_device *pd, struct packet_data *pkt) { - int uptodate; - pkt_dbg(2, pd, "pkt %d\n", pkt->id); for (;;) { @@ -1384,7 +1384,7 @@ static void pkt_run_state_machine(struct pktcdvd_device *pd, struct packet_data if (atomic_read(&pkt->io_wait) > 0) return; - if (test_bit(BIO_UPTODATE, &pkt->w_bio->bi_flags)) { + if (!pkt->w_bio->bi_error) { pkt_set_state(pkt, PACKET_FINISHED_STATE); } else { pkt_set_state(pkt, PACKET_RECOVERY_STATE); @@ -1401,8 +1401,7 @@ static void pkt_run_state_machine(struct pktcdvd_device *pd, struct packet_data break; case PACKET_FINISHED_STATE: - uptodate = test_bit(BIO_UPTODATE, &pkt->w_bio->bi_flags); - pkt_finish_packet(pkt, uptodate); + pkt_finish_packet(pkt, pkt->w_bio->bi_error); return; default: @@ -2332,13 +2331,14 @@ static void pkt_close(struct gendisk *disk, fmode_t mode) } -static void pkt_end_io_read_cloned(struct bio *bio, int err) +static void pkt_end_io_read_cloned(struct bio *bio) { struct packet_stacked_data *psd = bio->bi_private; struct pktcdvd_device *pd = psd->pd; + psd->bio->bi_error = bio->bi_error; bio_put(bio); - bio_endio(psd->bio, err); + bio_endio(psd->bio); mempool_free(psd, psd_pool); pkt_bio_finished(pd); } @@ -2447,6 +2447,10 @@ static void pkt_make_request(struct request_queue *q, struct bio *bio) char b[BDEVNAME_SIZE]; struct bio *split; + blk_queue_bounce(q, &bio); + + blk_queue_split(q, &bio, q->bio_split); + pd = q->queuedata; if (!pd) { pr_err("%s incorrect request queue\n", @@ -2477,8 +2481,6 @@ static void pkt_make_request(struct request_queue *q, struct bio *bio) goto end_io; } - blk_queue_bounce(q, &bio); - do { sector_t zone = get_zone(bio->bi_iter.bi_sector, pd); sector_t last_zone = get_zone(bio_end_sector(bio) - 1, pd); @@ -2504,26 +2506,6 @@ static void pkt_make_request(struct request_queue *q, struct bio *bio) -static int pkt_merge_bvec(struct request_queue *q, struct bvec_merge_data *bmd, - struct bio_vec *bvec) -{ - struct pktcdvd_device *pd = q->queuedata; - sector_t zone = get_zone(bmd->bi_sector, pd); - int used = ((bmd->bi_sector - zone) << 9) + bmd->bi_size; - int remaining = (pd->settings.size << 9) - used; - int remaining2; - - /* - * A bio <= PAGE_SIZE must be allowed. If it crosses a packet - * boundary, pkt_make_request() will split the bio. - */ - remaining2 = PAGE_SIZE - bmd->bi_size; - remaining = max(remaining, remaining2); - - BUG_ON(remaining < 0); - return remaining; -} - static void pkt_init_queue(struct pktcdvd_device *pd) { struct request_queue *q = pd->disk->queue; @@ -2531,7 +2513,6 @@ static void pkt_init_queue(struct pktcdvd_device *pd) blk_queue_make_request(q, pkt_make_request); blk_queue_logical_block_size(q, CD_FRAMESIZE); blk_queue_max_hw_sectors(q, PACKET_MAX_SECTORS); - blk_queue_merge_bvec(q, pkt_merge_bvec); q->queuedata = pd; } diff --git a/drivers/block/ps3vram.c b/drivers/block/ps3vram.c index b1612eb16172..d89fcac59515 100644 --- a/drivers/block/ps3vram.c +++ b/drivers/block/ps3vram.c @@ -593,7 +593,8 @@ static struct bio *ps3vram_do_bio(struct ps3_system_bus_device *dev, next = bio_list_peek(&priv->list); spin_unlock_irq(&priv->lock); - bio_endio(bio, error); + bio->bi_error = error; + bio_endio(bio); return next; } @@ -605,6 +606,8 @@ static void ps3vram_make_request(struct request_queue *q, struct bio *bio) dev_dbg(&dev->core, "%s\n", __func__); + blk_queue_split(q, &bio, q->bio_split); + spin_lock_irq(&priv->lock); busy = !bio_list_empty(&priv->list); bio_list_add(&priv->list, bio); diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index bc67a93aa4f4..d93a0372b37b 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -3474,52 +3474,6 @@ static int rbd_queue_rq(struct blk_mq_hw_ctx *hctx, return BLK_MQ_RQ_QUEUE_OK; } -/* - * a queue callback. Makes sure that we don't create a bio that spans across - * multiple osd objects. One exception would be with a single page bios, - * which we handle later at bio_chain_clone_range() - */ -static int rbd_merge_bvec(struct request_queue *q, struct bvec_merge_data *bmd, - struct bio_vec *bvec) -{ - struct rbd_device *rbd_dev = q->queuedata; - sector_t sector_offset; - sector_t sectors_per_obj; - sector_t obj_sector_offset; - int ret; - - /* - * Find how far into its rbd object the partition-relative - * bio start sector is to offset relative to the enclosing - * device. - */ - sector_offset = get_start_sect(bmd->bi_bdev) + bmd->bi_sector; - sectors_per_obj = 1 << (rbd_dev->header.obj_order - SECTOR_SHIFT); - obj_sector_offset = sector_offset & (sectors_per_obj - 1); - - /* - * Compute the number of bytes from that offset to the end - * of the object. Account for what's already used by the bio. - */ - ret = (int) (sectors_per_obj - obj_sector_offset) << SECTOR_SHIFT; - if (ret > bmd->bi_size) - ret -= bmd->bi_size; - else - ret = 0; - - /* - * Don't send back more than was asked for. And if the bio - * was empty, let the whole thing through because: "Note - * that a block device *must* allow a single page to be - * added to an empty bio." - */ - rbd_assert(bvec->bv_len <= PAGE_SIZE); - if (ret > (int) bvec->bv_len || !bmd->bi_size) - ret = (int) bvec->bv_len; - - return ret; -} - static void rbd_free_disk(struct rbd_device *rbd_dev) { struct gendisk *disk = rbd_dev->disk; @@ -3815,10 +3769,9 @@ static int rbd_init_disk(struct rbd_device *rbd_dev) queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); q->limits.discard_granularity = segment_size; q->limits.discard_alignment = segment_size; - q->limits.max_discard_sectors = segment_size / SECTOR_SIZE; + blk_queue_max_discard_sectors(q, segment_size / SECTOR_SIZE); q->limits.discard_zeroes_data = 1; - blk_queue_merge_bvec(q, rbd_merge_bvec); disk->queue = q; q->queuedata = rbd_dev; @@ -4720,7 +4673,10 @@ static int rbd_dev_v2_header_info(struct rbd_device *rbd_dev) } ret = rbd_dev_v2_snap_context(rbd_dev); - dout("rbd_dev_v2_snap_context returned %d\n", ret); + if (ret && first_time) { + kfree(rbd_dev->header.object_prefix); + rbd_dev->header.object_prefix = NULL; + } return ret; } @@ -5201,7 +5157,6 @@ static int rbd_dev_probe_parent(struct rbd_device *rbd_dev) out_err: if (parent) { rbd_dev_unparent(rbd_dev); - kfree(rbd_dev->header_name); rbd_dev_destroy(parent); } else { rbd_put_client(rbdc); diff --git a/drivers/block/rsxx/dev.c b/drivers/block/rsxx/dev.c index ac8c62cb4875..3163e4cdc2cc 100644 --- a/drivers/block/rsxx/dev.c +++ b/drivers/block/rsxx/dev.c @@ -137,7 +137,10 @@ static void bio_dma_done_cb(struct rsxx_cardinfo *card, if (!card->eeh_state && card->gendisk) disk_stats_complete(card, meta->bio, meta->start_time); - bio_endio(meta->bio, atomic_read(&meta->error) ? -EIO : 0); + if (atomic_read(&meta->error)) + bio_io_error(meta->bio); + else + bio_endio(meta->bio); kmem_cache_free(bio_meta_pool, meta); } } @@ -148,6 +151,8 @@ static void rsxx_make_request(struct request_queue *q, struct bio *bio) struct rsxx_bio_meta *bio_meta; int st = -EINVAL; + blk_queue_split(q, &bio, q->bio_split); + might_sleep(); if (!card) @@ -199,7 +204,9 @@ static void rsxx_make_request(struct request_queue *q, struct bio *bio) queue_err: kmem_cache_free(bio_meta_pool, bio_meta); req_err: - bio_endio(bio, st); + if (st) + bio->bi_error = st; + bio_endio(bio); } /*----------------- Device Setup -------------------*/ diff --git a/drivers/block/skd_main.c b/drivers/block/skd_main.c index 1e46eb2305c0..586f9168ffa4 100644 --- a/drivers/block/skd_main.c +++ b/drivers/block/skd_main.c @@ -4422,7 +4422,7 @@ static int skd_cons_disk(struct skd_device *skdev) /* DISCARD Flag initialization. */ q->limits.discard_granularity = 8192; q->limits.discard_alignment = 0; - q->limits.max_discard_sectors = UINT_MAX >> 9; + blk_queue_max_discard_sectors(q, UINT_MAX >> 9); q->limits.discard_zeroes_data = 1; queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q); queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q); diff --git a/drivers/block/umem.c b/drivers/block/umem.c index 4cf81b5bf0f7..04d65790a886 100644 --- a/drivers/block/umem.c +++ b/drivers/block/umem.c @@ -456,7 +456,7 @@ static void process_page(unsigned long data) PCI_DMA_TODEVICE : PCI_DMA_FROMDEVICE); if (control & DMASCR_HARD_ERROR) { /* error */ - clear_bit(BIO_UPTODATE, &bio->bi_flags); + bio->bi_error = -EIO; dev_printk(KERN_WARNING, &card->dev->dev, "I/O error on sector %d/%d\n", le32_to_cpu(desc->local_addr)>>9, @@ -505,7 +505,7 @@ static void process_page(unsigned long data) return_bio = bio->bi_next; bio->bi_next = NULL; - bio_endio(bio, 0); + bio_endio(bio); } } @@ -531,6 +531,8 @@ static void mm_make_request(struct request_queue *q, struct bio *bio) (unsigned long long)bio->bi_iter.bi_sector, bio->bi_iter.bi_size); + blk_queue_split(q, &bio, q->bio_split); + spin_lock_irq(&card->lock); *card->biotail = bio; bio->bi_next = NULL; diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index d4d05f064d39..e93899cc6f60 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -478,8 +478,7 @@ static int virtblk_get_cache_mode(struct virtio_device *vdev) struct virtio_blk_config, wce, &writeback); if (err) - writeback = virtio_has_feature(vdev, VIRTIO_BLK_F_WCE) || - virtio_has_feature(vdev, VIRTIO_F_VERSION_1); + writeback = virtio_has_feature(vdev, VIRTIO_BLK_F_WCE); return writeback; } @@ -657,6 +656,7 @@ static int virtblk_probe(struct virtio_device *vdev) vblk->disk->private_data = vblk; vblk->disk->fops = &virtblk_fops; vblk->disk->driverfs_dev = &vdev->dev; + vblk->disk->flags |= GENHD_FL_EXT_DEVT; vblk->index = index; /* configure queue flush support */ @@ -840,7 +840,7 @@ static unsigned int features_legacy[] = { static unsigned int features[] = { VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY, VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE, - VIRTIO_BLK_F_TOPOLOGY, + VIRTIO_BLK_F_WCE, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE, VIRTIO_BLK_F_MQ, }; diff --git a/drivers/block/xen-blkback/blkback.c b/drivers/block/xen-blkback/blkback.c index 954c0029fb3b..6a685aec6994 100644 --- a/drivers/block/xen-blkback/blkback.c +++ b/drivers/block/xen-blkback/blkback.c @@ -1078,9 +1078,9 @@ static void __end_block_io_op(struct pending_req *pending_req, int error) /* * bio callback. */ -static void end_block_io_op(struct bio *bio, int error) +static void end_block_io_op(struct bio *bio) { - __end_block_io_op(bio->bi_private, error); + __end_block_io_op(bio->bi_private, bio->bi_error); bio_put(bio); } diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c index 7a8a73f1fc04..0823a96902f8 100644 --- a/drivers/block/xen-blkfront.c +++ b/drivers/block/xen-blkfront.c @@ -37,6 +37,7 @@ #include #include +#include #include #include #include @@ -82,7 +83,6 @@ struct blk_shadow { struct split_bio { struct bio *bio; atomic_t pending; - int err; }; static DEFINE_MUTEX(blkfront_mutex); @@ -148,6 +148,7 @@ struct blkfront_info unsigned int feature_persistent:1; unsigned int max_indirect_segments; int is_ready; + struct blk_mq_tag_set tag_set; }; static unsigned int nr_minors; @@ -248,7 +249,7 @@ static struct grant *get_grant(grant_ref_t *gref_head, struct blkfront_info *info) { struct grant *gnt_list_entry; - unsigned long buffer_mfn; + unsigned long buffer_gfn; BUG_ON(list_empty(&info->grants)); gnt_list_entry = list_first_entry(&info->grants, struct grant, @@ -267,10 +268,10 @@ static struct grant *get_grant(grant_ref_t *gref_head, BUG_ON(!pfn); gnt_list_entry->pfn = pfn; } - buffer_mfn = pfn_to_mfn(gnt_list_entry->pfn); + buffer_gfn = pfn_to_gfn(gnt_list_entry->pfn); gnttab_grant_foreign_access_ref(gnt_list_entry->gref, info->xbdev->otherend_id, - buffer_mfn, 0); + buffer_gfn, 0); return gnt_list_entry; } @@ -617,54 +618,41 @@ static inline bool blkif_request_flush_invalid(struct request *req, !(info->feature_flush & REQ_FUA))); } -/* - * do_blkif_request - * read a block; request is in a request queue - */ -static void do_blkif_request(struct request_queue *rq) +static int blkif_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *qd) { - struct blkfront_info *info = NULL; - struct request *req; - int queued; + struct blkfront_info *info = qd->rq->rq_disk->private_data; - pr_debug("Entered do_blkif_request\n"); + blk_mq_start_request(qd->rq); + spin_lock_irq(&info->io_lock); + if (RING_FULL(&info->ring)) + goto out_busy; - queued = 0; + if (blkif_request_flush_invalid(qd->rq, info)) + goto out_err; - while ((req = blk_peek_request(rq)) != NULL) { - info = req->rq_disk->private_data; + if (blkif_queue_request(qd->rq)) + goto out_busy; - if (RING_FULL(&info->ring)) - goto wait; + flush_requests(info); + spin_unlock_irq(&info->io_lock); + return BLK_MQ_RQ_QUEUE_OK; - blk_start_request(req); +out_err: + spin_unlock_irq(&info->io_lock); + return BLK_MQ_RQ_QUEUE_ERROR; - if (blkif_request_flush_invalid(req, info)) { - __blk_end_request_all(req, -EOPNOTSUPP); - continue; - } - - pr_debug("do_blk_req %p: cmd %p, sec %lx, " - "(%u/%u) [%s]\n", - req, req->cmd, (unsigned long)blk_rq_pos(req), - blk_rq_cur_sectors(req), blk_rq_sectors(req), - rq_data_dir(req) ? "write" : "read"); - - if (blkif_queue_request(req)) { - blk_requeue_request(rq, req); -wait: - /* Avoid pointless unplugs. */ - blk_stop_queue(rq); - break; - } - - queued++; - } - - if (queued != 0) - flush_requests(info); +out_busy: + spin_unlock_irq(&info->io_lock); + blk_mq_stop_hw_queue(hctx); + return BLK_MQ_RQ_QUEUE_BUSY; } +static struct blk_mq_ops blkfront_mq_ops = { + .queue_rq = blkif_queue_rq, + .map_queue = blk_mq_map_queue, +}; + static int xlvbd_init_blk_queue(struct gendisk *gd, u16 sector_size, unsigned int physical_sector_size, unsigned int segments) @@ -672,9 +660,22 @@ static int xlvbd_init_blk_queue(struct gendisk *gd, u16 sector_size, struct request_queue *rq; struct blkfront_info *info = gd->private_data; - rq = blk_init_queue(do_blkif_request, &info->io_lock); - if (rq == NULL) + memset(&info->tag_set, 0, sizeof(info->tag_set)); + info->tag_set.ops = &blkfront_mq_ops; + info->tag_set.nr_hw_queues = 1; + info->tag_set.queue_depth = BLK_RING_SIZE(info); + info->tag_set.numa_node = NUMA_NO_NODE; + info->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE; + info->tag_set.cmd_size = 0; + info->tag_set.driver_data = info; + + if (blk_mq_alloc_tag_set(&info->tag_set)) return -1; + rq = blk_mq_init_queue(&info->tag_set); + if (IS_ERR(rq)) { + blk_mq_free_tag_set(&info->tag_set); + return -1; + } queue_flag_set_unlocked(QUEUE_FLAG_VIRT, rq); @@ -902,19 +903,15 @@ static int xlvbd_alloc_gendisk(blkif_sector_t capacity, static void xlvbd_release_gendisk(struct blkfront_info *info) { unsigned int minor, nr_minors; - unsigned long flags; if (info->rq == NULL) return; - spin_lock_irqsave(&info->io_lock, flags); - /* No more blkif_request(). */ - blk_stop_queue(info->rq); + blk_mq_stop_hw_queues(info->rq); /* No more gnttab callback work. */ gnttab_cancel_free_callback(&info->callback); - spin_unlock_irqrestore(&info->io_lock, flags); /* Flush gnttab callback work. Must be done with no locks held. */ flush_work(&info->work); @@ -926,20 +923,18 @@ static void xlvbd_release_gendisk(struct blkfront_info *info) xlbd_release_minors(minor, nr_minors); blk_cleanup_queue(info->rq); + blk_mq_free_tag_set(&info->tag_set); info->rq = NULL; put_disk(info->gd); info->gd = NULL; } +/* Must be called with io_lock holded */ static void kick_pending_request_queues(struct blkfront_info *info) { - if (!RING_FULL(&info->ring)) { - /* Re-enable calldowns. */ - blk_start_queue(info->rq); - /* Kick things off immediately. */ - do_blkif_request(info->rq); - } + if (!RING_FULL(&info->ring)) + blk_mq_start_stopped_hw_queues(info->rq, true); } static void blkif_restart_queue(struct work_struct *work) @@ -964,7 +959,7 @@ static void blkif_free(struct blkfront_info *info, int suspend) BLKIF_STATE_SUSPENDED : BLKIF_STATE_DISCONNECTED; /* No more blkif_request(). */ if (info->rq) - blk_stop_queue(info->rq); + blk_mq_stop_hw_queues(info->rq); /* Remove all persistent grants */ if (!list_empty(&info->grants)) { @@ -1147,7 +1142,6 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id) RING_IDX i, rp; unsigned long flags; struct blkfront_info *info = (struct blkfront_info *)dev_id; - int error; spin_lock_irqsave(&info->io_lock, flags); @@ -1188,37 +1182,37 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id) continue; } - error = (bret->status == BLKIF_RSP_OKAY) ? 0 : -EIO; + req->errors = (bret->status == BLKIF_RSP_OKAY) ? 0 : -EIO; switch (bret->operation) { case BLKIF_OP_DISCARD: if (unlikely(bret->status == BLKIF_RSP_EOPNOTSUPP)) { struct request_queue *rq = info->rq; printk(KERN_WARNING "blkfront: %s: %s op failed\n", info->gd->disk_name, op_name(bret->operation)); - error = -EOPNOTSUPP; + req->errors = -EOPNOTSUPP; info->feature_discard = 0; info->feature_secdiscard = 0; queue_flag_clear(QUEUE_FLAG_DISCARD, rq); queue_flag_clear(QUEUE_FLAG_SECDISCARD, rq); } - __blk_end_request_all(req, error); + blk_mq_complete_request(req); break; case BLKIF_OP_FLUSH_DISKCACHE: case BLKIF_OP_WRITE_BARRIER: if (unlikely(bret->status == BLKIF_RSP_EOPNOTSUPP)) { printk(KERN_WARNING "blkfront: %s: %s op failed\n", info->gd->disk_name, op_name(bret->operation)); - error = -EOPNOTSUPP; + req->errors = -EOPNOTSUPP; } if (unlikely(bret->status == BLKIF_RSP_ERROR && info->shadow[id].req.u.rw.nr_segments == 0)) { printk(KERN_WARNING "blkfront: %s: empty %s op failed\n", info->gd->disk_name, op_name(bret->operation)); - error = -EOPNOTSUPP; + req->errors = -EOPNOTSUPP; } - if (unlikely(error)) { - if (error == -EOPNOTSUPP) - error = 0; + if (unlikely(req->errors)) { + if (req->errors == -EOPNOTSUPP) + req->errors = 0; info->feature_flush = 0; xlvbd_flush(info); } @@ -1229,7 +1223,7 @@ static irqreturn_t blkif_interrupt(int irq, void *dev_id) dev_dbg(&info->xbdev->dev, "Bad return from blkdev data " "request: %x\n", bret->status); - __blk_end_request_all(req, error); + blk_mq_complete_request(req); break; default: BUG(); @@ -1481,16 +1475,14 @@ static int blkfront_probe(struct xenbus_device *dev, return 0; } -static void split_bio_end(struct bio *bio, int error) +static void split_bio_end(struct bio *bio) { struct split_bio *split_bio = bio->bi_private; - if (error) - split_bio->err = error; - if (atomic_dec_and_test(&split_bio->pending)) { split_bio->bio->bi_phys_segments = 0; - bio_endio(split_bio->bio, split_bio->err); + split_bio->bio->bi_error = bio->bi_error; + bio_endio(split_bio->bio); kfree(split_bio); } bio_put(bio); @@ -1558,28 +1550,6 @@ static int blkif_recover(struct blkfront_info *info) kfree(copy); - /* - * Empty the queue, this is important because we might have - * requests in the queue with more segments than what we - * can handle now. - */ - spin_lock_irq(&info->io_lock); - while ((req = blk_fetch_request(info->rq)) != NULL) { - if (req->cmd_flags & - (REQ_FLUSH | REQ_FUA | REQ_DISCARD | REQ_SECURE)) { - list_add(&req->queuelist, &requests); - continue; - } - merge_bio.head = req->bio; - merge_bio.tail = req->biotail; - bio_list_merge(&bio_list, &merge_bio); - req->bio = NULL; - if (req->cmd_flags & (REQ_FLUSH | REQ_FUA)) - pr_alert("diskcache flush request found!\n"); - __blk_end_request_all(req, 0); - } - spin_unlock_irq(&info->io_lock); - xenbus_switch_state(info->xbdev, XenbusStateConnected); spin_lock_irq(&info->io_lock); @@ -1594,9 +1564,10 @@ static int blkif_recover(struct blkfront_info *info) /* Requeue pending requests (flush or discard) */ list_del_init(&req->queuelist); BUG_ON(req->nr_phys_segments > segs); - blk_requeue_request(info->rq, req); + blk_mq_requeue_request(req); } spin_unlock_irq(&info->io_lock); + blk_mq_kick_requeue_list(info->rq); while ((bio = bio_list_pop(&bio_list)) != NULL) { /* Traverse the list of pending bios and re-queue them */ diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 763301c7828c..9fa15bb9d118 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -388,7 +388,6 @@ static ssize_t comp_algorithm_store(struct device *dev, static ssize_t compact_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { - unsigned long nr_migrated; struct zram *zram = dev_to_zram(dev); struct zram_meta *meta; @@ -399,8 +398,7 @@ static ssize_t compact_store(struct device *dev, } meta = zram->meta; - nr_migrated = zs_compact(meta->mem_pool); - atomic64_add(nr_migrated, &zram->stats.num_migrated); + zs_compact(meta->mem_pool); up_read(&zram->init_lock); return len; @@ -428,26 +426,31 @@ static ssize_t mm_stat_show(struct device *dev, struct device_attribute *attr, char *buf) { struct zram *zram = dev_to_zram(dev); + struct zs_pool_stats pool_stats; u64 orig_size, mem_used = 0; long max_used; ssize_t ret; + memset(&pool_stats, 0x00, sizeof(struct zs_pool_stats)); + down_read(&zram->init_lock); - if (init_done(zram)) + if (init_done(zram)) { mem_used = zs_get_total_pages(zram->meta->mem_pool); + zs_pool_stats(zram->meta->mem_pool, &pool_stats); + } orig_size = atomic64_read(&zram->stats.pages_stored); max_used = atomic_long_read(&zram->stats.max_used_pages); ret = scnprintf(buf, PAGE_SIZE, - "%8llu %8llu %8llu %8lu %8ld %8llu %8llu\n", + "%8llu %8llu %8llu %8lu %8ld %8llu %8lu\n", orig_size << PAGE_SHIFT, (u64)atomic64_read(&zram->stats.compr_data_size), mem_used << PAGE_SHIFT, zram->limit_pages << PAGE_SHIFT, max_used << PAGE_SHIFT, (u64)atomic64_read(&zram->stats.zero_pages), - (u64)atomic64_read(&zram->stats.num_migrated)); + pool_stats.pages_compacted); up_read(&zram->init_lock); return ret; @@ -619,7 +622,7 @@ static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, uncmem = user_mem; if (!uncmem) { - pr_info("Unable to allocate temp memory\n"); + pr_err("Unable to allocate temp memory\n"); ret = -ENOMEM; goto out_cleanup; } @@ -716,7 +719,7 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, handle = zs_malloc(meta->mem_pool, clen); if (!handle) { - pr_info("Error allocating memory for compressed page: %u, size=%zu\n", + pr_err("Error allocating memory for compressed page: %u, size=%zu\n", index, clen); ret = -ENOMEM; goto out; @@ -848,7 +851,7 @@ static void __zram_make_request(struct zram *zram, struct bio *bio) if (unlikely(bio->bi_rw & REQ_DISCARD)) { zram_bio_discard(zram, index, offset, bio); - bio_endio(bio, 0); + bio_endio(bio); return; } @@ -881,8 +884,7 @@ static void __zram_make_request(struct zram *zram, struct bio *bio) update_position(&index, &offset, &bvec); } - set_bit(BIO_UPTODATE, &bio->bi_flags); - bio_endio(bio, 0); + bio_endio(bio); return; out: @@ -899,6 +901,8 @@ static void zram_make_request(struct request_queue *queue, struct bio *bio) if (unlikely(!zram_meta_get(zram))) goto error; + blk_queue_split(queue, &bio, queue->bio_split); + if (!valid_io_request(zram, bio->bi_iter.bi_sector, bio->bi_iter.bi_size)) { atomic64_inc(&zram->stats.invalid_io); @@ -1035,7 +1039,7 @@ static ssize_t disksize_store(struct device *dev, comp = zcomp_create(zram->compressor, zram->max_comp_streams); if (IS_ERR(comp)) { - pr_info("Cannot initialise %s compressing backend\n", + pr_err("Cannot initialise %s compressing backend\n", zram->compressor); err = PTR_ERR(comp); goto out_free_meta; @@ -1213,7 +1217,7 @@ static int zram_add(void) /* gendisk structure */ zram->disk = alloc_disk(1); if (!zram->disk) { - pr_warn("Error allocating disk structure for device %d\n", + pr_err("Error allocating disk structure for device %d\n", device_id); ret = -ENOMEM; goto out_free_queue; @@ -1242,7 +1246,7 @@ static int zram_add(void) blk_queue_io_min(zram->disk->queue, PAGE_SIZE); blk_queue_io_opt(zram->disk->queue, PAGE_SIZE); zram->disk->queue->limits.discard_granularity = PAGE_SIZE; - zram->disk->queue->limits.max_discard_sectors = UINT_MAX; + blk_queue_max_discard_sectors(zram->disk->queue, UINT_MAX); /* * zram_bio_discard() will clear all logical blocks if logical block * size is identical with physical block size(PAGE_SIZE). But if it is @@ -1262,7 +1266,8 @@ static int zram_add(void) ret = sysfs_create_group(&disk_to_dev(zram->disk)->kobj, &zram_disk_attr_group); if (ret < 0) { - pr_warn("Error creating sysfs group"); + pr_err("Error creating sysfs group for device %d\n", + device_id); goto out_free_disk; } strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor)); @@ -1402,13 +1407,13 @@ static int __init zram_init(void) ret = class_register(&zram_control_class); if (ret) { - pr_warn("Unable to register zram-control class\n"); + pr_err("Unable to register zram-control class\n"); return ret; } zram_major = register_blkdev(0, "zram"); if (zram_major <= 0) { - pr_warn("Unable to get major number\n"); + pr_err("Unable to get major number\n"); class_unregister(&zram_control_class); return -EBUSY; } diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h index 6dbe2df506bf..8e92339686d7 100644 --- a/drivers/block/zram/zram_drv.h +++ b/drivers/block/zram/zram_drv.h @@ -78,7 +78,6 @@ struct zram_stats { atomic64_t compr_data_size; /* compressed size of pages stored */ atomic64_t num_reads; /* failed + successful */ atomic64_t num_writes; /* --do-- */ - atomic64_t num_migrated; /* no. of migrated object */ atomic64_t failed_reads; /* can happen when memory is too low */ atomic64_t failed_writes; /* can happen when memory is too low */ atomic64_t invalid_io; /* non-page-aligned I/O requests */ diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index 2e777071e1dc..0bd88c942a52 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -13,6 +13,10 @@ config BT_RTL tristate select FW_LOADER +config BT_QCA + tristate + select FW_LOADER + config BT_HCIBTUSB tristate "HCI USB driver" depends on USB @@ -132,6 +136,7 @@ config BT_HCIUART_3WIRE config BT_HCIUART_INTEL bool "Intel protocol support" depends on BT_HCIUART + select BT_HCIUART_H4 select BT_INTEL help The Intel protocol support enables Bluetooth HCI over serial @@ -150,6 +155,19 @@ config BT_HCIUART_BCM Say Y here to compile support for Broadcom protocol. +config BT_HCIUART_QCA + bool "Qualcomm Atheros protocol support" + depends on BT_HCIUART + select BT_HCIUART_H4 + select BT_QCA + help + The Qualcomm Atheros protocol supports HCI In-Band Sleep feature + over serial port interface(H4) between controller and host. + This protocol is required for UART clock control for QCA Bluetooth + devices. + + Say Y here to compile support for QCA protocol. + config BT_HCIBCM203X tristate "HCI BCM203x USB driver" depends on USB diff --git a/drivers/bluetooth/Makefile b/drivers/bluetooth/Makefile index f40e194e7080..07c9cf381e5a 100644 --- a/drivers/bluetooth/Makefile +++ b/drivers/bluetooth/Makefile @@ -22,6 +22,7 @@ obj-$(CONFIG_BT_MRVL_SDIO) += btmrvl_sdio.o obj-$(CONFIG_BT_WILINK) += btwilink.o obj-$(CONFIG_BT_BCM) += btbcm.o obj-$(CONFIG_BT_RTL) += btrtl.o +obj-$(CONFIG_BT_QCA) += btqca.o btmrvl-y := btmrvl_main.o btmrvl-$(CONFIG_DEBUG_FS) += btmrvl_debugfs.o @@ -34,6 +35,7 @@ hci_uart-$(CONFIG_BT_HCIUART_ATH3K) += hci_ath.o hci_uart-$(CONFIG_BT_HCIUART_3WIRE) += hci_h5.o hci_uart-$(CONFIG_BT_HCIUART_INTEL) += hci_intel.o hci_uart-$(CONFIG_BT_HCIUART_BCM) += hci_bcm.o +hci_uart-$(CONFIG_BT_HCIUART_QCA) += hci_qca.o hci_uart-objs := $(hci_uart-y) ccflags-y += -D__CHECK_ENDIAN__ diff --git a/drivers/bluetooth/bfusb.c b/drivers/bluetooth/bfusb.c index fcfb72e9e0ee..a5c4d0584389 100644 --- a/drivers/bluetooth/bfusb.c +++ b/drivers/bluetooth/bfusb.c @@ -492,7 +492,7 @@ static int bfusb_send_frame(struct hci_dev *hdev, struct sk_buff *skb) case HCI_SCODATA_PKT: hdev->stat.sco_tx++; break; - }; + } /* Prepend skb with frame type */ memcpy(skb_push(skb, 1), &bt_cb(skb)->pkt_type, 1); diff --git a/drivers/bluetooth/bt3c_cs.c b/drivers/bluetooth/bt3c_cs.c index 7aab65427d38..a00bb82eb7c6 100644 --- a/drivers/bluetooth/bt3c_cs.c +++ b/drivers/bluetooth/bt3c_cs.c @@ -427,7 +427,7 @@ static int bt3c_hci_send_frame(struct hci_dev *hdev, struct sk_buff *skb) case HCI_SCODATA_PKT: hdev->stat.sco_tx++; break; - }; + } /* Prepend skb with frame type */ memcpy(skb_push(skb, 1), &bt_cb(skb)->pkt_type, 1); diff --git a/drivers/bluetooth/btbcm.c b/drivers/bluetooth/btbcm.c index 9ceb8ac68fdc..02ed816a18f9 100644 --- a/drivers/bluetooth/btbcm.c +++ b/drivers/bluetooth/btbcm.c @@ -34,6 +34,7 @@ #define BDADDR_BCM20702A0 (&(bdaddr_t) {{0x00, 0xa0, 0x02, 0x70, 0x20, 0x00}}) #define BDADDR_BCM4324B3 (&(bdaddr_t) {{0x00, 0x00, 0x00, 0xb3, 0x24, 0x43}}) +#define BDADDR_BCM4330B1 (&(bdaddr_t) {{0x00, 0x00, 0x00, 0xb1, 0x30, 0x43}}) int btbcm_check_bdaddr(struct hci_dev *hdev) { @@ -66,9 +67,13 @@ int btbcm_check_bdaddr(struct hci_dev *hdev) * * The address 43:24:B3:00:00:00 indicates a BCM4324B3 controller * with waiting for configuration state. + * + * The address 43:30:B1:00:00:00 indicates a BCM4330B1 controller + * with waiting for configuration state. */ if (!bacmp(&bda->bdaddr, BDADDR_BCM20702A0) || - !bacmp(&bda->bdaddr, BDADDR_BCM4324B3)) { + !bacmp(&bda->bdaddr, BDADDR_BCM4324B3) || + !bacmp(&bda->bdaddr, BDADDR_BCM4330B1)) { BT_INFO("%s: BCM: Using default device address (%pMR)", hdev->name, &bda->bdaddr); set_bit(HCI_QUIRK_INVALID_BDADDR, &hdev->quirks); @@ -241,6 +246,7 @@ static const struct { u16 subver; const char *name; } bcm_uart_subver_table[] = { + { 0x4103, "BCM4330B1" }, /* 002.001.003 */ { 0x410e, "BCM43341B0" }, /* 002.001.014 */ { 0x4406, "BCM4324B3" }, /* 002.004.006 */ { 0x610c, "BCM4354" }, /* 003.001.012 */ diff --git a/drivers/bluetooth/btintel.c b/drivers/bluetooth/btintel.c index 828f2f8d1568..048423fd83bf 100644 --- a/drivers/bluetooth/btintel.c +++ b/drivers/bluetooth/btintel.c @@ -89,7 +89,89 @@ int btintel_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr) } EXPORT_SYMBOL_GPL(btintel_set_bdaddr); +void btintel_hw_error(struct hci_dev *hdev, u8 code) +{ + struct sk_buff *skb; + u8 type = 0x00; + + BT_ERR("%s: Hardware error 0x%2.2x", hdev->name, code); + + skb = __hci_cmd_sync(hdev, HCI_OP_RESET, 0, NULL, HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) { + BT_ERR("%s: Reset after hardware error failed (%ld)", + hdev->name, PTR_ERR(skb)); + return; + } + kfree_skb(skb); + + skb = __hci_cmd_sync(hdev, 0xfc22, 1, &type, HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) { + BT_ERR("%s: Retrieving Intel exception info failed (%ld)", + hdev->name, PTR_ERR(skb)); + return; + } + + if (skb->len != 13) { + BT_ERR("%s: Exception info size mismatch", hdev->name); + kfree_skb(skb); + return; + } + + BT_ERR("%s: Exception info %s", hdev->name, (char *)(skb->data + 1)); + + kfree_skb(skb); +} +EXPORT_SYMBOL_GPL(btintel_hw_error); + +void btintel_version_info(struct hci_dev *hdev, struct intel_version *ver) +{ + const char *variant; + + switch (ver->fw_variant) { + case 0x06: + variant = "Bootloader"; + break; + case 0x23: + variant = "Firmware"; + break; + default: + return; + } + + BT_INFO("%s: %s revision %u.%u build %u week %u %u", hdev->name, + variant, ver->fw_revision >> 4, ver->fw_revision & 0x0f, + ver->fw_build_num, ver->fw_build_ww, 2000 + ver->fw_build_yy); +} +EXPORT_SYMBOL_GPL(btintel_version_info); + +int btintel_secure_send(struct hci_dev *hdev, u8 fragment_type, u32 plen, + const void *param) +{ + while (plen > 0) { + struct sk_buff *skb; + u8 cmd_param[253], fragment_len = (plen > 252) ? 252 : plen; + + cmd_param[0] = fragment_type; + memcpy(cmd_param + 1, param, fragment_len); + + skb = __hci_cmd_sync(hdev, 0xfc09, fragment_len + 1, + cmd_param, HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) + return PTR_ERR(skb); + + kfree_skb(skb); + + plen -= fragment_len; + param += fragment_len; + } + + return 0; +} +EXPORT_SYMBOL_GPL(btintel_secure_send); + MODULE_AUTHOR("Marcel Holtmann "); MODULE_DESCRIPTION("Bluetooth support for Intel devices ver " VERSION); MODULE_VERSION(VERSION); MODULE_LICENSE("GPL"); +MODULE_FIRMWARE("intel/ibt-11-5.sfi"); +MODULE_FIRMWARE("intel/ibt-11-5.ddc"); diff --git a/drivers/bluetooth/btintel.h b/drivers/bluetooth/btintel.h index 4bda6ab34f60..b278d14758d5 100644 --- a/drivers/bluetooth/btintel.h +++ b/drivers/bluetooth/btintel.h @@ -73,6 +73,11 @@ struct intel_secure_send_result { int btintel_check_bdaddr(struct hci_dev *hdev); int btintel_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr); +void btintel_hw_error(struct hci_dev *hdev, u8 code); + +void btintel_version_info(struct hci_dev *hdev, struct intel_version *ver); +int btintel_secure_send(struct hci_dev *hdev, u8 fragment_type, u32 plen, + const void *param); #else @@ -86,4 +91,18 @@ static inline int btintel_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdadd return -EOPNOTSUPP; } +static inline void btintel_hw_error(struct hci_dev *hdev, u8 code) +{ +} + +static void btintel_version_info(struct hci_dev *hdev, struct intel_version *ver) +{ +} + +static inline int btintel_secure_send(struct hci_dev *hdev, u8 fragment_type, + u32 plen, const void *param) +{ + return -EOPNOTSUPP; +} + #endif diff --git a/drivers/bluetooth/btmrvl_drv.h b/drivers/bluetooth/btmrvl_drv.h index 086f0ec89580..27a9aac25583 100644 --- a/drivers/bluetooth/btmrvl_drv.h +++ b/drivers/bluetooth/btmrvl_drv.h @@ -95,10 +95,10 @@ struct btmrvl_private { struct btmrvl_device btmrvl_dev; struct btmrvl_adapter *adapter; struct btmrvl_thread main_thread; - int (*hw_host_to_card) (struct btmrvl_private *priv, + int (*hw_host_to_card)(struct btmrvl_private *priv, u8 *payload, u16 nb); - int (*hw_wakeup_firmware) (struct btmrvl_private *priv); - int (*hw_process_int_status) (struct btmrvl_private *priv); + int (*hw_wakeup_firmware)(struct btmrvl_private *priv); + int (*hw_process_int_status)(struct btmrvl_private *priv); void (*firmware_dump)(struct btmrvl_private *priv); spinlock_t driver_lock; /* spinlock used by driver */ #ifdef CONFIG_DEBUG_FS diff --git a/drivers/bluetooth/btmrvl_sdio.c b/drivers/bluetooth/btmrvl_sdio.c index b9a811900f6a..b9978a7ba0cc 100644 --- a/drivers/bluetooth/btmrvl_sdio.c +++ b/drivers/bluetooth/btmrvl_sdio.c @@ -1071,8 +1071,6 @@ static int btmrvl_sdio_download_fw(struct btmrvl_sdio_card *card) } } - sdio_release_host(card->func); - /* * winner or not, with this test the FW synchronizes when the * module can continue its initialization @@ -1082,6 +1080,8 @@ static int btmrvl_sdio_download_fw(struct btmrvl_sdio_card *card) return -ETIMEDOUT; } + sdio_release_host(card->func); + return 0; done: @@ -1376,8 +1376,7 @@ static void btmrvl_sdio_dump_firmware(struct btmrvl_private *priv) /* fw_dump_data will be free in device coredump release function after 5 min*/ - dev_coredumpv(&priv->btmrvl_dev.hcidev->dev, fw_dump_data, - fw_dump_len, GFP_KERNEL); + dev_coredumpv(&card->func->dev, fw_dump_data, fw_dump_len, GFP_KERNEL); BT_INFO("== btmrvl firmware dump to /sys/class/devcoredump end"); } diff --git a/drivers/bluetooth/btqca.c b/drivers/bluetooth/btqca.c new file mode 100644 index 000000000000..4a6208168850 --- /dev/null +++ b/drivers/bluetooth/btqca.c @@ -0,0 +1,392 @@ +/* + * Bluetooth supports for Qualcomm Atheros chips + * + * Copyright (c) 2015 The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ +#include +#include + +#include +#include + +#include "btqca.h" + +#define VERSION "0.1" + +static int rome_patch_ver_req(struct hci_dev *hdev, u32 *rome_version) +{ + struct sk_buff *skb; + struct edl_event_hdr *edl; + struct rome_version *ver; + char cmd; + int err = 0; + + BT_DBG("%s: ROME Patch Version Request", hdev->name); + + cmd = EDL_PATCH_VER_REQ_CMD; + skb = __hci_cmd_sync_ev(hdev, EDL_PATCH_CMD_OPCODE, EDL_PATCH_CMD_LEN, + &cmd, HCI_VENDOR_PKT, HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) { + err = PTR_ERR(skb); + BT_ERR("%s: Failed to read version of ROME (%d)", hdev->name, + err); + return err; + } + + if (skb->len != sizeof(*edl) + sizeof(*ver)) { + BT_ERR("%s: Version size mismatch len %d", hdev->name, + skb->len); + err = -EILSEQ; + goto out; + } + + edl = (struct edl_event_hdr *)(skb->data); + if (!edl || !edl->data) { + BT_ERR("%s: TLV with no header or no data", hdev->name); + err = -EILSEQ; + goto out; + } + + if (edl->cresp != EDL_CMD_REQ_RES_EVT || + edl->rtype != EDL_APP_VER_RES_EVT) { + BT_ERR("%s: Wrong packet received %d %d", hdev->name, + edl->cresp, edl->rtype); + err = -EIO; + goto out; + } + + ver = (struct rome_version *)(edl->data); + + BT_DBG("%s: Product:0x%08x", hdev->name, le32_to_cpu(ver->product_id)); + BT_DBG("%s: Patch :0x%08x", hdev->name, le16_to_cpu(ver->patch_ver)); + BT_DBG("%s: ROM :0x%08x", hdev->name, le16_to_cpu(ver->rome_ver)); + BT_DBG("%s: SOC :0x%08x", hdev->name, le32_to_cpu(ver->soc_id)); + + /* ROME chipset version can be decided by patch and SoC + * version, combination with upper 2 bytes from SoC + * and lower 2 bytes from patch will be used. + */ + *rome_version = (le32_to_cpu(ver->soc_id) << 16) | + (le16_to_cpu(ver->rome_ver) & 0x0000ffff); + +out: + kfree_skb(skb); + + return err; +} + +static int rome_reset(struct hci_dev *hdev) +{ + struct sk_buff *skb; + int err; + + BT_DBG("%s: ROME HCI_RESET", hdev->name); + + skb = __hci_cmd_sync(hdev, HCI_OP_RESET, 0, NULL, HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) { + err = PTR_ERR(skb); + BT_ERR("%s: Reset failed (%d)", hdev->name, err); + return err; + } + + kfree_skb(skb); + + return 0; +} + +static void rome_tlv_check_data(struct rome_config *config, + const struct firmware *fw) +{ + const u8 *data; + u32 type_len; + u16 tag_id, tag_len; + int idx, length; + struct tlv_type_hdr *tlv; + struct tlv_type_patch *tlv_patch; + struct tlv_type_nvm *tlv_nvm; + + tlv = (struct tlv_type_hdr *)fw->data; + + type_len = le32_to_cpu(tlv->type_len); + length = (type_len >> 8) & 0x00ffffff; + + BT_DBG("TLV Type\t\t : 0x%x", type_len & 0x000000ff); + BT_DBG("Length\t\t : %d bytes", length); + + switch (config->type) { + case TLV_TYPE_PATCH: + tlv_patch = (struct tlv_type_patch *)tlv->data; + BT_DBG("Total Length\t\t : %d bytes", + le32_to_cpu(tlv_patch->total_size)); + BT_DBG("Patch Data Length\t : %d bytes", + le32_to_cpu(tlv_patch->data_length)); + BT_DBG("Signing Format Version : 0x%x", + tlv_patch->format_version); + BT_DBG("Signature Algorithm\t : 0x%x", + tlv_patch->signature); + BT_DBG("Reserved\t\t : 0x%x", + le16_to_cpu(tlv_patch->reserved1)); + BT_DBG("Product ID\t\t : 0x%04x", + le16_to_cpu(tlv_patch->product_id)); + BT_DBG("Rom Build Version\t : 0x%04x", + le16_to_cpu(tlv_patch->rom_build)); + BT_DBG("Patch Version\t\t : 0x%04x", + le16_to_cpu(tlv_patch->patch_version)); + BT_DBG("Reserved\t\t : 0x%x", + le16_to_cpu(tlv_patch->reserved2)); + BT_DBG("Patch Entry Address\t : 0x%x", + le32_to_cpu(tlv_patch->entry)); + break; + + case TLV_TYPE_NVM: + idx = 0; + data = tlv->data; + while (idx < length) { + tlv_nvm = (struct tlv_type_nvm *)(data + idx); + + tag_id = le16_to_cpu(tlv_nvm->tag_id); + tag_len = le16_to_cpu(tlv_nvm->tag_len); + + /* Update NVM tags as needed */ + switch (tag_id) { + case EDL_TAG_ID_HCI: + /* HCI transport layer parameters + * enabling software inband sleep + * onto controller side. + */ + tlv_nvm->data[0] |= 0x80; + + /* UART Baud Rate */ + tlv_nvm->data[2] = config->user_baud_rate; + + break; + + case EDL_TAG_ID_DEEP_SLEEP: + /* Sleep enable mask + * enabling deep sleep feature on controller. + */ + tlv_nvm->data[0] |= 0x01; + + break; + } + + idx += (sizeof(u16) + sizeof(u16) + 8 + tag_len); + } + break; + + default: + BT_ERR("Unknown TLV type %d", config->type); + break; + } +} + +static int rome_tlv_send_segment(struct hci_dev *hdev, int idx, int seg_size, + const u8 *data) +{ + struct sk_buff *skb; + struct edl_event_hdr *edl; + struct tlv_seg_resp *tlv_resp; + u8 cmd[MAX_SIZE_PER_TLV_SEGMENT + 2]; + int err = 0; + + BT_DBG("%s: Download segment #%d size %d", hdev->name, idx, seg_size); + + cmd[0] = EDL_PATCH_TLV_REQ_CMD; + cmd[1] = seg_size; + memcpy(cmd + 2, data, seg_size); + + skb = __hci_cmd_sync_ev(hdev, EDL_PATCH_CMD_OPCODE, seg_size + 2, cmd, + HCI_VENDOR_PKT, HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) { + err = PTR_ERR(skb); + BT_ERR("%s: Failed to send TLV segment (%d)", hdev->name, err); + return err; + } + + if (skb->len != sizeof(*edl) + sizeof(*tlv_resp)) { + BT_ERR("%s: TLV response size mismatch", hdev->name); + err = -EILSEQ; + goto out; + } + + edl = (struct edl_event_hdr *)(skb->data); + if (!edl || !edl->data) { + BT_ERR("%s: TLV with no header or no data", hdev->name); + err = -EILSEQ; + goto out; + } + + tlv_resp = (struct tlv_seg_resp *)(edl->data); + + if (edl->cresp != EDL_CMD_REQ_RES_EVT || + edl->rtype != EDL_TVL_DNLD_RES_EVT || tlv_resp->result != 0x00) { + BT_ERR("%s: TLV with error stat 0x%x rtype 0x%x (0x%x)", + hdev->name, edl->cresp, edl->rtype, tlv_resp->result); + err = -EIO; + } + +out: + kfree_skb(skb); + + return err; +} + +static int rome_tlv_download_request(struct hci_dev *hdev, + const struct firmware *fw) +{ + const u8 *buffer, *data; + int total_segment, remain_size; + int ret, i; + + if (!fw || !fw->data) + return -EINVAL; + + total_segment = fw->size / MAX_SIZE_PER_TLV_SEGMENT; + remain_size = fw->size % MAX_SIZE_PER_TLV_SEGMENT; + + BT_DBG("%s: Total segment num %d remain size %d total size %zu", + hdev->name, total_segment, remain_size, fw->size); + + data = fw->data; + for (i = 0; i < total_segment; i++) { + buffer = data + i * MAX_SIZE_PER_TLV_SEGMENT; + ret = rome_tlv_send_segment(hdev, i, MAX_SIZE_PER_TLV_SEGMENT, + buffer); + if (ret < 0) + return -EIO; + } + + if (remain_size) { + buffer = data + total_segment * MAX_SIZE_PER_TLV_SEGMENT; + ret = rome_tlv_send_segment(hdev, total_segment, remain_size, + buffer); + if (ret < 0) + return -EIO; + } + + return 0; +} + +static int rome_download_firmware(struct hci_dev *hdev, + struct rome_config *config) +{ + const struct firmware *fw; + int ret; + + BT_INFO("%s: ROME Downloading %s", hdev->name, config->fwname); + + ret = request_firmware(&fw, config->fwname, &hdev->dev); + if (ret) { + BT_ERR("%s: Failed to request file: %s (%d)", hdev->name, + config->fwname, ret); + return ret; + } + + rome_tlv_check_data(config, fw); + + ret = rome_tlv_download_request(hdev, fw); + if (ret) { + BT_ERR("%s: Failed to download file: %s (%d)", hdev->name, + config->fwname, ret); + } + + release_firmware(fw); + + return ret; +} + +int qca_set_bdaddr_rome(struct hci_dev *hdev, const bdaddr_t *bdaddr) +{ + struct sk_buff *skb; + u8 cmd[9]; + int err; + + cmd[0] = EDL_NVM_ACCESS_SET_REQ_CMD; + cmd[1] = 0x02; /* TAG ID */ + cmd[2] = sizeof(bdaddr_t); /* size */ + memcpy(cmd + 3, bdaddr, sizeof(bdaddr_t)); + skb = __hci_cmd_sync_ev(hdev, EDL_NVM_ACCESS_OPCODE, sizeof(cmd), cmd, + HCI_VENDOR_PKT, HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) { + err = PTR_ERR(skb); + BT_ERR("%s: Change address command failed (%d)", + hdev->name, err); + return err; + } + + kfree_skb(skb); + + return 0; +} +EXPORT_SYMBOL_GPL(qca_set_bdaddr_rome); + +int qca_uart_setup_rome(struct hci_dev *hdev, uint8_t baudrate) +{ + u32 rome_ver = 0; + struct rome_config config; + int err; + + BT_DBG("%s: ROME setup on UART", hdev->name); + + config.user_baud_rate = baudrate; + + /* Get ROME version information */ + err = rome_patch_ver_req(hdev, &rome_ver); + if (err < 0 || rome_ver == 0) { + BT_ERR("%s: Failed to get version 0x%x", hdev->name, err); + return err; + } + + BT_INFO("%s: ROME controller version 0x%08x", hdev->name, rome_ver); + + /* Download rampatch file */ + config.type = TLV_TYPE_PATCH; + snprintf(config.fwname, sizeof(config.fwname), "qca/rampatch_%08x.bin", + rome_ver); + err = rome_download_firmware(hdev, &config); + if (err < 0) { + BT_ERR("%s: Failed to download patch (%d)", hdev->name, err); + return err; + } + + /* Download NVM configuration */ + config.type = TLV_TYPE_NVM; + snprintf(config.fwname, sizeof(config.fwname), "qca/nvm_%08x.bin", + rome_ver); + err = rome_download_firmware(hdev, &config); + if (err < 0) { + BT_ERR("%s: Failed to download NVM (%d)", hdev->name, err); + return err; + } + + /* Perform HCI reset */ + err = rome_reset(hdev); + if (err < 0) { + BT_ERR("%s: Failed to run HCI_RESET (%d)", hdev->name, err); + return err; + } + + BT_INFO("%s: ROME setup on UART is completed", hdev->name); + + return 0; +} +EXPORT_SYMBOL_GPL(qca_uart_setup_rome); + +MODULE_AUTHOR("Ben Young Tae Kim "); +MODULE_DESCRIPTION("Bluetooth support for Qualcomm Atheros family ver " VERSION); +MODULE_VERSION(VERSION); +MODULE_LICENSE("GPL"); diff --git a/drivers/bluetooth/btqca.h b/drivers/bluetooth/btqca.h new file mode 100644 index 000000000000..65e994b96c47 --- /dev/null +++ b/drivers/bluetooth/btqca.h @@ -0,0 +1,135 @@ +/* + * Bluetooth supports for Qualcomm Atheros ROME chips + * + * Copyright (c) 2015 The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#define EDL_PATCH_CMD_OPCODE (0xFC00) +#define EDL_NVM_ACCESS_OPCODE (0xFC0B) +#define EDL_PATCH_CMD_LEN (1) +#define EDL_PATCH_VER_REQ_CMD (0x19) +#define EDL_PATCH_TLV_REQ_CMD (0x1E) +#define EDL_NVM_ACCESS_SET_REQ_CMD (0x01) +#define MAX_SIZE_PER_TLV_SEGMENT (243) + +#define EDL_CMD_REQ_RES_EVT (0x00) +#define EDL_PATCH_VER_RES_EVT (0x19) +#define EDL_APP_VER_RES_EVT (0x02) +#define EDL_TVL_DNLD_RES_EVT (0x04) +#define EDL_CMD_EXE_STATUS_EVT (0x00) +#define EDL_SET_BAUDRATE_RSP_EVT (0x92) +#define EDL_NVM_ACCESS_CODE_EVT (0x0B) + +#define EDL_TAG_ID_HCI (17) +#define EDL_TAG_ID_DEEP_SLEEP (27) + +enum qca_bardrate { + QCA_BAUDRATE_115200 = 0, + QCA_BAUDRATE_57600, + QCA_BAUDRATE_38400, + QCA_BAUDRATE_19200, + QCA_BAUDRATE_9600, + QCA_BAUDRATE_230400, + QCA_BAUDRATE_250000, + QCA_BAUDRATE_460800, + QCA_BAUDRATE_500000, + QCA_BAUDRATE_720000, + QCA_BAUDRATE_921600, + QCA_BAUDRATE_1000000, + QCA_BAUDRATE_1250000, + QCA_BAUDRATE_2000000, + QCA_BAUDRATE_3000000, + QCA_BAUDRATE_4000000, + QCA_BAUDRATE_1600000, + QCA_BAUDRATE_3200000, + QCA_BAUDRATE_3500000, + QCA_BAUDRATE_AUTO = 0xFE, + QCA_BAUDRATE_RESERVED +}; + +enum rome_tlv_type { + TLV_TYPE_PATCH = 1, + TLV_TYPE_NVM +}; + +struct rome_config { + u8 type; + char fwname[64]; + uint8_t user_baud_rate; +}; + +struct edl_event_hdr { + __u8 cresp; + __u8 rtype; + __u8 data[0]; +} __packed; + +struct rome_version { + __le32 product_id; + __le16 patch_ver; + __le16 rome_ver; + __le32 soc_id; +} __packed; + +struct tlv_seg_resp { + __u8 result; +} __packed; + +struct tlv_type_patch { + __le32 total_size; + __le32 data_length; + __u8 format_version; + __u8 signature; + __le16 reserved1; + __le16 product_id; + __le16 rom_build; + __le16 patch_version; + __le16 reserved2; + __le32 entry; +} __packed; + +struct tlv_type_nvm { + __le16 tag_id; + __le16 tag_len; + __le32 reserve1; + __le32 reserve2; + __u8 data[0]; +} __packed; + +struct tlv_type_hdr { + __le32 type_len; + __u8 data[0]; +} __packed; + +#if IS_ENABLED(CONFIG_BT_QCA) + +int qca_set_bdaddr_rome(struct hci_dev *hdev, const bdaddr_t *bdaddr); +int qca_uart_setup_rome(struct hci_dev *hdev, uint8_t baudrate); + +#else + +static inline int qca_set_bdaddr_rome(struct hci_dev *hdev, const bdaddr_t *bdaddr) +{ + return -EOPNOTSUPP; +} + +static inline int qca_uart_setup_rome(struct hci_dev *hdev, int speed) +{ + return -EOPNOTSUPP; +} + +#endif diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index b4cf8d9c9dac..b6aceaf82aa8 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -68,6 +68,9 @@ static const struct usb_device_id btusb_table[] = { /* Generic Bluetooth AMP device */ { USB_DEVICE_INFO(0xe0, 0x01, 0x04), .driver_info = BTUSB_AMP }, + /* Generic Bluetooth USB interface */ + { USB_INTERFACE_INFO(0xe0, 0x01, 0x01) }, + /* Apple-specific (Broadcom) devices */ { USB_VENDOR_AND_INTERFACE_INFO(0x05ac, 0xff, 0x01, 0x01), .driver_info = BTUSB_BCM_APPLE }, @@ -319,6 +322,9 @@ static const struct usb_device_id blacklist_table[] = { { USB_DEVICE(0x13d3, 0x3461), .driver_info = BTUSB_REALTEK }, { USB_DEVICE(0x13d3, 0x3462), .driver_info = BTUSB_REALTEK }, + /* Silicon Wave based devices */ + { USB_DEVICE(0x0c10, 0x0000), .driver_info = BTUSB_SWAVE }, + { } /* Terminating entry */ }; @@ -1575,7 +1581,7 @@ static int btusb_setup_intel(struct hci_dev *hdev) /* fw_patch_num indicates the version of patch the device currently * have. If there is no patch data in the device, it is always 0x00. - * So, if it is other than 0x00, no need to patch the deivce again. + * So, if it is other than 0x00, no need to patch the device again. */ if (ver->fw_patch_num) { BT_INFO("%s: Intel device is already patched. patch num: %02x", @@ -1878,51 +1884,6 @@ static int btusb_send_frame_intel(struct hci_dev *hdev, struct sk_buff *skb) return -EILSEQ; } -static int btusb_intel_secure_send(struct hci_dev *hdev, u8 fragment_type, - u32 plen, const void *param) -{ - while (plen > 0) { - struct sk_buff *skb; - u8 cmd_param[253], fragment_len = (plen > 252) ? 252 : plen; - - cmd_param[0] = fragment_type; - memcpy(cmd_param + 1, param, fragment_len); - - skb = __hci_cmd_sync(hdev, 0xfc09, fragment_len + 1, - cmd_param, HCI_INIT_TIMEOUT); - if (IS_ERR(skb)) - return PTR_ERR(skb); - - kfree_skb(skb); - - plen -= fragment_len; - param += fragment_len; - } - - return 0; -} - -static void btusb_intel_version_info(struct hci_dev *hdev, - struct intel_version *ver) -{ - const char *variant; - - switch (ver->fw_variant) { - case 0x06: - variant = "Bootloader"; - break; - case 0x23: - variant = "Firmware"; - break; - default: - return; - } - - BT_INFO("%s: %s revision %u.%u build %u week %u %u", hdev->name, - variant, ver->fw_revision >> 4, ver->fw_revision & 0x0f, - ver->fw_build_num, ver->fw_build_ww, 2000 + ver->fw_build_yy); -} - static int btusb_setup_intel_new(struct hci_dev *hdev) { static const u8 reset_param[] = { 0x00, 0x01, 0x00, 0x01, @@ -1984,7 +1945,7 @@ static int btusb_setup_intel_new(struct hci_dev *hdev) return -EINVAL; } - btusb_intel_version_info(hdev, ver); + btintel_version_info(hdev, ver); /* The firmware variant determines if the device is in bootloader * mode or is running operational firmware. The value 0x06 identifies @@ -2104,7 +2065,7 @@ static int btusb_setup_intel_new(struct hci_dev *hdev) /* Start the firmware download transaction with the Init fragment * represented by the 128 bytes of CSS header. */ - err = btusb_intel_secure_send(hdev, 0x00, 128, fw->data); + err = btintel_secure_send(hdev, 0x00, 128, fw->data); if (err < 0) { BT_ERR("%s: Failed to send firmware header (%d)", hdev->name, err); @@ -2114,7 +2075,7 @@ static int btusb_setup_intel_new(struct hci_dev *hdev) /* Send the 256 bytes of public key information from the firmware * as the PKey fragment. */ - err = btusb_intel_secure_send(hdev, 0x03, 256, fw->data + 128); + err = btintel_secure_send(hdev, 0x03, 256, fw->data + 128); if (err < 0) { BT_ERR("%s: Failed to send firmware public key (%d)", hdev->name, err); @@ -2124,7 +2085,7 @@ static int btusb_setup_intel_new(struct hci_dev *hdev) /* Send the 256 bytes of signature information from the firmware * as the Sign fragment. */ - err = btusb_intel_secure_send(hdev, 0x02, 256, fw->data + 388); + err = btintel_secure_send(hdev, 0x02, 256, fw->data + 388); if (err < 0) { BT_ERR("%s: Failed to send firmware signature (%d)", hdev->name, err); @@ -2139,7 +2100,7 @@ static int btusb_setup_intel_new(struct hci_dev *hdev) frag_len += sizeof(*cmd) + cmd->plen; - /* The paramter length of the secure send command requires + /* The parameter length of the secure send command requires * a 4 byte alignment. It happens so that the firmware file * contains proper Intel_NOP commands to align the fragments * as needed. @@ -2148,8 +2109,7 @@ static int btusb_setup_intel_new(struct hci_dev *hdev) * firmware data buffer as a single Data fragement. */ if (!(frag_len % 4)) { - err = btusb_intel_secure_send(hdev, 0x01, frag_len, - fw_ptr); + err = btintel_secure_send(hdev, 0x01, frag_len, fw_ptr); if (err < 0) { BT_ERR("%s: Failed to send firmware data (%d)", hdev->name, err); @@ -2291,39 +2251,6 @@ static int btusb_setup_intel_new(struct hci_dev *hdev) return 0; } -static void btusb_hw_error_intel(struct hci_dev *hdev, u8 code) -{ - struct sk_buff *skb; - u8 type = 0x00; - - BT_ERR("%s: Hardware error 0x%2.2x", hdev->name, code); - - skb = __hci_cmd_sync(hdev, HCI_OP_RESET, 0, NULL, HCI_INIT_TIMEOUT); - if (IS_ERR(skb)) { - BT_ERR("%s: Reset after hardware error failed (%ld)", - hdev->name, PTR_ERR(skb)); - return; - } - kfree_skb(skb); - - skb = __hci_cmd_sync(hdev, 0xfc22, 1, &type, HCI_INIT_TIMEOUT); - if (IS_ERR(skb)) { - BT_ERR("%s: Retrieving Intel exception info failed (%ld)", - hdev->name, PTR_ERR(skb)); - return; - } - - if (skb->len != 13) { - BT_ERR("%s: Exception info size mismatch", hdev->name); - kfree_skb(skb); - return; - } - - BT_ERR("%s: Exception info %s", hdev->name, (char *)(skb->data + 1)); - - kfree_skb(skb); -} - static int btusb_shutdown_intel(struct hci_dev *hdev) { struct sk_buff *skb; @@ -2783,7 +2710,7 @@ static int btusb_probe(struct usb_interface *intf, if (id->driver_info & BTUSB_INTEL_NEW) { hdev->send = btusb_send_frame_intel; hdev->setup = btusb_setup_intel_new; - hdev->hw_error = btusb_hw_error_intel; + hdev->hw_error = btintel_hw_error; hdev->set_bdaddr = btintel_set_bdaddr; set_bit(HCI_QUIRK_STRICT_DUPLICATE_FILTER, &hdev->quirks); } diff --git a/drivers/bluetooth/dtl1_cs.c b/drivers/bluetooth/dtl1_cs.c index 78e10f0c65b2..84135c54ed2e 100644 --- a/drivers/bluetooth/dtl1_cs.c +++ b/drivers/bluetooth/dtl1_cs.c @@ -182,9 +182,9 @@ static void dtl1_control(struct dtl1_info *info, struct sk_buff *skb) int i; printk(KERN_INFO "Bluetooth: Nokia control data ="); - for (i = 0; i < skb->len; i++) { + for (i = 0; i < skb->len; i++) printk(" %02x", skb->data[i]); - } + printk("\n"); /* transition to active state */ @@ -406,7 +406,7 @@ static int dtl1_hci_send_frame(struct hci_dev *hdev, struct sk_buff *skb) break; default: return -EILSEQ; - }; + } nsh.zero = 0; nsh.len = skb->len; diff --git a/drivers/bluetooth/hci_bcm.c b/drivers/bluetooth/hci_bcm.c index 23523e140a9a..835bfab88ef5 100644 --- a/drivers/bluetooth/hci_bcm.c +++ b/drivers/bluetooth/hci_bcm.c @@ -25,6 +25,12 @@ #include #include #include +#include +#include +#include +#include +#include +#include #include #include @@ -32,11 +38,37 @@ #include "btbcm.h" #include "hci_uart.h" -struct bcm_data { - struct sk_buff *rx_skb; - struct sk_buff_head txq; +struct bcm_device { + struct list_head list; + + struct platform_device *pdev; + + const char *name; + struct gpio_desc *device_wakeup; + struct gpio_desc *shutdown; + + struct clk *clk; + bool clk_enabled; + + u32 init_speed; + +#ifdef CONFIG_PM_SLEEP + struct hci_uart *hu; + bool is_suspended; /* suspend/resume flag */ +#endif }; +struct bcm_data { + struct sk_buff *rx_skb; + struct sk_buff_head txq; + + struct bcm_device *dev; +}; + +/* List of BCM BT UART devices */ +static DEFINE_SPINLOCK(bcm_device_lock); +static LIST_HEAD(bcm_device_list); + static int bcm_set_baudrate(struct hci_uart *hu, unsigned int speed) { struct hci_dev *hdev = hu->hdev; @@ -86,9 +118,41 @@ static int bcm_set_baudrate(struct hci_uart *hu, unsigned int speed) return 0; } +/* bcm_device_exists should be protected by bcm_device_lock */ +static bool bcm_device_exists(struct bcm_device *device) +{ + struct list_head *p; + + list_for_each(p, &bcm_device_list) { + struct bcm_device *dev = list_entry(p, struct bcm_device, list); + + if (device == dev) + return true; + } + + return false; +} + +static int bcm_gpio_set_power(struct bcm_device *dev, bool powered) +{ + if (powered && !IS_ERR(dev->clk) && !dev->clk_enabled) + clk_enable(dev->clk); + + gpiod_set_value(dev->shutdown, powered); + gpiod_set_value(dev->device_wakeup, powered); + + if (!powered && !IS_ERR(dev->clk) && dev->clk_enabled) + clk_disable(dev->clk); + + dev->clk_enabled = powered; + + return 0; +} + static int bcm_open(struct hci_uart *hu) { struct bcm_data *bcm; + struct list_head *p; BT_DBG("hu %p", hu); @@ -99,6 +163,30 @@ static int bcm_open(struct hci_uart *hu) skb_queue_head_init(&bcm->txq); hu->priv = bcm; + + spin_lock(&bcm_device_lock); + list_for_each(p, &bcm_device_list) { + struct bcm_device *dev = list_entry(p, struct bcm_device, list); + + /* Retrieve saved bcm_device based on parent of the + * platform device (saved during device probe) and + * parent of tty device used by hci_uart + */ + if (hu->tty->dev->parent == dev->pdev->dev.parent) { + bcm->dev = dev; + hu->init_speed = dev->init_speed; +#ifdef CONFIG_PM_SLEEP + dev->hu = hu; +#endif + break; + } + } + + if (bcm->dev) + bcm_gpio_set_power(bcm->dev, true); + + spin_unlock(&bcm_device_lock); + return 0; } @@ -108,6 +196,16 @@ static int bcm_close(struct hci_uart *hu) BT_DBG("hu %p", hu); + /* Protect bcm->dev against removal of the device or driver */ + spin_lock(&bcm_device_lock); + if (bcm_device_exists(bcm->dev)) { + bcm_gpio_set_power(bcm->dev, false); +#ifdef CONFIG_PM_SLEEP + bcm->dev->hu = NULL; +#endif + } + spin_unlock(&bcm_device_lock); + skb_queue_purge(&bcm->txq); kfree_skb(bcm->rx_skb); kfree(bcm); @@ -232,6 +330,204 @@ static struct sk_buff *bcm_dequeue(struct hci_uart *hu) return skb_dequeue(&bcm->txq); } +#ifdef CONFIG_PM_SLEEP +/* Platform suspend callback */ +static int bcm_suspend(struct device *dev) +{ + struct bcm_device *bdev = platform_get_drvdata(to_platform_device(dev)); + + BT_DBG("suspend (%p): is_suspended %d", bdev, bdev->is_suspended); + + spin_lock(&bcm_device_lock); + + if (!bdev->hu) + goto unlock; + + if (!bdev->is_suspended) { + hci_uart_set_flow_control(bdev->hu, true); + + /* Once this callback returns, driver suspends BT via GPIO */ + bdev->is_suspended = true; + } + + /* Suspend the device */ + if (bdev->device_wakeup) { + gpiod_set_value(bdev->device_wakeup, false); + BT_DBG("suspend, delaying 15 ms"); + mdelay(15); + } + +unlock: + spin_unlock(&bcm_device_lock); + + return 0; +} + +/* Platform resume callback */ +static int bcm_resume(struct device *dev) +{ + struct bcm_device *bdev = platform_get_drvdata(to_platform_device(dev)); + + BT_DBG("resume (%p): is_suspended %d", bdev, bdev->is_suspended); + + spin_lock(&bcm_device_lock); + + if (!bdev->hu) + goto unlock; + + if (bdev->device_wakeup) { + gpiod_set_value(bdev->device_wakeup, true); + BT_DBG("resume, delaying 15 ms"); + mdelay(15); + } + + /* When this callback executes, the device has woken up already */ + if (bdev->is_suspended) { + bdev->is_suspended = false; + + hci_uart_set_flow_control(bdev->hu, false); + } + +unlock: + spin_unlock(&bcm_device_lock); + + return 0; +} +#endif + +static const struct acpi_gpio_params device_wakeup_gpios = { 0, 0, false }; +static const struct acpi_gpio_params shutdown_gpios = { 1, 0, false }; + +static const struct acpi_gpio_mapping acpi_bcm_default_gpios[] = { + { "device-wakeup-gpios", &device_wakeup_gpios, 1 }, + { "shutdown-gpios", &shutdown_gpios, 1 }, + { }, +}; + +#ifdef CONFIG_ACPI +static int bcm_resource(struct acpi_resource *ares, void *data) +{ + struct bcm_device *dev = data; + + if (ares->type == ACPI_RESOURCE_TYPE_SERIAL_BUS) { + struct acpi_resource_uart_serialbus *sb; + + sb = &ares->data.uart_serial_bus; + if (sb->type == ACPI_RESOURCE_SERIAL_TYPE_UART) + dev->init_speed = sb->default_baud_rate; + } + + /* Always tell the ACPI core to skip this resource */ + return 1; +} + +static int bcm_acpi_probe(struct bcm_device *dev) +{ + struct platform_device *pdev = dev->pdev; + const struct acpi_device_id *id; + struct acpi_device *adev; + LIST_HEAD(resources); + int ret; + + id = acpi_match_device(pdev->dev.driver->acpi_match_table, &pdev->dev); + if (!id) + return -ENODEV; + + /* Retrieve GPIO data */ + dev->name = dev_name(&pdev->dev); + ret = acpi_dev_add_driver_gpios(ACPI_COMPANION(&pdev->dev), + acpi_bcm_default_gpios); + if (ret) + return ret; + + dev->clk = devm_clk_get(&pdev->dev, NULL); + + dev->device_wakeup = devm_gpiod_get_optional(&pdev->dev, + "device-wakeup", + GPIOD_OUT_LOW); + if (IS_ERR(dev->device_wakeup)) + return PTR_ERR(dev->device_wakeup); + + dev->shutdown = devm_gpiod_get_optional(&pdev->dev, "shutdown", + GPIOD_OUT_LOW); + if (IS_ERR(dev->shutdown)) + return PTR_ERR(dev->shutdown); + + /* Make sure at-least one of the GPIO is defined and that + * a name is specified for this instance + */ + if ((!dev->device_wakeup && !dev->shutdown) || !dev->name) { + dev_err(&pdev->dev, "invalid platform data\n"); + return -EINVAL; + } + + /* Retrieve UART ACPI info */ + adev = ACPI_COMPANION(&dev->pdev->dev); + if (!adev) + return 0; + + acpi_dev_get_resources(adev, &resources, bcm_resource, dev); + + return 0; +} +#else +static int bcm_acpi_probe(struct bcm_device *dev) +{ + return -EINVAL; +} +#endif /* CONFIG_ACPI */ + +static int bcm_probe(struct platform_device *pdev) +{ + struct bcm_device *dev; + struct acpi_device_id *pdata = pdev->dev.platform_data; + int ret; + + dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL); + if (!dev) + return -ENOMEM; + + dev->pdev = pdev; + + if (ACPI_HANDLE(&pdev->dev)) { + ret = bcm_acpi_probe(dev); + if (ret) + return ret; + } else if (pdata) { + dev->name = pdata->id; + } else { + return -ENODEV; + } + + platform_set_drvdata(pdev, dev); + + dev_info(&pdev->dev, "%s device registered.\n", dev->name); + + /* Place this instance on the device list */ + spin_lock(&bcm_device_lock); + list_add_tail(&dev->list, &bcm_device_list); + spin_unlock(&bcm_device_lock); + + bcm_gpio_set_power(dev, false); + + return 0; +} + +static int bcm_remove(struct platform_device *pdev) +{ + struct bcm_device *dev = platform_get_drvdata(pdev); + + spin_lock(&bcm_device_lock); + list_del(&dev->list); + spin_unlock(&bcm_device_lock); + + acpi_dev_remove_driver_gpios(ACPI_COMPANION(&pdev->dev)); + + dev_info(&pdev->dev, "%s device unregistered.\n", dev->name); + + return 0; +} + static const struct hci_uart_proto bcm_proto = { .id = HCI_UART_BCM, .name = "BCM", @@ -247,12 +543,38 @@ static const struct hci_uart_proto bcm_proto = { .dequeue = bcm_dequeue, }; +#ifdef CONFIG_ACPI +static const struct acpi_device_id bcm_acpi_match[] = { + { "BCM2E39", 0 }, + { "BCM2E67", 0 }, + { }, +}; +MODULE_DEVICE_TABLE(acpi, bcm_acpi_match); +#endif + +/* Platform suspend and resume callbacks */ +static SIMPLE_DEV_PM_OPS(bcm_pm_ops, bcm_suspend, bcm_resume); + +static struct platform_driver bcm_driver = { + .probe = bcm_probe, + .remove = bcm_remove, + .driver = { + .name = "hci_bcm", + .acpi_match_table = ACPI_PTR(bcm_acpi_match), + .pm = &bcm_pm_ops, + }, +}; + int __init bcm_init(void) { + platform_driver_register(&bcm_driver); + return hci_uart_register_proto(&bcm_proto); } int __exit bcm_deinit(void) { + platform_driver_unregister(&bcm_driver); + return hci_uart_unregister_proto(&bcm_proto); } diff --git a/drivers/bluetooth/hci_h4.c b/drivers/bluetooth/hci_h4.c index 57faddc53645..eec3f28e4bb9 100644 --- a/drivers/bluetooth/hci_h4.c +++ b/drivers/bluetooth/hci_h4.c @@ -223,8 +223,7 @@ struct sk_buff *h4_recv_buf(struct hci_dev *hdev, struct sk_buff *skb, switch ((&pkts[i])->lsize) { case 0: /* No variable data length */ - (&pkts[i])->recv(hdev, skb); - skb = NULL; + dlen = 0; break; case 1: /* Single octet variable length */ @@ -252,6 +251,12 @@ struct sk_buff *h4_recv_buf(struct hci_dev *hdev, struct sk_buff *skb, kfree_skb(skb); return ERR_PTR(-EILSEQ); } + + if (!dlen) { + /* No more data, complete frame */ + (&pkts[i])->recv(hdev, skb); + skb = NULL; + } } else { /* Complete frame */ (&pkts[i])->recv(hdev, skb); diff --git a/drivers/bluetooth/hci_h5.c b/drivers/bluetooth/hci_h5.c index 3455cecc9ecf..b35b238a0380 100644 --- a/drivers/bluetooth/hci_h5.c +++ b/drivers/bluetooth/hci_h5.c @@ -75,7 +75,7 @@ struct h5 { size_t rx_pending; /* Expecting more bytes */ u8 rx_ack; /* Last ack number received */ - int (*rx_func) (struct hci_uart *hu, u8 c); + int (*rx_func)(struct hci_uart *hu, u8 c); struct timer_list timer; /* Retransmission timer */ diff --git a/drivers/bluetooth/hci_intel.c b/drivers/bluetooth/hci_intel.c index 5dd07bf05236..cf07d1121956 100644 --- a/drivers/bluetooth/hci_intel.c +++ b/drivers/bluetooth/hci_intel.c @@ -24,8 +24,864 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include #include #include #include "hci_uart.h" +#include "btintel.h" + +#define STATE_BOOTLOADER 0 +#define STATE_DOWNLOADING 1 +#define STATE_FIRMWARE_LOADED 2 +#define STATE_FIRMWARE_FAILED 3 +#define STATE_BOOTING 4 + +struct intel_device { + struct list_head list; + struct platform_device *pdev; + struct gpio_desc *reset; +}; + +static LIST_HEAD(intel_device_list); +static DEFINE_SPINLOCK(intel_device_list_lock); + +struct intel_data { + struct sk_buff *rx_skb; + struct sk_buff_head txq; + unsigned long flags; +}; + +static u8 intel_convert_speed(unsigned int speed) +{ + switch (speed) { + case 9600: + return 0x00; + case 19200: + return 0x01; + case 38400: + return 0x02; + case 57600: + return 0x03; + case 115200: + return 0x04; + case 230400: + return 0x05; + case 460800: + return 0x06; + case 921600: + return 0x07; + case 1843200: + return 0x08; + case 3250000: + return 0x09; + case 2000000: + return 0x0a; + case 3000000: + return 0x0b; + default: + return 0xff; + } +} + +static int intel_wait_booting(struct hci_uart *hu) +{ + struct intel_data *intel = hu->priv; + int err; + + err = wait_on_bit_timeout(&intel->flags, STATE_BOOTING, + TASK_INTERRUPTIBLE, + msecs_to_jiffies(1000)); + + if (err == 1) { + BT_ERR("%s: Device boot interrupted", hu->hdev->name); + return -EINTR; + } + + if (err) { + BT_ERR("%s: Device boot timeout", hu->hdev->name); + return -ETIMEDOUT; + } + + return err; +} + +static int intel_set_power(struct hci_uart *hu, bool powered) +{ + struct list_head *p; + int err = -ENODEV; + + spin_lock(&intel_device_list_lock); + + list_for_each(p, &intel_device_list) { + struct intel_device *idev = list_entry(p, struct intel_device, + list); + + /* tty device and pdev device should share the same parent + * which is the UART port. + */ + if (hu->tty->dev->parent != idev->pdev->dev.parent) + continue; + + if (!idev->reset) { + err = -ENOTSUPP; + break; + } + + BT_INFO("hu %p, Switching compatible pm device (%s) to %u", + hu, dev_name(&idev->pdev->dev), powered); + + gpiod_set_value(idev->reset, powered); + } + + spin_unlock(&intel_device_list_lock); + + return err; +} + +static int intel_open(struct hci_uart *hu) +{ + struct intel_data *intel; + + BT_DBG("hu %p", hu); + + intel = kzalloc(sizeof(*intel), GFP_KERNEL); + if (!intel) + return -ENOMEM; + + skb_queue_head_init(&intel->txq); + + hu->priv = intel; + + if (!intel_set_power(hu, true)) + set_bit(STATE_BOOTING, &intel->flags); + + return 0; +} + +static int intel_close(struct hci_uart *hu) +{ + struct intel_data *intel = hu->priv; + + BT_DBG("hu %p", hu); + + intel_set_power(hu, false); + + skb_queue_purge(&intel->txq); + kfree_skb(intel->rx_skb); + kfree(intel); + + hu->priv = NULL; + return 0; +} + +static int intel_flush(struct hci_uart *hu) +{ + struct intel_data *intel = hu->priv; + + BT_DBG("hu %p", hu); + + skb_queue_purge(&intel->txq); + + return 0; +} + +static int inject_cmd_complete(struct hci_dev *hdev, __u16 opcode) +{ + struct sk_buff *skb; + struct hci_event_hdr *hdr; + struct hci_ev_cmd_complete *evt; + + skb = bt_skb_alloc(sizeof(*hdr) + sizeof(*evt) + 1, GFP_ATOMIC); + if (!skb) + return -ENOMEM; + + hdr = (struct hci_event_hdr *)skb_put(skb, sizeof(*hdr)); + hdr->evt = HCI_EV_CMD_COMPLETE; + hdr->plen = sizeof(*evt) + 1; + + evt = (struct hci_ev_cmd_complete *)skb_put(skb, sizeof(*evt)); + evt->ncmd = 0x01; + evt->opcode = cpu_to_le16(opcode); + + *skb_put(skb, 1) = 0x00; + + bt_cb(skb)->pkt_type = HCI_EVENT_PKT; + + return hci_recv_frame(hdev, skb); +} + +static int intel_set_baudrate(struct hci_uart *hu, unsigned int speed) +{ + struct intel_data *intel = hu->priv; + struct hci_dev *hdev = hu->hdev; + u8 speed_cmd[] = { 0x06, 0xfc, 0x01, 0x00 }; + struct sk_buff *skb; + int err; + + /* This can be the first command sent to the chip, check + * that the controller is ready. + */ + err = intel_wait_booting(hu); + + clear_bit(STATE_BOOTING, &intel->flags); + + /* In case of timeout, try to continue anyway */ + if (err && err != ETIMEDOUT) + return err; + + BT_INFO("%s: Change controller speed to %d", hdev->name, speed); + + speed_cmd[3] = intel_convert_speed(speed); + if (speed_cmd[3] == 0xff) { + BT_ERR("%s: Unsupported speed", hdev->name); + return -EINVAL; + } + + /* Device will not accept speed change if Intel version has not been + * previously requested. + */ + skb = __hci_cmd_sync(hdev, 0xfc05, 0, NULL, HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) { + BT_ERR("%s: Reading Intel version information failed (%ld)", + hdev->name, PTR_ERR(skb)); + return PTR_ERR(skb); + } + kfree_skb(skb); + + skb = bt_skb_alloc(sizeof(speed_cmd), GFP_KERNEL); + if (!skb) { + BT_ERR("%s: Failed to allocate memory for baudrate packet", + hdev->name); + return -ENOMEM; + } + + memcpy(skb_put(skb, sizeof(speed_cmd)), speed_cmd, sizeof(speed_cmd)); + bt_cb(skb)->pkt_type = HCI_COMMAND_PKT; + + hci_uart_set_flow_control(hu, true); + + skb_queue_tail(&intel->txq, skb); + hci_uart_tx_wakeup(hu); + + /* wait 100ms to change baudrate on controller side */ + msleep(100); + + hci_uart_set_baudrate(hu, speed); + hci_uart_set_flow_control(hu, false); + + return 0; +} + +static int intel_setup(struct hci_uart *hu) +{ + static const u8 reset_param[] = { 0x00, 0x01, 0x00, 0x01, + 0x00, 0x08, 0x04, 0x00 }; + struct intel_data *intel = hu->priv; + struct hci_dev *hdev = hu->hdev; + struct sk_buff *skb; + struct intel_version *ver; + struct intel_boot_params *params; + const struct firmware *fw; + const u8 *fw_ptr; + char fwname[64]; + u32 frag_len; + ktime_t calltime, delta, rettime; + unsigned long long duration; + unsigned int init_speed, oper_speed; + int speed_change = 0; + int err; + + BT_DBG("%s", hdev->name); + + hu->hdev->set_bdaddr = btintel_set_bdaddr; + + calltime = ktime_get(); + + if (hu->init_speed) + init_speed = hu->init_speed; + else + init_speed = hu->proto->init_speed; + + if (hu->oper_speed) + oper_speed = hu->oper_speed; + else + oper_speed = hu->proto->oper_speed; + + if (oper_speed && init_speed && oper_speed != init_speed) + speed_change = 1; + + /* Check that the controller is ready */ + err = intel_wait_booting(hu); + + clear_bit(STATE_BOOTING, &intel->flags); + + /* In case of timeout, try to continue anyway */ + if (err && err != ETIMEDOUT) + return err; + + set_bit(STATE_BOOTLOADER, &intel->flags); + + /* Read the Intel version information to determine if the device + * is in bootloader mode or if it already has operational firmware + * loaded. + */ + skb = __hci_cmd_sync(hdev, 0xfc05, 0, NULL, HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) { + BT_ERR("%s: Reading Intel version information failed (%ld)", + hdev->name, PTR_ERR(skb)); + return PTR_ERR(skb); + } + + if (skb->len != sizeof(*ver)) { + BT_ERR("%s: Intel version event size mismatch", hdev->name); + kfree_skb(skb); + return -EILSEQ; + } + + ver = (struct intel_version *)skb->data; + if (ver->status) { + BT_ERR("%s: Intel version command failure (%02x)", + hdev->name, ver->status); + err = -bt_to_errno(ver->status); + kfree_skb(skb); + return err; + } + + /* The hardware platform number has a fixed value of 0x37 and + * for now only accept this single value. + */ + if (ver->hw_platform != 0x37) { + BT_ERR("%s: Unsupported Intel hardware platform (%u)", + hdev->name, ver->hw_platform); + kfree_skb(skb); + return -EINVAL; + } + + /* At the moment only the hardware variant iBT 3.0 (LnP/SfP) is + * supported by this firmware loading method. This check has been + * put in place to ensure correct forward compatibility options + * when newer hardware variants come along. + */ + if (ver->hw_variant != 0x0b) { + BT_ERR("%s: Unsupported Intel hardware variant (%u)", + hdev->name, ver->hw_variant); + kfree_skb(skb); + return -EINVAL; + } + + btintel_version_info(hdev, ver); + + /* The firmware variant determines if the device is in bootloader + * mode or is running operational firmware. The value 0x06 identifies + * the bootloader and the value 0x23 identifies the operational + * firmware. + * + * When the operational firmware is already present, then only + * the check for valid Bluetooth device address is needed. This + * determines if the device will be added as configured or + * unconfigured controller. + * + * It is not possible to use the Secure Boot Parameters in this + * case since that command is only available in bootloader mode. + */ + if (ver->fw_variant == 0x23) { + kfree_skb(skb); + clear_bit(STATE_BOOTLOADER, &intel->flags); + btintel_check_bdaddr(hdev); + return 0; + } + + /* If the device is not in bootloader mode, then the only possible + * choice is to return an error and abort the device initialization. + */ + if (ver->fw_variant != 0x06) { + BT_ERR("%s: Unsupported Intel firmware variant (%u)", + hdev->name, ver->fw_variant); + kfree_skb(skb); + return -ENODEV; + } + + kfree_skb(skb); + + /* Read the secure boot parameters to identify the operating + * details of the bootloader. + */ + skb = __hci_cmd_sync(hdev, 0xfc0d, 0, NULL, HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) { + BT_ERR("%s: Reading Intel boot parameters failed (%ld)", + hdev->name, PTR_ERR(skb)); + return PTR_ERR(skb); + } + + if (skb->len != sizeof(*params)) { + BT_ERR("%s: Intel boot parameters size mismatch", hdev->name); + kfree_skb(skb); + return -EILSEQ; + } + + params = (struct intel_boot_params *)skb->data; + if (params->status) { + BT_ERR("%s: Intel boot parameters command failure (%02x)", + hdev->name, params->status); + err = -bt_to_errno(params->status); + kfree_skb(skb); + return err; + } + + BT_INFO("%s: Device revision is %u", hdev->name, + le16_to_cpu(params->dev_revid)); + + BT_INFO("%s: Secure boot is %s", hdev->name, + params->secure_boot ? "enabled" : "disabled"); + + BT_INFO("%s: Minimum firmware build %u week %u %u", hdev->name, + params->min_fw_build_nn, params->min_fw_build_cw, + 2000 + params->min_fw_build_yy); + + /* It is required that every single firmware fragment is acknowledged + * with a command complete event. If the boot parameters indicate + * that this bootloader does not send them, then abort the setup. + */ + if (params->limited_cce != 0x00) { + BT_ERR("%s: Unsupported Intel firmware loading method (%u)", + hdev->name, params->limited_cce); + kfree_skb(skb); + return -EINVAL; + } + + /* If the OTP has no valid Bluetooth device address, then there will + * also be no valid address for the operational firmware. + */ + if (!bacmp(¶ms->otp_bdaddr, BDADDR_ANY)) { + BT_INFO("%s: No device address configured", hdev->name); + set_bit(HCI_QUIRK_INVALID_BDADDR, &hdev->quirks); + } + + /* With this Intel bootloader only the hardware variant and device + * revision information are used to select the right firmware. + * + * Currently this bootloader support is limited to hardware variant + * iBT 3.0 (LnP/SfP) which is identified by the value 11 (0x0b). + */ + snprintf(fwname, sizeof(fwname), "intel/ibt-11-%u.sfi", + le16_to_cpu(params->dev_revid)); + + err = request_firmware(&fw, fwname, &hdev->dev); + if (err < 0) { + BT_ERR("%s: Failed to load Intel firmware file (%d)", + hdev->name, err); + kfree_skb(skb); + return err; + } + + BT_INFO("%s: Found device firmware: %s", hdev->name, fwname); + + kfree_skb(skb); + + if (fw->size < 644) { + BT_ERR("%s: Invalid size of firmware file (%zu)", + hdev->name, fw->size); + err = -EBADF; + goto done; + } + + set_bit(STATE_DOWNLOADING, &intel->flags); + + /* Start the firmware download transaction with the Init fragment + * represented by the 128 bytes of CSS header. + */ + err = btintel_secure_send(hdev, 0x00, 128, fw->data); + if (err < 0) { + BT_ERR("%s: Failed to send firmware header (%d)", + hdev->name, err); + goto done; + } + + /* Send the 256 bytes of public key information from the firmware + * as the PKey fragment. + */ + err = btintel_secure_send(hdev, 0x03, 256, fw->data + 128); + if (err < 0) { + BT_ERR("%s: Failed to send firmware public key (%d)", + hdev->name, err); + goto done; + } + + /* Send the 256 bytes of signature information from the firmware + * as the Sign fragment. + */ + err = btintel_secure_send(hdev, 0x02, 256, fw->data + 388); + if (err < 0) { + BT_ERR("%s: Failed to send firmware signature (%d)", + hdev->name, err); + goto done; + } + + fw_ptr = fw->data + 644; + frag_len = 0; + + while (fw_ptr - fw->data < fw->size) { + struct hci_command_hdr *cmd = (void *)(fw_ptr + frag_len); + + frag_len += sizeof(*cmd) + cmd->plen; + + BT_DBG("%s: patching %td/%zu", hdev->name, + (fw_ptr - fw->data), fw->size); + + /* The parameter length of the secure send command requires + * a 4 byte alignment. It happens so that the firmware file + * contains proper Intel_NOP commands to align the fragments + * as needed. + * + * Send set of commands with 4 byte alignment from the + * firmware data buffer as a single Data fragement. + */ + if (frag_len % 4) + continue; + + /* Send each command from the firmware data buffer as + * a single Data fragment. + */ + err = btintel_secure_send(hdev, 0x01, frag_len, fw_ptr); + if (err < 0) { + BT_ERR("%s: Failed to send firmware data (%d)", + hdev->name, err); + goto done; + } + + fw_ptr += frag_len; + frag_len = 0; + } + + set_bit(STATE_FIRMWARE_LOADED, &intel->flags); + + BT_INFO("%s: Waiting for firmware download to complete", hdev->name); + + /* Before switching the device into operational mode and with that + * booting the loaded firmware, wait for the bootloader notification + * that all fragments have been successfully received. + * + * When the event processing receives the notification, then the + * STATE_DOWNLOADING flag will be cleared. + * + * The firmware loading should not take longer than 5 seconds + * and thus just timeout if that happens and fail the setup + * of this device. + */ + err = wait_on_bit_timeout(&intel->flags, STATE_DOWNLOADING, + TASK_INTERRUPTIBLE, + msecs_to_jiffies(5000)); + if (err == 1) { + BT_ERR("%s: Firmware loading interrupted", hdev->name); + err = -EINTR; + goto done; + } + + if (err) { + BT_ERR("%s: Firmware loading timeout", hdev->name); + err = -ETIMEDOUT; + goto done; + } + + if (test_bit(STATE_FIRMWARE_FAILED, &intel->flags)) { + BT_ERR("%s: Firmware loading failed", hdev->name); + err = -ENOEXEC; + goto done; + } + + rettime = ktime_get(); + delta = ktime_sub(rettime, calltime); + duration = (unsigned long long) ktime_to_ns(delta) >> 10; + + BT_INFO("%s: Firmware loaded in %llu usecs", hdev->name, duration); + +done: + release_firmware(fw); + + if (err < 0) + return err; + + /* We need to restore the default speed before Intel reset */ + if (speed_change) { + err = intel_set_baudrate(hu, init_speed); + if (err) + return err; + } + + calltime = ktime_get(); + + set_bit(STATE_BOOTING, &intel->flags); + + skb = __hci_cmd_sync(hdev, 0xfc01, sizeof(reset_param), reset_param, + HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) + return PTR_ERR(skb); + + kfree_skb(skb); + + /* The bootloader will not indicate when the device is ready. This + * is done by the operational firmware sending bootup notification. + * + * Booting into operational firmware should not take longer than + * 1 second. However if that happens, then just fail the setup + * since something went wrong. + */ + BT_INFO("%s: Waiting for device to boot", hdev->name); + + err = intel_wait_booting(hu); + if (err) + return err; + + clear_bit(STATE_BOOTING, &intel->flags); + + rettime = ktime_get(); + delta = ktime_sub(rettime, calltime); + duration = (unsigned long long) ktime_to_ns(delta) >> 10; + + BT_INFO("%s: Device booted in %llu usecs", hdev->name, duration); + + skb = __hci_cmd_sync(hdev, HCI_OP_RESET, 0, NULL, HCI_CMD_TIMEOUT); + if (IS_ERR(skb)) + return PTR_ERR(skb); + kfree_skb(skb); + + if (speed_change) { + err = intel_set_baudrate(hu, oper_speed); + if (err) + return err; + } + + BT_INFO("%s: Setup complete", hdev->name); + + clear_bit(STATE_BOOTLOADER, &intel->flags); + + return 0; +} + +static int intel_recv_event(struct hci_dev *hdev, struct sk_buff *skb) +{ + struct hci_uart *hu = hci_get_drvdata(hdev); + struct intel_data *intel = hu->priv; + struct hci_event_hdr *hdr; + + if (!test_bit(STATE_BOOTLOADER, &intel->flags) && + !test_bit(STATE_BOOTING, &intel->flags)) + goto recv; + + hdr = (void *)skb->data; + + /* When the firmware loading completes the device sends + * out a vendor specific event indicating the result of + * the firmware loading. + */ + if (skb->len == 7 && hdr->evt == 0xff && hdr->plen == 0x05 && + skb->data[2] == 0x06) { + if (skb->data[3] != 0x00) + set_bit(STATE_FIRMWARE_FAILED, &intel->flags); + + if (test_and_clear_bit(STATE_DOWNLOADING, &intel->flags) && + test_bit(STATE_FIRMWARE_LOADED, &intel->flags)) { + smp_mb__after_atomic(); + wake_up_bit(&intel->flags, STATE_DOWNLOADING); + } + + /* When switching to the operational firmware the device + * sends a vendor specific event indicating that the bootup + * completed. + */ + } else if (skb->len == 9 && hdr->evt == 0xff && hdr->plen == 0x07 && + skb->data[2] == 0x02) { + if (test_and_clear_bit(STATE_BOOTING, &intel->flags)) { + smp_mb__after_atomic(); + wake_up_bit(&intel->flags, STATE_BOOTING); + } + } +recv: + return hci_recv_frame(hdev, skb); +} + +static const struct h4_recv_pkt intel_recv_pkts[] = { + { H4_RECV_ACL, .recv = hci_recv_frame }, + { H4_RECV_SCO, .recv = hci_recv_frame }, + { H4_RECV_EVENT, .recv = intel_recv_event }, +}; + +static int intel_recv(struct hci_uart *hu, const void *data, int count) +{ + struct intel_data *intel = hu->priv; + + if (!test_bit(HCI_UART_REGISTERED, &hu->flags)) + return -EUNATCH; + + intel->rx_skb = h4_recv_buf(hu->hdev, intel->rx_skb, data, count, + intel_recv_pkts, + ARRAY_SIZE(intel_recv_pkts)); + if (IS_ERR(intel->rx_skb)) { + int err = PTR_ERR(intel->rx_skb); + BT_ERR("%s: Frame reassembly failed (%d)", hu->hdev->name, err); + intel->rx_skb = NULL; + return err; + } + + return count; +} + +static int intel_enqueue(struct hci_uart *hu, struct sk_buff *skb) +{ + struct intel_data *intel = hu->priv; + + BT_DBG("hu %p skb %p", hu, skb); + + skb_queue_tail(&intel->txq, skb); + + return 0; +} + +static struct sk_buff *intel_dequeue(struct hci_uart *hu) +{ + struct intel_data *intel = hu->priv; + struct sk_buff *skb; + + skb = skb_dequeue(&intel->txq); + if (!skb) + return skb; + + if (test_bit(STATE_BOOTLOADER, &intel->flags) && + (bt_cb(skb)->pkt_type == HCI_COMMAND_PKT)) { + struct hci_command_hdr *cmd = (void *)skb->data; + __u16 opcode = le16_to_cpu(cmd->opcode); + + /* When the 0xfc01 command is issued to boot into + * the operational firmware, it will actually not + * send a command complete event. To keep the flow + * control working inject that event here. + */ + if (opcode == 0xfc01) + inject_cmd_complete(hu->hdev, opcode); + } + + /* Prepend skb with frame type */ + memcpy(skb_push(skb, 1), &bt_cb(skb)->pkt_type, 1); + + return skb; +} + +static const struct hci_uart_proto intel_proto = { + .id = HCI_UART_INTEL, + .name = "Intel", + .init_speed = 115200, + .oper_speed = 3000000, + .open = intel_open, + .close = intel_close, + .flush = intel_flush, + .setup = intel_setup, + .set_baudrate = intel_set_baudrate, + .recv = intel_recv, + .enqueue = intel_enqueue, + .dequeue = intel_dequeue, +}; + +#ifdef CONFIG_ACPI +static const struct acpi_device_id intel_acpi_match[] = { + { "INT33E1", 0 }, + { }, +}; +MODULE_DEVICE_TABLE(acpi, intel_acpi_match); + +static int intel_acpi_probe(struct intel_device *idev) +{ + const struct acpi_device_id *id; + + id = acpi_match_device(intel_acpi_match, &idev->pdev->dev); + if (!id) + return -ENODEV; + + return 0; +} +#else +static int intel_acpi_probe(struct intel_device *idev) +{ + return -ENODEV; +} +#endif + +static int intel_probe(struct platform_device *pdev) +{ + struct intel_device *idev; + + idev = devm_kzalloc(&pdev->dev, sizeof(*idev), GFP_KERNEL); + if (!idev) + return -ENOMEM; + + idev->pdev = pdev; + + if (ACPI_HANDLE(&pdev->dev)) { + int err = intel_acpi_probe(idev); + if (err) + return err; + } else { + return -ENODEV; + } + + idev->reset = devm_gpiod_get_optional(&pdev->dev, "reset", + GPIOD_OUT_LOW); + if (IS_ERR(idev->reset)) { + dev_err(&pdev->dev, "Unable to retrieve gpio\n"); + return PTR_ERR(idev->reset); + } + + platform_set_drvdata(pdev, idev); + + /* Place this instance on the device list */ + spin_lock(&intel_device_list_lock); + list_add_tail(&idev->list, &intel_device_list); + spin_unlock(&intel_device_list_lock); + + dev_info(&pdev->dev, "registered.\n"); + + return 0; +} + +static int intel_remove(struct platform_device *pdev) +{ + struct intel_device *idev = platform_get_drvdata(pdev); + + spin_lock(&intel_device_list_lock); + list_del(&idev->list); + spin_unlock(&intel_device_list_lock); + + dev_info(&pdev->dev, "unregistered.\n"); + + return 0; +} + +static struct platform_driver intel_driver = { + .probe = intel_probe, + .remove = intel_remove, + .driver = { + .name = "hci_intel", + .acpi_match_table = ACPI_PTR(intel_acpi_match), + }, +}; + +int __init intel_init(void) +{ + platform_driver_register(&intel_driver); + + return hci_uart_register_proto(&intel_proto); +} + +int __exit intel_deinit(void) +{ + platform_driver_unregister(&intel_driver); + + return hci_uart_unregister_proto(&intel_proto); +} diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c index 177dd69fdd95..0d5a05a7c1fd 100644 --- a/drivers/bluetooth/hci_ldisc.c +++ b/drivers/bluetooth/hci_ldisc.c @@ -770,7 +770,7 @@ static int __init hci_uart_init(void) /* Register the tty discipline */ - memset(&hci_uart_ldisc, 0, sizeof (hci_uart_ldisc)); + memset(&hci_uart_ldisc, 0, sizeof(hci_uart_ldisc)); hci_uart_ldisc.magic = TTY_LDISC_MAGIC; hci_uart_ldisc.name = "n_hci"; hci_uart_ldisc.open = hci_uart_tty_open; @@ -804,9 +804,15 @@ static int __init hci_uart_init(void) #ifdef CONFIG_BT_HCIUART_3WIRE h5_init(); #endif +#ifdef CONFIG_BT_HCIUART_INTEL + intel_init(); +#endif #ifdef CONFIG_BT_HCIUART_BCM bcm_init(); #endif +#ifdef CONFIG_BT_HCIUART_QCA + qca_init(); +#endif return 0; } @@ -830,9 +836,15 @@ static void __exit hci_uart_exit(void) #ifdef CONFIG_BT_HCIUART_3WIRE h5_deinit(); #endif +#ifdef CONFIG_BT_HCIUART_INTEL + intel_deinit(); +#endif #ifdef CONFIG_BT_HCIUART_BCM bcm_deinit(); #endif +#ifdef CONFIG_BT_HCIUART_QCA + qca_deinit(); +#endif /* Release tty registration of line discipline */ err = tty_unregister_ldisc(N_HCI); diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c new file mode 100644 index 000000000000..6b9b91267959 --- /dev/null +++ b/drivers/bluetooth/hci_qca.c @@ -0,0 +1,969 @@ +/* + * Bluetooth Software UART Qualcomm protocol + * + * HCI_IBS (HCI In-Band Sleep) is Qualcomm's power management + * protocol extension to H4. + * + * Copyright (C) 2007 Texas Instruments, Inc. + * Copyright (c) 2010, 2012 The Linux Foundation. All rights reserved. + * + * Acknowledgements: + * This file is based on hci_ll.c, which was... + * Written by Ohad Ben-Cohen + * which was in turn based on hci_h4.c, which was written + * by Maxim Krasnyansky and Marcel Holtmann. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 + * as published by the Free Software Foundation + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#include +#include + +#include +#include + +#include "hci_uart.h" +#include "btqca.h" + +/* HCI_IBS protocol messages */ +#define HCI_IBS_SLEEP_IND 0xFE +#define HCI_IBS_WAKE_IND 0xFD +#define HCI_IBS_WAKE_ACK 0xFC +#define HCI_MAX_IBS_SIZE 10 + +/* Controller states */ +#define STATE_IN_BAND_SLEEP_ENABLED 1 + +#define IBS_WAKE_RETRANS_TIMEOUT_MS 100 +#define IBS_TX_IDLE_TIMEOUT_MS 2000 +#define BAUDRATE_SETTLE_TIMEOUT_MS 300 + +/* HCI_IBS transmit side sleep protocol states */ +enum tx_ibs_states { + HCI_IBS_TX_ASLEEP, + HCI_IBS_TX_WAKING, + HCI_IBS_TX_AWAKE, +}; + +/* HCI_IBS receive side sleep protocol states */ +enum rx_states { + HCI_IBS_RX_ASLEEP, + HCI_IBS_RX_AWAKE, +}; + +/* HCI_IBS transmit and receive side clock state vote */ +enum hci_ibs_clock_state_vote { + HCI_IBS_VOTE_STATS_UPDATE, + HCI_IBS_TX_VOTE_CLOCK_ON, + HCI_IBS_TX_VOTE_CLOCK_OFF, + HCI_IBS_RX_VOTE_CLOCK_ON, + HCI_IBS_RX_VOTE_CLOCK_OFF, +}; + +struct qca_data { + struct hci_uart *hu; + struct sk_buff *rx_skb; + struct sk_buff_head txq; + struct sk_buff_head tx_wait_q; /* HCI_IBS wait queue */ + spinlock_t hci_ibs_lock; /* HCI_IBS state lock */ + u8 tx_ibs_state; /* HCI_IBS transmit side power state*/ + u8 rx_ibs_state; /* HCI_IBS receive side power state */ + u32 tx_vote; /* Clock must be on for TX */ + u32 rx_vote; /* Clock must be on for RX */ + struct timer_list tx_idle_timer; + u32 tx_idle_delay; + struct timer_list wake_retrans_timer; + u32 wake_retrans; + struct workqueue_struct *workqueue; + struct work_struct ws_awake_rx; + struct work_struct ws_awake_device; + struct work_struct ws_rx_vote_off; + struct work_struct ws_tx_vote_off; + unsigned long flags; + + /* For debugging purpose */ + u64 ibs_sent_wacks; + u64 ibs_sent_slps; + u64 ibs_sent_wakes; + u64 ibs_recv_wacks; + u64 ibs_recv_slps; + u64 ibs_recv_wakes; + u64 vote_last_jif; + u32 vote_on_ms; + u32 vote_off_ms; + u64 tx_votes_on; + u64 rx_votes_on; + u64 tx_votes_off; + u64 rx_votes_off; + u64 votes_on; + u64 votes_off; +}; + +static void __serial_clock_on(struct tty_struct *tty) +{ + /* TODO: Some chipset requires to enable UART clock on client + * side to save power consumption or manual work is required. + * Please put your code to control UART clock here if needed + */ +} + +static void __serial_clock_off(struct tty_struct *tty) +{ + /* TODO: Some chipset requires to disable UART clock on client + * side to save power consumption or manual work is required. + * Please put your code to control UART clock off here if needed + */ +} + +/* serial_clock_vote needs to be called with the ibs lock held */ +static void serial_clock_vote(unsigned long vote, struct hci_uart *hu) +{ + struct qca_data *qca = hu->priv; + unsigned int diff; + + bool old_vote = (qca->tx_vote | qca->rx_vote); + bool new_vote; + + switch (vote) { + case HCI_IBS_VOTE_STATS_UPDATE: + diff = jiffies_to_msecs(jiffies - qca->vote_last_jif); + + if (old_vote) + qca->vote_off_ms += diff; + else + qca->vote_on_ms += diff; + return; + + case HCI_IBS_TX_VOTE_CLOCK_ON: + qca->tx_vote = true; + qca->tx_votes_on++; + new_vote = true; + break; + + case HCI_IBS_RX_VOTE_CLOCK_ON: + qca->rx_vote = true; + qca->rx_votes_on++; + new_vote = true; + break; + + case HCI_IBS_TX_VOTE_CLOCK_OFF: + qca->tx_vote = false; + qca->tx_votes_off++; + new_vote = qca->rx_vote | qca->tx_vote; + break; + + case HCI_IBS_RX_VOTE_CLOCK_OFF: + qca->rx_vote = false; + qca->rx_votes_off++; + new_vote = qca->rx_vote | qca->tx_vote; + break; + + default: + BT_ERR("Voting irregularity"); + return; + } + + if (new_vote != old_vote) { + if (new_vote) + __serial_clock_on(hu->tty); + else + __serial_clock_off(hu->tty); + + BT_DBG("Vote serial clock %s(%s)", new_vote? "true" : "false", + vote? "true" : "false"); + + diff = jiffies_to_msecs(jiffies - qca->vote_last_jif); + + if (new_vote) { + qca->votes_on++; + qca->vote_off_ms += diff; + } else { + qca->votes_off++; + qca->vote_on_ms += diff; + } + qca->vote_last_jif = jiffies; + } +} + +/* Builds and sends an HCI_IBS command packet. + * These are very simple packets with only 1 cmd byte. + */ +static int send_hci_ibs_cmd(u8 cmd, struct hci_uart *hu) +{ + int err = 0; + struct sk_buff *skb = NULL; + struct qca_data *qca = hu->priv; + + BT_DBG("hu %p send hci ibs cmd 0x%x", hu, cmd); + + skb = bt_skb_alloc(1, GFP_ATOMIC); + if (!skb) { + BT_ERR("Failed to allocate memory for HCI_IBS packet"); + return -ENOMEM; + } + + /* Assign HCI_IBS type */ + *skb_put(skb, 1) = cmd; + + skb_queue_tail(&qca->txq, skb); + + return err; +} + +static void qca_wq_awake_device(struct work_struct *work) +{ + struct qca_data *qca = container_of(work, struct qca_data, + ws_awake_device); + struct hci_uart *hu = qca->hu; + unsigned long retrans_delay; + + BT_DBG("hu %p wq awake device", hu); + + /* Vote for serial clock */ + serial_clock_vote(HCI_IBS_TX_VOTE_CLOCK_ON, hu); + + spin_lock(&qca->hci_ibs_lock); + + /* Send wake indication to device */ + if (send_hci_ibs_cmd(HCI_IBS_WAKE_IND, hu) < 0) + BT_ERR("Failed to send WAKE to device"); + + qca->ibs_sent_wakes++; + + /* Start retransmit timer */ + retrans_delay = msecs_to_jiffies(qca->wake_retrans); + mod_timer(&qca->wake_retrans_timer, jiffies + retrans_delay); + + spin_unlock(&qca->hci_ibs_lock); + + /* Actually send the packets */ + hci_uart_tx_wakeup(hu); +} + +static void qca_wq_awake_rx(struct work_struct *work) +{ + struct qca_data *qca = container_of(work, struct qca_data, + ws_awake_rx); + struct hci_uart *hu = qca->hu; + + BT_DBG("hu %p wq awake rx", hu); + + serial_clock_vote(HCI_IBS_RX_VOTE_CLOCK_ON, hu); + + spin_lock(&qca->hci_ibs_lock); + qca->rx_ibs_state = HCI_IBS_RX_AWAKE; + + /* Always acknowledge device wake up, + * sending IBS message doesn't count as TX ON. + */ + if (send_hci_ibs_cmd(HCI_IBS_WAKE_ACK, hu) < 0) + BT_ERR("Failed to acknowledge device wake up"); + + qca->ibs_sent_wacks++; + + spin_unlock(&qca->hci_ibs_lock); + + /* Actually send the packets */ + hci_uart_tx_wakeup(hu); +} + +static void qca_wq_serial_rx_clock_vote_off(struct work_struct *work) +{ + struct qca_data *qca = container_of(work, struct qca_data, + ws_rx_vote_off); + struct hci_uart *hu = qca->hu; + + BT_DBG("hu %p rx clock vote off", hu); + + serial_clock_vote(HCI_IBS_RX_VOTE_CLOCK_OFF, hu); +} + +static void qca_wq_serial_tx_clock_vote_off(struct work_struct *work) +{ + struct qca_data *qca = container_of(work, struct qca_data, + ws_tx_vote_off); + struct hci_uart *hu = qca->hu; + + BT_DBG("hu %p tx clock vote off", hu); + + /* Run HCI tx handling unlocked */ + hci_uart_tx_wakeup(hu); + + /* Now that message queued to tty driver, vote for tty clocks off. + * It is up to the tty driver to pend the clocks off until tx done. + */ + serial_clock_vote(HCI_IBS_TX_VOTE_CLOCK_OFF, hu); +} + +static void hci_ibs_tx_idle_timeout(unsigned long arg) +{ + struct hci_uart *hu = (struct hci_uart *)arg; + struct qca_data *qca = hu->priv; + unsigned long flags; + + BT_DBG("hu %p idle timeout in %d state", hu, qca->tx_ibs_state); + + spin_lock_irqsave_nested(&qca->hci_ibs_lock, + flags, SINGLE_DEPTH_NESTING); + + switch (qca->tx_ibs_state) { + case HCI_IBS_TX_AWAKE: + /* TX_IDLE, go to SLEEP */ + if (send_hci_ibs_cmd(HCI_IBS_SLEEP_IND, hu) < 0) { + BT_ERR("Failed to send SLEEP to device"); + break; + } + qca->tx_ibs_state = HCI_IBS_TX_ASLEEP; + qca->ibs_sent_slps++; + queue_work(qca->workqueue, &qca->ws_tx_vote_off); + break; + + case HCI_IBS_TX_ASLEEP: + case HCI_IBS_TX_WAKING: + /* Fall through */ + + default: + BT_ERR("Spurrious timeout tx state %d", qca->tx_ibs_state); + break; + } + + spin_unlock_irqrestore(&qca->hci_ibs_lock, flags); +} + +static void hci_ibs_wake_retrans_timeout(unsigned long arg) +{ + struct hci_uart *hu = (struct hci_uart *)arg; + struct qca_data *qca = hu->priv; + unsigned long flags, retrans_delay; + unsigned long retransmit = 0; + + BT_DBG("hu %p wake retransmit timeout in %d state", + hu, qca->tx_ibs_state); + + spin_lock_irqsave_nested(&qca->hci_ibs_lock, + flags, SINGLE_DEPTH_NESTING); + + switch (qca->tx_ibs_state) { + case HCI_IBS_TX_WAKING: + /* No WAKE_ACK, retransmit WAKE */ + retransmit = 1; + if (send_hci_ibs_cmd(HCI_IBS_WAKE_IND, hu) < 0) { + BT_ERR("Failed to acknowledge device wake up"); + break; + } + qca->ibs_sent_wakes++; + retrans_delay = msecs_to_jiffies(qca->wake_retrans); + mod_timer(&qca->wake_retrans_timer, jiffies + retrans_delay); + break; + + case HCI_IBS_TX_ASLEEP: + case HCI_IBS_TX_AWAKE: + /* Fall through */ + + default: + BT_ERR("Spurrious timeout tx state %d", qca->tx_ibs_state); + break; + } + + spin_unlock_irqrestore(&qca->hci_ibs_lock, flags); + + if (retransmit) + hci_uart_tx_wakeup(hu); +} + +/* Initialize protocol */ +static int qca_open(struct hci_uart *hu) +{ + struct qca_data *qca; + + BT_DBG("hu %p qca_open", hu); + + qca = kzalloc(sizeof(struct qca_data), GFP_ATOMIC); + if (!qca) + return -ENOMEM; + + skb_queue_head_init(&qca->txq); + skb_queue_head_init(&qca->tx_wait_q); + spin_lock_init(&qca->hci_ibs_lock); + qca->workqueue = create_singlethread_workqueue("qca_wq"); + if (!qca->workqueue) { + BT_ERR("QCA Workqueue not initialized properly"); + kfree(qca); + return -ENOMEM; + } + + INIT_WORK(&qca->ws_awake_rx, qca_wq_awake_rx); + INIT_WORK(&qca->ws_awake_device, qca_wq_awake_device); + INIT_WORK(&qca->ws_rx_vote_off, qca_wq_serial_rx_clock_vote_off); + INIT_WORK(&qca->ws_tx_vote_off, qca_wq_serial_tx_clock_vote_off); + + qca->hu = hu; + + /* Assume we start with both sides asleep -- extra wakes OK */ + qca->tx_ibs_state = HCI_IBS_TX_ASLEEP; + qca->rx_ibs_state = HCI_IBS_RX_ASLEEP; + + /* clocks actually on, but we start votes off */ + qca->tx_vote = false; + qca->rx_vote = false; + qca->flags = 0; + + qca->ibs_sent_wacks = 0; + qca->ibs_sent_slps = 0; + qca->ibs_sent_wakes = 0; + qca->ibs_recv_wacks = 0; + qca->ibs_recv_slps = 0; + qca->ibs_recv_wakes = 0; + qca->vote_last_jif = jiffies; + qca->vote_on_ms = 0; + qca->vote_off_ms = 0; + qca->votes_on = 0; + qca->votes_off = 0; + qca->tx_votes_on = 0; + qca->tx_votes_off = 0; + qca->rx_votes_on = 0; + qca->rx_votes_off = 0; + + hu->priv = qca; + + init_timer(&qca->wake_retrans_timer); + qca->wake_retrans_timer.function = hci_ibs_wake_retrans_timeout; + qca->wake_retrans_timer.data = (u_long)hu; + qca->wake_retrans = IBS_WAKE_RETRANS_TIMEOUT_MS; + + init_timer(&qca->tx_idle_timer); + qca->tx_idle_timer.function = hci_ibs_tx_idle_timeout; + qca->tx_idle_timer.data = (u_long)hu; + qca->tx_idle_delay = IBS_TX_IDLE_TIMEOUT_MS; + + BT_DBG("HCI_UART_QCA open, tx_idle_delay=%u, wake_retrans=%u", + qca->tx_idle_delay, qca->wake_retrans); + + return 0; +} + +static void qca_debugfs_init(struct hci_dev *hdev) +{ + struct hci_uart *hu = hci_get_drvdata(hdev); + struct qca_data *qca = hu->priv; + struct dentry *ibs_dir; + umode_t mode; + + if (!hdev->debugfs) + return; + + ibs_dir = debugfs_create_dir("ibs", hdev->debugfs); + + /* read only */ + mode = S_IRUGO; + debugfs_create_u8("tx_ibs_state", mode, ibs_dir, &qca->tx_ibs_state); + debugfs_create_u8("rx_ibs_state", mode, ibs_dir, &qca->rx_ibs_state); + debugfs_create_u64("ibs_sent_sleeps", mode, ibs_dir, + &qca->ibs_sent_slps); + debugfs_create_u64("ibs_sent_wakes", mode, ibs_dir, + &qca->ibs_sent_wakes); + debugfs_create_u64("ibs_sent_wake_acks", mode, ibs_dir, + &qca->ibs_sent_wacks); + debugfs_create_u64("ibs_recv_sleeps", mode, ibs_dir, + &qca->ibs_recv_slps); + debugfs_create_u64("ibs_recv_wakes", mode, ibs_dir, + &qca->ibs_recv_wakes); + debugfs_create_u64("ibs_recv_wake_acks", mode, ibs_dir, + &qca->ibs_recv_wacks); + debugfs_create_bool("tx_vote", mode, ibs_dir, &qca->tx_vote); + debugfs_create_u64("tx_votes_on", mode, ibs_dir, &qca->tx_votes_on); + debugfs_create_u64("tx_votes_off", mode, ibs_dir, &qca->tx_votes_off); + debugfs_create_bool("rx_vote", mode, ibs_dir, &qca->rx_vote); + debugfs_create_u64("rx_votes_on", mode, ibs_dir, &qca->rx_votes_on); + debugfs_create_u64("rx_votes_off", mode, ibs_dir, &qca->rx_votes_off); + debugfs_create_u64("votes_on", mode, ibs_dir, &qca->votes_on); + debugfs_create_u64("votes_off", mode, ibs_dir, &qca->votes_off); + debugfs_create_u32("vote_on_ms", mode, ibs_dir, &qca->vote_on_ms); + debugfs_create_u32("vote_off_ms", mode, ibs_dir, &qca->vote_off_ms); + + /* read/write */ + mode = S_IRUGO | S_IWUSR; + debugfs_create_u32("wake_retrans", mode, ibs_dir, &qca->wake_retrans); + debugfs_create_u32("tx_idle_delay", mode, ibs_dir, + &qca->tx_idle_delay); +} + +/* Flush protocol data */ +static int qca_flush(struct hci_uart *hu) +{ + struct qca_data *qca = hu->priv; + + BT_DBG("hu %p qca flush", hu); + + skb_queue_purge(&qca->tx_wait_q); + skb_queue_purge(&qca->txq); + + return 0; +} + +/* Close protocol */ +static int qca_close(struct hci_uart *hu) +{ + struct qca_data *qca = hu->priv; + + BT_DBG("hu %p qca close", hu); + + serial_clock_vote(HCI_IBS_VOTE_STATS_UPDATE, hu); + + skb_queue_purge(&qca->tx_wait_q); + skb_queue_purge(&qca->txq); + del_timer(&qca->tx_idle_timer); + del_timer(&qca->wake_retrans_timer); + destroy_workqueue(qca->workqueue); + qca->hu = NULL; + + kfree_skb(qca->rx_skb); + + hu->priv = NULL; + + kfree(qca); + + return 0; +} + +/* Called upon a wake-up-indication from the device. + */ +static void device_want_to_wakeup(struct hci_uart *hu) +{ + unsigned long flags; + struct qca_data *qca = hu->priv; + + BT_DBG("hu %p want to wake up", hu); + + spin_lock_irqsave(&qca->hci_ibs_lock, flags); + + qca->ibs_recv_wakes++; + + switch (qca->rx_ibs_state) { + case HCI_IBS_RX_ASLEEP: + /* Make sure clock is on - we may have turned clock off since + * receiving the wake up indicator awake rx clock. + */ + queue_work(qca->workqueue, &qca->ws_awake_rx); + spin_unlock_irqrestore(&qca->hci_ibs_lock, flags); + return; + + case HCI_IBS_RX_AWAKE: + /* Always acknowledge device wake up, + * sending IBS message doesn't count as TX ON. + */ + if (send_hci_ibs_cmd(HCI_IBS_WAKE_ACK, hu) < 0) { + BT_ERR("Failed to acknowledge device wake up"); + break; + } + qca->ibs_sent_wacks++; + break; + + default: + /* Any other state is illegal */ + BT_ERR("Received HCI_IBS_WAKE_IND in rx state %d", + qca->rx_ibs_state); + break; + } + + spin_unlock_irqrestore(&qca->hci_ibs_lock, flags); + + /* Actually send the packets */ + hci_uart_tx_wakeup(hu); +} + +/* Called upon a sleep-indication from the device. + */ +static void device_want_to_sleep(struct hci_uart *hu) +{ + unsigned long flags; + struct qca_data *qca = hu->priv; + + BT_DBG("hu %p want to sleep", hu); + + spin_lock_irqsave(&qca->hci_ibs_lock, flags); + + qca->ibs_recv_slps++; + + switch (qca->rx_ibs_state) { + case HCI_IBS_RX_AWAKE: + /* Update state */ + qca->rx_ibs_state = HCI_IBS_RX_ASLEEP; + /* Vote off rx clock under workqueue */ + queue_work(qca->workqueue, &qca->ws_rx_vote_off); + break; + + case HCI_IBS_RX_ASLEEP: + /* Fall through */ + + default: + /* Any other state is illegal */ + BT_ERR("Received HCI_IBS_SLEEP_IND in rx state %d", + qca->rx_ibs_state); + break; + } + + spin_unlock_irqrestore(&qca->hci_ibs_lock, flags); +} + +/* Called upon wake-up-acknowledgement from the device + */ +static void device_woke_up(struct hci_uart *hu) +{ + unsigned long flags, idle_delay; + struct qca_data *qca = hu->priv; + struct sk_buff *skb = NULL; + + BT_DBG("hu %p woke up", hu); + + spin_lock_irqsave(&qca->hci_ibs_lock, flags); + + qca->ibs_recv_wacks++; + + switch (qca->tx_ibs_state) { + case HCI_IBS_TX_AWAKE: + /* Expect one if we send 2 WAKEs */ + BT_DBG("Received HCI_IBS_WAKE_ACK in tx state %d", + qca->tx_ibs_state); + break; + + case HCI_IBS_TX_WAKING: + /* Send pending packets */ + while ((skb = skb_dequeue(&qca->tx_wait_q))) + skb_queue_tail(&qca->txq, skb); + + /* Switch timers and change state to HCI_IBS_TX_AWAKE */ + del_timer(&qca->wake_retrans_timer); + idle_delay = msecs_to_jiffies(qca->tx_idle_delay); + mod_timer(&qca->tx_idle_timer, jiffies + idle_delay); + qca->tx_ibs_state = HCI_IBS_TX_AWAKE; + break; + + case HCI_IBS_TX_ASLEEP: + /* Fall through */ + + default: + BT_ERR("Received HCI_IBS_WAKE_ACK in tx state %d", + qca->tx_ibs_state); + break; + } + + spin_unlock_irqrestore(&qca->hci_ibs_lock, flags); + + /* Actually send the packets */ + hci_uart_tx_wakeup(hu); +} + +/* Enqueue frame for transmittion (padding, crc, etc) may be called from + * two simultaneous tasklets. + */ +static int qca_enqueue(struct hci_uart *hu, struct sk_buff *skb) +{ + unsigned long flags = 0, idle_delay; + struct qca_data *qca = hu->priv; + + BT_DBG("hu %p qca enq skb %p tx_ibs_state %d", hu, skb, + qca->tx_ibs_state); + + /* Prepend skb with frame type */ + memcpy(skb_push(skb, 1), &bt_cb(skb)->pkt_type, 1); + + /* Don't go to sleep in middle of patch download or + * Out-Of-Band(GPIOs control) sleep is selected. + */ + if (!test_bit(STATE_IN_BAND_SLEEP_ENABLED, &qca->flags)) { + skb_queue_tail(&qca->txq, skb); + return 0; + } + + spin_lock_irqsave(&qca->hci_ibs_lock, flags); + + /* Act according to current state */ + switch (qca->tx_ibs_state) { + case HCI_IBS_TX_AWAKE: + BT_DBG("Device awake, sending normally"); + skb_queue_tail(&qca->txq, skb); + idle_delay = msecs_to_jiffies(qca->tx_idle_delay); + mod_timer(&qca->tx_idle_timer, jiffies + idle_delay); + break; + + case HCI_IBS_TX_ASLEEP: + BT_DBG("Device asleep, waking up and queueing packet"); + /* Save packet for later */ + skb_queue_tail(&qca->tx_wait_q, skb); + + qca->tx_ibs_state = HCI_IBS_TX_WAKING; + /* Schedule a work queue to wake up device */ + queue_work(qca->workqueue, &qca->ws_awake_device); + break; + + case HCI_IBS_TX_WAKING: + BT_DBG("Device waking up, queueing packet"); + /* Transient state; just keep packet for later */ + skb_queue_tail(&qca->tx_wait_q, skb); + break; + + default: + BT_ERR("Illegal tx state: %d (losing packet)", + qca->tx_ibs_state); + kfree_skb(skb); + break; + } + + spin_unlock_irqrestore(&qca->hci_ibs_lock, flags); + + return 0; +} + +static int qca_ibs_sleep_ind(struct hci_dev *hdev, struct sk_buff *skb) +{ + struct hci_uart *hu = hci_get_drvdata(hdev); + + BT_DBG("hu %p recv hci ibs cmd 0x%x", hu, HCI_IBS_SLEEP_IND); + + device_want_to_sleep(hu); + + kfree_skb(skb); + return 0; +} + +static int qca_ibs_wake_ind(struct hci_dev *hdev, struct sk_buff *skb) +{ + struct hci_uart *hu = hci_get_drvdata(hdev); + + BT_DBG("hu %p recv hci ibs cmd 0x%x", hu, HCI_IBS_WAKE_IND); + + device_want_to_wakeup(hu); + + kfree_skb(skb); + return 0; +} + +static int qca_ibs_wake_ack(struct hci_dev *hdev, struct sk_buff *skb) +{ + struct hci_uart *hu = hci_get_drvdata(hdev); + + BT_DBG("hu %p recv hci ibs cmd 0x%x", hu, HCI_IBS_WAKE_ACK); + + device_woke_up(hu); + + kfree_skb(skb); + return 0; +} + +#define QCA_IBS_SLEEP_IND_EVENT \ + .type = HCI_IBS_SLEEP_IND, \ + .hlen = 0, \ + .loff = 0, \ + .lsize = 0, \ + .maxlen = HCI_MAX_IBS_SIZE + +#define QCA_IBS_WAKE_IND_EVENT \ + .type = HCI_IBS_WAKE_IND, \ + .hlen = 0, \ + .loff = 0, \ + .lsize = 0, \ + .maxlen = HCI_MAX_IBS_SIZE + +#define QCA_IBS_WAKE_ACK_EVENT \ + .type = HCI_IBS_WAKE_ACK, \ + .hlen = 0, \ + .loff = 0, \ + .lsize = 0, \ + .maxlen = HCI_MAX_IBS_SIZE + +static const struct h4_recv_pkt qca_recv_pkts[] = { + { H4_RECV_ACL, .recv = hci_recv_frame }, + { H4_RECV_SCO, .recv = hci_recv_frame }, + { H4_RECV_EVENT, .recv = hci_recv_frame }, + { QCA_IBS_WAKE_IND_EVENT, .recv = qca_ibs_wake_ind }, + { QCA_IBS_WAKE_ACK_EVENT, .recv = qca_ibs_wake_ack }, + { QCA_IBS_SLEEP_IND_EVENT, .recv = qca_ibs_sleep_ind }, +}; + +static int qca_recv(struct hci_uart *hu, const void *data, int count) +{ + struct qca_data *qca = hu->priv; + + if (!test_bit(HCI_UART_REGISTERED, &hu->flags)) + return -EUNATCH; + + qca->rx_skb = h4_recv_buf(hu->hdev, qca->rx_skb, data, count, + qca_recv_pkts, ARRAY_SIZE(qca_recv_pkts)); + if (IS_ERR(qca->rx_skb)) { + int err = PTR_ERR(qca->rx_skb); + BT_ERR("%s: Frame reassembly failed (%d)", hu->hdev->name, err); + qca->rx_skb = NULL; + return err; + } + + return count; +} + +static struct sk_buff *qca_dequeue(struct hci_uart *hu) +{ + struct qca_data *qca = hu->priv; + + return skb_dequeue(&qca->txq); +} + +static uint8_t qca_get_baudrate_value(int speed) +{ + switch(speed) { + case 9600: + return QCA_BAUDRATE_9600; + case 19200: + return QCA_BAUDRATE_19200; + case 38400: + return QCA_BAUDRATE_38400; + case 57600: + return QCA_BAUDRATE_57600; + case 115200: + return QCA_BAUDRATE_115200; + case 230400: + return QCA_BAUDRATE_230400; + case 460800: + return QCA_BAUDRATE_460800; + case 500000: + return QCA_BAUDRATE_500000; + case 921600: + return QCA_BAUDRATE_921600; + case 1000000: + return QCA_BAUDRATE_1000000; + case 2000000: + return QCA_BAUDRATE_2000000; + case 3000000: + return QCA_BAUDRATE_3000000; + case 3500000: + return QCA_BAUDRATE_3500000; + default: + return QCA_BAUDRATE_115200; + } +} + +static int qca_set_baudrate(struct hci_dev *hdev, uint8_t baudrate) +{ + struct hci_uart *hu = hci_get_drvdata(hdev); + struct qca_data *qca = hu->priv; + struct sk_buff *skb; + u8 cmd[] = { 0x01, 0x48, 0xFC, 0x01, 0x00 }; + + if (baudrate > QCA_BAUDRATE_3000000) + return -EINVAL; + + cmd[4] = baudrate; + + skb = bt_skb_alloc(sizeof(cmd), GFP_ATOMIC); + if (!skb) { + BT_ERR("Failed to allocate memory for baudrate packet"); + return -ENOMEM; + } + + /* Assign commands to change baudrate and packet type. */ + memcpy(skb_put(skb, sizeof(cmd)), cmd, sizeof(cmd)); + bt_cb(skb)->pkt_type = HCI_COMMAND_PKT; + + skb_queue_tail(&qca->txq, skb); + hci_uart_tx_wakeup(hu); + + /* wait 300ms to change new baudrate on controller side + * controller will come back after they receive this HCI command + * then host can communicate with new baudrate to controller + */ + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_timeout(msecs_to_jiffies(BAUDRATE_SETTLE_TIMEOUT_MS)); + set_current_state(TASK_INTERRUPTIBLE); + + return 0; +} + +static int qca_setup(struct hci_uart *hu) +{ + struct hci_dev *hdev = hu->hdev; + struct qca_data *qca = hu->priv; + unsigned int speed, qca_baudrate = QCA_BAUDRATE_115200; + int ret; + + BT_INFO("%s: ROME setup", hdev->name); + + /* Patch downloading has to be done without IBS mode */ + clear_bit(STATE_IN_BAND_SLEEP_ENABLED, &qca->flags); + + /* Setup initial baudrate */ + speed = 0; + if (hu->init_speed) + speed = hu->init_speed; + else if (hu->proto->init_speed) + speed = hu->proto->init_speed; + + if (speed) + hci_uart_set_baudrate(hu, speed); + + /* Setup user speed if needed */ + speed = 0; + if (hu->oper_speed) + speed = hu->oper_speed; + else if (hu->proto->oper_speed) + speed = hu->proto->oper_speed; + + if (speed) { + qca_baudrate = qca_get_baudrate_value(speed); + + BT_INFO("%s: Set UART speed to %d", hdev->name, speed); + ret = qca_set_baudrate(hdev, qca_baudrate); + if (ret) { + BT_ERR("%s: Failed to change the baud rate (%d)", + hdev->name, ret); + return ret; + } + hci_uart_set_baudrate(hu, speed); + } + + /* Setup patch / NVM configurations */ + ret = qca_uart_setup_rome(hdev, qca_baudrate); + if (!ret) { + set_bit(STATE_IN_BAND_SLEEP_ENABLED, &qca->flags); + qca_debugfs_init(hdev); + } + + /* Setup bdaddr */ + hu->hdev->set_bdaddr = qca_set_bdaddr_rome; + + return ret; +} + +static struct hci_uart_proto qca_proto = { + .id = HCI_UART_QCA, + .name = "QCA", + .init_speed = 115200, + .oper_speed = 3000000, + .open = qca_open, + .close = qca_close, + .flush = qca_flush, + .setup = qca_setup, + .recv = qca_recv, + .enqueue = qca_enqueue, + .dequeue = qca_dequeue, +}; + +int __init qca_init(void) +{ + return hci_uart_register_proto(&qca_proto); +} + +int __exit qca_deinit(void) +{ + return hci_uart_unregister_proto(&qca_proto); +} diff --git a/drivers/bluetooth/hci_uart.h b/drivers/bluetooth/hci_uart.h index ce9c670956f5..495b9ef52bb0 100644 --- a/drivers/bluetooth/hci_uart.h +++ b/drivers/bluetooth/hci_uart.h @@ -35,7 +35,7 @@ #define HCIUARTGETFLAGS _IOR('U', 204, int) /* UART protocols */ -#define HCI_UART_MAX_PROTO 8 +#define HCI_UART_MAX_PROTO 9 #define HCI_UART_H4 0 #define HCI_UART_BCSP 1 @@ -45,6 +45,7 @@ #define HCI_UART_ATH3K 5 #define HCI_UART_INTEL 6 #define HCI_UART_BCM 7 +#define HCI_UART_QCA 8 #define HCI_UART_RAW_DEVICE 0 #define HCI_UART_RESET_ON_INIT 1 @@ -167,7 +168,17 @@ int h5_init(void); int h5_deinit(void); #endif +#ifdef CONFIG_BT_HCIUART_INTEL +int intel_init(void); +int intel_deinit(void); +#endif + #ifdef CONFIG_BT_HCIUART_BCM int bcm_init(void); int bcm_deinit(void); #endif + +#ifdef CONFIG_BT_HCIUART_QCA +int qca_init(void); +int qca_deinit(void); +#endif diff --git a/drivers/bus/mips_cdmm.c b/drivers/bus/mips_cdmm.c index ab3bde16ecb4..1c543effe062 100644 --- a/drivers/bus/mips_cdmm.c +++ b/drivers/bus/mips_cdmm.c @@ -331,6 +331,18 @@ static phys_addr_t mips_cdmm_cur_base(void) << MIPS_CDMMBASE_ADDR_START; } +/** + * mips_cdmm_phys_base() - Choose a physical base address for CDMM region. + * + * Picking a suitable physical address at which to map the CDMM region is + * platform specific, so this weak function can be overridden by platform + * code to pick a suitable value if none is configured by the bootloader. + */ +phys_addr_t __weak mips_cdmm_phys_base(void) +{ + return 0; +} + /** * mips_cdmm_setup() - Ensure the CDMM bus is initialised and usable. * @bus: Pointer to bus information for current CPU. @@ -368,7 +380,7 @@ static int mips_cdmm_setup(struct mips_cdmm_bus *bus) if (!bus->phys) bus->phys = mips_cdmm_cur_base(); /* Otherwise, ask platform code for suggestions */ - if (!bus->phys && mips_cdmm_phys_base) + if (!bus->phys) bus->phys = mips_cdmm_phys_base(); /* Otherwise, copy what other CPUs have done */ if (!bus->phys) diff --git a/drivers/bus/vexpress-config.c b/drivers/bus/vexpress-config.c index a64763b6b5fd..6575c0fe6a4e 100644 --- a/drivers/bus/vexpress-config.c +++ b/drivers/bus/vexpress-config.c @@ -107,7 +107,7 @@ struct regmap *devm_regmap_init_vexpress_config(struct device *dev) if (!res) return ERR_PTR(-ENOMEM); - regmap = bridge->ops->regmap_init(dev, bridge->context); + regmap = (bridge->ops->regmap_init)(dev, bridge->context); if (IS_ERR(regmap)) { devres_free(res); return regmap; diff --git a/drivers/char/agp/intel-gtt.c b/drivers/char/agp/intel-gtt.c index c6dea3f6917b..1341a94cc779 100644 --- a/drivers/char/agp/intel-gtt.c +++ b/drivers/char/agp/intel-gtt.c @@ -1408,8 +1408,8 @@ int intel_gmch_probe(struct pci_dev *bridge_pdev, struct pci_dev *gpu_pdev, } EXPORT_SYMBOL(intel_gmch_probe); -void intel_gtt_get(size_t *gtt_total, size_t *stolen_size, - phys_addr_t *mappable_base, unsigned long *mappable_end) +void intel_gtt_get(u64 *gtt_total, size_t *stolen_size, + phys_addr_t *mappable_base, u64 *mappable_end) { *gtt_total = intel_private.gtt_total_entries << PAGE_SHIFT; *stolen_size = intel_private.stolen_size; diff --git a/drivers/char/ipmi/ipmi_bt_sm.c b/drivers/char/ipmi/ipmi_bt_sm.c index 61e71616689b..feafdab734ae 100644 --- a/drivers/char/ipmi/ipmi_bt_sm.c +++ b/drivers/char/ipmi/ipmi_bt_sm.c @@ -694,7 +694,7 @@ static int bt_size(void) return sizeof(struct si_sm_data); } -struct si_sm_handlers bt_smi_handlers = { +const struct si_sm_handlers bt_smi_handlers = { .init_data = bt_init_data, .start_transaction = bt_start_transaction, .get_result = bt_get_result, diff --git a/drivers/char/ipmi/ipmi_kcs_sm.c b/drivers/char/ipmi/ipmi_kcs_sm.c index 8c25f596808a..1da61af7f576 100644 --- a/drivers/char/ipmi/ipmi_kcs_sm.c +++ b/drivers/char/ipmi/ipmi_kcs_sm.c @@ -540,7 +540,7 @@ static void kcs_cleanup(struct si_sm_data *kcs) { } -struct si_sm_handlers kcs_smi_handlers = { +const struct si_sm_handlers kcs_smi_handlers = { .init_data = init_kcs_data, .start_transaction = start_kcs_transaction, .get_result = get_kcs_result, diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c index bf75f6361773..e3536da05c88 100644 --- a/drivers/char/ipmi/ipmi_msghandler.c +++ b/drivers/char/ipmi/ipmi_msghandler.c @@ -342,7 +342,7 @@ struct ipmi_smi { * an umpreemptible region to use this. You must fetch the * value into a local variable and make sure it is not NULL. */ - struct ipmi_smi_handlers *handlers; + const struct ipmi_smi_handlers *handlers; void *send_info; #ifdef CONFIG_PROC_FS @@ -744,7 +744,13 @@ static void deliver_response(struct ipmi_recv_msg *msg) ipmi_inc_stat(intf, unhandled_local_responses); } ipmi_free_recv_msg(msg); - } else { + } else if (!oops_in_progress) { + /* + * If we are running in the panic context, calling the + * receive handler doesn't much meaning and has a deadlock + * risk. At this moment, simply skip it in that case. + */ + ipmi_user_t user = msg->user; user->handler->ipmi_recv_hndl(msg, user->handler_data); } @@ -1015,7 +1021,7 @@ int ipmi_get_smi_info(int if_num, struct ipmi_smi_info *data) { int rv = 0; ipmi_smi_t intf; - struct ipmi_smi_handlers *handlers; + const struct ipmi_smi_handlers *handlers; mutex_lock(&ipmi_interfaces_mutex); list_for_each_entry_rcu(intf, &ipmi_interfaces, link) { @@ -1501,7 +1507,7 @@ static struct ipmi_smi_msg *smi_add_send_msg(ipmi_smi_t intf, } -static void smi_send(ipmi_smi_t intf, struct ipmi_smi_handlers *handlers, +static void smi_send(ipmi_smi_t intf, const struct ipmi_smi_handlers *handlers, struct ipmi_smi_msg *smi_msg, int priority) { int run_to_completion = intf->run_to_completion; @@ -2747,7 +2753,7 @@ void ipmi_poll_interface(ipmi_user_t user) } EXPORT_SYMBOL(ipmi_poll_interface); -int ipmi_register_smi(struct ipmi_smi_handlers *handlers, +int ipmi_register_smi(const struct ipmi_smi_handlers *handlers, void *send_info, struct ipmi_device_id *device_id, struct device *si_dev, @@ -3959,6 +3965,10 @@ void ipmi_smi_msg_received(ipmi_smi_t intf, if (!run_to_completion) spin_lock_irqsave(&intf->xmit_msgs_lock, flags); + /* + * We can get an asynchronous event or receive message in addition + * to commands we send. + */ if (msg == intf->curr_msg) intf->curr_msg = NULL; if (!run_to_completion) @@ -4015,7 +4025,7 @@ static void check_msg_timeout(ipmi_smi_t intf, struct seq_table *ent, unsigned int *waiting_msgs) { struct ipmi_recv_msg *msg; - struct ipmi_smi_handlers *handlers; + const struct ipmi_smi_handlers *handlers; if (intf->in_shutdown) return; @@ -4082,7 +4092,7 @@ static void check_msg_timeout(ipmi_smi_t intf, struct seq_table *ent, ipmi_inc_stat(intf, retransmitted_ipmb_commands); - smi_send(intf, intf->handlers, smi_msg, 0); + smi_send(intf, handlers, smi_msg, 0); } else ipmi_free_smi_msg(smi_msg); @@ -4291,6 +4301,9 @@ static void ipmi_panic_request_and_wait(ipmi_smi_t intf, 0, 1); /* Don't retry, and don't wait. */ if (rv) atomic_sub(2, &panic_done_count); + else if (intf->handlers->flush_messages) + intf->handlers->flush_messages(intf->send_info); + while (atomic_read(&panic_done_count) != 0) ipmi_poll(intf); } @@ -4364,9 +4377,7 @@ static void send_panic_events(char *str) /* Interface is not ready. */ continue; - intf->run_to_completion = 1; /* Send the event announcing the panic. */ - intf->handlers->set_run_to_completion(intf->send_info, 1); ipmi_panic_request_and_wait(intf, &addr, &msg); } @@ -4506,6 +4517,23 @@ static int panic_event(struct notifier_block *this, /* Interface is not ready. */ continue; + /* + * If we were interrupted while locking xmit_msgs_lock or + * waiting_rcv_msgs_lock, the corresponding list may be + * corrupted. In this case, drop items on the list for + * the safety. + */ + if (!spin_trylock(&intf->xmit_msgs_lock)) { + INIT_LIST_HEAD(&intf->xmit_msgs); + INIT_LIST_HEAD(&intf->hp_xmit_msgs); + } else + spin_unlock(&intf->xmit_msgs_lock); + + if (!spin_trylock(&intf->waiting_rcv_msgs_lock)) + INIT_LIST_HEAD(&intf->waiting_rcv_msgs); + else + spin_unlock(&intf->waiting_rcv_msgs_lock); + intf->run_to_completion = 1; intf->handlers->set_run_to_completion(intf->send_info, 1); } diff --git a/drivers/char/ipmi/ipmi_powernv.c b/drivers/char/ipmi/ipmi_powernv.c index 9b409c0f14f7..6e658aa114f1 100644 --- a/drivers/char/ipmi/ipmi_powernv.c +++ b/drivers/char/ipmi/ipmi_powernv.c @@ -143,8 +143,15 @@ static int ipmi_powernv_recv(struct ipmi_smi_powernv *smi) pr_devel("%s: -> %d (size %lld)\n", __func__, rc, rc == 0 ? size : 0); if (rc) { + /* If came via the poll, and response was not yet ready */ + if (rc == OPAL_EMPTY) { + spin_unlock_irqrestore(&smi->msg_lock, flags); + return 0; + } + + smi->cur_msg = NULL; spin_unlock_irqrestore(&smi->msg_lock, flags); - ipmi_free_smi_msg(msg); + send_error_reply(smi, msg, IPMI_ERR_UNSPECIFIED); return 0; } @@ -300,7 +307,6 @@ static const struct of_device_id ipmi_powernv_match[] = { static struct platform_driver powernv_ipmi_driver = { .driver = { .name = "ipmi-powernv", - .owner = THIS_MODULE, .of_match_table = ipmi_powernv_match, }, .probe = ipmi_powernv_probe, diff --git a/drivers/char/ipmi/ipmi_si_intf.c b/drivers/char/ipmi/ipmi_si_intf.c index 8a45e92ff60c..654f6f36a071 100644 --- a/drivers/char/ipmi/ipmi_si_intf.c +++ b/drivers/char/ipmi/ipmi_si_intf.c @@ -64,7 +64,6 @@ #include #include #include -#include #include #include #include @@ -164,7 +163,7 @@ struct smi_info { int intf_num; ipmi_smi_t intf; struct si_sm_data *si_sm; - struct si_sm_handlers *handlers; + const struct si_sm_handlers *handlers; enum si_type si_type; spinlock_t si_lock; struct ipmi_smi_msg *waiting_msg; @@ -263,9 +262,21 @@ struct smi_info { bool supports_event_msg_buff; /* - * Can we clear the global enables receive irq bit? + * Can we disable interrupts the global enables receive irq + * bit? There are currently two forms of brokenness, some + * systems cannot disable the bit (which is technically within + * the spec but a bad idea) and some systems have the bit + * forced to zero even though interrupts work (which is + * clearly outside the spec). The next bool tells which form + * of brokenness is present. */ - bool cannot_clear_recv_irq_bit; + bool cannot_disable_irq; + + /* + * Some systems are broken and cannot set the irq enable + * bit, even if they support interrupts. + */ + bool irq_enable_broken; /* * Did we get an attention that we did not handle? @@ -309,9 +320,6 @@ static int num_force_kipmid; #ifdef CONFIG_PCI static bool pci_registered; #endif -#ifdef CONFIG_ACPI -static bool pnp_registered; -#endif #ifdef CONFIG_PARISC static bool parisc_registered; #endif @@ -558,13 +566,14 @@ static u8 current_global_enables(struct smi_info *smi_info, u8 base, if (smi_info->supports_event_msg_buff) enables |= IPMI_BMC_EVT_MSG_BUFF; - if ((smi_info->irq && !smi_info->interrupt_disabled) || - smi_info->cannot_clear_recv_irq_bit) + if (((smi_info->irq && !smi_info->interrupt_disabled) || + smi_info->cannot_disable_irq) && + !smi_info->irq_enable_broken) enables |= IPMI_BMC_RCV_MSG_INTR; if (smi_info->supports_event_msg_buff && - smi_info->irq && !smi_info->interrupt_disabled) - + smi_info->irq && !smi_info->interrupt_disabled && + !smi_info->irq_enable_broken) enables |= IPMI_BMC_EVT_MSG_INTR; *irq_on = enables & (IPMI_BMC_EVT_MSG_INTR | IPMI_BMC_RCV_MSG_INTR); @@ -928,33 +937,36 @@ static void check_start_timer_thread(struct smi_info *smi_info) } } +static void flush_messages(void *send_info) +{ + struct smi_info *smi_info = send_info; + enum si_sm_result result; + + /* + * Currently, this function is called only in run-to-completion + * mode. This means we are single-threaded, no need for locks. + */ + result = smi_event_handler(smi_info, 0); + while (result != SI_SM_IDLE) { + udelay(SI_SHORT_TIMEOUT_USEC); + result = smi_event_handler(smi_info, SI_SHORT_TIMEOUT_USEC); + } +} + static void sender(void *send_info, struct ipmi_smi_msg *msg) { struct smi_info *smi_info = send_info; - enum si_sm_result result; unsigned long flags; debug_timestamp("Enqueue"); if (smi_info->run_to_completion) { /* - * If we are running to completion, start it and run - * transactions until everything is clear. + * If we are running to completion, start it. Upper + * layer will call flush_messages to clear it out. */ smi_info->waiting_msg = msg; - - /* - * Run to completion means we are single-threaded, no - * need for locks. - */ - - result = smi_event_handler(smi_info, 0); - while (result != SI_SM_IDLE) { - udelay(SI_SHORT_TIMEOUT_USEC); - result = smi_event_handler(smi_info, - SI_SHORT_TIMEOUT_USEC); - } return; } @@ -975,17 +987,10 @@ static void sender(void *send_info, static void set_run_to_completion(void *send_info, bool i_run_to_completion) { struct smi_info *smi_info = send_info; - enum si_sm_result result; smi_info->run_to_completion = i_run_to_completion; - if (i_run_to_completion) { - result = smi_event_handler(smi_info, 0); - while (result != SI_SM_IDLE) { - udelay(SI_SHORT_TIMEOUT_USEC); - result = smi_event_handler(smi_info, - SI_SHORT_TIMEOUT_USEC); - } - } + if (i_run_to_completion) + flush_messages(smi_info); } /* @@ -1258,7 +1263,7 @@ static void set_maintenance_mode(void *send_info, bool enable) atomic_set(&smi_info->req_events, 0); } -static struct ipmi_smi_handlers handlers = { +static const struct ipmi_smi_handlers handlers = { .owner = THIS_MODULE, .start_processing = smi_start_processing, .get_smi_info = get_smi_info, @@ -1267,6 +1272,7 @@ static struct ipmi_smi_handlers handlers = { .set_need_watch = set_need_watch, .set_maintenance_mode = set_maintenance_mode, .set_run_to_completion = set_run_to_completion, + .flush_messages = flush_messages, .poll = poll, }; @@ -1283,14 +1289,14 @@ static int smi_num; /* Used to sequence the SMIs */ #define DEFAULT_REGSIZE 1 #ifdef CONFIG_ACPI -static bool si_tryacpi = 1; +static bool si_tryacpi = true; #endif #ifdef CONFIG_DMI -static bool si_trydmi = 1; +static bool si_trydmi = true; #endif -static bool si_tryplatform = 1; +static bool si_tryplatform = true; #ifdef CONFIG_PCI -static bool si_trypci = 1; +static bool si_trypci = true; #endif static bool si_trydefaults = IS_ENABLED(CONFIG_IPMI_SI_PROBE_DEFAULTS); static char *si_type[SI_MAX_PARMS]; @@ -1446,14 +1452,14 @@ static int std_irq_setup(struct smi_info *info) return rv; } -static unsigned char port_inb(struct si_sm_io *io, unsigned int offset) +static unsigned char port_inb(const struct si_sm_io *io, unsigned int offset) { unsigned int addr = io->addr_data; return inb(addr + (offset * io->regspacing)); } -static void port_outb(struct si_sm_io *io, unsigned int offset, +static void port_outb(const struct si_sm_io *io, unsigned int offset, unsigned char b) { unsigned int addr = io->addr_data; @@ -1461,14 +1467,14 @@ static void port_outb(struct si_sm_io *io, unsigned int offset, outb(b, addr + (offset * io->regspacing)); } -static unsigned char port_inw(struct si_sm_io *io, unsigned int offset) +static unsigned char port_inw(const struct si_sm_io *io, unsigned int offset) { unsigned int addr = io->addr_data; return (inw(addr + (offset * io->regspacing)) >> io->regshift) & 0xff; } -static void port_outw(struct si_sm_io *io, unsigned int offset, +static void port_outw(const struct si_sm_io *io, unsigned int offset, unsigned char b) { unsigned int addr = io->addr_data; @@ -1476,14 +1482,14 @@ static void port_outw(struct si_sm_io *io, unsigned int offset, outw(b << io->regshift, addr + (offset * io->regspacing)); } -static unsigned char port_inl(struct si_sm_io *io, unsigned int offset) +static unsigned char port_inl(const struct si_sm_io *io, unsigned int offset) { unsigned int addr = io->addr_data; return (inl(addr + (offset * io->regspacing)) >> io->regshift) & 0xff; } -static void port_outl(struct si_sm_io *io, unsigned int offset, +static void port_outl(const struct si_sm_io *io, unsigned int offset, unsigned char b) { unsigned int addr = io->addr_data; @@ -1556,49 +1562,52 @@ static int port_setup(struct smi_info *info) return 0; } -static unsigned char intf_mem_inb(struct si_sm_io *io, unsigned int offset) +static unsigned char intf_mem_inb(const struct si_sm_io *io, + unsigned int offset) { return readb((io->addr)+(offset * io->regspacing)); } -static void intf_mem_outb(struct si_sm_io *io, unsigned int offset, - unsigned char b) +static void intf_mem_outb(const struct si_sm_io *io, unsigned int offset, + unsigned char b) { writeb(b, (io->addr)+(offset * io->regspacing)); } -static unsigned char intf_mem_inw(struct si_sm_io *io, unsigned int offset) +static unsigned char intf_mem_inw(const struct si_sm_io *io, + unsigned int offset) { return (readw((io->addr)+(offset * io->regspacing)) >> io->regshift) & 0xff; } -static void intf_mem_outw(struct si_sm_io *io, unsigned int offset, - unsigned char b) +static void intf_mem_outw(const struct si_sm_io *io, unsigned int offset, + unsigned char b) { writeb(b << io->regshift, (io->addr)+(offset * io->regspacing)); } -static unsigned char intf_mem_inl(struct si_sm_io *io, unsigned int offset) +static unsigned char intf_mem_inl(const struct si_sm_io *io, + unsigned int offset) { return (readl((io->addr)+(offset * io->regspacing)) >> io->regshift) & 0xff; } -static void intf_mem_outl(struct si_sm_io *io, unsigned int offset, - unsigned char b) +static void intf_mem_outl(const struct si_sm_io *io, unsigned int offset, + unsigned char b) { writel(b << io->regshift, (io->addr)+(offset * io->regspacing)); } #ifdef readq -static unsigned char mem_inq(struct si_sm_io *io, unsigned int offset) +static unsigned char mem_inq(const struct si_sm_io *io, unsigned int offset) { return (readq((io->addr)+(offset * io->regspacing)) >> io->regshift) & 0xff; } -static void mem_outq(struct si_sm_io *io, unsigned int offset, +static void mem_outq(const struct si_sm_io *io, unsigned int offset, unsigned char b) { writeq(b << io->regshift, (io->addr)+(offset * io->regspacing)); @@ -2233,134 +2242,6 @@ static void spmi_find_bmc(void) try_init_spmi(spmi); } } - -static int ipmi_pnp_probe(struct pnp_dev *dev, - const struct pnp_device_id *dev_id) -{ - struct acpi_device *acpi_dev; - struct smi_info *info; - struct resource *res, *res_second; - acpi_handle handle; - acpi_status status; - unsigned long long tmp; - int rv = -EINVAL; - - acpi_dev = pnp_acpi_device(dev); - if (!acpi_dev) - return -ENODEV; - - info = smi_info_alloc(); - if (!info) - return -ENOMEM; - - info->addr_source = SI_ACPI; - printk(KERN_INFO PFX "probing via ACPI\n"); - - handle = acpi_dev->handle; - info->addr_info.acpi_info.acpi_handle = handle; - - /* _IFT tells us the interface type: KCS, BT, etc */ - status = acpi_evaluate_integer(handle, "_IFT", NULL, &tmp); - if (ACPI_FAILURE(status)) { - dev_err(&dev->dev, "Could not find ACPI IPMI interface type\n"); - goto err_free; - } - - switch (tmp) { - case 1: - info->si_type = SI_KCS; - break; - case 2: - info->si_type = SI_SMIC; - break; - case 3: - info->si_type = SI_BT; - break; - case 4: /* SSIF, just ignore */ - rv = -ENODEV; - goto err_free; - default: - dev_info(&dev->dev, "unknown IPMI type %lld\n", tmp); - goto err_free; - } - - res = pnp_get_resource(dev, IORESOURCE_IO, 0); - if (res) { - info->io_setup = port_setup; - info->io.addr_type = IPMI_IO_ADDR_SPACE; - } else { - res = pnp_get_resource(dev, IORESOURCE_MEM, 0); - if (res) { - info->io_setup = mem_setup; - info->io.addr_type = IPMI_MEM_ADDR_SPACE; - } - } - if (!res) { - dev_err(&dev->dev, "no I/O or memory address\n"); - goto err_free; - } - info->io.addr_data = res->start; - - info->io.regspacing = DEFAULT_REGSPACING; - res_second = pnp_get_resource(dev, - (info->io.addr_type == IPMI_IO_ADDR_SPACE) ? - IORESOURCE_IO : IORESOURCE_MEM, - 1); - if (res_second) { - if (res_second->start > info->io.addr_data) - info->io.regspacing = res_second->start - info->io.addr_data; - } - info->io.regsize = DEFAULT_REGSPACING; - info->io.regshift = 0; - - /* If _GPE exists, use it; otherwise use standard interrupts */ - status = acpi_evaluate_integer(handle, "_GPE", NULL, &tmp); - if (ACPI_SUCCESS(status)) { - info->irq = tmp; - info->irq_setup = acpi_gpe_irq_setup; - } else if (pnp_irq_valid(dev, 0)) { - info->irq = pnp_irq(dev, 0); - info->irq_setup = std_irq_setup; - } - - info->dev = &dev->dev; - pnp_set_drvdata(dev, info); - - dev_info(info->dev, "%pR regsize %d spacing %d irq %d\n", - res, info->io.regsize, info->io.regspacing, - info->irq); - - rv = add_smi(info); - if (rv) - kfree(info); - - return rv; - -err_free: - kfree(info); - return rv; -} - -static void ipmi_pnp_remove(struct pnp_dev *dev) -{ - struct smi_info *info = pnp_get_drvdata(dev); - - cleanup_one_si(info); -} - -static const struct pnp_device_id pnp_dev_table[] = { - {"IPI0001", 0}, - {"", 0}, -}; - -static struct pnp_driver ipmi_pnp_driver = { - .name = DEVICE_NAME, - .probe = ipmi_pnp_probe, - .remove = ipmi_pnp_remove, - .id_table = pnp_dev_table, -}; - -MODULE_DEVICE_TABLE(pnp, pnp_dev_table); #endif #ifdef CONFIG_DMI @@ -2654,7 +2535,7 @@ static void ipmi_pci_remove(struct pci_dev *pdev) pci_disable_device(pdev); } -static struct pci_device_id ipmi_pci_devices[] = { +static const struct pci_device_id ipmi_pci_devices[] = { { PCI_DEVICE(PCI_HP_VENDOR_ID, PCI_MMC_DEVICE_ID) }, { PCI_DEVICE_CLASS(PCI_ERMC_CLASSCODE, PCI_ERMC_CLASSCODE_MASK) }, { 0, } @@ -2669,10 +2550,19 @@ static struct pci_driver ipmi_pci_driver = { }; #endif /* CONFIG_PCI */ -static const struct of_device_id ipmi_match[]; -static int ipmi_probe(struct platform_device *dev) -{ #ifdef CONFIG_OF +static const struct of_device_id of_ipmi_match[] = { + { .type = "ipmi", .compatible = "ipmi-kcs", + .data = (void *)(unsigned long) SI_KCS }, + { .type = "ipmi", .compatible = "ipmi-smic", + .data = (void *)(unsigned long) SI_SMIC }, + { .type = "ipmi", .compatible = "ipmi-bt", + .data = (void *)(unsigned long) SI_BT }, + {}, +}; + +static int of_ipmi_probe(struct platform_device *dev) +{ const struct of_device_id *match; struct smi_info *info; struct resource resource; @@ -2683,9 +2573,9 @@ static int ipmi_probe(struct platform_device *dev) dev_info(&dev->dev, "probing via device tree\n"); - match = of_match_device(ipmi_match, &dev->dev); + match = of_match_device(of_ipmi_match, &dev->dev); if (!match) - return -EINVAL; + return -ENODEV; if (!of_device_is_available(np)) return -EINVAL; @@ -2754,33 +2644,160 @@ static int ipmi_probe(struct platform_device *dev) kfree(info); return ret; } -#endif return 0; } +MODULE_DEVICE_TABLE(of, of_ipmi_match); +#else +#define of_ipmi_match NULL +static int of_ipmi_probe(struct platform_device *dev) +{ + return -ENODEV; +} +#endif + +#ifdef CONFIG_ACPI +static int acpi_ipmi_probe(struct platform_device *dev) +{ + struct smi_info *info; + struct resource *res, *res_second; + acpi_handle handle; + acpi_status status; + unsigned long long tmp; + int rv = -EINVAL; + + handle = ACPI_HANDLE(&dev->dev); + if (!handle) + return -ENODEV; + + info = smi_info_alloc(); + if (!info) + return -ENOMEM; + + info->addr_source = SI_ACPI; + dev_info(&dev->dev, PFX "probing via ACPI\n"); + + info->addr_info.acpi_info.acpi_handle = handle; + + /* _IFT tells us the interface type: KCS, BT, etc */ + status = acpi_evaluate_integer(handle, "_IFT", NULL, &tmp); + if (ACPI_FAILURE(status)) { + dev_err(&dev->dev, "Could not find ACPI IPMI interface type\n"); + goto err_free; + } + + switch (tmp) { + case 1: + info->si_type = SI_KCS; + break; + case 2: + info->si_type = SI_SMIC; + break; + case 3: + info->si_type = SI_BT; + break; + case 4: /* SSIF, just ignore */ + rv = -ENODEV; + goto err_free; + default: + dev_info(&dev->dev, "unknown IPMI type %lld\n", tmp); + goto err_free; + } + + res = platform_get_resource(dev, IORESOURCE_IO, 0); + if (res) { + info->io_setup = port_setup; + info->io.addr_type = IPMI_IO_ADDR_SPACE; + } else { + res = platform_get_resource(dev, IORESOURCE_MEM, 0); + if (res) { + info->io_setup = mem_setup; + info->io.addr_type = IPMI_MEM_ADDR_SPACE; + } + } + if (!res) { + dev_err(&dev->dev, "no I/O or memory address\n"); + goto err_free; + } + info->io.addr_data = res->start; + + info->io.regspacing = DEFAULT_REGSPACING; + res_second = platform_get_resource(dev, + (info->io.addr_type == IPMI_IO_ADDR_SPACE) ? + IORESOURCE_IO : IORESOURCE_MEM, + 1); + if (res_second) { + if (res_second->start > info->io.addr_data) + info->io.regspacing = + res_second->start - info->io.addr_data; + } + info->io.regsize = DEFAULT_REGSPACING; + info->io.regshift = 0; + + /* If _GPE exists, use it; otherwise use standard interrupts */ + status = acpi_evaluate_integer(handle, "_GPE", NULL, &tmp); + if (ACPI_SUCCESS(status)) { + info->irq = tmp; + info->irq_setup = acpi_gpe_irq_setup; + } else { + int irq = platform_get_irq(dev, 0); + + if (irq > 0) { + info->irq = irq; + info->irq_setup = std_irq_setup; + } + } + + info->dev = &dev->dev; + platform_set_drvdata(dev, info); + + dev_info(info->dev, "%pR regsize %d spacing %d irq %d\n", + res, info->io.regsize, info->io.regspacing, + info->irq); + + rv = add_smi(info); + if (rv) + kfree(info); + + return rv; + +err_free: + kfree(info); + return rv; +} + +static const struct acpi_device_id acpi_ipmi_match[] = { + { "IPI0001", 0 }, + { }, +}; +MODULE_DEVICE_TABLE(acpi, acpi_ipmi_match); +#else +static int acpi_ipmi_probe(struct platform_device *dev) +{ + return -ENODEV; +} +#endif + +static int ipmi_probe(struct platform_device *dev) +{ + if (of_ipmi_probe(dev) == 0) + return 0; + + return acpi_ipmi_probe(dev); +} static int ipmi_remove(struct platform_device *dev) { -#ifdef CONFIG_OF - cleanup_one_si(dev_get_drvdata(&dev->dev)); -#endif + struct smi_info *info = dev_get_drvdata(&dev->dev); + + cleanup_one_si(info); return 0; } -static const struct of_device_id ipmi_match[] = -{ - { .type = "ipmi", .compatible = "ipmi-kcs", - .data = (void *)(unsigned long) SI_KCS }, - { .type = "ipmi", .compatible = "ipmi-smic", - .data = (void *)(unsigned long) SI_SMIC }, - { .type = "ipmi", .compatible = "ipmi-bt", - .data = (void *)(unsigned long) SI_BT }, - {}, -}; - static struct platform_driver ipmi_driver = { .driver = { .name = DEVICE_NAME, - .of_match_table = ipmi_match, + .of_match_table = of_ipmi_match, + .acpi_match_table = ACPI_PTR(acpi_ipmi_match), }, .probe = ipmi_probe, .remove = ipmi_remove, @@ -2905,12 +2922,7 @@ static int try_get_dev_id(struct smi_info *smi_info) return rv; } -/* - * Some BMCs do not support clearing the receive irq bit in the global - * enables (even if they don't support interrupts on the BMC). Check - * for this and handle it properly. - */ -static void check_clr_rcv_irq(struct smi_info *smi_info) +static int get_global_enables(struct smi_info *smi_info, u8 *enables) { unsigned char msg[3]; unsigned char *resp; @@ -2918,12 +2930,8 @@ static void check_clr_rcv_irq(struct smi_info *smi_info) int rv; resp = kmalloc(IPMI_MAX_MSG_LENGTH, GFP_KERNEL); - if (!resp) { - printk(KERN_WARNING PFX "Out of memory allocating response for" - " global enables command, cannot check recv irq bit" - " handling.\n"); - return; - } + if (!resp) + return -ENOMEM; msg[0] = IPMI_NETFN_APP_REQUEST << 2; msg[1] = IPMI_GET_BMC_GLOBAL_ENABLES_CMD; @@ -2931,9 +2939,9 @@ static void check_clr_rcv_irq(struct smi_info *smi_info) rv = wait_for_msg_done(smi_info); if (rv) { - printk(KERN_WARNING PFX "Error getting response from get" - " global enables command, cannot check recv irq bit" - " handling.\n"); + dev_warn(smi_info->dev, + "Error getting response from get global enables command: %d\n", + rv); goto out; } @@ -2944,27 +2952,44 @@ static void check_clr_rcv_irq(struct smi_info *smi_info) resp[0] != (IPMI_NETFN_APP_REQUEST | 1) << 2 || resp[1] != IPMI_GET_BMC_GLOBAL_ENABLES_CMD || resp[2] != 0) { - printk(KERN_WARNING PFX "Invalid return from get global" - " enables command, cannot check recv irq bit" - " handling.\n"); + dev_warn(smi_info->dev, + "Invalid return from get global enables command: %ld %x %x %x\n", + resp_len, resp[0], resp[1], resp[2]); rv = -EINVAL; goto out; + } else { + *enables = resp[3]; } - if ((resp[3] & IPMI_BMC_RCV_MSG_INTR) == 0) - /* Already clear, should work ok. */ - goto out; +out: + kfree(resp); + return rv; +} + +/* + * Returns 1 if it gets an error from the command. + */ +static int set_global_enables(struct smi_info *smi_info, u8 enables) +{ + unsigned char msg[3]; + unsigned char *resp; + unsigned long resp_len; + int rv; + + resp = kmalloc(IPMI_MAX_MSG_LENGTH, GFP_KERNEL); + if (!resp) + return -ENOMEM; msg[0] = IPMI_NETFN_APP_REQUEST << 2; msg[1] = IPMI_SET_BMC_GLOBAL_ENABLES_CMD; - msg[2] = resp[3] & ~IPMI_BMC_RCV_MSG_INTR; + msg[2] = enables; smi_info->handlers->start_transaction(smi_info->si_sm, msg, 3); rv = wait_for_msg_done(smi_info); if (rv) { - printk(KERN_WARNING PFX "Error getting response from set" - " global enables command, cannot check recv irq bit" - " handling.\n"); + dev_warn(smi_info->dev, + "Error getting response from set global enables command: %d\n", + rv); goto out; } @@ -2974,25 +2999,93 @@ static void check_clr_rcv_irq(struct smi_info *smi_info) if (resp_len < 3 || resp[0] != (IPMI_NETFN_APP_REQUEST | 1) << 2 || resp[1] != IPMI_SET_BMC_GLOBAL_ENABLES_CMD) { - printk(KERN_WARNING PFX "Invalid return from get global" - " enables command, cannot check recv irq bit" - " handling.\n"); + dev_warn(smi_info->dev, + "Invalid return from set global enables command: %ld %x %x\n", + resp_len, resp[0], resp[1]); rv = -EINVAL; goto out; } - if (resp[2] != 0) { + if (resp[2] != 0) + rv = 1; + +out: + kfree(resp); + return rv; +} + +/* + * Some BMCs do not support clearing the receive irq bit in the global + * enables (even if they don't support interrupts on the BMC). Check + * for this and handle it properly. + */ +static void check_clr_rcv_irq(struct smi_info *smi_info) +{ + u8 enables = 0; + int rv; + + rv = get_global_enables(smi_info, &enables); + if (!rv) { + if ((enables & IPMI_BMC_RCV_MSG_INTR) == 0) + /* Already clear, should work ok. */ + return; + + enables &= ~IPMI_BMC_RCV_MSG_INTR; + rv = set_global_enables(smi_info, enables); + } + + if (rv < 0) { + dev_err(smi_info->dev, + "Cannot check clearing the rcv irq: %d\n", rv); + return; + } + + if (rv) { /* * An error when setting the event buffer bit means * clearing the bit is not supported. */ - printk(KERN_WARNING PFX "The BMC does not support clearing" - " the recv irq bit, compensating, but the BMC needs to" - " be fixed.\n"); - smi_info->cannot_clear_recv_irq_bit = true; + dev_warn(smi_info->dev, + "The BMC does not support clearing the recv irq bit, compensating, but the BMC needs to be fixed.\n"); + smi_info->cannot_disable_irq = true; + } +} + +/* + * Some BMCs do not support setting the interrupt bits in the global + * enables even if they support interrupts. Clearly bad, but we can + * compensate. + */ +static void check_set_rcv_irq(struct smi_info *smi_info) +{ + u8 enables = 0; + int rv; + + if (!smi_info->irq) + return; + + rv = get_global_enables(smi_info, &enables); + if (!rv) { + enables |= IPMI_BMC_RCV_MSG_INTR; + rv = set_global_enables(smi_info, enables); + } + + if (rv < 0) { + dev_err(smi_info->dev, + "Cannot check setting the rcv irq: %d\n", rv); + return; + } + + if (rv) { + /* + * An error when setting the event buffer bit means + * setting the bit is not supported. + */ + dev_warn(smi_info->dev, + "The BMC does not support setting the recv irq bit, compensating, but the BMC needs to be fixed.\n"); + smi_info->cannot_disable_irq = true; + smi_info->irq_enable_broken = true; } - out: - kfree(resp); } static int try_enable_event_buffer(struct smi_info *smi_info) @@ -3313,6 +3406,12 @@ static void setup_xaction_handlers(struct smi_info *smi_info) setup_dell_poweredge_bt_xaction_handler(smi_info); } +static void check_for_broken_irqs(struct smi_info *smi_info) +{ + check_clr_rcv_irq(smi_info); + check_set_rcv_irq(smi_info); +} + static inline void wait_for_timer_and_thread(struct smi_info *smi_info) { if (smi_info->thread != NULL) @@ -3321,7 +3420,7 @@ static inline void wait_for_timer_and_thread(struct smi_info *smi_info) del_timer_sync(&smi_info->si_timer); } -static struct ipmi_default_vals +static const struct ipmi_default_vals { int type; int port; @@ -3490,10 +3589,9 @@ static int try_smi_init(struct smi_info *new_smi) goto out_err; } - check_clr_rcv_irq(new_smi); - setup_oem_data_handler(new_smi); setup_xaction_handlers(new_smi); + check_for_broken_irqs(new_smi); new_smi->waiting_msg = NULL; new_smi->curr_msg = NULL; @@ -3692,13 +3790,6 @@ static int init_ipmi_si(void) } #endif -#ifdef CONFIG_ACPI - if (si_tryacpi) { - pnp_register_driver(&ipmi_pnp_driver); - pnp_registered = true; - } -#endif - #ifdef CONFIG_DMI if (si_trydmi) dmi_find_bmc(); @@ -3850,10 +3941,6 @@ static void cleanup_ipmi_si(void) if (pci_registered) pci_unregister_driver(&ipmi_pci_driver); #endif -#ifdef CONFIG_ACPI - if (pnp_registered) - pnp_unregister_driver(&ipmi_pnp_driver); -#endif #ifdef CONFIG_PARISC if (parisc_registered) unregister_parisc_driver(&ipmi_parisc_driver); diff --git a/drivers/char/ipmi/ipmi_si_sm.h b/drivers/char/ipmi/ipmi_si_sm.h index df89f73475fb..a705027c0493 100644 --- a/drivers/char/ipmi/ipmi_si_sm.h +++ b/drivers/char/ipmi/ipmi_si_sm.h @@ -46,8 +46,8 @@ struct si_sm_data; * this interface. */ struct si_sm_io { - unsigned char (*inputb)(struct si_sm_io *io, unsigned int offset); - void (*outputb)(struct si_sm_io *io, + unsigned char (*inputb)(const struct si_sm_io *io, unsigned int offset); + void (*outputb)(const struct si_sm_io *io, unsigned int offset, unsigned char b); @@ -135,7 +135,7 @@ struct si_sm_handlers { }; /* Current state machines that we can use. */ -extern struct si_sm_handlers kcs_smi_handlers; -extern struct si_sm_handlers smic_smi_handlers; -extern struct si_sm_handlers bt_smi_handlers; +extern const struct si_sm_handlers kcs_smi_handlers; +extern const struct si_sm_handlers smic_smi_handlers; +extern const struct si_sm_handlers bt_smi_handlers; diff --git a/drivers/char/ipmi/ipmi_smic_sm.c b/drivers/char/ipmi/ipmi_smic_sm.c index c8e77afa8b96..8f7c73ff58f2 100644 --- a/drivers/char/ipmi/ipmi_smic_sm.c +++ b/drivers/char/ipmi/ipmi_smic_sm.c @@ -589,7 +589,7 @@ static int smic_size(void) return sizeof(struct si_sm_data); } -struct si_sm_handlers smic_smi_handlers = { +const struct si_sm_handlers smic_smi_handlers = { .init_data = init_smic_data, .start_transaction = start_smic_transaction, .get_result = smic_get_result, diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c index 207689c444a8..877205d22046 100644 --- a/drivers/char/ipmi/ipmi_ssif.c +++ b/drivers/char/ipmi/ipmi_ssif.c @@ -1136,6 +1136,10 @@ module_param_array(slave_addrs, int, &num_slave_addrs, 0); MODULE_PARM_DESC(slave_addrs, "The default IPMB slave address for the controller."); +static bool alerts_broken; +module_param(alerts_broken, bool, 0); +MODULE_PARM_DESC(alerts_broken, "Don't enable alerts for the controller."); + /* * Bit 0 enables message debugging, bit 1 enables state debugging, and * bit 2 enables timing debugging. This is an array indexed by @@ -1154,11 +1158,11 @@ static int use_thread; module_param(use_thread, int, 0); MODULE_PARM_DESC(use_thread, "Use the thread interface."); -static bool ssif_tryacpi = 1; +static bool ssif_tryacpi = true; module_param_named(tryacpi, ssif_tryacpi, bool, 0); MODULE_PARM_DESC(tryacpi, "Setting this to zero will disable the default scan of the interfaces identified via ACPI"); -static bool ssif_trydmi = 1; +static bool ssif_trydmi = true; module_param_named(trydmi, ssif_trydmi, bool, 0); MODULE_PARM_DESC(trydmi, "Setting this to zero will disable the default scan of the interfaces identified via DMI (SMBIOS)"); @@ -1582,6 +1586,10 @@ static int ssif_probe(struct i2c_client *client, const struct i2c_device_id *id) ssif_info->global_enables |= IPMI_BMC_EVT_MSG_BUFF; } + /* Some systems don't behave well if you enable alerts. */ + if (alerts_broken) + goto found; + msg[0] = IPMI_NETFN_APP_REQUEST << 2; msg[1] = IPMI_SET_BMC_GLOBAL_ENABLES_CMD; msg[2] = ssif_info->global_enables | IPMI_BMC_RCV_MSG_INTR; @@ -1787,7 +1795,7 @@ static unsigned short *ssif_address_list(void) } #ifdef CONFIG_ACPI -static struct acpi_device_id ssif_acpi_match[] = { +static const struct acpi_device_id ssif_acpi_match[] = { { "IPI0001", 0 }, { }, }; diff --git a/drivers/clk/rockchip/clk-rk3288.c b/drivers/clk/rockchip/clk-rk3288.c index 0df5bae9ddbf..9040878e3e2b 100644 --- a/drivers/clk/rockchip/clk-rk3288.c +++ b/drivers/clk/rockchip/clk-rk3288.c @@ -780,6 +780,7 @@ static const char *const rk3288_critical_clocks[] __initconst = { "aclk_cpu", "aclk_peri", "hclk_peri", + "pclk_pd_pmu", }; #ifdef CONFIG_PM_SLEEP diff --git a/drivers/clk/samsung/clk-exynos4.c b/drivers/clk/samsung/clk-exynos4.c index 251f48dcd12d..7f370d3e0983 100644 --- a/drivers/clk/samsung/clk-exynos4.c +++ b/drivers/clk/samsung/clk-exynos4.c @@ -1398,6 +1398,45 @@ static const struct exynos_cpuclk_cfg_data e4210_armclk_d[] __initconst = { { 0 }, }; +static const struct exynos_cpuclk_cfg_data e4212_armclk_d[] __initconst = { + { 1500000, E4210_CPU_DIV0(2, 1, 6, 0, 7, 3), E4210_CPU_DIV1(2, 6), }, + { 1400000, E4210_CPU_DIV0(2, 1, 6, 0, 7, 3), E4210_CPU_DIV1(2, 6), }, + { 1300000, E4210_CPU_DIV0(2, 1, 5, 0, 7, 3), E4210_CPU_DIV1(2, 5), }, + { 1200000, E4210_CPU_DIV0(2, 1, 5, 0, 7, 3), E4210_CPU_DIV1(2, 5), }, + { 1100000, E4210_CPU_DIV0(2, 1, 4, 0, 6, 3), E4210_CPU_DIV1(2, 4), }, + { 1000000, E4210_CPU_DIV0(1, 1, 4, 0, 5, 2), E4210_CPU_DIV1(2, 4), }, + { 900000, E4210_CPU_DIV0(1, 1, 3, 0, 5, 2), E4210_CPU_DIV1(2, 3), }, + { 800000, E4210_CPU_DIV0(1, 1, 3, 0, 5, 2), E4210_CPU_DIV1(2, 3), }, + { 700000, E4210_CPU_DIV0(1, 1, 3, 0, 4, 2), E4210_CPU_DIV1(2, 3), }, + { 600000, E4210_CPU_DIV0(1, 1, 3, 0, 4, 2), E4210_CPU_DIV1(2, 3), }, + { 500000, E4210_CPU_DIV0(1, 1, 3, 0, 4, 2), E4210_CPU_DIV1(2, 3), }, + { 400000, E4210_CPU_DIV0(1, 1, 3, 0, 4, 2), E4210_CPU_DIV1(2, 3), }, + { 300000, E4210_CPU_DIV0(1, 1, 2, 0, 4, 2), E4210_CPU_DIV1(2, 3), }, + { 200000, E4210_CPU_DIV0(1, 1, 1, 0, 3, 1), E4210_CPU_DIV1(2, 3), }, + { 0 }, +}; + +#define E4412_CPU_DIV1(cores, hpm, copy) \ + (((cores) << 8) | ((hpm) << 4) | ((copy) << 0)) + +static const struct exynos_cpuclk_cfg_data e4412_armclk_d[] __initconst = { + { 1500000, E4210_CPU_DIV0(2, 1, 6, 0, 7, 3), E4412_CPU_DIV1(7, 0, 6), }, + { 1400000, E4210_CPU_DIV0(2, 1, 6, 0, 7, 3), E4412_CPU_DIV1(6, 0, 6), }, + { 1300000, E4210_CPU_DIV0(2, 1, 5, 0, 7, 3), E4412_CPU_DIV1(6, 0, 5), }, + { 1200000, E4210_CPU_DIV0(2, 1, 5, 0, 7, 3), E4412_CPU_DIV1(5, 0, 5), }, + { 1100000, E4210_CPU_DIV0(2, 1, 4, 0, 6, 3), E4412_CPU_DIV1(5, 0, 4), }, + { 1000000, E4210_CPU_DIV0(1, 1, 4, 0, 5, 2), E4412_CPU_DIV1(4, 0, 4), }, + { 900000, E4210_CPU_DIV0(1, 1, 3, 0, 5, 2), E4412_CPU_DIV1(4, 0, 3), }, + { 800000, E4210_CPU_DIV0(1, 1, 3, 0, 5, 2), E4412_CPU_DIV1(3, 0, 3), }, + { 700000, E4210_CPU_DIV0(1, 1, 3, 0, 4, 2), E4412_CPU_DIV1(3, 0, 3), }, + { 600000, E4210_CPU_DIV0(1, 1, 3, 0, 4, 2), E4412_CPU_DIV1(2, 0, 3), }, + { 500000, E4210_CPU_DIV0(1, 1, 3, 0, 4, 2), E4412_CPU_DIV1(2, 0, 3), }, + { 400000, E4210_CPU_DIV0(1, 1, 3, 0, 4, 2), E4412_CPU_DIV1(1, 0, 3), }, + { 300000, E4210_CPU_DIV0(1, 1, 2, 0, 4, 2), E4412_CPU_DIV1(1, 0, 3), }, + { 200000, E4210_CPU_DIV0(1, 1, 1, 0, 3, 1), E4412_CPU_DIV1(0, 0, 3), }, + { 0 }, +}; + /* register exynos4 clocks */ static void __init exynos4_clk_init(struct device_node *np, enum exynos4_soc soc) @@ -1491,6 +1530,17 @@ static void __init exynos4_clk_init(struct device_node *np, samsung_clk_register_fixed_factor(ctx, exynos4x12_fixed_factor_clks, ARRAY_SIZE(exynos4x12_fixed_factor_clks)); + if (of_machine_is_compatible("samsung,exynos4412")) { + exynos_register_cpu_clock(ctx, CLK_ARM_CLK, "armclk", + mout_core_p4x12[0], mout_core_p4x12[1], 0x14200, + e4412_armclk_d, ARRAY_SIZE(e4412_armclk_d), + CLK_CPU_NEEDS_DEBUG_ALT_DIV | CLK_CPU_HAS_DIV1); + } else { + exynos_register_cpu_clock(ctx, CLK_ARM_CLK, "armclk", + mout_core_p4x12[0], mout_core_p4x12[1], 0x14200, + e4212_armclk_d, ARRAY_SIZE(e4212_armclk_d), + CLK_CPU_NEEDS_DEBUG_ALT_DIV | CLK_CPU_HAS_DIV1); + } } samsung_clk_register_alias(ctx, exynos4_aliases, diff --git a/drivers/clk/shmobile/clk-emev2.c b/drivers/clk/shmobile/clk-emev2.c index 5b60beb7d0eb..a91825471c79 100644 --- a/drivers/clk/shmobile/clk-emev2.c +++ b/drivers/clk/shmobile/clk-emev2.c @@ -28,6 +28,8 @@ #define USIBU1_RSTCTRL 0x0ac #define USIBU2_RSTCTRL 0x0b0 #define USIBU3_RSTCTRL 0x0b4 +#define IIC0_RSTCTRL 0x0dc +#define IIC1_RSTCTRL 0x0e0 #define STI_RSTCTRL 0x124 #define STI_CLKSEL 0x688 @@ -66,6 +68,10 @@ static void __init emev2_smu_init(void) emev2_smu_write(2, USIBU1_RSTCTRL); emev2_smu_write(2, USIBU2_RSTCTRL); emev2_smu_write(2, USIBU3_RSTCTRL); + + /* deassert reset for IIC0->IIC1 */ + emev2_smu_write(1, IIC0_RSTCTRL); + emev2_smu_write(1, IIC1_RSTCTRL); } static void __init emev2_smu_clkdiv_init(struct device_node *np) diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index 4e57730e0be4..a7726db13abb 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -111,6 +111,10 @@ config CLKSRC_LPC32XX select CLKSRC_MMIO select CLKSRC_OF +config CLKSRC_PISTACHIO + bool + select CLKSRC_OF + config CLKSRC_STM32 bool "Clocksource for STM32 SoCs" if !ARCH_STM32 depends on OF && ARM && (ARCH_STM32 || COMPILE_TEST) @@ -277,7 +281,7 @@ config CLKSRC_MIPS_GIC config CLKSRC_PXA def_bool y if ARCH_PXA || ARCH_SA1100 - select CLKSRC_OF if USE_OF + select CLKSRC_OF if OF help This enables OST0 support available on PXA and SA-11x0 platforms. @@ -293,4 +297,12 @@ config CLKSRC_IMX_GPT depends on ARM && CLKDEV_LOOKUP select CLKSRC_MMIO +config CLKSRC_ST_LPC + bool + depends on ARCH_STI + select CLKSRC_OF if OF + help + Enable this option to use the Low Power controller timer + as clocksource. + endmenu diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile index f228354961ca..5c00863c3e33 100644 --- a/drivers/clocksource/Makefile +++ b/drivers/clocksource/Makefile @@ -44,6 +44,7 @@ obj-$(CONFIG_FSL_FTM_TIMER) += fsl_ftm_timer.o obj-$(CONFIG_VF_PIT_TIMER) += vf_pit_timer.o obj-$(CONFIG_CLKSRC_QCOM) += qcom-timer.o obj-$(CONFIG_MTK_TIMER) += mtk_timer.o +obj-$(CONFIG_CLKSRC_PISTACHIO) += time-pistachio.o obj-$(CONFIG_ARM_ARCH_TIMER) += arm_arch_timer.o obj-$(CONFIG_ARM_GLOBAL_TIMER) += arm_global_timer.o @@ -60,3 +61,4 @@ obj-$(CONFIG_ASM9260_TIMER) += asm9260_timer.o obj-$(CONFIG_H8300) += h8300_timer8.o obj-$(CONFIG_H8300_TMR16) += h8300_timer16.o obj-$(CONFIG_H8300_TPU) += h8300_tpu.o +obj-$(CONFIG_CLKSRC_ST_LPC) += clksrc_st_lpc.o diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 0aa135ddbf80..d6e3e49399dd 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -181,44 +181,36 @@ static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id) return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt); } -static __always_inline void timer_set_mode(const int access, int mode, - struct clock_event_device *clk) +static __always_inline int timer_shutdown(const int access, + struct clock_event_device *clk) { unsigned long ctrl; - switch (mode) { - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); - ctrl &= ~ARCH_TIMER_CTRL_ENABLE; - arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); - break; - default: - break; - } + + ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk); + ctrl &= ~ARCH_TIMER_CTRL_ENABLE; + arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk); + + return 0; } -static void arch_timer_set_mode_virt(enum clock_event_mode mode, - struct clock_event_device *clk) +static int arch_timer_shutdown_virt(struct clock_event_device *clk) { - timer_set_mode(ARCH_TIMER_VIRT_ACCESS, mode, clk); + return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk); } -static void arch_timer_set_mode_phys(enum clock_event_mode mode, - struct clock_event_device *clk) +static int arch_timer_shutdown_phys(struct clock_event_device *clk) { - timer_set_mode(ARCH_TIMER_PHYS_ACCESS, mode, clk); + return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk); } -static void arch_timer_set_mode_virt_mem(enum clock_event_mode mode, - struct clock_event_device *clk) +static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk) { - timer_set_mode(ARCH_TIMER_MEM_VIRT_ACCESS, mode, clk); + return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk); } -static void arch_timer_set_mode_phys_mem(enum clock_event_mode mode, - struct clock_event_device *clk) +static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk) { - timer_set_mode(ARCH_TIMER_MEM_PHYS_ACCESS, mode, clk); + return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk); } static __always_inline void set_next_event(const int access, unsigned long evt, @@ -273,11 +265,11 @@ static void __arch_timer_setup(unsigned type, clk->cpumask = cpumask_of(smp_processor_id()); if (arch_timer_use_virtual) { clk->irq = arch_timer_ppi[VIRT_PPI]; - clk->set_mode = arch_timer_set_mode_virt; + clk->set_state_shutdown = arch_timer_shutdown_virt; clk->set_next_event = arch_timer_set_next_event_virt; } else { clk->irq = arch_timer_ppi[PHYS_SECURE_PPI]; - clk->set_mode = arch_timer_set_mode_phys; + clk->set_state_shutdown = arch_timer_shutdown_phys; clk->set_next_event = arch_timer_set_next_event_phys; } } else { @@ -286,17 +278,17 @@ static void __arch_timer_setup(unsigned type, clk->rating = 400; clk->cpumask = cpu_all_mask; if (arch_timer_mem_use_virtual) { - clk->set_mode = arch_timer_set_mode_virt_mem; + clk->set_state_shutdown = arch_timer_shutdown_virt_mem; clk->set_next_event = arch_timer_set_next_event_virt_mem; } else { - clk->set_mode = arch_timer_set_mode_phys_mem; + clk->set_state_shutdown = arch_timer_shutdown_phys_mem; clk->set_next_event = arch_timer_set_next_event_phys_mem; } } - clk->set_mode(CLOCK_EVT_MODE_SHUTDOWN, clk); + clk->set_state_shutdown(clk); clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff); } @@ -506,7 +498,7 @@ static void arch_timer_stop(struct clock_event_device *clk) disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]); } - clk->set_mode(CLOCK_EVT_MODE_UNUSED, clk); + clk->set_state_shutdown(clk); } static int arch_timer_cpu_notify(struct notifier_block *self, diff --git a/drivers/clocksource/arm_global_timer.c b/drivers/clocksource/arm_global_timer.c index e6833771a716..29ea50ac366a 100644 --- a/drivers/clocksource/arm_global_timer.c +++ b/drivers/clocksource/arm_global_timer.c @@ -107,26 +107,21 @@ static void gt_compare_set(unsigned long delta, int periodic) writel(ctrl, gt_base + GT_CONTROL); } -static void gt_clockevent_set_mode(enum clock_event_mode mode, - struct clock_event_device *clk) +static int gt_clockevent_shutdown(struct clock_event_device *evt) { unsigned long ctrl; - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - gt_compare_set(DIV_ROUND_CLOSEST(gt_clk_rate, HZ), 1); - break; - case CLOCK_EVT_MODE_ONESHOT: - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - ctrl = readl(gt_base + GT_CONTROL); - ctrl &= ~(GT_CONTROL_COMP_ENABLE | - GT_CONTROL_IRQ_ENABLE | GT_CONTROL_AUTO_INC); - writel(ctrl, gt_base + GT_CONTROL); - break; - default: - break; - } + ctrl = readl(gt_base + GT_CONTROL); + ctrl &= ~(GT_CONTROL_COMP_ENABLE | GT_CONTROL_IRQ_ENABLE | + GT_CONTROL_AUTO_INC); + writel(ctrl, gt_base + GT_CONTROL); + return 0; +} + +static int gt_clockevent_set_periodic(struct clock_event_device *evt) +{ + gt_compare_set(DIV_ROUND_CLOSEST(gt_clk_rate, HZ), 1); + return 0; } static int gt_clockevent_set_next_event(unsigned long evt, @@ -155,7 +150,7 @@ static irqreturn_t gt_clockevent_interrupt(int irq, void *dev_id) * the Global Timer flag _after_ having incremented * the Comparator register value to a higher value. */ - if (evt->mode == CLOCK_EVT_MODE_ONESHOT) + if (clockevent_state_oneshot(evt)) gt_compare_set(ULONG_MAX, 0); writel_relaxed(GT_INT_STATUS_EVENT_FLAG, gt_base + GT_INT_STATUS); @@ -171,7 +166,9 @@ static int gt_clockevents_init(struct clock_event_device *clk) clk->name = "arm_global_timer"; clk->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_PERCPU; - clk->set_mode = gt_clockevent_set_mode; + clk->set_state_shutdown = gt_clockevent_shutdown; + clk->set_state_periodic = gt_clockevent_set_periodic; + clk->set_state_oneshot = gt_clockevent_shutdown; clk->set_next_event = gt_clockevent_set_next_event; clk->cpumask = cpumask_of(cpu); clk->rating = 300; @@ -184,7 +181,7 @@ static int gt_clockevents_init(struct clock_event_device *clk) static void gt_clockevents_stop(struct clock_event_device *clk) { - gt_clockevent_set_mode(CLOCK_EVT_MODE_UNUSED, clk); + gt_clockevent_shutdown(clk); disable_percpu_irq(clk->irq); } diff --git a/drivers/clocksource/asm9260_timer.c b/drivers/clocksource/asm9260_timer.c index 4c2ba59897e8..217438d39eb3 100644 --- a/drivers/clocksource/asm9260_timer.c +++ b/drivers/clocksource/asm9260_timer.c @@ -120,38 +120,52 @@ static int asm9260_timer_set_next_event(unsigned long delta, return 0; } -static void asm9260_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static inline void __asm9260_timer_shutdown(struct clock_event_device *evt) { /* stop timer0 */ writel_relaxed(BM_C0_EN, priv.base + HW_TCR + CLR_REG); +} - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - /* disable reset and stop on match */ - writel_relaxed(BM_MCR_RES_EN(0) | BM_MCR_STOP_EN(0), - priv.base + HW_MCR + CLR_REG); - /* configure match count for TC0 */ - writel_relaxed(priv.ticks_per_jiffy, priv.base + HW_MR0); - /* enable TC0 */ - writel_relaxed(BM_C0_EN, priv.base + HW_TCR + SET_REG); - break; - case CLOCK_EVT_MODE_ONESHOT: - /* enable reset and stop on match */ - writel_relaxed(BM_MCR_RES_EN(0) | BM_MCR_STOP_EN(0), - priv.base + HW_MCR + SET_REG); - break; - default: - break; - } +static int asm9260_timer_shutdown(struct clock_event_device *evt) +{ + __asm9260_timer_shutdown(evt); + return 0; +} + +static int asm9260_timer_set_oneshot(struct clock_event_device *evt) +{ + __asm9260_timer_shutdown(evt); + + /* enable reset and stop on match */ + writel_relaxed(BM_MCR_RES_EN(0) | BM_MCR_STOP_EN(0), + priv.base + HW_MCR + SET_REG); + return 0; +} + +static int asm9260_timer_set_periodic(struct clock_event_device *evt) +{ + __asm9260_timer_shutdown(evt); + + /* disable reset and stop on match */ + writel_relaxed(BM_MCR_RES_EN(0) | BM_MCR_STOP_EN(0), + priv.base + HW_MCR + CLR_REG); + /* configure match count for TC0 */ + writel_relaxed(priv.ticks_per_jiffy, priv.base + HW_MR0); + /* enable TC0 */ + writel_relaxed(BM_C0_EN, priv.base + HW_TCR + SET_REG); + return 0; } static struct clock_event_device event_dev = { - .name = DRIVER_NAME, - .rating = 200, - .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .set_next_event = asm9260_timer_set_next_event, - .set_mode = asm9260_timer_set_mode, + .name = DRIVER_NAME, + .rating = 200, + .features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT, + .set_next_event = asm9260_timer_set_next_event, + .set_state_shutdown = asm9260_timer_shutdown, + .set_state_periodic = asm9260_timer_set_periodic, + .set_state_oneshot = asm9260_timer_set_oneshot, + .tick_resume = asm9260_timer_shutdown, }; static irqreturn_t asm9260_timer_interrupt(int irq, void *dev_id) diff --git a/drivers/clocksource/bcm2835_timer.c b/drivers/clocksource/bcm2835_timer.c index 26ed331b1aad..6f2822928963 100644 --- a/drivers/clocksource/bcm2835_timer.c +++ b/drivers/clocksource/bcm2835_timer.c @@ -54,21 +54,6 @@ static u64 notrace bcm2835_sched_read(void) return readl_relaxed(system_clock); } -static void bcm2835_time_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt_dev) -{ - switch (mode) { - case CLOCK_EVT_MODE_ONESHOT: - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_RESUME: - break; - default: - WARN(1, "%s: unhandled event mode %d\n", __func__, mode); - break; - } -} - static int bcm2835_time_set_next_event(unsigned long event, struct clock_event_device *evt_dev) { @@ -129,7 +114,6 @@ static void __init bcm2835_timer_init(struct device_node *node) timer->evt.name = node->name; timer->evt.rating = 300; timer->evt.features = CLOCK_EVT_FEAT_ONESHOT; - timer->evt.set_mode = bcm2835_time_set_mode; timer->evt.set_next_event = bcm2835_time_set_next_event; timer->evt.cpumask = cpumask_of(0); timer->act.name = node->name; diff --git a/drivers/clocksource/bcm_kona_timer.c b/drivers/clocksource/bcm_kona_timer.c index f1e33d08dd83..e717e87df9bc 100644 --- a/drivers/clocksource/bcm_kona_timer.c +++ b/drivers/clocksource/bcm_kona_timer.c @@ -127,25 +127,18 @@ static int kona_timer_set_next_event(unsigned long clc, return 0; } -static void kona_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *unused) +static int kona_timer_shutdown(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_ONESHOT: - /* by default mode is one shot don't do any thing */ - break; - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - default: - kona_timer_disable_and_clear(timers.tmr_regs); - } + kona_timer_disable_and_clear(timers.tmr_regs); + return 0; } static struct clock_event_device kona_clockevent_timer = { .name = "timer 1", .features = CLOCK_EVT_FEAT_ONESHOT, .set_next_event = kona_timer_set_next_event, - .set_mode = kona_timer_set_mode + .set_state_shutdown = kona_timer_shutdown, + .tick_resume = kona_timer_shutdown, }; static void __init kona_timer_clockevents_init(void) diff --git a/drivers/clocksource/cadence_ttc_timer.c b/drivers/clocksource/cadence_ttc_timer.c index 5ea91e3818d0..9be6018bd2b8 100644 --- a/drivers/clocksource/cadence_ttc_timer.c +++ b/drivers/clocksource/cadence_ttc_timer.c @@ -190,40 +190,42 @@ static int ttc_set_next_event(unsigned long cycles, } /** - * ttc_set_mode - Sets the mode of timer + * ttc_set_{shutdown|oneshot|periodic} - Sets the state of timer * - * @mode: Mode to be set * @evt: Address of clock event instance **/ -static void ttc_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int ttc_shutdown(struct clock_event_device *evt) { struct ttc_timer_clockevent *ttce = to_ttc_timer_clkevent(evt); struct ttc_timer *timer = &ttce->ttc; u32 ctrl_reg; - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - ttc_set_interval(timer, DIV_ROUND_CLOSEST(ttce->ttc.freq, - PRESCALE * HZ)); - break; - case CLOCK_EVT_MODE_ONESHOT: - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - ctrl_reg = readl_relaxed(timer->base_addr + - TTC_CNT_CNTRL_OFFSET); - ctrl_reg |= TTC_CNT_CNTRL_DISABLE_MASK; - writel_relaxed(ctrl_reg, - timer->base_addr + TTC_CNT_CNTRL_OFFSET); - break; - case CLOCK_EVT_MODE_RESUME: - ctrl_reg = readl_relaxed(timer->base_addr + - TTC_CNT_CNTRL_OFFSET); - ctrl_reg &= ~TTC_CNT_CNTRL_DISABLE_MASK; - writel_relaxed(ctrl_reg, - timer->base_addr + TTC_CNT_CNTRL_OFFSET); - break; - } + ctrl_reg = readl_relaxed(timer->base_addr + TTC_CNT_CNTRL_OFFSET); + ctrl_reg |= TTC_CNT_CNTRL_DISABLE_MASK; + writel_relaxed(ctrl_reg, timer->base_addr + TTC_CNT_CNTRL_OFFSET); + return 0; +} + +static int ttc_set_periodic(struct clock_event_device *evt) +{ + struct ttc_timer_clockevent *ttce = to_ttc_timer_clkevent(evt); + struct ttc_timer *timer = &ttce->ttc; + + ttc_set_interval(timer, + DIV_ROUND_CLOSEST(ttce->ttc.freq, PRESCALE * HZ)); + return 0; +} + +static int ttc_resume(struct clock_event_device *evt) +{ + struct ttc_timer_clockevent *ttce = to_ttc_timer_clkevent(evt); + struct ttc_timer *timer = &ttce->ttc; + u32 ctrl_reg; + + ctrl_reg = readl_relaxed(timer->base_addr + TTC_CNT_CNTRL_OFFSET); + ctrl_reg &= ~TTC_CNT_CNTRL_DISABLE_MASK; + writel_relaxed(ctrl_reg, timer->base_addr + TTC_CNT_CNTRL_OFFSET); + return 0; } static int ttc_rate_change_clocksource_cb(struct notifier_block *nb, @@ -429,7 +431,10 @@ static void __init ttc_setup_clockevent(struct clk *clk, ttcce->ce.name = "ttc_clockevent"; ttcce->ce.features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; ttcce->ce.set_next_event = ttc_set_next_event; - ttcce->ce.set_mode = ttc_set_mode; + ttcce->ce.set_state_shutdown = ttc_shutdown; + ttcce->ce.set_state_periodic = ttc_set_periodic; + ttcce->ce.set_state_oneshot = ttc_shutdown; + ttcce->ce.tick_resume = ttc_resume; ttcce->ce.rating = 200; ttcce->ce.irq = irq; ttcce->ce.cpumask = cpu_possible_mask; diff --git a/drivers/clocksource/clksrc_st_lpc.c b/drivers/clocksource/clksrc_st_lpc.c new file mode 100644 index 000000000000..65ec4674416d --- /dev/null +++ b/drivers/clocksource/clksrc_st_lpc.c @@ -0,0 +1,131 @@ +/* + * Clocksource using the Low Power Timer found in the Low Power Controller (LPC) + * + * Copyright (C) 2015 STMicroelectronics – All Rights Reserved + * + * Author(s): Francesco Virlinzi + * Ajit Pal Singh + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include +#include +#include +#include +#include +#include + +#include + +/* Low Power Timer */ +#define LPC_LPT_LSB_OFF 0x400 +#define LPC_LPT_MSB_OFF 0x404 +#define LPC_LPT_START_OFF 0x408 + +static struct st_clksrc_ddata { + struct clk *clk; + void __iomem *base; +} ddata; + +static void __init st_clksrc_reset(void) +{ + writel_relaxed(0, ddata.base + LPC_LPT_START_OFF); + writel_relaxed(0, ddata.base + LPC_LPT_MSB_OFF); + writel_relaxed(0, ddata.base + LPC_LPT_LSB_OFF); + writel_relaxed(1, ddata.base + LPC_LPT_START_OFF); +} + +static u64 notrace st_clksrc_sched_clock_read(void) +{ + return (u64)readl_relaxed(ddata.base + LPC_LPT_LSB_OFF); +} + +static int __init st_clksrc_init(void) +{ + unsigned long rate; + int ret; + + st_clksrc_reset(); + + rate = clk_get_rate(ddata.clk); + + sched_clock_register(st_clksrc_sched_clock_read, 32, rate); + + ret = clocksource_mmio_init(ddata.base + LPC_LPT_LSB_OFF, + "clksrc-st-lpc", rate, 300, 32, + clocksource_mmio_readl_up); + if (ret) { + pr_err("clksrc-st-lpc: Failed to register clocksource\n"); + return ret; + } + + return 0; +} + +static int __init st_clksrc_setup_clk(struct device_node *np) +{ + struct clk *clk; + + clk = of_clk_get(np, 0); + if (IS_ERR(clk)) { + pr_err("clksrc-st-lpc: Failed to get LPC clock\n"); + return PTR_ERR(clk); + } + + if (clk_prepare_enable(clk)) { + pr_err("clksrc-st-lpc: Failed to enable LPC clock\n"); + return -EINVAL; + } + + if (!clk_get_rate(clk)) { + pr_err("clksrc-st-lpc: Failed to get LPC clock rate\n"); + clk_disable_unprepare(clk); + return -EINVAL; + } + + ddata.clk = clk; + + return 0; +} + +static void __init st_clksrc_of_register(struct device_node *np) +{ + int ret; + uint32_t mode; + + ret = of_property_read_u32(np, "st,lpc-mode", &mode); + if (ret) { + pr_err("clksrc-st-lpc: An LPC mode must be provided\n"); + return; + } + + /* LPC can either run as a Clocksource or in RTC or WDT mode */ + if (mode != ST_LPC_MODE_CLKSRC) + return; + + ddata.base = of_iomap(np, 0); + if (!ddata.base) { + pr_err("clksrc-st-lpc: Unable to map iomem\n"); + return; + } + + if (st_clksrc_setup_clk(np)) { + iounmap(ddata.base); + return; + } + + if (st_clksrc_init()) { + clk_disable_unprepare(ddata.clk); + clk_put(ddata.clk); + iounmap(ddata.base); + return; + } + + pr_info("clksrc-st-lpc: clocksource initialised - running @ %luHz\n", + clk_get_rate(ddata.clk)); +} +CLOCKSOURCE_OF_DECLARE(ddata, "st,stih407-lpc", st_clksrc_of_register); diff --git a/drivers/clocksource/clps711x-timer.c b/drivers/clocksource/clps711x-timer.c index d83ec1f2fddc..cdd86e3525bb 100644 --- a/drivers/clocksource/clps711x-timer.c +++ b/drivers/clocksource/clps711x-timer.c @@ -61,11 +61,6 @@ static irqreturn_t clps711x_timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static void clps711x_clockevent_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) -{ -} - static int __init _clps711x_clkevt_init(struct clk *clock, void __iomem *base, unsigned int irq) { @@ -91,7 +86,6 @@ static int __init _clps711x_clkevt_init(struct clk *clock, void __iomem *base, clkevt->name = "clps711x-clockevent"; clkevt->rating = 300; clkevt->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_C3STOP; - clkevt->set_mode = clps711x_clockevent_set_mode; clkevt->cpumask = cpumask_of(0); clockevents_config_and_register(clkevt, HZ, 0, 0); diff --git a/drivers/clocksource/cs5535-clockevt.c b/drivers/clocksource/cs5535-clockevt.c index db2105290898..9a7e37cf56b0 100644 --- a/drivers/clocksource/cs5535-clockevt.c +++ b/drivers/clocksource/cs5535-clockevt.c @@ -42,7 +42,6 @@ MODULE_PARM_DESC(irq, "Which IRQ to use for the clock source MFGPT ticks."); * 256 128 .125 512.000 */ -static unsigned int cs5535_tick_mode = CLOCK_EVT_MODE_SHUTDOWN; static struct cs5535_mfgpt_timer *cs5535_event_clock; /* Selected from the table above */ @@ -77,15 +76,17 @@ static void start_timer(struct cs5535_mfgpt_timer *timer, uint16_t delta) MFGPT_SETUP_CNTEN | MFGPT_SETUP_CMP2); } -static void mfgpt_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int mfgpt_shutdown(struct clock_event_device *evt) { disable_timer(cs5535_event_clock); + return 0; +} - if (mode == CLOCK_EVT_MODE_PERIODIC) - start_timer(cs5535_event_clock, MFGPT_PERIODIC); - - cs5535_tick_mode = mode; +static int mfgpt_set_periodic(struct clock_event_device *evt) +{ + disable_timer(cs5535_event_clock); + start_timer(cs5535_event_clock, MFGPT_PERIODIC); + return 0; } static int mfgpt_next_event(unsigned long delta, struct clock_event_device *evt) @@ -97,7 +98,10 @@ static int mfgpt_next_event(unsigned long delta, struct clock_event_device *evt) static struct clock_event_device cs5535_clockevent = { .name = DRV_NAME, .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .set_mode = mfgpt_set_mode, + .set_state_shutdown = mfgpt_shutdown, + .set_state_periodic = mfgpt_set_periodic, + .set_state_oneshot = mfgpt_shutdown, + .tick_resume = mfgpt_shutdown, .set_next_event = mfgpt_next_event, .rating = 250, }; @@ -113,7 +117,7 @@ static irqreturn_t mfgpt_tick(int irq, void *dev_id) /* Turn off the clock (and clear the event) */ disable_timer(cs5535_event_clock); - if (cs5535_tick_mode == CLOCK_EVT_MODE_SHUTDOWN) + if (clockevent_state_shutdown(&cs5535_clockevent)) return IRQ_HANDLED; /* Clear the counter */ @@ -121,7 +125,7 @@ static irqreturn_t mfgpt_tick(int irq, void *dev_id) /* Restart the clock in periodic mode */ - if (cs5535_tick_mode == CLOCK_EVT_MODE_PERIODIC) + if (clockevent_state_periodic(&cs5535_clockevent)) cs5535_mfgpt_write(cs5535_event_clock, MFGPT_REG_SETUP, MFGPT_SETUP_CNTEN | MFGPT_SETUP_CMP2); diff --git a/drivers/clocksource/dummy_timer.c b/drivers/clocksource/dummy_timer.c index 31990600fcff..776b6c86dcd5 100644 --- a/drivers/clocksource/dummy_timer.c +++ b/drivers/clocksource/dummy_timer.c @@ -16,15 +16,6 @@ static DEFINE_PER_CPU(struct clock_event_device, dummy_timer_evt); -static void dummy_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) -{ - /* - * Core clockevents code will call this when exchanging timer devices. - * We don't need to do anything here. - */ -} - static void dummy_timer_setup(void) { int cpu = smp_processor_id(); @@ -35,7 +26,6 @@ static void dummy_timer_setup(void) CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_DUMMY; evt->rating = 100; - evt->set_mode = dummy_timer_set_mode; evt->cpumask = cpumask_of(cpu); clockevents_register_device(evt); diff --git a/drivers/clocksource/dw_apb_timer.c b/drivers/clocksource/dw_apb_timer.c index 35a88097af3c..c76c75006ea6 100644 --- a/drivers/clocksource/dw_apb_timer.c +++ b/drivers/clocksource/dw_apb_timer.c @@ -110,71 +110,87 @@ static void apbt_enable_int(struct dw_apb_timer *timer) apbt_writel(timer, ctrl, APBTMR_N_CONTROL); } -static void apbt_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int apbt_shutdown(struct clock_event_device *evt) { + struct dw_apb_clock_event_device *dw_ced = ced_to_dw_apb_ced(evt); unsigned long ctrl; - unsigned long period; + + pr_debug("%s CPU %d state=shutdown\n", __func__, + cpumask_first(evt->cpumask)); + + ctrl = apbt_readl(&dw_ced->timer, APBTMR_N_CONTROL); + ctrl &= ~APBTMR_CONTROL_ENABLE; + apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); + return 0; +} + +static int apbt_set_oneshot(struct clock_event_device *evt) +{ + struct dw_apb_clock_event_device *dw_ced = ced_to_dw_apb_ced(evt); + unsigned long ctrl; + + pr_debug("%s CPU %d state=oneshot\n", __func__, + cpumask_first(evt->cpumask)); + + ctrl = apbt_readl(&dw_ced->timer, APBTMR_N_CONTROL); + /* + * set free running mode, this mode will let timer reload max + * timeout which will give time (3min on 25MHz clock) to rearm + * the next event, therefore emulate the one-shot mode. + */ + ctrl &= ~APBTMR_CONTROL_ENABLE; + ctrl &= ~APBTMR_CONTROL_MODE_PERIODIC; + + apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); + /* write again to set free running mode */ + apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); + + /* + * DW APB p. 46, load counter with all 1s before starting free + * running mode. + */ + apbt_writel(&dw_ced->timer, ~0, APBTMR_N_LOAD_COUNT); + ctrl &= ~APBTMR_CONTROL_INT; + ctrl |= APBTMR_CONTROL_ENABLE; + apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); + return 0; +} + +static int apbt_set_periodic(struct clock_event_device *evt) +{ + struct dw_apb_clock_event_device *dw_ced = ced_to_dw_apb_ced(evt); + unsigned long period = DIV_ROUND_UP(dw_ced->timer.freq, HZ); + unsigned long ctrl; + + pr_debug("%s CPU %d state=periodic\n", __func__, + cpumask_first(evt->cpumask)); + + ctrl = apbt_readl(&dw_ced->timer, APBTMR_N_CONTROL); + ctrl |= APBTMR_CONTROL_MODE_PERIODIC; + apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); + /* + * DW APB p. 46, have to disable timer before load counter, + * may cause sync problem. + */ + ctrl &= ~APBTMR_CONTROL_ENABLE; + apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); + udelay(1); + pr_debug("Setting clock period %lu for HZ %d\n", period, HZ); + apbt_writel(&dw_ced->timer, period, APBTMR_N_LOAD_COUNT); + ctrl |= APBTMR_CONTROL_ENABLE; + apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); + return 0; +} + +static int apbt_resume(struct clock_event_device *evt) +{ struct dw_apb_clock_event_device *dw_ced = ced_to_dw_apb_ced(evt); - pr_debug("%s CPU %d mode=%d\n", __func__, - cpumask_first(evt->cpumask), - mode); + pr_debug("%s CPU %d state=resume\n", __func__, + cpumask_first(evt->cpumask)); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - period = DIV_ROUND_UP(dw_ced->timer.freq, HZ); - ctrl = apbt_readl(&dw_ced->timer, APBTMR_N_CONTROL); - ctrl |= APBTMR_CONTROL_MODE_PERIODIC; - apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); - /* - * DW APB p. 46, have to disable timer before load counter, - * may cause sync problem. - */ - ctrl &= ~APBTMR_CONTROL_ENABLE; - apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); - udelay(1); - pr_debug("Setting clock period %lu for HZ %d\n", period, HZ); - apbt_writel(&dw_ced->timer, period, APBTMR_N_LOAD_COUNT); - ctrl |= APBTMR_CONTROL_ENABLE; - apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); - break; - - case CLOCK_EVT_MODE_ONESHOT: - ctrl = apbt_readl(&dw_ced->timer, APBTMR_N_CONTROL); - /* - * set free running mode, this mode will let timer reload max - * timeout which will give time (3min on 25MHz clock) to rearm - * the next event, therefore emulate the one-shot mode. - */ - ctrl &= ~APBTMR_CONTROL_ENABLE; - ctrl &= ~APBTMR_CONTROL_MODE_PERIODIC; - - apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); - /* write again to set free running mode */ - apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); - - /* - * DW APB p. 46, load counter with all 1s before starting free - * running mode. - */ - apbt_writel(&dw_ced->timer, ~0, APBTMR_N_LOAD_COUNT); - ctrl &= ~APBTMR_CONTROL_INT; - ctrl |= APBTMR_CONTROL_ENABLE; - apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); - break; - - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - ctrl = apbt_readl(&dw_ced->timer, APBTMR_N_CONTROL); - ctrl &= ~APBTMR_CONTROL_ENABLE; - apbt_writel(&dw_ced->timer, ctrl, APBTMR_N_CONTROL); - break; - - case CLOCK_EVT_MODE_RESUME: - apbt_enable_int(&dw_ced->timer); - break; - } + apbt_enable_int(&dw_ced->timer); + return 0; } static int apbt_next_event(unsigned long delta, @@ -232,8 +248,12 @@ dw_apb_clockevent_init(int cpu, const char *name, unsigned rating, &dw_ced->ced); dw_ced->ced.min_delta_ns = clockevent_delta2ns(5000, &dw_ced->ced); dw_ced->ced.cpumask = cpumask_of(cpu); - dw_ced->ced.features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; - dw_ced->ced.set_mode = apbt_set_mode; + dw_ced->ced.features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_DYNIRQ; + dw_ced->ced.set_state_shutdown = apbt_shutdown; + dw_ced->ced.set_state_periodic = apbt_set_periodic; + dw_ced->ced.set_state_oneshot = apbt_set_oneshot; + dw_ced->ced.tick_resume = apbt_resume; dw_ced->ced.set_next_event = apbt_next_event; dw_ced->ced.irq = dw_ced->timer.irq; dw_ced->ced.rating = rating; diff --git a/drivers/clocksource/em_sti.c b/drivers/clocksource/em_sti.c index dc3c6ee04aaa..7a97a34dba70 100644 --- a/drivers/clocksource/em_sti.c +++ b/drivers/clocksource/em_sti.c @@ -251,33 +251,21 @@ static struct em_sti_priv *ced_to_em_sti(struct clock_event_device *ced) return container_of(ced, struct em_sti_priv, ced); } -static void em_sti_clock_event_mode(enum clock_event_mode mode, - struct clock_event_device *ced) +static int em_sti_clock_event_shutdown(struct clock_event_device *ced) +{ + struct em_sti_priv *p = ced_to_em_sti(ced); + em_sti_stop(p, USER_CLOCKEVENT); + return 0; +} + +static int em_sti_clock_event_set_oneshot(struct clock_event_device *ced) { struct em_sti_priv *p = ced_to_em_sti(ced); - /* deal with old setting first */ - switch (ced->mode) { - case CLOCK_EVT_MODE_ONESHOT: - em_sti_stop(p, USER_CLOCKEVENT); - break; - default: - break; - } - - switch (mode) { - case CLOCK_EVT_MODE_ONESHOT: - dev_info(&p->pdev->dev, "used for oneshot clock events\n"); - em_sti_start(p, USER_CLOCKEVENT); - clockevents_config(&p->ced, p->rate); - break; - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - em_sti_stop(p, USER_CLOCKEVENT); - break; - default: - break; - } + dev_info(&p->pdev->dev, "used for oneshot clock events\n"); + em_sti_start(p, USER_CLOCKEVENT); + clockevents_config(&p->ced, p->rate); + return 0; } static int em_sti_clock_event_next(unsigned long delta, @@ -303,11 +291,12 @@ static void em_sti_register_clockevent(struct em_sti_priv *p) ced->rating = 200; ced->cpumask = cpu_possible_mask; ced->set_next_event = em_sti_clock_event_next; - ced->set_mode = em_sti_clock_event_mode; + ced->set_state_shutdown = em_sti_clock_event_shutdown; + ced->set_state_oneshot = em_sti_clock_event_set_oneshot; dev_info(&p->pdev->dev, "used for clock events\n"); - /* Register with dummy 1 Hz value, gets updated in ->set_mode() */ + /* Register with dummy 1 Hz value, gets updated in ->set_state_oneshot() */ clockevents_config_and_register(ced, 1, 2, 0xffffffff); } diff --git a/drivers/clocksource/exynos_mct.c b/drivers/clocksource/exynos_mct.c index 9064ff743598..029f96ab131a 100644 --- a/drivers/clocksource/exynos_mct.c +++ b/drivers/clocksource/exynos_mct.c @@ -257,15 +257,14 @@ static void exynos4_mct_comp0_stop(void) exynos4_mct_write(0, EXYNOS4_MCT_G_INT_ENB); } -static void exynos4_mct_comp0_start(enum clock_event_mode mode, - unsigned long cycles) +static void exynos4_mct_comp0_start(bool periodic, unsigned long cycles) { unsigned int tcon; cycle_t comp_cycle; tcon = readl_relaxed(reg_base + EXYNOS4_MCT_G_TCON); - if (mode == CLOCK_EVT_MODE_PERIODIC) { + if (periodic) { tcon |= MCT_G_TCON_COMP0_AUTO_INC; exynos4_mct_write(cycles, EXYNOS4_MCT_G_COMP0_ADD_INCR); } @@ -283,38 +282,38 @@ static void exynos4_mct_comp0_start(enum clock_event_mode mode, static int exynos4_comp_set_next_event(unsigned long cycles, struct clock_event_device *evt) { - exynos4_mct_comp0_start(evt->mode, cycles); + exynos4_mct_comp0_start(false, cycles); return 0; } -static void exynos4_comp_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int mct_set_state_shutdown(struct clock_event_device *evt) +{ + exynos4_mct_comp0_stop(); + return 0; +} + +static int mct_set_state_periodic(struct clock_event_device *evt) { unsigned long cycles_per_jiffy; + + cycles_per_jiffy = (((unsigned long long)NSEC_PER_SEC / HZ * evt->mult) + >> evt->shift); exynos4_mct_comp0_stop(); - - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - cycles_per_jiffy = - (((unsigned long long) NSEC_PER_SEC / HZ * evt->mult) >> evt->shift); - exynos4_mct_comp0_start(mode, cycles_per_jiffy); - break; - - case CLOCK_EVT_MODE_ONESHOT: - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_RESUME: - break; - } + exynos4_mct_comp0_start(true, cycles_per_jiffy); + return 0; } static struct clock_event_device mct_comp_device = { - .name = "mct-comp", - .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .rating = 250, - .set_next_event = exynos4_comp_set_next_event, - .set_mode = exynos4_comp_set_mode, + .name = "mct-comp", + .features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT, + .rating = 250, + .set_next_event = exynos4_comp_set_next_event, + .set_state_periodic = mct_set_state_periodic, + .set_state_shutdown = mct_set_state_shutdown, + .set_state_oneshot = mct_set_state_shutdown, + .tick_resume = mct_set_state_shutdown, }; static irqreturn_t exynos4_mct_comp_isr(int irq, void *dev_id) @@ -390,39 +389,32 @@ static int exynos4_tick_set_next_event(unsigned long cycles, return 0; } -static inline void exynos4_tick_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int set_state_shutdown(struct clock_event_device *evt) +{ + exynos4_mct_tick_stop(this_cpu_ptr(&percpu_mct_tick)); + return 0; +} + +static int set_state_periodic(struct clock_event_device *evt) { struct mct_clock_event_device *mevt = this_cpu_ptr(&percpu_mct_tick); unsigned long cycles_per_jiffy; + cycles_per_jiffy = (((unsigned long long)NSEC_PER_SEC / HZ * evt->mult) + >> evt->shift); exynos4_mct_tick_stop(mevt); - - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - cycles_per_jiffy = - (((unsigned long long) NSEC_PER_SEC / HZ * evt->mult) >> evt->shift); - exynos4_mct_tick_start(cycles_per_jiffy, mevt); - break; - - case CLOCK_EVT_MODE_ONESHOT: - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_RESUME: - break; - } + exynos4_mct_tick_start(cycles_per_jiffy, mevt); + return 0; } static void exynos4_mct_tick_clear(struct mct_clock_event_device *mevt) { - struct clock_event_device *evt = &mevt->evt; - /* * This is for supporting oneshot mode. * Mct would generate interrupt periodically * without explicit stopping. */ - if (evt->mode != CLOCK_EVT_MODE_PERIODIC) + if (!clockevent_state_periodic(&mevt->evt)) exynos4_mct_tick_stop(mevt); /* Clear the MCT tick interrupt */ @@ -442,20 +434,21 @@ static irqreturn_t exynos4_mct_tick_isr(int irq, void *dev_id) return IRQ_HANDLED; } -static int exynos4_local_timer_setup(struct clock_event_device *evt) +static int exynos4_local_timer_setup(struct mct_clock_event_device *mevt) { - struct mct_clock_event_device *mevt; + struct clock_event_device *evt = &mevt->evt; unsigned int cpu = smp_processor_id(); - mevt = container_of(evt, struct mct_clock_event_device, evt); - mevt->base = EXYNOS4_MCT_L_BASE(cpu); snprintf(mevt->name, sizeof(mevt->name), "mct_tick%d", cpu); evt->name = mevt->name; evt->cpumask = cpumask_of(cpu); evt->set_next_event = exynos4_tick_set_next_event; - evt->set_mode = exynos4_tick_set_mode; + evt->set_state_periodic = set_state_periodic; + evt->set_state_shutdown = set_state_shutdown; + evt->set_state_oneshot = set_state_shutdown; + evt->tick_resume = set_state_shutdown; evt->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; evt->rating = 450; @@ -477,9 +470,11 @@ static int exynos4_local_timer_setup(struct clock_event_device *evt) return 0; } -static void exynos4_local_timer_stop(struct clock_event_device *evt) +static void exynos4_local_timer_stop(struct mct_clock_event_device *mevt) { - evt->set_mode(CLOCK_EVT_MODE_UNUSED, evt); + struct clock_event_device *evt = &mevt->evt; + + evt->set_state_shutdown(evt); if (mct_int_type == MCT_INT_SPI) { if (evt->irq != -1) disable_irq_nosync(evt->irq); @@ -500,11 +495,11 @@ static int exynos4_mct_cpu_notify(struct notifier_block *self, switch (action & ~CPU_TASKS_FROZEN) { case CPU_STARTING: mevt = this_cpu_ptr(&percpu_mct_tick); - exynos4_local_timer_setup(&mevt->evt); + exynos4_local_timer_setup(mevt); break; case CPU_DYING: mevt = this_cpu_ptr(&percpu_mct_tick); - exynos4_local_timer_stop(&mevt->evt); + exynos4_local_timer_stop(mevt); break; } @@ -570,7 +565,7 @@ static void __init exynos4_timer_resources(struct device_node *np, void __iomem goto out_irq; /* Immediately configure the timer on the boot CPU */ - exynos4_local_timer_setup(&mevt->evt); + exynos4_local_timer_setup(mevt); return; out_irq: diff --git a/drivers/clocksource/fsl_ftm_timer.c b/drivers/clocksource/fsl_ftm_timer.c index 454227d4f895..ef434699c80a 100644 --- a/drivers/clocksource/fsl_ftm_timer.c +++ b/drivers/clocksource/fsl_ftm_timer.c @@ -153,19 +153,16 @@ static int ftm_set_next_event(unsigned long delta, return 0; } -static void ftm_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int ftm_set_oneshot(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - ftm_set_next_event(priv->periodic_cyc, evt); - break; - case CLOCK_EVT_MODE_ONESHOT: - ftm_counter_disable(priv->clkevt_base); - break; - default: - return; - } + ftm_counter_disable(priv->clkevt_base); + return 0; +} + +static int ftm_set_periodic(struct clock_event_device *evt) +{ + ftm_set_next_event(priv->periodic_cyc, evt); + return 0; } static irqreturn_t ftm_evt_interrupt(int irq, void *dev_id) @@ -174,7 +171,7 @@ static irqreturn_t ftm_evt_interrupt(int irq, void *dev_id) ftm_irq_acknowledge(priv->clkevt_base); - if (likely(evt->mode == CLOCK_EVT_MODE_ONESHOT)) { + if (likely(clockevent_state_oneshot(evt))) { ftm_irq_disable(priv->clkevt_base); ftm_counter_disable(priv->clkevt_base); } @@ -185,11 +182,13 @@ static irqreturn_t ftm_evt_interrupt(int irq, void *dev_id) } static struct clock_event_device ftm_clockevent = { - .name = "Freescale ftm timer", - .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .set_mode = ftm_set_mode, - .set_next_event = ftm_set_next_event, - .rating = 300, + .name = "Freescale ftm timer", + .features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT, + .set_state_periodic = ftm_set_periodic, + .set_state_oneshot = ftm_set_oneshot, + .set_next_event = ftm_set_next_event, + .rating = 300, }; static struct irqaction ftm_timer_irq = { diff --git a/drivers/clocksource/h8300_timer8.c b/drivers/clocksource/h8300_timer8.c index 0214cb3a7f5e..f9b3b7033a97 100644 --- a/drivers/clocksource/h8300_timer8.c +++ b/drivers/clocksource/h8300_timer8.c @@ -81,7 +81,7 @@ static irqreturn_t timer8_interrupt(int irq, void *dev_id) p->flags |= FLAG_IRQCONTEXT; ctrl_outw(p->tcora, p->mapbase + TCORA); if (!(p->flags & FLAG_SKIPEVENT)) { - if (p->ced.mode == CLOCK_EVT_MODE_ONESHOT) + if (clockevent_state_oneshot(&p->ced)) ctrl_outw(0x0000, p->mapbase + _8TCR); p->ced.event_handler(&p->ced); } @@ -169,29 +169,32 @@ static void timer8_clock_event_start(struct timer8_priv *p, int periodic) timer8_set_next(p, periodic?(p->rate + HZ/2) / HZ:0x10000); } -static void timer8_clock_event_mode(enum clock_event_mode mode, - struct clock_event_device *ced) +static int timer8_clock_event_shutdown(struct clock_event_device *ced) +{ + timer8_stop(ced_to_priv(ced)); + return 0; +} + +static int timer8_clock_event_periodic(struct clock_event_device *ced) { struct timer8_priv *p = ced_to_priv(ced); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - dev_info(&p->pdev->dev, "used for periodic clock events\n"); - timer8_stop(p); - timer8_clock_event_start(p, PERIODIC); - break; - case CLOCK_EVT_MODE_ONESHOT: - dev_info(&p->pdev->dev, "used for oneshot clock events\n"); - timer8_stop(p); - timer8_clock_event_start(p, ONESHOT); - break; - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - timer8_stop(p); - break; - default: - break; - } + dev_info(&p->pdev->dev, "used for periodic clock events\n"); + timer8_stop(p); + timer8_clock_event_start(p, PERIODIC); + + return 0; +} + +static int timer8_clock_event_oneshot(struct clock_event_device *ced) +{ + struct timer8_priv *p = ced_to_priv(ced); + + dev_info(&p->pdev->dev, "used for oneshot clock events\n"); + timer8_stop(p); + timer8_clock_event_start(p, ONESHOT); + + return 0; } static int timer8_clock_event_next(unsigned long delta, @@ -199,7 +202,7 @@ static int timer8_clock_event_next(unsigned long delta, { struct timer8_priv *p = ced_to_priv(ced); - BUG_ON(ced->mode != CLOCK_EVT_MODE_ONESHOT); + BUG_ON(!clockevent_state_oneshot(ced)); timer8_set_next(p, delta - 1); return 0; @@ -246,7 +249,9 @@ static int timer8_setup(struct timer8_priv *p, p->ced.rating = 200; p->ced.cpumask = cpumask_of(0); p->ced.set_next_event = timer8_clock_event_next; - p->ced.set_mode = timer8_clock_event_mode; + p->ced.set_state_shutdown = timer8_clock_event_shutdown; + p->ced.set_state_periodic = timer8_clock_event_periodic; + p->ced.set_state_oneshot = timer8_clock_event_oneshot; ret = setup_irq(irq, &p->irqaction); if (ret < 0) { diff --git a/drivers/clocksource/i8253.c b/drivers/clocksource/i8253.c index 14ee3efcc404..0efd36e483ab 100644 --- a/drivers/clocksource/i8253.c +++ b/drivers/clocksource/i8253.c @@ -100,44 +100,40 @@ int __init clocksource_i8253_init(void) #endif #ifdef CONFIG_CLKEVT_I8253 -/* - * Initialize the PIT timer. - * - * This is also called after resume to bring the PIT into operation again. - */ -static void init_pit_timer(enum clock_event_mode mode, - struct clock_event_device *evt) +static int pit_shutdown(struct clock_event_device *evt) +{ + if (!clockevent_state_oneshot(evt) && !clockevent_state_periodic(evt)) + return 0; + + raw_spin_lock(&i8253_lock); + + outb_p(0x30, PIT_MODE); + outb_p(0, PIT_CH0); + outb_p(0, PIT_CH0); + + raw_spin_unlock(&i8253_lock); + return 0; +} + +static int pit_set_oneshot(struct clock_event_device *evt) +{ + raw_spin_lock(&i8253_lock); + outb_p(0x38, PIT_MODE); + raw_spin_unlock(&i8253_lock); + return 0; +} + +static int pit_set_periodic(struct clock_event_device *evt) { raw_spin_lock(&i8253_lock); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - /* binary, mode 2, LSB/MSB, ch 0 */ - outb_p(0x34, PIT_MODE); - outb_p(PIT_LATCH & 0xff , PIT_CH0); /* LSB */ - outb_p(PIT_LATCH >> 8 , PIT_CH0); /* MSB */ - break; + /* binary, mode 2, LSB/MSB, ch 0 */ + outb_p(0x34, PIT_MODE); + outb_p(PIT_LATCH & 0xff, PIT_CH0); /* LSB */ + outb_p(PIT_LATCH >> 8, PIT_CH0); /* MSB */ - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - if (evt->mode == CLOCK_EVT_MODE_PERIODIC || - evt->mode == CLOCK_EVT_MODE_ONESHOT) { - outb_p(0x30, PIT_MODE); - outb_p(0, PIT_CH0); - outb_p(0, PIT_CH0); - } - break; - - case CLOCK_EVT_MODE_ONESHOT: - /* One shot setup */ - outb_p(0x38, PIT_MODE); - break; - - case CLOCK_EVT_MODE_RESUME: - /* Nothing to do here */ - break; - } raw_spin_unlock(&i8253_lock); + return 0; } /* @@ -160,10 +156,11 @@ static int pit_next_event(unsigned long delta, struct clock_event_device *evt) * it can be solely used for the global tick. */ struct clock_event_device i8253_clockevent = { - .name = "pit", - .features = CLOCK_EVT_FEAT_PERIODIC, - .set_mode = init_pit_timer, - .set_next_event = pit_next_event, + .name = "pit", + .features = CLOCK_EVT_FEAT_PERIODIC, + .set_state_shutdown = pit_shutdown, + .set_state_periodic = pit_set_periodic, + .set_next_event = pit_next_event, }; /* @@ -172,8 +169,10 @@ struct clock_event_device i8253_clockevent = { */ void __init clockevent_i8253_init(bool oneshot) { - if (oneshot) + if (oneshot) { i8253_clockevent.features |= CLOCK_EVT_FEAT_ONESHOT; + i8253_clockevent.set_state_oneshot = pit_set_oneshot; + } /* * Start pit with the boot cpu mask. x86 might make it global * when it is used as broadcast device later. diff --git a/drivers/clocksource/meson6_timer.c b/drivers/clocksource/meson6_timer.c index 5c15cba41dca..1fa22c4d2d49 100644 --- a/drivers/clocksource/meson6_timer.c +++ b/drivers/clocksource/meson6_timer.c @@ -67,25 +67,25 @@ static void meson6_clkevt_time_start(unsigned char timer, bool periodic) writel(val | TIMER_ENABLE_BIT(timer), timer_base + TIMER_ISA_MUX); } -static void meson6_clkevt_mode(enum clock_event_mode mode, - struct clock_event_device *clk) +static int meson6_shutdown(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - meson6_clkevt_time_stop(CED_ID); - meson6_clkevt_time_setup(CED_ID, USEC_PER_SEC/HZ - 1); - meson6_clkevt_time_start(CED_ID, true); - break; - case CLOCK_EVT_MODE_ONESHOT: - meson6_clkevt_time_stop(CED_ID); - meson6_clkevt_time_start(CED_ID, false); - break; - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - default: - meson6_clkevt_time_stop(CED_ID); - break; - } + meson6_clkevt_time_stop(CED_ID); + return 0; +} + +static int meson6_set_oneshot(struct clock_event_device *evt) +{ + meson6_clkevt_time_stop(CED_ID); + meson6_clkevt_time_start(CED_ID, false); + return 0; +} + +static int meson6_set_periodic(struct clock_event_device *evt) +{ + meson6_clkevt_time_stop(CED_ID); + meson6_clkevt_time_setup(CED_ID, USEC_PER_SEC / HZ - 1); + meson6_clkevt_time_start(CED_ID, true); + return 0; } static int meson6_clkevt_next_event(unsigned long evt, @@ -99,11 +99,15 @@ static int meson6_clkevt_next_event(unsigned long evt, } static struct clock_event_device meson6_clockevent = { - .name = "meson6_tick", - .rating = 400, - .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .set_mode = meson6_clkevt_mode, - .set_next_event = meson6_clkevt_next_event, + .name = "meson6_tick", + .rating = 400, + .features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT, + .set_state_shutdown = meson6_shutdown, + .set_state_periodic = meson6_set_periodic, + .set_state_oneshot = meson6_set_oneshot, + .tick_resume = meson6_shutdown, + .set_next_event = meson6_clkevt_next_event, }; static irqreturn_t meson6_timer_interrupt(int irq, void *dev_id) diff --git a/drivers/clocksource/metag_generic.c b/drivers/clocksource/metag_generic.c index b7384b853e5a..bcd5c0d602a0 100644 --- a/drivers/clocksource/metag_generic.c +++ b/drivers/clocksource/metag_generic.c @@ -56,25 +56,6 @@ static int metag_timer_set_next_event(unsigned long delta, return 0; } -static void metag_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) -{ - switch (mode) { - case CLOCK_EVT_MODE_ONESHOT: - case CLOCK_EVT_MODE_RESUME: - break; - - case CLOCK_EVT_MODE_SHUTDOWN: - /* We should disable the IRQ here */ - break; - - case CLOCK_EVT_MODE_PERIODIC: - case CLOCK_EVT_MODE_UNUSED: - WARN_ON(1); - break; - }; -} - static cycle_t metag_clocksource_read(struct clocksource *cs) { return __core_reg_get(TXTIMER); @@ -129,7 +110,6 @@ static void arch_timer_setup(unsigned int cpu) clk->rating = 200, clk->shift = 12, clk->irq = tbisig_map(TBID_SIGNUM_TRT), - clk->set_mode = metag_timer_set_mode, clk->set_next_event = metag_timer_set_next_event, clk->mult = div_sc(hwtimer_freq, NSEC_PER_SEC, clk->shift); diff --git a/drivers/clocksource/mips-gic-timer.c b/drivers/clocksource/mips-gic-timer.c index b81ed1a5342d..02a1945e5093 100644 --- a/drivers/clocksource/mips-gic-timer.c +++ b/drivers/clocksource/mips-gic-timer.c @@ -33,12 +33,6 @@ static int gic_next_event(unsigned long delta, struct clock_event_device *evt) return res; } -static void gic_set_clock_mode(enum clock_event_mode mode, - struct clock_event_device *evt) -{ - /* Nothing to do ... */ -} - static irqreturn_t gic_compare_interrupt(int irq, void *dev_id) { struct clock_event_device *cd = dev_id; @@ -67,7 +61,6 @@ static void gic_clockevent_cpu_init(struct clock_event_device *cd) cd->irq = gic_timer_irq; cd->cpumask = cpumask_of(cpu); cd->set_next_event = gic_next_event; - cd->set_mode = gic_set_clock_mode; clockevents_config_and_register(cd, gic_frequency, 0x300, 0x7fffffff); @@ -79,6 +72,13 @@ static void gic_clockevent_cpu_exit(struct clock_event_device *cd) disable_percpu_irq(gic_timer_irq); } +static void gic_update_frequency(void *data) +{ + unsigned long rate = (unsigned long)data; + + clockevents_update_freq(this_cpu_ptr(&gic_clockevent_device), rate); +} + static int gic_cpu_notifier(struct notifier_block *nb, unsigned long action, void *data) { @@ -94,18 +94,40 @@ static int gic_cpu_notifier(struct notifier_block *nb, unsigned long action, return NOTIFY_OK; } +static int gic_clk_notifier(struct notifier_block *nb, unsigned long action, + void *data) +{ + struct clk_notifier_data *cnd = data; + + if (action == POST_RATE_CHANGE) + on_each_cpu(gic_update_frequency, (void *)cnd->new_rate, 1); + + return NOTIFY_OK; +} + + static struct notifier_block gic_cpu_nb = { .notifier_call = gic_cpu_notifier, }; +static struct notifier_block gic_clk_nb = { + .notifier_call = gic_clk_notifier, +}; + static int gic_clockevent_init(void) { + int ret; + if (!cpu_has_counter || !gic_frequency) return -ENXIO; - setup_percpu_irq(gic_timer_irq, &gic_compare_irqaction); + ret = setup_percpu_irq(gic_timer_irq, &gic_compare_irqaction); + if (ret < 0) + return ret; - register_cpu_notifier(&gic_cpu_nb); + ret = register_cpu_notifier(&gic_cpu_nb); + if (ret < 0) + pr_warn("GIC: Unable to register CPU notifier\n"); gic_clockevent_cpu_init(this_cpu_ptr(&gic_clockevent_device)); @@ -125,18 +147,17 @@ static struct clocksource gic_clocksource = { static void __init __gic_clocksource_init(void) { + int ret; + /* Set clocksource mask. */ gic_clocksource.mask = CLOCKSOURCE_MASK(gic_get_count_width()); /* Calculate a somewhat reasonable rating value. */ gic_clocksource.rating = 200 + gic_frequency / 10000000; - clocksource_register_hz(&gic_clocksource, gic_frequency); - - gic_clockevent_init(); - - /* And finally start the counter */ - gic_start_count(); + ret = clocksource_register_hz(&gic_clocksource, gic_frequency); + if (ret < 0) + pr_warn("GIC: Unable to register clocksource\n"); } void __init gic_clocksource_init(unsigned int frequency) @@ -146,11 +167,16 @@ void __init gic_clocksource_init(unsigned int frequency) GIC_LOCAL_TO_HWIRQ(GIC_LOCAL_INT_COMPARE); __gic_clocksource_init(); + gic_clockevent_init(); + + /* And finally start the counter */ + gic_start_count(); } static void __init gic_clocksource_of_init(struct device_node *node) { struct clk *clk; + int ret; if (WARN_ON(!gic_present || !node->parent || !of_device_is_compatible(node->parent, "mti,gic"))) @@ -158,8 +184,13 @@ static void __init gic_clocksource_of_init(struct device_node *node) clk = of_clk_get(node, 0); if (!IS_ERR(clk)) { + if (clk_prepare_enable(clk) < 0) { + pr_err("GIC failed to enable clock\n"); + clk_put(clk); + return; + } + gic_frequency = clk_get_rate(clk); - clk_put(clk); } else if (of_property_read_u32(node, "clock-frequency", &gic_frequency)) { pr_err("GIC frequency not specified.\n"); @@ -172,6 +203,15 @@ static void __init gic_clocksource_of_init(struct device_node *node) } __gic_clocksource_init(); + + ret = gic_clockevent_init(); + if (!ret && !IS_ERR(clk)) { + if (clk_notifier_register(clk, &gic_clk_nb) < 0) + pr_warn("GIC: Unable to register clock notifier\n"); + } + + /* And finally start the counter */ + gic_start_count(); } CLOCKSOURCE_OF_DECLARE(mips_gic_timer, "mti,gic-timer", gic_clocksource_of_init); diff --git a/drivers/clocksource/moxart_timer.c b/drivers/clocksource/moxart_timer.c index 5eb2c35932b1..19857af651c1 100644 --- a/drivers/clocksource/moxart_timer.c +++ b/drivers/clocksource/moxart_timer.c @@ -58,25 +58,24 @@ static void __iomem *base; static unsigned int clock_count_per_tick; -static void moxart_clkevt_mode(enum clock_event_mode mode, - struct clock_event_device *clk) +static int moxart_shutdown(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_RESUME: - case CLOCK_EVT_MODE_ONESHOT: - writel(TIMER1_DISABLE, base + TIMER_CR); - writel(~0, base + TIMER1_BASE + REG_LOAD); - break; - case CLOCK_EVT_MODE_PERIODIC: - writel(clock_count_per_tick, base + TIMER1_BASE + REG_LOAD); - writel(TIMER1_ENABLE, base + TIMER_CR); - break; - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - default: - writel(TIMER1_DISABLE, base + TIMER_CR); - break; - } + writel(TIMER1_DISABLE, base + TIMER_CR); + return 0; +} + +static int moxart_set_oneshot(struct clock_event_device *evt) +{ + writel(TIMER1_DISABLE, base + TIMER_CR); + writel(~0, base + TIMER1_BASE + REG_LOAD); + return 0; +} + +static int moxart_set_periodic(struct clock_event_device *evt) +{ + writel(clock_count_per_tick, base + TIMER1_BASE + REG_LOAD); + writel(TIMER1_ENABLE, base + TIMER_CR); + return 0; } static int moxart_clkevt_next_event(unsigned long cycles, @@ -95,11 +94,15 @@ static int moxart_clkevt_next_event(unsigned long cycles, } static struct clock_event_device moxart_clockevent = { - .name = "moxart_timer", - .rating = 200, - .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .set_mode = moxart_clkevt_mode, - .set_next_event = moxart_clkevt_next_event, + .name = "moxart_timer", + .rating = 200, + .features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT, + .set_state_shutdown = moxart_shutdown, + .set_state_periodic = moxart_set_periodic, + .set_state_oneshot = moxart_set_oneshot, + .tick_resume = moxart_set_oneshot, + .set_next_event = moxart_clkevt_next_event, }; static irqreturn_t moxart_timer_interrupt(int irq, void *dev_id) diff --git a/drivers/clocksource/mtk_timer.c b/drivers/clocksource/mtk_timer.c index 68ab42356d0e..50f0641c65b6 100644 --- a/drivers/clocksource/mtk_timer.c +++ b/drivers/clocksource/mtk_timer.c @@ -102,27 +102,20 @@ static void mtk_clkevt_time_start(struct mtk_clock_event_device *evt, evt->gpt_base + TIMER_CTRL_REG(timer)); } -static void mtk_clkevt_mode(enum clock_event_mode mode, - struct clock_event_device *clk) +static int mtk_clkevt_shutdown(struct clock_event_device *clk) +{ + mtk_clkevt_time_stop(to_mtk_clk(clk), GPT_CLK_EVT); + return 0; +} + +static int mtk_clkevt_set_periodic(struct clock_event_device *clk) { struct mtk_clock_event_device *evt = to_mtk_clk(clk); mtk_clkevt_time_stop(evt, GPT_CLK_EVT); - - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - mtk_clkevt_time_setup(evt, evt->ticks_per_jiffy, GPT_CLK_EVT); - mtk_clkevt_time_start(evt, true, GPT_CLK_EVT); - break; - case CLOCK_EVT_MODE_ONESHOT: - /* Timer is enabled in set_next_event */ - break; - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - default: - /* No more interrupts will occur as source is disabled */ - break; - } + mtk_clkevt_time_setup(evt, evt->ticks_per_jiffy, GPT_CLK_EVT); + mtk_clkevt_time_start(evt, true, GPT_CLK_EVT); + return 0; } static int mtk_clkevt_next_event(unsigned long event, @@ -196,7 +189,10 @@ static void __init mtk_timer_init(struct device_node *node) evt->dev.name = "mtk_tick"; evt->dev.rating = 300; evt->dev.features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; - evt->dev.set_mode = mtk_clkevt_mode; + evt->dev.set_state_shutdown = mtk_clkevt_shutdown; + evt->dev.set_state_periodic = mtk_clkevt_set_periodic; + evt->dev.set_state_oneshot = mtk_clkevt_shutdown; + evt->dev.tick_resume = mtk_clkevt_shutdown; evt->dev.set_next_event = mtk_clkevt_next_event; evt->dev.cpumask = cpu_possible_mask; diff --git a/drivers/clocksource/mxs_timer.c b/drivers/clocksource/mxs_timer.c index 445b68a01dc5..f5ce2961c0d6 100644 --- a/drivers/clocksource/mxs_timer.c +++ b/drivers/clocksource/mxs_timer.c @@ -77,7 +77,6 @@ #define BV_TIMROTv2_TIMCTRLn_SELECT__TICK_ALWAYS 0xf static struct clock_event_device mxs_clockevent_device; -static enum clock_event_mode mxs_clockevent_mode = CLOCK_EVT_MODE_UNUSED; static void __iomem *mxs_timrot_base; static u32 timrot_major_version; @@ -141,64 +140,49 @@ static struct irqaction mxs_timer_irq = { .handler = mxs_timer_interrupt, }; -#ifdef DEBUG -static const char *clock_event_mode_label[] const = { - [CLOCK_EVT_MODE_PERIODIC] = "CLOCK_EVT_MODE_PERIODIC", - [CLOCK_EVT_MODE_ONESHOT] = "CLOCK_EVT_MODE_ONESHOT", - [CLOCK_EVT_MODE_SHUTDOWN] = "CLOCK_EVT_MODE_SHUTDOWN", - [CLOCK_EVT_MODE_UNUSED] = "CLOCK_EVT_MODE_UNUSED" -}; -#endif /* DEBUG */ - -static void mxs_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static void mxs_irq_clear(char *state) { /* Disable interrupt in timer module */ timrot_irq_disable(); - if (mode != mxs_clockevent_mode) { - /* Set event time into the furthest future */ - if (timrot_is_v1()) - __raw_writel(0xffff, - mxs_timrot_base + HW_TIMROT_TIMCOUNTn(1)); - else - __raw_writel(0xffffffff, - mxs_timrot_base + HW_TIMROT_FIXED_COUNTn(1)); + /* Set event time into the furthest future */ + if (timrot_is_v1()) + __raw_writel(0xffff, mxs_timrot_base + HW_TIMROT_TIMCOUNTn(1)); + else + __raw_writel(0xffffffff, + mxs_timrot_base + HW_TIMROT_FIXED_COUNTn(1)); - /* Clear pending interrupt */ - timrot_irq_acknowledge(); - } + /* Clear pending interrupt */ + timrot_irq_acknowledge(); #ifdef DEBUG - pr_info("%s: changing mode from %s to %s\n", __func__, - clock_event_mode_label[mxs_clockevent_mode], - clock_event_mode_label[mode]); + pr_info("%s: changing mode to %s\n", __func__, state) #endif /* DEBUG */ +} - /* Remember timer mode */ - mxs_clockevent_mode = mode; +static int mxs_shutdown(struct clock_event_device *evt) +{ + mxs_irq_clear("shutdown"); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - pr_err("%s: Periodic mode is not implemented\n", __func__); - break; - case CLOCK_EVT_MODE_ONESHOT: - timrot_irq_enable(); - break; - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_RESUME: - /* Left event sources disabled, no more interrupts appear */ - break; - } + return 0; +} + +static int mxs_set_oneshot(struct clock_event_device *evt) +{ + if (clockevent_state_oneshot(evt)) + mxs_irq_clear("oneshot"); + timrot_irq_enable(); + return 0; } static struct clock_event_device mxs_clockevent_device = { - .name = "mxs_timrot", - .features = CLOCK_EVT_FEAT_ONESHOT, - .set_mode = mxs_set_mode, - .set_next_event = timrotv2_set_next_event, - .rating = 200, + .name = "mxs_timrot", + .features = CLOCK_EVT_FEAT_ONESHOT, + .set_state_shutdown = mxs_shutdown, + .set_state_oneshot = mxs_set_oneshot, + .tick_resume = mxs_shutdown, + .set_next_event = timrotv2_set_next_event, + .rating = 200, }; static int __init mxs_clockevent_init(struct clk *timer_clk) diff --git a/drivers/clocksource/nomadik-mtu.c b/drivers/clocksource/nomadik-mtu.c index a709cfa49d85..bc8dd443c727 100644 --- a/drivers/clocksource/nomadik-mtu.c +++ b/drivers/clocksource/nomadik-mtu.c @@ -119,28 +119,27 @@ static void nmdk_clkevt_reset(void) } } -static void nmdk_clkevt_mode(enum clock_event_mode mode, - struct clock_event_device *dev) +static int nmdk_clkevt_shutdown(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - clkevt_periodic = true; - nmdk_clkevt_reset(); - break; - case CLOCK_EVT_MODE_ONESHOT: - clkevt_periodic = false; - break; - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - writel(0, mtu_base + MTU_IMSC); - /* disable timer */ - writel(0, mtu_base + MTU_CR(1)); - /* load some high default value */ - writel(0xffffffff, mtu_base + MTU_LR(1)); - break; - case CLOCK_EVT_MODE_RESUME: - break; - } + writel(0, mtu_base + MTU_IMSC); + /* disable timer */ + writel(0, mtu_base + MTU_CR(1)); + /* load some high default value */ + writel(0xffffffff, mtu_base + MTU_LR(1)); + return 0; +} + +static int nmdk_clkevt_set_oneshot(struct clock_event_device *evt) +{ + clkevt_periodic = false; + return 0; +} + +static int nmdk_clkevt_set_periodic(struct clock_event_device *evt) +{ + clkevt_periodic = true; + nmdk_clkevt_reset(); + return 0; } static void nmdk_clksrc_reset(void) @@ -163,13 +162,16 @@ static void nmdk_clkevt_resume(struct clock_event_device *cedev) } static struct clock_event_device nmdk_clkevt = { - .name = "mtu_1", - .features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_PERIODIC | - CLOCK_EVT_FEAT_DYNIRQ, - .rating = 200, - .set_mode = nmdk_clkevt_mode, - .set_next_event = nmdk_clkevt_next, - .resume = nmdk_clkevt_resume, + .name = "mtu_1", + .features = CLOCK_EVT_FEAT_ONESHOT | + CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_DYNIRQ, + .rating = 200, + .set_state_shutdown = nmdk_clkevt_shutdown, + .set_state_periodic = nmdk_clkevt_set_periodic, + .set_state_oneshot = nmdk_clkevt_set_oneshot, + .set_next_event = nmdk_clkevt_next, + .resume = nmdk_clkevt_resume, }; /* diff --git a/drivers/clocksource/pxa_timer.c b/drivers/clocksource/pxa_timer.c index d9438af2bbd6..45b6a4999713 100644 --- a/drivers/clocksource/pxa_timer.c +++ b/drivers/clocksource/pxa_timer.c @@ -88,26 +88,12 @@ pxa_osmr0_set_next_event(unsigned long delta, struct clock_event_device *dev) return (signed)(next - oscr) <= MIN_OSCR_DELTA ? -ETIME : 0; } -static void -pxa_osmr0_set_mode(enum clock_event_mode mode, struct clock_event_device *dev) +static int pxa_osmr0_shutdown(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_ONESHOT: - timer_writel(timer_readl(OIER) & ~OIER_E0, OIER); - timer_writel(OSSR_M0, OSSR); - break; - - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - /* initializing, released, or preparing for suspend */ - timer_writel(timer_readl(OIER) & ~OIER_E0, OIER); - timer_writel(OSSR_M0, OSSR); - break; - - case CLOCK_EVT_MODE_RESUME: - case CLOCK_EVT_MODE_PERIODIC: - break; - } + /* initializing, released, or preparing for suspend */ + timer_writel(timer_readl(OIER) & ~OIER_E0, OIER); + timer_writel(OSSR_M0, OSSR); + return 0; } #ifdef CONFIG_PM @@ -147,13 +133,14 @@ static void pxa_timer_resume(struct clock_event_device *cedev) #endif static struct clock_event_device ckevt_pxa_osmr0 = { - .name = "osmr0", - .features = CLOCK_EVT_FEAT_ONESHOT, - .rating = 200, - .set_next_event = pxa_osmr0_set_next_event, - .set_mode = pxa_osmr0_set_mode, - .suspend = pxa_timer_suspend, - .resume = pxa_timer_resume, + .name = "osmr0", + .features = CLOCK_EVT_FEAT_ONESHOT, + .rating = 200, + .set_next_event = pxa_osmr0_set_next_event, + .set_state_shutdown = pxa_osmr0_shutdown, + .set_state_oneshot = pxa_osmr0_shutdown, + .suspend = pxa_timer_suspend, + .resume = pxa_timer_resume, }; static struct irqaction pxa_ost0_irq = { diff --git a/drivers/clocksource/qcom-timer.c b/drivers/clocksource/qcom-timer.c index cba2d015564c..f8e09f923651 100644 --- a/drivers/clocksource/qcom-timer.c +++ b/drivers/clocksource/qcom-timer.c @@ -47,7 +47,7 @@ static irqreturn_t msm_timer_interrupt(int irq, void *dev_id) { struct clock_event_device *evt = dev_id; /* Stop the timer tick */ - if (evt->mode == CLOCK_EVT_MODE_ONESHOT) { + if (clockevent_state_oneshot(evt)) { u32 ctrl = readl_relaxed(event_base + TIMER_ENABLE); ctrl &= ~TIMER_ENABLE_EN; writel_relaxed(ctrl, event_base + TIMER_ENABLE); @@ -75,26 +75,14 @@ static int msm_timer_set_next_event(unsigned long cycles, return 0; } -static void msm_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int msm_timer_shutdown(struct clock_event_device *evt) { u32 ctrl; ctrl = readl_relaxed(event_base + TIMER_ENABLE); ctrl &= ~(TIMER_ENABLE_EN | TIMER_ENABLE_CLR_ON_MATCH_EN); - - switch (mode) { - case CLOCK_EVT_MODE_RESUME: - case CLOCK_EVT_MODE_PERIODIC: - break; - case CLOCK_EVT_MODE_ONESHOT: - /* Timer is enabled in set_next_event */ - break; - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - break; - } writel_relaxed(ctrl, event_base + TIMER_ENABLE); + return 0; } static struct clock_event_device __percpu *msm_evt; @@ -126,7 +114,9 @@ static int msm_local_timer_setup(struct clock_event_device *evt) evt->name = "msm_timer"; evt->features = CLOCK_EVT_FEAT_ONESHOT; evt->rating = 200; - evt->set_mode = msm_timer_set_mode; + evt->set_state_shutdown = msm_timer_shutdown; + evt->set_state_oneshot = msm_timer_shutdown; + evt->tick_resume = msm_timer_shutdown; evt->set_next_event = msm_timer_set_next_event; evt->cpumask = cpumask_of(cpu); @@ -147,7 +137,7 @@ static int msm_local_timer_setup(struct clock_event_device *evt) static void msm_local_timer_stop(struct clock_event_device *evt) { - evt->set_mode(CLOCK_EVT_MODE_UNUSED, evt); + evt->set_state_shutdown(evt); disable_percpu_irq(evt->irq); } diff --git a/drivers/clocksource/rockchip_timer.c b/drivers/clocksource/rockchip_timer.c index a35993bafb20..bb2c2b050964 100644 --- a/drivers/clocksource/rockchip_timer.c +++ b/drivers/clocksource/rockchip_timer.c @@ -82,23 +82,18 @@ static inline int rk_timer_set_next_event(unsigned long cycles, return 0; } -static inline void rk_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *ce) +static int rk_timer_shutdown(struct clock_event_device *ce) { - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - rk_timer_disable(ce); - rk_timer_update_counter(rk_timer(ce)->freq / HZ - 1, ce); - rk_timer_enable(ce, TIMER_MODE_FREE_RUNNING); - break; - case CLOCK_EVT_MODE_ONESHOT: - case CLOCK_EVT_MODE_RESUME: - break; - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - rk_timer_disable(ce); - break; - } + rk_timer_disable(ce); + return 0; +} + +static int rk_timer_set_periodic(struct clock_event_device *ce) +{ + rk_timer_disable(ce); + rk_timer_update_counter(rk_timer(ce)->freq / HZ - 1, ce); + rk_timer_enable(ce, TIMER_MODE_FREE_RUNNING); + return 0; } static irqreturn_t rk_timer_interrupt(int irq, void *dev_id) @@ -107,7 +102,7 @@ static irqreturn_t rk_timer_interrupt(int irq, void *dev_id) rk_timer_interrupt_clear(ce); - if (ce->mode == CLOCK_EVT_MODE_ONESHOT) + if (clockevent_state_oneshot(ce)) rk_timer_disable(ce); ce->event_handler(ce); @@ -161,7 +156,8 @@ static void __init rk_timer_init(struct device_node *np) ce->name = TIMER_NAME; ce->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; ce->set_next_event = rk_timer_set_next_event; - ce->set_mode = rk_timer_set_mode; + ce->set_state_shutdown = rk_timer_shutdown; + ce->set_state_periodic = rk_timer_set_periodic; ce->irq = irq; ce->cpumask = cpumask_of(0); ce->rating = 250; diff --git a/drivers/clocksource/samsung_pwm_timer.c b/drivers/clocksource/samsung_pwm_timer.c index 5645cfc90c41..bc90e13338cc 100644 --- a/drivers/clocksource/samsung_pwm_timer.c +++ b/drivers/clocksource/samsung_pwm_timer.c @@ -207,25 +207,18 @@ static int samsung_set_next_event(unsigned long cycles, return 0; } -static void samsung_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int samsung_shutdown(struct clock_event_device *evt) { samsung_time_stop(pwm.event_id); + return 0; +} - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - samsung_time_setup(pwm.event_id, pwm.clock_count_per_tick - 1); - samsung_time_start(pwm.event_id, true); - break; - - case CLOCK_EVT_MODE_ONESHOT: - break; - - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_RESUME: - break; - } +static int samsung_set_periodic(struct clock_event_device *evt) +{ + samsung_time_stop(pwm.event_id); + samsung_time_setup(pwm.event_id, pwm.clock_count_per_tick - 1); + samsung_time_start(pwm.event_id, true); + return 0; } static void samsung_clockevent_resume(struct clock_event_device *cev) @@ -240,12 +233,16 @@ static void samsung_clockevent_resume(struct clock_event_device *cev) } static struct clock_event_device time_event_device = { - .name = "samsung_event_timer", - .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .rating = 200, - .set_next_event = samsung_set_next_event, - .set_mode = samsung_set_mode, - .resume = samsung_clockevent_resume, + .name = "samsung_event_timer", + .features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT, + .rating = 200, + .set_next_event = samsung_set_next_event, + .set_state_shutdown = samsung_shutdown, + .set_state_periodic = samsung_set_periodic, + .set_state_oneshot = samsung_shutdown, + .tick_resume = samsung_shutdown, + .resume = samsung_clockevent_resume, }; static irqreturn_t samsung_clock_event_isr(int irq, void *dev_id) diff --git a/drivers/clocksource/sh_cmt.c b/drivers/clocksource/sh_cmt.c index c96de14036a0..ba73a6eb8d66 100644 --- a/drivers/clocksource/sh_cmt.c +++ b/drivers/clocksource/sh_cmt.c @@ -538,7 +538,7 @@ static irqreturn_t sh_cmt_interrupt(int irq, void *dev_id) if (ch->flags & FLAG_CLOCKEVENT) { if (!(ch->flags & FLAG_SKIPEVENT)) { - if (ch->ced.mode == CLOCK_EVT_MODE_ONESHOT) { + if (clockevent_state_oneshot(&ch->ced)) { ch->next_match_value = ch->max_match_value; ch->flags |= FLAG_REPROGRAM; } @@ -554,7 +554,7 @@ static irqreturn_t sh_cmt_interrupt(int irq, void *dev_id) sh_cmt_clock_event_program_verify(ch, 1); if (ch->flags & FLAG_CLOCKEVENT) - if ((ch->ced.mode == CLOCK_EVT_MODE_SHUTDOWN) + if ((clockevent_state_shutdown(&ch->ced)) || (ch->match_value == ch->next_match_value)) ch->flags &= ~FLAG_REPROGRAM; } @@ -726,39 +726,37 @@ static void sh_cmt_clock_event_start(struct sh_cmt_channel *ch, int periodic) sh_cmt_set_next(ch, ch->max_match_value); } -static void sh_cmt_clock_event_mode(enum clock_event_mode mode, - struct clock_event_device *ced) +static int sh_cmt_clock_event_shutdown(struct clock_event_device *ced) +{ + struct sh_cmt_channel *ch = ced_to_sh_cmt(ced); + + sh_cmt_stop(ch, FLAG_CLOCKEVENT); + return 0; +} + +static int sh_cmt_clock_event_set_state(struct clock_event_device *ced, + int periodic) { struct sh_cmt_channel *ch = ced_to_sh_cmt(ced); /* deal with old setting first */ - switch (ced->mode) { - case CLOCK_EVT_MODE_PERIODIC: - case CLOCK_EVT_MODE_ONESHOT: + if (clockevent_state_oneshot(ced) || clockevent_state_periodic(ced)) sh_cmt_stop(ch, FLAG_CLOCKEVENT); - break; - default: - break; - } - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - dev_info(&ch->cmt->pdev->dev, - "ch%u: used for periodic clock events\n", ch->index); - sh_cmt_clock_event_start(ch, 1); - break; - case CLOCK_EVT_MODE_ONESHOT: - dev_info(&ch->cmt->pdev->dev, - "ch%u: used for oneshot clock events\n", ch->index); - sh_cmt_clock_event_start(ch, 0); - break; - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - sh_cmt_stop(ch, FLAG_CLOCKEVENT); - break; - default: - break; - } + dev_info(&ch->cmt->pdev->dev, "ch%u: used for %s clock events\n", + ch->index, periodic ? "periodic" : "oneshot"); + sh_cmt_clock_event_start(ch, periodic); + return 0; +} + +static int sh_cmt_clock_event_set_oneshot(struct clock_event_device *ced) +{ + return sh_cmt_clock_event_set_state(ced, 0); +} + +static int sh_cmt_clock_event_set_periodic(struct clock_event_device *ced) +{ + return sh_cmt_clock_event_set_state(ced, 1); } static int sh_cmt_clock_event_next(unsigned long delta, @@ -766,7 +764,7 @@ static int sh_cmt_clock_event_next(unsigned long delta, { struct sh_cmt_channel *ch = ced_to_sh_cmt(ced); - BUG_ON(ced->mode != CLOCK_EVT_MODE_ONESHOT); + BUG_ON(!clockevent_state_oneshot(ced)); if (likely(ch->flags & FLAG_IRQCONTEXT)) ch->next_match_value = delta - 1; else @@ -820,7 +818,9 @@ static int sh_cmt_register_clockevent(struct sh_cmt_channel *ch, ced->rating = 125; ced->cpumask = cpu_possible_mask; ced->set_next_event = sh_cmt_clock_event_next; - ced->set_mode = sh_cmt_clock_event_mode; + ced->set_state_shutdown = sh_cmt_clock_event_shutdown; + ced->set_state_periodic = sh_cmt_clock_event_set_periodic; + ced->set_state_oneshot = sh_cmt_clock_event_set_oneshot; ced->suspend = sh_cmt_clock_event_suspend; ced->resume = sh_cmt_clock_event_resume; @@ -935,9 +935,6 @@ static int sh_cmt_map_memory(struct sh_cmt_device *cmt) static const struct platform_device_id sh_cmt_id_table[] = { { "sh-cmt-16", (kernel_ulong_t)&sh_cmt_info[SH_CMT_16BIT] }, { "sh-cmt-32", (kernel_ulong_t)&sh_cmt_info[SH_CMT_32BIT] }, - { "sh-cmt-32-fast", (kernel_ulong_t)&sh_cmt_info[SH_CMT_32BIT_FAST] }, - { "sh-cmt-48", (kernel_ulong_t)&sh_cmt_info[SH_CMT_48BIT] }, - { "sh-cmt-48-gen2", (kernel_ulong_t)&sh_cmt_info[SH_CMT_48BIT_GEN2] }, { } }; MODULE_DEVICE_TABLE(platform, sh_cmt_id_table); diff --git a/drivers/clocksource/sh_mtu2.c b/drivers/clocksource/sh_mtu2.c index 3d88698cf2b8..f1985da8113f 100644 --- a/drivers/clocksource/sh_mtu2.c +++ b/drivers/clocksource/sh_mtu2.c @@ -276,36 +276,25 @@ static struct sh_mtu2_channel *ced_to_sh_mtu2(struct clock_event_device *ced) return container_of(ced, struct sh_mtu2_channel, ced); } -static void sh_mtu2_clock_event_mode(enum clock_event_mode mode, - struct clock_event_device *ced) +static int sh_mtu2_clock_event_shutdown(struct clock_event_device *ced) { struct sh_mtu2_channel *ch = ced_to_sh_mtu2(ced); - int disabled = 0; - /* deal with old setting first */ - switch (ced->mode) { - case CLOCK_EVT_MODE_PERIODIC: + sh_mtu2_disable(ch); + return 0; +} + +static int sh_mtu2_clock_event_set_periodic(struct clock_event_device *ced) +{ + struct sh_mtu2_channel *ch = ced_to_sh_mtu2(ced); + + if (clockevent_state_periodic(ced)) sh_mtu2_disable(ch); - disabled = 1; - break; - default: - break; - } - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - dev_info(&ch->mtu->pdev->dev, - "ch%u: used for periodic clock events\n", ch->index); - sh_mtu2_enable(ch); - break; - case CLOCK_EVT_MODE_UNUSED: - if (!disabled) - sh_mtu2_disable(ch); - break; - case CLOCK_EVT_MODE_SHUTDOWN: - default: - break; - } + dev_info(&ch->mtu->pdev->dev, "ch%u: used for periodic clock events\n", + ch->index); + sh_mtu2_enable(ch); + return 0; } static void sh_mtu2_clock_event_suspend(struct clock_event_device *ced) @@ -327,7 +316,8 @@ static void sh_mtu2_register_clockevent(struct sh_mtu2_channel *ch, ced->features = CLOCK_EVT_FEAT_PERIODIC; ced->rating = 200; ced->cpumask = cpu_possible_mask; - ced->set_mode = sh_mtu2_clock_event_mode; + ced->set_state_shutdown = sh_mtu2_clock_event_shutdown; + ced->set_state_periodic = sh_mtu2_clock_event_set_periodic; ced->suspend = sh_mtu2_clock_event_suspend; ced->resume = sh_mtu2_clock_event_resume; diff --git a/drivers/clocksource/sh_tmu.c b/drivers/clocksource/sh_tmu.c index b6b8fa3cd211..469e776ec17a 100644 --- a/drivers/clocksource/sh_tmu.c +++ b/drivers/clocksource/sh_tmu.c @@ -240,7 +240,7 @@ static irqreturn_t sh_tmu_interrupt(int irq, void *dev_id) struct sh_tmu_channel *ch = dev_id; /* disable or acknowledge interrupt */ - if (ch->ced.mode == CLOCK_EVT_MODE_ONESHOT) + if (clockevent_state_oneshot(&ch->ced)) sh_tmu_write(ch, TCR, TCR_TPSC_CLK4); else sh_tmu_write(ch, TCR, TCR_UNIE | TCR_TPSC_CLK4); @@ -358,42 +358,38 @@ static void sh_tmu_clock_event_start(struct sh_tmu_channel *ch, int periodic) } } -static void sh_tmu_clock_event_mode(enum clock_event_mode mode, - struct clock_event_device *ced) +static int sh_tmu_clock_event_shutdown(struct clock_event_device *ced) +{ + struct sh_tmu_channel *ch = ced_to_sh_tmu(ced); + + if (clockevent_state_oneshot(ced) || clockevent_state_periodic(ced)) + sh_tmu_disable(ch); + return 0; +} + +static int sh_tmu_clock_event_set_state(struct clock_event_device *ced, + int periodic) { struct sh_tmu_channel *ch = ced_to_sh_tmu(ced); - int disabled = 0; /* deal with old setting first */ - switch (ced->mode) { - case CLOCK_EVT_MODE_PERIODIC: - case CLOCK_EVT_MODE_ONESHOT: + if (clockevent_state_oneshot(ced) || clockevent_state_periodic(ced)) sh_tmu_disable(ch); - disabled = 1; - break; - default: - break; - } - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - dev_info(&ch->tmu->pdev->dev, - "ch%u: used for periodic clock events\n", ch->index); - sh_tmu_clock_event_start(ch, 1); - break; - case CLOCK_EVT_MODE_ONESHOT: - dev_info(&ch->tmu->pdev->dev, - "ch%u: used for oneshot clock events\n", ch->index); - sh_tmu_clock_event_start(ch, 0); - break; - case CLOCK_EVT_MODE_UNUSED: - if (!disabled) - sh_tmu_disable(ch); - break; - case CLOCK_EVT_MODE_SHUTDOWN: - default: - break; - } + dev_info(&ch->tmu->pdev->dev, "ch%u: used for %s clock events\n", + ch->index, periodic ? "periodic" : "oneshot"); + sh_tmu_clock_event_start(ch, periodic); + return 0; +} + +static int sh_tmu_clock_event_set_oneshot(struct clock_event_device *ced) +{ + return sh_tmu_clock_event_set_state(ced, 0); +} + +static int sh_tmu_clock_event_set_periodic(struct clock_event_device *ced) +{ + return sh_tmu_clock_event_set_state(ced, 1); } static int sh_tmu_clock_event_next(unsigned long delta, @@ -401,7 +397,7 @@ static int sh_tmu_clock_event_next(unsigned long delta, { struct sh_tmu_channel *ch = ced_to_sh_tmu(ced); - BUG_ON(ced->mode != CLOCK_EVT_MODE_ONESHOT); + BUG_ON(!clockevent_state_oneshot(ced)); /* program new delta value */ sh_tmu_set_next(ch, delta, 0); @@ -430,7 +426,9 @@ static void sh_tmu_register_clockevent(struct sh_tmu_channel *ch, ced->rating = 200; ced->cpumask = cpu_possible_mask; ced->set_next_event = sh_tmu_clock_event_next; - ced->set_mode = sh_tmu_clock_event_mode; + ced->set_state_shutdown = sh_tmu_clock_event_shutdown; + ced->set_state_periodic = sh_tmu_clock_event_set_periodic; + ced->set_state_oneshot = sh_tmu_clock_event_set_oneshot; ced->suspend = sh_tmu_clock_event_suspend; ced->resume = sh_tmu_clock_event_resume; diff --git a/drivers/clocksource/sun4i_timer.c b/drivers/clocksource/sun4i_timer.c index 1928a8912584..6f3719d73390 100644 --- a/drivers/clocksource/sun4i_timer.c +++ b/drivers/clocksource/sun4i_timer.c @@ -81,25 +81,25 @@ static void sun4i_clkevt_time_start(u8 timer, bool periodic) timer_base + TIMER_CTL_REG(timer)); } -static void sun4i_clkevt_mode(enum clock_event_mode mode, - struct clock_event_device *clk) +static int sun4i_clkevt_shutdown(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - sun4i_clkevt_time_stop(0); - sun4i_clkevt_time_setup(0, ticks_per_jiffy); - sun4i_clkevt_time_start(0, true); - break; - case CLOCK_EVT_MODE_ONESHOT: - sun4i_clkevt_time_stop(0); - sun4i_clkevt_time_start(0, false); - break; - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - default: - sun4i_clkevt_time_stop(0); - break; - } + sun4i_clkevt_time_stop(0); + return 0; +} + +static int sun4i_clkevt_set_oneshot(struct clock_event_device *evt) +{ + sun4i_clkevt_time_stop(0); + sun4i_clkevt_time_start(0, false); + return 0; +} + +static int sun4i_clkevt_set_periodic(struct clock_event_device *evt) +{ + sun4i_clkevt_time_stop(0); + sun4i_clkevt_time_setup(0, ticks_per_jiffy); + sun4i_clkevt_time_start(0, true); + return 0; } static int sun4i_clkevt_next_event(unsigned long evt, @@ -116,7 +116,10 @@ static struct clock_event_device sun4i_clockevent = { .name = "sun4i_tick", .rating = 350, .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .set_mode = sun4i_clkevt_mode, + .set_state_shutdown = sun4i_clkevt_shutdown, + .set_state_periodic = sun4i_clkevt_set_periodic, + .set_state_oneshot = sun4i_clkevt_set_oneshot, + .tick_resume = sun4i_clkevt_shutdown, .set_next_event = sun4i_clkevt_next_event, }; diff --git a/drivers/clocksource/tcb_clksrc.c b/drivers/clocksource/tcb_clksrc.c index 8bdbc45c6dad..d28d2fe798d5 100644 --- a/drivers/clocksource/tcb_clksrc.c +++ b/drivers/clocksource/tcb_clksrc.c @@ -91,55 +91,62 @@ static struct tc_clkevt_device *to_tc_clkevt(struct clock_event_device *clkevt) */ static u32 timer_clock; -static void tc_mode(enum clock_event_mode m, struct clock_event_device *d) +static int tc_shutdown(struct clock_event_device *d) { struct tc_clkevt_device *tcd = to_tc_clkevt(d); void __iomem *regs = tcd->regs; - if (tcd->clkevt.mode == CLOCK_EVT_MODE_PERIODIC - || tcd->clkevt.mode == CLOCK_EVT_MODE_ONESHOT) { - __raw_writel(0xff, regs + ATMEL_TC_REG(2, IDR)); - __raw_writel(ATMEL_TC_CLKDIS, regs + ATMEL_TC_REG(2, CCR)); - clk_disable(tcd->clk); - } + __raw_writel(0xff, regs + ATMEL_TC_REG(2, IDR)); + __raw_writel(ATMEL_TC_CLKDIS, regs + ATMEL_TC_REG(2, CCR)); + clk_disable(tcd->clk); - switch (m) { + return 0; +} + +static int tc_set_oneshot(struct clock_event_device *d) +{ + struct tc_clkevt_device *tcd = to_tc_clkevt(d); + void __iomem *regs = tcd->regs; + + if (clockevent_state_oneshot(d) || clockevent_state_periodic(d)) + tc_shutdown(d); + + clk_enable(tcd->clk); + + /* slow clock, count up to RC, then irq and stop */ + __raw_writel(timer_clock | ATMEL_TC_CPCSTOP | ATMEL_TC_WAVE | + ATMEL_TC_WAVESEL_UP_AUTO, regs + ATMEL_TC_REG(2, CMR)); + __raw_writel(ATMEL_TC_CPCS, regs + ATMEL_TC_REG(2, IER)); + + /* set_next_event() configures and starts the timer */ + return 0; +} + +static int tc_set_periodic(struct clock_event_device *d) +{ + struct tc_clkevt_device *tcd = to_tc_clkevt(d); + void __iomem *regs = tcd->regs; + + if (clockevent_state_oneshot(d) || clockevent_state_periodic(d)) + tc_shutdown(d); /* By not making the gentime core emulate periodic mode on top * of oneshot, we get lower overhead and improved accuracy. */ - case CLOCK_EVT_MODE_PERIODIC: - clk_enable(tcd->clk); + clk_enable(tcd->clk); - /* slow clock, count up to RC, then irq and restart */ - __raw_writel(timer_clock - | ATMEL_TC_WAVE | ATMEL_TC_WAVESEL_UP_AUTO, - regs + ATMEL_TC_REG(2, CMR)); - __raw_writel((32768 + HZ/2) / HZ, tcaddr + ATMEL_TC_REG(2, RC)); + /* slow clock, count up to RC, then irq and restart */ + __raw_writel(timer_clock | ATMEL_TC_WAVE | ATMEL_TC_WAVESEL_UP_AUTO, + regs + ATMEL_TC_REG(2, CMR)); + __raw_writel((32768 + HZ / 2) / HZ, tcaddr + ATMEL_TC_REG(2, RC)); - /* Enable clock and interrupts on RC compare */ - __raw_writel(ATMEL_TC_CPCS, regs + ATMEL_TC_REG(2, IER)); + /* Enable clock and interrupts on RC compare */ + __raw_writel(ATMEL_TC_CPCS, regs + ATMEL_TC_REG(2, IER)); - /* go go gadget! */ - __raw_writel(ATMEL_TC_CLKEN | ATMEL_TC_SWTRG, - regs + ATMEL_TC_REG(2, CCR)); - break; - - case CLOCK_EVT_MODE_ONESHOT: - clk_enable(tcd->clk); - - /* slow clock, count up to RC, then irq and stop */ - __raw_writel(timer_clock | ATMEL_TC_CPCSTOP - | ATMEL_TC_WAVE | ATMEL_TC_WAVESEL_UP_AUTO, - regs + ATMEL_TC_REG(2, CMR)); - __raw_writel(ATMEL_TC_CPCS, regs + ATMEL_TC_REG(2, IER)); - - /* set_next_event() configures and starts the timer */ - break; - - default: - break; - } + /* go go gadget! */ + __raw_writel(ATMEL_TC_CLKEN | ATMEL_TC_SWTRG, regs + + ATMEL_TC_REG(2, CCR)); + return 0; } static int tc_next_event(unsigned long delta, struct clock_event_device *d) @@ -154,13 +161,15 @@ static int tc_next_event(unsigned long delta, struct clock_event_device *d) static struct tc_clkevt_device clkevt = { .clkevt = { - .name = "tc_clkevt", - .features = CLOCK_EVT_FEAT_PERIODIC - | CLOCK_EVT_FEAT_ONESHOT, + .name = "tc_clkevt", + .features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT, /* Should be lower than at91rm9200's system timer */ - .rating = 125, - .set_next_event = tc_next_event, - .set_mode = tc_mode, + .rating = 125, + .set_next_event = tc_next_event, + .set_state_shutdown = tc_shutdown, + .set_state_periodic = tc_set_periodic, + .set_state_oneshot = tc_set_oneshot, }, }; diff --git a/drivers/clocksource/tegra20_timer.c b/drivers/clocksource/tegra20_timer.c index 5a112d72fc2d..6ebda1177e79 100644 --- a/drivers/clocksource/tegra20_timer.c +++ b/drivers/clocksource/tegra20_timer.c @@ -72,33 +72,36 @@ static int tegra_timer_set_next_event(unsigned long cycles, return 0; } -static void tegra_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static inline void timer_shutdown(struct clock_event_device *evt) { - u32 reg; - timer_writel(0, TIMER3_BASE + TIMER_PTV); +} - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - reg = 0xC0000000 | ((1000000/HZ)-1); - timer_writel(reg, TIMER3_BASE + TIMER_PTV); - break; - case CLOCK_EVT_MODE_ONESHOT: - break; - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_RESUME: - break; - } +static int tegra_timer_shutdown(struct clock_event_device *evt) +{ + timer_shutdown(evt); + return 0; +} + +static int tegra_timer_set_periodic(struct clock_event_device *evt) +{ + u32 reg = 0xC0000000 | ((1000000 / HZ) - 1); + + timer_shutdown(evt); + timer_writel(reg, TIMER3_BASE + TIMER_PTV); + return 0; } static struct clock_event_device tegra_clockevent = { - .name = "timer0", - .rating = 300, - .features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_PERIODIC, - .set_next_event = tegra_timer_set_next_event, - .set_mode = tegra_timer_set_mode, + .name = "timer0", + .rating = 300, + .features = CLOCK_EVT_FEAT_ONESHOT | + CLOCK_EVT_FEAT_PERIODIC, + .set_next_event = tegra_timer_set_next_event, + .set_state_shutdown = tegra_timer_shutdown, + .set_state_periodic = tegra_timer_set_periodic, + .set_state_oneshot = tegra_timer_shutdown, + .tick_resume = tegra_timer_shutdown, }; static u64 notrace tegra_read_sched_clock(void) diff --git a/drivers/clocksource/time-armada-370-xp.c b/drivers/clocksource/time-armada-370-xp.c index 0c8c5e337540..2162796fd504 100644 --- a/drivers/clocksource/time-armada-370-xp.c +++ b/drivers/clocksource/time-armada-370-xp.c @@ -121,33 +121,33 @@ armada_370_xp_clkevt_next_event(unsigned long delta, return 0; } -static void -armada_370_xp_clkevt_mode(enum clock_event_mode mode, - struct clock_event_device *dev) +static int armada_370_xp_clkevt_shutdown(struct clock_event_device *evt) { - if (mode == CLOCK_EVT_MODE_PERIODIC) { + /* + * Disable timer. + */ + local_timer_ctrl_clrset(TIMER0_EN, 0); - /* - * Setup timer to fire at 1/HZ intervals. - */ - writel(ticks_per_jiffy - 1, local_base + TIMER0_RELOAD_OFF); - writel(ticks_per_jiffy - 1, local_base + TIMER0_VAL_OFF); + /* + * ACK pending timer interrupt. + */ + writel(TIMER0_CLR_MASK, local_base + LCL_TIMER_EVENTS_STATUS); + return 0; +} - /* - * Enable timer. - */ - local_timer_ctrl_clrset(0, TIMER0_RELOAD_EN | enable_mask); - } else { - /* - * Disable timer. - */ - local_timer_ctrl_clrset(TIMER0_EN, 0); +static int armada_370_xp_clkevt_set_periodic(struct clock_event_device *evt) +{ + /* + * Setup timer to fire at 1/HZ intervals. + */ + writel(ticks_per_jiffy - 1, local_base + TIMER0_RELOAD_OFF); + writel(ticks_per_jiffy - 1, local_base + TIMER0_VAL_OFF); - /* - * ACK pending timer interrupt. - */ - writel(TIMER0_CLR_MASK, local_base + LCL_TIMER_EVENTS_STATUS); - } + /* + * Enable timer. + */ + local_timer_ctrl_clrset(0, TIMER0_RELOAD_EN | enable_mask); + return 0; } static int armada_370_xp_clkevt_irq; @@ -185,7 +185,10 @@ static int armada_370_xp_timer_setup(struct clock_event_device *evt) evt->shift = 32, evt->rating = 300, evt->set_next_event = armada_370_xp_clkevt_next_event, - evt->set_mode = armada_370_xp_clkevt_mode, + evt->set_state_shutdown = armada_370_xp_clkevt_shutdown; + evt->set_state_periodic = armada_370_xp_clkevt_set_periodic; + evt->set_state_oneshot = armada_370_xp_clkevt_shutdown; + evt->tick_resume = armada_370_xp_clkevt_shutdown; evt->irq = armada_370_xp_clkevt_irq; evt->cpumask = cpumask_of(cpu); @@ -197,7 +200,7 @@ static int armada_370_xp_timer_setup(struct clock_event_device *evt) static void armada_370_xp_timer_stop(struct clock_event_device *evt) { - evt->set_mode(CLOCK_EVT_MODE_UNUSED, evt); + evt->set_state_shutdown(evt); disable_percpu_irq(evt->irq); } diff --git a/drivers/clocksource/time-efm32.c b/drivers/clocksource/time-efm32.c index 5b6e3d5644c9..b06e4c2be406 100644 --- a/drivers/clocksource/time-efm32.c +++ b/drivers/clocksource/time-efm32.c @@ -48,40 +48,42 @@ struct efm32_clock_event_ddata { unsigned periodic_top; }; -static void efm32_clock_event_set_mode(enum clock_event_mode mode, - struct clock_event_device *evtdev) +static int efm32_clock_event_shutdown(struct clock_event_device *evtdev) { struct efm32_clock_event_ddata *ddata = container_of(evtdev, struct efm32_clock_event_ddata, evtdev); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - writel_relaxed(TIMERn_CMD_STOP, ddata->base + TIMERn_CMD); - writel_relaxed(ddata->periodic_top, ddata->base + TIMERn_TOP); - writel_relaxed(TIMERn_CTRL_PRESC_1024 | - TIMERn_CTRL_CLKSEL_PRESCHFPERCLK | - TIMERn_CTRL_MODE_DOWN, - ddata->base + TIMERn_CTRL); - writel_relaxed(TIMERn_CMD_START, ddata->base + TIMERn_CMD); - break; + writel_relaxed(TIMERn_CMD_STOP, ddata->base + TIMERn_CMD); + return 0; +} - case CLOCK_EVT_MODE_ONESHOT: - writel_relaxed(TIMERn_CMD_STOP, ddata->base + TIMERn_CMD); - writel_relaxed(TIMERn_CTRL_PRESC_1024 | - TIMERn_CTRL_CLKSEL_PRESCHFPERCLK | - TIMERn_CTRL_OSMEN | - TIMERn_CTRL_MODE_DOWN, - ddata->base + TIMERn_CTRL); - break; +static int efm32_clock_event_set_oneshot(struct clock_event_device *evtdev) +{ + struct efm32_clock_event_ddata *ddata = + container_of(evtdev, struct efm32_clock_event_ddata, evtdev); - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - writel_relaxed(TIMERn_CMD_STOP, ddata->base + TIMERn_CMD); - break; + writel_relaxed(TIMERn_CMD_STOP, ddata->base + TIMERn_CMD); + writel_relaxed(TIMERn_CTRL_PRESC_1024 | + TIMERn_CTRL_CLKSEL_PRESCHFPERCLK | + TIMERn_CTRL_OSMEN | + TIMERn_CTRL_MODE_DOWN, + ddata->base + TIMERn_CTRL); + return 0; +} - case CLOCK_EVT_MODE_RESUME: - break; - } +static int efm32_clock_event_set_periodic(struct clock_event_device *evtdev) +{ + struct efm32_clock_event_ddata *ddata = + container_of(evtdev, struct efm32_clock_event_ddata, evtdev); + + writel_relaxed(TIMERn_CMD_STOP, ddata->base + TIMERn_CMD); + writel_relaxed(ddata->periodic_top, ddata->base + TIMERn_TOP); + writel_relaxed(TIMERn_CTRL_PRESC_1024 | + TIMERn_CTRL_CLKSEL_PRESCHFPERCLK | + TIMERn_CTRL_MODE_DOWN, + ddata->base + TIMERn_CTRL); + writel_relaxed(TIMERn_CMD_START, ddata->base + TIMERn_CMD); + return 0; } static int efm32_clock_event_set_next_event(unsigned long evt, @@ -112,7 +114,9 @@ static struct efm32_clock_event_ddata clock_event_ddata = { .evtdev = { .name = "efm32 clockevent", .features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_PERIODIC, - .set_mode = efm32_clock_event_set_mode, + .set_state_shutdown = efm32_clock_event_shutdown, + .set_state_periodic = efm32_clock_event_set_periodic, + .set_state_oneshot = efm32_clock_event_set_oneshot, .set_next_event = efm32_clock_event_set_next_event, .rating = 200, }, diff --git a/drivers/clocksource/time-orion.c b/drivers/clocksource/time-orion.c index 0b3ce0399c51..0ece7427b497 100644 --- a/drivers/clocksource/time-orion.c +++ b/drivers/clocksource/time-orion.c @@ -60,30 +60,36 @@ static int orion_clkevt_next_event(unsigned long delta, return 0; } -static void orion_clkevt_mode(enum clock_event_mode mode, - struct clock_event_device *dev) +static int orion_clkevt_shutdown(struct clock_event_device *dev) { - if (mode == CLOCK_EVT_MODE_PERIODIC) { - /* setup and enable periodic timer at 1/HZ intervals */ - writel(ticks_per_jiffy - 1, timer_base + TIMER1_RELOAD); - writel(ticks_per_jiffy - 1, timer_base + TIMER1_VAL); - atomic_io_modify(timer_base + TIMER_CTRL, - TIMER1_RELOAD_EN | TIMER1_EN, - TIMER1_RELOAD_EN | TIMER1_EN); - } else { - /* disable timer */ - atomic_io_modify(timer_base + TIMER_CTRL, - TIMER1_RELOAD_EN | TIMER1_EN, 0); - } + /* disable timer */ + atomic_io_modify(timer_base + TIMER_CTRL, + TIMER1_RELOAD_EN | TIMER1_EN, 0); + return 0; +} + +static int orion_clkevt_set_periodic(struct clock_event_device *dev) +{ + /* setup and enable periodic timer at 1/HZ intervals */ + writel(ticks_per_jiffy - 1, timer_base + TIMER1_RELOAD); + writel(ticks_per_jiffy - 1, timer_base + TIMER1_VAL); + atomic_io_modify(timer_base + TIMER_CTRL, + TIMER1_RELOAD_EN | TIMER1_EN, + TIMER1_RELOAD_EN | TIMER1_EN); + return 0; } static struct clock_event_device orion_clkevt = { - .name = "orion_event", - .features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_PERIODIC, - .shift = 32, - .rating = 300, - .set_next_event = orion_clkevt_next_event, - .set_mode = orion_clkevt_mode, + .name = "orion_event", + .features = CLOCK_EVT_FEAT_ONESHOT | + CLOCK_EVT_FEAT_PERIODIC, + .shift = 32, + .rating = 300, + .set_next_event = orion_clkevt_next_event, + .set_state_shutdown = orion_clkevt_shutdown, + .set_state_periodic = orion_clkevt_set_periodic, + .set_state_oneshot = orion_clkevt_shutdown, + .tick_resume = orion_clkevt_shutdown, }; static irqreturn_t orion_clkevt_irq_handler(int irq, void *dev_id) diff --git a/drivers/clocksource/time-pistachio.c b/drivers/clocksource/time-pistachio.c new file mode 100644 index 000000000000..18d4266c2986 --- /dev/null +++ b/drivers/clocksource/time-pistachio.c @@ -0,0 +1,217 @@ +/* + * Pistachio clocksource based on general-purpose timers + * + * Copyright (C) 2015 Imagination Technologies + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ + +#define pr_fmt(fmt) "%s: " fmt, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Top level reg */ +#define CR_TIMER_CTRL_CFG 0x00 +#define TIMER_ME_GLOBAL BIT(0) +#define CR_TIMER_REV 0x10 + +/* Timer specific registers */ +#define TIMER_CFG 0x20 +#define TIMER_ME_LOCAL BIT(0) +#define TIMER_RELOAD_VALUE 0x24 +#define TIMER_CURRENT_VALUE 0x28 +#define TIMER_CURRENT_OVERFLOW_VALUE 0x2C +#define TIMER_IRQ_STATUS 0x30 +#define TIMER_IRQ_CLEAR 0x34 +#define TIMER_IRQ_MASK 0x38 + +#define PERIP_TIMER_CONTROL 0x90 + +/* Timer specific configuration Values */ +#define RELOAD_VALUE 0xffffffff + +struct pistachio_clocksource { + void __iomem *base; + raw_spinlock_t lock; + struct clocksource cs; +}; + +static struct pistachio_clocksource pcs_gpt; + +#define to_pistachio_clocksource(cs) \ + container_of(cs, struct pistachio_clocksource, cs) + +static inline u32 gpt_readl(void __iomem *base, u32 offset, u32 gpt_id) +{ + return readl(base + 0x20 * gpt_id + offset); +} + +static inline void gpt_writel(void __iomem *base, u32 value, u32 offset, + u32 gpt_id) +{ + writel(value, base + 0x20 * gpt_id + offset); +} + +static cycle_t pistachio_clocksource_read_cycles(struct clocksource *cs) +{ + struct pistachio_clocksource *pcs = to_pistachio_clocksource(cs); + u32 counter, overflw; + unsigned long flags; + + /* + * The counter value is only refreshed after the overflow value is read. + * And they must be read in strict order, hence raw spin lock added. + */ + + raw_spin_lock_irqsave(&pcs->lock, flags); + overflw = gpt_readl(pcs->base, TIMER_CURRENT_OVERFLOW_VALUE, 0); + counter = gpt_readl(pcs->base, TIMER_CURRENT_VALUE, 0); + raw_spin_unlock_irqrestore(&pcs->lock, flags); + + return ~(cycle_t)counter; +} + +static u64 notrace pistachio_read_sched_clock(void) +{ + return pistachio_clocksource_read_cycles(&pcs_gpt.cs); +} + +static void pistachio_clksrc_set_mode(struct clocksource *cs, int timeridx, + int enable) +{ + struct pistachio_clocksource *pcs = to_pistachio_clocksource(cs); + u32 val; + + val = gpt_readl(pcs->base, TIMER_CFG, timeridx); + if (enable) + val |= TIMER_ME_LOCAL; + else + val &= ~TIMER_ME_LOCAL; + + gpt_writel(pcs->base, val, TIMER_CFG, timeridx); +} + +static void pistachio_clksrc_enable(struct clocksource *cs, int timeridx) +{ + struct pistachio_clocksource *pcs = to_pistachio_clocksource(cs); + + /* Disable GPT local before loading reload value */ + pistachio_clksrc_set_mode(cs, timeridx, false); + gpt_writel(pcs->base, RELOAD_VALUE, TIMER_RELOAD_VALUE, timeridx); + pistachio_clksrc_set_mode(cs, timeridx, true); +} + +static void pistachio_clksrc_disable(struct clocksource *cs, int timeridx) +{ + /* Disable GPT local */ + pistachio_clksrc_set_mode(cs, timeridx, false); +} + +static int pistachio_clocksource_enable(struct clocksource *cs) +{ + pistachio_clksrc_enable(cs, 0); + return 0; +} + +static void pistachio_clocksource_disable(struct clocksource *cs) +{ + pistachio_clksrc_disable(cs, 0); +} + +/* Desirable clock source for pistachio platform */ +static struct pistachio_clocksource pcs_gpt = { + .cs = { + .name = "gptimer", + .rating = 300, + .enable = pistachio_clocksource_enable, + .disable = pistachio_clocksource_disable, + .read = pistachio_clocksource_read_cycles, + .mask = CLOCKSOURCE_MASK(32), + .flags = CLOCK_SOURCE_IS_CONTINUOUS | + CLOCK_SOURCE_SUSPEND_NONSTOP, + }, +}; + +static void __init pistachio_clksrc_of_init(struct device_node *node) +{ + struct clk *sys_clk, *fast_clk; + struct regmap *periph_regs; + unsigned long rate; + int ret; + + pcs_gpt.base = of_iomap(node, 0); + if (!pcs_gpt.base) { + pr_err("cannot iomap\n"); + return; + } + + periph_regs = syscon_regmap_lookup_by_phandle(node, "img,cr-periph"); + if (IS_ERR(periph_regs)) { + pr_err("cannot get peripheral regmap (%lu)\n", + PTR_ERR(periph_regs)); + return; + } + + /* Switch to using the fast counter clock */ + ret = regmap_update_bits(periph_regs, PERIP_TIMER_CONTROL, + 0xf, 0x0); + if (ret) + return; + + sys_clk = of_clk_get_by_name(node, "sys"); + if (IS_ERR(sys_clk)) { + pr_err("clock get failed (%lu)\n", PTR_ERR(sys_clk)); + return; + } + + fast_clk = of_clk_get_by_name(node, "fast"); + if (IS_ERR(fast_clk)) { + pr_err("clock get failed (%lu)\n", PTR_ERR(fast_clk)); + return; + } + + ret = clk_prepare_enable(sys_clk); + if (ret < 0) { + pr_err("failed to enable clock (%d)\n", ret); + return; + } + + ret = clk_prepare_enable(fast_clk); + if (ret < 0) { + pr_err("failed to enable clock (%d)\n", ret); + clk_disable_unprepare(sys_clk); + return; + } + + rate = clk_get_rate(fast_clk); + + /* Disable irq's for clocksource usage */ + gpt_writel(&pcs_gpt.base, 0, TIMER_IRQ_MASK, 0); + gpt_writel(&pcs_gpt.base, 0, TIMER_IRQ_MASK, 1); + gpt_writel(&pcs_gpt.base, 0, TIMER_IRQ_MASK, 2); + gpt_writel(&pcs_gpt.base, 0, TIMER_IRQ_MASK, 3); + + /* Enable timer block */ + writel(TIMER_ME_GLOBAL, pcs_gpt.base); + + raw_spin_lock_init(&pcs_gpt.lock); + sched_clock_register(pistachio_read_sched_clock, 32, rate); + clocksource_register_hz(&pcs_gpt.cs, rate); +} +CLOCKSOURCE_OF_DECLARE(pistachio_gptimer, "img,pistachio-gptimer", + pistachio_clksrc_of_init); diff --git a/drivers/clocksource/timer-atlas7.c b/drivers/clocksource/timer-atlas7.c index 60f9de3438b0..27fa13680be1 100644 --- a/drivers/clocksource/timer-atlas7.c +++ b/drivers/clocksource/timer-atlas7.c @@ -76,7 +76,7 @@ static irqreturn_t sirfsoc_timer_interrupt(int irq, void *dev_id) /* clear timer interrupt */ writel_relaxed(BIT(cpu), sirfsoc_timer_base + SIRFSOC_TIMER_INTR_STATUS); - if (ce->mode == CLOCK_EVT_MODE_ONESHOT) + if (clockevent_state_oneshot(ce)) sirfsoc_timer_count_disable(cpu); ce->event_handler(ce); @@ -117,18 +117,11 @@ static int sirfsoc_timer_set_next_event(unsigned long delta, return 0; } -static void sirfsoc_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *ce) +/* Oneshot is enabled in set_next_event */ +static int sirfsoc_timer_shutdown(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_ONESHOT: - /* enable in set_next_event */ - break; - default: - break; - } - sirfsoc_timer_count_disable(smp_processor_id()); + return 0; } static void sirfsoc_clocksource_suspend(struct clocksource *cs) @@ -193,7 +186,9 @@ static int sirfsoc_local_timer_setup(struct clock_event_device *ce) ce->name = "local_timer"; ce->features = CLOCK_EVT_FEAT_ONESHOT; ce->rating = 200; - ce->set_mode = sirfsoc_timer_set_mode; + ce->set_state_shutdown = sirfsoc_timer_shutdown; + ce->set_state_oneshot = sirfsoc_timer_shutdown; + ce->tick_resume = sirfsoc_timer_shutdown; ce->set_next_event = sirfsoc_timer_set_next_event; clockevents_calc_mult_shift(ce, atlas7_timer_rate, 60); ce->max_delta_ns = clockevent_delta2ns(-2, ce); diff --git a/drivers/clocksource/timer-atmel-pit.c b/drivers/clocksource/timer-atmel-pit.c index c0304ff608b0..d911c5dca8f1 100644 --- a/drivers/clocksource/timer-atmel-pit.c +++ b/drivers/clocksource/timer-atmel-pit.c @@ -90,33 +90,27 @@ static cycle_t read_pit_clk(struct clocksource *cs) return elapsed; } -/* - * Clockevent device: interrupts every 1/HZ (== pit_cycles * MCK/16) - */ -static void -pit_clkevt_mode(enum clock_event_mode mode, struct clock_event_device *dev) +static int pit_clkevt_shutdown(struct clock_event_device *dev) { struct pit_data *data = clkevt_to_pit_data(dev); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - /* update clocksource counter */ - data->cnt += data->cycle * PIT_PICNT(pit_read(data->base, AT91_PIT_PIVR)); - pit_write(data->base, AT91_PIT_MR, - (data->cycle - 1) | AT91_PIT_PITEN | AT91_PIT_PITIEN); - break; - case CLOCK_EVT_MODE_ONESHOT: - BUG(); - /* FALLTHROUGH */ - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - /* disable irq, leaving the clocksource active */ - pit_write(data->base, AT91_PIT_MR, - (data->cycle - 1) | AT91_PIT_PITEN); - break; - case CLOCK_EVT_MODE_RESUME: - break; - } + /* disable irq, leaving the clocksource active */ + pit_write(data->base, AT91_PIT_MR, (data->cycle - 1) | AT91_PIT_PITEN); + return 0; +} + +/* + * Clockevent device: interrupts every 1/HZ (== pit_cycles * MCK/16) + */ +static int pit_clkevt_set_periodic(struct clock_event_device *dev) +{ + struct pit_data *data = clkevt_to_pit_data(dev); + + /* update clocksource counter */ + data->cnt += data->cycle * PIT_PICNT(pit_read(data->base, AT91_PIT_PIVR)); + pit_write(data->base, AT91_PIT_MR, + (data->cycle - 1) | AT91_PIT_PITEN | AT91_PIT_PITIEN); + return 0; } static void at91sam926x_pit_suspend(struct clock_event_device *cedev) @@ -162,7 +156,7 @@ static irqreturn_t at91sam926x_pit_interrupt(int irq, void *dev_id) WARN_ON_ONCE(!irqs_disabled()); /* The PIT interrupt may be disabled, and is shared */ - if ((data->clkevt.mode == CLOCK_EVT_MODE_PERIODIC) && + if (clockevent_state_periodic(&data->clkevt) && (pit_read(data->base, AT91_PIT_SR) & AT91_PIT_PITS)) { unsigned nr_ticks; @@ -208,8 +202,8 @@ static void __init at91sam926x_pit_common_init(struct pit_data *data) data->clksrc.mask = CLOCKSOURCE_MASK(bits); data->clksrc.name = "pit"; data->clksrc.rating = 175; - data->clksrc.read = read_pit_clk, - data->clksrc.flags = CLOCK_SOURCE_IS_CONTINUOUS, + data->clksrc.read = read_pit_clk; + data->clksrc.flags = CLOCK_SOURCE_IS_CONTINUOUS; clocksource_register_hz(&data->clksrc, pit_rate); /* Set up irq handler */ @@ -227,7 +221,8 @@ static void __init at91sam926x_pit_common_init(struct pit_data *data) data->clkevt.rating = 100; data->clkevt.cpumask = cpumask_of(0); - data->clkevt.set_mode = pit_clkevt_mode; + data->clkevt.set_state_shutdown = pit_clkevt_shutdown; + data->clkevt.set_state_periodic = pit_clkevt_set_periodic; data->clkevt.resume = at91sam926x_pit_resume; data->clkevt.suspend = at91sam926x_pit_suspend; clockevents_register_device(&data->clkevt); diff --git a/drivers/clocksource/timer-atmel-st.c b/drivers/clocksource/timer-atmel-st.c index 1692e17e096b..41b7b6dc1d0d 100644 --- a/drivers/clocksource/timer-atmel-st.c +++ b/drivers/clocksource/timer-atmel-st.c @@ -106,36 +106,47 @@ static struct clocksource clk32k = { .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; -static void -clkevt32k_mode(enum clock_event_mode mode, struct clock_event_device *dev) +static void clkdev32k_disable_and_flush_irq(void) { unsigned int val; /* Disable and flush pending timer interrupts */ regmap_write(regmap_st, AT91_ST_IDR, AT91_ST_PITS | AT91_ST_ALMS); regmap_read(regmap_st, AT91_ST_SR, &val); - last_crtr = read_CRTR(); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - /* PIT for periodic irqs; fixed rate of 1/HZ */ - irqmask = AT91_ST_PITS; - regmap_write(regmap_st, AT91_ST_PIMR, RM9200_TIMER_LATCH); - break; - case CLOCK_EVT_MODE_ONESHOT: - /* ALM for oneshot irqs, set by next_event() - * before 32 seconds have passed - */ - irqmask = AT91_ST_ALMS; - regmap_write(regmap_st, AT91_ST_RTAR, last_crtr); - break; - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_RESUME: - irqmask = 0; - break; - } +} + +static int clkevt32k_shutdown(struct clock_event_device *evt) +{ + clkdev32k_disable_and_flush_irq(); + irqmask = 0; regmap_write(regmap_st, AT91_ST_IER, irqmask); + return 0; +} + +static int clkevt32k_set_oneshot(struct clock_event_device *dev) +{ + clkdev32k_disable_and_flush_irq(); + + /* + * ALM for oneshot irqs, set by next_event() + * before 32 seconds have passed. + */ + irqmask = AT91_ST_ALMS; + regmap_write(regmap_st, AT91_ST_RTAR, last_crtr); + regmap_write(regmap_st, AT91_ST_IER, irqmask); + return 0; +} + +static int clkevt32k_set_periodic(struct clock_event_device *dev) +{ + clkdev32k_disable_and_flush_irq(); + + /* PIT for periodic irqs; fixed rate of 1/HZ */ + irqmask = AT91_ST_PITS; + regmap_write(regmap_st, AT91_ST_PIMR, RM9200_TIMER_LATCH); + regmap_write(regmap_st, AT91_ST_IER, irqmask); + return 0; } static int @@ -170,11 +181,15 @@ clkevt32k_next_event(unsigned long delta, struct clock_event_device *dev) } static struct clock_event_device clkevt = { - .name = "at91_tick", - .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .rating = 150, - .set_next_event = clkevt32k_next_event, - .set_mode = clkevt32k_mode, + .name = "at91_tick", + .features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT, + .rating = 150, + .set_next_event = clkevt32k_next_event, + .set_state_shutdown = clkevt32k_shutdown, + .set_state_periodic = clkevt32k_set_periodic, + .set_state_oneshot = clkevt32k_set_oneshot, + .tick_resume = clkevt32k_shutdown, }; /* diff --git a/drivers/clocksource/timer-digicolor.c b/drivers/clocksource/timer-digicolor.c index 7f8388cfa810..e73947f0f86d 100644 --- a/drivers/clocksource/timer-digicolor.c +++ b/drivers/clocksource/timer-digicolor.c @@ -87,27 +87,27 @@ static inline void dc_timer_set_count(struct clock_event_device *ce, writel(count, dt->base + COUNT(dt->timer_id)); } -static void digicolor_clkevt_mode(enum clock_event_mode mode, - struct clock_event_device *ce) +static int digicolor_clkevt_shutdown(struct clock_event_device *ce) +{ + dc_timer_disable(ce); + return 0; +} + +static int digicolor_clkevt_set_oneshot(struct clock_event_device *ce) +{ + dc_timer_disable(ce); + dc_timer_enable(ce, CONTROL_MODE_ONESHOT); + return 0; +} + +static int digicolor_clkevt_set_periodic(struct clock_event_device *ce) { struct digicolor_timer *dt = dc_timer(ce); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - dc_timer_disable(ce); - dc_timer_set_count(ce, dt->ticks_per_jiffy); - dc_timer_enable(ce, CONTROL_MODE_PERIODIC); - break; - case CLOCK_EVT_MODE_ONESHOT: - dc_timer_disable(ce); - dc_timer_enable(ce, CONTROL_MODE_ONESHOT); - break; - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - default: - dc_timer_disable(ce); - break; - } + dc_timer_disable(ce); + dc_timer_set_count(ce, dt->ticks_per_jiffy); + dc_timer_enable(ce, CONTROL_MODE_PERIODIC); + return 0; } static int digicolor_clkevt_next_event(unsigned long evt, @@ -125,7 +125,10 @@ static struct digicolor_timer dc_timer_dev = { .name = "digicolor_tick", .rating = 340, .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .set_mode = digicolor_clkevt_mode, + .set_state_shutdown = digicolor_clkevt_shutdown, + .set_state_periodic = digicolor_clkevt_set_periodic, + .set_state_oneshot = digicolor_clkevt_set_oneshot, + .tick_resume = digicolor_clkevt_shutdown, .set_next_event = digicolor_clkevt_next_event, }, .timer_id = TIMER_C, diff --git a/drivers/clocksource/timer-imx-gpt.c b/drivers/clocksource/timer-imx-gpt.c index 86c7eb66bdfb..839aba92fc39 100644 --- a/drivers/clocksource/timer-imx-gpt.c +++ b/drivers/clocksource/timer-imx-gpt.c @@ -83,7 +83,6 @@ struct imx_timer { struct clk *clk_ipg; const struct imx_gpt_data *gpt; struct clock_event_device ced; - enum clock_event_mode cem; struct irqaction act; }; @@ -212,18 +211,38 @@ static int v2_set_next_event(unsigned long evt, -ETIME : 0; } +static int mxc_shutdown(struct clock_event_device *ced) +{ + struct imx_timer *imxtm = to_imx_timer(ced); + unsigned long flags; + u32 tcn; + + /* + * The timer interrupt generation is disabled at least + * for enough time to call mxc_set_next_event() + */ + local_irq_save(flags); + + /* Disable interrupt in GPT module */ + imxtm->gpt->gpt_irq_disable(imxtm); + + tcn = readl_relaxed(imxtm->base + imxtm->gpt->reg_tcn); + /* Set event time into far-far future */ + writel_relaxed(tcn - 3, imxtm->base + imxtm->gpt->reg_tcmp); + + /* Clear pending interrupt */ + imxtm->gpt->gpt_irq_acknowledge(imxtm); + #ifdef DEBUG -static const char *clock_event_mode_label[] = { - [CLOCK_EVT_MODE_PERIODIC] = "CLOCK_EVT_MODE_PERIODIC", - [CLOCK_EVT_MODE_ONESHOT] = "CLOCK_EVT_MODE_ONESHOT", - [CLOCK_EVT_MODE_SHUTDOWN] = "CLOCK_EVT_MODE_SHUTDOWN", - [CLOCK_EVT_MODE_UNUSED] = "CLOCK_EVT_MODE_UNUSED", - [CLOCK_EVT_MODE_RESUME] = "CLOCK_EVT_MODE_RESUME", -}; + printk(KERN_INFO "%s: changing mode\n", __func__); #endif /* DEBUG */ -static void mxc_set_mode(enum clock_event_mode mode, - struct clock_event_device *ced) + local_irq_restore(flags); + + return 0; +} + +static int mxc_set_oneshot(struct clock_event_device *ced) { struct imx_timer *imxtm = to_imx_timer(ced); unsigned long flags; @@ -237,7 +256,7 @@ static void mxc_set_mode(enum clock_event_mode mode, /* Disable interrupt in GPT module */ imxtm->gpt->gpt_irq_disable(imxtm); - if (mode != imxtm->cem) { + if (!clockevent_state_oneshot(ced)) { u32 tcn = readl_relaxed(imxtm->base + imxtm->gpt->reg_tcn); /* Set event time into far-far future */ writel_relaxed(tcn - 3, imxtm->base + imxtm->gpt->reg_tcmp); @@ -247,37 +266,19 @@ static void mxc_set_mode(enum clock_event_mode mode, } #ifdef DEBUG - printk(KERN_INFO "mxc_set_mode: changing mode from %s to %s\n", - clock_event_mode_label[imxtm->cem], - clock_event_mode_label[mode]); + printk(KERN_INFO "%s: changing mode\n", __func__); #endif /* DEBUG */ - /* Remember timer mode */ - imxtm->cem = mode; - local_irq_restore(flags); - - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - printk(KERN_ERR"mxc_set_mode: Periodic mode is not " - "supported for i.MX\n"); - break; - case CLOCK_EVT_MODE_ONESHOT: /* * Do not put overhead of interrupt enable/disable into * mxc_set_next_event(), the core has about 4 minutes * to call mxc_set_next_event() or shutdown clock after * mode switching */ - local_irq_save(flags); - imxtm->gpt->gpt_irq_enable(imxtm); - local_irq_restore(flags); - break; - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_RESUME: - /* Left event sources disabled, no more interrupts appear */ - break; - } + imxtm->gpt->gpt_irq_enable(imxtm); + local_irq_restore(flags); + + return 0; } /* @@ -303,11 +304,11 @@ static int __init mxc_clockevent_init(struct imx_timer *imxtm) struct clock_event_device *ced = &imxtm->ced; struct irqaction *act = &imxtm->act; - imxtm->cem = CLOCK_EVT_MODE_UNUSED; - ced->name = "mxc_timer1"; ced->features = CLOCK_EVT_FEAT_ONESHOT; - ced->set_mode = mxc_set_mode; + ced->set_state_shutdown = mxc_shutdown; + ced->set_state_oneshot = mxc_set_oneshot; + ced->tick_resume = mxc_shutdown; ced->set_next_event = imxtm->gpt->set_next_event; ced->rating = 200; ced->cpumask = cpumask_of(0); diff --git a/drivers/clocksource/timer-integrator-ap.c b/drivers/clocksource/timer-integrator-ap.c index a68866e0ecd4..3f59ac2180dc 100644 --- a/drivers/clocksource/timer-integrator-ap.c +++ b/drivers/clocksource/timer-integrator-ap.c @@ -75,33 +75,37 @@ static irqreturn_t integrator_timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static void clkevt_set_mode(enum clock_event_mode mode, struct clock_event_device *evt) +static int clkevt_shutdown(struct clock_event_device *evt) +{ + u32 ctrl = readl(clkevt_base + TIMER_CTRL) & ~TIMER_CTRL_ENABLE; + + /* Disable timer */ + writel(ctrl, clkevt_base + TIMER_CTRL); + return 0; +} + +static int clkevt_set_oneshot(struct clock_event_device *evt) +{ + u32 ctrl = readl(clkevt_base + TIMER_CTRL) & + ~(TIMER_CTRL_ENABLE | TIMER_CTRL_PERIODIC); + + /* Leave the timer disabled, .set_next_event will enable it */ + writel(ctrl, clkevt_base + TIMER_CTRL); + return 0; +} + +static int clkevt_set_periodic(struct clock_event_device *evt) { u32 ctrl = readl(clkevt_base + TIMER_CTRL) & ~TIMER_CTRL_ENABLE; /* Disable timer */ writel(ctrl, clkevt_base + TIMER_CTRL); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - /* Enable the timer and start the periodic tick */ - writel(timer_reload, clkevt_base + TIMER_LOAD); - ctrl |= TIMER_CTRL_PERIODIC | TIMER_CTRL_ENABLE; - writel(ctrl, clkevt_base + TIMER_CTRL); - break; - case CLOCK_EVT_MODE_ONESHOT: - /* Leave the timer disabled, .set_next_event will enable it */ - ctrl &= ~TIMER_CTRL_PERIODIC; - writel(ctrl, clkevt_base + TIMER_CTRL); - break; - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_RESUME: - default: - /* Just leave in disabled state */ - break; - } - + /* Enable the timer and start the periodic tick */ + writel(timer_reload, clkevt_base + TIMER_LOAD); + ctrl |= TIMER_CTRL_PERIODIC | TIMER_CTRL_ENABLE; + writel(ctrl, clkevt_base + TIMER_CTRL); + return 0; } static int clkevt_set_next_event(unsigned long next, struct clock_event_device *evt) @@ -116,11 +120,15 @@ static int clkevt_set_next_event(unsigned long next, struct clock_event_device * } static struct clock_event_device integrator_clockevent = { - .name = "timer1", - .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .set_mode = clkevt_set_mode, - .set_next_event = clkevt_set_next_event, - .rating = 300, + .name = "timer1", + .features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT, + .set_state_shutdown = clkevt_shutdown, + .set_state_periodic = clkevt_set_periodic, + .set_state_oneshot = clkevt_set_oneshot, + .tick_resume = clkevt_shutdown, + .set_next_event = clkevt_set_next_event, + .rating = 300, }; static struct irqaction integrator_timer_irq = { diff --git a/drivers/clocksource/timer-keystone.c b/drivers/clocksource/timer-keystone.c index 0250354f7e55..edacf3902e10 100644 --- a/drivers/clocksource/timer-keystone.c +++ b/drivers/clocksource/timer-keystone.c @@ -72,10 +72,10 @@ static inline void keystone_timer_barrier(void) /** * keystone_timer_config: configures timer to work in oneshot/periodic modes. - * @ mode: mode to configure + * @ mask: mask of the mode to configure * @ period: cycles number to configure for */ -static int keystone_timer_config(u64 period, enum clock_event_mode mode) +static int keystone_timer_config(u64 period, int mask) { u32 tcr; u32 off; @@ -84,16 +84,7 @@ static int keystone_timer_config(u64 period, enum clock_event_mode mode) off = tcr & ~(TCR_ENAMODE_MASK); /* set enable mode */ - switch (mode) { - case CLOCK_EVT_MODE_ONESHOT: - tcr |= TCR_ENAMODE_ONESHOT_MASK; - break; - case CLOCK_EVT_MODE_PERIODIC: - tcr |= TCR_ENAMODE_PERIODIC_MASK; - break; - default: - return -1; - } + tcr |= mask; /* disable timer */ keystone_timer_writel(off, TCR); @@ -138,24 +129,19 @@ static irqreturn_t keystone_timer_interrupt(int irq, void *dev_id) static int keystone_set_next_event(unsigned long cycles, struct clock_event_device *evt) { - return keystone_timer_config(cycles, evt->mode); + return keystone_timer_config(cycles, TCR_ENAMODE_ONESHOT_MASK); } -static void keystone_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int keystone_shutdown(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - keystone_timer_config(timer.hz_period, CLOCK_EVT_MODE_PERIODIC); - break; - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_ONESHOT: - keystone_timer_disable(); - break; - default: - break; - } + keystone_timer_disable(); + return 0; +} + +static int keystone_set_periodic(struct clock_event_device *evt) +{ + keystone_timer_config(timer.hz_period, TCR_ENAMODE_PERIODIC_MASK); + return 0; } static void __init keystone_timer_init(struct device_node *np) @@ -222,7 +208,9 @@ static void __init keystone_timer_init(struct device_node *np) /* setup clockevent */ event_dev->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; event_dev->set_next_event = keystone_set_next_event; - event_dev->set_mode = keystone_set_mode; + event_dev->set_state_shutdown = keystone_shutdown; + event_dev->set_state_periodic = keystone_set_periodic; + event_dev->set_state_oneshot = keystone_shutdown; event_dev->cpumask = cpu_all_mask; event_dev->owner = THIS_MODULE; event_dev->name = TIMER_NAME; diff --git a/drivers/clocksource/timer-prima2.c b/drivers/clocksource/timer-prima2.c index ce18d570e1cd..78de982cc640 100644 --- a/drivers/clocksource/timer-prima2.c +++ b/drivers/clocksource/timer-prima2.c @@ -104,26 +104,21 @@ static int sirfsoc_timer_set_next_event(unsigned long delta, return next - now > delta ? -ETIME : 0; } -static void sirfsoc_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *ce) +static int sirfsoc_timer_shutdown(struct clock_event_device *evt) { u32 val = readl_relaxed(sirfsoc_timer_base + SIRFSOC_TIMER_INT_EN); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - WARN_ON(1); - break; - case CLOCK_EVT_MODE_ONESHOT: - writel_relaxed(val | BIT(0), - sirfsoc_timer_base + SIRFSOC_TIMER_INT_EN); - break; - case CLOCK_EVT_MODE_SHUTDOWN: - writel_relaxed(val & ~BIT(0), - sirfsoc_timer_base + SIRFSOC_TIMER_INT_EN); - break; - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_RESUME: - break; - } + + writel_relaxed(val & ~BIT(0), + sirfsoc_timer_base + SIRFSOC_TIMER_INT_EN); + return 0; +} + +static int sirfsoc_timer_set_oneshot(struct clock_event_device *evt) +{ + u32 val = readl_relaxed(sirfsoc_timer_base + SIRFSOC_TIMER_INT_EN); + + writel_relaxed(val | BIT(0), sirfsoc_timer_base + SIRFSOC_TIMER_INT_EN); + return 0; } static void sirfsoc_clocksource_suspend(struct clocksource *cs) @@ -157,7 +152,8 @@ static struct clock_event_device sirfsoc_clockevent = { .name = "sirfsoc_clockevent", .rating = 200, .features = CLOCK_EVT_FEAT_ONESHOT, - .set_mode = sirfsoc_timer_set_mode, + .set_state_shutdown = sirfsoc_timer_shutdown, + .set_state_oneshot = sirfsoc_timer_set_oneshot, .set_next_event = sirfsoc_timer_set_next_event, }; diff --git a/drivers/clocksource/timer-sp804.c b/drivers/clocksource/timer-sp804.c index ca02503f17d1..5f45b9adef60 100644 --- a/drivers/clocksource/timer-sp804.c +++ b/drivers/clocksource/timer-sp804.c @@ -133,50 +133,50 @@ static irqreturn_t sp804_timer_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static void sp804_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static inline void timer_shutdown(struct clock_event_device *evt) { - unsigned long ctrl = TIMER_CTRL_32BIT | TIMER_CTRL_IE; - - writel(ctrl, clkevt_base + TIMER_CTRL); - - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - writel(clkevt_reload, clkevt_base + TIMER_LOAD); - ctrl |= TIMER_CTRL_PERIODIC | TIMER_CTRL_ENABLE; - break; - - case CLOCK_EVT_MODE_ONESHOT: - /* period set, and timer enabled in 'next_event' hook */ - ctrl |= TIMER_CTRL_ONESHOT; - break; - - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - default: - break; - } + writel(0, clkevt_base + TIMER_CTRL); +} +static int sp804_shutdown(struct clock_event_device *evt) +{ + timer_shutdown(evt); + return 0; +} + +static int sp804_set_periodic(struct clock_event_device *evt) +{ + unsigned long ctrl = TIMER_CTRL_32BIT | TIMER_CTRL_IE | + TIMER_CTRL_PERIODIC | TIMER_CTRL_ENABLE; + + timer_shutdown(evt); + writel(clkevt_reload, clkevt_base + TIMER_LOAD); writel(ctrl, clkevt_base + TIMER_CTRL); + return 0; } static int sp804_set_next_event(unsigned long next, struct clock_event_device *evt) { - unsigned long ctrl = readl(clkevt_base + TIMER_CTRL); + unsigned long ctrl = TIMER_CTRL_32BIT | TIMER_CTRL_IE | + TIMER_CTRL_ONESHOT | TIMER_CTRL_ENABLE; writel(next, clkevt_base + TIMER_LOAD); - writel(ctrl | TIMER_CTRL_ENABLE, clkevt_base + TIMER_CTRL); + writel(ctrl, clkevt_base + TIMER_CTRL); return 0; } static struct clock_event_device sp804_clockevent = { - .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT | - CLOCK_EVT_FEAT_DYNIRQ, - .set_mode = sp804_set_mode, - .set_next_event = sp804_set_next_event, - .rating = 300, + .features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT | + CLOCK_EVT_FEAT_DYNIRQ, + .set_state_shutdown = sp804_shutdown, + .set_state_periodic = sp804_set_periodic, + .set_state_oneshot = sp804_shutdown, + .tick_resume = sp804_shutdown, + .set_next_event = sp804_set_next_event, + .rating = 300, }; static struct irqaction sp804_timer_irq = { diff --git a/drivers/clocksource/timer-stm32.c b/drivers/clocksource/timer-stm32.c index a97e8b50701c..f3dcb76799b4 100644 --- a/drivers/clocksource/timer-stm32.c +++ b/drivers/clocksource/timer-stm32.c @@ -40,24 +40,25 @@ struct stm32_clock_event_ddata { void __iomem *base; }; -static void stm32_clock_event_set_mode(enum clock_event_mode mode, - struct clock_event_device *evtdev) +static int stm32_clock_event_shutdown(struct clock_event_device *evtdev) { struct stm32_clock_event_ddata *data = container_of(evtdev, struct stm32_clock_event_ddata, evtdev); void *base = data->base; - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - writel_relaxed(data->periodic_top, base + TIM_ARR); - writel_relaxed(TIM_CR1_ARPE | TIM_CR1_CEN, base + TIM_CR1); - break; + writel_relaxed(0, base + TIM_CR1); + return 0; +} - case CLOCK_EVT_MODE_ONESHOT: - default: - writel_relaxed(0, base + TIM_CR1); - break; - } +static int stm32_clock_event_set_periodic(struct clock_event_device *evtdev) +{ + struct stm32_clock_event_ddata *data = + container_of(evtdev, struct stm32_clock_event_ddata, evtdev); + void *base = data->base; + + writel_relaxed(data->periodic_top, base + TIM_ARR); + writel_relaxed(TIM_CR1_ARPE | TIM_CR1_CEN, base + TIM_CR1); + return 0; } static int stm32_clock_event_set_next_event(unsigned long evt, @@ -88,7 +89,10 @@ static struct stm32_clock_event_ddata clock_event_ddata = { .evtdev = { .name = "stm32 clockevent", .features = CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_PERIODIC, - .set_mode = stm32_clock_event_set_mode, + .set_state_shutdown = stm32_clock_event_shutdown, + .set_state_periodic = stm32_clock_event_set_periodic, + .set_state_oneshot = stm32_clock_event_shutdown, + .tick_resume = stm32_clock_event_shutdown, .set_next_event = stm32_clock_event_set_next_event, .rating = 200, }, diff --git a/drivers/clocksource/timer-sun5i.c b/drivers/clocksource/timer-sun5i.c index 0ffb4ea7c925..bca9573e036a 100644 --- a/drivers/clocksource/timer-sun5i.c +++ b/drivers/clocksource/timer-sun5i.c @@ -103,27 +103,31 @@ static void sun5i_clkevt_time_start(struct sun5i_timer_clkevt *ce, u8 timer, boo ce->timer.base + TIMER_CTL_REG(timer)); } -static void sun5i_clkevt_mode(enum clock_event_mode mode, - struct clock_event_device *clkevt) +static int sun5i_clkevt_shutdown(struct clock_event_device *clkevt) { struct sun5i_timer_clkevt *ce = to_sun5i_timer_clkevt(clkevt); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - sun5i_clkevt_time_stop(ce, 0); - sun5i_clkevt_time_setup(ce, 0, ce->timer.ticks_per_jiffy); - sun5i_clkevt_time_start(ce, 0, true); - break; - case CLOCK_EVT_MODE_ONESHOT: - sun5i_clkevt_time_stop(ce, 0); - sun5i_clkevt_time_start(ce, 0, false); - break; - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - default: - sun5i_clkevt_time_stop(ce, 0); - break; - } + sun5i_clkevt_time_stop(ce, 0); + return 0; +} + +static int sun5i_clkevt_set_oneshot(struct clock_event_device *clkevt) +{ + struct sun5i_timer_clkevt *ce = to_sun5i_timer_clkevt(clkevt); + + sun5i_clkevt_time_stop(ce, 0); + sun5i_clkevt_time_start(ce, 0, false); + return 0; +} + +static int sun5i_clkevt_set_periodic(struct clock_event_device *clkevt) +{ + struct sun5i_timer_clkevt *ce = to_sun5i_timer_clkevt(clkevt); + + sun5i_clkevt_time_stop(ce, 0); + sun5i_clkevt_time_setup(ce, 0, ce->timer.ticks_per_jiffy); + sun5i_clkevt_time_start(ce, 0, true); + return 0; } static int sun5i_clkevt_next_event(unsigned long evt, @@ -286,7 +290,10 @@ static int __init sun5i_setup_clockevent(struct device_node *node, void __iomem ce->clkevt.name = node->name; ce->clkevt.features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; ce->clkevt.set_next_event = sun5i_clkevt_next_event; - ce->clkevt.set_mode = sun5i_clkevt_mode; + ce->clkevt.set_state_shutdown = sun5i_clkevt_shutdown; + ce->clkevt.set_state_periodic = sun5i_clkevt_set_periodic; + ce->clkevt.set_state_oneshot = sun5i_clkevt_set_oneshot; + ce->clkevt.tick_resume = sun5i_clkevt_shutdown; ce->clkevt.rating = 340; ce->clkevt.irq = irq; ce->clkevt.cpumask = cpu_possible_mask; diff --git a/drivers/clocksource/timer-u300.c b/drivers/clocksource/timer-u300.c index 5dcf756970e7..1744b243898a 100644 --- a/drivers/clocksource/timer-u300.c +++ b/drivers/clocksource/timer-u300.c @@ -187,85 +187,82 @@ struct u300_clockevent_data { unsigned ticks_per_jiffy; }; +static int u300_shutdown(struct clock_event_device *evt) +{ + /* Disable interrupts on GP1 */ + writel(U300_TIMER_APP_GPT1IE_IRQ_DISABLE, + u300_timer_base + U300_TIMER_APP_GPT1IE); + /* Disable GP1 */ + writel(U300_TIMER_APP_DGPT1_TIMER_DISABLE, + u300_timer_base + U300_TIMER_APP_DGPT1); + return 0; +} + /* - * The u300_set_mode() function is always called first, if we - * have oneshot timer active, the oneshot scheduling function + * If we have oneshot timer active, the oneshot scheduling function * u300_set_next_event() is called immediately after. */ -static void u300_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int u300_set_oneshot(struct clock_event_device *evt) +{ + /* Just return; here? */ + /* + * The actual event will be programmed by the next event hook, + * so we just set a dummy value somewhere at the end of the + * universe here. + */ + /* Disable interrupts on GPT1 */ + writel(U300_TIMER_APP_GPT1IE_IRQ_DISABLE, + u300_timer_base + U300_TIMER_APP_GPT1IE); + /* Disable GP1 while we're reprogramming it. */ + writel(U300_TIMER_APP_DGPT1_TIMER_DISABLE, + u300_timer_base + U300_TIMER_APP_DGPT1); + /* + * Expire far in the future, u300_set_next_event() will be + * called soon... + */ + writel(0xFFFFFFFF, u300_timer_base + U300_TIMER_APP_GPT1TC); + /* We run one shot per tick here! */ + writel(U300_TIMER_APP_SGPT1M_MODE_ONE_SHOT, + u300_timer_base + U300_TIMER_APP_SGPT1M); + /* Enable interrupts for this timer */ + writel(U300_TIMER_APP_GPT1IE_IRQ_ENABLE, + u300_timer_base + U300_TIMER_APP_GPT1IE); + /* Enable timer */ + writel(U300_TIMER_APP_EGPT1_TIMER_ENABLE, + u300_timer_base + U300_TIMER_APP_EGPT1); + return 0; +} + +static int u300_set_periodic(struct clock_event_device *evt) { struct u300_clockevent_data *cevdata = container_of(evt, struct u300_clockevent_data, cevd); - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - /* Disable interrupts on GPT1 */ - writel(U300_TIMER_APP_GPT1IE_IRQ_DISABLE, - u300_timer_base + U300_TIMER_APP_GPT1IE); - /* Disable GP1 while we're reprogramming it. */ - writel(U300_TIMER_APP_DGPT1_TIMER_DISABLE, - u300_timer_base + U300_TIMER_APP_DGPT1); - /* - * Set the periodic mode to a certain number of ticks per - * jiffy. - */ - writel(cevdata->ticks_per_jiffy, - u300_timer_base + U300_TIMER_APP_GPT1TC); - /* - * Set continuous mode, so the timer keeps triggering - * interrupts. - */ - writel(U300_TIMER_APP_SGPT1M_MODE_CONTINUOUS, - u300_timer_base + U300_TIMER_APP_SGPT1M); - /* Enable timer interrupts */ - writel(U300_TIMER_APP_GPT1IE_IRQ_ENABLE, - u300_timer_base + U300_TIMER_APP_GPT1IE); - /* Then enable the OS timer again */ - writel(U300_TIMER_APP_EGPT1_TIMER_ENABLE, - u300_timer_base + U300_TIMER_APP_EGPT1); - break; - case CLOCK_EVT_MODE_ONESHOT: - /* Just break; here? */ - /* - * The actual event will be programmed by the next event hook, - * so we just set a dummy value somewhere at the end of the - * universe here. - */ - /* Disable interrupts on GPT1 */ - writel(U300_TIMER_APP_GPT1IE_IRQ_DISABLE, - u300_timer_base + U300_TIMER_APP_GPT1IE); - /* Disable GP1 while we're reprogramming it. */ - writel(U300_TIMER_APP_DGPT1_TIMER_DISABLE, - u300_timer_base + U300_TIMER_APP_DGPT1); - /* - * Expire far in the future, u300_set_next_event() will be - * called soon... - */ - writel(0xFFFFFFFF, u300_timer_base + U300_TIMER_APP_GPT1TC); - /* We run one shot per tick here! */ - writel(U300_TIMER_APP_SGPT1M_MODE_ONE_SHOT, - u300_timer_base + U300_TIMER_APP_SGPT1M); - /* Enable interrupts for this timer */ - writel(U300_TIMER_APP_GPT1IE_IRQ_ENABLE, - u300_timer_base + U300_TIMER_APP_GPT1IE); - /* Enable timer */ - writel(U300_TIMER_APP_EGPT1_TIMER_ENABLE, - u300_timer_base + U300_TIMER_APP_EGPT1); - break; - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - /* Disable interrupts on GP1 */ - writel(U300_TIMER_APP_GPT1IE_IRQ_DISABLE, - u300_timer_base + U300_TIMER_APP_GPT1IE); - /* Disable GP1 */ - writel(U300_TIMER_APP_DGPT1_TIMER_DISABLE, - u300_timer_base + U300_TIMER_APP_DGPT1); - break; - case CLOCK_EVT_MODE_RESUME: - /* Ignore this call */ - break; - } + /* Disable interrupts on GPT1 */ + writel(U300_TIMER_APP_GPT1IE_IRQ_DISABLE, + u300_timer_base + U300_TIMER_APP_GPT1IE); + /* Disable GP1 while we're reprogramming it. */ + writel(U300_TIMER_APP_DGPT1_TIMER_DISABLE, + u300_timer_base + U300_TIMER_APP_DGPT1); + /* + * Set the periodic mode to a certain number of ticks per + * jiffy. + */ + writel(cevdata->ticks_per_jiffy, + u300_timer_base + U300_TIMER_APP_GPT1TC); + /* + * Set continuous mode, so the timer keeps triggering + * interrupts. + */ + writel(U300_TIMER_APP_SGPT1M_MODE_CONTINUOUS, + u300_timer_base + U300_TIMER_APP_SGPT1M); + /* Enable timer interrupts */ + writel(U300_TIMER_APP_GPT1IE_IRQ_ENABLE, + u300_timer_base + U300_TIMER_APP_GPT1IE); + /* Then enable the OS timer again */ + writel(U300_TIMER_APP_EGPT1_TIMER_ENABLE, + u300_timer_base + U300_TIMER_APP_EGPT1); + return 0; } /* @@ -309,13 +306,15 @@ static int u300_set_next_event(unsigned long cycles, static struct u300_clockevent_data u300_clockevent_data = { /* Use general purpose timer 1 as clock event */ .cevd = { - .name = "GPT1", + .name = "GPT1", /* Reasonably fast and accurate clock event */ - .rating = 300, - .features = CLOCK_EVT_FEAT_PERIODIC | - CLOCK_EVT_FEAT_ONESHOT, - .set_next_event = u300_set_next_event, - .set_mode = u300_set_mode, + .rating = 300, + .features = CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_ONESHOT, + .set_next_event = u300_set_next_event, + .set_state_shutdown = u300_shutdown, + .set_state_periodic = u300_set_periodic, + .set_state_oneshot = u300_set_oneshot, }, }; diff --git a/drivers/clocksource/vf_pit_timer.c b/drivers/clocksource/vf_pit_timer.c index b45ac6229b57..f07ba9932171 100644 --- a/drivers/clocksource/vf_pit_timer.c +++ b/drivers/clocksource/vf_pit_timer.c @@ -86,20 +86,16 @@ static int pit_set_next_event(unsigned long delta, return 0; } -static void pit_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int pit_shutdown(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_PERIODIC: - pit_set_next_event(cycle_per_jiffy, evt); - break; - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - pit_timer_disable(); - break; - default: - break; - } + pit_timer_disable(); + return 0; +} + +static int pit_set_periodic(struct clock_event_device *evt) +{ + pit_set_next_event(cycle_per_jiffy, evt); + return 0; } static irqreturn_t pit_timer_interrupt(int irq, void *dev_id) @@ -114,7 +110,7 @@ static irqreturn_t pit_timer_interrupt(int irq, void *dev_id) * and start the counter again. So software need to disable the timer * to stop the counter loop in ONESHOT mode. */ - if (likely(evt->mode == CLOCK_EVT_MODE_ONESHOT)) + if (likely(clockevent_state_oneshot(evt))) pit_timer_disable(); evt->event_handler(evt); @@ -125,7 +121,8 @@ static irqreturn_t pit_timer_interrupt(int irq, void *dev_id) static struct clock_event_device clockevent_pit = { .name = "VF pit timer", .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .set_mode = pit_set_mode, + .set_state_shutdown = pit_shutdown, + .set_state_periodic = pit_set_periodic, .set_next_event = pit_set_next_event, .rating = 300, }; diff --git a/drivers/clocksource/vt8500_timer.c b/drivers/clocksource/vt8500_timer.c index 1098ed3b9b89..a92e94b40b5b 100644 --- a/drivers/clocksource/vt8500_timer.c +++ b/drivers/clocksource/vt8500_timer.c @@ -88,29 +88,20 @@ static int vt8500_timer_set_next_event(unsigned long cycles, return 0; } -static void vt8500_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) +static int vt8500_shutdown(struct clock_event_device *evt) { - switch (mode) { - case CLOCK_EVT_MODE_RESUME: - case CLOCK_EVT_MODE_PERIODIC: - break; - case CLOCK_EVT_MODE_ONESHOT: - case CLOCK_EVT_MODE_UNUSED: - case CLOCK_EVT_MODE_SHUTDOWN: - writel(readl(regbase + TIMER_CTRL_VAL) | 1, - regbase + TIMER_CTRL_VAL); - writel(0, regbase + TIMER_IER_VAL); - break; - } + writel(readl(regbase + TIMER_CTRL_VAL) | 1, regbase + TIMER_CTRL_VAL); + writel(0, regbase + TIMER_IER_VAL); + return 0; } static struct clock_event_device clockevent = { - .name = "vt8500_timer", - .features = CLOCK_EVT_FEAT_ONESHOT, - .rating = 200, - .set_next_event = vt8500_timer_set_next_event, - .set_mode = vt8500_timer_set_mode, + .name = "vt8500_timer", + .features = CLOCK_EVT_FEAT_ONESHOT, + .rating = 200, + .set_next_event = vt8500_timer_set_next_event, + .set_state_shutdown = vt8500_shutdown, + .set_state_oneshot = vt8500_shutdown, }; static irqreturn_t vt8500_timer_interrupt(int irq, void *dev_id) diff --git a/drivers/clocksource/zevio-timer.c b/drivers/clocksource/zevio-timer.c index 7ce442148c3f..ceaa6133f9c2 100644 --- a/drivers/clocksource/zevio-timer.c +++ b/drivers/clocksource/zevio-timer.c @@ -76,32 +76,28 @@ static int zevio_timer_set_event(unsigned long delta, return 0; } -static void zevio_timer_set_mode(enum clock_event_mode mode, - struct clock_event_device *dev) +static int zevio_timer_shutdown(struct clock_event_device *dev) { struct zevio_timer *timer = container_of(dev, struct zevio_timer, clkevt); - switch (mode) { - case CLOCK_EVT_MODE_RESUME: - case CLOCK_EVT_MODE_ONESHOT: - /* Enable timer interrupts */ - writel(TIMER_INTR_MSK, timer->interrupt_regs + IO_INTR_MSK); - writel(TIMER_INTR_ALL, timer->interrupt_regs + IO_INTR_ACK); - break; - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - /* Disable timer interrupts */ - writel(0, timer->interrupt_regs + IO_INTR_MSK); - writel(TIMER_INTR_ALL, timer->interrupt_regs + IO_INTR_ACK); - /* Stop timer */ - writel(CNTL_STOP_TIMER, timer->timer1 + IO_CONTROL); - break; - case CLOCK_EVT_MODE_PERIODIC: - default: - /* Unsupported */ - break; - } + /* Disable timer interrupts */ + writel(0, timer->interrupt_regs + IO_INTR_MSK); + writel(TIMER_INTR_ALL, timer->interrupt_regs + IO_INTR_ACK); + /* Stop timer */ + writel(CNTL_STOP_TIMER, timer->timer1 + IO_CONTROL); + return 0; +} + +static int zevio_timer_set_oneshot(struct clock_event_device *dev) +{ + struct zevio_timer *timer = container_of(dev, struct zevio_timer, + clkevt); + + /* Enable timer interrupts */ + writel(TIMER_INTR_MSK, timer->interrupt_regs + IO_INTR_MSK); + writel(TIMER_INTR_ALL, timer->interrupt_regs + IO_INTR_ACK); + return 0; } static irqreturn_t zevio_timer_interrupt(int irq, void *dev_id) @@ -162,7 +158,9 @@ static int __init zevio_timer_add(struct device_node *node) if (timer->interrupt_regs && irqnr) { timer->clkevt.name = timer->clockevent_name; timer->clkevt.set_next_event = zevio_timer_set_event; - timer->clkevt.set_mode = zevio_timer_set_mode; + timer->clkevt.set_state_shutdown = zevio_timer_shutdown; + timer->clkevt.set_state_oneshot = zevio_timer_set_oneshot; + timer->clkevt.tick_resume = zevio_timer_set_oneshot; timer->clkevt.rating = 200; timer->clkevt.cpumask = cpu_all_mask; timer->clkevt.features = CLOCK_EVT_FEAT_ONESHOT; diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm index 24e5c664683f..cd0391e46c6d 100644 --- a/drivers/cpufreq/Kconfig.arm +++ b/drivers/cpufreq/Kconfig.arm @@ -24,55 +24,6 @@ config ARM_VEXPRESS_SPC_CPUFREQ This add the CPUfreq driver support for Versatile Express big.LITTLE platforms using SPC for power management. - -config ARM_EXYNOS_CPUFREQ - tristate "SAMSUNG EXYNOS CPUfreq Driver" - depends on CPU_EXYNOS4210 || SOC_EXYNOS4212 || SOC_EXYNOS4412 || SOC_EXYNOS5250 - depends on THERMAL - help - This adds the CPUFreq driver for Samsung EXYNOS platforms. - Supported SoC versions are: - Exynos4210, Exynos4212, Exynos4412, and Exynos5250. - - If in doubt, say N. - -config ARM_EXYNOS4X12_CPUFREQ - bool "SAMSUNG EXYNOS4x12" - depends on SOC_EXYNOS4212 || SOC_EXYNOS4412 - depends on ARM_EXYNOS_CPUFREQ - default y - help - This adds the CPUFreq driver for Samsung EXYNOS4X12 - SoC (EXYNOS4212 or EXYNOS4412). - - If in doubt, say N. - -config ARM_EXYNOS5250_CPUFREQ - bool "SAMSUNG EXYNOS5250" - depends on SOC_EXYNOS5250 - depends on ARM_EXYNOS_CPUFREQ - default y - help - This adds the CPUFreq driver for Samsung EXYNOS5250 - SoC. - - If in doubt, say N. - -config ARM_EXYNOS_CPU_FREQ_BOOST_SW - bool "EXYNOS Frequency Overclocking - Software" - depends on ARM_EXYNOS_CPUFREQ && THERMAL - select CPU_FREQ_BOOST_SW - select EXYNOS_THERMAL - help - This driver supports software managed overclocking (BOOST). - It allows usage of special frequencies for Samsung Exynos - processors if thermal conditions are appropriate. - - It requires, for safe operation, thermal framework with properly - defined trip points. - - If in doubt, say N. - config ARM_EXYNOS5440_CPUFREQ tristate "SAMSUNG EXYNOS5440" depends on SOC_EXYNOS5440 @@ -130,6 +81,14 @@ config ARM_KIRKWOOD_CPUFREQ This adds the CPUFreq driver for Marvell Kirkwood SoCs. +config ARM_MT8173_CPUFREQ + bool "Mediatek MT8173 CPUFreq support" + depends on ARCH_MEDIATEK && REGULATOR + depends on !CPU_THERMAL || THERMAL=y + select PM_OPP + help + This adds the CPUFreq driver support for Mediatek MT8173 SoC. + config ARM_OMAP2PLUS_CPUFREQ bool "TI OMAP2+" depends on ARCH_OMAP2PLUS diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index 032745de8fcc..41340384f11f 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -52,16 +52,13 @@ obj-$(CONFIG_ARM_DT_BL_CPUFREQ) += arm_big_little_dt.o obj-$(CONFIG_ARCH_DAVINCI) += davinci-cpufreq.o obj-$(CONFIG_UX500_SOC_DB8500) += dbx500-cpufreq.o -obj-$(CONFIG_ARM_EXYNOS_CPUFREQ) += arm-exynos-cpufreq.o -arm-exynos-cpufreq-y := exynos-cpufreq.o -arm-exynos-cpufreq-$(CONFIG_ARM_EXYNOS4X12_CPUFREQ) += exynos4x12-cpufreq.o -arm-exynos-cpufreq-$(CONFIG_ARM_EXYNOS5250_CPUFREQ) += exynos5250-cpufreq.o obj-$(CONFIG_ARM_EXYNOS5440_CPUFREQ) += exynos5440-cpufreq.o obj-$(CONFIG_ARM_HIGHBANK_CPUFREQ) += highbank-cpufreq.o obj-$(CONFIG_ARM_HISI_ACPU_CPUFREQ) += hisi-acpu-cpufreq.o obj-$(CONFIG_ARM_IMX6Q_CPUFREQ) += imx6q-cpufreq.o obj-$(CONFIG_ARM_INTEGRATOR) += integrator-cpufreq.o obj-$(CONFIG_ARM_KIRKWOOD_CPUFREQ) += kirkwood-cpufreq.o +obj-$(CONFIG_ARM_MT8173_CPUFREQ) += mt8173-cpufreq.o obj-$(CONFIG_ARM_OMAP2PLUS_CPUFREQ) += omap-cpufreq.o obj-$(CONFIG_ARM_PXA2xx_CPUFREQ) += pxa2xx-cpufreq.o obj-$(CONFIG_PXA3xx) += pxa3xx-cpufreq.o diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index 0136dfcdabf0..15b921a9248c 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -65,18 +65,21 @@ enum { #define MSR_K7_HWCR_CPB_DIS (1ULL << 25) struct acpi_cpufreq_data { - struct acpi_processor_performance *acpi_data; struct cpufreq_frequency_table *freq_table; unsigned int resume; unsigned int cpu_feature; + unsigned int acpi_perf_cpu; cpumask_var_t freqdomain_cpus; }; -static DEFINE_PER_CPU(struct acpi_cpufreq_data *, acfreq_data); - /* acpi_perf_data is a pointer to percpu data. */ static struct acpi_processor_performance __percpu *acpi_perf_data; +static inline struct acpi_processor_performance *to_perf_data(struct acpi_cpufreq_data *data) +{ + return per_cpu_ptr(acpi_perf_data, data->acpi_perf_cpu); +} + static struct cpufreq_driver acpi_cpufreq_driver; static unsigned int acpi_pstate_strict; @@ -144,7 +147,7 @@ static int _store_boost(int val) static ssize_t show_freqdomain_cpus(struct cpufreq_policy *policy, char *buf) { - struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); + struct acpi_cpufreq_data *data = policy->driver_data; return cpufreq_show_cpus(data->freqdomain_cpus, buf); } @@ -202,7 +205,7 @@ static unsigned extract_io(u32 value, struct acpi_cpufreq_data *data) struct acpi_processor_performance *perf; int i; - perf = data->acpi_data; + perf = to_perf_data(data); for (i = 0; i < perf->state_count; i++) { if (value == perf->states[i].status) @@ -221,7 +224,7 @@ static unsigned extract_msr(u32 msr, struct acpi_cpufreq_data *data) else msr &= INTEL_MSR_RANGE; - perf = data->acpi_data; + perf = to_perf_data(data); cpufreq_for_each_entry(pos, data->freq_table) if (msr == perf->states[pos->driver_data].status) @@ -327,7 +330,8 @@ static void drv_write(struct drv_cmd *cmd) put_cpu(); } -static u32 get_cur_val(const struct cpumask *mask) +static u32 +get_cur_val(const struct cpumask *mask, struct acpi_cpufreq_data *data) { struct acpi_processor_performance *perf; struct drv_cmd cmd; @@ -335,7 +339,7 @@ static u32 get_cur_val(const struct cpumask *mask) if (unlikely(cpumask_empty(mask))) return 0; - switch (per_cpu(acfreq_data, cpumask_first(mask))->cpu_feature) { + switch (data->cpu_feature) { case SYSTEM_INTEL_MSR_CAPABLE: cmd.type = SYSTEM_INTEL_MSR_CAPABLE; cmd.addr.msr.reg = MSR_IA32_PERF_CTL; @@ -346,7 +350,7 @@ static u32 get_cur_val(const struct cpumask *mask) break; case SYSTEM_IO_CAPABLE: cmd.type = SYSTEM_IO_CAPABLE; - perf = per_cpu(acfreq_data, cpumask_first(mask))->acpi_data; + perf = to_perf_data(data); cmd.addr.io.port = perf->control_register.address; cmd.addr.io.bit_width = perf->control_register.bit_width; break; @@ -364,19 +368,24 @@ static u32 get_cur_val(const struct cpumask *mask) static unsigned int get_cur_freq_on_cpu(unsigned int cpu) { - struct acpi_cpufreq_data *data = per_cpu(acfreq_data, cpu); + struct acpi_cpufreq_data *data; + struct cpufreq_policy *policy; unsigned int freq; unsigned int cached_freq; pr_debug("get_cur_freq_on_cpu (%d)\n", cpu); - if (unlikely(data == NULL || - data->acpi_data == NULL || data->freq_table == NULL)) { + policy = cpufreq_cpu_get(cpu); + if (unlikely(!policy)) return 0; - } - cached_freq = data->freq_table[data->acpi_data->state].frequency; - freq = extract_freq(get_cur_val(cpumask_of(cpu)), data); + data = policy->driver_data; + cpufreq_cpu_put(policy); + if (unlikely(!data || !data->freq_table)) + return 0; + + cached_freq = data->freq_table[to_perf_data(data)->state].frequency; + freq = extract_freq(get_cur_val(cpumask_of(cpu), data), data); if (freq != cached_freq) { /* * The dreaded BIOS frequency change behind our back. @@ -397,7 +406,7 @@ static unsigned int check_freqs(const struct cpumask *mask, unsigned int freq, unsigned int i; for (i = 0; i < 100; i++) { - cur_freq = extract_freq(get_cur_val(mask), data); + cur_freq = extract_freq(get_cur_val(mask, data), data); if (cur_freq == freq) return 1; udelay(10); @@ -408,18 +417,17 @@ static unsigned int check_freqs(const struct cpumask *mask, unsigned int freq, static int acpi_cpufreq_target(struct cpufreq_policy *policy, unsigned int index) { - struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); + struct acpi_cpufreq_data *data = policy->driver_data; struct acpi_processor_performance *perf; struct drv_cmd cmd; unsigned int next_perf_state = 0; /* Index into perf table */ int result = 0; - if (unlikely(data == NULL || - data->acpi_data == NULL || data->freq_table == NULL)) { + if (unlikely(data == NULL || data->freq_table == NULL)) { return -ENODEV; } - perf = data->acpi_data; + perf = to_perf_data(data); next_perf_state = data->freq_table[index].driver_data; if (perf->state == next_perf_state) { if (unlikely(data->resume)) { @@ -482,8 +490,9 @@ static int acpi_cpufreq_target(struct cpufreq_policy *policy, static unsigned long acpi_cpufreq_guess_freq(struct acpi_cpufreq_data *data, unsigned int cpu) { - struct acpi_processor_performance *perf = data->acpi_data; + struct acpi_processor_performance *perf; + perf = to_perf_data(data); if (cpu_khz) { /* search the closest match to cpu_khz */ unsigned int i; @@ -672,17 +681,17 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) goto err_free; } - data->acpi_data = per_cpu_ptr(acpi_perf_data, cpu); - per_cpu(acfreq_data, cpu) = data; + perf = per_cpu_ptr(acpi_perf_data, cpu); + data->acpi_perf_cpu = cpu; + policy->driver_data = data; if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) acpi_cpufreq_driver.flags |= CPUFREQ_CONST_LOOPS; - result = acpi_processor_register_performance(data->acpi_data, cpu); + result = acpi_processor_register_performance(perf, cpu); if (result) goto err_free_mask; - perf = data->acpi_data; policy->shared_type = perf->shared_type; /* @@ -838,26 +847,25 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) err_freqfree: kfree(data->freq_table); err_unreg: - acpi_processor_unregister_performance(perf, cpu); + acpi_processor_unregister_performance(cpu); err_free_mask: free_cpumask_var(data->freqdomain_cpus); err_free: kfree(data); - per_cpu(acfreq_data, cpu) = NULL; + policy->driver_data = NULL; return result; } static int acpi_cpufreq_cpu_exit(struct cpufreq_policy *policy) { - struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); + struct acpi_cpufreq_data *data = policy->driver_data; pr_debug("acpi_cpufreq_cpu_exit\n"); if (data) { - per_cpu(acfreq_data, policy->cpu) = NULL; - acpi_processor_unregister_performance(data->acpi_data, - policy->cpu); + policy->driver_data = NULL; + acpi_processor_unregister_performance(data->acpi_perf_cpu); free_cpumask_var(data->freqdomain_cpus); kfree(data->freq_table); kfree(data); @@ -868,7 +876,7 @@ static int acpi_cpufreq_cpu_exit(struct cpufreq_policy *policy) static int acpi_cpufreq_resume(struct cpufreq_policy *policy) { - struct acpi_cpufreq_data *data = per_cpu(acfreq_data, policy->cpu); + struct acpi_cpufreq_data *data = policy->driver_data; pr_debug("acpi_cpufreq_resume\n"); @@ -880,7 +888,9 @@ static int acpi_cpufreq_resume(struct cpufreq_policy *policy) static struct freq_attr *acpi_cpufreq_attr[] = { &cpufreq_freq_attr_scaling_available_freqs, &freqdomain_cpus, - NULL, /* this is a placeholder for cpb, do not remove */ +#ifdef CONFIG_X86_ACPI_CPUFREQ_CPB + &cpb, +#endif NULL, }; @@ -953,17 +963,16 @@ static int __init acpi_cpufreq_init(void) * only if configured. This is considered legacy code, which * will probably be removed at some point in the future. */ - if (check_amd_hwpstate_cpu(0)) { - struct freq_attr **iter; + if (!check_amd_hwpstate_cpu(0)) { + struct freq_attr **attr; - pr_debug("adding sysfs entry for cpb\n"); + pr_debug("CPB unsupported, do not expose it\n"); - for (iter = acpi_cpufreq_attr; *iter != NULL; iter++) - ; - - /* make sure there is a terminator behind it */ - if (iter[1] == NULL) - *iter = &cpb; + for (attr = acpi_cpufreq_attr; *attr; attr++) + if (*attr == &cpb) { + *attr = NULL; + break; + } } #endif acpi_cpufreq_boost_init(); diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c index 528a82bf5038..7c0d70e2a861 100644 --- a/drivers/cpufreq/cpufreq-dt.c +++ b/drivers/cpufreq/cpufreq-dt.c @@ -36,6 +36,12 @@ struct private_data { unsigned int voltage_tolerance; /* in percentage */ }; +static struct freq_attr *cpufreq_dt_attr[] = { + &cpufreq_freq_attr_scaling_available_freqs, + NULL, /* Extra space for boost-attr if required */ + NULL, +}; + static int set_target(struct cpufreq_policy *policy, unsigned int index) { struct dev_pm_opp *opp; @@ -184,15 +190,16 @@ static int allocate_resources(int cpu, struct device **cdev, static int cpufreq_init(struct cpufreq_policy *policy) { - struct cpufreq_dt_platform_data *pd; struct cpufreq_frequency_table *freq_table; struct device_node *np; struct private_data *priv; struct device *cpu_dev; struct regulator *cpu_reg; struct clk *cpu_clk; + struct dev_pm_opp *suspend_opp; unsigned long min_uV = ~0, max_uV = 0; unsigned int transition_latency; + bool need_update = false; int ret; ret = allocate_resources(policy->cpu, &cpu_dev, &cpu_reg, &cpu_clk); @@ -208,8 +215,30 @@ static int cpufreq_init(struct cpufreq_policy *policy) goto out_put_reg_clk; } - /* OPPs might be populated at runtime, don't check for error here */ - of_init_opp_table(cpu_dev); + /* Get OPP-sharing information from "operating-points-v2" bindings */ + ret = of_get_cpus_sharing_opps(cpu_dev, policy->cpus); + if (ret) { + /* + * operating-points-v2 not supported, fallback to old method of + * finding shared-OPPs for backward compatibility. + */ + if (ret == -ENOENT) + need_update = true; + else + goto out_node_put; + } + + /* + * Initialize OPP tables for all policy->cpus. They will be shared by + * all CPUs which have marked their CPUs shared with OPP bindings. + * + * For platforms not using operating-points-v2 bindings, we do this + * before updating policy->cpus. Otherwise, we will end up creating + * duplicate OPPs for policy->cpus. + * + * OPPs might be populated at runtime, don't check for error here + */ + of_cpumask_init_opp_table(policy->cpus); /* * But we need OPP table to function so if it is not there let's @@ -222,6 +251,26 @@ static int cpufreq_init(struct cpufreq_policy *policy) goto out_free_opp; } + if (need_update) { + struct cpufreq_dt_platform_data *pd = cpufreq_get_driver_data(); + + if (!pd || !pd->independent_clocks) + cpumask_setall(policy->cpus); + + /* + * OPP tables are initialized only for policy->cpu, do it for + * others as well. + */ + ret = set_cpus_sharing_opps(cpu_dev, policy->cpus); + if (ret) + dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", + __func__, ret); + + of_property_read_u32(np, "clock-latency", &transition_latency); + } else { + transition_latency = dev_pm_opp_get_max_clock_latency(cpu_dev); + } + priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) { ret = -ENOMEM; @@ -230,7 +279,7 @@ static int cpufreq_init(struct cpufreq_policy *policy) of_property_read_u32(np, "voltage-tolerance", &priv->voltage_tolerance); - if (of_property_read_u32(np, "clock-latency", &transition_latency)) + if (!transition_latency) transition_latency = CPUFREQ_ETERNAL; if (!IS_ERR(cpu_reg)) { @@ -255,7 +304,8 @@ static int cpufreq_init(struct cpufreq_policy *policy) rcu_read_unlock(); tol_uV = opp_uV * priv->voltage_tolerance / 100; - if (regulator_is_supported_voltage(cpu_reg, opp_uV, + if (regulator_is_supported_voltage(cpu_reg, + opp_uV - tol_uV, opp_uV + tol_uV)) { if (opp_uV < min_uV) min_uV = opp_uV; @@ -284,6 +334,13 @@ static int cpufreq_init(struct cpufreq_policy *policy) policy->driver_data = priv; policy->clk = cpu_clk; + + rcu_read_lock(); + suspend_opp = dev_pm_opp_get_suspend_opp(cpu_dev); + if (suspend_opp) + policy->suspend_freq = dev_pm_opp_get_freq(suspend_opp) / 1000; + rcu_read_unlock(); + ret = cpufreq_table_validate_and_show(policy, freq_table); if (ret) { dev_err(cpu_dev, "%s: invalid frequency table: %d\n", __func__, @@ -291,11 +348,16 @@ static int cpufreq_init(struct cpufreq_policy *policy) goto out_free_cpufreq_table; } - policy->cpuinfo.transition_latency = transition_latency; + /* Support turbo/boost mode */ + if (policy_has_boost_freq(policy)) { + /* This gets disabled by core on driver unregister */ + ret = cpufreq_enable_boost_support(); + if (ret) + goto out_free_cpufreq_table; + cpufreq_dt_attr[1] = &cpufreq_freq_attr_scaling_boost_freqs; + } - pd = cpufreq_get_driver_data(); - if (!pd || !pd->independent_clocks) - cpumask_setall(policy->cpus); + policy->cpuinfo.transition_latency = transition_latency; of_node_put(np); @@ -306,7 +368,8 @@ static int cpufreq_init(struct cpufreq_policy *policy) out_free_priv: kfree(priv); out_free_opp: - of_free_opp_table(cpu_dev); + of_cpumask_free_opp_table(policy->cpus); +out_node_put: of_node_put(np); out_put_reg_clk: clk_put(cpu_clk); @@ -322,7 +385,7 @@ static int cpufreq_exit(struct cpufreq_policy *policy) cpufreq_cooling_unregister(priv->cdev); dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); - of_free_opp_table(priv->cpu_dev); + of_cpumask_free_opp_table(policy->related_cpus); clk_put(policy->clk); if (!IS_ERR(priv->cpu_reg)) regulator_put(priv->cpu_reg); @@ -367,7 +430,8 @@ static struct cpufreq_driver dt_cpufreq_driver = { .exit = cpufreq_exit, .ready = cpufreq_ready, .name = "cpufreq-dt", - .attr = cpufreq_generic_attr, + .attr = cpufreq_dt_attr, + .suspend = cpufreq_generic_suspend, }; static int dt_cpufreq_probe(struct platform_device *pdev) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 9bb09ce98d04..6633b3fa996e 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -112,12 +112,6 @@ static inline bool has_target(void) return cpufreq_driver->target_index || cpufreq_driver->target; } -/* - * rwsem to guarantee that cpufreq driver module doesn't unload during critical - * sections - */ -static DECLARE_RWSEM(cpufreq_rwsem); - /* internal prototypes */ static int __cpufreq_governor(struct cpufreq_policy *policy, unsigned int event); @@ -245,7 +239,7 @@ int cpufreq_generic_init(struct cpufreq_policy *policy, EXPORT_SYMBOL_GPL(cpufreq_generic_init); /* Only for cpufreq core internal use */ -struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu) +static struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu) { struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); @@ -277,10 +271,6 @@ EXPORT_SYMBOL_GPL(cpufreq_generic_get); * If corresponding call cpufreq_cpu_put() isn't made, the policy wouldn't be * freed as that depends on the kobj count. * - * It also takes a read-lock of 'cpufreq_rwsem' and doesn't put it back if a - * valid policy is found. This is done to make sure the driver doesn't get - * unregistered while the policy is being used. - * * Return: A valid policy on success, otherwise NULL on failure. */ struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu) @@ -291,9 +281,6 @@ struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu) if (WARN_ON(cpu >= nr_cpu_ids)) return NULL; - if (!down_read_trylock(&cpufreq_rwsem)) - return NULL; - /* get the cpufreq driver */ read_lock_irqsave(&cpufreq_driver_lock, flags); @@ -306,9 +293,6 @@ struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu) read_unlock_irqrestore(&cpufreq_driver_lock, flags); - if (!policy) - up_read(&cpufreq_rwsem); - return policy; } EXPORT_SYMBOL_GPL(cpufreq_cpu_get); @@ -320,13 +304,10 @@ EXPORT_SYMBOL_GPL(cpufreq_cpu_get); * * This decrements the kobject reference count incremented earlier by calling * cpufreq_cpu_get(). - * - * It also drops the read-lock of 'cpufreq_rwsem' taken at cpufreq_cpu_get(). */ void cpufreq_cpu_put(struct cpufreq_policy *policy) { kobject_put(&policy->kobj); - up_read(&cpufreq_rwsem); } EXPORT_SYMBOL_GPL(cpufreq_cpu_put); @@ -539,9 +520,6 @@ static int cpufreq_parse_governor(char *str_governor, unsigned int *policy, { int err = -EINVAL; - if (!cpufreq_driver) - goto out; - if (cpufreq_driver->setpolicy) { if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) { *policy = CPUFREQ_POLICY_PERFORMANCE; @@ -576,7 +554,6 @@ static int cpufreq_parse_governor(char *str_governor, unsigned int *policy, mutex_unlock(&cpufreq_governor_mutex); } -out: return err; } @@ -625,9 +602,7 @@ static ssize_t store_##file_name \ int ret, temp; \ struct cpufreq_policy new_policy; \ \ - ret = cpufreq_get_policy(&new_policy, policy->cpu); \ - if (ret) \ - return -EINVAL; \ + memcpy(&new_policy, policy, sizeof(*policy)); \ \ ret = sscanf(buf, "%u", &new_policy.object); \ if (ret != 1) \ @@ -681,9 +656,7 @@ static ssize_t store_scaling_governor(struct cpufreq_policy *policy, char str_governor[16]; struct cpufreq_policy new_policy; - ret = cpufreq_get_policy(&new_policy, policy->cpu); - if (ret) - return ret; + memcpy(&new_policy, policy, sizeof(*policy)); ret = sscanf(buf, "%15s", str_governor); if (ret != 1) @@ -694,14 +667,7 @@ static ssize_t store_scaling_governor(struct cpufreq_policy *policy, return -EINVAL; ret = cpufreq_set_policy(policy, &new_policy); - - policy->user_policy.policy = policy->policy; - policy->user_policy.governor = policy->governor; - - if (ret) - return ret; - else - return count; + return ret ? ret : count; } /** @@ -851,9 +817,6 @@ static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf) struct freq_attr *fattr = to_attr(attr); ssize_t ret; - if (!down_read_trylock(&cpufreq_rwsem)) - return -EINVAL; - down_read(&policy->rwsem); if (fattr->show) @@ -862,7 +825,6 @@ static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf) ret = -EIO; up_read(&policy->rwsem); - up_read(&cpufreq_rwsem); return ret; } @@ -879,9 +841,6 @@ static ssize_t store(struct kobject *kobj, struct attribute *attr, if (!cpu_online(policy->cpu)) goto unlock; - if (!down_read_trylock(&cpufreq_rwsem)) - goto unlock; - down_write(&policy->rwsem); /* Updating inactive policies is invalid, so avoid doing that. */ @@ -897,8 +856,6 @@ static ssize_t store(struct kobject *kobj, struct attribute *attr, unlock_policy_rwsem: up_write(&policy->rwsem); - - up_read(&cpufreq_rwsem); unlock: put_online_cpus(); @@ -1027,8 +984,7 @@ static void cpufreq_remove_dev_symlink(struct cpufreq_policy *policy) } } -static int cpufreq_add_dev_interface(struct cpufreq_policy *policy, - struct device *dev) +static int cpufreq_add_dev_interface(struct cpufreq_policy *policy) { struct freq_attr **drv_attr; int ret = 0; @@ -1060,11 +1016,10 @@ static int cpufreq_add_dev_interface(struct cpufreq_policy *policy, return cpufreq_add_dev_symlink(policy); } -static void cpufreq_init_policy(struct cpufreq_policy *policy) +static int cpufreq_init_policy(struct cpufreq_policy *policy) { struct cpufreq_governor *gov = NULL; struct cpufreq_policy new_policy; - int ret = 0; memcpy(&new_policy, policy, sizeof(*policy)); @@ -1083,16 +1038,10 @@ static void cpufreq_init_policy(struct cpufreq_policy *policy) cpufreq_parse_governor(gov->name, &new_policy.policy, NULL); /* set default policy */ - ret = cpufreq_set_policy(policy, &new_policy); - if (ret) { - pr_debug("setting policy failed\n"); - if (cpufreq_driver->exit) - cpufreq_driver->exit(policy); - } + return cpufreq_set_policy(policy, &new_policy); } -static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, - unsigned int cpu, struct device *dev) +static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu) { int ret = 0; @@ -1126,33 +1075,15 @@ static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, return 0; } -static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu) -{ - struct cpufreq_policy *policy; - unsigned long flags; - - read_lock_irqsave(&cpufreq_driver_lock, flags); - policy = per_cpu(cpufreq_cpu_data, cpu); - read_unlock_irqrestore(&cpufreq_driver_lock, flags); - - if (likely(policy)) { - /* Policy should be inactive here */ - WARN_ON(!policy_is_inactive(policy)); - - down_write(&policy->rwsem); - policy->cpu = cpu; - policy->governor = NULL; - up_write(&policy->rwsem); - } - - return policy; -} - -static struct cpufreq_policy *cpufreq_policy_alloc(struct device *dev) +static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) { + struct device *dev = get_cpu_device(cpu); struct cpufreq_policy *policy; int ret; + if (WARN_ON(!dev)) + return NULL; + policy = kzalloc(sizeof(*policy), GFP_KERNEL); if (!policy) return NULL; @@ -1180,10 +1111,10 @@ static struct cpufreq_policy *cpufreq_policy_alloc(struct device *dev) init_completion(&policy->kobj_unregister); INIT_WORK(&policy->update, handle_update); - policy->cpu = dev->id; + policy->cpu = cpu; /* Set this once on allocation */ - policy->kobj_cpu = dev->id; + policy->kobj_cpu = cpu; return policy; @@ -1245,59 +1176,34 @@ static void cpufreq_policy_free(struct cpufreq_policy *policy, bool notify) kfree(policy); } -/** - * cpufreq_add_dev - add a CPU device - * - * Adds the cpufreq interface for a CPU device. - * - * The Oracle says: try running cpufreq registration/unregistration concurrently - * with with cpu hotplugging and all hell will break loose. Tried to clean this - * mess up, but more thorough testing is needed. - Mathieu - */ -static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) +static int cpufreq_online(unsigned int cpu) { - unsigned int j, cpu = dev->id; - int ret = -ENOMEM; struct cpufreq_policy *policy; + bool new_policy; unsigned long flags; - bool recover_policy = !sif; + unsigned int j; + int ret; - pr_debug("adding CPU %u\n", cpu); - - if (cpu_is_offline(cpu)) { - /* - * Only possible if we are here from the subsys_interface add - * callback. A hotplug notifier will follow and we will handle - * it as CPU online then. For now, just create the sysfs link, - * unless there is no policy or the link is already present. - */ - policy = per_cpu(cpufreq_cpu_data, cpu); - return policy && !cpumask_test_and_set_cpu(cpu, policy->real_cpus) - ? add_cpu_dev_symlink(policy, cpu) : 0; - } - - if (!down_read_trylock(&cpufreq_rwsem)) - return 0; + pr_debug("%s: bringing CPU%u online\n", __func__, cpu); /* Check if this CPU already has a policy to manage it */ policy = per_cpu(cpufreq_cpu_data, cpu); - if (policy && !policy_is_inactive(policy)) { + if (policy) { WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus)); - ret = cpufreq_add_policy_cpu(policy, cpu, dev); - up_read(&cpufreq_rwsem); - return ret; - } + if (!policy_is_inactive(policy)) + return cpufreq_add_policy_cpu(policy, cpu); - /* - * Restore the saved policy when doing light-weight init and fall back - * to the full init if that fails. - */ - policy = recover_policy ? cpufreq_policy_restore(cpu) : NULL; - if (!policy) { - recover_policy = false; - policy = cpufreq_policy_alloc(dev); + /* This is the only online CPU for the policy. Start over. */ + new_policy = false; + down_write(&policy->rwsem); + policy->cpu = cpu; + policy->governor = NULL; + up_write(&policy->rwsem); + } else { + new_policy = true; + policy = cpufreq_policy_alloc(cpu); if (!policy) - goto nomem_out; + return -ENOMEM; } cpumask_copy(policy->cpus, cpumask_of(cpu)); @@ -1308,17 +1214,17 @@ static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) ret = cpufreq_driver->init(policy); if (ret) { pr_debug("initialization failed\n"); - goto err_set_policy_cpu; + goto out_free_policy; } down_write(&policy->rwsem); - /* related cpus should atleast have policy->cpus */ - cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus); - - /* Remember which CPUs have been present at the policy creation time. */ - if (!recover_policy) + if (new_policy) { + /* related_cpus should at least include policy->cpus. */ + cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus); + /* Remember CPUs present at the policy creation time. */ cpumask_and(policy->real_cpus, policy->cpus, cpu_present_mask); + } /* * affected cpus must always be the one, which are online. We aren't @@ -1326,7 +1232,7 @@ static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) */ cpumask_and(policy->cpus, policy->cpus, cpu_online_mask); - if (!recover_policy) { + if (new_policy) { policy->user_policy.min = policy->min; policy->user_policy.max = policy->max; @@ -1340,7 +1246,7 @@ static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) policy->cur = cpufreq_driver->get(policy->cpu); if (!policy->cur) { pr_err("%s: ->get() failed\n", __func__); - goto err_get_freq; + goto out_exit_policy; } } @@ -1387,10 +1293,10 @@ static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) blocking_notifier_call_chain(&cpufreq_policy_notifier_list, CPUFREQ_START, policy); - if (!recover_policy) { - ret = cpufreq_add_dev_interface(policy, dev); + if (new_policy) { + ret = cpufreq_add_dev_interface(policy); if (ret) - goto err_out_unregister; + goto out_exit_policy; blocking_notifier_call_chain(&cpufreq_policy_notifier_list, CPUFREQ_CREATE_POLICY, policy); @@ -1399,18 +1305,19 @@ static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) write_unlock_irqrestore(&cpufreq_driver_lock, flags); } - cpufreq_init_policy(policy); - - if (!recover_policy) { - policy->user_policy.policy = policy->policy; - policy->user_policy.governor = policy->governor; + ret = cpufreq_init_policy(policy); + if (ret) { + pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n", + __func__, cpu, ret); + /* cpufreq_policy_free() will notify based on this */ + new_policy = false; + goto out_exit_policy; } + up_write(&policy->rwsem); kobject_uevent(&policy->kobj, KOBJ_ADD); - up_read(&cpufreq_rwsem); - /* Callback for handling stuff after policy is ready */ if (cpufreq_driver->ready) cpufreq_driver->ready(policy); @@ -1419,24 +1326,47 @@ static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) return 0; -err_out_unregister: -err_get_freq: +out_exit_policy: up_write(&policy->rwsem); if (cpufreq_driver->exit) cpufreq_driver->exit(policy); -err_set_policy_cpu: - cpufreq_policy_free(policy, recover_policy); -nomem_out: - up_read(&cpufreq_rwsem); +out_free_policy: + cpufreq_policy_free(policy, !new_policy); + return ret; +} + +/** + * cpufreq_add_dev - the cpufreq interface for a CPU device. + * @dev: CPU device. + * @sif: Subsystem interface structure pointer (not used) + */ +static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif) +{ + unsigned cpu = dev->id; + int ret; + + dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu); + + if (cpu_online(cpu)) { + ret = cpufreq_online(cpu); + } else { + /* + * A hotplug notifier will follow and we will handle it as CPU + * online then. For now, just create the sysfs link, unless + * there is no policy or the link is already present. + */ + struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); + + ret = policy && !cpumask_test_and_set_cpu(cpu, policy->real_cpus) + ? add_cpu_dev_symlink(policy, cpu) : 0; + } return ret; } -static int __cpufreq_remove_dev_prepare(struct device *dev) +static void cpufreq_offline_prepare(unsigned int cpu) { - unsigned int cpu = dev->id; - int ret = 0; struct cpufreq_policy *policy; pr_debug("%s: unregistering CPU %u\n", __func__, cpu); @@ -1444,11 +1374,11 @@ static int __cpufreq_remove_dev_prepare(struct device *dev) policy = cpufreq_cpu_get_raw(cpu); if (!policy) { pr_debug("%s: No cpu_data found\n", __func__); - return -EINVAL; + return; } if (has_target()) { - ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP); + int ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP); if (ret) pr_err("%s: Failed to stop governor\n", __func__); } @@ -1469,7 +1399,7 @@ static int __cpufreq_remove_dev_prepare(struct device *dev) /* Start governor again for active policy */ if (!policy_is_inactive(policy)) { if (has_target()) { - ret = __cpufreq_governor(policy, CPUFREQ_GOV_START); + int ret = __cpufreq_governor(policy, CPUFREQ_GOV_START); if (!ret) ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS); @@ -1479,28 +1409,24 @@ static int __cpufreq_remove_dev_prepare(struct device *dev) } else if (cpufreq_driver->stop_cpu) { cpufreq_driver->stop_cpu(policy); } - - return ret; } -static int __cpufreq_remove_dev_finish(struct device *dev) +static void cpufreq_offline_finish(unsigned int cpu) { - unsigned int cpu = dev->id; - int ret; struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); if (!policy) { pr_debug("%s: No cpu_data found\n", __func__); - return -EINVAL; + return; } /* Only proceed for inactive policies */ if (!policy_is_inactive(policy)) - return 0; + return; /* If cpu is last user of policy, free policy */ if (has_target()) { - ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); + int ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); if (ret) pr_err("%s: Failed to exit governor\n", __func__); } @@ -1512,8 +1438,6 @@ static int __cpufreq_remove_dev_finish(struct device *dev) */ if (cpufreq_driver->exit) cpufreq_driver->exit(policy); - - return 0; } /** @@ -1530,8 +1454,8 @@ static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif) return; if (cpu_online(cpu)) { - __cpufreq_remove_dev_prepare(dev); - __cpufreq_remove_dev_finish(dev); + cpufreq_offline_prepare(cpu); + cpufreq_offline_finish(cpu); } cpumask_clear_cpu(cpu, policy->real_cpus); @@ -1702,8 +1626,8 @@ int cpufreq_generic_suspend(struct cpufreq_policy *policy) int ret; if (!policy->suspend_freq) { - pr_err("%s: suspend_freq can't be zero\n", __func__); - return -EINVAL; + pr_debug("%s: suspend_freq not defined\n", __func__); + return 0; } pr_debug("%s: Setting suspend-freq: %u\n", __func__, @@ -2107,8 +2031,7 @@ static int __cpufreq_governor(struct cpufreq_policy *policy, if (!try_module_get(policy->governor->owner)) return -EINVAL; - pr_debug("__cpufreq_governor for CPU %u, event %u\n", - policy->cpu, event); + pr_debug("%s: for CPU %u, event %u\n", __func__, policy->cpu, event); mutex_lock(&cpufreq_governor_lock); if ((policy->governor_enabled && event == CPUFREQ_GOV_START) @@ -2245,7 +2168,11 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo)); - if (new_policy->min > policy->max || new_policy->max < policy->min) + /* + * This check works well when we store new min/max freq attributes, + * because new_policy is a copy of policy with one field updated. + */ + if (new_policy->min > new_policy->max) return -EINVAL; /* verify the cpu speed can be set within this limit */ @@ -2257,10 +2184,6 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, blocking_notifier_call_chain(&cpufreq_policy_notifier_list, CPUFREQ_ADJUST, new_policy); - /* adjust if necessary - hardware incompatibility*/ - blocking_notifier_call_chain(&cpufreq_policy_notifier_list, - CPUFREQ_INCOMPATIBLE, new_policy); - /* * verify the cpu speed can be set within this limit, which might be * different to the first one @@ -2294,16 +2217,31 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, old_gov = policy->governor; /* end old governor */ if (old_gov) { - __cpufreq_governor(policy, CPUFREQ_GOV_STOP); + ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP); + if (ret) { + /* This can happen due to race with other operations */ + pr_debug("%s: Failed to Stop Governor: %s (%d)\n", + __func__, old_gov->name, ret); + return ret; + } + up_write(&policy->rwsem); - __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); + ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); down_write(&policy->rwsem); + + if (ret) { + pr_err("%s: Failed to Exit Governor: %s (%d)\n", + __func__, old_gov->name, ret); + return ret; + } } /* start new governor */ policy->governor = new_policy->governor; - if (!__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) { - if (!__cpufreq_governor(policy, CPUFREQ_GOV_START)) + ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT); + if (!ret) { + ret = __cpufreq_governor(policy, CPUFREQ_GOV_START); + if (!ret) goto out; up_write(&policy->rwsem); @@ -2315,11 +2253,13 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, pr_debug("starting governor %s failed\n", policy->governor->name); if (old_gov) { policy->governor = old_gov; - __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT); - __cpufreq_governor(policy, CPUFREQ_GOV_START); + if (__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) + policy->governor = NULL; + else + __cpufreq_governor(policy, CPUFREQ_GOV_START); } - return -EINVAL; + return ret; out: pr_debug("governor: change or update limits\n"); @@ -2348,8 +2288,6 @@ int cpufreq_update_policy(unsigned int cpu) memcpy(&new_policy, policy, sizeof(*policy)); new_policy.min = policy->user_policy.min; new_policy.max = policy->user_policy.max; - new_policy.policy = policy->user_policy.policy; - new_policy.governor = policy->user_policy.governor; /* * BIOS might change freq behind our back @@ -2385,27 +2323,23 @@ static int cpufreq_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; - struct device *dev; - dev = get_cpu_device(cpu); - if (dev) { - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_ONLINE: - cpufreq_add_dev(dev, NULL); - break; + switch (action & ~CPU_TASKS_FROZEN) { + case CPU_ONLINE: + cpufreq_online(cpu); + break; - case CPU_DOWN_PREPARE: - __cpufreq_remove_dev_prepare(dev); - break; + case CPU_DOWN_PREPARE: + cpufreq_offline_prepare(cpu); + break; - case CPU_POST_DEAD: - __cpufreq_remove_dev_finish(dev); - break; + case CPU_POST_DEAD: + cpufreq_offline_finish(cpu); + break; - case CPU_DOWN_FAILED: - cpufreq_add_dev(dev, NULL); - break; - } + case CPU_DOWN_FAILED: + cpufreq_online(cpu); + break; } return NOTIFY_OK; } @@ -2475,6 +2409,49 @@ int cpufreq_boost_supported(void) } EXPORT_SYMBOL_GPL(cpufreq_boost_supported); +static int create_boost_sysfs_file(void) +{ + int ret; + + if (!cpufreq_boost_supported()) + return 0; + + /* + * Check if driver provides function to enable boost - + * if not, use cpufreq_boost_set_sw as default + */ + if (!cpufreq_driver->set_boost) + cpufreq_driver->set_boost = cpufreq_boost_set_sw; + + ret = cpufreq_sysfs_create_file(&boost.attr); + if (ret) + pr_err("%s: cannot register global BOOST sysfs file\n", + __func__); + + return ret; +} + +static void remove_boost_sysfs_file(void) +{ + if (cpufreq_boost_supported()) + cpufreq_sysfs_remove_file(&boost.attr); +} + +int cpufreq_enable_boost_support(void) +{ + if (!cpufreq_driver) + return -EINVAL; + + if (cpufreq_boost_supported()) + return 0; + + cpufreq_driver->boost_supported = true; + + /* This will get removed on driver unregister */ + return create_boost_sysfs_file(); +} +EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support); + int cpufreq_boost_enabled(void) { return cpufreq_driver->boost_enabled; @@ -2513,10 +2490,14 @@ int cpufreq_register_driver(struct cpufreq_driver *driver_data) pr_debug("trying to register driver %s\n", driver_data->name); + /* Protect against concurrent CPU online/offline. */ + get_online_cpus(); + write_lock_irqsave(&cpufreq_driver_lock, flags); if (cpufreq_driver) { write_unlock_irqrestore(&cpufreq_driver_lock, flags); - return -EEXIST; + ret = -EEXIST; + goto out; } cpufreq_driver = driver_data; write_unlock_irqrestore(&cpufreq_driver_lock, flags); @@ -2524,21 +2505,9 @@ int cpufreq_register_driver(struct cpufreq_driver *driver_data) if (driver_data->setpolicy) driver_data->flags |= CPUFREQ_CONST_LOOPS; - if (cpufreq_boost_supported()) { - /* - * Check if driver provides function to enable boost - - * if not, use cpufreq_boost_set_sw as default - */ - if (!cpufreq_driver->set_boost) - cpufreq_driver->set_boost = cpufreq_boost_set_sw; - - ret = cpufreq_sysfs_create_file(&boost.attr); - if (ret) { - pr_err("%s: cannot register global BOOST sysfs file\n", - __func__); - goto err_null_driver; - } - } + ret = create_boost_sysfs_file(); + if (ret) + goto err_null_driver; ret = subsys_interface_register(&cpufreq_interface); if (ret) @@ -2555,17 +2524,19 @@ int cpufreq_register_driver(struct cpufreq_driver *driver_data) register_hotcpu_notifier(&cpufreq_cpu_notifier); pr_debug("driver %s up and running\n", driver_data->name); - return 0; +out: + put_online_cpus(); + return ret; + err_if_unreg: subsys_interface_unregister(&cpufreq_interface); err_boost_unreg: - if (cpufreq_boost_supported()) - cpufreq_sysfs_remove_file(&boost.attr); + remove_boost_sysfs_file(); err_null_driver: write_lock_irqsave(&cpufreq_driver_lock, flags); cpufreq_driver = NULL; write_unlock_irqrestore(&cpufreq_driver_lock, flags); - return ret; + goto out; } EXPORT_SYMBOL_GPL(cpufreq_register_driver); @@ -2586,19 +2557,18 @@ int cpufreq_unregister_driver(struct cpufreq_driver *driver) pr_debug("unregistering driver %s\n", driver->name); + /* Protect against concurrent cpu hotplug */ + get_online_cpus(); subsys_interface_unregister(&cpufreq_interface); - if (cpufreq_boost_supported()) - cpufreq_sysfs_remove_file(&boost.attr); - + remove_boost_sysfs_file(); unregister_hotcpu_notifier(&cpufreq_cpu_notifier); - down_write(&cpufreq_rwsem); write_lock_irqsave(&cpufreq_driver_lock, flags); cpufreq_driver = NULL; write_unlock_irqrestore(&cpufreq_driver_lock, flags); - up_write(&cpufreq_rwsem); + put_online_cpus(); return 0; } diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c index c86a10c30912..84a1506950a7 100644 --- a/drivers/cpufreq/cpufreq_conservative.c +++ b/drivers/cpufreq/cpufreq_conservative.c @@ -47,7 +47,7 @@ static inline unsigned int get_freq_target(struct cs_dbs_tuners *cs_tuners, static void cs_check_cpu(int cpu, unsigned int load) { struct cs_cpu_dbs_info_s *dbs_info = &per_cpu(cs_cpu_dbs_info, cpu); - struct cpufreq_policy *policy = dbs_info->cdbs.cur_policy; + struct cpufreq_policy *policy = dbs_info->cdbs.shared->policy; struct dbs_data *dbs_data = policy->governor_data; struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; @@ -102,26 +102,15 @@ static void cs_check_cpu(int cpu, unsigned int load) } } -static void cs_dbs_timer(struct work_struct *work) +static unsigned int cs_dbs_timer(struct cpu_dbs_info *cdbs, + struct dbs_data *dbs_data, bool modify_all) { - struct cs_cpu_dbs_info_s *dbs_info = container_of(work, - struct cs_cpu_dbs_info_s, cdbs.work.work); - unsigned int cpu = dbs_info->cdbs.cur_policy->cpu; - struct cs_cpu_dbs_info_s *core_dbs_info = &per_cpu(cs_cpu_dbs_info, - cpu); - struct dbs_data *dbs_data = dbs_info->cdbs.cur_policy->governor_data; struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; - int delay = delay_for_sampling_rate(cs_tuners->sampling_rate); - bool modify_all = true; - mutex_lock(&core_dbs_info->cdbs.timer_mutex); - if (!need_load_eval(&core_dbs_info->cdbs, cs_tuners->sampling_rate)) - modify_all = false; - else - dbs_check_cpu(dbs_data, cpu); + if (modify_all) + dbs_check_cpu(dbs_data, cdbs->shared->policy->cpu); - gov_queue_work(dbs_data, dbs_info->cdbs.cur_policy, delay, modify_all); - mutex_unlock(&core_dbs_info->cdbs.timer_mutex); + return delay_for_sampling_rate(cs_tuners->sampling_rate); } static int dbs_cpufreq_notifier(struct notifier_block *nb, unsigned long val, @@ -135,7 +124,7 @@ static int dbs_cpufreq_notifier(struct notifier_block *nb, unsigned long val, if (!dbs_info->enable) return 0; - policy = dbs_info->cdbs.cur_policy; + policy = dbs_info->cdbs.shared->policy; /* * we only care if our internally tracked freq moves outside the 'valid' diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c index 57a39f8a92b7..939197ffa4ac 100644 --- a/drivers/cpufreq/cpufreq_governor.c +++ b/drivers/cpufreq/cpufreq_governor.c @@ -32,10 +32,10 @@ static struct attribute_group *get_sysfs_attr(struct dbs_data *dbs_data) void dbs_check_cpu(struct dbs_data *dbs_data, int cpu) { - struct cpu_dbs_common_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu); + struct cpu_dbs_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu); struct od_dbs_tuners *od_tuners = dbs_data->tuners; struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; - struct cpufreq_policy *policy; + struct cpufreq_policy *policy = cdbs->shared->policy; unsigned int sampling_rate; unsigned int max_load = 0; unsigned int ignore_nice; @@ -60,11 +60,9 @@ void dbs_check_cpu(struct dbs_data *dbs_data, int cpu) ignore_nice = cs_tuners->ignore_nice_load; } - policy = cdbs->cur_policy; - /* Get Absolute Load */ for_each_cpu(j, policy->cpus) { - struct cpu_dbs_common_info *j_cdbs; + struct cpu_dbs_info *j_cdbs; u64 cur_wall_time, cur_idle_time; unsigned int idle_time, wall_time; unsigned int load; @@ -163,9 +161,9 @@ EXPORT_SYMBOL_GPL(dbs_check_cpu); static inline void __gov_queue_work(int cpu, struct dbs_data *dbs_data, unsigned int delay) { - struct cpu_dbs_common_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu); + struct cpu_dbs_info *cdbs = dbs_data->cdata->get_cpu_cdbs(cpu); - mod_delayed_work_on(cpu, system_wq, &cdbs->work, delay); + mod_delayed_work_on(cpu, system_wq, &cdbs->dwork, delay); } void gov_queue_work(struct dbs_data *dbs_data, struct cpufreq_policy *policy, @@ -199,33 +197,63 @@ EXPORT_SYMBOL_GPL(gov_queue_work); static inline void gov_cancel_work(struct dbs_data *dbs_data, struct cpufreq_policy *policy) { - struct cpu_dbs_common_info *cdbs; + struct cpu_dbs_info *cdbs; int i; for_each_cpu(i, policy->cpus) { cdbs = dbs_data->cdata->get_cpu_cdbs(i); - cancel_delayed_work_sync(&cdbs->work); + cancel_delayed_work_sync(&cdbs->dwork); } } /* Will return if we need to evaluate cpu load again or not */ -bool need_load_eval(struct cpu_dbs_common_info *cdbs, - unsigned int sampling_rate) +static bool need_load_eval(struct cpu_common_dbs_info *shared, + unsigned int sampling_rate) { - if (policy_is_shared(cdbs->cur_policy)) { + if (policy_is_shared(shared->policy)) { ktime_t time_now = ktime_get(); - s64 delta_us = ktime_us_delta(time_now, cdbs->time_stamp); + s64 delta_us = ktime_us_delta(time_now, shared->time_stamp); /* Do nothing if we recently have sampled */ if (delta_us < (s64)(sampling_rate / 2)) return false; else - cdbs->time_stamp = time_now; + shared->time_stamp = time_now; } return true; } -EXPORT_SYMBOL_GPL(need_load_eval); + +static void dbs_timer(struct work_struct *work) +{ + struct cpu_dbs_info *cdbs = container_of(work, struct cpu_dbs_info, + dwork.work); + struct cpu_common_dbs_info *shared = cdbs->shared; + struct cpufreq_policy *policy = shared->policy; + struct dbs_data *dbs_data = policy->governor_data; + unsigned int sampling_rate, delay; + bool modify_all = true; + + mutex_lock(&shared->timer_mutex); + + if (dbs_data->cdata->governor == GOV_CONSERVATIVE) { + struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; + + sampling_rate = cs_tuners->sampling_rate; + } else { + struct od_dbs_tuners *od_tuners = dbs_data->tuners; + + sampling_rate = od_tuners->sampling_rate; + } + + if (!need_load_eval(cdbs->shared, sampling_rate)) + modify_all = false; + + delay = dbs_data->cdata->gov_dbs_timer(cdbs, dbs_data, modify_all); + gov_queue_work(dbs_data, policy, delay, modify_all); + + mutex_unlock(&shared->timer_mutex); +} static void set_sampling_rate(struct dbs_data *dbs_data, unsigned int sampling_rate) @@ -239,6 +267,37 @@ static void set_sampling_rate(struct dbs_data *dbs_data, } } +static int alloc_common_dbs_info(struct cpufreq_policy *policy, + struct common_dbs_data *cdata) +{ + struct cpu_common_dbs_info *shared; + int j; + + /* Allocate memory for the common information for policy->cpus */ + shared = kzalloc(sizeof(*shared), GFP_KERNEL); + if (!shared) + return -ENOMEM; + + /* Set shared for all CPUs, online+offline */ + for_each_cpu(j, policy->related_cpus) + cdata->get_cpu_cdbs(j)->shared = shared; + + return 0; +} + +static void free_common_dbs_info(struct cpufreq_policy *policy, + struct common_dbs_data *cdata) +{ + struct cpu_dbs_info *cdbs = cdata->get_cpu_cdbs(policy->cpu); + struct cpu_common_dbs_info *shared = cdbs->shared; + int j; + + for_each_cpu(j, policy->cpus) + cdata->get_cpu_cdbs(j)->shared = NULL; + + kfree(shared); +} + static int cpufreq_governor_init(struct cpufreq_policy *policy, struct dbs_data *dbs_data, struct common_dbs_data *cdata) @@ -246,9 +305,18 @@ static int cpufreq_governor_init(struct cpufreq_policy *policy, unsigned int latency; int ret; + /* State should be equivalent to EXIT */ + if (policy->governor_data) + return -EBUSY; + if (dbs_data) { if (WARN_ON(have_governor_per_policy())) return -EINVAL; + + ret = alloc_common_dbs_info(policy, cdata); + if (ret) + return ret; + dbs_data->usage_count++; policy->governor_data = dbs_data; return 0; @@ -258,12 +326,16 @@ static int cpufreq_governor_init(struct cpufreq_policy *policy, if (!dbs_data) return -ENOMEM; + ret = alloc_common_dbs_info(policy, cdata); + if (ret) + goto free_dbs_data; + dbs_data->cdata = cdata; dbs_data->usage_count = 1; ret = cdata->init(dbs_data, !policy->governor->initialized); if (ret) - goto free_dbs_data; + goto free_common_dbs_info; /* policy latency is in ns. Convert it to us first */ latency = policy->cpuinfo.transition_latency / 1000; @@ -300,15 +372,22 @@ static int cpufreq_governor_init(struct cpufreq_policy *policy, } cdata_exit: cdata->exit(dbs_data, !policy->governor->initialized); +free_common_dbs_info: + free_common_dbs_info(policy, cdata); free_dbs_data: kfree(dbs_data); return ret; } -static void cpufreq_governor_exit(struct cpufreq_policy *policy, - struct dbs_data *dbs_data) +static int cpufreq_governor_exit(struct cpufreq_policy *policy, + struct dbs_data *dbs_data) { struct common_dbs_data *cdata = dbs_data->cdata; + struct cpu_dbs_info *cdbs = cdata->get_cpu_cdbs(policy->cpu); + + /* State should be equivalent to INIT */ + if (!cdbs->shared || cdbs->shared->policy) + return -EBUSY; policy->governor_data = NULL; if (!--dbs_data->usage_count) { @@ -323,6 +402,9 @@ static void cpufreq_governor_exit(struct cpufreq_policy *policy, cdata->exit(dbs_data, policy->governor->initialized == 1); kfree(dbs_data); } + + free_common_dbs_info(policy, cdata); + return 0; } static int cpufreq_governor_start(struct cpufreq_policy *policy, @@ -330,12 +412,17 @@ static int cpufreq_governor_start(struct cpufreq_policy *policy, { struct common_dbs_data *cdata = dbs_data->cdata; unsigned int sampling_rate, ignore_nice, j, cpu = policy->cpu; - struct cpu_dbs_common_info *cpu_cdbs = cdata->get_cpu_cdbs(cpu); + struct cpu_dbs_info *cdbs = cdata->get_cpu_cdbs(cpu); + struct cpu_common_dbs_info *shared = cdbs->shared; int io_busy = 0; if (!policy->cur) return -EINVAL; + /* State should be equivalent to INIT */ + if (!shared || shared->policy) + return -EBUSY; + if (cdata->governor == GOV_CONSERVATIVE) { struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; @@ -349,12 +436,14 @@ static int cpufreq_governor_start(struct cpufreq_policy *policy, io_busy = od_tuners->io_is_busy; } + shared->policy = policy; + shared->time_stamp = ktime_get(); + mutex_init(&shared->timer_mutex); + for_each_cpu(j, policy->cpus) { - struct cpu_dbs_common_info *j_cdbs = cdata->get_cpu_cdbs(j); + struct cpu_dbs_info *j_cdbs = cdata->get_cpu_cdbs(j); unsigned int prev_load; - j_cdbs->cpu = j; - j_cdbs->cur_policy = policy; j_cdbs->prev_cpu_idle = get_cpu_idle_time(j, &j_cdbs->prev_cpu_wall, io_busy); @@ -366,8 +455,7 @@ static int cpufreq_governor_start(struct cpufreq_policy *policy, if (ignore_nice) j_cdbs->prev_cpu_nice = kcpustat_cpu(j).cpustat[CPUTIME_NICE]; - mutex_init(&j_cdbs->timer_mutex); - INIT_DEFERRABLE_WORK(&j_cdbs->work, cdata->gov_dbs_timer); + INIT_DEFERRABLE_WORK(&j_cdbs->dwork, dbs_timer); } if (cdata->governor == GOV_CONSERVATIVE) { @@ -386,20 +474,24 @@ static int cpufreq_governor_start(struct cpufreq_policy *policy, od_ops->powersave_bias_init_cpu(cpu); } - /* Initiate timer time stamp */ - cpu_cdbs->time_stamp = ktime_get(); - gov_queue_work(dbs_data, policy, delay_for_sampling_rate(sampling_rate), true); return 0; } -static void cpufreq_governor_stop(struct cpufreq_policy *policy, - struct dbs_data *dbs_data) +static int cpufreq_governor_stop(struct cpufreq_policy *policy, + struct dbs_data *dbs_data) { struct common_dbs_data *cdata = dbs_data->cdata; unsigned int cpu = policy->cpu; - struct cpu_dbs_common_info *cpu_cdbs = cdata->get_cpu_cdbs(cpu); + struct cpu_dbs_info *cdbs = cdata->get_cpu_cdbs(cpu); + struct cpu_common_dbs_info *shared = cdbs->shared; + + /* State should be equivalent to START */ + if (!shared || !shared->policy) + return -EBUSY; + + gov_cancel_work(dbs_data, policy); if (cdata->governor == GOV_CONSERVATIVE) { struct cs_cpu_dbs_info_s *cs_dbs_info = @@ -408,38 +500,40 @@ static void cpufreq_governor_stop(struct cpufreq_policy *policy, cs_dbs_info->enable = 0; } - gov_cancel_work(dbs_data, policy); - - mutex_destroy(&cpu_cdbs->timer_mutex); - cpu_cdbs->cur_policy = NULL; + shared->policy = NULL; + mutex_destroy(&shared->timer_mutex); + return 0; } -static void cpufreq_governor_limits(struct cpufreq_policy *policy, - struct dbs_data *dbs_data) +static int cpufreq_governor_limits(struct cpufreq_policy *policy, + struct dbs_data *dbs_data) { struct common_dbs_data *cdata = dbs_data->cdata; unsigned int cpu = policy->cpu; - struct cpu_dbs_common_info *cpu_cdbs = cdata->get_cpu_cdbs(cpu); + struct cpu_dbs_info *cdbs = cdata->get_cpu_cdbs(cpu); - if (!cpu_cdbs->cur_policy) - return; + /* State should be equivalent to START */ + if (!cdbs->shared || !cdbs->shared->policy) + return -EBUSY; - mutex_lock(&cpu_cdbs->timer_mutex); - if (policy->max < cpu_cdbs->cur_policy->cur) - __cpufreq_driver_target(cpu_cdbs->cur_policy, policy->max, + mutex_lock(&cdbs->shared->timer_mutex); + if (policy->max < cdbs->shared->policy->cur) + __cpufreq_driver_target(cdbs->shared->policy, policy->max, CPUFREQ_RELATION_H); - else if (policy->min > cpu_cdbs->cur_policy->cur) - __cpufreq_driver_target(cpu_cdbs->cur_policy, policy->min, + else if (policy->min > cdbs->shared->policy->cur) + __cpufreq_driver_target(cdbs->shared->policy, policy->min, CPUFREQ_RELATION_L); dbs_check_cpu(dbs_data, cpu); - mutex_unlock(&cpu_cdbs->timer_mutex); + mutex_unlock(&cdbs->shared->timer_mutex); + + return 0; } int cpufreq_governor_dbs(struct cpufreq_policy *policy, struct common_dbs_data *cdata, unsigned int event) { struct dbs_data *dbs_data; - int ret = 0; + int ret; /* Lock governor to block concurrent initialization of governor */ mutex_lock(&cdata->mutex); @@ -449,7 +543,7 @@ int cpufreq_governor_dbs(struct cpufreq_policy *policy, else dbs_data = cdata->gdbs_data; - if (WARN_ON(!dbs_data && (event != CPUFREQ_GOV_POLICY_INIT))) { + if (!dbs_data && (event != CPUFREQ_GOV_POLICY_INIT)) { ret = -EINVAL; goto unlock; } @@ -459,17 +553,19 @@ int cpufreq_governor_dbs(struct cpufreq_policy *policy, ret = cpufreq_governor_init(policy, dbs_data, cdata); break; case CPUFREQ_GOV_POLICY_EXIT: - cpufreq_governor_exit(policy, dbs_data); + ret = cpufreq_governor_exit(policy, dbs_data); break; case CPUFREQ_GOV_START: ret = cpufreq_governor_start(policy, dbs_data); break; case CPUFREQ_GOV_STOP: - cpufreq_governor_stop(policy, dbs_data); + ret = cpufreq_governor_stop(policy, dbs_data); break; case CPUFREQ_GOV_LIMITS: - cpufreq_governor_limits(policy, dbs_data); + ret = cpufreq_governor_limits(policy, dbs_data); break; + default: + ret = -EINVAL; } unlock: diff --git a/drivers/cpufreq/cpufreq_governor.h b/drivers/cpufreq/cpufreq_governor.h index 34736f5e869d..50f171796632 100644 --- a/drivers/cpufreq/cpufreq_governor.h +++ b/drivers/cpufreq/cpufreq_governor.h @@ -109,7 +109,7 @@ store_one(_gov, file_name) /* create helper routines */ #define define_get_cpu_dbs_routines(_dbs_info) \ -static struct cpu_dbs_common_info *get_cpu_cdbs(int cpu) \ +static struct cpu_dbs_info *get_cpu_cdbs(int cpu) \ { \ return &per_cpu(_dbs_info, cpu).cdbs; \ } \ @@ -128,9 +128,20 @@ static void *get_cpu_dbs_info_s(int cpu) \ * cs_*: Conservative governor */ +/* Common to all CPUs of a policy */ +struct cpu_common_dbs_info { + struct cpufreq_policy *policy; + /* + * percpu mutex that serializes governor limit change with dbs_timer + * invocation. We do not want dbs_timer to run when user is changing + * the governor or limits. + */ + struct mutex timer_mutex; + ktime_t time_stamp; +}; + /* Per cpu structures */ -struct cpu_dbs_common_info { - int cpu; +struct cpu_dbs_info { u64 prev_cpu_idle; u64 prev_cpu_wall; u64 prev_cpu_nice; @@ -141,19 +152,12 @@ struct cpu_dbs_common_info { * wake-up from idle. */ unsigned int prev_load; - struct cpufreq_policy *cur_policy; - struct delayed_work work; - /* - * percpu mutex that serializes governor limit change with gov_dbs_timer - * invocation. We do not want gov_dbs_timer to run when user is changing - * the governor or limits. - */ - struct mutex timer_mutex; - ktime_t time_stamp; + struct delayed_work dwork; + struct cpu_common_dbs_info *shared; }; struct od_cpu_dbs_info_s { - struct cpu_dbs_common_info cdbs; + struct cpu_dbs_info cdbs; struct cpufreq_frequency_table *freq_table; unsigned int freq_lo; unsigned int freq_lo_jiffies; @@ -163,7 +167,7 @@ struct od_cpu_dbs_info_s { }; struct cs_cpu_dbs_info_s { - struct cpu_dbs_common_info cdbs; + struct cpu_dbs_info cdbs; unsigned int down_skip; unsigned int requested_freq; unsigned int enable:1; @@ -204,9 +208,11 @@ struct common_dbs_data { */ struct dbs_data *gdbs_data; - struct cpu_dbs_common_info *(*get_cpu_cdbs)(int cpu); + struct cpu_dbs_info *(*get_cpu_cdbs)(int cpu); void *(*get_cpu_dbs_info_s)(int cpu); - void (*gov_dbs_timer)(struct work_struct *work); + unsigned int (*gov_dbs_timer)(struct cpu_dbs_info *cdbs, + struct dbs_data *dbs_data, + bool modify_all); void (*gov_check_cpu)(int cpu, unsigned int load); int (*init)(struct dbs_data *dbs_data, bool notify); void (*exit)(struct dbs_data *dbs_data, bool notify); @@ -265,8 +271,6 @@ static ssize_t show_sampling_rate_min_gov_pol \ extern struct mutex cpufreq_governor_lock; void dbs_check_cpu(struct dbs_data *dbs_data, int cpu); -bool need_load_eval(struct cpu_dbs_common_info *cdbs, - unsigned int sampling_rate); int cpufreq_governor_dbs(struct cpufreq_policy *policy, struct common_dbs_data *cdata, unsigned int event); void gov_queue_work(struct dbs_data *dbs_data, struct cpufreq_policy *policy, diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c index 3c1e10f2304c..1fa9088c84a8 100644 --- a/drivers/cpufreq/cpufreq_ondemand.c +++ b/drivers/cpufreq/cpufreq_ondemand.c @@ -155,7 +155,7 @@ static void dbs_freq_increase(struct cpufreq_policy *policy, unsigned int freq) static void od_check_cpu(int cpu, unsigned int load) { struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu); - struct cpufreq_policy *policy = dbs_info->cdbs.cur_policy; + struct cpufreq_policy *policy = dbs_info->cdbs.shared->policy; struct dbs_data *dbs_data = policy->governor_data; struct od_dbs_tuners *od_tuners = dbs_data->tuners; @@ -191,46 +191,40 @@ static void od_check_cpu(int cpu, unsigned int load) } } -static void od_dbs_timer(struct work_struct *work) +static unsigned int od_dbs_timer(struct cpu_dbs_info *cdbs, + struct dbs_data *dbs_data, bool modify_all) { - struct od_cpu_dbs_info_s *dbs_info = - container_of(work, struct od_cpu_dbs_info_s, cdbs.work.work); - unsigned int cpu = dbs_info->cdbs.cur_policy->cpu; - struct od_cpu_dbs_info_s *core_dbs_info = &per_cpu(od_cpu_dbs_info, + struct cpufreq_policy *policy = cdbs->shared->policy; + unsigned int cpu = policy->cpu; + struct od_cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, cpu); - struct dbs_data *dbs_data = dbs_info->cdbs.cur_policy->governor_data; struct od_dbs_tuners *od_tuners = dbs_data->tuners; - int delay = 0, sample_type = core_dbs_info->sample_type; - bool modify_all = true; + int delay = 0, sample_type = dbs_info->sample_type; - mutex_lock(&core_dbs_info->cdbs.timer_mutex); - if (!need_load_eval(&core_dbs_info->cdbs, od_tuners->sampling_rate)) { - modify_all = false; + if (!modify_all) goto max_delay; - } /* Common NORMAL_SAMPLE setup */ - core_dbs_info->sample_type = OD_NORMAL_SAMPLE; + dbs_info->sample_type = OD_NORMAL_SAMPLE; if (sample_type == OD_SUB_SAMPLE) { - delay = core_dbs_info->freq_lo_jiffies; - __cpufreq_driver_target(core_dbs_info->cdbs.cur_policy, - core_dbs_info->freq_lo, CPUFREQ_RELATION_H); + delay = dbs_info->freq_lo_jiffies; + __cpufreq_driver_target(policy, dbs_info->freq_lo, + CPUFREQ_RELATION_H); } else { dbs_check_cpu(dbs_data, cpu); - if (core_dbs_info->freq_lo) { + if (dbs_info->freq_lo) { /* Setup timer for SUB_SAMPLE */ - core_dbs_info->sample_type = OD_SUB_SAMPLE; - delay = core_dbs_info->freq_hi_jiffies; + dbs_info->sample_type = OD_SUB_SAMPLE; + delay = dbs_info->freq_hi_jiffies; } } max_delay: if (!delay) delay = delay_for_sampling_rate(od_tuners->sampling_rate - * core_dbs_info->rate_mult); + * dbs_info->rate_mult); - gov_queue_work(dbs_data, dbs_info->cdbs.cur_policy, delay, modify_all); - mutex_unlock(&core_dbs_info->cdbs.timer_mutex); + return delay; } /************************** sysfs interface ************************/ @@ -273,27 +267,27 @@ static void update_sampling_rate(struct dbs_data *dbs_data, dbs_info = &per_cpu(od_cpu_dbs_info, cpu); cpufreq_cpu_put(policy); - mutex_lock(&dbs_info->cdbs.timer_mutex); + mutex_lock(&dbs_info->cdbs.shared->timer_mutex); - if (!delayed_work_pending(&dbs_info->cdbs.work)) { - mutex_unlock(&dbs_info->cdbs.timer_mutex); + if (!delayed_work_pending(&dbs_info->cdbs.dwork)) { + mutex_unlock(&dbs_info->cdbs.shared->timer_mutex); continue; } next_sampling = jiffies + usecs_to_jiffies(new_rate); - appointed_at = dbs_info->cdbs.work.timer.expires; + appointed_at = dbs_info->cdbs.dwork.timer.expires; if (time_before(next_sampling, appointed_at)) { - mutex_unlock(&dbs_info->cdbs.timer_mutex); - cancel_delayed_work_sync(&dbs_info->cdbs.work); - mutex_lock(&dbs_info->cdbs.timer_mutex); + mutex_unlock(&dbs_info->cdbs.shared->timer_mutex); + cancel_delayed_work_sync(&dbs_info->cdbs.dwork); + mutex_lock(&dbs_info->cdbs.shared->timer_mutex); - gov_queue_work(dbs_data, dbs_info->cdbs.cur_policy, - usecs_to_jiffies(new_rate), true); + gov_queue_work(dbs_data, policy, + usecs_to_jiffies(new_rate), true); } - mutex_unlock(&dbs_info->cdbs.timer_mutex); + mutex_unlock(&dbs_info->cdbs.shared->timer_mutex); } } @@ -556,13 +550,16 @@ static void od_set_powersave_bias(unsigned int powersave_bias) get_online_cpus(); for_each_online_cpu(cpu) { + struct cpu_common_dbs_info *shared; + if (cpumask_test_cpu(cpu, &done)) continue; - policy = per_cpu(od_cpu_dbs_info, cpu).cdbs.cur_policy; - if (!policy) + shared = per_cpu(od_cpu_dbs_info, cpu).cdbs.shared; + if (!shared) continue; + policy = shared->policy; cpumask_or(&done, &done, policy->cpus); if (policy->governor != &cpufreq_gov_ondemand) diff --git a/drivers/cpufreq/cpufreq_opp.c b/drivers/cpufreq/cpufreq_opp.c index 773bcde893c0..0f5e6d5f6da0 100644 --- a/drivers/cpufreq/cpufreq_opp.c +++ b/drivers/cpufreq/cpufreq_opp.c @@ -75,6 +75,10 @@ int dev_pm_opp_init_cpufreq_table(struct device *dev, } freq_table[i].driver_data = i; freq_table[i].frequency = rate / 1000; + + /* Is Boost/turbo opp ? */ + if (dev_pm_opp_is_turbo(opp)) + freq_table[i].flags = CPUFREQ_BOOST_FREQ; } freq_table[i].driver_data = i; diff --git a/drivers/cpufreq/e_powersaver.c b/drivers/cpufreq/e_powersaver.c index a0d2a423cea9..4085244c8a67 100644 --- a/drivers/cpufreq/e_powersaver.c +++ b/drivers/cpufreq/e_powersaver.c @@ -78,7 +78,7 @@ static int eps_acpi_init(void) static int eps_acpi_exit(struct cpufreq_policy *policy) { if (eps_acpi_cpu_perf) { - acpi_processor_unregister_performance(eps_acpi_cpu_perf, 0); + acpi_processor_unregister_performance(0); free_cpumask_var(eps_acpi_cpu_perf->shared_cpu_map); kfree(eps_acpi_cpu_perf); eps_acpi_cpu_perf = NULL; diff --git a/drivers/cpufreq/exynos-cpufreq.c b/drivers/cpufreq/exynos-cpufreq.c deleted file mode 100644 index fa3dd840a837..000000000000 --- a/drivers/cpufreq/exynos-cpufreq.c +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Copyright (c) 2010-2011 Samsung Electronics Co., Ltd. - * http://www.samsung.com - * - * EXYNOS - CPU frequency scaling support for EXYNOS series - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. -*/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "exynos-cpufreq.h" - -static struct exynos_dvfs_info *exynos_info; -static struct thermal_cooling_device *cdev; -static struct regulator *arm_regulator; -static unsigned int locking_frequency; - -static int exynos_cpufreq_get_index(unsigned int freq) -{ - struct cpufreq_frequency_table *freq_table = exynos_info->freq_table; - struct cpufreq_frequency_table *pos; - - cpufreq_for_each_entry(pos, freq_table) - if (pos->frequency == freq) - break; - - if (pos->frequency == CPUFREQ_TABLE_END) - return -EINVAL; - - return pos - freq_table; -} - -static int exynos_cpufreq_scale(unsigned int target_freq) -{ - struct cpufreq_frequency_table *freq_table = exynos_info->freq_table; - unsigned int *volt_table = exynos_info->volt_table; - struct cpufreq_policy *policy = cpufreq_cpu_get(0); - unsigned int arm_volt, safe_arm_volt = 0; - unsigned int mpll_freq_khz = exynos_info->mpll_freq_khz; - struct device *dev = exynos_info->dev; - unsigned int old_freq; - int index, old_index; - int ret = 0; - - old_freq = policy->cur; - - /* - * The policy max have been changed so that we cannot get proper - * old_index with cpufreq_frequency_table_target(). Thus, ignore - * policy and get the index from the raw frequency table. - */ - old_index = exynos_cpufreq_get_index(old_freq); - if (old_index < 0) { - ret = old_index; - goto out; - } - - index = exynos_cpufreq_get_index(target_freq); - if (index < 0) { - ret = index; - goto out; - } - - /* - * ARM clock source will be changed APLL to MPLL temporary - * To support this level, need to control regulator for - * required voltage level - */ - if (exynos_info->need_apll_change != NULL) { - if (exynos_info->need_apll_change(old_index, index) && - (freq_table[index].frequency < mpll_freq_khz) && - (freq_table[old_index].frequency < mpll_freq_khz)) - safe_arm_volt = volt_table[exynos_info->pll_safe_idx]; - } - arm_volt = volt_table[index]; - - /* When the new frequency is higher than current frequency */ - if ((target_freq > old_freq) && !safe_arm_volt) { - /* Firstly, voltage up to increase frequency */ - ret = regulator_set_voltage(arm_regulator, arm_volt, arm_volt); - if (ret) { - dev_err(dev, "failed to set cpu voltage to %d\n", - arm_volt); - return ret; - } - } - - if (safe_arm_volt) { - ret = regulator_set_voltage(arm_regulator, safe_arm_volt, - safe_arm_volt); - if (ret) { - dev_err(dev, "failed to set cpu voltage to %d\n", - safe_arm_volt); - return ret; - } - } - - exynos_info->set_freq(old_index, index); - - /* When the new frequency is lower than current frequency */ - if ((target_freq < old_freq) || - ((target_freq > old_freq) && safe_arm_volt)) { - /* down the voltage after frequency change */ - ret = regulator_set_voltage(arm_regulator, arm_volt, - arm_volt); - if (ret) { - dev_err(dev, "failed to set cpu voltage to %d\n", - arm_volt); - goto out; - } - } - -out: - cpufreq_cpu_put(policy); - - return ret; -} - -static int exynos_target(struct cpufreq_policy *policy, unsigned int index) -{ - return exynos_cpufreq_scale(exynos_info->freq_table[index].frequency); -} - -static int exynos_cpufreq_cpu_init(struct cpufreq_policy *policy) -{ - policy->clk = exynos_info->cpu_clk; - policy->suspend_freq = locking_frequency; - return cpufreq_generic_init(policy, exynos_info->freq_table, 100000); -} - -static struct cpufreq_driver exynos_driver = { - .flags = CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK, - .verify = cpufreq_generic_frequency_table_verify, - .target_index = exynos_target, - .get = cpufreq_generic_get, - .init = exynos_cpufreq_cpu_init, - .name = "exynos_cpufreq", - .attr = cpufreq_generic_attr, -#ifdef CONFIG_ARM_EXYNOS_CPU_FREQ_BOOST_SW - .boost_supported = true, -#endif -#ifdef CONFIG_PM - .suspend = cpufreq_generic_suspend, -#endif -}; - -static int exynos_cpufreq_probe(struct platform_device *pdev) -{ - struct device_node *cpu0; - int ret = -EINVAL; - - exynos_info = kzalloc(sizeof(*exynos_info), GFP_KERNEL); - if (!exynos_info) - return -ENOMEM; - - exynos_info->dev = &pdev->dev; - - if (of_machine_is_compatible("samsung,exynos4212")) { - exynos_info->type = EXYNOS_SOC_4212; - ret = exynos4x12_cpufreq_init(exynos_info); - } else if (of_machine_is_compatible("samsung,exynos4412")) { - exynos_info->type = EXYNOS_SOC_4412; - ret = exynos4x12_cpufreq_init(exynos_info); - } else if (of_machine_is_compatible("samsung,exynos5250")) { - exynos_info->type = EXYNOS_SOC_5250; - ret = exynos5250_cpufreq_init(exynos_info); - } else { - pr_err("%s: Unknown SoC type\n", __func__); - ret = -ENODEV; - } - - if (ret) - goto err_vdd_arm; - - if (exynos_info->set_freq == NULL) { - dev_err(&pdev->dev, "No set_freq function (ERR)\n"); - ret = -EINVAL; - goto err_vdd_arm; - } - - arm_regulator = regulator_get(NULL, "vdd_arm"); - if (IS_ERR(arm_regulator)) { - dev_err(&pdev->dev, "failed to get resource vdd_arm\n"); - ret = -EINVAL; - goto err_vdd_arm; - } - - /* Done here as we want to capture boot frequency */ - locking_frequency = clk_get_rate(exynos_info->cpu_clk) / 1000; - - ret = cpufreq_register_driver(&exynos_driver); - if (ret) - goto err_cpufreq_reg; - - cpu0 = of_get_cpu_node(0, NULL); - if (!cpu0) { - pr_err("failed to find cpu0 node\n"); - return 0; - } - - if (of_find_property(cpu0, "#cooling-cells", NULL)) { - cdev = of_cpufreq_cooling_register(cpu0, - cpu_present_mask); - if (IS_ERR(cdev)) - pr_err("running cpufreq without cooling device: %ld\n", - PTR_ERR(cdev)); - } - - return 0; - -err_cpufreq_reg: - dev_err(&pdev->dev, "failed to register cpufreq driver\n"); - regulator_put(arm_regulator); -err_vdd_arm: - kfree(exynos_info); - return ret; -} - -static struct platform_driver exynos_cpufreq_platdrv = { - .driver = { - .name = "exynos-cpufreq", - }, - .probe = exynos_cpufreq_probe, -}; -module_platform_driver(exynos_cpufreq_platdrv); diff --git a/drivers/cpufreq/exynos-cpufreq.h b/drivers/cpufreq/exynos-cpufreq.h deleted file mode 100644 index a3855e4d913d..000000000000 --- a/drivers/cpufreq/exynos-cpufreq.h +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2010 Samsung Electronics Co., Ltd. - * http://www.samsung.com - * - * EXYNOS - CPUFreq support - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. -*/ - -enum cpufreq_level_index { - L0, L1, L2, L3, L4, - L5, L6, L7, L8, L9, - L10, L11, L12, L13, L14, - L15, L16, L17, L18, L19, - L20, -}; - -enum exynos_soc_type { - EXYNOS_SOC_4212, - EXYNOS_SOC_4412, - EXYNOS_SOC_5250, -}; - -#define APLL_FREQ(f, a0, a1, a2, a3, a4, a5, a6, a7, b0, b1, b2, m, p, s) \ - { \ - .freq = (f) * 1000, \ - .clk_div_cpu0 = ((a0) | (a1) << 4 | (a2) << 8 | (a3) << 12 | \ - (a4) << 16 | (a5) << 20 | (a6) << 24 | (a7) << 28), \ - .clk_div_cpu1 = (b0 << 0 | b1 << 4 | b2 << 8), \ - .mps = ((m) << 16 | (p) << 8 | (s)), \ - } - -struct apll_freq { - unsigned int freq; - u32 clk_div_cpu0; - u32 clk_div_cpu1; - u32 mps; -}; - -struct exynos_dvfs_info { - enum exynos_soc_type type; - struct device *dev; - unsigned long mpll_freq_khz; - unsigned int pll_safe_idx; - struct clk *cpu_clk; - unsigned int *volt_table; - struct cpufreq_frequency_table *freq_table; - void (*set_freq)(unsigned int, unsigned int); - bool (*need_apll_change)(unsigned int, unsigned int); - void __iomem *cmu_regs; -}; - -#ifdef CONFIG_ARM_EXYNOS4X12_CPUFREQ -extern int exynos4x12_cpufreq_init(struct exynos_dvfs_info *); -#else -static inline int exynos4x12_cpufreq_init(struct exynos_dvfs_info *info) -{ - return -EOPNOTSUPP; -} -#endif -#ifdef CONFIG_ARM_EXYNOS5250_CPUFREQ -extern int exynos5250_cpufreq_init(struct exynos_dvfs_info *); -#else -static inline int exynos5250_cpufreq_init(struct exynos_dvfs_info *info) -{ - return -EOPNOTSUPP; -} -#endif - -#define EXYNOS4_CLKSRC_CPU 0x14200 -#define EXYNOS4_CLKMUX_STATCPU 0x14400 - -#define EXYNOS4_CLKDIV_CPU 0x14500 -#define EXYNOS4_CLKDIV_CPU1 0x14504 -#define EXYNOS4_CLKDIV_STATCPU 0x14600 -#define EXYNOS4_CLKDIV_STATCPU1 0x14604 - -#define EXYNOS4_CLKSRC_CPU_MUXCORE_SHIFT (16) -#define EXYNOS4_CLKMUX_STATCPU_MUXCORE_MASK (0x7 << EXYNOS4_CLKSRC_CPU_MUXCORE_SHIFT) - -#define EXYNOS5_APLL_LOCK 0x00000 -#define EXYNOS5_APLL_CON0 0x00100 -#define EXYNOS5_CLKMUX_STATCPU 0x00400 -#define EXYNOS5_CLKDIV_CPU0 0x00500 -#define EXYNOS5_CLKDIV_CPU1 0x00504 -#define EXYNOS5_CLKDIV_STATCPU0 0x00600 -#define EXYNOS5_CLKDIV_STATCPU1 0x00604 diff --git a/drivers/cpufreq/exynos4x12-cpufreq.c b/drivers/cpufreq/exynos4x12-cpufreq.c deleted file mode 100644 index 9e78a850e29f..000000000000 --- a/drivers/cpufreq/exynos4x12-cpufreq.c +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Copyright (c) 2010-2012 Samsung Electronics Co., Ltd. - * http://www.samsung.com - * - * EXYNOS4X12 - CPU frequency scaling support - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. -*/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "exynos-cpufreq.h" - -static struct clk *cpu_clk; -static struct clk *moutcore; -static struct clk *mout_mpll; -static struct clk *mout_apll; -static struct exynos_dvfs_info *cpufreq; - -static unsigned int exynos4x12_volt_table[] = { - 1350000, 1287500, 1250000, 1187500, 1137500, 1087500, 1037500, - 1000000, 987500, 975000, 950000, 925000, 900000, 900000 -}; - -static struct cpufreq_frequency_table exynos4x12_freq_table[] = { - {CPUFREQ_BOOST_FREQ, L0, 1500 * 1000}, - {0, L1, 1400 * 1000}, - {0, L2, 1300 * 1000}, - {0, L3, 1200 * 1000}, - {0, L4, 1100 * 1000}, - {0, L5, 1000 * 1000}, - {0, L6, 900 * 1000}, - {0, L7, 800 * 1000}, - {0, L8, 700 * 1000}, - {0, L9, 600 * 1000}, - {0, L10, 500 * 1000}, - {0, L11, 400 * 1000}, - {0, L12, 300 * 1000}, - {0, L13, 200 * 1000}, - {0, 0, CPUFREQ_TABLE_END}, -}; - -static struct apll_freq *apll_freq_4x12; - -static struct apll_freq apll_freq_4212[] = { - /* - * values: - * freq - * clock divider for CORE, COREM0, COREM1, PERIPH, ATB, PCLK_DBG, APLL, CORE2 - * clock divider for COPY, HPM, RESERVED - * PLL M, P, S - */ - APLL_FREQ(1500, 0, 3, 7, 0, 6, 1, 2, 0, 6, 2, 0, 250, 4, 0), - APLL_FREQ(1400, 0, 3, 7, 0, 6, 1, 2, 0, 6, 2, 0, 175, 3, 0), - APLL_FREQ(1300, 0, 3, 7, 0, 5, 1, 2, 0, 5, 2, 0, 325, 6, 0), - APLL_FREQ(1200, 0, 3, 7, 0, 5, 1, 2, 0, 5, 2, 0, 200, 4, 0), - APLL_FREQ(1100, 0, 3, 6, 0, 4, 1, 2, 0, 4, 2, 0, 275, 6, 0), - APLL_FREQ(1000, 0, 2, 5, 0, 4, 1, 1, 0, 4, 2, 0, 125, 3, 0), - APLL_FREQ(900, 0, 2, 5, 0, 3, 1, 1, 0, 3, 2, 0, 150, 4, 0), - APLL_FREQ(800, 0, 2, 5, 0, 3, 1, 1, 0, 3, 2, 0, 100, 3, 0), - APLL_FREQ(700, 0, 2, 4, 0, 3, 1, 1, 0, 3, 2, 0, 175, 3, 1), - APLL_FREQ(600, 0, 2, 4, 0, 3, 1, 1, 0, 3, 2, 0, 200, 4, 1), - APLL_FREQ(500, 0, 2, 4, 0, 3, 1, 1, 0, 3, 2, 0, 125, 3, 1), - APLL_FREQ(400, 0, 2, 4, 0, 3, 1, 1, 0, 3, 2, 0, 100, 3, 1), - APLL_FREQ(300, 0, 2, 4, 0, 2, 1, 1, 0, 3, 2, 0, 200, 4, 2), - APLL_FREQ(200, 0, 1, 3, 0, 1, 1, 1, 0, 3, 2, 0, 100, 3, 2), -}; - -static struct apll_freq apll_freq_4412[] = { - /* - * values: - * freq - * clock divider for CORE, COREM0, COREM1, PERIPH, ATB, PCLK_DBG, APLL, CORE2 - * clock divider for COPY, HPM, CORES - * PLL M, P, S - */ - APLL_FREQ(1500, 0, 3, 7, 0, 6, 1, 2, 0, 6, 0, 7, 250, 4, 0), - APLL_FREQ(1400, 0, 3, 7, 0, 6, 1, 2, 0, 6, 0, 6, 175, 3, 0), - APLL_FREQ(1300, 0, 3, 7, 0, 5, 1, 2, 0, 5, 0, 6, 325, 6, 0), - APLL_FREQ(1200, 0, 3, 7, 0, 5, 1, 2, 0, 5, 0, 5, 200, 4, 0), - APLL_FREQ(1100, 0, 3, 6, 0, 4, 1, 2, 0, 4, 0, 5, 275, 6, 0), - APLL_FREQ(1000, 0, 2, 5, 0, 4, 1, 1, 0, 4, 0, 4, 125, 3, 0), - APLL_FREQ(900, 0, 2, 5, 0, 3, 1, 1, 0, 3, 0, 4, 150, 4, 0), - APLL_FREQ(800, 0, 2, 5, 0, 3, 1, 1, 0, 3, 0, 3, 100, 3, 0), - APLL_FREQ(700, 0, 2, 4, 0, 3, 1, 1, 0, 3, 0, 3, 175, 3, 1), - APLL_FREQ(600, 0, 2, 4, 0, 3, 1, 1, 0, 3, 0, 2, 200, 4, 1), - APLL_FREQ(500, 0, 2, 4, 0, 3, 1, 1, 0, 3, 0, 2, 125, 3, 1), - APLL_FREQ(400, 0, 2, 4, 0, 3, 1, 1, 0, 3, 0, 1, 100, 3, 1), - APLL_FREQ(300, 0, 2, 4, 0, 2, 1, 1, 0, 3, 0, 1, 200, 4, 2), - APLL_FREQ(200, 0, 1, 3, 0, 1, 1, 1, 0, 3, 0, 0, 100, 3, 2), -}; - -static void exynos4x12_set_clkdiv(unsigned int div_index) -{ - unsigned int tmp; - - /* Change Divider - CPU0 */ - - tmp = apll_freq_4x12[div_index].clk_div_cpu0; - - __raw_writel(tmp, cpufreq->cmu_regs + EXYNOS4_CLKDIV_CPU); - - while (__raw_readl(cpufreq->cmu_regs + EXYNOS4_CLKDIV_STATCPU) - & 0x11111111) - cpu_relax(); - - /* Change Divider - CPU1 */ - tmp = apll_freq_4x12[div_index].clk_div_cpu1; - - __raw_writel(tmp, cpufreq->cmu_regs + EXYNOS4_CLKDIV_CPU1); - - do { - cpu_relax(); - tmp = __raw_readl(cpufreq->cmu_regs + EXYNOS4_CLKDIV_STATCPU1); - } while (tmp != 0x0); -} - -static void exynos4x12_set_apll(unsigned int index) -{ - unsigned int tmp, freq = apll_freq_4x12[index].freq; - - /* MUX_CORE_SEL = MPLL, ARMCLK uses MPLL for lock time */ - clk_set_parent(moutcore, mout_mpll); - - do { - cpu_relax(); - tmp = (__raw_readl(cpufreq->cmu_regs + EXYNOS4_CLKMUX_STATCPU) - >> EXYNOS4_CLKSRC_CPU_MUXCORE_SHIFT); - tmp &= 0x7; - } while (tmp != 0x2); - - clk_set_rate(mout_apll, freq * 1000); - - /* MUX_CORE_SEL = APLL */ - clk_set_parent(moutcore, mout_apll); - - do { - cpu_relax(); - tmp = __raw_readl(cpufreq->cmu_regs + EXYNOS4_CLKMUX_STATCPU); - tmp &= EXYNOS4_CLKMUX_STATCPU_MUXCORE_MASK; - } while (tmp != (0x1 << EXYNOS4_CLKSRC_CPU_MUXCORE_SHIFT)); -} - -static void exynos4x12_set_frequency(unsigned int old_index, - unsigned int new_index) -{ - if (old_index > new_index) { - exynos4x12_set_clkdiv(new_index); - exynos4x12_set_apll(new_index); - } else if (old_index < new_index) { - exynos4x12_set_apll(new_index); - exynos4x12_set_clkdiv(new_index); - } -} - -int exynos4x12_cpufreq_init(struct exynos_dvfs_info *info) -{ - struct device_node *np; - unsigned long rate; - - /* - * HACK: This is a temporary workaround to get access to clock - * controller registers directly and remove static mappings and - * dependencies on platform headers. It is necessary to enable - * Exynos multi-platform support and will be removed together with - * this whole driver as soon as Exynos gets migrated to use - * cpufreq-dt driver. - */ - np = of_find_compatible_node(NULL, NULL, "samsung,exynos4412-clock"); - if (!np) { - pr_err("%s: failed to find clock controller DT node\n", - __func__); - return -ENODEV; - } - - info->cmu_regs = of_iomap(np, 0); - if (!info->cmu_regs) { - pr_err("%s: failed to map CMU registers\n", __func__); - return -EFAULT; - } - - cpu_clk = clk_get(NULL, "armclk"); - if (IS_ERR(cpu_clk)) - return PTR_ERR(cpu_clk); - - moutcore = clk_get(NULL, "moutcore"); - if (IS_ERR(moutcore)) - goto err_moutcore; - - mout_mpll = clk_get(NULL, "mout_mpll"); - if (IS_ERR(mout_mpll)) - goto err_mout_mpll; - - rate = clk_get_rate(mout_mpll) / 1000; - - mout_apll = clk_get(NULL, "mout_apll"); - if (IS_ERR(mout_apll)) - goto err_mout_apll; - - if (info->type == EXYNOS_SOC_4212) - apll_freq_4x12 = apll_freq_4212; - else - apll_freq_4x12 = apll_freq_4412; - - info->mpll_freq_khz = rate; - /* 800Mhz */ - info->pll_safe_idx = L7; - info->cpu_clk = cpu_clk; - info->volt_table = exynos4x12_volt_table; - info->freq_table = exynos4x12_freq_table; - info->set_freq = exynos4x12_set_frequency; - - cpufreq = info; - - return 0; - -err_mout_apll: - clk_put(mout_mpll); -err_mout_mpll: - clk_put(moutcore); -err_moutcore: - clk_put(cpu_clk); - - pr_debug("%s: failed initialization\n", __func__); - return -EINVAL; -} diff --git a/drivers/cpufreq/exynos5250-cpufreq.c b/drivers/cpufreq/exynos5250-cpufreq.c deleted file mode 100644 index 3eafdc7ba787..000000000000 --- a/drivers/cpufreq/exynos5250-cpufreq.c +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Copyright (c) 2010-20122Samsung Electronics Co., Ltd. - * http://www.samsung.com - * - * EXYNOS5250 - CPU frequency scaling support - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. -*/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "exynos-cpufreq.h" - -static struct clk *cpu_clk; -static struct clk *moutcore; -static struct clk *mout_mpll; -static struct clk *mout_apll; -static struct exynos_dvfs_info *cpufreq; - -static unsigned int exynos5250_volt_table[] = { - 1300000, 1250000, 1225000, 1200000, 1150000, - 1125000, 1100000, 1075000, 1050000, 1025000, - 1012500, 1000000, 975000, 950000, 937500, - 925000 -}; - -static struct cpufreq_frequency_table exynos5250_freq_table[] = { - {0, L0, 1700 * 1000}, - {0, L1, 1600 * 1000}, - {0, L2, 1500 * 1000}, - {0, L3, 1400 * 1000}, - {0, L4, 1300 * 1000}, - {0, L5, 1200 * 1000}, - {0, L6, 1100 * 1000}, - {0, L7, 1000 * 1000}, - {0, L8, 900 * 1000}, - {0, L9, 800 * 1000}, - {0, L10, 700 * 1000}, - {0, L11, 600 * 1000}, - {0, L12, 500 * 1000}, - {0, L13, 400 * 1000}, - {0, L14, 300 * 1000}, - {0, L15, 200 * 1000}, - {0, 0, CPUFREQ_TABLE_END}, -}; - -static struct apll_freq apll_freq_5250[] = { - /* - * values: - * freq - * clock divider for ARM, CPUD, ACP, PERIPH, ATB, PCLK_DBG, APLL, ARM2 - * clock divider for COPY, HPM, RESERVED - * PLL M, P, S - */ - APLL_FREQ(1700, 0, 3, 7, 7, 7, 3, 5, 0, 0, 2, 0, 425, 6, 0), - APLL_FREQ(1600, 0, 3, 7, 7, 7, 1, 4, 0, 0, 2, 0, 200, 3, 0), - APLL_FREQ(1500, 0, 2, 7, 7, 7, 1, 4, 0, 0, 2, 0, 250, 4, 0), - APLL_FREQ(1400, 0, 2, 7, 7, 6, 1, 4, 0, 0, 2, 0, 175, 3, 0), - APLL_FREQ(1300, 0, 2, 7, 7, 6, 1, 3, 0, 0, 2, 0, 325, 6, 0), - APLL_FREQ(1200, 0, 2, 7, 7, 5, 1, 3, 0, 0, 2, 0, 200, 4, 0), - APLL_FREQ(1100, 0, 3, 7, 7, 5, 1, 3, 0, 0, 2, 0, 275, 6, 0), - APLL_FREQ(1000, 0, 1, 7, 7, 4, 1, 2, 0, 0, 2, 0, 125, 3, 0), - APLL_FREQ(900, 0, 1, 7, 7, 4, 1, 2, 0, 0, 2, 0, 150, 4, 0), - APLL_FREQ(800, 0, 1, 7, 7, 4, 1, 2, 0, 0, 2, 0, 100, 3, 0), - APLL_FREQ(700, 0, 1, 7, 7, 3, 1, 1, 0, 0, 2, 0, 175, 3, 1), - APLL_FREQ(600, 0, 1, 7, 7, 3, 1, 1, 0, 0, 2, 0, 200, 4, 1), - APLL_FREQ(500, 0, 1, 7, 7, 2, 1, 1, 0, 0, 2, 0, 125, 3, 1), - APLL_FREQ(400, 0, 1, 7, 7, 2, 1, 1, 0, 0, 2, 0, 100, 3, 1), - APLL_FREQ(300, 0, 1, 7, 7, 1, 1, 1, 0, 0, 2, 0, 200, 4, 2), - APLL_FREQ(200, 0, 1, 7, 7, 1, 1, 1, 0, 0, 2, 0, 100, 3, 2), -}; - -static void set_clkdiv(unsigned int div_index) -{ - unsigned int tmp; - - /* Change Divider - CPU0 */ - - tmp = apll_freq_5250[div_index].clk_div_cpu0; - - __raw_writel(tmp, cpufreq->cmu_regs + EXYNOS5_CLKDIV_CPU0); - - while (__raw_readl(cpufreq->cmu_regs + EXYNOS5_CLKDIV_STATCPU0) - & 0x11111111) - cpu_relax(); - - /* Change Divider - CPU1 */ - tmp = apll_freq_5250[div_index].clk_div_cpu1; - - __raw_writel(tmp, cpufreq->cmu_regs + EXYNOS5_CLKDIV_CPU1); - - while (__raw_readl(cpufreq->cmu_regs + EXYNOS5_CLKDIV_STATCPU1) & 0x11) - cpu_relax(); -} - -static void set_apll(unsigned int index) -{ - unsigned int tmp; - unsigned int freq = apll_freq_5250[index].freq; - - /* MUX_CORE_SEL = MPLL, ARMCLK uses MPLL for lock time */ - clk_set_parent(moutcore, mout_mpll); - - do { - cpu_relax(); - tmp = (__raw_readl(cpufreq->cmu_regs + EXYNOS5_CLKMUX_STATCPU) - >> 16); - tmp &= 0x7; - } while (tmp != 0x2); - - clk_set_rate(mout_apll, freq * 1000); - - /* MUX_CORE_SEL = APLL */ - clk_set_parent(moutcore, mout_apll); - - do { - cpu_relax(); - tmp = __raw_readl(cpufreq->cmu_regs + EXYNOS5_CLKMUX_STATCPU); - tmp &= (0x7 << 16); - } while (tmp != (0x1 << 16)); -} - -static void exynos5250_set_frequency(unsigned int old_index, - unsigned int new_index) -{ - if (old_index > new_index) { - set_clkdiv(new_index); - set_apll(new_index); - } else if (old_index < new_index) { - set_apll(new_index); - set_clkdiv(new_index); - } -} - -int exynos5250_cpufreq_init(struct exynos_dvfs_info *info) -{ - struct device_node *np; - unsigned long rate; - - /* - * HACK: This is a temporary workaround to get access to clock - * controller registers directly and remove static mappings and - * dependencies on platform headers. It is necessary to enable - * Exynos multi-platform support and will be removed together with - * this whole driver as soon as Exynos gets migrated to use - * cpufreq-dt driver. - */ - np = of_find_compatible_node(NULL, NULL, "samsung,exynos5250-clock"); - if (!np) { - pr_err("%s: failed to find clock controller DT node\n", - __func__); - return -ENODEV; - } - - info->cmu_regs = of_iomap(np, 0); - if (!info->cmu_regs) { - pr_err("%s: failed to map CMU registers\n", __func__); - return -EFAULT; - } - - cpu_clk = clk_get(NULL, "armclk"); - if (IS_ERR(cpu_clk)) - return PTR_ERR(cpu_clk); - - moutcore = clk_get(NULL, "mout_cpu"); - if (IS_ERR(moutcore)) - goto err_moutcore; - - mout_mpll = clk_get(NULL, "mout_mpll"); - if (IS_ERR(mout_mpll)) - goto err_mout_mpll; - - rate = clk_get_rate(mout_mpll) / 1000; - - mout_apll = clk_get(NULL, "mout_apll"); - if (IS_ERR(mout_apll)) - goto err_mout_apll; - - info->mpll_freq_khz = rate; - /* 800Mhz */ - info->pll_safe_idx = L9; - info->cpu_clk = cpu_clk; - info->volt_table = exynos5250_volt_table; - info->freq_table = exynos5250_freq_table; - info->set_freq = exynos5250_set_frequency; - - cpufreq = info; - - return 0; - -err_mout_apll: - clk_put(mout_mpll); -err_mout_mpll: - clk_put(moutcore); -err_moutcore: - clk_put(cpu_clk); - - pr_err("%s: failed initialization\n", __func__); - return -EINVAL; -} diff --git a/drivers/cpufreq/freq_table.c b/drivers/cpufreq/freq_table.c index dfbbf981ed56..a8f1daffc9bc 100644 --- a/drivers/cpufreq/freq_table.c +++ b/drivers/cpufreq/freq_table.c @@ -18,6 +18,21 @@ * FREQUENCY TABLE HELPERS * *********************************************************************/ +bool policy_has_boost_freq(struct cpufreq_policy *policy) +{ + struct cpufreq_frequency_table *pos, *table = policy->freq_table; + + if (!table) + return false; + + cpufreq_for_each_valid_entry(pos, table) + if (pos->flags & CPUFREQ_BOOST_FREQ) + return true; + + return false; +} +EXPORT_SYMBOL_GPL(policy_has_boost_freq); + int cpufreq_frequency_table_cpuinfo(struct cpufreq_policy *policy, struct cpufreq_frequency_table *table) { diff --git a/drivers/cpufreq/ia64-acpi-cpufreq.c b/drivers/cpufreq/ia64-acpi-cpufreq.c index c30aaa6a54e8..0202429f1c5b 100644 --- a/drivers/cpufreq/ia64-acpi-cpufreq.c +++ b/drivers/cpufreq/ia64-acpi-cpufreq.c @@ -29,7 +29,6 @@ MODULE_LICENSE("GPL"); struct cpufreq_acpi_io { struct acpi_processor_performance acpi_data; - struct cpufreq_frequency_table *freq_table; unsigned int resume; }; @@ -221,6 +220,7 @@ acpi_cpufreq_cpu_init ( unsigned int cpu = policy->cpu; struct cpufreq_acpi_io *data; unsigned int result = 0; + struct cpufreq_frequency_table *freq_table; pr_debug("acpi_cpufreq_cpu_init\n"); @@ -254,10 +254,10 @@ acpi_cpufreq_cpu_init ( } /* alloc freq_table */ - data->freq_table = kzalloc(sizeof(*data->freq_table) * + freq_table = kzalloc(sizeof(*freq_table) * (data->acpi_data.state_count + 1), GFP_KERNEL); - if (!data->freq_table) { + if (!freq_table) { result = -ENOMEM; goto err_unreg; } @@ -276,14 +276,14 @@ acpi_cpufreq_cpu_init ( for (i = 0; i <= data->acpi_data.state_count; i++) { if (i < data->acpi_data.state_count) { - data->freq_table[i].frequency = + freq_table[i].frequency = data->acpi_data.states[i].core_frequency * 1000; } else { - data->freq_table[i].frequency = CPUFREQ_TABLE_END; + freq_table[i].frequency = CPUFREQ_TABLE_END; } } - result = cpufreq_table_validate_and_show(policy, data->freq_table); + result = cpufreq_table_validate_and_show(policy, freq_table); if (result) { goto err_freqfree; } @@ -311,9 +311,9 @@ acpi_cpufreq_cpu_init ( return (result); err_freqfree: - kfree(data->freq_table); + kfree(freq_table); err_unreg: - acpi_processor_unregister_performance(&data->acpi_data, cpu); + acpi_processor_unregister_performance(cpu); err_free: kfree(data); acpi_io_data[cpu] = NULL; @@ -332,8 +332,8 @@ acpi_cpufreq_cpu_exit ( if (data) { acpi_io_data[policy->cpu] = NULL; - acpi_processor_unregister_performance(&data->acpi_data, - policy->cpu); + acpi_processor_unregister_performance(policy->cpu); + kfree(policy->freq_table); kfree(data); } diff --git a/drivers/cpufreq/integrator-cpufreq.c b/drivers/cpufreq/integrator-cpufreq.c index 129e266f7621..2faa4216bf2a 100644 --- a/drivers/cpufreq/integrator-cpufreq.c +++ b/drivers/cpufreq/integrator-cpufreq.c @@ -98,11 +98,10 @@ static int integrator_set_target(struct cpufreq_policy *policy, /* get current setting */ cm_osc = __raw_readl(cm_base + INTEGRATOR_HDR_OSC_OFFSET); - if (machine_is_integrator()) { + if (machine_is_integrator()) vco.s = (cm_osc >> 8) & 7; - } else if (machine_is_cintegrator()) { + else if (machine_is_cintegrator()) vco.s = 1; - } vco.v = cm_osc & 255; vco.r = 22; freqs.old = icst_hz(&cclk_params, vco) / 1000; @@ -163,11 +162,10 @@ static unsigned int integrator_get(unsigned int cpu) /* detect memory etc. */ cm_osc = __raw_readl(cm_base + INTEGRATOR_HDR_OSC_OFFSET); - if (machine_is_integrator()) { + if (machine_is_integrator()) vco.s = (cm_osc >> 8) & 7; - } else { + else vco.s = 1; - } vco.v = cm_osc & 255; vco.r = 22; @@ -203,7 +201,7 @@ static int __init integrator_cpufreq_probe(struct platform_device *pdev) struct resource *res; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) + if (!res) return -ENODEV; cm_base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); @@ -234,6 +232,6 @@ static struct platform_driver integrator_cpufreq_driver = { module_platform_driver_probe(integrator_cpufreq_driver, integrator_cpufreq_probe); -MODULE_AUTHOR ("Russell M. King"); -MODULE_DESCRIPTION ("cpufreq driver for ARM Integrator CPUs"); -MODULE_LICENSE ("GPL"); +MODULE_AUTHOR("Russell M. King"); +MODULE_DESCRIPTION("cpufreq driver for ARM Integrator CPUs"); +MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 7898de054f4e..3af9dd7332e6 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -260,24 +260,31 @@ static inline void update_turbo_state(void) cpu->pstate.max_pstate == cpu->pstate.turbo_pstate); } -#define PCT_TO_HWP(x) (x * 255 / 100) static void intel_pstate_hwp_set(void) { - int min, max, cpu; - u64 value, freq; + int min, hw_min, max, hw_max, cpu, range, adj_range; + u64 value, cap; + + rdmsrl(MSR_HWP_CAPABILITIES, cap); + hw_min = HWP_LOWEST_PERF(cap); + hw_max = HWP_HIGHEST_PERF(cap); + range = hw_max - hw_min; get_online_cpus(); for_each_online_cpu(cpu) { rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value); - min = PCT_TO_HWP(limits.min_perf_pct); + adj_range = limits.min_perf_pct * range / 100; + min = hw_min + adj_range; value &= ~HWP_MIN_PERF(~0L); value |= HWP_MIN_PERF(min); - max = PCT_TO_HWP(limits.max_perf_pct); + adj_range = limits.max_perf_pct * range / 100; + max = hw_min + adj_range; if (limits.no_turbo) { - rdmsrl( MSR_HWP_CAPABILITIES, freq); - max = HWP_GUARANTEED_PERF(freq); + hw_max = HWP_GUARANTEED_PERF(cap); + if (hw_max < max) + max = hw_max; } value &= ~HWP_MAX_PERF(~0L); @@ -423,6 +430,8 @@ static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b, limits.max_sysfs_pct = clamp_t(int, input, 0 , 100); limits.max_perf_pct = min(limits.max_policy_pct, limits.max_sysfs_pct); + limits.max_perf_pct = max(limits.min_policy_pct, limits.max_perf_pct); + limits.max_perf_pct = max(limits.min_perf_pct, limits.max_perf_pct); limits.max_perf = div_fp(int_tofp(limits.max_perf_pct), int_tofp(100)); if (hwp_active) @@ -442,6 +451,8 @@ static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b, limits.min_sysfs_pct = clamp_t(int, input, 0 , 100); limits.min_perf_pct = max(limits.min_policy_pct, limits.min_sysfs_pct); + limits.min_perf_pct = min(limits.max_policy_pct, limits.min_perf_pct); + limits.min_perf_pct = min(limits.max_perf_pct, limits.min_perf_pct); limits.min_perf = div_fp(int_tofp(limits.min_perf_pct), int_tofp(100)); if (hwp_active) @@ -484,12 +495,11 @@ static void __init intel_pstate_sysfs_expose_params(void) } /************************** sysfs end ************************/ -static void intel_pstate_hwp_enable(void) +static void intel_pstate_hwp_enable(struct cpudata *cpudata) { - hwp_active++; pr_info("intel_pstate: HWP enabled\n"); - wrmsrl( MSR_PM_ENABLE, 0x1); + wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1); } static int byt_get_min_pstate(void) @@ -522,7 +532,7 @@ static void byt_set_pstate(struct cpudata *cpudata, int pstate) int32_t vid_fp; u32 vid; - val = pstate << 8; + val = (u64)pstate << 8; if (limits.no_turbo && !limits.turbo_disabled) val |= (u64)1 << 32; @@ -611,7 +621,7 @@ static void core_set_pstate(struct cpudata *cpudata, int pstate) { u64 val; - val = pstate << 8; + val = (u64)pstate << 8; if (limits.no_turbo && !limits.turbo_disabled) val |= (u64)1 << 32; @@ -909,6 +919,7 @@ static const struct x86_cpu_id intel_pstate_cpu_ids[] = { ICPU(0x4c, byt_params), ICPU(0x4e, core_params), ICPU(0x4f, core_params), + ICPU(0x5e, core_params), ICPU(0x56, core_params), ICPU(0x57, knl_params), {} @@ -933,6 +944,10 @@ static int intel_pstate_init_cpu(unsigned int cpunum) cpu = all_cpu_data[cpunum]; cpu->cpu = cpunum; + + if (hwp_active) + intel_pstate_hwp_enable(cpu); + intel_pstate_get_cpu_pstates(cpu); init_timer_deferrable(&cpu->timer); @@ -985,12 +1000,19 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy) limits.min_policy_pct = (policy->min * 100) / policy->cpuinfo.max_freq; limits.min_policy_pct = clamp_t(int, limits.min_policy_pct, 0 , 100); - limits.min_perf_pct = max(limits.min_policy_pct, limits.min_sysfs_pct); - limits.min_perf = div_fp(int_tofp(limits.min_perf_pct), int_tofp(100)); - limits.max_policy_pct = (policy->max * 100) / policy->cpuinfo.max_freq; limits.max_policy_pct = clamp_t(int, limits.max_policy_pct, 0 , 100); + + /* Normalize user input to [min_policy_pct, max_policy_pct] */ + limits.min_perf_pct = max(limits.min_policy_pct, limits.min_sysfs_pct); + limits.min_perf_pct = min(limits.max_policy_pct, limits.min_perf_pct); limits.max_perf_pct = min(limits.max_policy_pct, limits.max_sysfs_pct); + limits.max_perf_pct = max(limits.min_policy_pct, limits.max_perf_pct); + + /* Make sure min_perf_pct <= max_perf_pct */ + limits.min_perf_pct = min(limits.max_perf_pct, limits.min_perf_pct); + + limits.min_perf = div_fp(int_tofp(limits.min_perf_pct), int_tofp(100)); limits.max_perf = div_fp(int_tofp(limits.max_perf_pct), int_tofp(100)); if (hwp_active) @@ -1170,6 +1192,10 @@ static struct hw_vendor_info vendor_info[] = { {1, "ORACLE", "X4270M3 ", PPC}, {1, "ORACLE", "X4270M2 ", PPC}, {1, "ORACLE", "X4170M2 ", PPC}, + {1, "ORACLE", "X4170 M3", PPC}, + {1, "ORACLE", "X4275 M3", PPC}, + {1, "ORACLE", "X6-2 ", PPC}, + {1, "ORACLE", "Sudbury ", PPC}, {0, "", ""}, }; @@ -1246,7 +1272,7 @@ static int __init intel_pstate_init(void) return -ENOMEM; if (static_cpu_has_safe(X86_FEATURE_HWP) && !no_hwp) - intel_pstate_hwp_enable(); + hwp_active++; if (!hwp_active && hwp_only) goto out; diff --git a/drivers/cpufreq/mt8173-cpufreq.c b/drivers/cpufreq/mt8173-cpufreq.c new file mode 100644 index 000000000000..49caed293a3b --- /dev/null +++ b/drivers/cpufreq/mt8173-cpufreq.c @@ -0,0 +1,527 @@ +/* + * Copyright (c) 2015 Linaro Ltd. + * Author: Pi-Cheng Chen + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MIN_VOLT_SHIFT (100000) +#define MAX_VOLT_SHIFT (200000) +#define MAX_VOLT_LIMIT (1150000) +#define VOLT_TOL (10000) + +/* + * The struct mtk_cpu_dvfs_info holds necessary information for doing CPU DVFS + * on each CPU power/clock domain of Mediatek SoCs. Each CPU cluster in + * Mediatek SoCs has two voltage inputs, Vproc and Vsram. In some cases the two + * voltage inputs need to be controlled under a hardware limitation: + * 100mV < Vsram - Vproc < 200mV + * + * When scaling the clock frequency of a CPU clock domain, the clock source + * needs to be switched to another stable PLL clock temporarily until + * the original PLL becomes stable at target frequency. + */ +struct mtk_cpu_dvfs_info { + struct device *cpu_dev; + struct regulator *proc_reg; + struct regulator *sram_reg; + struct clk *cpu_clk; + struct clk *inter_clk; + struct thermal_cooling_device *cdev; + int intermediate_voltage; + bool need_voltage_tracking; +}; + +static int mtk_cpufreq_voltage_tracking(struct mtk_cpu_dvfs_info *info, + int new_vproc) +{ + struct regulator *proc_reg = info->proc_reg; + struct regulator *sram_reg = info->sram_reg; + int old_vproc, old_vsram, new_vsram, vsram, vproc, ret; + + old_vproc = regulator_get_voltage(proc_reg); + old_vsram = regulator_get_voltage(sram_reg); + /* Vsram should not exceed the maximum allowed voltage of SoC. */ + new_vsram = min(new_vproc + MIN_VOLT_SHIFT, MAX_VOLT_LIMIT); + + if (old_vproc < new_vproc) { + /* + * When scaling up voltages, Vsram and Vproc scale up step + * by step. At each step, set Vsram to (Vproc + 200mV) first, + * then set Vproc to (Vsram - 100mV). + * Keep doing it until Vsram and Vproc hit target voltages. + */ + do { + old_vsram = regulator_get_voltage(sram_reg); + old_vproc = regulator_get_voltage(proc_reg); + + vsram = min(new_vsram, old_vproc + MAX_VOLT_SHIFT); + + if (vsram + VOLT_TOL >= MAX_VOLT_LIMIT) { + vsram = MAX_VOLT_LIMIT; + + /* + * If the target Vsram hits the maximum voltage, + * try to set the exact voltage value first. + */ + ret = regulator_set_voltage(sram_reg, vsram, + vsram); + if (ret) + ret = regulator_set_voltage(sram_reg, + vsram - VOLT_TOL, + vsram); + + vproc = new_vproc; + } else { + ret = regulator_set_voltage(sram_reg, vsram, + vsram + VOLT_TOL); + + vproc = vsram - MIN_VOLT_SHIFT; + } + if (ret) + return ret; + + ret = regulator_set_voltage(proc_reg, vproc, + vproc + VOLT_TOL); + if (ret) { + regulator_set_voltage(sram_reg, old_vsram, + old_vsram); + return ret; + } + } while (vproc < new_vproc || vsram < new_vsram); + } else if (old_vproc > new_vproc) { + /* + * When scaling down voltages, Vsram and Vproc scale down step + * by step. At each step, set Vproc to (Vsram - 200mV) first, + * then set Vproc to (Vproc + 100mV). + * Keep doing it until Vsram and Vproc hit target voltages. + */ + do { + old_vproc = regulator_get_voltage(proc_reg); + old_vsram = regulator_get_voltage(sram_reg); + + vproc = max(new_vproc, old_vsram - MAX_VOLT_SHIFT); + ret = regulator_set_voltage(proc_reg, vproc, + vproc + VOLT_TOL); + if (ret) + return ret; + + if (vproc == new_vproc) + vsram = new_vsram; + else + vsram = max(new_vsram, vproc + MIN_VOLT_SHIFT); + + if (vsram + VOLT_TOL >= MAX_VOLT_LIMIT) { + vsram = MAX_VOLT_LIMIT; + + /* + * If the target Vsram hits the maximum voltage, + * try to set the exact voltage value first. + */ + ret = regulator_set_voltage(sram_reg, vsram, + vsram); + if (ret) + ret = regulator_set_voltage(sram_reg, + vsram - VOLT_TOL, + vsram); + } else { + ret = regulator_set_voltage(sram_reg, vsram, + vsram + VOLT_TOL); + } + + if (ret) { + regulator_set_voltage(proc_reg, old_vproc, + old_vproc); + return ret; + } + } while (vproc > new_vproc + VOLT_TOL || + vsram > new_vsram + VOLT_TOL); + } + + return 0; +} + +static int mtk_cpufreq_set_voltage(struct mtk_cpu_dvfs_info *info, int vproc) +{ + if (info->need_voltage_tracking) + return mtk_cpufreq_voltage_tracking(info, vproc); + else + return regulator_set_voltage(info->proc_reg, vproc, + vproc + VOLT_TOL); +} + +static int mtk_cpufreq_set_target(struct cpufreq_policy *policy, + unsigned int index) +{ + struct cpufreq_frequency_table *freq_table = policy->freq_table; + struct clk *cpu_clk = policy->clk; + struct clk *armpll = clk_get_parent(cpu_clk); + struct mtk_cpu_dvfs_info *info = policy->driver_data; + struct device *cpu_dev = info->cpu_dev; + struct dev_pm_opp *opp; + long freq_hz, old_freq_hz; + int vproc, old_vproc, inter_vproc, target_vproc, ret; + + inter_vproc = info->intermediate_voltage; + + old_freq_hz = clk_get_rate(cpu_clk); + old_vproc = regulator_get_voltage(info->proc_reg); + + freq_hz = freq_table[index].frequency * 1000; + + rcu_read_lock(); + opp = dev_pm_opp_find_freq_ceil(cpu_dev, &freq_hz); + if (IS_ERR(opp)) { + rcu_read_unlock(); + pr_err("cpu%d: failed to find OPP for %ld\n", + policy->cpu, freq_hz); + return PTR_ERR(opp); + } + vproc = dev_pm_opp_get_voltage(opp); + rcu_read_unlock(); + + /* + * If the new voltage or the intermediate voltage is higher than the + * current voltage, scale up voltage first. + */ + target_vproc = (inter_vproc > vproc) ? inter_vproc : vproc; + if (old_vproc < target_vproc) { + ret = mtk_cpufreq_set_voltage(info, target_vproc); + if (ret) { + pr_err("cpu%d: failed to scale up voltage!\n", + policy->cpu); + mtk_cpufreq_set_voltage(info, old_vproc); + return ret; + } + } + + /* Reparent the CPU clock to intermediate clock. */ + ret = clk_set_parent(cpu_clk, info->inter_clk); + if (ret) { + pr_err("cpu%d: failed to re-parent cpu clock!\n", + policy->cpu); + mtk_cpufreq_set_voltage(info, old_vproc); + WARN_ON(1); + return ret; + } + + /* Set the original PLL to target rate. */ + ret = clk_set_rate(armpll, freq_hz); + if (ret) { + pr_err("cpu%d: failed to scale cpu clock rate!\n", + policy->cpu); + clk_set_parent(cpu_clk, armpll); + mtk_cpufreq_set_voltage(info, old_vproc); + return ret; + } + + /* Set parent of CPU clock back to the original PLL. */ + ret = clk_set_parent(cpu_clk, armpll); + if (ret) { + pr_err("cpu%d: failed to re-parent cpu clock!\n", + policy->cpu); + mtk_cpufreq_set_voltage(info, inter_vproc); + WARN_ON(1); + return ret; + } + + /* + * If the new voltage is lower than the intermediate voltage or the + * original voltage, scale down to the new voltage. + */ + if (vproc < inter_vproc || vproc < old_vproc) { + ret = mtk_cpufreq_set_voltage(info, vproc); + if (ret) { + pr_err("cpu%d: failed to scale down voltage!\n", + policy->cpu); + clk_set_parent(cpu_clk, info->inter_clk); + clk_set_rate(armpll, old_freq_hz); + clk_set_parent(cpu_clk, armpll); + return ret; + } + } + + return 0; +} + +static void mtk_cpufreq_ready(struct cpufreq_policy *policy) +{ + struct mtk_cpu_dvfs_info *info = policy->driver_data; + struct device_node *np = of_node_get(info->cpu_dev->of_node); + + if (WARN_ON(!np)) + return; + + if (of_find_property(np, "#cooling-cells", NULL)) { + info->cdev = of_cpufreq_cooling_register(np, + policy->related_cpus); + + if (IS_ERR(info->cdev)) { + dev_err(info->cpu_dev, + "running cpufreq without cooling device: %ld\n", + PTR_ERR(info->cdev)); + + info->cdev = NULL; + } + } + + of_node_put(np); +} + +static int mtk_cpu_dvfs_info_init(struct mtk_cpu_dvfs_info *info, int cpu) +{ + struct device *cpu_dev; + struct regulator *proc_reg = ERR_PTR(-ENODEV); + struct regulator *sram_reg = ERR_PTR(-ENODEV); + struct clk *cpu_clk = ERR_PTR(-ENODEV); + struct clk *inter_clk = ERR_PTR(-ENODEV); + struct dev_pm_opp *opp; + unsigned long rate; + int ret; + + cpu_dev = get_cpu_device(cpu); + if (!cpu_dev) { + pr_err("failed to get cpu%d device\n", cpu); + return -ENODEV; + } + + cpu_clk = clk_get(cpu_dev, "cpu"); + if (IS_ERR(cpu_clk)) { + if (PTR_ERR(cpu_clk) == -EPROBE_DEFER) + pr_warn("cpu clk for cpu%d not ready, retry.\n", cpu); + else + pr_err("failed to get cpu clk for cpu%d\n", cpu); + + ret = PTR_ERR(cpu_clk); + return ret; + } + + inter_clk = clk_get(cpu_dev, "intermediate"); + if (IS_ERR(inter_clk)) { + if (PTR_ERR(inter_clk) == -EPROBE_DEFER) + pr_warn("intermediate clk for cpu%d not ready, retry.\n", + cpu); + else + pr_err("failed to get intermediate clk for cpu%d\n", + cpu); + + ret = PTR_ERR(inter_clk); + goto out_free_resources; + } + + proc_reg = regulator_get_exclusive(cpu_dev, "proc"); + if (IS_ERR(proc_reg)) { + if (PTR_ERR(proc_reg) == -EPROBE_DEFER) + pr_warn("proc regulator for cpu%d not ready, retry.\n", + cpu); + else + pr_err("failed to get proc regulator for cpu%d\n", + cpu); + + ret = PTR_ERR(proc_reg); + goto out_free_resources; + } + + /* Both presence and absence of sram regulator are valid cases. */ + sram_reg = regulator_get_exclusive(cpu_dev, "sram"); + + ret = of_init_opp_table(cpu_dev); + if (ret) { + pr_warn("no OPP table for cpu%d\n", cpu); + goto out_free_resources; + } + + /* Search a safe voltage for intermediate frequency. */ + rate = clk_get_rate(inter_clk); + rcu_read_lock(); + opp = dev_pm_opp_find_freq_ceil(cpu_dev, &rate); + if (IS_ERR(opp)) { + rcu_read_unlock(); + pr_err("failed to get intermediate opp for cpu%d\n", cpu); + ret = PTR_ERR(opp); + goto out_free_opp_table; + } + info->intermediate_voltage = dev_pm_opp_get_voltage(opp); + rcu_read_unlock(); + + info->cpu_dev = cpu_dev; + info->proc_reg = proc_reg; + info->sram_reg = IS_ERR(sram_reg) ? NULL : sram_reg; + info->cpu_clk = cpu_clk; + info->inter_clk = inter_clk; + + /* + * If SRAM regulator is present, software "voltage tracking" is needed + * for this CPU power domain. + */ + info->need_voltage_tracking = !IS_ERR(sram_reg); + + return 0; + +out_free_opp_table: + of_free_opp_table(cpu_dev); + +out_free_resources: + if (!IS_ERR(proc_reg)) + regulator_put(proc_reg); + if (!IS_ERR(sram_reg)) + regulator_put(sram_reg); + if (!IS_ERR(cpu_clk)) + clk_put(cpu_clk); + if (!IS_ERR(inter_clk)) + clk_put(inter_clk); + + return ret; +} + +static void mtk_cpu_dvfs_info_release(struct mtk_cpu_dvfs_info *info) +{ + if (!IS_ERR(info->proc_reg)) + regulator_put(info->proc_reg); + if (!IS_ERR(info->sram_reg)) + regulator_put(info->sram_reg); + if (!IS_ERR(info->cpu_clk)) + clk_put(info->cpu_clk); + if (!IS_ERR(info->inter_clk)) + clk_put(info->inter_clk); + + of_free_opp_table(info->cpu_dev); +} + +static int mtk_cpufreq_init(struct cpufreq_policy *policy) +{ + struct mtk_cpu_dvfs_info *info; + struct cpufreq_frequency_table *freq_table; + int ret; + + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (!info) + return -ENOMEM; + + ret = mtk_cpu_dvfs_info_init(info, policy->cpu); + if (ret) { + pr_err("%s failed to initialize dvfs info for cpu%d\n", + __func__, policy->cpu); + goto out_free_dvfs_info; + } + + ret = dev_pm_opp_init_cpufreq_table(info->cpu_dev, &freq_table); + if (ret) { + pr_err("failed to init cpufreq table for cpu%d: %d\n", + policy->cpu, ret); + goto out_release_dvfs_info; + } + + ret = cpufreq_table_validate_and_show(policy, freq_table); + if (ret) { + pr_err("%s: invalid frequency table: %d\n", __func__, ret); + goto out_free_cpufreq_table; + } + + /* CPUs in the same cluster share a clock and power domain. */ + cpumask_copy(policy->cpus, &cpu_topology[policy->cpu].core_sibling); + policy->driver_data = info; + policy->clk = info->cpu_clk; + + return 0; + +out_free_cpufreq_table: + dev_pm_opp_free_cpufreq_table(info->cpu_dev, &freq_table); + +out_release_dvfs_info: + mtk_cpu_dvfs_info_release(info); + +out_free_dvfs_info: + kfree(info); + + return ret; +} + +static int mtk_cpufreq_exit(struct cpufreq_policy *policy) +{ + struct mtk_cpu_dvfs_info *info = policy->driver_data; + + cpufreq_cooling_unregister(info->cdev); + dev_pm_opp_free_cpufreq_table(info->cpu_dev, &policy->freq_table); + mtk_cpu_dvfs_info_release(info); + kfree(info); + + return 0; +} + +static struct cpufreq_driver mt8173_cpufreq_driver = { + .flags = CPUFREQ_STICKY | CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = mtk_cpufreq_set_target, + .get = cpufreq_generic_get, + .init = mtk_cpufreq_init, + .exit = mtk_cpufreq_exit, + .ready = mtk_cpufreq_ready, + .name = "mtk-cpufreq", + .attr = cpufreq_generic_attr, +}; + +static int mt8173_cpufreq_probe(struct platform_device *pdev) +{ + int ret; + + ret = cpufreq_register_driver(&mt8173_cpufreq_driver); + if (ret) + pr_err("failed to register mtk cpufreq driver\n"); + + return ret; +} + +static struct platform_driver mt8173_cpufreq_platdrv = { + .driver = { + .name = "mt8173-cpufreq", + }, + .probe = mt8173_cpufreq_probe, +}; + +static int mt8173_cpufreq_driver_init(void) +{ + struct platform_device *pdev; + int err; + + if (!of_machine_is_compatible("mediatek,mt8173")) + return -ENODEV; + + err = platform_driver_register(&mt8173_cpufreq_platdrv); + if (err) + return err; + + /* + * Since there's no place to hold device registration code and no + * device tree based way to match cpufreq driver yet, both the driver + * and the device registration codes are put here to handle defer + * probing. + */ + pdev = platform_device_register_simple("mt8173-cpufreq", -1, NULL, 0); + if (IS_ERR(pdev)) { + pr_err("failed to register mtk-cpufreq platform device\n"); + return PTR_ERR(pdev); + } + + return 0; +} +device_initcall(mt8173_cpufreq_driver_init); diff --git a/drivers/cpufreq/powernow-k7.c b/drivers/cpufreq/powernow-k7.c index 37c5742482d8..c1ae1999770a 100644 --- a/drivers/cpufreq/powernow-k7.c +++ b/drivers/cpufreq/powernow-k7.c @@ -421,7 +421,7 @@ static int powernow_acpi_init(void) return 0; err2: - acpi_processor_unregister_performance(acpi_processor_perf, 0); + acpi_processor_unregister_performance(0); err1: free_cpumask_var(acpi_processor_perf->shared_cpu_map); err05: @@ -661,7 +661,7 @@ static int powernow_cpu_exit(struct cpufreq_policy *policy) { #ifdef CONFIG_X86_POWERNOW_K7_ACPI if (acpi_processor_perf) { - acpi_processor_unregister_performance(acpi_processor_perf, 0); + acpi_processor_unregister_performance(0); free_cpumask_var(acpi_processor_perf->shared_cpu_map); kfree(acpi_processor_perf); } diff --git a/drivers/cpufreq/powernow-k8.c b/drivers/cpufreq/powernow-k8.c index 5c035d04d827..0b5bf135b090 100644 --- a/drivers/cpufreq/powernow-k8.c +++ b/drivers/cpufreq/powernow-k8.c @@ -795,7 +795,7 @@ static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data) kfree(powernow_table); err_out: - acpi_processor_unregister_performance(&data->acpi_data, data->cpu); + acpi_processor_unregister_performance(data->cpu); /* data->acpi_data.state_count informs us at ->exit() * whether ACPI was used */ @@ -863,8 +863,7 @@ static int fill_powernow_table_fidvid(struct powernow_k8_data *data, static void powernow_k8_cpu_exit_acpi(struct powernow_k8_data *data) { if (data->acpi_data.state_count) - acpi_processor_unregister_performance(&data->acpi_data, - data->cpu); + acpi_processor_unregister_performance(data->cpu); free_cpumask_var(data->acpi_data.shared_cpu_map); } diff --git a/drivers/cpufreq/powernv-cpufreq.c b/drivers/cpufreq/powernv-cpufreq.c index ebef0d8279c7..64994e10638e 100644 --- a/drivers/cpufreq/powernv-cpufreq.c +++ b/drivers/cpufreq/powernv-cpufreq.c @@ -27,20 +27,31 @@ #include #include #include +#include #include #include #include #include /* Required for cpu_sibling_mask() in UP configs */ +#include #define POWERNV_MAX_PSTATES 256 #define PMSR_PSAFE_ENABLE (1UL << 30) #define PMSR_SPR_EM_DISABLE (1UL << 31) #define PMSR_MAX(x) ((x >> 32) & 0xFF) -#define PMSR_LP(x) ((x >> 48) & 0xFF) static struct cpufreq_frequency_table powernv_freqs[POWERNV_MAX_PSTATES+1]; -static bool rebooting, throttled; +static bool rebooting, throttled, occ_reset; + +static struct chip { + unsigned int id; + bool throttled; + cpumask_t mask; + struct work_struct throttle; + bool restore; +} *chips; + +static int nr_chips; /* * Note: The set of pstates consists of contiguous integers, the @@ -298,28 +309,35 @@ static inline unsigned int get_nominal_index(void) return powernv_pstate_info.max - powernv_pstate_info.nominal; } -static void powernv_cpufreq_throttle_check(unsigned int cpu) +static void powernv_cpufreq_throttle_check(void *data) { + unsigned int cpu = smp_processor_id(); unsigned long pmsr; - int pmsr_pmax, pmsr_lp; + int pmsr_pmax, i; pmsr = get_pmspr(SPRN_PMSR); + for (i = 0; i < nr_chips; i++) + if (chips[i].id == cpu_to_chip_id(cpu)) + break; + /* Check for Pmax Capping */ pmsr_pmax = (s8)PMSR_MAX(pmsr); if (pmsr_pmax != powernv_pstate_info.max) { - throttled = true; - pr_info("CPU %d Pmax is reduced to %d\n", cpu, pmsr_pmax); - pr_info("Max allowed Pstate is capped\n"); + if (chips[i].throttled) + goto next; + chips[i].throttled = true; + pr_info("CPU %d on Chip %u has Pmax reduced to %d\n", cpu, + chips[i].id, pmsr_pmax); + } else if (chips[i].throttled) { + chips[i].throttled = false; + pr_info("CPU %d on Chip %u has Pmax restored to %d\n", cpu, + chips[i].id, pmsr_pmax); } - /* - * Check for Psafe by reading LocalPstate - * or check if Psafe_mode_active is set in PMSR. - */ - pmsr_lp = (s8)PMSR_LP(pmsr); - if ((pmsr_lp < powernv_pstate_info.min) || - (pmsr & PMSR_PSAFE_ENABLE)) { + /* Check if Psafe_mode_active is set in PMSR. */ +next: + if (pmsr & PMSR_PSAFE_ENABLE) { throttled = true; pr_info("Pstate set to safe frequency\n"); } @@ -350,7 +368,7 @@ static int powernv_cpufreq_target_index(struct cpufreq_policy *policy, return 0; if (!throttled) - powernv_cpufreq_throttle_check(smp_processor_id()); + powernv_cpufreq_throttle_check(NULL); freq_data.pstate_id = powernv_freqs[new_index].driver_data; @@ -395,6 +413,119 @@ static struct notifier_block powernv_cpufreq_reboot_nb = { .notifier_call = powernv_cpufreq_reboot_notifier, }; +void powernv_cpufreq_work_fn(struct work_struct *work) +{ + struct chip *chip = container_of(work, struct chip, throttle); + unsigned int cpu; + cpumask_var_t mask; + + smp_call_function_any(&chip->mask, + powernv_cpufreq_throttle_check, NULL, 0); + + if (!chip->restore) + return; + + chip->restore = false; + cpumask_copy(mask, &chip->mask); + for_each_cpu_and(cpu, mask, cpu_online_mask) { + int index, tcpu; + struct cpufreq_policy policy; + + cpufreq_get_policy(&policy, cpu); + cpufreq_frequency_table_target(&policy, policy.freq_table, + policy.cur, + CPUFREQ_RELATION_C, &index); + powernv_cpufreq_target_index(&policy, index); + for_each_cpu(tcpu, policy.cpus) + cpumask_clear_cpu(tcpu, mask); + } +} + +static char throttle_reason[][30] = { + "No throttling", + "Power Cap", + "Processor Over Temperature", + "Power Supply Failure", + "Over Current", + "OCC Reset" + }; + +static int powernv_cpufreq_occ_msg(struct notifier_block *nb, + unsigned long msg_type, void *_msg) +{ + struct opal_msg *msg = _msg; + struct opal_occ_msg omsg; + int i; + + if (msg_type != OPAL_MSG_OCC) + return 0; + + omsg.type = be64_to_cpu(msg->params[0]); + + switch (omsg.type) { + case OCC_RESET: + occ_reset = true; + pr_info("OCC (On Chip Controller - enforces hard thermal/power limits) Resetting\n"); + /* + * powernv_cpufreq_throttle_check() is called in + * target() callback which can detect the throttle state + * for governors like ondemand. + * But static governors will not call target() often thus + * report throttling here. + */ + if (!throttled) { + throttled = true; + pr_crit("CPU frequency is throttled for duration\n"); + } + + break; + case OCC_LOAD: + pr_info("OCC Loading, CPU frequency is throttled until OCC is started\n"); + break; + case OCC_THROTTLE: + omsg.chip = be64_to_cpu(msg->params[1]); + omsg.throttle_status = be64_to_cpu(msg->params[2]); + + if (occ_reset) { + occ_reset = false; + throttled = false; + pr_info("OCC Active, CPU frequency is no longer throttled\n"); + + for (i = 0; i < nr_chips; i++) { + chips[i].restore = true; + schedule_work(&chips[i].throttle); + } + + return 0; + } + + if (omsg.throttle_status && + omsg.throttle_status <= OCC_MAX_THROTTLE_STATUS) + pr_info("OCC: Chip %u Pmax reduced due to %s\n", + (unsigned int)omsg.chip, + throttle_reason[omsg.throttle_status]); + else if (!omsg.throttle_status) + pr_info("OCC: Chip %u %s\n", (unsigned int)omsg.chip, + throttle_reason[omsg.throttle_status]); + else + return 0; + + for (i = 0; i < nr_chips; i++) + if (chips[i].id == omsg.chip) { + if (!omsg.throttle_status) + chips[i].restore = true; + schedule_work(&chips[i].throttle); + } + } + return 0; +} + +static struct notifier_block powernv_cpufreq_opal_nb = { + .notifier_call = powernv_cpufreq_occ_msg, + .next = NULL, + .priority = 0, +}; + static void powernv_cpufreq_stop_cpu(struct cpufreq_policy *policy) { struct powernv_smp_call_data freq_data; @@ -414,6 +545,36 @@ static struct cpufreq_driver powernv_cpufreq_driver = { .attr = powernv_cpu_freq_attr, }; +static int init_chip_info(void) +{ + unsigned int chip[256]; + unsigned int cpu, i; + unsigned int prev_chip_id = UINT_MAX; + + for_each_possible_cpu(cpu) { + unsigned int id = cpu_to_chip_id(cpu); + + if (prev_chip_id != id) { + prev_chip_id = id; + chip[nr_chips++] = id; + } + } + + chips = kmalloc_array(nr_chips, sizeof(struct chip), GFP_KERNEL); + if (!chips) + return -ENOMEM; + + for (i = 0; i < nr_chips; i++) { + chips[i].id = chip[i]; + chips[i].throttled = false; + cpumask_copy(&chips[i].mask, cpumask_of_node(chip[i])); + INIT_WORK(&chips[i].throttle, powernv_cpufreq_work_fn); + chips[i].restore = false; + } + + return 0; +} + static int __init powernv_cpufreq_init(void) { int rc = 0; @@ -429,7 +590,13 @@ static int __init powernv_cpufreq_init(void) return rc; } + /* Populate chip info */ + rc = init_chip_info(); + if (rc) + return rc; + register_reboot_notifier(&powernv_cpufreq_reboot_nb); + opal_message_notifier_register(OPAL_MSG_OCC, &powernv_cpufreq_opal_nb); return cpufreq_register_driver(&powernv_cpufreq_driver); } module_init(powernv_cpufreq_init); @@ -437,6 +604,8 @@ module_init(powernv_cpufreq_init); static void __exit powernv_cpufreq_exit(void) { unregister_reboot_notifier(&powernv_cpufreq_reboot_nb); + opal_message_notifier_unregister(OPAL_MSG_OCC, + &powernv_cpufreq_opal_nb); cpufreq_unregister_driver(&powernv_cpufreq_driver); } module_exit(powernv_cpufreq_exit); diff --git a/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c b/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c index d29e8da396a0..7969f7690498 100644 --- a/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c +++ b/drivers/cpufreq/ppc_cbe_cpufreq_pmi.c @@ -97,8 +97,8 @@ static int pmi_notifier(struct notifier_block *nb, struct cpufreq_frequency_table *cbe_freqs; u8 node; - /* Should this really be called for CPUFREQ_ADJUST, CPUFREQ_INCOMPATIBLE - * and CPUFREQ_NOTIFY policy events?) + /* Should this really be called for CPUFREQ_ADJUST and CPUFREQ_NOTIFY + * policy events?) */ if (event == CPUFREQ_START) return 0; diff --git a/drivers/cpufreq/sfi-cpufreq.c b/drivers/cpufreq/sfi-cpufreq.c index ffa3389e535b..992ce6f9abec 100644 --- a/drivers/cpufreq/sfi-cpufreq.c +++ b/drivers/cpufreq/sfi-cpufreq.c @@ -45,12 +45,10 @@ static int sfi_parse_freq(struct sfi_table_header *table) pentry = (struct sfi_freq_table_entry *)sb->pentry; totallen = num_freq_table_entries * sizeof(*pentry); - sfi_cpufreq_array = kzalloc(totallen, GFP_KERNEL); + sfi_cpufreq_array = kmemdup(pentry, totallen, GFP_KERNEL); if (!sfi_cpufreq_array) return -ENOMEM; - memcpy(sfi_cpufreq_array, pentry, totallen); - return 0; } diff --git a/drivers/cpufreq/speedstep-lib.c b/drivers/cpufreq/speedstep-lib.c index 4ab7a2156672..15d3214aaa00 100644 --- a/drivers/cpufreq/speedstep-lib.c +++ b/drivers/cpufreq/speedstep-lib.c @@ -386,7 +386,7 @@ unsigned int speedstep_get_freqs(enum speedstep_processor processor, unsigned int prev_speed; unsigned int ret = 0; unsigned long flags; - struct timeval tv1, tv2; + ktime_t tv1, tv2; if ((!processor) || (!low_speed) || (!high_speed) || (!set_state)) return -EINVAL; @@ -415,14 +415,14 @@ unsigned int speedstep_get_freqs(enum speedstep_processor processor, /* start latency measurement */ if (transition_latency) - do_gettimeofday(&tv1); + tv1 = ktime_get(); /* switch to high state */ set_state(SPEEDSTEP_HIGH); /* end latency measurement */ if (transition_latency) - do_gettimeofday(&tv2); + tv2 = ktime_get(); *high_speed = speedstep_get_frequency(processor); if (!*high_speed) { @@ -442,8 +442,7 @@ unsigned int speedstep_get_freqs(enum speedstep_processor processor, set_state(SPEEDSTEP_LOW); if (transition_latency) { - *transition_latency = (tv2.tv_sec - tv1.tv_sec) * USEC_PER_SEC + - tv2.tv_usec - tv1.tv_usec; + *transition_latency = ktime_to_us(ktime_sub(tv2, tv1)); pr_debug("transition latency is %u uSec\n", *transition_latency); /* convert uSec to nSec and add 20% for safety reasons */ diff --git a/drivers/cpuidle/coupled.c b/drivers/cpuidle/coupled.c index 7936dce4b878..344058f8501a 100644 --- a/drivers/cpuidle/coupled.c +++ b/drivers/cpuidle/coupled.c @@ -176,18 +176,38 @@ void cpuidle_coupled_parallel_barrier(struct cpuidle_device *dev, atomic_t *a) /** * cpuidle_state_is_coupled - check if a state is part of a coupled set - * @dev: struct cpuidle_device for the current cpu * @drv: struct cpuidle_driver for the platform * @state: index of the target state in drv->states * * Returns true if the target state is coupled with cpus besides this one */ -bool cpuidle_state_is_coupled(struct cpuidle_device *dev, - struct cpuidle_driver *drv, int state) +bool cpuidle_state_is_coupled(struct cpuidle_driver *drv, int state) { return drv->states[state].flags & CPUIDLE_FLAG_COUPLED; } +/** + * cpuidle_coupled_state_verify - check if the coupled states are correctly set. + * @drv: struct cpuidle_driver for the platform + * + * Returns 0 for valid state values, a negative error code otherwise: + * * -EINVAL if any coupled state(safe_state_index) is wrongly set. + */ +int cpuidle_coupled_state_verify(struct cpuidle_driver *drv) +{ + int i; + + for (i = drv->state_count - 1; i >= 0; i--) { + if (cpuidle_state_is_coupled(drv, i) && + (drv->safe_state_index == i || + drv->safe_state_index < 0 || + drv->safe_state_index >= drv->state_count)) + return -EINVAL; + } + + return 0; +} + /** * cpuidle_coupled_set_ready - mark a cpu as ready * @coupled: the struct coupled that contains the current cpu @@ -473,7 +493,7 @@ int cpuidle_enter_state_coupled(struct cpuidle_device *dev, return entered_state; } entered_state = cpuidle_enter_state(dev, drv, - dev->safe_state_index); + drv->safe_state_index); local_irq_disable(); } @@ -521,7 +541,7 @@ int cpuidle_enter_state_coupled(struct cpuidle_device *dev, } entered_state = cpuidle_enter_state(dev, drv, - dev->safe_state_index); + drv->safe_state_index); local_irq_disable(); } diff --git a/drivers/cpuidle/cpuidle-calxeda.c b/drivers/cpuidle/cpuidle-calxeda.c index c13feec89ea1..ea9728fde9b3 100644 --- a/drivers/cpuidle/cpuidle-calxeda.c +++ b/drivers/cpuidle/cpuidle-calxeda.c @@ -25,16 +25,21 @@ #include #include #include +#include + #include #include -#include + +#include + +#define CALXEDA_IDLE_PARAM \ + ((0 << PSCI_0_2_POWER_STATE_ID_SHIFT) | \ + (0 << PSCI_0_2_POWER_STATE_AFFL_SHIFT) | \ + (PSCI_POWER_STATE_TYPE_POWER_DOWN << PSCI_0_2_POWER_STATE_TYPE_SHIFT)) static int calxeda_idle_finish(unsigned long val) { - const struct psci_power_state ps = { - .type = PSCI_POWER_STATE_TYPE_POWER_DOWN, - }; - return psci_ops.cpu_suspend(ps, __pa(cpu_resume)); + return psci_ops.cpu_suspend(CALXEDA_IDLE_PARAM, __pa(cpu_resume)); } static int calxeda_pwrdown_idle(struct cpuidle_device *dev, diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c index 33253930247f..17a6dc0e2111 100644 --- a/drivers/cpuidle/cpuidle.c +++ b/drivers/cpuidle/cpuidle.c @@ -214,7 +214,7 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv, tick_broadcast_exit(); } - if (!cpuidle_state_is_coupled(dev, drv, entered_state)) + if (!cpuidle_state_is_coupled(drv, entered_state)) local_irq_enable(); diff = ktime_to_us(ktime_sub(time_end, time_start)); @@ -263,7 +263,7 @@ int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev) int cpuidle_enter(struct cpuidle_driver *drv, struct cpuidle_device *dev, int index) { - if (cpuidle_state_is_coupled(dev, drv, index)) + if (cpuidle_state_is_coupled(drv, index)) return cpuidle_enter_state_coupled(dev, drv, index); return cpuidle_enter_state(dev, drv, index); } diff --git a/drivers/cpuidle/cpuidle.h b/drivers/cpuidle/cpuidle.h index ee97e9672ecf..f87f399b0540 100644 --- a/drivers/cpuidle/cpuidle.h +++ b/drivers/cpuidle/cpuidle.h @@ -34,19 +34,24 @@ extern int cpuidle_add_sysfs(struct cpuidle_device *dev); extern void cpuidle_remove_sysfs(struct cpuidle_device *dev); #ifdef CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED -bool cpuidle_state_is_coupled(struct cpuidle_device *dev, - struct cpuidle_driver *drv, int state); +bool cpuidle_state_is_coupled(struct cpuidle_driver *drv, int state); +int cpuidle_coupled_state_verify(struct cpuidle_driver *drv); int cpuidle_enter_state_coupled(struct cpuidle_device *dev, struct cpuidle_driver *drv, int next_state); int cpuidle_coupled_register_device(struct cpuidle_device *dev); void cpuidle_coupled_unregister_device(struct cpuidle_device *dev); #else -static inline bool cpuidle_state_is_coupled(struct cpuidle_device *dev, - struct cpuidle_driver *drv, int state) +static inline +bool cpuidle_state_is_coupled(struct cpuidle_driver *drv, int state) { return false; } +static inline int cpuidle_coupled_state_verify(struct cpuidle_driver *drv) +{ + return 0; +} + static inline int cpuidle_enter_state_coupled(struct cpuidle_device *dev, struct cpuidle_driver *drv, int next_state) { diff --git a/drivers/cpuidle/driver.c b/drivers/cpuidle/driver.c index 5db147859b90..389ade4572be 100644 --- a/drivers/cpuidle/driver.c +++ b/drivers/cpuidle/driver.c @@ -227,6 +227,10 @@ static int __cpuidle_register_driver(struct cpuidle_driver *drv) if (!drv || !drv->state_count) return -EINVAL; + ret = cpuidle_coupled_state_verify(drv); + if (ret) + return ret; + if (cpuidle_disabled()) return -ENODEV; diff --git a/drivers/crypto/bfin_crc.c b/drivers/crypto/bfin_crc.c index d9af9403ab6c..2f0b3337505d 100644 --- a/drivers/crypto/bfin_crc.c +++ b/drivers/crypto/bfin_crc.c @@ -370,8 +370,7 @@ static int bfin_crypto_crc_handle_queue(struct bfin_crypto_crc *crc, sg_init_table(ctx->bufsl, nsg); sg_set_buf(ctx->bufsl, ctx->buflast, ctx->buflast_len); if (nsg > 1) - scatterwalk_sg_chain(ctx->bufsl, nsg, - req->src); + sg_chain(ctx->bufsl, nsg, req->src); ctx->sg = ctx->bufsl; } else ctx->sg = req->src; diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index b2024c95a3cf..48adb2a0903e 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -588,7 +588,7 @@ static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr, * the dmaengine may try to DMA the incorrect amount of data. */ sg_init_table(&ctx->sgl, 1); - ctx->sgl.page_link = ctx->sg->page_link; + sg_assign_page(&ctx->sgl, sg_page(ctx->sg)); ctx->sgl.offset = ctx->sg->offset; sg_dma_len(&ctx->sgl) = len32; sg_dma_address(&ctx->sgl) = sg_dma_address(ctx->sg); diff --git a/drivers/crypto/qat/qat_common/adf_transport_debug.c b/drivers/crypto/qat/qat_common/adf_transport_debug.c index e41986967294..52340b9bb387 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_debug.c +++ b/drivers/crypto/qat/qat_common/adf_transport_debug.c @@ -86,9 +86,7 @@ static int adf_ring_show(struct seq_file *sfile, void *v) { struct adf_etr_ring_data *ring = sfile->private; struct adf_etr_bank_data *bank = ring->bank; - uint32_t *msg = v; void __iomem *csr = ring->bank->csr_addr; - int i, x; if (v == SEQ_START_TOKEN) { int head, tail, empty; @@ -113,18 +111,8 @@ static int adf_ring_show(struct seq_file *sfile, void *v) seq_puts(sfile, "----------- Ring data ------------\n"); return 0; } - seq_printf(sfile, "%p:", msg); - x = 0; - i = 0; - for (; i < (ADF_MSG_SIZE_TO_BYTES(ring->msg_size) >> 2); i++) { - seq_printf(sfile, " %08X", *(msg + i)); - if ((ADF_MSG_SIZE_TO_BYTES(ring->msg_size) >> 2) != i + 1 && - (++x == 8)) { - seq_printf(sfile, "\n%p:", msg + i + 1); - x = 0; - } - } - seq_puts(sfile, "\n"); + seq_hex_dump(sfile, "", DUMP_PREFIX_ADDRESS, 32, 4, + v, ADF_MSG_SIZE_TO_BYTES(ring->msg_size), false); return 0; } diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c index 5c5df1d17f90..be2f5049256a 100644 --- a/drivers/crypto/qce/sha.c +++ b/drivers/crypto/qce/sha.c @@ -296,7 +296,7 @@ static int qce_ahash_update(struct ahash_request *req) if (rctx->buflen) { sg_init_table(rctx->sg, 2); sg_set_buf(rctx->sg, rctx->tmpbuf, rctx->buflen); - scatterwalk_sg_chain(rctx->sg, 2, req->src); + sg_chain(rctx->sg, 2, req->src); req->src = rctx->sg; } diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index 1c19e44c3146..820dc3acb28c 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -999,7 +999,7 @@ static int sahara_sha_prepare_request(struct ahash_request *req) sg_init_table(rctx->in_sg_chain, 2); sg_set_buf(rctx->in_sg_chain, rctx->rembuf, rctx->buf_cnt); - scatterwalk_sg_chain(rctx->in_sg_chain, 2, req->src); + sg_chain(rctx->in_sg_chain, 2, req->src); rctx->total = req->nbytes + rctx->buf_cnt; rctx->in_sg = rctx->in_sg_chain; diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index cd774534d987..3b20a1bce703 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1929,7 +1929,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) sg_init_table(req_ctx->bufsl, nsg); sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf); if (nsg > 1) - scatterwalk_sg_chain(req_ctx->bufsl, 2, areq->src); + sg_chain(req_ctx->bufsl, 2, areq->src); req_ctx->psrc = req_ctx->bufsl; } else req_ctx->psrc = areq->src; diff --git a/drivers/devfreq/event/exynos-ppmu.c b/drivers/devfreq/event/exynos-ppmu.c index 7d99d13bacd8..f9901f52a225 100644 --- a/drivers/devfreq/event/exynos-ppmu.c +++ b/drivers/devfreq/event/exynos-ppmu.c @@ -1,7 +1,7 @@ /* * exynos_ppmu.c - EXYNOS PPMU (Platform Performance Monitoring Unit) support * - * Copyright (c) 2014 Samsung Electronics Co., Ltd. + * Copyright (c) 2014-2015 Samsung Electronics Co., Ltd. * Author : Chanwoo Choi * * This program is free software; you can redistribute it and/or modify @@ -82,6 +82,15 @@ struct __exynos_ppmu_events { PPMU_EVENT(mscl), PPMU_EVENT(fimd0x), PPMU_EVENT(fimd1x), + + /* Only for Exynos5433 SoCs */ + PPMU_EVENT(d0-cpu), + PPMU_EVENT(d0-general), + PPMU_EVENT(d0-rt), + PPMU_EVENT(d1-cpu), + PPMU_EVENT(d1-general), + PPMU_EVENT(d1-rt), + { /* sentinel */ }, }; @@ -96,6 +105,9 @@ static int exynos_ppmu_find_ppmu_id(struct devfreq_event_dev *edev) return -EINVAL; } +/* + * The devfreq-event ops structure for PPMU v1.1 + */ static int exynos_ppmu_disable(struct devfreq_event_dev *edev) { struct exynos_ppmu *info = devfreq_event_get_drvdata(edev); @@ -200,10 +212,158 @@ static const struct devfreq_event_ops exynos_ppmu_ops = { .get_event = exynos_ppmu_get_event, }; +/* + * The devfreq-event ops structure for PPMU v2.0 + */ +static int exynos_ppmu_v2_disable(struct devfreq_event_dev *edev) +{ + struct exynos_ppmu *info = devfreq_event_get_drvdata(edev); + u32 pmnc, clear; + + /* Disable all counters */ + clear = (PPMU_CCNT_MASK | PPMU_PMCNT0_MASK | PPMU_PMCNT1_MASK + | PPMU_PMCNT2_MASK | PPMU_PMCNT3_MASK); + + __raw_writel(clear, info->ppmu.base + PPMU_V2_FLAG); + __raw_writel(clear, info->ppmu.base + PPMU_V2_INTENC); + __raw_writel(clear, info->ppmu.base + PPMU_V2_CNTENC); + __raw_writel(clear, info->ppmu.base + PPMU_V2_CNT_RESET); + + __raw_writel(0x0, info->ppmu.base + PPMU_V2_CIG_CFG0); + __raw_writel(0x0, info->ppmu.base + PPMU_V2_CIG_CFG1); + __raw_writel(0x0, info->ppmu.base + PPMU_V2_CIG_CFG2); + __raw_writel(0x0, info->ppmu.base + PPMU_V2_CIG_RESULT); + __raw_writel(0x0, info->ppmu.base + PPMU_V2_CNT_AUTO); + __raw_writel(0x0, info->ppmu.base + PPMU_V2_CH_EV0_TYPE); + __raw_writel(0x0, info->ppmu.base + PPMU_V2_CH_EV1_TYPE); + __raw_writel(0x0, info->ppmu.base + PPMU_V2_CH_EV2_TYPE); + __raw_writel(0x0, info->ppmu.base + PPMU_V2_CH_EV3_TYPE); + __raw_writel(0x0, info->ppmu.base + PPMU_V2_SM_ID_V); + __raw_writel(0x0, info->ppmu.base + PPMU_V2_SM_ID_A); + __raw_writel(0x0, info->ppmu.base + PPMU_V2_SM_OTHERS_V); + __raw_writel(0x0, info->ppmu.base + PPMU_V2_SM_OTHERS_A); + __raw_writel(0x0, info->ppmu.base + PPMU_V2_INTERRUPT_RESET); + + /* Disable PPMU */ + pmnc = __raw_readl(info->ppmu.base + PPMU_V2_PMNC); + pmnc &= ~PPMU_PMNC_ENABLE_MASK; + __raw_writel(pmnc, info->ppmu.base + PPMU_V2_PMNC); + + return 0; +} + +static int exynos_ppmu_v2_set_event(struct devfreq_event_dev *edev) +{ + struct exynos_ppmu *info = devfreq_event_get_drvdata(edev); + int id = exynos_ppmu_find_ppmu_id(edev); + u32 pmnc, cntens; + + /* Enable all counters */ + cntens = __raw_readl(info->ppmu.base + PPMU_V2_CNTENS); + cntens |= (PPMU_CCNT_MASK | (PPMU_ENABLE << id)); + __raw_writel(cntens, info->ppmu.base + PPMU_V2_CNTENS); + + /* Set the event of Read/Write data count */ + switch (id) { + case PPMU_PMNCNT0: + case PPMU_PMNCNT1: + case PPMU_PMNCNT2: + __raw_writel(PPMU_V2_RO_DATA_CNT | PPMU_V2_WO_DATA_CNT, + info->ppmu.base + PPMU_V2_CH_EVx_TYPE(id)); + break; + case PPMU_PMNCNT3: + __raw_writel(PPMU_V2_EVT3_RW_DATA_CNT, + info->ppmu.base + PPMU_V2_CH_EVx_TYPE(id)); + break; + } + + /* Reset cycle counter/performance counter and enable PPMU */ + pmnc = __raw_readl(info->ppmu.base + PPMU_V2_PMNC); + pmnc &= ~(PPMU_PMNC_ENABLE_MASK + | PPMU_PMNC_COUNTER_RESET_MASK + | PPMU_PMNC_CC_RESET_MASK + | PPMU_PMNC_CC_DIVIDER_MASK + | PPMU_V2_PMNC_START_MODE_MASK); + pmnc |= (PPMU_ENABLE << PPMU_PMNC_ENABLE_SHIFT); + pmnc |= (PPMU_ENABLE << PPMU_PMNC_COUNTER_RESET_SHIFT); + pmnc |= (PPMU_ENABLE << PPMU_PMNC_CC_RESET_SHIFT); + pmnc |= (PPMU_V2_MODE_MANUAL << PPMU_V2_PMNC_START_MODE_SHIFT); + __raw_writel(pmnc, info->ppmu.base + PPMU_V2_PMNC); + + return 0; +} + +static int exynos_ppmu_v2_get_event(struct devfreq_event_dev *edev, + struct devfreq_event_data *edata) +{ + struct exynos_ppmu *info = devfreq_event_get_drvdata(edev); + int id = exynos_ppmu_find_ppmu_id(edev); + u32 pmnc, cntenc; + u32 pmcnt_high, pmcnt_low; + u64 load_count = 0; + + /* Disable PPMU */ + pmnc = __raw_readl(info->ppmu.base + PPMU_V2_PMNC); + pmnc &= ~PPMU_PMNC_ENABLE_MASK; + __raw_writel(pmnc, info->ppmu.base + PPMU_V2_PMNC); + + /* Read cycle count and performance count */ + edata->total_count = __raw_readl(info->ppmu.base + PPMU_V2_CCNT); + + switch (id) { + case PPMU_PMNCNT0: + case PPMU_PMNCNT1: + case PPMU_PMNCNT2: + load_count = __raw_readl(info->ppmu.base + PPMU_V2_PMNCT(id)); + break; + case PPMU_PMNCNT3: + pmcnt_high = __raw_readl(info->ppmu.base + PPMU_V2_PMCNT3_HIGH); + pmcnt_low = __raw_readl(info->ppmu.base + PPMU_V2_PMCNT3_LOW); + load_count = (u64)((pmcnt_high & 0xff) << 32) + (u64)pmcnt_low; + break; + } + edata->load_count = load_count; + + /* Disable all counters */ + cntenc = __raw_readl(info->ppmu.base + PPMU_V2_CNTENC); + cntenc |= (PPMU_CCNT_MASK | (PPMU_ENABLE << id)); + __raw_writel(cntenc, info->ppmu.base + PPMU_V2_CNTENC); + + dev_dbg(&edev->dev, "%25s (load: %ld / %ld)\n", edev->desc->name, + edata->load_count, edata->total_count); + return 0; +} + +static const struct devfreq_event_ops exynos_ppmu_v2_ops = { + .disable = exynos_ppmu_v2_disable, + .set_event = exynos_ppmu_v2_set_event, + .get_event = exynos_ppmu_v2_get_event, +}; + +static const struct of_device_id exynos_ppmu_id_match[] = { + { + .compatible = "samsung,exynos-ppmu", + .data = (void *)&exynos_ppmu_ops, + }, { + .compatible = "samsung,exynos-ppmu-v2", + .data = (void *)&exynos_ppmu_v2_ops, + }, + { /* sentinel */ }, +}; + +static struct devfreq_event_ops *exynos_bus_get_ops(struct device_node *np) +{ + const struct of_device_id *match; + + match = of_match_node(exynos_ppmu_id_match, np); + return (struct devfreq_event_ops *)match->data; +} + static int of_get_devfreq_events(struct device_node *np, struct exynos_ppmu *info) { struct devfreq_event_desc *desc; + struct devfreq_event_ops *event_ops; struct device *dev = info->dev; struct device_node *events_np, *node; int i, j, count; @@ -214,6 +374,7 @@ static int of_get_devfreq_events(struct device_node *np, "failed to get child node of devfreq-event devices\n"); return -EINVAL; } + event_ops = exynos_bus_get_ops(np); count = of_get_child_count(events_np); desc = devm_kzalloc(dev, sizeof(*desc) * count, GFP_KERNEL); @@ -238,7 +399,7 @@ static int of_get_devfreq_events(struct device_node *np, continue; } - desc[j].ops = &exynos_ppmu_ops; + desc[j].ops = event_ops; desc[j].driver_data = info; of_property_read_string(node, "event-name", &desc[j].name); @@ -354,11 +515,6 @@ static int exynos_ppmu_remove(struct platform_device *pdev) return 0; } -static struct of_device_id exynos_ppmu_id_match[] = { - { .compatible = "samsung,exynos-ppmu", }, - { /* sentinel */ }, -}; - static struct platform_driver exynos_ppmu_driver = { .probe = exynos_ppmu_probe, .remove = exynos_ppmu_remove, diff --git a/drivers/devfreq/event/exynos-ppmu.h b/drivers/devfreq/event/exynos-ppmu.h index 4e831d48c138..05774c449137 100644 --- a/drivers/devfreq/event/exynos-ppmu.h +++ b/drivers/devfreq/event/exynos-ppmu.h @@ -26,6 +26,9 @@ enum ppmu_counter { PPMU_PMNCNT_MAX, }; +/*** + * PPMUv1.1 Definitions + */ enum ppmu_event_type { PPMU_RO_BUSY_CYCLE_CNT = 0x0, PPMU_WO_BUSY_CYCLE_CNT = 0x1, @@ -90,4 +93,71 @@ enum ppmu_reg { #define PPMU_PMNCT(x) (PPMU_PMCNT0 + (0x10 * x)) #define PPMU_BEVTxSEL(x) (PPMU_BEVT0SEL + (0x100 * x)) +/*** + * PPMU_V2.0 definitions + */ +enum ppmu_v2_mode { + PPMU_V2_MODE_MANUAL = 0, + PPMU_V2_MODE_AUTO = 1, + PPMU_V2_MODE_CIG = 2, /* CIG (Conditional Interrupt Generation) */ +}; + +enum ppmu_v2_event_type { + PPMU_V2_RO_DATA_CNT = 0x4, + PPMU_V2_WO_DATA_CNT = 0x5, + + PPMU_V2_EVT3_RW_DATA_CNT = 0x22, /* Only for Event3 */ +}; + +enum ppmu_V2_reg { + /* PPC control register */ + PPMU_V2_PMNC = 0x04, + PPMU_V2_CNTENS = 0x08, + PPMU_V2_CNTENC = 0x0c, + PPMU_V2_INTENS = 0x10, + PPMU_V2_INTENC = 0x14, + PPMU_V2_FLAG = 0x18, + + /* Cycle Counter and Performance Event Counter Register */ + PPMU_V2_CCNT = 0x48, + PPMU_V2_PMCNT0 = 0x34, + PPMU_V2_PMCNT1 = 0x38, + PPMU_V2_PMCNT2 = 0x3c, + PPMU_V2_PMCNT3_LOW = 0x40, + PPMU_V2_PMCNT3_HIGH = 0x44, + + /* Bus Event Generator */ + PPMU_V2_CIG_CFG0 = 0x1c, + PPMU_V2_CIG_CFG1 = 0x20, + PPMU_V2_CIG_CFG2 = 0x24, + PPMU_V2_CIG_RESULT = 0x28, + PPMU_V2_CNT_RESET = 0x2c, + PPMU_V2_CNT_AUTO = 0x30, + PPMU_V2_CH_EV0_TYPE = 0x200, + PPMU_V2_CH_EV1_TYPE = 0x204, + PPMU_V2_CH_EV2_TYPE = 0x208, + PPMU_V2_CH_EV3_TYPE = 0x20c, + PPMU_V2_SM_ID_V = 0x220, + PPMU_V2_SM_ID_A = 0x224, + PPMU_V2_SM_OTHERS_V = 0x228, + PPMU_V2_SM_OTHERS_A = 0x22c, + PPMU_V2_INTERRUPT_RESET = 0x260, +}; + +/* PMNC register */ +#define PPMU_V2_PMNC_START_MODE_SHIFT 20 +#define PPMU_V2_PMNC_START_MODE_MASK (0x3 << PPMU_V2_PMNC_START_MODE_SHIFT) + +#define PPMU_PMNC_CC_RESET_SHIFT 2 +#define PPMU_PMNC_COUNTER_RESET_SHIFT 1 +#define PPMU_PMNC_ENABLE_SHIFT 0 +#define PPMU_PMNC_START_MODE_MASK BIT(16) +#define PPMU_PMNC_CC_DIVIDER_MASK BIT(3) +#define PPMU_PMNC_CC_RESET_MASK BIT(2) +#define PPMU_PMNC_COUNTER_RESET_MASK BIT(1) +#define PPMU_PMNC_ENABLE_MASK BIT(0) + +#define PPMU_V2_PMNCT(x) (PPMU_V2_PMCNT0 + (0x4 * x)) +#define PPMU_V2_CH_EVx_TYPE(x) (PPMU_V2_CH_EV0_TYPE + (0x4 * x)) + #endif /* __EXYNOS_PPMU_H__ */ diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 88d474b78076..b4584757dae0 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -33,27 +33,29 @@ if DMADEVICES comment "DMA Devices" -config INTEL_MIC_X100_DMA - tristate "Intel MIC X100 DMA Driver" - depends on 64BIT && X86 && INTEL_MIC_BUS - select DMA_ENGINE - help - This enables DMA support for the Intel Many Integrated Core - (MIC) family of PCIe form factor coprocessor X100 devices that - run a 64 bit Linux OS. This driver will be used by both MIC - host and card drivers. - - If you are building host kernel with a MIC device or a card - kernel for a MIC device, then say M (recommended) or Y, else - say N. If unsure say N. - - More information about the Intel MIC family as well as the Linux - OS and tools for MIC to use with this driver are available from - . - +#core config ASYNC_TX_ENABLE_CHANNEL_SWITCH bool +config ARCH_HAS_ASYNC_TX_FIND_CHANNEL + bool + +config DMA_ENGINE + bool + +config DMA_VIRTUAL_CHANNELS + tristate + +config DMA_ACPI + def_bool y + depends on ACPI + +config DMA_OF + def_bool y + depends on OF + select DMA_ENGINE + +#devices config AMBA_PL08X bool "ARM PrimeCell PL080 or PL081 support" depends on ARM_AMBA @@ -63,9 +65,181 @@ config AMBA_PL08X Platform has a PL08x DMAC device which can provide DMA engine support +config AMCC_PPC440SPE_ADMA + tristate "AMCC PPC440SPe ADMA support" + depends on 440SPe || 440SP + select DMA_ENGINE + select DMA_ENGINE_RAID + select ARCH_HAS_ASYNC_TX_FIND_CHANNEL + select ASYNC_TX_ENABLE_CHANNEL_SWITCH + help + Enable support for the AMCC PPC440SPe RAID engines. + +config AT_HDMAC + tristate "Atmel AHB DMA support" + depends on ARCH_AT91 + select DMA_ENGINE + help + Support the Atmel AHB DMA controller. + +config AT_XDMAC + tristate "Atmel XDMA support" + depends on ARCH_AT91 + select DMA_ENGINE + help + Support the Atmel XDMA controller. + +config AXI_DMAC + tristate "Analog Devices AXI-DMAC DMA support" + depends on MICROBLAZE || NIOS2 || ARCH_ZYNQ || ARCH_SOCFPGA || COMPILE_TEST + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the Analog Devices AXI-DMAC peripheral. This DMA + controller is often used in Analog Device's reference designs for FPGA + platforms. + +config COH901318 + bool "ST-Ericsson COH901318 DMA support" + select DMA_ENGINE + depends on ARCH_U300 + help + Enable support for ST-Ericsson COH 901 318 DMA. + +config DMA_BCM2835 + tristate "BCM2835 DMA engine support" + depends on ARCH_BCM2835 + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + +config DMA_JZ4740 + tristate "JZ4740 DMA support" + depends on MACH_JZ4740 + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + +config DMA_JZ4780 + tristate "JZ4780 DMA support" + depends on MACH_JZ4780 + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + This selects support for the DMA controller in Ingenic JZ4780 SoCs. + If you have a board based on such a SoC and wish to use DMA for + devices which can use the DMA controller, say Y or M here. + +config DMA_OMAP + tristate "OMAP DMA support" + depends on ARCH_OMAP + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + select TI_DMA_CROSSBAR if SOC_DRA7XX + +config DMA_SA11X0 + tristate "SA-11x0 DMA support" + depends on ARCH_SA1100 + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Support the DMA engine found on Intel StrongARM SA-1100 and + SA-1110 SoCs. This DMA engine can only be used with on-chip + devices. + +config DMA_SUN4I + tristate "Allwinner A10 DMA SoCs support" + depends on MACH_SUN4I || MACH_SUN5I || MACH_SUN7I + default (MACH_SUN4I || MACH_SUN5I || MACH_SUN7I) + select DMA_ENGINE + select DMA_OF + select DMA_VIRTUAL_CHANNELS + help + Enable support for the DMA controller present in the sun4i, + sun5i and sun7i Allwinner ARM SoCs. + +config DMA_SUN6I + tristate "Allwinner A31 SoCs DMA support" + depends on MACH_SUN6I || MACH_SUN8I || COMPILE_TEST + depends on RESET_CONTROLLER + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Support for the DMA engine first found in Allwinner A31 SoCs. + +config EP93XX_DMA + bool "Cirrus Logic EP93xx DMA support" + depends on ARCH_EP93XX + select DMA_ENGINE + help + Enable support for the Cirrus Logic EP93xx M2P/M2M DMA controller. + +config FSL_DMA + tristate "Freescale Elo series DMA support" + depends on FSL_SOC + select DMA_ENGINE + select ASYNC_TX_ENABLE_CHANNEL_SWITCH + ---help--- + Enable support for the Freescale Elo series DMA controllers. + The Elo is the DMA controller on some mpc82xx and mpc83xx parts, the + EloPlus is on mpc85xx and mpc86xx and Pxxx parts, and the Elo3 is on + some Txxx and Bxxx parts. + +config FSL_EDMA + tristate "Freescale eDMA engine support" + depends on OF + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Support the Freescale eDMA engine with programmable channel + multiplexing capability for DMA request sources(slot). + This module can be found on Freescale Vybrid and LS-1 SoCs. + +config FSL_RAID + tristate "Freescale RAID engine Support" + depends on FSL_SOC && !ASYNC_TX_ENABLE_CHANNEL_SWITCH + select DMA_ENGINE + select DMA_ENGINE_RAID + ---help--- + Enable support for Freescale RAID Engine. RAID Engine is + available on some QorIQ SoCs (like P5020/P5040). It has + the capability to offload memcpy, xor and pq computation + for raid5/6. + +config IMG_MDC_DMA + tristate "IMG MDC support" + depends on MIPS || COMPILE_TEST + depends on MFD_SYSCON + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the IMG multi-threaded DMA controller (MDC). + +config IMX_DMA + tristate "i.MX DMA support" + depends on ARCH_MXC + select DMA_ENGINE + help + Support the i.MX DMA engine. This engine is integrated into + Freescale i.MX1/21/27 chips. + +config IMX_SDMA + tristate "i.MX SDMA support" + depends on ARCH_MXC + select DMA_ENGINE + help + Support the i.MX SDMA engine. This engine is integrated into + Freescale i.MX25/31/35/51/53/6 chips. + +config IDMA64 + tristate "Intel integrated DMA 64-bit support" + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable DMA support for Intel Low Power Subsystem such as found on + Intel Skylake PCH. + config INTEL_IOATDMA tristate "Intel I/OAT DMA support" - depends on PCI && X86 + depends on PCI && X86_64 select DMA_ENGINE select DMA_ENGINE_RAID select DCA @@ -85,45 +259,69 @@ config INTEL_IOP_ADMA help Enable support for the Intel(R) IOP Series RAID engines. -source "drivers/dma/dw/Kconfig" - -config AT_HDMAC - tristate "Atmel AHB DMA support" - depends on ARCH_AT91 +config INTEL_MIC_X100_DMA + tristate "Intel MIC X100 DMA Driver" + depends on 64BIT && X86 && INTEL_MIC_BUS select DMA_ENGINE help - Support the Atmel AHB DMA controller. + This enables DMA support for the Intel Many Integrated Core + (MIC) family of PCIe form factor coprocessor X100 devices that + run a 64 bit Linux OS. This driver will be used by both MIC + host and card drivers. -config AT_XDMAC - tristate "Atmel XDMA support" - depends on ARCH_AT91 + If you are building host kernel with a MIC device or a card + kernel for a MIC device, then say M (recommended) or Y, else + say N. If unsure say N. + + More information about the Intel MIC family as well as the Linux + OS and tools for MIC to use with this driver are available from + . + +config K3_DMA + tristate "Hisilicon K3 DMA support" + depends on ARCH_HI3xxx + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Support the DMA engine for Hisilicon K3 platform + devices. + +config LPC18XX_DMAMUX + bool "NXP LPC18xx/43xx DMA MUX for PL080" + depends on ARCH_LPC18XX || COMPILE_TEST + depends on OF && AMBA_PL08X + select MFD_SYSCON + help + Enable support for DMA on NXP LPC18xx/43xx platforms + with PL080 and multiplexed DMA request lines. + +config MMP_PDMA + bool "MMP PDMA support" + depends on (ARCH_MMP || ARCH_PXA) select DMA_ENGINE help - Support the Atmel XDMA controller. + Support the MMP PDMA engine for PXA and MMP platform. -config FSL_DMA - tristate "Freescale Elo series DMA support" - depends on FSL_SOC +config MMP_TDMA + bool "MMP Two-Channel DMA support" + depends on ARCH_MMP select DMA_ENGINE - select ASYNC_TX_ENABLE_CHANNEL_SWITCH - ---help--- - Enable support for the Freescale Elo series DMA controllers. - The Elo is the DMA controller on some mpc82xx and mpc83xx parts, the - EloPlus is on mpc85xx and mpc86xx and Pxxx parts, and the Elo3 is on - some Txxx and Bxxx parts. + select MMP_SRAM + help + Support the MMP Two-Channel DMA engine. + This engine used for MMP Audio DMA and pxa910 SQU. + It needs sram driver under mach-mmp. -config FSL_RAID - tristate "Freescale RAID engine Support" - depends on FSL_SOC && !ASYNC_TX_ENABLE_CHANNEL_SWITCH - select DMA_ENGINE - select DMA_ENGINE_RAID - ---help--- - Enable support for Freescale RAID Engine. RAID Engine is - available on some QorIQ SoCs (like P5020/P5040). It has - the capability to offload memcpy, xor and pq computation - for raid5/6. - -source "drivers/dma/hsu/Kconfig" +config MOXART_DMA + tristate "MOXART DMA support" + depends on ARCH_MOXART + select DMA_ENGINE + select DMA_OF + select DMA_VIRTUAL_CHANNELS + help + Enable support for the MOXA ART SoC DMA controller. + + Say Y here if you enabled MMP ADMA, otherwise say N. config MPC512X_DMA tristate "Freescale MPC512x built-in DMA engine support" @@ -132,8 +330,6 @@ config MPC512X_DMA ---help--- Enable support for the Freescale MPC512x built-in DMA engine. -source "drivers/dma/bestcomm/Kconfig" - config MV_XOR bool "Marvell XOR engine support" depends on PLAT_ORION @@ -143,6 +339,15 @@ config MV_XOR ---help--- Enable support for the Marvell XOR engine. +config MXS_DMA + bool "MXS DMA support" + depends on SOC_IMX23 || SOC_IMX28 || SOC_IMX6Q + select STMP_DEVICE + select DMA_ENGINE + help + Support the MXS DMA engine. This engine including APBH-DMA + and APBX-DMA is integrated into Freescale i.MX23/28/MX6Q/MX6DL chips. + config MX3_IPU bool "MX3x Image Processing Unit support" depends on ARCH_MXC @@ -162,6 +367,36 @@ config MX3_IPU_IRQS To avoid bloating the irq_desc[] array we allocate a sufficient number of IRQ slots and map them dynamically to specific sources. +config NBPFAXI_DMA + tristate "Renesas Type-AXI NBPF DMA support" + select DMA_ENGINE + depends on ARM || COMPILE_TEST + help + Support for "Type-AXI" NBPF DMA IPs from Renesas + +config PCH_DMA + tristate "Intel EG20T PCH / LAPIS Semicon IOH(ML7213/ML7223/ML7831) DMA" + depends on PCI && (X86_32 || COMPILE_TEST) + select DMA_ENGINE + help + Enable support for Intel EG20T PCH DMA engine. + + This driver also can be used for LAPIS Semiconductor IOH(Input/ + Output Hub), ML7213, ML7223 and ML7831. + ML7213 IOH is for IVI(In-Vehicle Infotainment) use, ML7223 IOH is + for MP(Media Phone) use and ML7831 IOH is for general purpose use. + ML7213/ML7223/ML7831 is companion chip for Intel Atom E6xx series. + ML7213/ML7223/ML7831 is completely compatible for Intel EG20T PCH. + +config PL330_DMA + tristate "DMA API Driver for PL330" + select DMA_ENGINE + depends on ARM_AMBA + help + Select if your platform has one or more PL330 DMACs. + You need to provide platform specific settings via + platform_data for a dma-pl330 device. + config PXA_DMA bool "PXA DMA support" depends on (ARCH_MMP || ARCH_PXA) @@ -173,6 +408,41 @@ config PXA_DMA 16 to 32 channels for peripheral to memory or memory to memory transfers. +config QCOM_BAM_DMA + tristate "QCOM BAM DMA support" + depends on ARCH_QCOM || (COMPILE_TEST && OF && ARM) + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + ---help--- + Enable support for the QCOM BAM DMA controller. This controller + provides DMA capabilities for a variety of on-chip devices. + +config SIRF_DMA + tristate "CSR SiRFprimaII/SiRFmarco DMA support" + depends on ARCH_SIRF + select DMA_ENGINE + help + Enable support for the CSR SiRFprimaII DMA engine. + +config STE_DMA40 + bool "ST-Ericsson DMA40 support" + depends on ARCH_U8500 + select DMA_ENGINE + help + Support for ST-Ericsson DMA40 controller + +config S3C24XX_DMAC + tristate "Samsung S3C24XX DMA support" + depends on ARCH_S3C24XX + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Support for the Samsung S3C24XX DMA controller driver. The + DMA controller is having multiple DMA channels which can be + configured for different peripherals like audio, UART, SPI. + The DMA controller can transfer data from memory to peripheral, + periphal to memory, periphal to periphal and memory to memory. + config TXX9_DMAC tristate "Toshiba TXx9 SoC DMA support" depends on MACH_TX49XX || MACH_TX39XX @@ -193,44 +463,6 @@ config TEGRA20_APB_DMA This DMA controller transfers data from memory to peripheral fifo or vice versa. It does not support memory to memory data transfer. -config S3C24XX_DMAC - tristate "Samsung S3C24XX DMA support" - depends on ARCH_S3C24XX - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - Support for the Samsung S3C24XX DMA controller driver. The - DMA controller is having multiple DMA channels which can be - configured for different peripherals like audio, UART, SPI. - The DMA controller can transfer data from memory to peripheral, - periphal to memory, periphal to periphal and memory to memory. - -source "drivers/dma/sh/Kconfig" - -config COH901318 - bool "ST-Ericsson COH901318 DMA support" - select DMA_ENGINE - depends on ARCH_U300 - help - Enable support for ST-Ericsson COH 901 318 DMA. - -config STE_DMA40 - bool "ST-Ericsson DMA40 support" - depends on ARCH_U8500 - select DMA_ENGINE - help - Support for ST-Ericsson DMA40 controller - -config AMCC_PPC440SPE_ADMA - tristate "AMCC PPC440SPe ADMA support" - depends on 440SPe || 440SP - select DMA_ENGINE - select DMA_ENGINE_RAID - select ARCH_HAS_ASYNC_TX_FIND_CHANNEL - select ASYNC_TX_ENABLE_CHANNEL_SWITCH - help - Enable support for the AMCC PPC440SPe RAID engines. - config TIMB_DMA tristate "Timberdale FPGA DMA support" depends on MFD_TIMBERDALE @@ -238,12 +470,16 @@ config TIMB_DMA help Enable support for the Timberdale FPGA DMA engine. -config SIRF_DMA - tristate "CSR SiRFprimaII/SiRFmarco DMA support" - depends on ARCH_SIRF +config TI_CPPI41 + tristate "AM33xx CPPI41 DMA support" + depends on ARCH_OMAP select DMA_ENGINE help - Enable support for the CSR SiRFprimaII DMA engine. + The Communications Port Programming Interface (CPPI) 4.1 DMA engine + is currently used by the USB driver on AM335x platforms. + +config TI_DMA_CROSSBAR + bool config TI_EDMA bool "TI EDMA support" @@ -256,160 +492,14 @@ config TI_EDMA Enable support for the TI EDMA controller. This DMA engine is found on TI DaVinci and AM33xx parts. -config TI_DMA_CROSSBAR - bool - -config ARCH_HAS_ASYNC_TX_FIND_CHANNEL - bool - -config PL330_DMA - tristate "DMA API Driver for PL330" +config XGENE_DMA + tristate "APM X-Gene DMA support" + depends on ARCH_XGENE || COMPILE_TEST select DMA_ENGINE - depends on ARM_AMBA + select DMA_ENGINE_RAID + select ASYNC_TX_ENABLE_CHANNEL_SWITCH help - Select if your platform has one or more PL330 DMACs. - You need to provide platform specific settings via - platform_data for a dma-pl330 device. - -config PCH_DMA - tristate "Intel EG20T PCH / LAPIS Semicon IOH(ML7213/ML7223/ML7831) DMA" - depends on PCI && (X86_32 || COMPILE_TEST) - select DMA_ENGINE - help - Enable support for Intel EG20T PCH DMA engine. - - This driver also can be used for LAPIS Semiconductor IOH(Input/ - Output Hub), ML7213, ML7223 and ML7831. - ML7213 IOH is for IVI(In-Vehicle Infotainment) use, ML7223 IOH is - for MP(Media Phone) use and ML7831 IOH is for general purpose use. - ML7213/ML7223/ML7831 is companion chip for Intel Atom E6xx series. - ML7213/ML7223/ML7831 is completely compatible for Intel EG20T PCH. - -config IMX_SDMA - tristate "i.MX SDMA support" - depends on ARCH_MXC - select DMA_ENGINE - help - Support the i.MX SDMA engine. This engine is integrated into - Freescale i.MX25/31/35/51/53/6 chips. - -config IMX_DMA - tristate "i.MX DMA support" - depends on ARCH_MXC - select DMA_ENGINE - help - Support the i.MX DMA engine. This engine is integrated into - Freescale i.MX1/21/27 chips. - -config MXS_DMA - bool "MXS DMA support" - depends on SOC_IMX23 || SOC_IMX28 || SOC_IMX6Q - select STMP_DEVICE - select DMA_ENGINE - help - Support the MXS DMA engine. This engine including APBH-DMA - and APBX-DMA is integrated into Freescale i.MX23/28/MX6Q/MX6DL chips. - -config EP93XX_DMA - bool "Cirrus Logic EP93xx DMA support" - depends on ARCH_EP93XX - select DMA_ENGINE - help - Enable support for the Cirrus Logic EP93xx M2P/M2M DMA controller. - -config DMA_SA11X0 - tristate "SA-11x0 DMA support" - depends on ARCH_SA1100 - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - Support the DMA engine found on Intel StrongARM SA-1100 and - SA-1110 SoCs. This DMA engine can only be used with on-chip - devices. - -config MMP_TDMA - bool "MMP Two-Channel DMA support" - depends on ARCH_MMP - select DMA_ENGINE - select MMP_SRAM - help - Support the MMP Two-Channel DMA engine. - This engine used for MMP Audio DMA and pxa910 SQU. - It needs sram driver under mach-mmp. - - Say Y here if you enabled MMP ADMA, otherwise say N. - -config DMA_OMAP - tristate "OMAP DMA support" - depends on ARCH_OMAP - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - select TI_DMA_CROSSBAR if SOC_DRA7XX - -config DMA_BCM2835 - tristate "BCM2835 DMA engine support" - depends on ARCH_BCM2835 - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - -config TI_CPPI41 - tristate "AM33xx CPPI41 DMA support" - depends on ARCH_OMAP - select DMA_ENGINE - help - The Communications Port Programming Interface (CPPI) 4.1 DMA engine - is currently used by the USB driver on AM335x platforms. - -config MMP_PDMA - bool "MMP PDMA support" - depends on (ARCH_MMP || ARCH_PXA) - select DMA_ENGINE - help - Support the MMP PDMA engine for PXA and MMP platform. - -config DMA_JZ4740 - tristate "JZ4740 DMA support" - depends on MACH_JZ4740 - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - -config DMA_JZ4780 - tristate "JZ4780 DMA support" - depends on MACH_JZ4780 - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - This selects support for the DMA controller in Ingenic JZ4780 SoCs. - If you have a board based on such a SoC and wish to use DMA for - devices which can use the DMA controller, say Y or M here. - -config K3_DMA - tristate "Hisilicon K3 DMA support" - depends on ARCH_HI3xxx - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - Support the DMA engine for Hisilicon K3 platform - devices. - -config MOXART_DMA - tristate "MOXART DMA support" - depends on ARCH_MOXART - select DMA_ENGINE - select DMA_OF - select DMA_VIRTUAL_CHANNELS - help - Enable support for the MOXA ART SoC DMA controller. - -config FSL_EDMA - tristate "Freescale eDMA engine support" - depends on OF - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - Support the Freescale eDMA engine with programmable channel - multiplexing capability for DMA request sources(slot). - This module can be found on Freescale Vybrid and LS-1 SoCs. + Enable support for the APM X-Gene SoC DMA engine. config XILINX_VDMA tristate "Xilinx AXI VDMA Engine" @@ -425,55 +515,25 @@ config XILINX_VDMA channels, Memory Mapped to Stream (MM2S) and Stream to Memory Mapped (S2MM) for the data transfers. -config DMA_SUN6I - tristate "Allwinner A31 SoCs DMA support" - depends on MACH_SUN6I || MACH_SUN8I || COMPILE_TEST - depends on RESET_CONTROLLER +config ZX_DMA + tristate "ZTE ZX296702 DMA support" + depends on ARCH_ZX select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help - Support for the DMA engine first found in Allwinner A31 SoCs. + Support the DMA engine for ZTE ZX296702 platform devices. -config NBPFAXI_DMA - tristate "Renesas Type-AXI NBPF DMA support" - select DMA_ENGINE - depends on ARM || COMPILE_TEST - help - Support for "Type-AXI" NBPF DMA IPs from Renesas -config IMG_MDC_DMA - tristate "IMG MDC support" - depends on MIPS || COMPILE_TEST - depends on MFD_SYSCON - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - Enable support for the IMG multi-threaded DMA controller (MDC). +# driver files +source "drivers/dma/bestcomm/Kconfig" -config XGENE_DMA - tristate "APM X-Gene DMA support" - depends on ARCH_XGENE || COMPILE_TEST - select DMA_ENGINE - select DMA_ENGINE_RAID - select ASYNC_TX_ENABLE_CHANNEL_SWITCH - help - Enable support for the APM X-Gene SoC DMA engine. +source "drivers/dma/dw/Kconfig" -config DMA_ENGINE - bool +source "drivers/dma/hsu/Kconfig" -config DMA_VIRTUAL_CHANNELS - tristate - -config DMA_ACPI - def_bool y - depends on ACPI - -config DMA_OF - def_bool y - depends on OF - select DMA_ENGINE +source "drivers/dma/sh/Kconfig" +# clients comment "DMA Clients" depends on DMA_ENGINE @@ -498,13 +558,4 @@ config DMATEST config DMA_ENGINE_RAID bool -config QCOM_BAM_DMA - tristate "QCOM BAM DMA support" - depends on ARCH_QCOM || (COMPILE_TEST && OF && ARM) - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - ---help--- - Enable support for the QCOM BAM DMA controller. This controller - provides DMA capabilities for a variety of on-chip devices. - endif diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 6a4d6f2827da..7711a7180726 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -1,58 +1,69 @@ +#dmaengine debug flags subdir-ccflags-$(CONFIG_DMADEVICES_DEBUG) := -DDEBUG subdir-ccflags-$(CONFIG_DMADEVICES_VDEBUG) += -DVERBOSE_DEBUG +#core obj-$(CONFIG_DMA_ENGINE) += dmaengine.o obj-$(CONFIG_DMA_VIRTUAL_CHANNELS) += virt-dma.o obj-$(CONFIG_DMA_ACPI) += acpi-dma.o obj-$(CONFIG_DMA_OF) += of-dma.o +#dmatest obj-$(CONFIG_DMATEST) += dmatest.o -obj-$(CONFIG_INTEL_IOATDMA) += ioat/ -obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o -obj-$(CONFIG_FSL_DMA) += fsldma.o -obj-$(CONFIG_HSU_DMA) += hsu/ -obj-$(CONFIG_MPC512X_DMA) += mpc512x_dma.o -obj-$(CONFIG_PPC_BESTCOMM) += bestcomm/ -obj-$(CONFIG_MV_XOR) += mv_xor.o -obj-$(CONFIG_DW_DMAC_CORE) += dw/ + +#devices +obj-$(CONFIG_AMBA_PL08X) += amba-pl08x.o +obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/ obj-$(CONFIG_AT_HDMAC) += at_hdmac.o obj-$(CONFIG_AT_XDMAC) += at_xdmac.o -obj-$(CONFIG_MX3_IPU) += ipu/ -obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o -obj-$(CONFIG_RENESAS_DMA) += sh/ +obj-$(CONFIG_AXI_DMAC) += dma-axi-dmac.o obj-$(CONFIG_COH901318) += coh901318.o coh901318_lli.o -obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/ -obj-$(CONFIG_IMX_SDMA) += imx-sdma.o -obj-$(CONFIG_IMX_DMA) += imx-dma.o -obj-$(CONFIG_MXS_DMA) += mxs-dma.o -obj-$(CONFIG_PXA_DMA) += pxa_dma.o -obj-$(CONFIG_TIMB_DMA) += timb_dma.o -obj-$(CONFIG_SIRF_DMA) += sirf-dma.o -obj-$(CONFIG_TI_EDMA) += edma.o -obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o -obj-$(CONFIG_TEGRA20_APB_DMA) += tegra20-apb-dma.o -obj-$(CONFIG_S3C24XX_DMAC) += s3c24xx-dma.o -obj-$(CONFIG_PL330_DMA) += pl330.o -obj-$(CONFIG_PCH_DMA) += pch_dma.o -obj-$(CONFIG_AMBA_PL08X) += amba-pl08x.o -obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o -obj-$(CONFIG_DMA_SA11X0) += sa11x0-dma.o -obj-$(CONFIG_MMP_TDMA) += mmp_tdma.o -obj-$(CONFIG_DMA_OMAP) += omap-dma.o -obj-$(CONFIG_TI_DMA_CROSSBAR) += ti-dma-crossbar.o obj-$(CONFIG_DMA_BCM2835) += bcm2835-dma.o -obj-$(CONFIG_MMP_PDMA) += mmp_pdma.o obj-$(CONFIG_DMA_JZ4740) += dma-jz4740.o obj-$(CONFIG_DMA_JZ4780) += dma-jz4780.o -obj-$(CONFIG_TI_CPPI41) += cppi41.o -obj-$(CONFIG_K3_DMA) += k3dma.o -obj-$(CONFIG_MOXART_DMA) += moxart-dma.o -obj-$(CONFIG_FSL_RAID) += fsl_raid.o -obj-$(CONFIG_FSL_EDMA) += fsl-edma.o -obj-$(CONFIG_QCOM_BAM_DMA) += qcom_bam_dma.o -obj-y += xilinx/ -obj-$(CONFIG_INTEL_MIC_X100_DMA) += mic_x100_dma.o -obj-$(CONFIG_NBPFAXI_DMA) += nbpfaxi.o +obj-$(CONFIG_DMA_OMAP) += omap-dma.o +obj-$(CONFIG_DMA_SA11X0) += sa11x0-dma.o +obj-$(CONFIG_DMA_SUN4I) += sun4i-dma.o obj-$(CONFIG_DMA_SUN6I) += sun6i-dma.o +obj-$(CONFIG_DW_DMAC_CORE) += dw/ +obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o +obj-$(CONFIG_FSL_DMA) += fsldma.o +obj-$(CONFIG_FSL_EDMA) += fsl-edma.o +obj-$(CONFIG_FSL_RAID) += fsl_raid.o +obj-$(CONFIG_HSU_DMA) += hsu/ obj-$(CONFIG_IMG_MDC_DMA) += img-mdc-dma.o +obj-$(CONFIG_IMX_DMA) += imx-dma.o +obj-$(CONFIG_IMX_SDMA) += imx-sdma.o +obj-$(CONFIG_IDMA64) += idma64.o +obj-$(CONFIG_INTEL_IOATDMA) += ioat/ +obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o +obj-$(CONFIG_INTEL_MIC_X100_DMA) += mic_x100_dma.o +obj-$(CONFIG_K3_DMA) += k3dma.o +obj-$(CONFIG_LPC18XX_DMAMUX) += lpc18xx-dmamux.o +obj-$(CONFIG_MMP_PDMA) += mmp_pdma.o +obj-$(CONFIG_MMP_TDMA) += mmp_tdma.o +obj-$(CONFIG_MOXART_DMA) += moxart-dma.o +obj-$(CONFIG_MPC512X_DMA) += mpc512x_dma.o +obj-$(CONFIG_MV_XOR) += mv_xor.o +obj-$(CONFIG_MXS_DMA) += mxs-dma.o +obj-$(CONFIG_MX3_IPU) += ipu/ +obj-$(CONFIG_NBPFAXI_DMA) += nbpfaxi.o +obj-$(CONFIG_PCH_DMA) += pch_dma.o +obj-$(CONFIG_PL330_DMA) += pl330.o +obj-$(CONFIG_PPC_BESTCOMM) += bestcomm/ +obj-$(CONFIG_PXA_DMA) += pxa_dma.o +obj-$(CONFIG_QCOM_BAM_DMA) += qcom_bam_dma.o +obj-$(CONFIG_RENESAS_DMA) += sh/ +obj-$(CONFIG_SIRF_DMA) += sirf-dma.o +obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o +obj-$(CONFIG_S3C24XX_DMAC) += s3c24xx-dma.o +obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o +obj-$(CONFIG_TEGRA20_APB_DMA) += tegra20-apb-dma.o +obj-$(CONFIG_TIMB_DMA) += timb_dma.o +obj-$(CONFIG_TI_CPPI41) += cppi41.o +obj-$(CONFIG_TI_DMA_CROSSBAR) += ti-dma-crossbar.o +obj-$(CONFIG_TI_EDMA) += edma.o obj-$(CONFIG_XGENE_DMA) += xgene-dma.o +obj-$(CONFIG_ZX_DMA) += zx296702_dma.o + +obj-y += xilinx/ diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c index 5de3cf453f35..9b42c0588550 100644 --- a/drivers/dma/amba-pl08x.c +++ b/drivers/dma/amba-pl08x.c @@ -83,6 +83,8 @@ #include #include #include +#include +#include #include #include #include @@ -2030,10 +2032,188 @@ static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x) } #endif +#ifdef CONFIG_OF +static struct dma_chan *pl08x_find_chan_id(struct pl08x_driver_data *pl08x, + u32 id) +{ + struct pl08x_dma_chan *chan; + + list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) { + if (chan->signal == id) + return &chan->vc.chan; + } + + return NULL; +} + +static struct dma_chan *pl08x_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct pl08x_driver_data *pl08x = ofdma->of_dma_data; + struct pl08x_channel_data *data; + struct pl08x_dma_chan *chan; + struct dma_chan *dma_chan; + + if (!pl08x) + return NULL; + + if (dma_spec->args_count != 2) + return NULL; + + dma_chan = pl08x_find_chan_id(pl08x, dma_spec->args[0]); + if (dma_chan) + return dma_get_slave_channel(dma_chan); + + chan = devm_kzalloc(pl08x->slave.dev, sizeof(*chan) + sizeof(*data), + GFP_KERNEL); + if (!chan) + return NULL; + + data = (void *)&chan[1]; + data->bus_id = "(none)"; + data->periph_buses = dma_spec->args[1]; + + chan->cd = data; + chan->host = pl08x; + chan->slave = true; + chan->name = data->bus_id; + chan->state = PL08X_CHAN_IDLE; + chan->signal = dma_spec->args[0]; + chan->vc.desc_free = pl08x_desc_free; + + vchan_init(&chan->vc, &pl08x->slave); + + return dma_get_slave_channel(&chan->vc.chan); +} + +static int pl08x_of_probe(struct amba_device *adev, + struct pl08x_driver_data *pl08x, + struct device_node *np) +{ + struct pl08x_platform_data *pd; + u32 cctl_memcpy = 0; + u32 val; + int ret; + + pd = devm_kzalloc(&adev->dev, sizeof(*pd), GFP_KERNEL); + if (!pd) + return -ENOMEM; + + /* Eligible bus masters for fetching LLIs */ + if (of_property_read_bool(np, "lli-bus-interface-ahb1")) + pd->lli_buses |= PL08X_AHB1; + if (of_property_read_bool(np, "lli-bus-interface-ahb2")) + pd->lli_buses |= PL08X_AHB2; + if (!pd->lli_buses) { + dev_info(&adev->dev, "no bus masters for LLIs stated, assume all\n"); + pd->lli_buses |= PL08X_AHB1 | PL08X_AHB2; + } + + /* Eligible bus masters for memory access */ + if (of_property_read_bool(np, "mem-bus-interface-ahb1")) + pd->mem_buses |= PL08X_AHB1; + if (of_property_read_bool(np, "mem-bus-interface-ahb2")) + pd->mem_buses |= PL08X_AHB2; + if (!pd->mem_buses) { + dev_info(&adev->dev, "no bus masters for memory stated, assume all\n"); + pd->mem_buses |= PL08X_AHB1 | PL08X_AHB2; + } + + /* Parse the memcpy channel properties */ + ret = of_property_read_u32(np, "memcpy-burst-size", &val); + if (ret) { + dev_info(&adev->dev, "no memcpy burst size specified, using 1 byte\n"); + val = 1; + } + switch (val) { + default: + dev_err(&adev->dev, "illegal burst size for memcpy, set to 1\n"); + /* Fall through */ + case 1: + cctl_memcpy |= PL080_BSIZE_1 << PL080_CONTROL_SB_SIZE_SHIFT | + PL080_BSIZE_1 << PL080_CONTROL_DB_SIZE_SHIFT; + break; + case 4: + cctl_memcpy |= PL080_BSIZE_4 << PL080_CONTROL_SB_SIZE_SHIFT | + PL080_BSIZE_4 << PL080_CONTROL_DB_SIZE_SHIFT; + break; + case 8: + cctl_memcpy |= PL080_BSIZE_8 << PL080_CONTROL_SB_SIZE_SHIFT | + PL080_BSIZE_8 << PL080_CONTROL_DB_SIZE_SHIFT; + break; + case 16: + cctl_memcpy |= PL080_BSIZE_16 << PL080_CONTROL_SB_SIZE_SHIFT | + PL080_BSIZE_16 << PL080_CONTROL_DB_SIZE_SHIFT; + break; + case 32: + cctl_memcpy |= PL080_BSIZE_32 << PL080_CONTROL_SB_SIZE_SHIFT | + PL080_BSIZE_32 << PL080_CONTROL_DB_SIZE_SHIFT; + break; + case 64: + cctl_memcpy |= PL080_BSIZE_64 << PL080_CONTROL_SB_SIZE_SHIFT | + PL080_BSIZE_64 << PL080_CONTROL_DB_SIZE_SHIFT; + break; + case 128: + cctl_memcpy |= PL080_BSIZE_128 << PL080_CONTROL_SB_SIZE_SHIFT | + PL080_BSIZE_128 << PL080_CONTROL_DB_SIZE_SHIFT; + break; + case 256: + cctl_memcpy |= PL080_BSIZE_256 << PL080_CONTROL_SB_SIZE_SHIFT | + PL080_BSIZE_256 << PL080_CONTROL_DB_SIZE_SHIFT; + break; + } + + ret = of_property_read_u32(np, "memcpy-bus-width", &val); + if (ret) { + dev_info(&adev->dev, "no memcpy bus width specified, using 8 bits\n"); + val = 8; + } + switch (val) { + default: + dev_err(&adev->dev, "illegal bus width for memcpy, set to 8 bits\n"); + /* Fall through */ + case 8: + cctl_memcpy |= PL080_WIDTH_8BIT << PL080_CONTROL_SWIDTH_SHIFT | + PL080_WIDTH_8BIT << PL080_CONTROL_DWIDTH_SHIFT; + break; + case 16: + cctl_memcpy |= PL080_WIDTH_16BIT << PL080_CONTROL_SWIDTH_SHIFT | + PL080_WIDTH_16BIT << PL080_CONTROL_DWIDTH_SHIFT; + break; + case 32: + cctl_memcpy |= PL080_WIDTH_32BIT << PL080_CONTROL_SWIDTH_SHIFT | + PL080_WIDTH_32BIT << PL080_CONTROL_DWIDTH_SHIFT; + break; + } + + /* This is currently the only thing making sense */ + cctl_memcpy |= PL080_CONTROL_PROT_SYS; + + /* Set up memcpy channel */ + pd->memcpy_channel.bus_id = "memcpy"; + pd->memcpy_channel.cctl_memcpy = cctl_memcpy; + /* Use the buses that can access memory, obviously */ + pd->memcpy_channel.periph_buses = pd->mem_buses; + + pl08x->pd = pd; + + return of_dma_controller_register(adev->dev.of_node, pl08x_of_xlate, + pl08x); +} +#else +static inline int pl08x_of_probe(struct amba_device *adev, + struct pl08x_driver_data *pl08x, + struct device_node *np) +{ + return -EINVAL; +} +#endif + static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) { struct pl08x_driver_data *pl08x; const struct vendor_data *vd = id->data; + struct device_node *np = adev->dev.of_node; u32 tsfr_size; int ret = 0; int i; @@ -2093,9 +2273,15 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) /* Get the platform data */ pl08x->pd = dev_get_platdata(&adev->dev); if (!pl08x->pd) { - dev_err(&adev->dev, "no platform data supplied\n"); - ret = -EINVAL; - goto out_no_platdata; + if (np) { + ret = pl08x_of_probe(adev, pl08x, np); + if (ret) + goto out_no_platdata; + } else { + dev_err(&adev->dev, "no platform data supplied\n"); + ret = -EINVAL; + goto out_no_platdata; + } } /* Assign useful pointers to the driver state */ diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index d3629b7482dd..58d406230d89 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -448,6 +448,7 @@ static void atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) { struct dma_async_tx_descriptor *txd = &desc->txd; + struct at_dma *atdma = to_at_dma(atchan->chan_common.device); dev_vdbg(chan2dev(&atchan->chan_common), "descriptor %u complete\n", txd->cookie); @@ -456,6 +457,13 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) if (!atc_chan_is_cyclic(atchan)) dma_cookie_complete(txd); + /* If the transfer was a memset, free our temporary buffer */ + if (desc->memset) { + dma_pool_free(atdma->memset_pool, desc->memset_vaddr, + desc->memset_paddr); + desc->memset = false; + } + /* move children to free_list */ list_splice_init(&desc->tx_list, &atchan->free_list); /* move myself to free_list */ @@ -717,14 +725,14 @@ atc_prep_dma_interleaved(struct dma_chan *chan, size_t len = 0; int i; + if (unlikely(!xt || xt->numf != 1 || !xt->frame_size)) + return NULL; + dev_info(chan2dev(chan), "%s: src=0x%08x, dest=0x%08x, numf=%d, frame_size=%d, flags=0x%lx\n", __func__, xt->src_start, xt->dst_start, xt->numf, xt->frame_size, flags); - if (unlikely(!xt || xt->numf != 1 || !xt->frame_size)) - return NULL; - /* * The controller can only "skip" X bytes every Y bytes, so we * need to make sure we are given a template that fit that @@ -873,6 +881,93 @@ atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, return NULL; } +/** + * atc_prep_dma_memset - prepare a memcpy operation + * @chan: the channel to prepare operation on + * @dest: operation virtual destination address + * @value: value to set memory buffer to + * @len: operation length + * @flags: tx descriptor status flags + */ +static struct dma_async_tx_descriptor * +atc_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value, + size_t len, unsigned long flags) +{ + struct at_dma_chan *atchan = to_at_dma_chan(chan); + struct at_dma *atdma = to_at_dma(chan->device); + struct at_desc *desc = NULL; + size_t xfer_count; + u32 ctrla; + u32 ctrlb; + + dev_vdbg(chan2dev(chan), "%s: d0x%x v0x%x l0x%zx f0x%lx\n", __func__, + dest, value, len, flags); + + if (unlikely(!len)) { + dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__); + return NULL; + } + + if (!is_dma_fill_aligned(chan->device, dest, 0, len)) { + dev_dbg(chan2dev(chan), "%s: buffer is not aligned\n", + __func__); + return NULL; + } + + xfer_count = len >> 2; + if (xfer_count > ATC_BTSIZE_MAX) { + dev_err(chan2dev(chan), "%s: buffer is too big\n", + __func__); + return NULL; + } + + ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN + | ATC_SRC_ADDR_MODE_FIXED + | ATC_DST_ADDR_MODE_INCR + | ATC_FC_MEM2MEM; + + ctrla = ATC_SRC_WIDTH(2) | + ATC_DST_WIDTH(2); + + desc = atc_desc_get(atchan); + if (!desc) { + dev_err(chan2dev(chan), "%s: can't get a descriptor\n", + __func__); + return NULL; + } + + desc->memset_vaddr = dma_pool_alloc(atdma->memset_pool, GFP_ATOMIC, + &desc->memset_paddr); + if (!desc->memset_vaddr) { + dev_err(chan2dev(chan), "%s: couldn't allocate buffer\n", + __func__); + goto err_put_desc; + } + + *desc->memset_vaddr = value; + desc->memset = true; + + desc->lli.saddr = desc->memset_paddr; + desc->lli.daddr = dest; + desc->lli.ctrla = ctrla | xfer_count; + desc->lli.ctrlb = ctrlb; + + desc->txd.cookie = -EBUSY; + desc->len = len; + desc->total_len = len; + + /* set end-of-link on the descriptor */ + set_desc_eol(desc); + + desc->txd.flags = flags; + + return &desc->txd; + +err_put_desc: + atc_desc_put(atchan, desc); + return NULL; +} + /** * atc_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction @@ -1755,6 +1850,8 @@ static int __init at_dma_probe(struct platform_device *pdev) dma_cap_set(DMA_SG, at91sam9rl_config.cap_mask); dma_cap_set(DMA_INTERLEAVE, at91sam9g45_config.cap_mask); dma_cap_set(DMA_MEMCPY, at91sam9g45_config.cap_mask); + dma_cap_set(DMA_MEMSET, at91sam9g45_config.cap_mask); + dma_cap_set(DMA_PRIVATE, at91sam9g45_config.cap_mask); dma_cap_set(DMA_SLAVE, at91sam9g45_config.cap_mask); dma_cap_set(DMA_SG, at91sam9g45_config.cap_mask); @@ -1818,7 +1915,16 @@ static int __init at_dma_probe(struct platform_device *pdev) if (!atdma->dma_desc_pool) { dev_err(&pdev->dev, "No memory for descriptors dma pool\n"); err = -ENOMEM; - goto err_pool_create; + goto err_desc_pool_create; + } + + /* create a pool of consistent memory blocks for memset blocks */ + atdma->memset_pool = dma_pool_create("at_hdmac_memset_pool", + &pdev->dev, sizeof(int), 4, 0); + if (!atdma->memset_pool) { + dev_err(&pdev->dev, "No memory for memset dma pool\n"); + err = -ENOMEM; + goto err_memset_pool_create; } /* clear any pending interrupt */ @@ -1864,6 +1970,11 @@ static int __init at_dma_probe(struct platform_device *pdev) if (dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask)) atdma->dma_common.device_prep_dma_memcpy = atc_prep_dma_memcpy; + if (dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask)) { + atdma->dma_common.device_prep_dma_memset = atc_prep_dma_memset; + atdma->dma_common.fill_align = DMAENGINE_ALIGN_4_BYTES; + } + if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) { atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg; /* controller can do slave DMA: can trigger cyclic transfers */ @@ -1884,8 +1995,9 @@ static int __init at_dma_probe(struct platform_device *pdev) dma_writel(atdma, EN, AT_DMA_ENABLE); - dev_info(&pdev->dev, "Atmel AHB DMA Controller ( %s%s%s), %d channels\n", + dev_info(&pdev->dev, "Atmel AHB DMA Controller ( %s%s%s%s), %d channels\n", dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask) ? "cpy " : "", + dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask) ? "set " : "", dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask) ? "slave " : "", dma_has_cap(DMA_SG, atdma->dma_common.cap_mask) ? "sg-cpy " : "", plat_dat->nr_channels); @@ -1910,8 +2022,10 @@ static int __init at_dma_probe(struct platform_device *pdev) err_of_dma_controller_register: dma_async_device_unregister(&atdma->dma_common); + dma_pool_destroy(atdma->memset_pool); +err_memset_pool_create: dma_pool_destroy(atdma->dma_desc_pool); -err_pool_create: +err_desc_pool_create: free_irq(platform_get_irq(pdev, 0), atdma); err_irq: clk_disable_unprepare(atdma->clk); @@ -1936,6 +2050,7 @@ static int at_dma_remove(struct platform_device *pdev) at_dma_off(atdma); dma_async_device_unregister(&atdma->dma_common); + dma_pool_destroy(atdma->memset_pool); dma_pool_destroy(atdma->dma_desc_pool); free_irq(platform_get_irq(pdev, 0), atdma); diff --git a/drivers/dma/at_hdmac_regs.h b/drivers/dma/at_hdmac_regs.h index 7f5a08230f76..c3bebbe899ac 100644 --- a/drivers/dma/at_hdmac_regs.h +++ b/drivers/dma/at_hdmac_regs.h @@ -200,6 +200,11 @@ struct at_desc { size_t boundary; size_t dst_hole; size_t src_hole; + + /* Memset temporary buffer */ + bool memset; + dma_addr_t memset_paddr; + int *memset_vaddr; }; static inline struct at_desc * @@ -330,6 +335,7 @@ struct at_dma { u8 all_chan_mask; struct dma_pool *dma_desc_pool; + struct dma_pool *memset_pool; /* AT THE END channels table */ struct at_dma_chan chan[0]; }; diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c index 40afa2a16cfc..a165b4bfd330 100644 --- a/drivers/dma/at_xdmac.c +++ b/drivers/dma/at_xdmac.c @@ -625,12 +625,12 @@ at_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction direction, unsigned long flags, void *context) { - struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); - struct at_xdmac_desc *first = NULL, *prev = NULL; - struct scatterlist *sg; - int i; - unsigned int xfer_size = 0; - unsigned long irqflags; + struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); + struct at_xdmac_desc *first = NULL, *prev = NULL; + struct scatterlist *sg; + int i; + unsigned int xfer_size = 0; + unsigned long irqflags; struct dma_async_tx_descriptor *ret = NULL; if (!sgl) @@ -797,10 +797,7 @@ at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, list_add_tail(&desc->desc_node, &first->descs_list); } - prev->lld.mbr_nda = first->tx_dma_desc.phys; - dev_dbg(chan2dev(chan), - "%s: chain lld: prev=0x%p, mbr_nda=%pad\n", - __func__, prev, &prev->lld.mbr_nda); + at_xdmac_queue_desc(chan, prev, first); first->tx_dma_desc.flags = flags; first->xfer_size = buf_len; first->direction = direction; @@ -1135,7 +1132,7 @@ static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan, * SAMA5D4x), so we can use the same interface for source and dest, * that solves the fact we don't know the direction. */ - u32 chan_cc = AT_XDMAC_CC_DAM_INCREMENTED_AM + u32 chan_cc = AT_XDMAC_CC_DAM_UBS_AM | AT_XDMAC_CC_SAM_INCREMENTED_AM | AT_XDMAC_CC_DIF(0) | AT_XDMAC_CC_SIF(0) @@ -1203,6 +1200,168 @@ at_xdmac_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value, return &desc->tx_dma_desc; } +static struct dma_async_tx_descriptor * +at_xdmac_prep_dma_memset_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, int value, + unsigned long flags) +{ + struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); + struct at_xdmac_desc *desc, *pdesc = NULL, + *ppdesc = NULL, *first = NULL; + struct scatterlist *sg, *psg = NULL, *ppsg = NULL; + size_t stride = 0, pstride = 0, len = 0; + int i; + + if (!sgl) + return NULL; + + dev_dbg(chan2dev(chan), "%s: sg_len=%d, value=0x%x, flags=0x%lx\n", + __func__, sg_len, value, flags); + + /* Prepare descriptors. */ + for_each_sg(sgl, sg, sg_len, i) { + dev_dbg(chan2dev(chan), "%s: dest=0x%08x, len=%d, pattern=0x%x, flags=0x%lx\n", + __func__, sg_dma_address(sg), sg_dma_len(sg), + value, flags); + desc = at_xdmac_memset_create_desc(chan, atchan, + sg_dma_address(sg), + sg_dma_len(sg), + value); + if (!desc && first) + list_splice_init(&first->descs_list, + &atchan->free_descs_list); + + if (!first) + first = desc; + + /* Update our strides */ + pstride = stride; + if (psg) + stride = sg_dma_address(sg) - + (sg_dma_address(psg) + sg_dma_len(psg)); + + /* + * The scatterlist API gives us only the address and + * length of each elements. + * + * Unfortunately, we don't have the stride, which we + * will need to compute. + * + * That make us end up in a situation like this one: + * len stride len stride len + * +-------+ +-------+ +-------+ + * | N-2 | | N-1 | | N | + * +-------+ +-------+ +-------+ + * + * We need all these three elements (N-2, N-1 and N) + * to actually take the decision on whether we need to + * queue N-1 or reuse N-2. + * + * We will only consider N if it is the last element. + */ + if (ppdesc && pdesc) { + if ((stride == pstride) && + (sg_dma_len(ppsg) == sg_dma_len(psg))) { + dev_dbg(chan2dev(chan), + "%s: desc 0x%p can be merged with desc 0x%p\n", + __func__, pdesc, ppdesc); + + /* + * Increment the block count of the + * N-2 descriptor + */ + at_xdmac_increment_block_count(chan, ppdesc); + ppdesc->lld.mbr_dus = stride; + + /* + * Put back the N-1 descriptor in the + * free descriptor list + */ + list_add_tail(&pdesc->desc_node, + &atchan->free_descs_list); + + /* + * Make our N-1 descriptor pointer + * point to the N-2 since they were + * actually merged. + */ + pdesc = ppdesc; + + /* + * Rule out the case where we don't have + * pstride computed yet (our second sg + * element) + * + * We also want to catch the case where there + * would be a negative stride, + */ + } else if (pstride || + sg_dma_address(sg) < sg_dma_address(psg)) { + /* + * Queue the N-1 descriptor after the + * N-2 + */ + at_xdmac_queue_desc(chan, ppdesc, pdesc); + + /* + * Add the N-1 descriptor to the list + * of the descriptors used for this + * transfer + */ + list_add_tail(&desc->desc_node, + &first->descs_list); + dev_dbg(chan2dev(chan), + "%s: add desc 0x%p to descs_list 0x%p\n", + __func__, desc, first); + } + } + + /* + * If we are the last element, just see if we have the + * same size than the previous element. + * + * If so, we can merge it with the previous descriptor + * since we don't care about the stride anymore. + */ + if ((i == (sg_len - 1)) && + sg_dma_len(ppsg) == sg_dma_len(psg)) { + dev_dbg(chan2dev(chan), + "%s: desc 0x%p can be merged with desc 0x%p\n", + __func__, desc, pdesc); + + /* + * Increment the block count of the N-1 + * descriptor + */ + at_xdmac_increment_block_count(chan, pdesc); + pdesc->lld.mbr_dus = stride; + + /* + * Put back the N descriptor in the free + * descriptor list + */ + list_add_tail(&desc->desc_node, + &atchan->free_descs_list); + } + + /* Update our descriptors */ + ppdesc = pdesc; + pdesc = desc; + + /* Update our scatter pointers */ + ppsg = psg; + psg = sg; + + len += sg_dma_len(sg); + } + + first->tx_dma_desc.cookie = -EBUSY; + first->tx_dma_desc.flags = flags; + first->xfer_size = len; + + return &first->tx_dma_desc; +} + static enum dma_status at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, struct dma_tx_state *txstate) @@ -1736,6 +1895,7 @@ static int at_xdmac_probe(struct platform_device *pdev) dma_cap_set(DMA_INTERLEAVE, atxdmac->dma.cap_mask); dma_cap_set(DMA_MEMCPY, atxdmac->dma.cap_mask); dma_cap_set(DMA_MEMSET, atxdmac->dma.cap_mask); + dma_cap_set(DMA_MEMSET_SG, atxdmac->dma.cap_mask); dma_cap_set(DMA_SLAVE, atxdmac->dma.cap_mask); /* * Without DMA_PRIVATE the driver is not able to allocate more than @@ -1751,6 +1911,7 @@ static int at_xdmac_probe(struct platform_device *pdev) atxdmac->dma.device_prep_interleaved_dma = at_xdmac_prep_interleaved; atxdmac->dma.device_prep_dma_memcpy = at_xdmac_prep_dma_memcpy; atxdmac->dma.device_prep_dma_memset = at_xdmac_prep_dma_memset; + atxdmac->dma.device_prep_dma_memset_sg = at_xdmac_prep_dma_memset_sg; atxdmac->dma.device_prep_slave_sg = at_xdmac_prep_slave_sg; atxdmac->dma.device_config = at_xdmac_device_config; atxdmac->dma.device_pause = at_xdmac_device_pause; diff --git a/drivers/dma/coh901318.c b/drivers/dma/coh901318.c index fd22dd36985f..c340ca9bd2b5 100644 --- a/drivers/dma/coh901318.c +++ b/drivers/dma/coh901318.c @@ -2730,7 +2730,7 @@ static int __init coh901318_probe(struct platform_device *pdev) * This controller can only access address at even 32bit boundaries, * i.e. 2^2 */ - base->dma_memcpy.copy_align = 2; + base->dma_memcpy.copy_align = DMAENGINE_ALIGN_4_BYTES; err = dma_async_device_register(&base->dma_memcpy); if (err) diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c new file mode 100644 index 000000000000..5b2395e7e04d --- /dev/null +++ b/drivers/dma/dma-axi-dmac.c @@ -0,0 +1,691 @@ +/* + * Driver for the Analog Devices AXI-DMAC core + * + * Copyright 2013-2015 Analog Devices Inc. + * Author: Lars-Peter Clausen + * + * Licensed under the GPL-2. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "dmaengine.h" +#include "virt-dma.h" + +/* + * The AXI-DMAC is a soft IP core that is used in FPGA designs. The core has + * various instantiation parameters which decided the exact feature set support + * by the core. + * + * Each channel of the core has a source interface and a destination interface. + * The number of channels and the type of the channel interfaces is selected at + * configuration time. A interface can either be a connected to a central memory + * interconnect, which allows access to system memory, or it can be connected to + * a dedicated bus which is directly connected to a data port on a peripheral. + * Given that those are configuration options of the core that are selected when + * it is instantiated this means that they can not be changed by software at + * runtime. By extension this means that each channel is uni-directional. It can + * either be device to memory or memory to device, but not both. Also since the + * device side is a dedicated data bus only connected to a single peripheral + * there is no address than can or needs to be configured for the device side. + */ + +#define AXI_DMAC_REG_IRQ_MASK 0x80 +#define AXI_DMAC_REG_IRQ_PENDING 0x84 +#define AXI_DMAC_REG_IRQ_SOURCE 0x88 + +#define AXI_DMAC_REG_CTRL 0x400 +#define AXI_DMAC_REG_TRANSFER_ID 0x404 +#define AXI_DMAC_REG_START_TRANSFER 0x408 +#define AXI_DMAC_REG_FLAGS 0x40c +#define AXI_DMAC_REG_DEST_ADDRESS 0x410 +#define AXI_DMAC_REG_SRC_ADDRESS 0x414 +#define AXI_DMAC_REG_X_LENGTH 0x418 +#define AXI_DMAC_REG_Y_LENGTH 0x41c +#define AXI_DMAC_REG_DEST_STRIDE 0x420 +#define AXI_DMAC_REG_SRC_STRIDE 0x424 +#define AXI_DMAC_REG_TRANSFER_DONE 0x428 +#define AXI_DMAC_REG_ACTIVE_TRANSFER_ID 0x42c +#define AXI_DMAC_REG_STATUS 0x430 +#define AXI_DMAC_REG_CURRENT_SRC_ADDR 0x434 +#define AXI_DMAC_REG_CURRENT_DEST_ADDR 0x438 + +#define AXI_DMAC_CTRL_ENABLE BIT(0) +#define AXI_DMAC_CTRL_PAUSE BIT(1) + +#define AXI_DMAC_IRQ_SOT BIT(0) +#define AXI_DMAC_IRQ_EOT BIT(1) + +#define AXI_DMAC_FLAG_CYCLIC BIT(0) + +struct axi_dmac_sg { + dma_addr_t src_addr; + dma_addr_t dest_addr; + unsigned int x_len; + unsigned int y_len; + unsigned int dest_stride; + unsigned int src_stride; + unsigned int id; +}; + +struct axi_dmac_desc { + struct virt_dma_desc vdesc; + bool cyclic; + + unsigned int num_submitted; + unsigned int num_completed; + unsigned int num_sgs; + struct axi_dmac_sg sg[]; +}; + +struct axi_dmac_chan { + struct virt_dma_chan vchan; + + struct axi_dmac_desc *next_desc; + struct list_head active_descs; + enum dma_transfer_direction direction; + + unsigned int src_width; + unsigned int dest_width; + unsigned int src_type; + unsigned int dest_type; + + unsigned int max_length; + unsigned int align_mask; + + bool hw_cyclic; + bool hw_2d; +}; + +struct axi_dmac { + void __iomem *base; + int irq; + + struct clk *clk; + + struct dma_device dma_dev; + struct axi_dmac_chan chan; + + struct device_dma_parameters dma_parms; +}; + +static struct axi_dmac *chan_to_axi_dmac(struct axi_dmac_chan *chan) +{ + return container_of(chan->vchan.chan.device, struct axi_dmac, + dma_dev); +} + +static struct axi_dmac_chan *to_axi_dmac_chan(struct dma_chan *c) +{ + return container_of(c, struct axi_dmac_chan, vchan.chan); +} + +static struct axi_dmac_desc *to_axi_dmac_desc(struct virt_dma_desc *vdesc) +{ + return container_of(vdesc, struct axi_dmac_desc, vdesc); +} + +static void axi_dmac_write(struct axi_dmac *axi_dmac, unsigned int reg, + unsigned int val) +{ + writel(val, axi_dmac->base + reg); +} + +static int axi_dmac_read(struct axi_dmac *axi_dmac, unsigned int reg) +{ + return readl(axi_dmac->base + reg); +} + +static int axi_dmac_src_is_mem(struct axi_dmac_chan *chan) +{ + return chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM; +} + +static int axi_dmac_dest_is_mem(struct axi_dmac_chan *chan) +{ + return chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM; +} + +static bool axi_dmac_check_len(struct axi_dmac_chan *chan, unsigned int len) +{ + if (len == 0 || len > chan->max_length) + return false; + if ((len & chan->align_mask) != 0) /* Not aligned */ + return false; + return true; +} + +static bool axi_dmac_check_addr(struct axi_dmac_chan *chan, dma_addr_t addr) +{ + if ((addr & chan->align_mask) != 0) /* Not aligned */ + return false; + return true; +} + +static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) +{ + struct axi_dmac *dmac = chan_to_axi_dmac(chan); + struct virt_dma_desc *vdesc; + struct axi_dmac_desc *desc; + struct axi_dmac_sg *sg; + unsigned int flags = 0; + unsigned int val; + + val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); + if (val) /* Queue is full, wait for the next SOT IRQ */ + return; + + desc = chan->next_desc; + + if (!desc) { + vdesc = vchan_next_desc(&chan->vchan); + if (!vdesc) + return; + list_move_tail(&vdesc->node, &chan->active_descs); + desc = to_axi_dmac_desc(vdesc); + } + sg = &desc->sg[desc->num_submitted]; + + desc->num_submitted++; + if (desc->num_submitted == desc->num_sgs) + chan->next_desc = NULL; + else + chan->next_desc = desc; + + sg->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID); + + if (axi_dmac_dest_is_mem(chan)) { + axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->dest_addr); + axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->dest_stride); + } + + if (axi_dmac_src_is_mem(chan)) { + axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->src_addr); + axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->src_stride); + } + + /* + * If the hardware supports cyclic transfers and there is no callback to + * call, enable hw cyclic mode to avoid unnecessary interrupts. + */ + if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback) + flags |= AXI_DMAC_FLAG_CYCLIC; + + axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->x_len - 1); + axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->y_len - 1); + axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, flags); + axi_dmac_write(dmac, AXI_DMAC_REG_START_TRANSFER, 1); +} + +static struct axi_dmac_desc *axi_dmac_active_desc(struct axi_dmac_chan *chan) +{ + return list_first_entry_or_null(&chan->active_descs, + struct axi_dmac_desc, vdesc.node); +} + +static void axi_dmac_transfer_done(struct axi_dmac_chan *chan, + unsigned int completed_transfers) +{ + struct axi_dmac_desc *active; + struct axi_dmac_sg *sg; + + active = axi_dmac_active_desc(chan); + if (!active) + return; + + if (active->cyclic) { + vchan_cyclic_callback(&active->vdesc); + } else { + do { + sg = &active->sg[active->num_completed]; + if (!(BIT(sg->id) & completed_transfers)) + break; + active->num_completed++; + if (active->num_completed == active->num_sgs) { + list_del(&active->vdesc.node); + vchan_cookie_complete(&active->vdesc); + active = axi_dmac_active_desc(chan); + } + } while (active); + } +} + +static irqreturn_t axi_dmac_interrupt_handler(int irq, void *devid) +{ + struct axi_dmac *dmac = devid; + unsigned int pending; + + pending = axi_dmac_read(dmac, AXI_DMAC_REG_IRQ_PENDING); + axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_PENDING, pending); + + spin_lock(&dmac->chan.vchan.lock); + /* One or more transfers have finished */ + if (pending & AXI_DMAC_IRQ_EOT) { + unsigned int completed; + + completed = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_DONE); + axi_dmac_transfer_done(&dmac->chan, completed); + } + /* Space has become available in the descriptor queue */ + if (pending & AXI_DMAC_IRQ_SOT) + axi_dmac_start_transfer(&dmac->chan); + spin_unlock(&dmac->chan.vchan.lock); + + return IRQ_HANDLED; +} + +static int axi_dmac_terminate_all(struct dma_chan *c) +{ + struct axi_dmac_chan *chan = to_axi_dmac_chan(c); + struct axi_dmac *dmac = chan_to_axi_dmac(chan); + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&chan->vchan.lock, flags); + axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, 0); + chan->next_desc = NULL; + vchan_get_all_descriptors(&chan->vchan, &head); + list_splice_tail_init(&chan->active_descs, &head); + spin_unlock_irqrestore(&chan->vchan.lock, flags); + + vchan_dma_desc_free_list(&chan->vchan, &head); + + return 0; +} + +static void axi_dmac_issue_pending(struct dma_chan *c) +{ + struct axi_dmac_chan *chan = to_axi_dmac_chan(c); + struct axi_dmac *dmac = chan_to_axi_dmac(chan); + unsigned long flags; + + axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, AXI_DMAC_CTRL_ENABLE); + + spin_lock_irqsave(&chan->vchan.lock, flags); + if (vchan_issue_pending(&chan->vchan)) + axi_dmac_start_transfer(chan); + spin_unlock_irqrestore(&chan->vchan.lock, flags); +} + +static struct axi_dmac_desc *axi_dmac_alloc_desc(unsigned int num_sgs) +{ + struct axi_dmac_desc *desc; + + desc = kzalloc(sizeof(struct axi_dmac_desc) + + sizeof(struct axi_dmac_sg) * num_sgs, GFP_NOWAIT); + if (!desc) + return NULL; + + desc->num_sgs = num_sgs; + + return desc; +} + +static struct dma_async_tx_descriptor *axi_dmac_prep_slave_sg( + struct dma_chan *c, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct axi_dmac_chan *chan = to_axi_dmac_chan(c); + struct axi_dmac_desc *desc; + struct scatterlist *sg; + unsigned int i; + + if (direction != chan->direction) + return NULL; + + desc = axi_dmac_alloc_desc(sg_len); + if (!desc) + return NULL; + + for_each_sg(sgl, sg, sg_len, i) { + if (!axi_dmac_check_addr(chan, sg_dma_address(sg)) || + !axi_dmac_check_len(chan, sg_dma_len(sg))) { + kfree(desc); + return NULL; + } + + if (direction == DMA_DEV_TO_MEM) + desc->sg[i].dest_addr = sg_dma_address(sg); + else + desc->sg[i].src_addr = sg_dma_address(sg); + desc->sg[i].x_len = sg_dma_len(sg); + desc->sg[i].y_len = 1; + } + + desc->cyclic = false; + + return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); +} + +static struct dma_async_tx_descriptor *axi_dmac_prep_dma_cyclic( + struct dma_chan *c, dma_addr_t buf_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct axi_dmac_chan *chan = to_axi_dmac_chan(c); + struct axi_dmac_desc *desc; + unsigned int num_periods, i; + + if (direction != chan->direction) + return NULL; + + if (!axi_dmac_check_len(chan, buf_len) || + !axi_dmac_check_addr(chan, buf_addr)) + return NULL; + + if (period_len == 0 || buf_len % period_len) + return NULL; + + num_periods = buf_len / period_len; + + desc = axi_dmac_alloc_desc(num_periods); + if (!desc) + return NULL; + + for (i = 0; i < num_periods; i++) { + if (direction == DMA_DEV_TO_MEM) + desc->sg[i].dest_addr = buf_addr; + else + desc->sg[i].src_addr = buf_addr; + desc->sg[i].x_len = period_len; + desc->sg[i].y_len = 1; + buf_addr += period_len; + } + + desc->cyclic = true; + + return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); +} + +static struct dma_async_tx_descriptor *axi_dmac_prep_interleaved( + struct dma_chan *c, struct dma_interleaved_template *xt, + unsigned long flags) +{ + struct axi_dmac_chan *chan = to_axi_dmac_chan(c); + struct axi_dmac_desc *desc; + size_t dst_icg, src_icg; + + if (xt->frame_size != 1) + return NULL; + + if (xt->dir != chan->direction) + return NULL; + + if (axi_dmac_src_is_mem(chan)) { + if (!xt->src_inc || !axi_dmac_check_addr(chan, xt->src_start)) + return NULL; + } + + if (axi_dmac_dest_is_mem(chan)) { + if (!xt->dst_inc || !axi_dmac_check_addr(chan, xt->dst_start)) + return NULL; + } + + dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]); + src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]); + + if (chan->hw_2d) { + if (!axi_dmac_check_len(chan, xt->sgl[0].size) || + !axi_dmac_check_len(chan, xt->numf)) + return NULL; + if (xt->sgl[0].size + dst_icg > chan->max_length || + xt->sgl[0].size + src_icg > chan->max_length) + return NULL; + } else { + if (dst_icg != 0 || src_icg != 0) + return NULL; + if (chan->max_length / xt->sgl[0].size < xt->numf) + return NULL; + if (!axi_dmac_check_len(chan, xt->sgl[0].size * xt->numf)) + return NULL; + } + + desc = axi_dmac_alloc_desc(1); + if (!desc) + return NULL; + + if (axi_dmac_src_is_mem(chan)) { + desc->sg[0].src_addr = xt->src_start; + desc->sg[0].src_stride = xt->sgl[0].size + src_icg; + } + + if (axi_dmac_dest_is_mem(chan)) { + desc->sg[0].dest_addr = xt->dst_start; + desc->sg[0].dest_stride = xt->sgl[0].size + dst_icg; + } + + if (chan->hw_2d) { + desc->sg[0].x_len = xt->sgl[0].size; + desc->sg[0].y_len = xt->numf; + } else { + desc->sg[0].x_len = xt->sgl[0].size * xt->numf; + desc->sg[0].y_len = 1; + } + + return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); +} + +static void axi_dmac_free_chan_resources(struct dma_chan *c) +{ + vchan_free_chan_resources(to_virt_chan(c)); +} + +static void axi_dmac_desc_free(struct virt_dma_desc *vdesc) +{ + kfree(container_of(vdesc, struct axi_dmac_desc, vdesc)); +} + +/* + * The configuration stored in the devicetree matches the configuration + * parameters of the peripheral instance and allows the driver to know which + * features are implemented and how it should behave. + */ +static int axi_dmac_parse_chan_dt(struct device_node *of_chan, + struct axi_dmac_chan *chan) +{ + u32 val; + int ret; + + ret = of_property_read_u32(of_chan, "reg", &val); + if (ret) + return ret; + + /* We only support 1 channel for now */ + if (val != 0) + return -EINVAL; + + ret = of_property_read_u32(of_chan, "adi,source-bus-type", &val); + if (ret) + return ret; + if (val > AXI_DMAC_BUS_TYPE_FIFO) + return -EINVAL; + chan->src_type = val; + + ret = of_property_read_u32(of_chan, "adi,destination-bus-type", &val); + if (ret) + return ret; + if (val > AXI_DMAC_BUS_TYPE_FIFO) + return -EINVAL; + chan->dest_type = val; + + ret = of_property_read_u32(of_chan, "adi,source-bus-width", &val); + if (ret) + return ret; + chan->src_width = val / 8; + + ret = of_property_read_u32(of_chan, "adi,destination-bus-width", &val); + if (ret) + return ret; + chan->dest_width = val / 8; + + ret = of_property_read_u32(of_chan, "adi,length-width", &val); + if (ret) + return ret; + + if (val >= 32) + chan->max_length = UINT_MAX; + else + chan->max_length = (1ULL << val) - 1; + + chan->align_mask = max(chan->dest_width, chan->src_width) - 1; + + if (axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan)) + chan->direction = DMA_MEM_TO_MEM; + else if (!axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan)) + chan->direction = DMA_MEM_TO_DEV; + else if (axi_dmac_dest_is_mem(chan) && !axi_dmac_src_is_mem(chan)) + chan->direction = DMA_DEV_TO_MEM; + else + chan->direction = DMA_DEV_TO_DEV; + + chan->hw_cyclic = of_property_read_bool(of_chan, "adi,cyclic"); + chan->hw_2d = of_property_read_bool(of_chan, "adi,2d"); + + return 0; +} + +static int axi_dmac_probe(struct platform_device *pdev) +{ + struct device_node *of_channels, *of_chan; + struct dma_device *dma_dev; + struct axi_dmac *dmac; + struct resource *res; + int ret; + + dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); + if (!dmac) + return -ENOMEM; + + dmac->irq = platform_get_irq(pdev, 0); + if (dmac->irq <= 0) + return -EINVAL; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + dmac->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(dmac->base)) + return PTR_ERR(dmac->base); + + dmac->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(dmac->clk)) + return PTR_ERR(dmac->clk); + + INIT_LIST_HEAD(&dmac->chan.active_descs); + + of_channels = of_get_child_by_name(pdev->dev.of_node, "adi,channels"); + if (of_channels == NULL) + return -ENODEV; + + for_each_child_of_node(of_channels, of_chan) { + ret = axi_dmac_parse_chan_dt(of_chan, &dmac->chan); + if (ret) { + of_node_put(of_chan); + of_node_put(of_channels); + return -EINVAL; + } + } + of_node_put(of_channels); + + pdev->dev.dma_parms = &dmac->dma_parms; + dma_set_max_seg_size(&pdev->dev, dmac->chan.max_length); + + dma_dev = &dmac->dma_dev; + dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); + dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask); + dma_dev->device_free_chan_resources = axi_dmac_free_chan_resources; + dma_dev->device_tx_status = dma_cookie_status; + dma_dev->device_issue_pending = axi_dmac_issue_pending; + dma_dev->device_prep_slave_sg = axi_dmac_prep_slave_sg; + dma_dev->device_prep_dma_cyclic = axi_dmac_prep_dma_cyclic; + dma_dev->device_prep_interleaved_dma = axi_dmac_prep_interleaved; + dma_dev->device_terminate_all = axi_dmac_terminate_all; + dma_dev->dev = &pdev->dev; + dma_dev->chancnt = 1; + dma_dev->src_addr_widths = BIT(dmac->chan.src_width); + dma_dev->dst_addr_widths = BIT(dmac->chan.dest_width); + dma_dev->directions = BIT(dmac->chan.direction); + dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; + INIT_LIST_HEAD(&dma_dev->channels); + + dmac->chan.vchan.desc_free = axi_dmac_desc_free; + vchan_init(&dmac->chan.vchan, dma_dev); + + ret = clk_prepare_enable(dmac->clk); + if (ret < 0) + return ret; + + axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_MASK, 0x00); + + ret = dma_async_device_register(dma_dev); + if (ret) + goto err_clk_disable; + + ret = of_dma_controller_register(pdev->dev.of_node, + of_dma_xlate_by_chan_id, dma_dev); + if (ret) + goto err_unregister_device; + + ret = request_irq(dmac->irq, axi_dmac_interrupt_handler, 0, + dev_name(&pdev->dev), dmac); + if (ret) + goto err_unregister_of; + + platform_set_drvdata(pdev, dmac); + + return 0; + +err_unregister_of: + of_dma_controller_free(pdev->dev.of_node); +err_unregister_device: + dma_async_device_unregister(&dmac->dma_dev); +err_clk_disable: + clk_disable_unprepare(dmac->clk); + + return ret; +} + +static int axi_dmac_remove(struct platform_device *pdev) +{ + struct axi_dmac *dmac = platform_get_drvdata(pdev); + + of_dma_controller_free(pdev->dev.of_node); + free_irq(dmac->irq, dmac); + tasklet_kill(&dmac->chan.vchan.task); + dma_async_device_unregister(&dmac->dma_dev); + clk_disable_unprepare(dmac->clk); + + return 0; +} + +static const struct of_device_id axi_dmac_of_match_table[] = { + { .compatible = "adi,axi-dmac-1.00.a" }, + { }, +}; + +static struct platform_driver axi_dmac_driver = { + .driver = { + .name = "dma-axi-dmac", + .of_match_table = axi_dmac_of_match_table, + }, + .probe = axi_dmac_probe, + .remove = axi_dmac_remove, +}; +module_platform_driver(axi_dmac_driver); + +MODULE_AUTHOR("Lars-Peter Clausen "); +MODULE_DESCRIPTION("DMA controller driver for the AXI-DMAC controller"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/dma-jz4780.c b/drivers/dma/dma-jz4780.c index 26d2f0e09ea3..dade7c47ff18 100644 --- a/drivers/dma/dma-jz4780.c +++ b/drivers/dma/dma-jz4780.c @@ -145,7 +145,8 @@ struct jz4780_dma_dev { struct jz4780_dma_chan chan[JZ_DMA_NR_CHANNELS]; }; -struct jz4780_dma_data { +struct jz4780_dma_filter_data { + struct device_node *of_node; uint32_t transfer_type; int channel; }; @@ -214,11 +215,25 @@ static void jz4780_dma_desc_free(struct virt_dma_desc *vdesc) kfree(desc); } -static uint32_t jz4780_dma_transfer_size(unsigned long val, int *ord) +static uint32_t jz4780_dma_transfer_size(unsigned long val, uint32_t *shift) { - *ord = ffs(val) - 1; + int ord = ffs(val) - 1; - switch (*ord) { + /* + * 8 byte transfer sizes unsupported so fall back on 4. If it's larger + * than the maximum, just limit it. It is perfectly safe to fall back + * in this way since we won't exceed the maximum burst size supported + * by the device, the only effect is reduced efficiency. This is better + * than refusing to perform the request at all. + */ + if (ord == 3) + ord = 2; + else if (ord > 7) + ord = 7; + + *shift = ord; + + switch (ord) { case 0: return JZ_DMA_SIZE_1_BYTE; case 1: @@ -231,20 +246,17 @@ static uint32_t jz4780_dma_transfer_size(unsigned long val, int *ord) return JZ_DMA_SIZE_32_BYTE; case 6: return JZ_DMA_SIZE_64_BYTE; - case 7: - return JZ_DMA_SIZE_128_BYTE; default: - return -EINVAL; + return JZ_DMA_SIZE_128_BYTE; } } -static uint32_t jz4780_dma_setup_hwdesc(struct jz4780_dma_chan *jzchan, +static int jz4780_dma_setup_hwdesc(struct jz4780_dma_chan *jzchan, struct jz4780_dma_hwdesc *desc, dma_addr_t addr, size_t len, enum dma_transfer_direction direction) { struct dma_slave_config *config = &jzchan->config; uint32_t width, maxburst, tsz; - int ord; if (direction == DMA_MEM_TO_DEV) { desc->dcm = JZ_DMA_DCM_SAI; @@ -271,8 +283,8 @@ static uint32_t jz4780_dma_setup_hwdesc(struct jz4780_dma_chan *jzchan, * divisible by the transfer size, and we must not use more than the * maximum burst specified by the user. */ - tsz = jz4780_dma_transfer_size(addr | len | (width * maxburst), &ord); - jzchan->transfer_shift = ord; + tsz = jz4780_dma_transfer_size(addr | len | (width * maxburst), + &jzchan->transfer_shift); switch (width) { case DMA_SLAVE_BUSWIDTH_1_BYTE: @@ -289,12 +301,14 @@ static uint32_t jz4780_dma_setup_hwdesc(struct jz4780_dma_chan *jzchan, desc->dcm |= width << JZ_DMA_DCM_SP_SHIFT; desc->dcm |= width << JZ_DMA_DCM_DP_SHIFT; - desc->dtc = len >> ord; + desc->dtc = len >> jzchan->transfer_shift; + return 0; } static struct dma_async_tx_descriptor *jz4780_dma_prep_slave_sg( struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, - enum dma_transfer_direction direction, unsigned long flags) + enum dma_transfer_direction direction, unsigned long flags, + void *context) { struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); struct jz4780_dma_desc *desc; @@ -307,12 +321,11 @@ static struct dma_async_tx_descriptor *jz4780_dma_prep_slave_sg( for (i = 0; i < sg_len; i++) { err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], - sg_dma_address(&sgl[i]), - sg_dma_len(&sgl[i]), - direction); + sg_dma_address(&sgl[i]), + sg_dma_len(&sgl[i]), + direction); if (err < 0) - return ERR_PTR(err); - + return NULL; desc->desc[i].dcm |= JZ_DMA_DCM_TIE; @@ -354,9 +367,9 @@ static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_cyclic( for (i = 0; i < periods; i++) { err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], buf_addr, - period_len, direction); + period_len, direction); if (err < 0) - return ERR_PTR(err); + return NULL; buf_addr += period_len; @@ -390,15 +403,13 @@ struct dma_async_tx_descriptor *jz4780_dma_prep_dma_memcpy( struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); struct jz4780_dma_desc *desc; uint32_t tsz; - int ord; desc = jz4780_dma_desc_alloc(jzchan, 1, DMA_MEMCPY); if (!desc) return NULL; - tsz = jz4780_dma_transfer_size(dest | src | len, &ord); - if (tsz < 0) - return ERR_PTR(tsz); + tsz = jz4780_dma_transfer_size(dest | src | len, + &jzchan->transfer_shift); desc->desc[0].dsa = src; desc->desc[0].dta = dest; @@ -407,7 +418,7 @@ struct dma_async_tx_descriptor *jz4780_dma_prep_dma_memcpy( tsz << JZ_DMA_DCM_TSZ_SHIFT | JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_SP_SHIFT | JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_DP_SHIFT; - desc->desc[0].dtc = len >> ord; + desc->desc[0].dtc = len >> jzchan->transfer_shift; return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); } @@ -484,8 +495,9 @@ static void jz4780_dma_issue_pending(struct dma_chan *chan) spin_unlock_irqrestore(&jzchan->vchan.lock, flags); } -static int jz4780_dma_terminate_all(struct jz4780_dma_chan *jzchan) +static int jz4780_dma_terminate_all(struct dma_chan *chan) { + struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); unsigned long flags; LIST_HEAD(head); @@ -507,9 +519,11 @@ static int jz4780_dma_terminate_all(struct jz4780_dma_chan *jzchan) return 0; } -static int jz4780_dma_slave_config(struct jz4780_dma_chan *jzchan, - const struct dma_slave_config *config) +static int jz4780_dma_config(struct dma_chan *chan, + struct dma_slave_config *config) { + struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); + if ((config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) || (config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)) return -EINVAL; @@ -567,8 +581,8 @@ static enum dma_status jz4780_dma_tx_status(struct dma_chan *chan, txstate->residue = 0; if (vdesc && jzchan->desc && vdesc == &jzchan->desc->vdesc - && jzchan->desc->status & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) - status = DMA_ERROR; + && jzchan->desc->status & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) + status = DMA_ERROR; spin_unlock_irqrestore(&jzchan->vchan.lock, flags); return status; @@ -671,7 +685,10 @@ static bool jz4780_dma_filter_fn(struct dma_chan *chan, void *param) { struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); - struct jz4780_dma_data *data = param; + struct jz4780_dma_filter_data *data = param; + + if (jzdma->dma_device.dev->of_node != data->of_node) + return false; if (data->channel > -1) { if (data->channel != jzchan->id) @@ -690,11 +707,12 @@ static struct dma_chan *jz4780_of_dma_xlate(struct of_phandle_args *dma_spec, { struct jz4780_dma_dev *jzdma = ofdma->of_dma_data; dma_cap_mask_t mask = jzdma->dma_device.cap_mask; - struct jz4780_dma_data data; + struct jz4780_dma_filter_data data; if (dma_spec->args_count != 2) return NULL; + data.of_node = ofdma->of_node; data.transfer_type = dma_spec->args[0]; data.channel = dma_spec->args[1]; @@ -713,9 +731,14 @@ static struct dma_chan *jz4780_of_dma_xlate(struct of_phandle_args *dma_spec, data.channel); return NULL; } - } - return dma_request_channel(mask, jz4780_dma_filter_fn, &data); + jzdma->chan[data.channel].transfer_type = data.transfer_type; + + return dma_get_slave_channel( + &jzdma->chan[data.channel].vchan.chan); + } else { + return dma_request_channel(mask, jz4780_dma_filter_fn, &data); + } } static int jz4780_dma_probe(struct platform_device *pdev) @@ -743,23 +766,26 @@ static int jz4780_dma_probe(struct platform_device *pdev) if (IS_ERR(jzdma->base)) return PTR_ERR(jzdma->base); - jzdma->irq = platform_get_irq(pdev, 0); - if (jzdma->irq < 0) { + ret = platform_get_irq(pdev, 0); + if (ret < 0) { dev_err(dev, "failed to get IRQ: %d\n", ret); - return jzdma->irq; + return ret; } - ret = devm_request_irq(dev, jzdma->irq, jz4780_dma_irq_handler, 0, - dev_name(dev), jzdma); + jzdma->irq = ret; + + ret = request_irq(jzdma->irq, jz4780_dma_irq_handler, 0, dev_name(dev), + jzdma); if (ret) { dev_err(dev, "failed to request IRQ %u!\n", jzdma->irq); - return -EINVAL; + return ret; } jzdma->clk = devm_clk_get(dev, NULL); if (IS_ERR(jzdma->clk)) { dev_err(dev, "failed to get clock\n"); - return PTR_ERR(jzdma->clk); + ret = PTR_ERR(jzdma->clk); + goto err_free_irq; } clk_prepare_enable(jzdma->clk); @@ -775,13 +801,13 @@ static int jz4780_dma_probe(struct platform_device *pdev) dma_cap_set(DMA_CYCLIC, dd->cap_mask); dd->dev = dev; - dd->copy_align = 2; /* 2^2 = 4 byte alignment */ + dd->copy_align = DMAENGINE_ALIGN_4_BYTES; dd->device_alloc_chan_resources = jz4780_dma_alloc_chan_resources; dd->device_free_chan_resources = jz4780_dma_free_chan_resources; dd->device_prep_slave_sg = jz4780_dma_prep_slave_sg; dd->device_prep_dma_cyclic = jz4780_dma_prep_dma_cyclic; dd->device_prep_dma_memcpy = jz4780_dma_prep_dma_memcpy; - dd->device_config = jz4780_dma_slave_config; + dd->device_config = jz4780_dma_config; dd->device_terminate_all = jz4780_dma_terminate_all; dd->device_tx_status = jz4780_dma_tx_status; dd->device_issue_pending = jz4780_dma_issue_pending; @@ -790,7 +816,6 @@ static int jz4780_dma_probe(struct platform_device *pdev) dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; - /* * Enable DMA controller, mark all channels as not programmable. * Also set the FMSC bit - it increases MSC performance, so it makes @@ -832,15 +857,24 @@ static int jz4780_dma_probe(struct platform_device *pdev) err_disable_clk: clk_disable_unprepare(jzdma->clk); + +err_free_irq: + free_irq(jzdma->irq, jzdma); return ret; } static int jz4780_dma_remove(struct platform_device *pdev) { struct jz4780_dma_dev *jzdma = platform_get_drvdata(pdev); + int i; of_dma_controller_free(pdev->dev.of_node); - devm_free_irq(&pdev->dev, jzdma->irq, jzdma); + + free_irq(jzdma->irq, jzdma); + + for (i = 0; i < JZ_DMA_NR_CHANNELS; i++) + tasklet_kill(&jzdma->chan[i].vchan.task); + dma_async_device_unregister(&jzdma->dma_device); return 0; } diff --git a/drivers/dma/dw/Kconfig b/drivers/dma/dw/Kconfig index 36e02f0f645e..e00c9b022964 100644 --- a/drivers/dma/dw/Kconfig +++ b/drivers/dma/dw/Kconfig @@ -6,6 +6,9 @@ config DW_DMAC_CORE tristate select DMA_ENGINE +config DW_DMAC_BIG_ENDIAN_IO + bool + config DW_DMAC tristate "Synopsys DesignWare AHB DMA platform driver" select DW_DMAC_CORE @@ -23,6 +26,3 @@ config DW_DMAC_PCI Support the Synopsys DesignWare AHB DMA controller on the platfroms that enumerate it as a PCI device. For example, Intel Medfield has integrated this GPDMA controller. - -config DW_DMAC_BIG_ENDIAN_IO - bool diff --git a/drivers/dma/edma.c b/drivers/dma/edma.c index 88853af69489..3e5d4f193005 100644 --- a/drivers/dma/edma.c +++ b/drivers/dma/edma.c @@ -1000,7 +1000,7 @@ static void edma_dma_init(struct edma_cc *ecc, struct dma_device *dma, * code using dma memcpy must make sure alignment of * length is at dma->copy_align boundary. */ - dma->copy_align = DMA_SLAVE_BUSWIDTH_4_BYTES; + dma->copy_align = DMAENGINE_ALIGN_4_BYTES; INIT_LIST_HEAD(&dma->channels); } diff --git a/drivers/dma/hsu/hsu.c b/drivers/dma/hsu/hsu.c index f42f71e37e73..7669c7dd1e34 100644 --- a/drivers/dma/hsu/hsu.c +++ b/drivers/dma/hsu/hsu.c @@ -99,21 +99,13 @@ static void hsu_dma_chan_start(struct hsu_dma_chan *hsuc) static void hsu_dma_stop_channel(struct hsu_dma_chan *hsuc) { - unsigned long flags; - - spin_lock_irqsave(&hsuc->lock, flags); hsu_chan_disable(hsuc); hsu_chan_writel(hsuc, HSU_CH_DCR, 0); - spin_unlock_irqrestore(&hsuc->lock, flags); } static void hsu_dma_start_channel(struct hsu_dma_chan *hsuc) { - unsigned long flags; - - spin_lock_irqsave(&hsuc->lock, flags); hsu_dma_chan_start(hsuc); - spin_unlock_irqrestore(&hsuc->lock, flags); } static void hsu_dma_start_transfer(struct hsu_dma_chan *hsuc) @@ -139,9 +131,9 @@ static u32 hsu_dma_chan_get_sr(struct hsu_dma_chan *hsuc) unsigned long flags; u32 sr; - spin_lock_irqsave(&hsuc->lock, flags); + spin_lock_irqsave(&hsuc->vchan.lock, flags); sr = hsu_chan_readl(hsuc, HSU_CH_SR); - spin_unlock_irqrestore(&hsuc->lock, flags); + spin_unlock_irqrestore(&hsuc->vchan.lock, flags); return sr; } @@ -273,14 +265,11 @@ static size_t hsu_dma_active_desc_size(struct hsu_dma_chan *hsuc) struct hsu_dma_desc *desc = hsuc->desc; size_t bytes = hsu_dma_desc_size(desc); int i; - unsigned long flags; - spin_lock_irqsave(&hsuc->lock, flags); i = desc->active % HSU_DMA_CHAN_NR_DESC; do { bytes += hsu_chan_readl(hsuc, HSU_CH_DxTSR(i)); } while (--i >= 0); - spin_unlock_irqrestore(&hsuc->lock, flags); return bytes; } @@ -327,24 +316,6 @@ static int hsu_dma_slave_config(struct dma_chan *chan, return 0; } -static void hsu_dma_chan_deactivate(struct hsu_dma_chan *hsuc) -{ - unsigned long flags; - - spin_lock_irqsave(&hsuc->lock, flags); - hsu_chan_disable(hsuc); - spin_unlock_irqrestore(&hsuc->lock, flags); -} - -static void hsu_dma_chan_activate(struct hsu_dma_chan *hsuc) -{ - unsigned long flags; - - spin_lock_irqsave(&hsuc->lock, flags); - hsu_chan_enable(hsuc); - spin_unlock_irqrestore(&hsuc->lock, flags); -} - static int hsu_dma_pause(struct dma_chan *chan) { struct hsu_dma_chan *hsuc = to_hsu_dma_chan(chan); @@ -352,7 +323,7 @@ static int hsu_dma_pause(struct dma_chan *chan) spin_lock_irqsave(&hsuc->vchan.lock, flags); if (hsuc->desc && hsuc->desc->status == DMA_IN_PROGRESS) { - hsu_dma_chan_deactivate(hsuc); + hsu_chan_disable(hsuc); hsuc->desc->status = DMA_PAUSED; } spin_unlock_irqrestore(&hsuc->vchan.lock, flags); @@ -368,7 +339,7 @@ static int hsu_dma_resume(struct dma_chan *chan) spin_lock_irqsave(&hsuc->vchan.lock, flags); if (hsuc->desc && hsuc->desc->status == DMA_PAUSED) { hsuc->desc->status = DMA_IN_PROGRESS; - hsu_dma_chan_activate(hsuc); + hsu_chan_enable(hsuc); } spin_unlock_irqrestore(&hsuc->vchan.lock, flags); @@ -441,8 +412,6 @@ int hsu_dma_probe(struct hsu_dma_chip *chip) hsuc->direction = (i & 0x1) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV; hsuc->reg = addr + i * HSU_DMA_CHAN_LENGTH; - - spin_lock_init(&hsuc->lock); } dma_cap_set(DMA_SLAVE, hsu->dma.cap_mask); diff --git a/drivers/dma/hsu/hsu.h b/drivers/dma/hsu/hsu.h index 0275233cf550..eeb9fff66967 100644 --- a/drivers/dma/hsu/hsu.h +++ b/drivers/dma/hsu/hsu.h @@ -78,7 +78,6 @@ struct hsu_dma_chan { struct virt_dma_chan vchan; void __iomem *reg; - spinlock_t lock; /* hardware configuration */ enum dma_transfer_direction direction; diff --git a/drivers/dma/idma64.c b/drivers/dma/idma64.c new file mode 100644 index 000000000000..18c14e1f1414 --- /dev/null +++ b/drivers/dma/idma64.c @@ -0,0 +1,710 @@ +/* + * Core driver for the Intel integrated DMA 64-bit + * + * Copyright (C) 2015 Intel Corporation + * Author: Andy Shevchenko + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "idma64.h" + +/* Platform driver name */ +#define DRV_NAME "idma64" + +/* For now we support only two channels */ +#define IDMA64_NR_CHAN 2 + +/* ---------------------------------------------------------------------- */ + +static struct device *chan2dev(struct dma_chan *chan) +{ + return &chan->dev->device; +} + +/* ---------------------------------------------------------------------- */ + +static void idma64_off(struct idma64 *idma64) +{ + unsigned short count = 100; + + dma_writel(idma64, CFG, 0); + + channel_clear_bit(idma64, MASK(XFER), idma64->all_chan_mask); + channel_clear_bit(idma64, MASK(BLOCK), idma64->all_chan_mask); + channel_clear_bit(idma64, MASK(SRC_TRAN), idma64->all_chan_mask); + channel_clear_bit(idma64, MASK(DST_TRAN), idma64->all_chan_mask); + channel_clear_bit(idma64, MASK(ERROR), idma64->all_chan_mask); + + do { + cpu_relax(); + } while (dma_readl(idma64, CFG) & IDMA64_CFG_DMA_EN && --count); +} + +static void idma64_on(struct idma64 *idma64) +{ + dma_writel(idma64, CFG, IDMA64_CFG_DMA_EN); +} + +/* ---------------------------------------------------------------------- */ + +static void idma64_chan_init(struct idma64 *idma64, struct idma64_chan *idma64c) +{ + u32 cfghi = IDMA64C_CFGH_SRC_PER(1) | IDMA64C_CFGH_DST_PER(0); + u32 cfglo = 0; + + /* Enforce FIFO drain when channel is suspended */ + cfglo |= IDMA64C_CFGL_CH_DRAIN; + + /* Set default burst alignment */ + cfglo |= IDMA64C_CFGL_DST_BURST_ALIGN | IDMA64C_CFGL_SRC_BURST_ALIGN; + + channel_writel(idma64c, CFG_LO, cfglo); + channel_writel(idma64c, CFG_HI, cfghi); + + /* Enable interrupts */ + channel_set_bit(idma64, MASK(XFER), idma64c->mask); + channel_set_bit(idma64, MASK(ERROR), idma64c->mask); + + /* + * Enforce the controller to be turned on. + * + * The iDMA is turned off in ->probe() and looses context during system + * suspend / resume cycle. That's why we have to enable it each time we + * use it. + */ + idma64_on(idma64); +} + +static void idma64_chan_stop(struct idma64 *idma64, struct idma64_chan *idma64c) +{ + channel_clear_bit(idma64, CH_EN, idma64c->mask); +} + +static void idma64_chan_start(struct idma64 *idma64, struct idma64_chan *idma64c) +{ + struct idma64_desc *desc = idma64c->desc; + struct idma64_hw_desc *hw = &desc->hw[0]; + + channel_writeq(idma64c, SAR, 0); + channel_writeq(idma64c, DAR, 0); + + channel_writel(idma64c, CTL_HI, IDMA64C_CTLH_BLOCK_TS(~0UL)); + channel_writel(idma64c, CTL_LO, IDMA64C_CTLL_LLP_S_EN | IDMA64C_CTLL_LLP_D_EN); + + channel_writeq(idma64c, LLP, hw->llp); + + channel_set_bit(idma64, CH_EN, idma64c->mask); +} + +static void idma64_stop_transfer(struct idma64_chan *idma64c) +{ + struct idma64 *idma64 = to_idma64(idma64c->vchan.chan.device); + + idma64_chan_stop(idma64, idma64c); +} + +static void idma64_start_transfer(struct idma64_chan *idma64c) +{ + struct idma64 *idma64 = to_idma64(idma64c->vchan.chan.device); + struct virt_dma_desc *vdesc; + + /* Get the next descriptor */ + vdesc = vchan_next_desc(&idma64c->vchan); + if (!vdesc) { + idma64c->desc = NULL; + return; + } + + list_del(&vdesc->node); + idma64c->desc = to_idma64_desc(vdesc); + + /* Configure the channel */ + idma64_chan_init(idma64, idma64c); + + /* Start the channel with a new descriptor */ + idma64_chan_start(idma64, idma64c); +} + +/* ---------------------------------------------------------------------- */ + +static void idma64_chan_irq(struct idma64 *idma64, unsigned short c, + u32 status_err, u32 status_xfer) +{ + struct idma64_chan *idma64c = &idma64->chan[c]; + struct idma64_desc *desc; + unsigned long flags; + + spin_lock_irqsave(&idma64c->vchan.lock, flags); + desc = idma64c->desc; + if (desc) { + if (status_err & (1 << c)) { + dma_writel(idma64, CLEAR(ERROR), idma64c->mask); + desc->status = DMA_ERROR; + } else if (status_xfer & (1 << c)) { + dma_writel(idma64, CLEAR(XFER), idma64c->mask); + desc->status = DMA_COMPLETE; + vchan_cookie_complete(&desc->vdesc); + idma64_start_transfer(idma64c); + } + + /* idma64_start_transfer() updates idma64c->desc */ + if (idma64c->desc == NULL || desc->status == DMA_ERROR) + idma64_stop_transfer(idma64c); + } + spin_unlock_irqrestore(&idma64c->vchan.lock, flags); +} + +static irqreturn_t idma64_irq(int irq, void *dev) +{ + struct idma64 *idma64 = dev; + u32 status = dma_readl(idma64, STATUS_INT); + u32 status_xfer; + u32 status_err; + unsigned short i; + + dev_vdbg(idma64->dma.dev, "%s: status=%#x\n", __func__, status); + + /* Check if we have any interrupt from the DMA controller */ + if (!status) + return IRQ_NONE; + + /* Disable interrupts */ + channel_clear_bit(idma64, MASK(XFER), idma64->all_chan_mask); + channel_clear_bit(idma64, MASK(ERROR), idma64->all_chan_mask); + + status_xfer = dma_readl(idma64, RAW(XFER)); + status_err = dma_readl(idma64, RAW(ERROR)); + + for (i = 0; i < idma64->dma.chancnt; i++) + idma64_chan_irq(idma64, i, status_err, status_xfer); + + /* Re-enable interrupts */ + channel_set_bit(idma64, MASK(XFER), idma64->all_chan_mask); + channel_set_bit(idma64, MASK(ERROR), idma64->all_chan_mask); + + return IRQ_HANDLED; +} + +/* ---------------------------------------------------------------------- */ + +static struct idma64_desc *idma64_alloc_desc(unsigned int ndesc) +{ + struct idma64_desc *desc; + + desc = kzalloc(sizeof(*desc), GFP_NOWAIT); + if (!desc) + return NULL; + + desc->hw = kcalloc(ndesc, sizeof(*desc->hw), GFP_NOWAIT); + if (!desc->hw) { + kfree(desc); + return NULL; + } + + return desc; +} + +static void idma64_desc_free(struct idma64_chan *idma64c, + struct idma64_desc *desc) +{ + struct idma64_hw_desc *hw; + + if (desc->ndesc) { + unsigned int i = desc->ndesc; + + do { + hw = &desc->hw[--i]; + dma_pool_free(idma64c->pool, hw->lli, hw->llp); + } while (i); + } + + kfree(desc->hw); + kfree(desc); +} + +static void idma64_vdesc_free(struct virt_dma_desc *vdesc) +{ + struct idma64_chan *idma64c = to_idma64_chan(vdesc->tx.chan); + + idma64_desc_free(idma64c, to_idma64_desc(vdesc)); +} + +static u64 idma64_hw_desc_fill(struct idma64_hw_desc *hw, + struct dma_slave_config *config, + enum dma_transfer_direction direction, u64 llp) +{ + struct idma64_lli *lli = hw->lli; + u64 sar, dar; + u32 ctlhi = IDMA64C_CTLH_BLOCK_TS(hw->len); + u32 ctllo = IDMA64C_CTLL_LLP_S_EN | IDMA64C_CTLL_LLP_D_EN; + u32 src_width, dst_width; + + if (direction == DMA_MEM_TO_DEV) { + sar = hw->phys; + dar = config->dst_addr; + ctllo |= IDMA64C_CTLL_DST_FIX | IDMA64C_CTLL_SRC_INC | + IDMA64C_CTLL_FC_M2P; + src_width = min_t(u32, 2, __fls(sar | hw->len)); + dst_width = __fls(config->dst_addr_width); + } else { /* DMA_DEV_TO_MEM */ + sar = config->src_addr; + dar = hw->phys; + ctllo |= IDMA64C_CTLL_DST_INC | IDMA64C_CTLL_SRC_FIX | + IDMA64C_CTLL_FC_P2M; + src_width = __fls(config->src_addr_width); + dst_width = min_t(u32, 2, __fls(dar | hw->len)); + } + + lli->sar = sar; + lli->dar = dar; + + lli->ctlhi = ctlhi; + lli->ctllo = ctllo | + IDMA64C_CTLL_SRC_MSIZE(config->src_maxburst) | + IDMA64C_CTLL_DST_MSIZE(config->dst_maxburst) | + IDMA64C_CTLL_DST_WIDTH(dst_width) | + IDMA64C_CTLL_SRC_WIDTH(src_width); + + lli->llp = llp; + return hw->llp; +} + +static void idma64_desc_fill(struct idma64_chan *idma64c, + struct idma64_desc *desc) +{ + struct dma_slave_config *config = &idma64c->config; + struct idma64_hw_desc *hw = &desc->hw[desc->ndesc - 1]; + struct idma64_lli *lli = hw->lli; + u64 llp = 0; + unsigned int i = desc->ndesc; + + /* Fill the hardware descriptors and link them to a list */ + do { + hw = &desc->hw[--i]; + llp = idma64_hw_desc_fill(hw, config, desc->direction, llp); + desc->length += hw->len; + } while (i); + + /* Trigger interrupt after last block */ + lli->ctllo |= IDMA64C_CTLL_INT_EN; +} + +static struct dma_async_tx_descriptor *idma64_prep_slave_sg( + struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct idma64_chan *idma64c = to_idma64_chan(chan); + struct idma64_desc *desc; + struct scatterlist *sg; + unsigned int i; + + desc = idma64_alloc_desc(sg_len); + if (!desc) + return NULL; + + for_each_sg(sgl, sg, sg_len, i) { + struct idma64_hw_desc *hw = &desc->hw[i]; + + /* Allocate DMA capable memory for hardware descriptor */ + hw->lli = dma_pool_alloc(idma64c->pool, GFP_NOWAIT, &hw->llp); + if (!hw->lli) { + desc->ndesc = i; + idma64_desc_free(idma64c, desc); + return NULL; + } + + hw->phys = sg_dma_address(sg); + hw->len = sg_dma_len(sg); + } + + desc->ndesc = sg_len; + desc->direction = direction; + desc->status = DMA_IN_PROGRESS; + + idma64_desc_fill(idma64c, desc); + return vchan_tx_prep(&idma64c->vchan, &desc->vdesc, flags); +} + +static void idma64_issue_pending(struct dma_chan *chan) +{ + struct idma64_chan *idma64c = to_idma64_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&idma64c->vchan.lock, flags); + if (vchan_issue_pending(&idma64c->vchan) && !idma64c->desc) + idma64_start_transfer(idma64c); + spin_unlock_irqrestore(&idma64c->vchan.lock, flags); +} + +static size_t idma64_active_desc_size(struct idma64_chan *idma64c) +{ + struct idma64_desc *desc = idma64c->desc; + struct idma64_hw_desc *hw; + size_t bytes = desc->length; + u64 llp; + u32 ctlhi; + unsigned int i = 0; + + llp = channel_readq(idma64c, LLP); + do { + hw = &desc->hw[i]; + } while ((hw->llp != llp) && (++i < desc->ndesc)); + + if (!i) + return bytes; + + do { + bytes -= desc->hw[--i].len; + } while (i); + + ctlhi = channel_readl(idma64c, CTL_HI); + return bytes - IDMA64C_CTLH_BLOCK_TS(ctlhi); +} + +static enum dma_status idma64_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, struct dma_tx_state *state) +{ + struct idma64_chan *idma64c = to_idma64_chan(chan); + struct virt_dma_desc *vdesc; + enum dma_status status; + size_t bytes; + unsigned long flags; + + status = dma_cookie_status(chan, cookie, state); + if (status == DMA_COMPLETE) + return status; + + spin_lock_irqsave(&idma64c->vchan.lock, flags); + vdesc = vchan_find_desc(&idma64c->vchan, cookie); + if (idma64c->desc && cookie == idma64c->desc->vdesc.tx.cookie) { + bytes = idma64_active_desc_size(idma64c); + dma_set_residue(state, bytes); + status = idma64c->desc->status; + } else if (vdesc) { + bytes = to_idma64_desc(vdesc)->length; + dma_set_residue(state, bytes); + } + spin_unlock_irqrestore(&idma64c->vchan.lock, flags); + + return status; +} + +static void convert_burst(u32 *maxburst) +{ + if (*maxburst) + *maxburst = __fls(*maxburst); + else + *maxburst = 0; +} + +static int idma64_slave_config(struct dma_chan *chan, + struct dma_slave_config *config) +{ + struct idma64_chan *idma64c = to_idma64_chan(chan); + + /* Check if chan will be configured for slave transfers */ + if (!is_slave_direction(config->direction)) + return -EINVAL; + + memcpy(&idma64c->config, config, sizeof(idma64c->config)); + + convert_burst(&idma64c->config.src_maxburst); + convert_burst(&idma64c->config.dst_maxburst); + + return 0; +} + +static void idma64_chan_deactivate(struct idma64_chan *idma64c) +{ + unsigned short count = 100; + u32 cfglo; + + cfglo = channel_readl(idma64c, CFG_LO); + channel_writel(idma64c, CFG_LO, cfglo | IDMA64C_CFGL_CH_SUSP); + do { + udelay(1); + cfglo = channel_readl(idma64c, CFG_LO); + } while (!(cfglo & IDMA64C_CFGL_FIFO_EMPTY) && --count); +} + +static void idma64_chan_activate(struct idma64_chan *idma64c) +{ + u32 cfglo; + + cfglo = channel_readl(idma64c, CFG_LO); + channel_writel(idma64c, CFG_LO, cfglo & ~IDMA64C_CFGL_CH_SUSP); +} + +static int idma64_pause(struct dma_chan *chan) +{ + struct idma64_chan *idma64c = to_idma64_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&idma64c->vchan.lock, flags); + if (idma64c->desc && idma64c->desc->status == DMA_IN_PROGRESS) { + idma64_chan_deactivate(idma64c); + idma64c->desc->status = DMA_PAUSED; + } + spin_unlock_irqrestore(&idma64c->vchan.lock, flags); + + return 0; +} + +static int idma64_resume(struct dma_chan *chan) +{ + struct idma64_chan *idma64c = to_idma64_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&idma64c->vchan.lock, flags); + if (idma64c->desc && idma64c->desc->status == DMA_PAUSED) { + idma64c->desc->status = DMA_IN_PROGRESS; + idma64_chan_activate(idma64c); + } + spin_unlock_irqrestore(&idma64c->vchan.lock, flags); + + return 0; +} + +static int idma64_terminate_all(struct dma_chan *chan) +{ + struct idma64_chan *idma64c = to_idma64_chan(chan); + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&idma64c->vchan.lock, flags); + idma64_chan_deactivate(idma64c); + idma64_stop_transfer(idma64c); + if (idma64c->desc) { + idma64_vdesc_free(&idma64c->desc->vdesc); + idma64c->desc = NULL; + } + vchan_get_all_descriptors(&idma64c->vchan, &head); + spin_unlock_irqrestore(&idma64c->vchan.lock, flags); + + vchan_dma_desc_free_list(&idma64c->vchan, &head); + return 0; +} + +static int idma64_alloc_chan_resources(struct dma_chan *chan) +{ + struct idma64_chan *idma64c = to_idma64_chan(chan); + + /* Create a pool of consistent memory blocks for hardware descriptors */ + idma64c->pool = dma_pool_create(dev_name(chan2dev(chan)), + chan->device->dev, + sizeof(struct idma64_lli), 8, 0); + if (!idma64c->pool) { + dev_err(chan2dev(chan), "No memory for descriptors\n"); + return -ENOMEM; + } + + return 0; +} + +static void idma64_free_chan_resources(struct dma_chan *chan) +{ + struct idma64_chan *idma64c = to_idma64_chan(chan); + + vchan_free_chan_resources(to_virt_chan(chan)); + dma_pool_destroy(idma64c->pool); + idma64c->pool = NULL; +} + +/* ---------------------------------------------------------------------- */ + +#define IDMA64_BUSWIDTHS \ + BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) + +static int idma64_probe(struct idma64_chip *chip) +{ + struct idma64 *idma64; + unsigned short nr_chan = IDMA64_NR_CHAN; + unsigned short i; + int ret; + + idma64 = devm_kzalloc(chip->dev, sizeof(*idma64), GFP_KERNEL); + if (!idma64) + return -ENOMEM; + + idma64->regs = chip->regs; + chip->idma64 = idma64; + + idma64->chan = devm_kcalloc(chip->dev, nr_chan, sizeof(*idma64->chan), + GFP_KERNEL); + if (!idma64->chan) + return -ENOMEM; + + idma64->all_chan_mask = (1 << nr_chan) - 1; + + /* Turn off iDMA controller */ + idma64_off(idma64); + + ret = devm_request_irq(chip->dev, chip->irq, idma64_irq, IRQF_SHARED, + dev_name(chip->dev), idma64); + if (ret) + return ret; + + INIT_LIST_HEAD(&idma64->dma.channels); + for (i = 0; i < nr_chan; i++) { + struct idma64_chan *idma64c = &idma64->chan[i]; + + idma64c->vchan.desc_free = idma64_vdesc_free; + vchan_init(&idma64c->vchan, &idma64->dma); + + idma64c->regs = idma64->regs + i * IDMA64_CH_LENGTH; + idma64c->mask = BIT(i); + } + + dma_cap_set(DMA_SLAVE, idma64->dma.cap_mask); + dma_cap_set(DMA_PRIVATE, idma64->dma.cap_mask); + + idma64->dma.device_alloc_chan_resources = idma64_alloc_chan_resources; + idma64->dma.device_free_chan_resources = idma64_free_chan_resources; + + idma64->dma.device_prep_slave_sg = idma64_prep_slave_sg; + + idma64->dma.device_issue_pending = idma64_issue_pending; + idma64->dma.device_tx_status = idma64_tx_status; + + idma64->dma.device_config = idma64_slave_config; + idma64->dma.device_pause = idma64_pause; + idma64->dma.device_resume = idma64_resume; + idma64->dma.device_terminate_all = idma64_terminate_all; + + idma64->dma.src_addr_widths = IDMA64_BUSWIDTHS; + idma64->dma.dst_addr_widths = IDMA64_BUSWIDTHS; + idma64->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + idma64->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + + idma64->dma.dev = chip->dev; + + ret = dma_async_device_register(&idma64->dma); + if (ret) + return ret; + + dev_info(chip->dev, "Found Intel integrated DMA 64-bit\n"); + return 0; +} + +static int idma64_remove(struct idma64_chip *chip) +{ + struct idma64 *idma64 = chip->idma64; + unsigned short i; + + dma_async_device_unregister(&idma64->dma); + + /* + * Explicitly call devm_request_irq() to avoid the side effects with + * the scheduled tasklets. + */ + devm_free_irq(chip->dev, chip->irq, idma64); + + for (i = 0; i < idma64->dma.chancnt; i++) { + struct idma64_chan *idma64c = &idma64->chan[i]; + + tasklet_kill(&idma64c->vchan.task); + } + + return 0; +} + +/* ---------------------------------------------------------------------- */ + +static int idma64_platform_probe(struct platform_device *pdev) +{ + struct idma64_chip *chip; + struct device *dev = &pdev->dev; + struct resource *mem; + int ret; + + chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL); + if (!chip) + return -ENOMEM; + + chip->irq = platform_get_irq(pdev, 0); + if (chip->irq < 0) + return chip->irq; + + mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + chip->regs = devm_ioremap_resource(dev, mem); + if (IS_ERR(chip->regs)) + return PTR_ERR(chip->regs); + + ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (ret) + return ret; + + chip->dev = dev; + + ret = idma64_probe(chip); + if (ret) + return ret; + + platform_set_drvdata(pdev, chip); + return 0; +} + +static int idma64_platform_remove(struct platform_device *pdev) +{ + struct idma64_chip *chip = platform_get_drvdata(pdev); + + return idma64_remove(chip); +} + +#ifdef CONFIG_PM_SLEEP + +static int idma64_pm_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct idma64_chip *chip = platform_get_drvdata(pdev); + + idma64_off(chip->idma64); + return 0; +} + +static int idma64_pm_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct idma64_chip *chip = platform_get_drvdata(pdev); + + idma64_on(chip->idma64); + return 0; +} + +#endif /* CONFIG_PM_SLEEP */ + +static const struct dev_pm_ops idma64_dev_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(idma64_pm_suspend, idma64_pm_resume) +}; + +static struct platform_driver idma64_platform_driver = { + .probe = idma64_platform_probe, + .remove = idma64_platform_remove, + .driver = { + .name = DRV_NAME, + .pm = &idma64_dev_pm_ops, + }, +}; + +module_platform_driver(idma64_platform_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("iDMA64 core driver"); +MODULE_AUTHOR("Andy Shevchenko "); +MODULE_ALIAS("platform:" DRV_NAME); diff --git a/drivers/dma/idma64.h b/drivers/dma/idma64.h new file mode 100644 index 000000000000..a4d99685a7c4 --- /dev/null +++ b/drivers/dma/idma64.h @@ -0,0 +1,233 @@ +/* + * Driver for the Intel integrated DMA 64-bit + * + * Copyright (C) 2015 Intel Corporation + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __DMA_IDMA64_H__ +#define __DMA_IDMA64_H__ + +#include +#include +#include +#include + +#include "virt-dma.h" + +/* Channel registers */ + +#define IDMA64_CH_SAR 0x00 /* Source Address Register */ +#define IDMA64_CH_DAR 0x08 /* Destination Address Register */ +#define IDMA64_CH_LLP 0x10 /* Linked List Pointer */ +#define IDMA64_CH_CTL_LO 0x18 /* Control Register Low */ +#define IDMA64_CH_CTL_HI 0x1c /* Control Register High */ +#define IDMA64_CH_SSTAT 0x20 +#define IDMA64_CH_DSTAT 0x28 +#define IDMA64_CH_SSTATAR 0x30 +#define IDMA64_CH_DSTATAR 0x38 +#define IDMA64_CH_CFG_LO 0x40 /* Configuration Register Low */ +#define IDMA64_CH_CFG_HI 0x44 /* Configuration Register High */ +#define IDMA64_CH_SGR 0x48 +#define IDMA64_CH_DSR 0x50 + +#define IDMA64_CH_LENGTH 0x58 + +/* Bitfields in CTL_LO */ +#define IDMA64C_CTLL_INT_EN (1 << 0) /* irqs enabled? */ +#define IDMA64C_CTLL_DST_WIDTH(x) ((x) << 1) /* bytes per element */ +#define IDMA64C_CTLL_SRC_WIDTH(x) ((x) << 4) +#define IDMA64C_CTLL_DST_INC (0 << 8) /* DAR update/not */ +#define IDMA64C_CTLL_DST_FIX (1 << 8) +#define IDMA64C_CTLL_SRC_INC (0 << 10) /* SAR update/not */ +#define IDMA64C_CTLL_SRC_FIX (1 << 10) +#define IDMA64C_CTLL_DST_MSIZE(x) ((x) << 11) /* burst, #elements */ +#define IDMA64C_CTLL_SRC_MSIZE(x) ((x) << 14) +#define IDMA64C_CTLL_FC_M2P (1 << 20) /* mem-to-periph */ +#define IDMA64C_CTLL_FC_P2M (2 << 20) /* periph-to-mem */ +#define IDMA64C_CTLL_LLP_D_EN (1 << 27) /* dest block chain */ +#define IDMA64C_CTLL_LLP_S_EN (1 << 28) /* src block chain */ + +/* Bitfields in CTL_HI */ +#define IDMA64C_CTLH_BLOCK_TS(x) ((x) & ((1 << 17) - 1)) +#define IDMA64C_CTLH_DONE (1 << 17) + +/* Bitfields in CFG_LO */ +#define IDMA64C_CFGL_DST_BURST_ALIGN (1 << 0) /* dst burst align */ +#define IDMA64C_CFGL_SRC_BURST_ALIGN (1 << 1) /* src burst align */ +#define IDMA64C_CFGL_CH_SUSP (1 << 8) +#define IDMA64C_CFGL_FIFO_EMPTY (1 << 9) +#define IDMA64C_CFGL_CH_DRAIN (1 << 10) /* drain FIFO */ +#define IDMA64C_CFGL_DST_OPT_BL (1 << 20) /* optimize dst burst length */ +#define IDMA64C_CFGL_SRC_OPT_BL (1 << 21) /* optimize src burst length */ + +/* Bitfields in CFG_HI */ +#define IDMA64C_CFGH_SRC_PER(x) ((x) << 0) /* src peripheral */ +#define IDMA64C_CFGH_DST_PER(x) ((x) << 4) /* dst peripheral */ +#define IDMA64C_CFGH_RD_ISSUE_THD(x) ((x) << 8) +#define IDMA64C_CFGH_RW_ISSUE_THD(x) ((x) << 18) + +/* Interrupt registers */ + +#define IDMA64_INT_XFER 0x00 +#define IDMA64_INT_BLOCK 0x08 +#define IDMA64_INT_SRC_TRAN 0x10 +#define IDMA64_INT_DST_TRAN 0x18 +#define IDMA64_INT_ERROR 0x20 + +#define IDMA64_RAW(x) (0x2c0 + IDMA64_INT_##x) /* r */ +#define IDMA64_STATUS(x) (0x2e8 + IDMA64_INT_##x) /* r (raw & mask) */ +#define IDMA64_MASK(x) (0x310 + IDMA64_INT_##x) /* rw (set = irq enabled) */ +#define IDMA64_CLEAR(x) (0x338 + IDMA64_INT_##x) /* w (ack, affects "raw") */ + +/* Common registers */ + +#define IDMA64_STATUS_INT 0x360 /* r */ +#define IDMA64_CFG 0x398 +#define IDMA64_CH_EN 0x3a0 + +/* Bitfields in CFG */ +#define IDMA64_CFG_DMA_EN (1 << 0) + +/* Hardware descriptor for Linked LIst transfers */ +struct idma64_lli { + u64 sar; + u64 dar; + u64 llp; + u32 ctllo; + u32 ctlhi; + u32 sstat; + u32 dstat; +}; + +struct idma64_hw_desc { + struct idma64_lli *lli; + dma_addr_t llp; + dma_addr_t phys; + unsigned int len; +}; + +struct idma64_desc { + struct virt_dma_desc vdesc; + enum dma_transfer_direction direction; + struct idma64_hw_desc *hw; + unsigned int ndesc; + size_t length; + enum dma_status status; +}; + +static inline struct idma64_desc *to_idma64_desc(struct virt_dma_desc *vdesc) +{ + return container_of(vdesc, struct idma64_desc, vdesc); +} + +struct idma64_chan { + struct virt_dma_chan vchan; + + void __iomem *regs; + + /* hardware configuration */ + enum dma_transfer_direction direction; + unsigned int mask; + struct dma_slave_config config; + + void *pool; + struct idma64_desc *desc; +}; + +static inline struct idma64_chan *to_idma64_chan(struct dma_chan *chan) +{ + return container_of(chan, struct idma64_chan, vchan.chan); +} + +#define channel_set_bit(idma64, reg, mask) \ + dma_writel(idma64, reg, ((mask) << 8) | (mask)) +#define channel_clear_bit(idma64, reg, mask) \ + dma_writel(idma64, reg, ((mask) << 8) | 0) + +static inline u32 idma64c_readl(struct idma64_chan *idma64c, int offset) +{ + return readl(idma64c->regs + offset); +} + +static inline void idma64c_writel(struct idma64_chan *idma64c, int offset, + u32 value) +{ + writel(value, idma64c->regs + offset); +} + +#define channel_readl(idma64c, reg) \ + idma64c_readl(idma64c, IDMA64_CH_##reg) +#define channel_writel(idma64c, reg, value) \ + idma64c_writel(idma64c, IDMA64_CH_##reg, (value)) + +static inline u64 idma64c_readq(struct idma64_chan *idma64c, int offset) +{ + u64 l, h; + + l = idma64c_readl(idma64c, offset); + h = idma64c_readl(idma64c, offset + 4); + + return l | (h << 32); +} + +static inline void idma64c_writeq(struct idma64_chan *idma64c, int offset, + u64 value) +{ + idma64c_writel(idma64c, offset, value); + idma64c_writel(idma64c, offset + 4, value >> 32); +} + +#define channel_readq(idma64c, reg) \ + idma64c_readq(idma64c, IDMA64_CH_##reg) +#define channel_writeq(idma64c, reg, value) \ + idma64c_writeq(idma64c, IDMA64_CH_##reg, (value)) + +struct idma64 { + struct dma_device dma; + + void __iomem *regs; + + /* channels */ + unsigned short all_chan_mask; + struct idma64_chan *chan; +}; + +static inline struct idma64 *to_idma64(struct dma_device *ddev) +{ + return container_of(ddev, struct idma64, dma); +} + +static inline u32 idma64_readl(struct idma64 *idma64, int offset) +{ + return readl(idma64->regs + offset); +} + +static inline void idma64_writel(struct idma64 *idma64, int offset, u32 value) +{ + writel(value, idma64->regs + offset); +} + +#define dma_readl(idma64, reg) \ + idma64_readl(idma64, IDMA64_##reg) +#define dma_writel(idma64, reg, value) \ + idma64_writel(idma64, IDMA64_##reg, (value)) + +/** + * struct idma64_chip - representation of DesignWare DMA controller hardware + * @dev: struct device of the DMA controller + * @irq: irq line + * @regs: memory mapped I/O space + * @idma64: struct idma64 that is filed by idma64_probe() + */ +struct idma64_chip { + struct device *dev; + int irq; + void __iomem *regs; + struct idma64 *idma64; +}; + +#endif /* __DMA_IDMA64_H__ */ diff --git a/drivers/dma/imx-dma.c b/drivers/dma/imx-dma.c index 865501fcc67d..48d85f8b95fe 100644 --- a/drivers/dma/imx-dma.c +++ b/drivers/dma/imx-dma.c @@ -1083,8 +1083,12 @@ static int __init imxdma_probe(struct platform_device *pdev) if (IS_ERR(imxdma->dma_ahb)) return PTR_ERR(imxdma->dma_ahb); - clk_prepare_enable(imxdma->dma_ipg); - clk_prepare_enable(imxdma->dma_ahb); + ret = clk_prepare_enable(imxdma->dma_ipg); + if (ret) + return ret; + ret = clk_prepare_enable(imxdma->dma_ahb); + if (ret) + goto disable_dma_ipg_clk; /* reset DMA module */ imx_dmav1_writel(imxdma, DCR_DRST, DMA_DCR); @@ -1094,20 +1098,20 @@ static int __init imxdma_probe(struct platform_device *pdev) dma_irq_handler, 0, "DMA", imxdma); if (ret) { dev_warn(imxdma->dev, "Can't register IRQ for DMA\n"); - goto err; + goto disable_dma_ahb_clk; } irq_err = platform_get_irq(pdev, 1); if (irq_err < 0) { ret = irq_err; - goto err; + goto disable_dma_ahb_clk; } ret = devm_request_irq(&pdev->dev, irq_err, imxdma_err_handler, 0, "DMA", imxdma); if (ret) { dev_warn(imxdma->dev, "Can't register ERRIRQ for DMA\n"); - goto err; + goto disable_dma_ahb_clk; } } @@ -1144,7 +1148,7 @@ static int __init imxdma_probe(struct platform_device *pdev) dev_warn(imxdma->dev, "Can't register IRQ %d " "for DMA channel %d\n", irq + i, i); - goto err; + goto disable_dma_ahb_clk; } init_timer(&imxdmac->watchdog); imxdmac->watchdog.function = &imxdma_watchdog; @@ -1183,14 +1187,14 @@ static int __init imxdma_probe(struct platform_device *pdev) platform_set_drvdata(pdev, imxdma); - imxdma->dma_device.copy_align = 2; /* 2^2 = 4 bytes alignment */ + imxdma->dma_device.copy_align = DMAENGINE_ALIGN_4_BYTES; imxdma->dma_device.dev->dma_parms = &imxdma->dma_parms; dma_set_max_seg_size(imxdma->dma_device.dev, 0xffffff); ret = dma_async_device_register(&imxdma->dma_device); if (ret) { dev_err(&pdev->dev, "unable to register\n"); - goto err; + goto disable_dma_ahb_clk; } if (pdev->dev.of_node) { @@ -1206,9 +1210,10 @@ static int __init imxdma_probe(struct platform_device *pdev) err_of_dma_controller: dma_async_device_unregister(&imxdma->dma_device); -err: - clk_disable_unprepare(imxdma->dma_ipg); +disable_dma_ahb_clk: clk_disable_unprepare(imxdma->dma_ahb); +disable_dma_ipg_clk: + clk_disable_unprepare(imxdma->dma_ipg); return ret; } diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 77b6aab04f47..9d375bc7590a 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -35,12 +35,16 @@ #include #include #include +#include #include #include #include #include #include +#include +#include +#include #include "dmaengine.h" @@ -123,6 +127,56 @@ */ #define CHANGE_ENDIANNESS 0x80 +/* + * p_2_p watermark_level description + * Bits Name Description + * 0-7 Lower WML Lower watermark level + * 8 PS 1: Pad Swallowing + * 0: No Pad Swallowing + * 9 PA 1: Pad Adding + * 0: No Pad Adding + * 10 SPDIF If this bit is set both source + * and destination are on SPBA + * 11 Source Bit(SP) 1: Source on SPBA + * 0: Source on AIPS + * 12 Destination Bit(DP) 1: Destination on SPBA + * 0: Destination on AIPS + * 13-15 --------- MUST BE 0 + * 16-23 Higher WML HWML + * 24-27 N Total number of samples after + * which Pad adding/Swallowing + * must be done. It must be odd. + * 28 Lower WML Event(LWE) SDMA events reg to check for + * LWML event mask + * 0: LWE in EVENTS register + * 1: LWE in EVENTS2 register + * 29 Higher WML Event(HWE) SDMA events reg to check for + * HWML event mask + * 0: HWE in EVENTS register + * 1: HWE in EVENTS2 register + * 30 --------- MUST BE 0 + * 31 CONT 1: Amount of samples to be + * transferred is unknown and + * script will keep on + * transferring samples as long as + * both events are detected and + * script must be manually stopped + * by the application + * 0: The amount of samples to be + * transferred is equal to the + * count field of mode word + */ +#define SDMA_WATERMARK_LEVEL_LWML 0xFF +#define SDMA_WATERMARK_LEVEL_PS BIT(8) +#define SDMA_WATERMARK_LEVEL_PA BIT(9) +#define SDMA_WATERMARK_LEVEL_SPDIF BIT(10) +#define SDMA_WATERMARK_LEVEL_SP BIT(11) +#define SDMA_WATERMARK_LEVEL_DP BIT(12) +#define SDMA_WATERMARK_LEVEL_HWML (0xFF << 16) +#define SDMA_WATERMARK_LEVEL_LWE BIT(28) +#define SDMA_WATERMARK_LEVEL_HWE BIT(29) +#define SDMA_WATERMARK_LEVEL_CONT BIT(31) + /* * Mode/Count of data node descriptors - IPCv2 */ @@ -259,8 +313,9 @@ struct sdma_channel { struct sdma_buffer_descriptor *bd; dma_addr_t bd_phys; unsigned int pc_from_device, pc_to_device; + unsigned int device_to_device; unsigned long flags; - dma_addr_t per_address; + dma_addr_t per_address, per_address2; unsigned long event_mask[2]; unsigned long watermark_level; u32 shp_addr, per_addr; @@ -328,6 +383,8 @@ struct sdma_engine { u32 script_number; struct sdma_script_start_addrs *script_addrs; const struct sdma_driver_data *drvdata; + u32 spba_start_addr; + u32 spba_end_addr; }; static struct sdma_driver_data sdma_imx31 = { @@ -705,6 +762,7 @@ static void sdma_get_pc(struct sdma_channel *sdmac, sdmac->pc_from_device = 0; sdmac->pc_to_device = 0; + sdmac->device_to_device = 0; switch (peripheral_type) { case IMX_DMATYPE_MEMORY: @@ -780,6 +838,7 @@ static void sdma_get_pc(struct sdma_channel *sdmac, sdmac->pc_from_device = per_2_emi; sdmac->pc_to_device = emi_2_per; + sdmac->device_to_device = per_2_per; } static int sdma_load_context(struct sdma_channel *sdmac) @@ -792,11 +851,12 @@ static int sdma_load_context(struct sdma_channel *sdmac) int ret; unsigned long flags; - if (sdmac->direction == DMA_DEV_TO_MEM) { + if (sdmac->direction == DMA_DEV_TO_MEM) load_address = sdmac->pc_from_device; - } else { + else if (sdmac->direction == DMA_DEV_TO_DEV) + load_address = sdmac->device_to_device; + else load_address = sdmac->pc_to_device; - } if (load_address < 0) return load_address; @@ -851,6 +911,46 @@ static int sdma_disable_channel(struct dma_chan *chan) return 0; } +static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac) +{ + struct sdma_engine *sdma = sdmac->sdma; + + int lwml = sdmac->watermark_level & SDMA_WATERMARK_LEVEL_LWML; + int hwml = (sdmac->watermark_level & SDMA_WATERMARK_LEVEL_HWML) >> 16; + + set_bit(sdmac->event_id0 % 32, &sdmac->event_mask[1]); + set_bit(sdmac->event_id1 % 32, &sdmac->event_mask[0]); + + if (sdmac->event_id0 > 31) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_LWE; + + if (sdmac->event_id1 > 31) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_HWE; + + /* + * If LWML(src_maxburst) > HWML(dst_maxburst), we need + * swap LWML and HWML of INFO(A.3.2.5.1), also need swap + * r0(event_mask[1]) and r1(event_mask[0]). + */ + if (lwml > hwml) { + sdmac->watermark_level &= ~(SDMA_WATERMARK_LEVEL_LWML | + SDMA_WATERMARK_LEVEL_HWML); + sdmac->watermark_level |= hwml; + sdmac->watermark_level |= lwml << 16; + swap(sdmac->event_mask[0], sdmac->event_mask[1]); + } + + if (sdmac->per_address2 >= sdma->spba_start_addr && + sdmac->per_address2 <= sdma->spba_end_addr) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SP; + + if (sdmac->per_address >= sdma->spba_start_addr && + sdmac->per_address <= sdma->spba_end_addr) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP; + + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT; +} + static int sdma_config_channel(struct dma_chan *chan) { struct sdma_channel *sdmac = to_sdma_chan(chan); @@ -869,6 +969,12 @@ static int sdma_config_channel(struct dma_chan *chan) sdma_event_enable(sdmac, sdmac->event_id0); } + if (sdmac->event_id1) { + if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events) + return -EINVAL; + sdma_event_enable(sdmac, sdmac->event_id1); + } + switch (sdmac->peripheral_type) { case IMX_DMATYPE_DSP: sdma_config_ownership(sdmac, false, true, true); @@ -887,19 +993,17 @@ static int sdma_config_channel(struct dma_chan *chan) (sdmac->peripheral_type != IMX_DMATYPE_DSP)) { /* Handle multiple event channels differently */ if (sdmac->event_id1) { - sdmac->event_mask[1] = BIT(sdmac->event_id1 % 32); - if (sdmac->event_id1 > 31) - __set_bit(31, &sdmac->watermark_level); - sdmac->event_mask[0] = BIT(sdmac->event_id0 % 32); - if (sdmac->event_id0 > 31) - __set_bit(30, &sdmac->watermark_level); - } else { + if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP || + sdmac->peripheral_type == IMX_DMATYPE_ASRC) + sdma_set_watermarklevel_for_p2p(sdmac); + } else __set_bit(sdmac->event_id0, sdmac->event_mask); - } + /* Watermark Level */ sdmac->watermark_level |= sdmac->watermark_level; /* Address */ sdmac->shp_addr = sdmac->per_address; + sdmac->per_addr = sdmac->per_address2; } else { sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */ } @@ -987,17 +1091,22 @@ static int sdma_alloc_chan_resources(struct dma_chan *chan) sdmac->peripheral_type = data->peripheral_type; sdmac->event_id0 = data->dma_request; + sdmac->event_id1 = data->dma_request2; - clk_enable(sdmac->sdma->clk_ipg); - clk_enable(sdmac->sdma->clk_ahb); + ret = clk_enable(sdmac->sdma->clk_ipg); + if (ret) + return ret; + ret = clk_enable(sdmac->sdma->clk_ahb); + if (ret) + goto disable_clk_ipg; ret = sdma_request_channel(sdmac); if (ret) - return ret; + goto disable_clk_ahb; ret = sdma_set_channel_priority(sdmac, prio); if (ret) - return ret; + goto disable_clk_ahb; dma_async_tx_descriptor_init(&sdmac->desc, chan); sdmac->desc.tx_submit = sdma_tx_submit; @@ -1005,6 +1114,12 @@ static int sdma_alloc_chan_resources(struct dma_chan *chan) sdmac->desc.flags = DMA_CTRL_ACK; return 0; + +disable_clk_ahb: + clk_disable(sdmac->sdma->clk_ahb); +disable_clk_ipg: + clk_disable(sdmac->sdma->clk_ipg); + return ret; } static void sdma_free_chan_resources(struct dma_chan *chan) @@ -1221,6 +1336,14 @@ static int sdma_config(struct dma_chan *chan, sdmac->watermark_level = dmaengine_cfg->src_maxburst * dmaengine_cfg->src_addr_width; sdmac->word_size = dmaengine_cfg->src_addr_width; + } else if (dmaengine_cfg->direction == DMA_DEV_TO_DEV) { + sdmac->per_address2 = dmaengine_cfg->src_addr; + sdmac->per_address = dmaengine_cfg->dst_addr; + sdmac->watermark_level = dmaengine_cfg->src_maxburst & + SDMA_WATERMARK_LEVEL_LWML; + sdmac->watermark_level |= (dmaengine_cfg->dst_maxburst << 16) & + SDMA_WATERMARK_LEVEL_HWML; + sdmac->word_size = dmaengine_cfg->dst_addr_width; } else { sdmac->per_address = dmaengine_cfg->dst_addr; sdmac->watermark_level = dmaengine_cfg->dst_maxburst * @@ -1337,6 +1460,72 @@ static void sdma_load_firmware(const struct firmware *fw, void *context) release_firmware(fw); } +#define EVENT_REMAP_CELLS 3 + +static int __init sdma_event_remap(struct sdma_engine *sdma) +{ + struct device_node *np = sdma->dev->of_node; + struct device_node *gpr_np = of_parse_phandle(np, "gpr", 0); + struct property *event_remap; + struct regmap *gpr; + char propname[] = "fsl,sdma-event-remap"; + u32 reg, val, shift, num_map, i; + int ret = 0; + + if (IS_ERR(np) || IS_ERR(gpr_np)) + goto out; + + event_remap = of_find_property(np, propname, NULL); + num_map = event_remap ? (event_remap->length / sizeof(u32)) : 0; + if (!num_map) { + dev_warn(sdma->dev, "no event needs to be remapped\n"); + goto out; + } else if (num_map % EVENT_REMAP_CELLS) { + dev_err(sdma->dev, "the property %s must modulo %d\n", + propname, EVENT_REMAP_CELLS); + ret = -EINVAL; + goto out; + } + + gpr = syscon_node_to_regmap(gpr_np); + if (IS_ERR(gpr)) { + dev_err(sdma->dev, "failed to get gpr regmap\n"); + ret = PTR_ERR(gpr); + goto out; + } + + for (i = 0; i < num_map; i += EVENT_REMAP_CELLS) { + ret = of_property_read_u32_index(np, propname, i, ®); + if (ret) { + dev_err(sdma->dev, "failed to read property %s index %d\n", + propname, i); + goto out; + } + + ret = of_property_read_u32_index(np, propname, i + 1, &shift); + if (ret) { + dev_err(sdma->dev, "failed to read property %s index %d\n", + propname, i + 1); + goto out; + } + + ret = of_property_read_u32_index(np, propname, i + 2, &val); + if (ret) { + dev_err(sdma->dev, "failed to read property %s index %d\n", + propname, i + 2); + goto out; + } + + regmap_update_bits(gpr, reg, BIT(shift), val << shift); + } + +out: + if (!IS_ERR(gpr_np)) + of_node_put(gpr_np); + + return ret; +} + static int sdma_get_firmware(struct sdma_engine *sdma, const char *fw_name) { @@ -1354,8 +1543,12 @@ static int sdma_init(struct sdma_engine *sdma) int i, ret; dma_addr_t ccb_phys; - clk_enable(sdma->clk_ipg); - clk_enable(sdma->clk_ahb); + ret = clk_enable(sdma->clk_ipg); + if (ret) + return ret; + ret = clk_enable(sdma->clk_ahb); + if (ret) + goto disable_clk_ipg; /* Be sure SDMA has not started yet */ writel_relaxed(0, sdma->regs + SDMA_H_C0PTR); @@ -1411,8 +1604,9 @@ static int sdma_init(struct sdma_engine *sdma) return 0; err_dma_alloc: - clk_disable(sdma->clk_ipg); clk_disable(sdma->clk_ahb); +disable_clk_ipg: + clk_disable(sdma->clk_ipg); dev_err(sdma->dev, "initialisation failed with %d\n", ret); return ret; } @@ -1444,6 +1638,14 @@ static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec, data.dma_request = dma_spec->args[0]; data.peripheral_type = dma_spec->args[1]; data.priority = dma_spec->args[2]; + /* + * init dma_request2 to zero, which is not used by the dts. + * For P2P, dma_request2 is init from dma_request_channel(), + * chan->private will point to the imx_dma_data, and in + * device_alloc_chan_resources(), imx_dma_data.dma_request2 will + * be set to sdmac->event_id1. + */ + data.dma_request2 = 0; return dma_request_channel(mask, sdma_filter_fn, &data); } @@ -1453,10 +1655,12 @@ static int sdma_probe(struct platform_device *pdev) const struct of_device_id *of_id = of_match_device(sdma_dt_ids, &pdev->dev); struct device_node *np = pdev->dev.of_node; + struct device_node *spba_bus; const char *fw_name; int ret; int irq; struct resource *iores; + struct resource spba_res; struct sdma_platform_data *pdata = dev_get_platdata(&pdev->dev); int i; struct sdma_engine *sdma; @@ -1551,6 +1755,10 @@ static int sdma_probe(struct platform_device *pdev) if (ret) goto err_init; + ret = sdma_event_remap(sdma); + if (ret) + goto err_init; + if (sdma->drvdata->script_addrs) sdma_add_scripts(sdma, sdma->drvdata->script_addrs); if (pdata && pdata->script_addrs) @@ -1608,6 +1816,14 @@ static int sdma_probe(struct platform_device *pdev) dev_err(&pdev->dev, "failed to register controller\n"); goto err_register; } + + spba_bus = of_find_compatible_node(NULL, NULL, "fsl,spba-bus"); + ret = of_address_to_resource(spba_bus, 0, &spba_res); + if (!ret) { + sdma->spba_start_addr = spba_res.start; + sdma->spba_end_addr = spba_res.end; + } + of_node_put(spba_bus); } dev_info(sdma->dev, "initialized\n"); diff --git a/drivers/dma/ioat/Makefile b/drivers/dma/ioat/Makefile index 0ff7270af25b..cf5fedbe2b75 100644 --- a/drivers/dma/ioat/Makefile +++ b/drivers/dma/ioat/Makefile @@ -1,2 +1,2 @@ obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o -ioatdma-y := pci.o dma.o dma_v2.o dma_v3.o dca.o +ioatdma-y := init.o dma.o prep.o dca.o sysfs.o diff --git a/drivers/dma/ioat/dca.c b/drivers/dma/ioat/dca.c index ea1e107ae884..2cb7c308d5c7 100644 --- a/drivers/dma/ioat/dca.c +++ b/drivers/dma/ioat/dca.c @@ -31,7 +31,6 @@ #include "dma.h" #include "registers.h" -#include "dma_v2.h" /* * Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6 @@ -71,14 +70,6 @@ static inline int dca2_tag_map_valid(u8 *tag_map) #define APICID_BIT(x) (DCA_TAG_MAP_VALID | (x)) #define IOAT_TAG_MAP_LEN 8 -static u8 ioat_tag_map_BNB[IOAT_TAG_MAP_LEN] = { - 1, APICID_BIT(1), APICID_BIT(2), APICID_BIT(2), }; -static u8 ioat_tag_map_SCNB[IOAT_TAG_MAP_LEN] = { - 1, APICID_BIT(1), APICID_BIT(2), APICID_BIT(2), }; -static u8 ioat_tag_map_CNB[IOAT_TAG_MAP_LEN] = { - 1, APICID_BIT(1), APICID_BIT(3), APICID_BIT(4), APICID_BIT(2), }; -static u8 ioat_tag_map_UNISYS[IOAT_TAG_MAP_LEN] = { 0 }; - /* pack PCI B/D/F into a u16 */ static inline u16 dcaid_from_pcidev(struct pci_dev *pci) { @@ -126,96 +117,6 @@ struct ioat_dca_priv { struct ioat_dca_slot req_slots[0]; }; -/* 5000 series chipset DCA Port Requester ID Table Entry Format - * [15:8] PCI-Express Bus Number - * [7:3] PCI-Express Device Number - * [2:0] PCI-Express Function Number - * - * 5000 series chipset DCA control register format - * [7:1] Reserved (0) - * [0] Ignore Function Number - */ - -static int ioat_dca_add_requester(struct dca_provider *dca, struct device *dev) -{ - struct ioat_dca_priv *ioatdca = dca_priv(dca); - struct pci_dev *pdev; - int i; - u16 id; - - /* This implementation only supports PCI-Express */ - if (!dev_is_pci(dev)) - return -ENODEV; - pdev = to_pci_dev(dev); - id = dcaid_from_pcidev(pdev); - - if (ioatdca->requester_count == ioatdca->max_requesters) - return -ENODEV; - - for (i = 0; i < ioatdca->max_requesters; i++) { - if (ioatdca->req_slots[i].pdev == NULL) { - /* found an empty slot */ - ioatdca->requester_count++; - ioatdca->req_slots[i].pdev = pdev; - ioatdca->req_slots[i].rid = id; - writew(id, ioatdca->dca_base + (i * 4)); - /* make sure the ignore function bit is off */ - writeb(0, ioatdca->dca_base + (i * 4) + 2); - return i; - } - } - /* Error, ioatdma->requester_count is out of whack */ - return -EFAULT; -} - -static int ioat_dca_remove_requester(struct dca_provider *dca, - struct device *dev) -{ - struct ioat_dca_priv *ioatdca = dca_priv(dca); - struct pci_dev *pdev; - int i; - - /* This implementation only supports PCI-Express */ - if (!dev_is_pci(dev)) - return -ENODEV; - pdev = to_pci_dev(dev); - - for (i = 0; i < ioatdca->max_requesters; i++) { - if (ioatdca->req_slots[i].pdev == pdev) { - writew(0, ioatdca->dca_base + (i * 4)); - ioatdca->req_slots[i].pdev = NULL; - ioatdca->req_slots[i].rid = 0; - ioatdca->requester_count--; - return i; - } - } - return -ENODEV; -} - -static u8 ioat_dca_get_tag(struct dca_provider *dca, - struct device *dev, - int cpu) -{ - struct ioat_dca_priv *ioatdca = dca_priv(dca); - int i, apic_id, bit, value; - u8 entry, tag; - - tag = 0; - apic_id = cpu_physical_id(cpu); - - for (i = 0; i < IOAT_TAG_MAP_LEN; i++) { - entry = ioatdca->tag_map[i]; - if (entry & DCA_TAG_MAP_VALID) { - bit = entry & ~DCA_TAG_MAP_VALID; - value = (apic_id & (1 << bit)) ? 1 : 0; - } else { - value = entry ? 1 : 0; - } - tag |= (value << i); - } - return tag; -} - static int ioat_dca_dev_managed(struct dca_provider *dca, struct device *dev) { @@ -231,260 +132,7 @@ static int ioat_dca_dev_managed(struct dca_provider *dca, return 0; } -static struct dca_ops ioat_dca_ops = { - .add_requester = ioat_dca_add_requester, - .remove_requester = ioat_dca_remove_requester, - .get_tag = ioat_dca_get_tag, - .dev_managed = ioat_dca_dev_managed, -}; - - -struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase) -{ - struct dca_provider *dca; - struct ioat_dca_priv *ioatdca; - u8 *tag_map = NULL; - int i; - int err; - u8 version; - u8 max_requesters; - - if (!system_has_dca_enabled(pdev)) - return NULL; - - /* I/OAT v1 systems must have a known tag_map to support DCA */ - switch (pdev->vendor) { - case PCI_VENDOR_ID_INTEL: - switch (pdev->device) { - case PCI_DEVICE_ID_INTEL_IOAT: - tag_map = ioat_tag_map_BNB; - break; - case PCI_DEVICE_ID_INTEL_IOAT_CNB: - tag_map = ioat_tag_map_CNB; - break; - case PCI_DEVICE_ID_INTEL_IOAT_SCNB: - tag_map = ioat_tag_map_SCNB; - break; - } - break; - case PCI_VENDOR_ID_UNISYS: - switch (pdev->device) { - case PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR: - tag_map = ioat_tag_map_UNISYS; - break; - } - break; - } - if (tag_map == NULL) - return NULL; - - version = readb(iobase + IOAT_VER_OFFSET); - if (version == IOAT_VER_3_0) - max_requesters = IOAT3_DCA_MAX_REQ; - else - max_requesters = IOAT_DCA_MAX_REQ; - - dca = alloc_dca_provider(&ioat_dca_ops, - sizeof(*ioatdca) + - (sizeof(struct ioat_dca_slot) * max_requesters)); - if (!dca) - return NULL; - - ioatdca = dca_priv(dca); - ioatdca->max_requesters = max_requesters; - ioatdca->dca_base = iobase + 0x54; - - /* copy over the APIC ID to DCA tag mapping */ - for (i = 0; i < IOAT_TAG_MAP_LEN; i++) - ioatdca->tag_map[i] = tag_map[i]; - - err = register_dca_provider(dca, &pdev->dev); - if (err) { - free_dca_provider(dca); - return NULL; - } - - return dca; -} - - -static int ioat2_dca_add_requester(struct dca_provider *dca, struct device *dev) -{ - struct ioat_dca_priv *ioatdca = dca_priv(dca); - struct pci_dev *pdev; - int i; - u16 id; - u16 global_req_table; - - /* This implementation only supports PCI-Express */ - if (!dev_is_pci(dev)) - return -ENODEV; - pdev = to_pci_dev(dev); - id = dcaid_from_pcidev(pdev); - - if (ioatdca->requester_count == ioatdca->max_requesters) - return -ENODEV; - - for (i = 0; i < ioatdca->max_requesters; i++) { - if (ioatdca->req_slots[i].pdev == NULL) { - /* found an empty slot */ - ioatdca->requester_count++; - ioatdca->req_slots[i].pdev = pdev; - ioatdca->req_slots[i].rid = id; - global_req_table = - readw(ioatdca->dca_base + IOAT_DCA_GREQID_OFFSET); - writel(id | IOAT_DCA_GREQID_VALID, - ioatdca->iobase + global_req_table + (i * 4)); - return i; - } - } - /* Error, ioatdma->requester_count is out of whack */ - return -EFAULT; -} - -static int ioat2_dca_remove_requester(struct dca_provider *dca, - struct device *dev) -{ - struct ioat_dca_priv *ioatdca = dca_priv(dca); - struct pci_dev *pdev; - int i; - u16 global_req_table; - - /* This implementation only supports PCI-Express */ - if (!dev_is_pci(dev)) - return -ENODEV; - pdev = to_pci_dev(dev); - - for (i = 0; i < ioatdca->max_requesters; i++) { - if (ioatdca->req_slots[i].pdev == pdev) { - global_req_table = - readw(ioatdca->dca_base + IOAT_DCA_GREQID_OFFSET); - writel(0, ioatdca->iobase + global_req_table + (i * 4)); - ioatdca->req_slots[i].pdev = NULL; - ioatdca->req_slots[i].rid = 0; - ioatdca->requester_count--; - return i; - } - } - return -ENODEV; -} - -static u8 ioat2_dca_get_tag(struct dca_provider *dca, - struct device *dev, - int cpu) -{ - u8 tag; - - tag = ioat_dca_get_tag(dca, dev, cpu); - tag = (~tag) & 0x1F; - return tag; -} - -static struct dca_ops ioat2_dca_ops = { - .add_requester = ioat2_dca_add_requester, - .remove_requester = ioat2_dca_remove_requester, - .get_tag = ioat2_dca_get_tag, - .dev_managed = ioat_dca_dev_managed, -}; - -static int ioat2_dca_count_dca_slots(void __iomem *iobase, u16 dca_offset) -{ - int slots = 0; - u32 req; - u16 global_req_table; - - global_req_table = readw(iobase + dca_offset + IOAT_DCA_GREQID_OFFSET); - if (global_req_table == 0) - return 0; - do { - req = readl(iobase + global_req_table + (slots * sizeof(u32))); - slots++; - } while ((req & IOAT_DCA_GREQID_LASTID) == 0); - - return slots; -} - -struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase) -{ - struct dca_provider *dca; - struct ioat_dca_priv *ioatdca; - int slots; - int i; - int err; - u32 tag_map; - u16 dca_offset; - u16 csi_fsb_control; - u16 pcie_control; - u8 bit; - - if (!system_has_dca_enabled(pdev)) - return NULL; - - dca_offset = readw(iobase + IOAT_DCAOFFSET_OFFSET); - if (dca_offset == 0) - return NULL; - - slots = ioat2_dca_count_dca_slots(iobase, dca_offset); - if (slots == 0) - return NULL; - - dca = alloc_dca_provider(&ioat2_dca_ops, - sizeof(*ioatdca) - + (sizeof(struct ioat_dca_slot) * slots)); - if (!dca) - return NULL; - - ioatdca = dca_priv(dca); - ioatdca->iobase = iobase; - ioatdca->dca_base = iobase + dca_offset; - ioatdca->max_requesters = slots; - - /* some bios might not know to turn these on */ - csi_fsb_control = readw(ioatdca->dca_base + IOAT_FSB_CAP_ENABLE_OFFSET); - if ((csi_fsb_control & IOAT_FSB_CAP_ENABLE_PREFETCH) == 0) { - csi_fsb_control |= IOAT_FSB_CAP_ENABLE_PREFETCH; - writew(csi_fsb_control, - ioatdca->dca_base + IOAT_FSB_CAP_ENABLE_OFFSET); - } - pcie_control = readw(ioatdca->dca_base + IOAT_PCI_CAP_ENABLE_OFFSET); - if ((pcie_control & IOAT_PCI_CAP_ENABLE_MEMWR) == 0) { - pcie_control |= IOAT_PCI_CAP_ENABLE_MEMWR; - writew(pcie_control, - ioatdca->dca_base + IOAT_PCI_CAP_ENABLE_OFFSET); - } - - - /* TODO version, compatibility and configuration checks */ - - /* copy out the APIC to DCA tag map */ - tag_map = readl(ioatdca->dca_base + IOAT_APICID_TAG_MAP_OFFSET); - for (i = 0; i < 5; i++) { - bit = (tag_map >> (4 * i)) & 0x0f; - if (bit < 8) - ioatdca->tag_map[i] = bit | DCA_TAG_MAP_VALID; - else - ioatdca->tag_map[i] = 0; - } - - if (!dca2_tag_map_valid(ioatdca->tag_map)) { - WARN_TAINT_ONCE(1, TAINT_FIRMWARE_WORKAROUND, - "%s %s: APICID_TAG_MAP set incorrectly by BIOS, disabling DCA\n", - dev_driver_string(&pdev->dev), - dev_name(&pdev->dev)); - free_dca_provider(dca); - return NULL; - } - - err = register_dca_provider(dca, &pdev->dev); - if (err) { - free_dca_provider(dca); - return NULL; - } - - return dca; -} - -static int ioat3_dca_add_requester(struct dca_provider *dca, struct device *dev) +static int ioat_dca_add_requester(struct dca_provider *dca, struct device *dev) { struct ioat_dca_priv *ioatdca = dca_priv(dca); struct pci_dev *pdev; @@ -518,7 +166,7 @@ static int ioat3_dca_add_requester(struct dca_provider *dca, struct device *dev) return -EFAULT; } -static int ioat3_dca_remove_requester(struct dca_provider *dca, +static int ioat_dca_remove_requester(struct dca_provider *dca, struct device *dev) { struct ioat_dca_priv *ioatdca = dca_priv(dca); @@ -545,7 +193,7 @@ static int ioat3_dca_remove_requester(struct dca_provider *dca, return -ENODEV; } -static u8 ioat3_dca_get_tag(struct dca_provider *dca, +static u8 ioat_dca_get_tag(struct dca_provider *dca, struct device *dev, int cpu) { @@ -576,14 +224,14 @@ static u8 ioat3_dca_get_tag(struct dca_provider *dca, return tag; } -static struct dca_ops ioat3_dca_ops = { - .add_requester = ioat3_dca_add_requester, - .remove_requester = ioat3_dca_remove_requester, - .get_tag = ioat3_dca_get_tag, +static struct dca_ops ioat_dca_ops = { + .add_requester = ioat_dca_add_requester, + .remove_requester = ioat_dca_remove_requester, + .get_tag = ioat_dca_get_tag, .dev_managed = ioat_dca_dev_managed, }; -static int ioat3_dca_count_dca_slots(void *iobase, u16 dca_offset) +static int ioat_dca_count_dca_slots(void *iobase, u16 dca_offset) { int slots = 0; u32 req; @@ -618,7 +266,7 @@ static inline int dca3_tag_map_invalid(u8 *tag_map) (tag_map[4] == DCA_TAG_MAP_VALID)); } -struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase) +struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase) { struct dca_provider *dca; struct ioat_dca_priv *ioatdca; @@ -645,11 +293,11 @@ struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase) if (dca_offset == 0) return NULL; - slots = ioat3_dca_count_dca_slots(iobase, dca_offset); + slots = ioat_dca_count_dca_slots(iobase, dca_offset); if (slots == 0) return NULL; - dca = alloc_dca_provider(&ioat3_dca_ops, + dca = alloc_dca_provider(&ioat_dca_ops, sizeof(*ioatdca) + (sizeof(struct ioat_dca_slot) * slots)); if (!dca) diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index ee0aa9f4ccfa..f66b7e640610 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -1,6 +1,6 @@ /* * Intel I/OAT DMA Linux driver - * Copyright(c) 2004 - 2009 Intel Corporation. + * Copyright(c) 2004 - 2015 Intel Corporation. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -31,31 +31,23 @@ #include #include #include -#include #include "dma.h" #include "registers.h" #include "hw.h" #include "../dmaengine.h" -int ioat_pending_level = 4; -module_param(ioat_pending_level, int, 0644); -MODULE_PARM_DESC(ioat_pending_level, - "high-water mark for pushing ioat descriptors (default: 4)"); - -/* internal functions */ -static void ioat1_cleanup(struct ioat_dma_chan *ioat); -static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat); +static void ioat_eh(struct ioatdma_chan *ioat_chan); /** * ioat_dma_do_interrupt - handler used for single vector interrupt mode * @irq: interrupt id * @data: interrupt data */ -static irqreturn_t ioat_dma_do_interrupt(int irq, void *data) +irqreturn_t ioat_dma_do_interrupt(int irq, void *data) { struct ioatdma_device *instance = data; - struct ioat_chan_common *chan; + struct ioatdma_chan *ioat_chan; unsigned long attnstatus; int bit; u8 intrctrl; @@ -72,9 +64,9 @@ static irqreturn_t ioat_dma_do_interrupt(int irq, void *data) attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET); for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) { - chan = ioat_chan_by_index(instance, bit); - if (test_bit(IOAT_RUN, &chan->state)) - tasklet_schedule(&chan->cleanup_task); + ioat_chan = ioat_chan_by_index(instance, bit); + if (test_bit(IOAT_RUN, &ioat_chan->state)) + tasklet_schedule(&ioat_chan->cleanup_task); } writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); @@ -86,294 +78,32 @@ static irqreturn_t ioat_dma_do_interrupt(int irq, void *data) * @irq: interrupt id * @data: interrupt data */ -static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data) +irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data) { - struct ioat_chan_common *chan = data; + struct ioatdma_chan *ioat_chan = data; - if (test_bit(IOAT_RUN, &chan->state)) - tasklet_schedule(&chan->cleanup_task); + if (test_bit(IOAT_RUN, &ioat_chan->state)) + tasklet_schedule(&ioat_chan->cleanup_task); return IRQ_HANDLED; } -/* common channel initialization */ -void ioat_init_channel(struct ioatdma_device *device, struct ioat_chan_common *chan, int idx) +void ioat_stop(struct ioatdma_chan *ioat_chan) { - struct dma_device *dma = &device->common; - struct dma_chan *c = &chan->common; - unsigned long data = (unsigned long) c; - - chan->device = device; - chan->reg_base = device->reg_base + (0x80 * (idx + 1)); - spin_lock_init(&chan->cleanup_lock); - chan->common.device = dma; - dma_cookie_init(&chan->common); - list_add_tail(&chan->common.device_node, &dma->channels); - device->idx[idx] = chan; - init_timer(&chan->timer); - chan->timer.function = device->timer_fn; - chan->timer.data = data; - tasklet_init(&chan->cleanup_task, device->cleanup_fn, data); -} - -/** - * ioat1_dma_enumerate_channels - find and initialize the device's channels - * @device: the device to be enumerated - */ -static int ioat1_enumerate_channels(struct ioatdma_device *device) -{ - u8 xfercap_scale; - u32 xfercap; - int i; - struct ioat_dma_chan *ioat; - struct device *dev = &device->pdev->dev; - struct dma_device *dma = &device->common; - - INIT_LIST_HEAD(&dma->channels); - dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET); - dma->chancnt &= 0x1f; /* bits [4:0] valid */ - if (dma->chancnt > ARRAY_SIZE(device->idx)) { - dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n", - dma->chancnt, ARRAY_SIZE(device->idx)); - dma->chancnt = ARRAY_SIZE(device->idx); - } - xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET); - xfercap_scale &= 0x1f; /* bits [4:0] valid */ - xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale)); - dev_dbg(dev, "%s: xfercap = %d\n", __func__, xfercap); - -#ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL - if (i7300_idle_platform_probe(NULL, NULL, 1) == 0) - dma->chancnt--; -#endif - for (i = 0; i < dma->chancnt; i++) { - ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL); - if (!ioat) - break; - - ioat_init_channel(device, &ioat->base, i); - ioat->xfercap = xfercap; - spin_lock_init(&ioat->desc_lock); - INIT_LIST_HEAD(&ioat->free_desc); - INIT_LIST_HEAD(&ioat->used_desc); - } - dma->chancnt = i; - return i; -} - -/** - * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended - * descriptors to hw - * @chan: DMA channel handle - */ -static inline void -__ioat1_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat) -{ - void __iomem *reg_base = ioat->base.reg_base; - - dev_dbg(to_dev(&ioat->base), "%s: pending: %d\n", - __func__, ioat->pending); - ioat->pending = 0; - writeb(IOAT_CHANCMD_APPEND, reg_base + IOAT1_CHANCMD_OFFSET); -} - -static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan) -{ - struct ioat_dma_chan *ioat = to_ioat_chan(chan); - - if (ioat->pending > 0) { - spin_lock_bh(&ioat->desc_lock); - __ioat1_dma_memcpy_issue_pending(ioat); - spin_unlock_bh(&ioat->desc_lock); - } -} - -/** - * ioat1_reset_channel - restart a channel - * @ioat: IOAT DMA channel handle - */ -static void ioat1_reset_channel(struct ioat_dma_chan *ioat) -{ - struct ioat_chan_common *chan = &ioat->base; - void __iomem *reg_base = chan->reg_base; - u32 chansts, chanerr; - - dev_warn(to_dev(chan), "reset\n"); - chanerr = readl(reg_base + IOAT_CHANERR_OFFSET); - chansts = *chan->completion & IOAT_CHANSTS_STATUS; - if (chanerr) { - dev_err(to_dev(chan), - "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n", - chan_num(chan), chansts, chanerr); - writel(chanerr, reg_base + IOAT_CHANERR_OFFSET); - } - - /* - * whack it upside the head with a reset - * and wait for things to settle out. - * force the pending count to a really big negative - * to make sure no one forces an issue_pending - * while we're waiting. - */ - - ioat->pending = INT_MIN; - writeb(IOAT_CHANCMD_RESET, - reg_base + IOAT_CHANCMD_OFFSET(chan->device->version)); - set_bit(IOAT_RESET_PENDING, &chan->state); - mod_timer(&chan->timer, jiffies + RESET_DELAY); -} - -static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx) -{ - struct dma_chan *c = tx->chan; - struct ioat_dma_chan *ioat = to_ioat_chan(c); - struct ioat_desc_sw *desc = tx_to_ioat_desc(tx); - struct ioat_chan_common *chan = &ioat->base; - struct ioat_desc_sw *first; - struct ioat_desc_sw *chain_tail; - dma_cookie_t cookie; - - spin_lock_bh(&ioat->desc_lock); - /* cookie incr and addition to used_list must be atomic */ - cookie = dma_cookie_assign(tx); - dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie); - - /* write address into NextDescriptor field of last desc in chain */ - first = to_ioat_desc(desc->tx_list.next); - chain_tail = to_ioat_desc(ioat->used_desc.prev); - /* make descriptor updates globally visible before chaining */ - wmb(); - chain_tail->hw->next = first->txd.phys; - list_splice_tail_init(&desc->tx_list, &ioat->used_desc); - dump_desc_dbg(ioat, chain_tail); - dump_desc_dbg(ioat, first); - - if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state)) - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); - - ioat->active += desc->hw->tx_cnt; - ioat->pending += desc->hw->tx_cnt; - if (ioat->pending >= ioat_pending_level) - __ioat1_dma_memcpy_issue_pending(ioat); - spin_unlock_bh(&ioat->desc_lock); - - return cookie; -} - -/** - * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair - * @ioat: the channel supplying the memory pool for the descriptors - * @flags: allocation flags - */ -static struct ioat_desc_sw * -ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat, gfp_t flags) -{ - struct ioat_dma_descriptor *desc; - struct ioat_desc_sw *desc_sw; - struct ioatdma_device *ioatdma_device; - dma_addr_t phys; - - ioatdma_device = ioat->base.device; - desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys); - if (unlikely(!desc)) - return NULL; - - desc_sw = kzalloc(sizeof(*desc_sw), flags); - if (unlikely(!desc_sw)) { - pci_pool_free(ioatdma_device->dma_pool, desc, phys); - return NULL; - } - - memset(desc, 0, sizeof(*desc)); - - INIT_LIST_HEAD(&desc_sw->tx_list); - dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common); - desc_sw->txd.tx_submit = ioat1_tx_submit; - desc_sw->hw = desc; - desc_sw->txd.phys = phys; - set_desc_id(desc_sw, -1); - - return desc_sw; -} - -static int ioat_initial_desc_count = 256; -module_param(ioat_initial_desc_count, int, 0644); -MODULE_PARM_DESC(ioat_initial_desc_count, - "ioat1: initial descriptors per channel (default: 256)"); -/** - * ioat1_dma_alloc_chan_resources - returns the number of allocated descriptors - * @chan: the channel to be filled out - */ -static int ioat1_dma_alloc_chan_resources(struct dma_chan *c) -{ - struct ioat_dma_chan *ioat = to_ioat_chan(c); - struct ioat_chan_common *chan = &ioat->base; - struct ioat_desc_sw *desc; - u32 chanerr; - int i; - LIST_HEAD(tmp_list); - - /* have we already been set up? */ - if (!list_empty(&ioat->free_desc)) - return ioat->desccount; - - /* Setup register to interrupt and write completion status on error */ - writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET); - - chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); - if (chanerr) { - dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr); - writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET); - } - - /* Allocate descriptors */ - for (i = 0; i < ioat_initial_desc_count; i++) { - desc = ioat_dma_alloc_descriptor(ioat, GFP_KERNEL); - if (!desc) { - dev_err(to_dev(chan), "Only %d initial descriptors\n", i); - break; - } - set_desc_id(desc, i); - list_add_tail(&desc->node, &tmp_list); - } - spin_lock_bh(&ioat->desc_lock); - ioat->desccount = i; - list_splice(&tmp_list, &ioat->free_desc); - spin_unlock_bh(&ioat->desc_lock); - - /* allocate a completion writeback area */ - /* doing 2 32bit writes to mmio since 1 64b write doesn't work */ - chan->completion = pci_pool_alloc(chan->device->completion_pool, - GFP_KERNEL, &chan->completion_dma); - memset(chan->completion, 0, sizeof(*chan->completion)); - writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF, - chan->reg_base + IOAT_CHANCMP_OFFSET_LOW); - writel(((u64) chan->completion_dma) >> 32, - chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); - - set_bit(IOAT_RUN, &chan->state); - ioat1_dma_start_null_desc(ioat); /* give chain to dma device */ - dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n", - __func__, ioat->desccount); - return ioat->desccount; -} - -void ioat_stop(struct ioat_chan_common *chan) -{ - struct ioatdma_device *device = chan->device; - struct pci_dev *pdev = device->pdev; - int chan_id = chan_num(chan); + struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; + struct pci_dev *pdev = ioat_dma->pdev; + int chan_id = chan_num(ioat_chan); struct msix_entry *msix; /* 1/ stop irq from firing tasklets * 2/ stop the tasklet from re-arming irqs */ - clear_bit(IOAT_RUN, &chan->state); + clear_bit(IOAT_RUN, &ioat_chan->state); /* flush inflight interrupts */ - switch (device->irq_mode) { + switch (ioat_dma->irq_mode) { case IOAT_MSIX: - msix = &device->msix_entries[chan_id]; + msix = &ioat_dma->msix_entries[chan_id]; synchronize_irq(msix->vector); break; case IOAT_MSI: @@ -385,398 +115,67 @@ void ioat_stop(struct ioat_chan_common *chan) } /* flush inflight timers */ - del_timer_sync(&chan->timer); + del_timer_sync(&ioat_chan->timer); /* flush inflight tasklet runs */ - tasklet_kill(&chan->cleanup_task); + tasklet_kill(&ioat_chan->cleanup_task); /* final cleanup now that everything is quiesced and can't re-arm */ - device->cleanup_fn((unsigned long) &chan->common); + ioat_cleanup_event((unsigned long)&ioat_chan->dma_chan); } -/** - * ioat1_dma_free_chan_resources - release all the descriptors - * @chan: the channel to be cleaned - */ -static void ioat1_dma_free_chan_resources(struct dma_chan *c) +static void __ioat_issue_pending(struct ioatdma_chan *ioat_chan) { - struct ioat_dma_chan *ioat = to_ioat_chan(c); - struct ioat_chan_common *chan = &ioat->base; - struct ioatdma_device *ioatdma_device = chan->device; - struct ioat_desc_sw *desc, *_desc; - int in_use_descs = 0; + ioat_chan->dmacount += ioat_ring_pending(ioat_chan); + ioat_chan->issued = ioat_chan->head; + writew(ioat_chan->dmacount, + ioat_chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET); + dev_dbg(to_dev(ioat_chan), + "%s: head: %#x tail: %#x issued: %#x count: %#x\n", + __func__, ioat_chan->head, ioat_chan->tail, + ioat_chan->issued, ioat_chan->dmacount); +} - /* Before freeing channel resources first check - * if they have been previously allocated for this channel. - */ - if (ioat->desccount == 0) - return; +void ioat_issue_pending(struct dma_chan *c) +{ + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); - ioat_stop(chan); - - /* Delay 100ms after reset to allow internal DMA logic to quiesce - * before removing DMA descriptor resources. - */ - writeb(IOAT_CHANCMD_RESET, - chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version)); - mdelay(100); - - spin_lock_bh(&ioat->desc_lock); - list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) { - dev_dbg(to_dev(chan), "%s: freeing %d from used list\n", - __func__, desc_id(desc)); - dump_desc_dbg(ioat, desc); - in_use_descs++; - list_del(&desc->node); - pci_pool_free(ioatdma_device->dma_pool, desc->hw, - desc->txd.phys); - kfree(desc); + if (ioat_ring_pending(ioat_chan)) { + spin_lock_bh(&ioat_chan->prep_lock); + __ioat_issue_pending(ioat_chan); + spin_unlock_bh(&ioat_chan->prep_lock); } - list_for_each_entry_safe(desc, _desc, - &ioat->free_desc, node) { - list_del(&desc->node); - pci_pool_free(ioatdma_device->dma_pool, desc->hw, - desc->txd.phys); - kfree(desc); - } - spin_unlock_bh(&ioat->desc_lock); - - pci_pool_free(ioatdma_device->completion_pool, - chan->completion, - chan->completion_dma); - - /* one is ok since we left it on there on purpose */ - if (in_use_descs > 1) - dev_err(to_dev(chan), "Freeing %d in use descriptors!\n", - in_use_descs - 1); - - chan->last_completion = 0; - chan->completion_dma = 0; - ioat->pending = 0; - ioat->desccount = 0; } /** - * ioat1_dma_get_next_descriptor - return the next available descriptor - * @ioat: IOAT DMA channel handle + * ioat_update_pending - log pending descriptors + * @ioat: ioat+ channel * - * Gets the next descriptor from the chain, and must be called with the - * channel's desc_lock held. Allocates more descriptors if the channel - * has run out. + * Check if the number of unsubmitted descriptors has exceeded the + * watermark. Called with prep_lock held */ -static struct ioat_desc_sw * -ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat) +static void ioat_update_pending(struct ioatdma_chan *ioat_chan) { - struct ioat_desc_sw *new; - - if (!list_empty(&ioat->free_desc)) { - new = to_ioat_desc(ioat->free_desc.next); - list_del(&new->node); - } else { - /* try to get another desc */ - new = ioat_dma_alloc_descriptor(ioat, GFP_ATOMIC); - if (!new) { - dev_err(to_dev(&ioat->base), "alloc failed\n"); - return NULL; - } - } - dev_dbg(to_dev(&ioat->base), "%s: allocated: %d\n", - __func__, desc_id(new)); - prefetch(new->hw); - return new; + if (ioat_ring_pending(ioat_chan) > ioat_pending_level) + __ioat_issue_pending(ioat_chan); } -static struct dma_async_tx_descriptor * -ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest, - dma_addr_t dma_src, size_t len, unsigned long flags) +static void __ioat_start_null_desc(struct ioatdma_chan *ioat_chan) { - struct ioat_dma_chan *ioat = to_ioat_chan(c); - struct ioat_desc_sw *desc; - size_t copy; - LIST_HEAD(chain); - dma_addr_t src = dma_src; - dma_addr_t dest = dma_dest; - size_t total_len = len; - struct ioat_dma_descriptor *hw = NULL; - int tx_cnt = 0; - - spin_lock_bh(&ioat->desc_lock); - desc = ioat1_dma_get_next_descriptor(ioat); - do { - if (!desc) - break; - - tx_cnt++; - copy = min_t(size_t, len, ioat->xfercap); - - hw = desc->hw; - hw->size = copy; - hw->ctl = 0; - hw->src_addr = src; - hw->dst_addr = dest; - - list_add_tail(&desc->node, &chain); - - len -= copy; - dest += copy; - src += copy; - if (len) { - struct ioat_desc_sw *next; - - async_tx_ack(&desc->txd); - next = ioat1_dma_get_next_descriptor(ioat); - hw->next = next ? next->txd.phys : 0; - dump_desc_dbg(ioat, desc); - desc = next; - } else - hw->next = 0; - } while (len); - - if (!desc) { - struct ioat_chan_common *chan = &ioat->base; - - dev_err(to_dev(chan), - "chan%d - get_next_desc failed\n", chan_num(chan)); - list_splice(&chain, &ioat->free_desc); - spin_unlock_bh(&ioat->desc_lock); - return NULL; - } - spin_unlock_bh(&ioat->desc_lock); - - desc->txd.flags = flags; - desc->len = total_len; - list_splice(&chain, &desc->tx_list); - hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); - hw->ctl_f.compl_write = 1; - hw->tx_cnt = tx_cnt; - dump_desc_dbg(ioat, desc); - - return &desc->txd; -} - -static void ioat1_cleanup_event(unsigned long data) -{ - struct ioat_dma_chan *ioat = to_ioat_chan((void *) data); - struct ioat_chan_common *chan = &ioat->base; - - ioat1_cleanup(ioat); - if (!test_bit(IOAT_RUN, &chan->state)) - return; - writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); -} - -dma_addr_t ioat_get_current_completion(struct ioat_chan_common *chan) -{ - dma_addr_t phys_complete; - u64 completion; - - completion = *chan->completion; - phys_complete = ioat_chansts_to_addr(completion); - - dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__, - (unsigned long long) phys_complete); - - if (is_ioat_halted(completion)) { - u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); - dev_err(to_dev(chan), "Channel halted, chanerr = %x\n", - chanerr); - - /* TODO do something to salvage the situation */ - } - - return phys_complete; -} - -bool ioat_cleanup_preamble(struct ioat_chan_common *chan, - dma_addr_t *phys_complete) -{ - *phys_complete = ioat_get_current_completion(chan); - if (*phys_complete == chan->last_completion) - return false; - clear_bit(IOAT_COMPLETION_ACK, &chan->state); - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); - - return true; -} - -static void __cleanup(struct ioat_dma_chan *ioat, dma_addr_t phys_complete) -{ - struct ioat_chan_common *chan = &ioat->base; - struct list_head *_desc, *n; - struct dma_async_tx_descriptor *tx; - - dev_dbg(to_dev(chan), "%s: phys_complete: %llx\n", - __func__, (unsigned long long) phys_complete); - list_for_each_safe(_desc, n, &ioat->used_desc) { - struct ioat_desc_sw *desc; - - prefetch(n); - desc = list_entry(_desc, typeof(*desc), node); - tx = &desc->txd; - /* - * Incoming DMA requests may use multiple descriptors, - * due to exceeding xfercap, perhaps. If so, only the - * last one will have a cookie, and require unmapping. - */ - dump_desc_dbg(ioat, desc); - if (tx->cookie) { - dma_cookie_complete(tx); - dma_descriptor_unmap(tx); - ioat->active -= desc->hw->tx_cnt; - if (tx->callback) { - tx->callback(tx->callback_param); - tx->callback = NULL; - } - } - - if (tx->phys != phys_complete) { - /* - * a completed entry, but not the last, so clean - * up if the client is done with the descriptor - */ - if (async_tx_test_ack(tx)) - list_move_tail(&desc->node, &ioat->free_desc); - } else { - /* - * last used desc. Do not remove, so we can - * append from it. - */ - - /* if nothing else is pending, cancel the - * completion timeout - */ - if (n == &ioat->used_desc) { - dev_dbg(to_dev(chan), - "%s cancel completion timeout\n", - __func__); - clear_bit(IOAT_COMPLETION_PENDING, &chan->state); - } - - /* TODO check status bits? */ - break; - } - } - - chan->last_completion = phys_complete; -} - -/** - * ioat1_cleanup - cleanup up finished descriptors - * @chan: ioat channel to be cleaned up - * - * To prevent lock contention we defer cleanup when the locks are - * contended with a terminal timeout that forces cleanup and catches - * completion notification errors. - */ -static void ioat1_cleanup(struct ioat_dma_chan *ioat) -{ - struct ioat_chan_common *chan = &ioat->base; - dma_addr_t phys_complete; - - prefetch(chan->completion); - - if (!spin_trylock_bh(&chan->cleanup_lock)) - return; - - if (!ioat_cleanup_preamble(chan, &phys_complete)) { - spin_unlock_bh(&chan->cleanup_lock); - return; - } - - if (!spin_trylock_bh(&ioat->desc_lock)) { - spin_unlock_bh(&chan->cleanup_lock); - return; - } - - __cleanup(ioat, phys_complete); - - spin_unlock_bh(&ioat->desc_lock); - spin_unlock_bh(&chan->cleanup_lock); -} - -static void ioat1_timer_event(unsigned long data) -{ - struct ioat_dma_chan *ioat = to_ioat_chan((void *) data); - struct ioat_chan_common *chan = &ioat->base; - - dev_dbg(to_dev(chan), "%s: state: %lx\n", __func__, chan->state); - - spin_lock_bh(&chan->cleanup_lock); - if (test_and_clear_bit(IOAT_RESET_PENDING, &chan->state)) { - struct ioat_desc_sw *desc; - - spin_lock_bh(&ioat->desc_lock); - - /* restart active descriptors */ - desc = to_ioat_desc(ioat->used_desc.prev); - ioat_set_chainaddr(ioat, desc->txd.phys); - ioat_start(chan); - - ioat->pending = 0; - set_bit(IOAT_COMPLETION_PENDING, &chan->state); - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); - spin_unlock_bh(&ioat->desc_lock); - } else if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) { - dma_addr_t phys_complete; - - spin_lock_bh(&ioat->desc_lock); - /* if we haven't made progress and we have already - * acknowledged a pending completion once, then be more - * forceful with a restart - */ - if (ioat_cleanup_preamble(chan, &phys_complete)) - __cleanup(ioat, phys_complete); - else if (test_bit(IOAT_COMPLETION_ACK, &chan->state)) - ioat1_reset_channel(ioat); - else { - u64 status = ioat_chansts(chan); - - /* manually update the last completion address */ - if (ioat_chansts_to_addr(status) != 0) - *chan->completion = status; - - set_bit(IOAT_COMPLETION_ACK, &chan->state); - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); - } - spin_unlock_bh(&ioat->desc_lock); - } - spin_unlock_bh(&chan->cleanup_lock); -} - -enum dma_status -ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie, - struct dma_tx_state *txstate) -{ - struct ioat_chan_common *chan = to_chan_common(c); - struct ioatdma_device *device = chan->device; - enum dma_status ret; - - ret = dma_cookie_status(c, cookie, txstate); - if (ret == DMA_COMPLETE) - return ret; - - device->cleanup_fn((unsigned long) c); - - return dma_cookie_status(c, cookie, txstate); -} - -static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat) -{ - struct ioat_chan_common *chan = &ioat->base; - struct ioat_desc_sw *desc; + struct ioat_ring_ent *desc; struct ioat_dma_descriptor *hw; - spin_lock_bh(&ioat->desc_lock); - - desc = ioat1_dma_get_next_descriptor(ioat); - - if (!desc) { - dev_err(to_dev(chan), - "Unable to start null desc - get next desc failed\n"); - spin_unlock_bh(&ioat->desc_lock); + if (ioat_ring_space(ioat_chan) < 1) { + dev_err(to_dev(ioat_chan), + "Unable to start null desc - ring full\n"); return; } + dev_dbg(to_dev(ioat_chan), + "%s: head: %#x tail: %#x issued: %#x\n", + __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued); + desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head); + hw = desc->hw; hw->ctl = 0; hw->ctl_f.null = 1; @@ -787,460 +186,804 @@ static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat) hw->src_addr = 0; hw->dst_addr = 0; async_tx_ack(&desc->txd); - hw->next = 0; - list_add_tail(&desc->node, &ioat->used_desc); - dump_desc_dbg(ioat, desc); - - ioat_set_chainaddr(ioat, desc->txd.phys); - ioat_start(chan); - spin_unlock_bh(&ioat->desc_lock); + ioat_set_chainaddr(ioat_chan, desc->txd.phys); + dump_desc_dbg(ioat_chan, desc); + /* make sure descriptors are written before we submit */ + wmb(); + ioat_chan->head += 1; + __ioat_issue_pending(ioat_chan); } -/* - * Perform a IOAT transaction to verify the HW works. - */ -#define IOAT_TEST_SIZE 2000 - -static void ioat_dma_test_callback(void *dma_async_param) +void ioat_start_null_desc(struct ioatdma_chan *ioat_chan) { - struct completion *cmp = dma_async_param; - - complete(cmp); + spin_lock_bh(&ioat_chan->prep_lock); + __ioat_start_null_desc(ioat_chan); + spin_unlock_bh(&ioat_chan->prep_lock); } -/** - * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works. - * @device: device to be tested - */ -int ioat_dma_self_test(struct ioatdma_device *device) +static void __ioat_restart_chan(struct ioatdma_chan *ioat_chan) { - int i; - u8 *src; - u8 *dest; - struct dma_device *dma = &device->common; - struct device *dev = &device->pdev->dev; - struct dma_chan *dma_chan; - struct dma_async_tx_descriptor *tx; - dma_addr_t dma_dest, dma_src; - dma_cookie_t cookie; + /* set the tail to be re-issued */ + ioat_chan->issued = ioat_chan->tail; + ioat_chan->dmacount = 0; + mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); + + dev_dbg(to_dev(ioat_chan), + "%s: head: %#x tail: %#x issued: %#x count: %#x\n", + __func__, ioat_chan->head, ioat_chan->tail, + ioat_chan->issued, ioat_chan->dmacount); + + if (ioat_ring_pending(ioat_chan)) { + struct ioat_ring_ent *desc; + + desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail); + ioat_set_chainaddr(ioat_chan, desc->txd.phys); + __ioat_issue_pending(ioat_chan); + } else + __ioat_start_null_desc(ioat_chan); +} + +static int ioat_quiesce(struct ioatdma_chan *ioat_chan, unsigned long tmo) +{ + unsigned long end = jiffies + tmo; int err = 0; - struct completion cmp; - unsigned long tmo; - unsigned long flags; + u32 status; - src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL); - if (!src) - return -ENOMEM; - dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL); - if (!dest) { - kfree(src); - return -ENOMEM; + status = ioat_chansts(ioat_chan); + if (is_ioat_active(status) || is_ioat_idle(status)) + ioat_suspend(ioat_chan); + while (is_ioat_active(status) || is_ioat_idle(status)) { + if (tmo && time_after(jiffies, end)) { + err = -ETIMEDOUT; + break; + } + status = ioat_chansts(ioat_chan); + cpu_relax(); } - /* Fill in src buffer */ - for (i = 0; i < IOAT_TEST_SIZE; i++) - src[i] = (u8)i; - - /* Start copy, using first DMA channel */ - dma_chan = container_of(dma->channels.next, struct dma_chan, - device_node); - if (dma->device_alloc_chan_resources(dma_chan) < 1) { - dev_err(dev, "selftest cannot allocate chan resource\n"); - err = -ENODEV; - goto out; - } - - dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE); - if (dma_mapping_error(dev, dma_src)) { - dev_err(dev, "mapping src buffer failed\n"); - goto free_resources; - } - dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE); - if (dma_mapping_error(dev, dma_dest)) { - dev_err(dev, "mapping dest buffer failed\n"); - goto unmap_src; - } - flags = DMA_PREP_INTERRUPT; - tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src, - IOAT_TEST_SIZE, flags); - if (!tx) { - dev_err(dev, "Self-test prep failed, disabling\n"); - err = -ENODEV; - goto unmap_dma; - } - - async_tx_ack(tx); - init_completion(&cmp); - tx->callback = ioat_dma_test_callback; - tx->callback_param = &cmp; - cookie = tx->tx_submit(tx); - if (cookie < 0) { - dev_err(dev, "Self-test setup failed, disabling\n"); - err = -ENODEV; - goto unmap_dma; - } - dma->device_issue_pending(dma_chan); - - tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); - - if (tmo == 0 || - dma->device_tx_status(dma_chan, cookie, NULL) - != DMA_COMPLETE) { - dev_err(dev, "Self-test copy timed out, disabling\n"); - err = -ENODEV; - goto unmap_dma; - } - if (memcmp(src, dest, IOAT_TEST_SIZE)) { - dev_err(dev, "Self-test copy failed compare, disabling\n"); - err = -ENODEV; - goto free_resources; - } - -unmap_dma: - dma_unmap_single(dev, dma_dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE); -unmap_src: - dma_unmap_single(dev, dma_src, IOAT_TEST_SIZE, DMA_TO_DEVICE); -free_resources: - dma->device_free_chan_resources(dma_chan); -out: - kfree(src); - kfree(dest); return err; } -static char ioat_interrupt_style[32] = "msix"; -module_param_string(ioat_interrupt_style, ioat_interrupt_style, - sizeof(ioat_interrupt_style), 0644); -MODULE_PARM_DESC(ioat_interrupt_style, - "set ioat interrupt style: msix (default), msi, intx"); +static int ioat_reset_sync(struct ioatdma_chan *ioat_chan, unsigned long tmo) +{ + unsigned long end = jiffies + tmo; + int err = 0; + + ioat_reset(ioat_chan); + while (ioat_reset_pending(ioat_chan)) { + if (end && time_after(jiffies, end)) { + err = -ETIMEDOUT; + break; + } + cpu_relax(); + } + + return err; +} + +static dma_cookie_t ioat_tx_submit_unlock(struct dma_async_tx_descriptor *tx) + __releases(&ioat_chan->prep_lock) +{ + struct dma_chan *c = tx->chan; + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); + dma_cookie_t cookie; + + cookie = dma_cookie_assign(tx); + dev_dbg(to_dev(ioat_chan), "%s: cookie: %d\n", __func__, cookie); + + if (!test_and_set_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state)) + mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); + + /* make descriptor updates visible before advancing ioat->head, + * this is purposefully not smp_wmb() since we are also + * publishing the descriptor updates to a dma device + */ + wmb(); + + ioat_chan->head += ioat_chan->produce; + + ioat_update_pending(ioat_chan); + spin_unlock_bh(&ioat_chan->prep_lock); + + return cookie; +} + +static struct ioat_ring_ent * +ioat_alloc_ring_ent(struct dma_chan *chan, gfp_t flags) +{ + struct ioat_dma_descriptor *hw; + struct ioat_ring_ent *desc; + struct ioatdma_device *ioat_dma; + dma_addr_t phys; + + ioat_dma = to_ioatdma_device(chan->device); + hw = pci_pool_alloc(ioat_dma->dma_pool, flags, &phys); + if (!hw) + return NULL; + memset(hw, 0, sizeof(*hw)); + + desc = kmem_cache_zalloc(ioat_cache, flags); + if (!desc) { + pci_pool_free(ioat_dma->dma_pool, hw, phys); + return NULL; + } + + dma_async_tx_descriptor_init(&desc->txd, chan); + desc->txd.tx_submit = ioat_tx_submit_unlock; + desc->hw = hw; + desc->txd.phys = phys; + return desc; +} + +void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan) +{ + struct ioatdma_device *ioat_dma; + + ioat_dma = to_ioatdma_device(chan->device); + pci_pool_free(ioat_dma->dma_pool, desc->hw, desc->txd.phys); + kmem_cache_free(ioat_cache, desc); +} + +struct ioat_ring_ent ** +ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags) +{ + struct ioat_ring_ent **ring; + int descs = 1 << order; + int i; + + if (order > ioat_get_max_alloc_order()) + return NULL; + + /* allocate the array to hold the software ring */ + ring = kcalloc(descs, sizeof(*ring), flags); + if (!ring) + return NULL; + for (i = 0; i < descs; i++) { + ring[i] = ioat_alloc_ring_ent(c, flags); + if (!ring[i]) { + while (i--) + ioat_free_ring_ent(ring[i], c); + kfree(ring); + return NULL; + } + set_desc_id(ring[i], i); + } + + /* link descs */ + for (i = 0; i < descs-1; i++) { + struct ioat_ring_ent *next = ring[i+1]; + struct ioat_dma_descriptor *hw = ring[i]->hw; + + hw->next = next->txd.phys; + } + ring[i]->hw->next = ring[0]->txd.phys; + + return ring; +} + +static bool reshape_ring(struct ioatdma_chan *ioat_chan, int order) +{ + /* reshape differs from normal ring allocation in that we want + * to allocate a new software ring while only + * extending/truncating the hardware ring + */ + struct dma_chan *c = &ioat_chan->dma_chan; + const u32 curr_size = ioat_ring_size(ioat_chan); + const u16 active = ioat_ring_active(ioat_chan); + const u32 new_size = 1 << order; + struct ioat_ring_ent **ring; + u32 i; + + if (order > ioat_get_max_alloc_order()) + return false; + + /* double check that we have at least 1 free descriptor */ + if (active == curr_size) + return false; + + /* when shrinking, verify that we can hold the current active + * set in the new ring + */ + if (active >= new_size) + return false; + + /* allocate the array to hold the software ring */ + ring = kcalloc(new_size, sizeof(*ring), GFP_NOWAIT); + if (!ring) + return false; + + /* allocate/trim descriptors as needed */ + if (new_size > curr_size) { + /* copy current descriptors to the new ring */ + for (i = 0; i < curr_size; i++) { + u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1); + u16 new_idx = (ioat_chan->tail+i) & (new_size-1); + + ring[new_idx] = ioat_chan->ring[curr_idx]; + set_desc_id(ring[new_idx], new_idx); + } + + /* add new descriptors to the ring */ + for (i = curr_size; i < new_size; i++) { + u16 new_idx = (ioat_chan->tail+i) & (new_size-1); + + ring[new_idx] = ioat_alloc_ring_ent(c, GFP_NOWAIT); + if (!ring[new_idx]) { + while (i--) { + u16 new_idx = (ioat_chan->tail+i) & + (new_size-1); + + ioat_free_ring_ent(ring[new_idx], c); + } + kfree(ring); + return false; + } + set_desc_id(ring[new_idx], new_idx); + } + + /* hw link new descriptors */ + for (i = curr_size-1; i < new_size; i++) { + u16 new_idx = (ioat_chan->tail+i) & (new_size-1); + struct ioat_ring_ent *next = + ring[(new_idx+1) & (new_size-1)]; + struct ioat_dma_descriptor *hw = ring[new_idx]->hw; + + hw->next = next->txd.phys; + } + } else { + struct ioat_dma_descriptor *hw; + struct ioat_ring_ent *next; + + /* copy current descriptors to the new ring, dropping the + * removed descriptors + */ + for (i = 0; i < new_size; i++) { + u16 curr_idx = (ioat_chan->tail+i) & (curr_size-1); + u16 new_idx = (ioat_chan->tail+i) & (new_size-1); + + ring[new_idx] = ioat_chan->ring[curr_idx]; + set_desc_id(ring[new_idx], new_idx); + } + + /* free deleted descriptors */ + for (i = new_size; i < curr_size; i++) { + struct ioat_ring_ent *ent; + + ent = ioat_get_ring_ent(ioat_chan, ioat_chan->tail+i); + ioat_free_ring_ent(ent, c); + } + + /* fix up hardware ring */ + hw = ring[(ioat_chan->tail+new_size-1) & (new_size-1)]->hw; + next = ring[(ioat_chan->tail+new_size) & (new_size-1)]; + hw->next = next->txd.phys; + } + + dev_dbg(to_dev(ioat_chan), "%s: allocated %d descriptors\n", + __func__, new_size); + + kfree(ioat_chan->ring); + ioat_chan->ring = ring; + ioat_chan->alloc_order = order; + + return true; +} /** - * ioat_dma_setup_interrupts - setup interrupt handler - * @device: ioat device + * ioat_check_space_lock - verify space and grab ring producer lock + * @ioat: ioat,3 channel (ring) to operate on + * @num_descs: allocation length */ -int ioat_dma_setup_interrupts(struct ioatdma_device *device) +int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs) + __acquires(&ioat_chan->prep_lock) { - struct ioat_chan_common *chan; - struct pci_dev *pdev = device->pdev; - struct device *dev = &pdev->dev; - struct msix_entry *msix; - int i, j, msixcnt; - int err = -EINVAL; - u8 intrctrl = 0; + bool retry; - if (!strcmp(ioat_interrupt_style, "msix")) - goto msix; - if (!strcmp(ioat_interrupt_style, "msi")) - goto msi; - if (!strcmp(ioat_interrupt_style, "intx")) - goto intx; - dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style); - goto err_no_irq; + retry: + spin_lock_bh(&ioat_chan->prep_lock); + /* never allow the last descriptor to be consumed, we need at + * least one free at all times to allow for on-the-fly ring + * resizing. + */ + if (likely(ioat_ring_space(ioat_chan) > num_descs)) { + dev_dbg(to_dev(ioat_chan), "%s: num_descs: %d (%x:%x:%x)\n", + __func__, num_descs, ioat_chan->head, + ioat_chan->tail, ioat_chan->issued); + ioat_chan->produce = num_descs; + return 0; /* with ioat->prep_lock held */ + } + retry = test_and_set_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state); + spin_unlock_bh(&ioat_chan->prep_lock); -msix: - /* The number of MSI-X vectors should equal the number of channels */ - msixcnt = device->common.chancnt; - for (i = 0; i < msixcnt; i++) - device->msix_entries[i].entry = i; + /* is another cpu already trying to expand the ring? */ + if (retry) + goto retry; - err = pci_enable_msix_exact(pdev, device->msix_entries, msixcnt); - if (err) - goto msi; + spin_lock_bh(&ioat_chan->cleanup_lock); + spin_lock_bh(&ioat_chan->prep_lock); + retry = reshape_ring(ioat_chan, ioat_chan->alloc_order + 1); + clear_bit(IOAT_RESHAPE_PENDING, &ioat_chan->state); + spin_unlock_bh(&ioat_chan->prep_lock); + spin_unlock_bh(&ioat_chan->cleanup_lock); - for (i = 0; i < msixcnt; i++) { - msix = &device->msix_entries[i]; - chan = ioat_chan_by_index(device, i); - err = devm_request_irq(dev, msix->vector, - ioat_dma_do_interrupt_msix, 0, - "ioat-msix", chan); - if (err) { - for (j = 0; j < i; j++) { - msix = &device->msix_entries[j]; - chan = ioat_chan_by_index(device, j); - devm_free_irq(dev, msix->vector, chan); + /* if we were able to expand the ring retry the allocation */ + if (retry) + goto retry; + + dev_dbg_ratelimited(to_dev(ioat_chan), + "%s: ring full! num_descs: %d (%x:%x:%x)\n", + __func__, num_descs, ioat_chan->head, + ioat_chan->tail, ioat_chan->issued); + + /* progress reclaim in the allocation failure case we may be + * called under bh_disabled so we need to trigger the timer + * event directly + */ + if (time_is_before_jiffies(ioat_chan->timer.expires) + && timer_pending(&ioat_chan->timer)) { + mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); + ioat_timer_event((unsigned long)ioat_chan); + } + + return -ENOMEM; +} + +static bool desc_has_ext(struct ioat_ring_ent *desc) +{ + struct ioat_dma_descriptor *hw = desc->hw; + + if (hw->ctl_f.op == IOAT_OP_XOR || + hw->ctl_f.op == IOAT_OP_XOR_VAL) { + struct ioat_xor_descriptor *xor = desc->xor; + + if (src_cnt_to_sw(xor->ctl_f.src_cnt) > 5) + return true; + } else if (hw->ctl_f.op == IOAT_OP_PQ || + hw->ctl_f.op == IOAT_OP_PQ_VAL) { + struct ioat_pq_descriptor *pq = desc->pq; + + if (src_cnt_to_sw(pq->ctl_f.src_cnt) > 3) + return true; + } + + return false; +} + +static void +ioat_free_sed(struct ioatdma_device *ioat_dma, struct ioat_sed_ent *sed) +{ + if (!sed) + return; + + dma_pool_free(ioat_dma->sed_hw_pool[sed->hw_pool], sed->hw, sed->dma); + kmem_cache_free(ioat_sed_cache, sed); +} + +static u64 ioat_get_current_completion(struct ioatdma_chan *ioat_chan) +{ + u64 phys_complete; + u64 completion; + + completion = *ioat_chan->completion; + phys_complete = ioat_chansts_to_addr(completion); + + dev_dbg(to_dev(ioat_chan), "%s: phys_complete: %#llx\n", __func__, + (unsigned long long) phys_complete); + + return phys_complete; +} + +static bool ioat_cleanup_preamble(struct ioatdma_chan *ioat_chan, + u64 *phys_complete) +{ + *phys_complete = ioat_get_current_completion(ioat_chan); + if (*phys_complete == ioat_chan->last_completion) + return false; + + clear_bit(IOAT_COMPLETION_ACK, &ioat_chan->state); + mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); + + return true; +} + +static void +desc_get_errstat(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc) +{ + struct ioat_dma_descriptor *hw = desc->hw; + + switch (hw->ctl_f.op) { + case IOAT_OP_PQ_VAL: + case IOAT_OP_PQ_VAL_16S: + { + struct ioat_pq_descriptor *pq = desc->pq; + + /* check if there's error written */ + if (!pq->dwbes_f.wbes) + return; + + /* need to set a chanerr var for checking to clear later */ + + if (pq->dwbes_f.p_val_err) + *desc->result |= SUM_CHECK_P_RESULT; + + if (pq->dwbes_f.q_val_err) + *desc->result |= SUM_CHECK_Q_RESULT; + + return; + } + default: + return; + } +} + +/** + * __cleanup - reclaim used descriptors + * @ioat: channel (ring) to clean + */ +static void __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete) +{ + struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; + struct ioat_ring_ent *desc; + bool seen_current = false; + int idx = ioat_chan->tail, i; + u16 active; + + dev_dbg(to_dev(ioat_chan), "%s: head: %#x tail: %#x issued: %#x\n", + __func__, ioat_chan->head, ioat_chan->tail, ioat_chan->issued); + + /* + * At restart of the channel, the completion address and the + * channel status will be 0 due to starting a new chain. Since + * it's new chain and the first descriptor "fails", there is + * nothing to clean up. We do not want to reap the entire submitted + * chain due to this 0 address value and then BUG. + */ + if (!phys_complete) + return; + + active = ioat_ring_active(ioat_chan); + for (i = 0; i < active && !seen_current; i++) { + struct dma_async_tx_descriptor *tx; + + smp_read_barrier_depends(); + prefetch(ioat_get_ring_ent(ioat_chan, idx + i + 1)); + desc = ioat_get_ring_ent(ioat_chan, idx + i); + dump_desc_dbg(ioat_chan, desc); + + /* set err stat if we are using dwbes */ + if (ioat_dma->cap & IOAT_CAP_DWBES) + desc_get_errstat(ioat_chan, desc); + + tx = &desc->txd; + if (tx->cookie) { + dma_cookie_complete(tx); + dma_descriptor_unmap(tx); + if (tx->callback) { + tx->callback(tx->callback_param); + tx->callback = NULL; } - goto msi; + } + + if (tx->phys == phys_complete) + seen_current = true; + + /* skip extended descriptors */ + if (desc_has_ext(desc)) { + BUG_ON(i + 1 >= active); + i++; + } + + /* cleanup super extended descriptors */ + if (desc->sed) { + ioat_free_sed(ioat_dma, desc->sed); + desc->sed = NULL; } } - intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL; - device->irq_mode = IOAT_MSIX; - goto done; -msi: - err = pci_enable_msi(pdev); - if (err) - goto intx; + /* finish all descriptor reads before incrementing tail */ + smp_mb(); + ioat_chan->tail = idx + i; + /* no active descs have written a completion? */ + BUG_ON(active && !seen_current); + ioat_chan->last_completion = phys_complete; - err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0, - "ioat-msi", device); - if (err) { + if (active - i == 0) { + dev_dbg(to_dev(ioat_chan), "%s: cancel completion timeout\n", + __func__); + mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); + } + + /* 5 microsecond delay per pending descriptor */ + writew(min((5 * (active - i)), IOAT_INTRDELAY_MASK), + ioat_chan->ioat_dma->reg_base + IOAT_INTRDELAY_OFFSET); +} + +static void ioat_cleanup(struct ioatdma_chan *ioat_chan) +{ + u64 phys_complete; + + spin_lock_bh(&ioat_chan->cleanup_lock); + + if (ioat_cleanup_preamble(ioat_chan, &phys_complete)) + __cleanup(ioat_chan, phys_complete); + + if (is_ioat_halted(*ioat_chan->completion)) { + u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); + + if (chanerr & IOAT_CHANERR_HANDLE_MASK) { + mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); + ioat_eh(ioat_chan); + } + } + + spin_unlock_bh(&ioat_chan->cleanup_lock); +} + +void ioat_cleanup_event(unsigned long data) +{ + struct ioatdma_chan *ioat_chan = to_ioat_chan((void *)data); + + ioat_cleanup(ioat_chan); + if (!test_bit(IOAT_RUN, &ioat_chan->state)) + return; + writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); +} + +static void ioat_restart_channel(struct ioatdma_chan *ioat_chan) +{ + u64 phys_complete; + + ioat_quiesce(ioat_chan, 0); + if (ioat_cleanup_preamble(ioat_chan, &phys_complete)) + __cleanup(ioat_chan, phys_complete); + + __ioat_restart_chan(ioat_chan); +} + +static void ioat_eh(struct ioatdma_chan *ioat_chan) +{ + struct pci_dev *pdev = to_pdev(ioat_chan); + struct ioat_dma_descriptor *hw; + struct dma_async_tx_descriptor *tx; + u64 phys_complete; + struct ioat_ring_ent *desc; + u32 err_handled = 0; + u32 chanerr_int; + u32 chanerr; + + /* cleanup so tail points to descriptor that caused the error */ + if (ioat_cleanup_preamble(ioat_chan, &phys_complete)) + __cleanup(ioat_chan, phys_complete); + + chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); + pci_read_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, &chanerr_int); + + dev_dbg(to_dev(ioat_chan), "%s: error = %x:%x\n", + __func__, chanerr, chanerr_int); + + desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail); + hw = desc->hw; + dump_desc_dbg(ioat_chan, desc); + + switch (hw->ctl_f.op) { + case IOAT_OP_XOR_VAL: + if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) { + *desc->result |= SUM_CHECK_P_RESULT; + err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR; + } + break; + case IOAT_OP_PQ_VAL: + case IOAT_OP_PQ_VAL_16S: + if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) { + *desc->result |= SUM_CHECK_P_RESULT; + err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR; + } + if (chanerr & IOAT_CHANERR_XOR_Q_ERR) { + *desc->result |= SUM_CHECK_Q_RESULT; + err_handled |= IOAT_CHANERR_XOR_Q_ERR; + } + break; + } + + /* fault on unhandled error or spurious halt */ + if (chanerr ^ err_handled || chanerr == 0) { + dev_err(to_dev(ioat_chan), "%s: fatal error (%x:%x)\n", + __func__, chanerr, err_handled); + BUG(); + } else { /* cleanup the faulty descriptor */ + tx = &desc->txd; + if (tx->cookie) { + dma_cookie_complete(tx); + dma_descriptor_unmap(tx); + if (tx->callback) { + tx->callback(tx->callback_param); + tx->callback = NULL; + } + } + } + + writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET); + pci_write_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, chanerr_int); + + /* mark faulting descriptor as complete */ + *ioat_chan->completion = desc->txd.phys; + + spin_lock_bh(&ioat_chan->prep_lock); + ioat_restart_channel(ioat_chan); + spin_unlock_bh(&ioat_chan->prep_lock); +} + +static void check_active(struct ioatdma_chan *ioat_chan) +{ + if (ioat_ring_active(ioat_chan)) { + mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); + return; + } + + if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state)) + mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); + else if (ioat_chan->alloc_order > ioat_get_alloc_order()) { + /* if the ring is idle, empty, and oversized try to step + * down the size + */ + reshape_ring(ioat_chan, ioat_chan->alloc_order - 1); + + /* keep shrinking until we get back to our minimum + * default size + */ + if (ioat_chan->alloc_order > ioat_get_alloc_order()) + mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); + } + +} + +void ioat_timer_event(unsigned long data) +{ + struct ioatdma_chan *ioat_chan = to_ioat_chan((void *)data); + dma_addr_t phys_complete; + u64 status; + + status = ioat_chansts(ioat_chan); + + /* when halted due to errors check for channel + * programming errors before advancing the completion state + */ + if (is_ioat_halted(status)) { + u32 chanerr; + + chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); + dev_err(to_dev(ioat_chan), "%s: Channel halted (%x)\n", + __func__, chanerr); + if (test_bit(IOAT_RUN, &ioat_chan->state)) + BUG_ON(is_ioat_bug(chanerr)); + else /* we never got off the ground */ + return; + } + + /* if we haven't made progress and we have already + * acknowledged a pending completion once, then be more + * forceful with a restart + */ + spin_lock_bh(&ioat_chan->cleanup_lock); + if (ioat_cleanup_preamble(ioat_chan, &phys_complete)) + __cleanup(ioat_chan, phys_complete); + else if (test_bit(IOAT_COMPLETION_ACK, &ioat_chan->state)) { + spin_lock_bh(&ioat_chan->prep_lock); + ioat_restart_channel(ioat_chan); + spin_unlock_bh(&ioat_chan->prep_lock); + spin_unlock_bh(&ioat_chan->cleanup_lock); + return; + } else { + set_bit(IOAT_COMPLETION_ACK, &ioat_chan->state); + mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); + } + + + if (ioat_ring_active(ioat_chan)) + mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); + else { + spin_lock_bh(&ioat_chan->prep_lock); + check_active(ioat_chan); + spin_unlock_bh(&ioat_chan->prep_lock); + } + spin_unlock_bh(&ioat_chan->cleanup_lock); +} + +enum dma_status +ioat_tx_status(struct dma_chan *c, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); + enum dma_status ret; + + ret = dma_cookie_status(c, cookie, txstate); + if (ret == DMA_COMPLETE) + return ret; + + ioat_cleanup(ioat_chan); + + return dma_cookie_status(c, cookie, txstate); +} + +static int ioat_irq_reinit(struct ioatdma_device *ioat_dma) +{ + struct pci_dev *pdev = ioat_dma->pdev; + int irq = pdev->irq, i; + + if (!is_bwd_ioat(pdev)) + return 0; + + switch (ioat_dma->irq_mode) { + case IOAT_MSIX: + for (i = 0; i < ioat_dma->dma_dev.chancnt; i++) { + struct msix_entry *msix = &ioat_dma->msix_entries[i]; + struct ioatdma_chan *ioat_chan; + + ioat_chan = ioat_chan_by_index(ioat_dma, i); + devm_free_irq(&pdev->dev, msix->vector, ioat_chan); + } + + pci_disable_msix(pdev); + break; + case IOAT_MSI: pci_disable_msi(pdev); - goto intx; + /* fall through */ + case IOAT_INTX: + devm_free_irq(&pdev->dev, irq, ioat_dma); + break; + default: + return 0; } - device->irq_mode = IOAT_MSI; - goto done; + ioat_dma->irq_mode = IOAT_NOIRQ; -intx: - err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, - IRQF_SHARED, "ioat-intx", device); - if (err) - goto err_no_irq; - - device->irq_mode = IOAT_INTX; -done: - if (device->intr_quirk) - device->intr_quirk(device); - intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN; - writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET); - return 0; - -err_no_irq: - /* Disable all interrupt generation */ - writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET); - device->irq_mode = IOAT_NOIRQ; - dev_err(dev, "no usable interrupts\n"); - return err; -} -EXPORT_SYMBOL(ioat_dma_setup_interrupts); - -static void ioat_disable_interrupts(struct ioatdma_device *device) -{ - /* Disable all interrupt generation */ - writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET); + return ioat_dma_setup_interrupts(ioat_dma); } -int ioat_probe(struct ioatdma_device *device) +int ioat_reset_hw(struct ioatdma_chan *ioat_chan) { - int err = -ENODEV; - struct dma_device *dma = &device->common; - struct pci_dev *pdev = device->pdev; - struct device *dev = &pdev->dev; - - /* DMA coherent memory pool for DMA descriptor allocations */ - device->dma_pool = pci_pool_create("dma_desc_pool", pdev, - sizeof(struct ioat_dma_descriptor), - 64, 0); - if (!device->dma_pool) { - err = -ENOMEM; - goto err_dma_pool; - } - - device->completion_pool = pci_pool_create("completion_pool", pdev, - sizeof(u64), SMP_CACHE_BYTES, - SMP_CACHE_BYTES); - - if (!device->completion_pool) { - err = -ENOMEM; - goto err_completion_pool; - } - - device->enumerate_channels(device); - - dma_cap_set(DMA_MEMCPY, dma->cap_mask); - dma->dev = &pdev->dev; - - if (!dma->chancnt) { - dev_err(dev, "channel enumeration error\n"); - goto err_setup_interrupts; - } - - err = ioat_dma_setup_interrupts(device); - if (err) - goto err_setup_interrupts; - - err = device->self_test(device); - if (err) - goto err_self_test; - - return 0; - -err_self_test: - ioat_disable_interrupts(device); -err_setup_interrupts: - pci_pool_destroy(device->completion_pool); -err_completion_pool: - pci_pool_destroy(device->dma_pool); -err_dma_pool: - return err; -} - -int ioat_register(struct ioatdma_device *device) -{ - int err = dma_async_device_register(&device->common); - - if (err) { - ioat_disable_interrupts(device); - pci_pool_destroy(device->completion_pool); - pci_pool_destroy(device->dma_pool); - } - - return err; -} - -/* ioat1_intr_quirk - fix up dma ctrl register to enable / disable msi */ -static void ioat1_intr_quirk(struct ioatdma_device *device) -{ - struct pci_dev *pdev = device->pdev; - u32 dmactrl; - - pci_read_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, &dmactrl); - if (pdev->msi_enabled) - dmactrl |= IOAT_PCI_DMACTRL_MSI_EN; - else - dmactrl &= ~IOAT_PCI_DMACTRL_MSI_EN; - pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl); -} - -static ssize_t ring_size_show(struct dma_chan *c, char *page) -{ - struct ioat_dma_chan *ioat = to_ioat_chan(c); - - return sprintf(page, "%d\n", ioat->desccount); -} -static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size); - -static ssize_t ring_active_show(struct dma_chan *c, char *page) -{ - struct ioat_dma_chan *ioat = to_ioat_chan(c); - - return sprintf(page, "%d\n", ioat->active); -} -static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active); - -static ssize_t cap_show(struct dma_chan *c, char *page) -{ - struct dma_device *dma = c->device; - - return sprintf(page, "copy%s%s%s%s%s\n", - dma_has_cap(DMA_PQ, dma->cap_mask) ? " pq" : "", - dma_has_cap(DMA_PQ_VAL, dma->cap_mask) ? " pq_val" : "", - dma_has_cap(DMA_XOR, dma->cap_mask) ? " xor" : "", - dma_has_cap(DMA_XOR_VAL, dma->cap_mask) ? " xor_val" : "", - dma_has_cap(DMA_INTERRUPT, dma->cap_mask) ? " intr" : ""); - -} -struct ioat_sysfs_entry ioat_cap_attr = __ATTR_RO(cap); - -static ssize_t version_show(struct dma_chan *c, char *page) -{ - struct dma_device *dma = c->device; - struct ioatdma_device *device = to_ioatdma_device(dma); - - return sprintf(page, "%d.%d\n", - device->version >> 4, device->version & 0xf); -} -struct ioat_sysfs_entry ioat_version_attr = __ATTR_RO(version); - -static struct attribute *ioat1_attrs[] = { - &ring_size_attr.attr, - &ring_active_attr.attr, - &ioat_cap_attr.attr, - &ioat_version_attr.attr, - NULL, -}; - -static ssize_t -ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page) -{ - struct ioat_sysfs_entry *entry; - struct ioat_chan_common *chan; - - entry = container_of(attr, struct ioat_sysfs_entry, attr); - chan = container_of(kobj, struct ioat_chan_common, kobj); - - if (!entry->show) - return -EIO; - return entry->show(&chan->common, page); -} - -const struct sysfs_ops ioat_sysfs_ops = { - .show = ioat_attr_show, -}; - -static struct kobj_type ioat1_ktype = { - .sysfs_ops = &ioat_sysfs_ops, - .default_attrs = ioat1_attrs, -}; - -void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type) -{ - struct dma_device *dma = &device->common; - struct dma_chan *c; - - list_for_each_entry(c, &dma->channels, device_node) { - struct ioat_chan_common *chan = to_chan_common(c); - struct kobject *parent = &c->dev->device.kobj; - int err; - - err = kobject_init_and_add(&chan->kobj, type, parent, "quickdata"); - if (err) { - dev_warn(to_dev(chan), - "sysfs init error (%d), continuing...\n", err); - kobject_put(&chan->kobj); - set_bit(IOAT_KOBJ_INIT_FAIL, &chan->state); - } - } -} - -void ioat_kobject_del(struct ioatdma_device *device) -{ - struct dma_device *dma = &device->common; - struct dma_chan *c; - - list_for_each_entry(c, &dma->channels, device_node) { - struct ioat_chan_common *chan = to_chan_common(c); - - if (!test_bit(IOAT_KOBJ_INIT_FAIL, &chan->state)) { - kobject_del(&chan->kobj); - kobject_put(&chan->kobj); - } - } -} - -int ioat1_dma_probe(struct ioatdma_device *device, int dca) -{ - struct pci_dev *pdev = device->pdev; - struct dma_device *dma; + /* throw away whatever the channel was doing and get it + * initialized, with ioat3 specific workarounds + */ + struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; + struct pci_dev *pdev = ioat_dma->pdev; + u32 chanerr; + u16 dev_id; int err; - device->intr_quirk = ioat1_intr_quirk; - device->enumerate_channels = ioat1_enumerate_channels; - device->self_test = ioat_dma_self_test; - device->timer_fn = ioat1_timer_event; - device->cleanup_fn = ioat1_cleanup_event; - dma = &device->common; - dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy; - dma->device_issue_pending = ioat1_dma_memcpy_issue_pending; - dma->device_alloc_chan_resources = ioat1_dma_alloc_chan_resources; - dma->device_free_chan_resources = ioat1_dma_free_chan_resources; - dma->device_tx_status = ioat_dma_tx_status; + ioat_quiesce(ioat_chan, msecs_to_jiffies(100)); - err = ioat_probe(device); - if (err) - return err; - err = ioat_register(device); - if (err) - return err; - ioat_kobject_add(device, &ioat1_ktype); + chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); + writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET); - if (dca) - device->dca = ioat_dca_init(pdev, device->reg_base); + if (ioat_dma->version < IOAT_VER_3_3) { + /* clear any pending errors */ + err = pci_read_config_dword(pdev, + IOAT_PCI_CHANERR_INT_OFFSET, &chanerr); + if (err) { + dev_err(&pdev->dev, + "channel error register unreachable\n"); + return err; + } + pci_write_config_dword(pdev, + IOAT_PCI_CHANERR_INT_OFFSET, chanerr); + + /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit + * (workaround for spurious config parity error after restart) + */ + pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id); + if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0) { + pci_write_config_dword(pdev, + IOAT_PCI_DMAUNCERRSTS_OFFSET, + 0x10); + } + } + + err = ioat_reset_sync(ioat_chan, msecs_to_jiffies(200)); + if (!err) + err = ioat_irq_reinit(ioat_dma); + + if (err) + dev_err(&pdev->dev, "Failed to reset: %d\n", err); return err; } - -void ioat_dma_remove(struct ioatdma_device *device) -{ - struct dma_device *dma = &device->common; - - ioat_disable_interrupts(device); - - ioat_kobject_del(device); - - dma_async_device_unregister(dma); - - pci_pool_destroy(device->dma_pool); - pci_pool_destroy(device->completion_pool); - - INIT_LIST_HEAD(&dma->channels); -} diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index 30f5c7eede16..1bc084986646 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h @@ -18,26 +18,32 @@ #define IOATDMA_H #include -#include "hw.h" -#include "registers.h" #include #include #include #include -#include +#include +#include +#include "registers.h" +#include "hw.h" #define IOAT_DMA_VERSION "4.00" -#define IOAT_LOW_COMPLETION_MASK 0xffffffc0 #define IOAT_DMA_DCA_ANY_CPU ~0 -#define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, common) -#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node) -#define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, txd) -#define to_dev(ioat_chan) (&(ioat_chan)->device->pdev->dev) -#define to_pdev(ioat_chan) ((ioat_chan)->device->pdev) +#define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, dma_dev) +#define to_dev(ioat_chan) (&(ioat_chan)->ioat_dma->pdev->dev) +#define to_pdev(ioat_chan) ((ioat_chan)->ioat_dma->pdev) -#define chan_num(ch) ((int)((ch)->reg_base - (ch)->device->reg_base) / 0x80) +#define chan_num(ch) ((int)((ch)->reg_base - (ch)->ioat_dma->reg_base) / 0x80) + +/* ioat hardware assumes at least two sources for raid operations */ +#define src_cnt_to_sw(x) ((x) + 2) +#define src_cnt_to_hw(x) ((x) - 2) +#define ndest_to_sw(x) ((x) + 1) +#define ndest_to_hw(x) ((x) - 1) +#define src16_cnt_to_sw(x) ((x) + 9) +#define src16_cnt_to_hw(x) ((x) - 9) /* * workaround for IOAT ver.3.0 null descriptor issue @@ -57,19 +63,15 @@ enum ioat_irq_mode { * @pdev: PCI-Express device * @reg_base: MMIO register space base address * @dma_pool: for allocating DMA descriptors - * @common: embedded struct dma_device + * @completion_pool: DMA buffers for completion ops + * @sed_hw_pool: DMA super descriptor pools + * @dma_dev: embedded struct dma_device * @version: version of ioatdma device * @msix_entries: irq handlers * @idx: per channel data * @dca: direct cache access context - * @intr_quirk: interrupt setup quirk (for ioat_v1 devices) - * @enumerate_channels: hw version specific channel enumeration - * @reset_hw: hw version specific channel (re)initialization - * @cleanup_fn: select between the v2 and v3 cleanup routines - * @timer_fn: select between the v2 and v3 timer watchdog routines - * @self_test: hardware version specific self test for each supported op type - * - * Note: the v3 cleanup routine supports raid operations + * @irq_mode: interrupt mode (INTX, MSI, MSIX) + * @cap: read DMA capabilities register */ struct ioatdma_device { struct pci_dev *pdev; @@ -78,28 +80,21 @@ struct ioatdma_device { struct pci_pool *completion_pool; #define MAX_SED_POOLS 5 struct dma_pool *sed_hw_pool[MAX_SED_POOLS]; - struct dma_device common; + struct dma_device dma_dev; u8 version; struct msix_entry msix_entries[4]; - struct ioat_chan_common *idx[4]; + struct ioatdma_chan *idx[4]; struct dca_provider *dca; enum ioat_irq_mode irq_mode; u32 cap; - void (*intr_quirk)(struct ioatdma_device *device); - int (*enumerate_channels)(struct ioatdma_device *device); - int (*reset_hw)(struct ioat_chan_common *chan); - void (*cleanup_fn)(unsigned long data); - void (*timer_fn)(unsigned long data); - int (*self_test)(struct ioatdma_device *device); }; -struct ioat_chan_common { - struct dma_chan common; +struct ioatdma_chan { + struct dma_chan dma_chan; void __iomem *reg_base; dma_addr_t last_completion; spinlock_t cleanup_lock; unsigned long state; - #define IOAT_COMPLETION_PENDING 0 #define IOAT_COMPLETION_ACK 1 #define IOAT_RESET_PENDING 2 #define IOAT_KOBJ_INIT_FAIL 3 @@ -110,11 +105,32 @@ struct ioat_chan_common { #define COMPLETION_TIMEOUT msecs_to_jiffies(100) #define IDLE_TIMEOUT msecs_to_jiffies(2000) #define RESET_DELAY msecs_to_jiffies(100) - struct ioatdma_device *device; + struct ioatdma_device *ioat_dma; dma_addr_t completion_dma; u64 *completion; struct tasklet_struct cleanup_task; struct kobject kobj; + +/* ioat v2 / v3 channel attributes + * @xfercap_log; log2 of channel max transfer length (for fast division) + * @head: allocated index + * @issued: hardware notification point + * @tail: cleanup index + * @dmacount: identical to 'head' except for occasionally resetting to zero + * @alloc_order: log2 of the number of allocated descriptors + * @produce: number of descriptors to produce at submit time + * @ring: software ring buffer implementation of hardware ring + * @prep_lock: serializes descriptor preparation (producers) + */ + size_t xfercap_log; + u16 head; + u16 issued; + u16 tail; + u16 dmacount; + u16 alloc_order; + u16 produce; + struct ioat_ring_ent **ring; + spinlock_t prep_lock; }; struct ioat_sysfs_entry { @@ -122,29 +138,12 @@ struct ioat_sysfs_entry { ssize_t (*show)(struct dma_chan *, char *); }; -/** - * struct ioat_dma_chan - internal representation of a DMA channel - */ -struct ioat_dma_chan { - struct ioat_chan_common base; - - size_t xfercap; /* XFERCAP register value expanded out */ - - spinlock_t desc_lock; - struct list_head free_desc; - struct list_head used_desc; - - int pending; - u16 desccount; - u16 active; -}; - /** * struct ioat_sed_ent - wrapper around super extended hardware descriptor * @hw: hardware SED - * @sed_dma: dma address for the SED - * @list: list member + * @dma: dma address for the SED * @parent: point to the dma descriptor that's the parent + * @hw_pool: descriptor pool index */ struct ioat_sed_ent { struct ioat_sed_raw_descriptor *hw; @@ -153,39 +152,57 @@ struct ioat_sed_ent { unsigned int hw_pool; }; -static inline struct ioat_chan_common *to_chan_common(struct dma_chan *c) -{ - return container_of(c, struct ioat_chan_common, common); -} - -static inline struct ioat_dma_chan *to_ioat_chan(struct dma_chan *c) -{ - struct ioat_chan_common *chan = to_chan_common(c); - - return container_of(chan, struct ioat_dma_chan, base); -} - -/* wrapper around hardware descriptor format + additional software fields */ - /** - * struct ioat_desc_sw - wrapper around hardware descriptor + * struct ioat_ring_ent - wrapper around hardware descriptor * @hw: hardware DMA descriptor (for memcpy) - * @node: this descriptor will either be on the free list, - * or attached to a transaction list (tx_list) + * @xor: hardware xor descriptor + * @xor_ex: hardware xor extension descriptor + * @pq: hardware pq descriptor + * @pq_ex: hardware pq extension descriptor + * @pqu: hardware pq update descriptor + * @raw: hardware raw (un-typed) descriptor * @txd: the generic software descriptor for all engines + * @len: total transaction length for unmap + * @result: asynchronous result of validate operations * @id: identifier for debug + * @sed: pointer to super extended descriptor sw desc */ -struct ioat_desc_sw { - struct ioat_dma_descriptor *hw; - struct list_head node; + +struct ioat_ring_ent { + union { + struct ioat_dma_descriptor *hw; + struct ioat_xor_descriptor *xor; + struct ioat_xor_ext_descriptor *xor_ex; + struct ioat_pq_descriptor *pq; + struct ioat_pq_ext_descriptor *pq_ex; + struct ioat_pq_update_descriptor *pqu; + struct ioat_raw_descriptor *raw; + }; size_t len; - struct list_head tx_list; struct dma_async_tx_descriptor txd; + enum sum_check_flags *result; #ifdef DEBUG int id; #endif + struct ioat_sed_ent *sed; }; +extern const struct sysfs_ops ioat_sysfs_ops; +extern struct ioat_sysfs_entry ioat_version_attr; +extern struct ioat_sysfs_entry ioat_cap_attr; +extern int ioat_pending_level; +extern int ioat_ring_alloc_order; +extern struct kobj_type ioat_ktype; +extern struct kmem_cache *ioat_cache; +extern int ioat_ring_max_alloc_order; +extern struct kmem_cache *ioat_sed_cache; + +static inline struct ioatdma_chan *to_ioat_chan(struct dma_chan *c) +{ + return container_of(c, struct ioatdma_chan, dma_chan); +} + +/* wrapper around hardware descriptor format + additional software fields */ #ifdef DEBUG #define set_desc_id(desc, i) ((desc)->id = (i)) #define desc_id(desc) ((desc)->id) @@ -195,10 +212,10 @@ struct ioat_desc_sw { #endif static inline void -__dump_desc_dbg(struct ioat_chan_common *chan, struct ioat_dma_descriptor *hw, +__dump_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_dma_descriptor *hw, struct dma_async_tx_descriptor *tx, int id) { - struct device *dev = to_dev(chan); + struct device *dev = to_dev(ioat_chan); dev_dbg(dev, "desc[%d]: (%#llx->%#llx) cookie: %d flags: %#x" " ctl: %#10.8x (op: %#x int_en: %d compl: %d)\n", id, @@ -208,25 +225,25 @@ __dump_desc_dbg(struct ioat_chan_common *chan, struct ioat_dma_descriptor *hw, } #define dump_desc_dbg(c, d) \ - ({ if (d) __dump_desc_dbg(&c->base, d->hw, &d->txd, desc_id(d)); 0; }) + ({ if (d) __dump_desc_dbg(c, d->hw, &d->txd, desc_id(d)); 0; }) -static inline struct ioat_chan_common * -ioat_chan_by_index(struct ioatdma_device *device, int index) +static inline struct ioatdma_chan * +ioat_chan_by_index(struct ioatdma_device *ioat_dma, int index) { - return device->idx[index]; + return ioat_dma->idx[index]; } -static inline u64 ioat_chansts_32(struct ioat_chan_common *chan) +static inline u64 ioat_chansts_32(struct ioatdma_chan *ioat_chan) { - u8 ver = chan->device->version; + u8 ver = ioat_chan->ioat_dma->version; u64 status; u32 status_lo; /* We need to read the low address first as this causes the * chipset to latch the upper bits for the subsequent read */ - status_lo = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_LOW(ver)); - status = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_HIGH(ver)); + status_lo = readl(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET_LOW(ver)); + status = readl(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET_HIGH(ver)); status <<= 32; status |= status_lo; @@ -235,16 +252,16 @@ static inline u64 ioat_chansts_32(struct ioat_chan_common *chan) #if BITS_PER_LONG == 64 -static inline u64 ioat_chansts(struct ioat_chan_common *chan) +static inline u64 ioat_chansts(struct ioatdma_chan *ioat_chan) { - u8 ver = chan->device->version; + u8 ver = ioat_chan->ioat_dma->version; u64 status; /* With IOAT v3.3 the status register is 64bit. */ if (ver >= IOAT_VER_3_3) - status = readq(chan->reg_base + IOAT_CHANSTS_OFFSET(ver)); + status = readq(ioat_chan->reg_base + IOAT_CHANSTS_OFFSET(ver)); else - status = ioat_chansts_32(chan); + status = ioat_chansts_32(ioat_chan); return status; } @@ -253,56 +270,41 @@ static inline u64 ioat_chansts(struct ioat_chan_common *chan) #define ioat_chansts ioat_chansts_32 #endif -static inline void ioat_start(struct ioat_chan_common *chan) -{ - u8 ver = chan->device->version; - - writeb(IOAT_CHANCMD_START, chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); -} - static inline u64 ioat_chansts_to_addr(u64 status) { return status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR; } -static inline u32 ioat_chanerr(struct ioat_chan_common *chan) +static inline u32 ioat_chanerr(struct ioatdma_chan *ioat_chan) { - return readl(chan->reg_base + IOAT_CHANERR_OFFSET); + return readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); } -static inline void ioat_suspend(struct ioat_chan_common *chan) +static inline void ioat_suspend(struct ioatdma_chan *ioat_chan) { - u8 ver = chan->device->version; + u8 ver = ioat_chan->ioat_dma->version; - writeb(IOAT_CHANCMD_SUSPEND, chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); + writeb(IOAT_CHANCMD_SUSPEND, + ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); } -static inline void ioat_reset(struct ioat_chan_common *chan) +static inline void ioat_reset(struct ioatdma_chan *ioat_chan) { - u8 ver = chan->device->version; + u8 ver = ioat_chan->ioat_dma->version; - writeb(IOAT_CHANCMD_RESET, chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); + writeb(IOAT_CHANCMD_RESET, + ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); } -static inline bool ioat_reset_pending(struct ioat_chan_common *chan) +static inline bool ioat_reset_pending(struct ioatdma_chan *ioat_chan) { - u8 ver = chan->device->version; + u8 ver = ioat_chan->ioat_dma->version; u8 cmd; - cmd = readb(chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); + cmd = readb(ioat_chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); return (cmd & IOAT_CHANCMD_RESET) == IOAT_CHANCMD_RESET; } -static inline void ioat_set_chainaddr(struct ioat_dma_chan *ioat, u64 addr) -{ - struct ioat_chan_common *chan = &ioat->base; - - writel(addr & 0x00000000FFFFFFFF, - chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW); - writel(addr >> 32, - chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH); -} - static inline bool is_ioat_active(unsigned long status) { return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_ACTIVE); @@ -329,24 +331,111 @@ static inline bool is_ioat_bug(unsigned long err) return !!err; } -int ioat_probe(struct ioatdma_device *device); -int ioat_register(struct ioatdma_device *device); -int ioat1_dma_probe(struct ioatdma_device *dev, int dca); -int ioat_dma_self_test(struct ioatdma_device *device); -void ioat_dma_remove(struct ioatdma_device *device); +#define IOAT_MAX_ORDER 16 +#define ioat_get_alloc_order() \ + (min(ioat_ring_alloc_order, IOAT_MAX_ORDER)) +#define ioat_get_max_alloc_order() \ + (min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER)) + +static inline u32 ioat_ring_size(struct ioatdma_chan *ioat_chan) +{ + return 1 << ioat_chan->alloc_order; +} + +/* count of descriptors in flight with the engine */ +static inline u16 ioat_ring_active(struct ioatdma_chan *ioat_chan) +{ + return CIRC_CNT(ioat_chan->head, ioat_chan->tail, + ioat_ring_size(ioat_chan)); +} + +/* count of descriptors pending submission to hardware */ +static inline u16 ioat_ring_pending(struct ioatdma_chan *ioat_chan) +{ + return CIRC_CNT(ioat_chan->head, ioat_chan->issued, + ioat_ring_size(ioat_chan)); +} + +static inline u32 ioat_ring_space(struct ioatdma_chan *ioat_chan) +{ + return ioat_ring_size(ioat_chan) - ioat_ring_active(ioat_chan); +} + +static inline u16 +ioat_xferlen_to_descs(struct ioatdma_chan *ioat_chan, size_t len) +{ + u16 num_descs = len >> ioat_chan->xfercap_log; + + num_descs += !!(len & ((1 << ioat_chan->xfercap_log) - 1)); + return num_descs; +} + +static inline struct ioat_ring_ent * +ioat_get_ring_ent(struct ioatdma_chan *ioat_chan, u16 idx) +{ + return ioat_chan->ring[idx & (ioat_ring_size(ioat_chan) - 1)]; +} + +static inline void +ioat_set_chainaddr(struct ioatdma_chan *ioat_chan, u64 addr) +{ + writel(addr & 0x00000000FFFFFFFF, + ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW); + writel(addr >> 32, + ioat_chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH); +} + +/* IOAT Prep functions */ +struct dma_async_tx_descriptor * +ioat_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, + dma_addr_t dma_src, size_t len, unsigned long flags); +struct dma_async_tx_descriptor * +ioat_prep_interrupt_lock(struct dma_chan *c, unsigned long flags); +struct dma_async_tx_descriptor * +ioat_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, + unsigned int src_cnt, size_t len, unsigned long flags); +struct dma_async_tx_descriptor * +ioat_prep_xor_val(struct dma_chan *chan, dma_addr_t *src, + unsigned int src_cnt, size_t len, + enum sum_check_flags *result, unsigned long flags); +struct dma_async_tx_descriptor * +ioat_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, + unsigned int src_cnt, const unsigned char *scf, size_t len, + unsigned long flags); +struct dma_async_tx_descriptor * +ioat_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, + unsigned int src_cnt, const unsigned char *scf, size_t len, + enum sum_check_flags *pqres, unsigned long flags); +struct dma_async_tx_descriptor * +ioat_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src, + unsigned int src_cnt, size_t len, unsigned long flags); +struct dma_async_tx_descriptor * +ioat_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src, + unsigned int src_cnt, size_t len, + enum sum_check_flags *result, unsigned long flags); + +/* IOAT Operation functions */ +irqreturn_t ioat_dma_do_interrupt(int irq, void *data); +irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data); +struct ioat_ring_ent ** +ioat_alloc_ring(struct dma_chan *c, int order, gfp_t flags); +void ioat_start_null_desc(struct ioatdma_chan *ioat_chan); +void ioat_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan); +int ioat_reset_hw(struct ioatdma_chan *ioat_chan); +enum dma_status +ioat_tx_status(struct dma_chan *c, dma_cookie_t cookie, + struct dma_tx_state *txstate); +void ioat_cleanup_event(unsigned long data); +void ioat_timer_event(unsigned long data); +int ioat_check_space_lock(struct ioatdma_chan *ioat_chan, int num_descs); +void ioat_issue_pending(struct dma_chan *chan); +void ioat_timer_event(unsigned long data); + +/* IOAT Init functions */ +bool is_bwd_ioat(struct pci_dev *pdev); struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase); -dma_addr_t ioat_get_current_completion(struct ioat_chan_common *chan); -void ioat_init_channel(struct ioatdma_device *device, - struct ioat_chan_common *chan, int idx); -enum dma_status ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie, - struct dma_tx_state *txstate); -bool ioat_cleanup_preamble(struct ioat_chan_common *chan, - dma_addr_t *phys_complete); -void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type); -void ioat_kobject_del(struct ioatdma_device *device); -int ioat_dma_setup_interrupts(struct ioatdma_device *device); -void ioat_stop(struct ioat_chan_common *chan); -extern const struct sysfs_ops ioat_sysfs_ops; -extern struct ioat_sysfs_entry ioat_version_attr; -extern struct ioat_sysfs_entry ioat_cap_attr; +void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type); +void ioat_kobject_del(struct ioatdma_device *ioat_dma); +int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma); +void ioat_stop(struct ioatdma_chan *ioat_chan); #endif /* IOATDMA_H */ diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c deleted file mode 100644 index 69c7dfcad023..000000000000 --- a/drivers/dma/ioat/dma_v2.c +++ /dev/null @@ -1,916 +0,0 @@ -/* - * Intel I/OAT DMA Linux driver - * Copyright(c) 2004 - 2009 Intel Corporation. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * The full GNU General Public License is included in this distribution in - * the file called "COPYING". - * - */ - -/* - * This driver supports an Intel I/OAT DMA engine (versions >= 2), which - * does asynchronous data movement and checksumming operations. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "dma.h" -#include "dma_v2.h" -#include "registers.h" -#include "hw.h" - -#include "../dmaengine.h" - -int ioat_ring_alloc_order = 8; -module_param(ioat_ring_alloc_order, int, 0644); -MODULE_PARM_DESC(ioat_ring_alloc_order, - "ioat2+: allocate 2^n descriptors per channel" - " (default: 8 max: 16)"); -static int ioat_ring_max_alloc_order = IOAT_MAX_ORDER; -module_param(ioat_ring_max_alloc_order, int, 0644); -MODULE_PARM_DESC(ioat_ring_max_alloc_order, - "ioat2+: upper limit for ring size (default: 16)"); - -void __ioat2_issue_pending(struct ioat2_dma_chan *ioat) -{ - struct ioat_chan_common *chan = &ioat->base; - - ioat->dmacount += ioat2_ring_pending(ioat); - ioat->issued = ioat->head; - writew(ioat->dmacount, chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET); - dev_dbg(to_dev(chan), - "%s: head: %#x tail: %#x issued: %#x count: %#x\n", - __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount); -} - -void ioat2_issue_pending(struct dma_chan *c) -{ - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - - if (ioat2_ring_pending(ioat)) { - spin_lock_bh(&ioat->prep_lock); - __ioat2_issue_pending(ioat); - spin_unlock_bh(&ioat->prep_lock); - } -} - -/** - * ioat2_update_pending - log pending descriptors - * @ioat: ioat2+ channel - * - * Check if the number of unsubmitted descriptors has exceeded the - * watermark. Called with prep_lock held - */ -static void ioat2_update_pending(struct ioat2_dma_chan *ioat) -{ - if (ioat2_ring_pending(ioat) > ioat_pending_level) - __ioat2_issue_pending(ioat); -} - -static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat) -{ - struct ioat_ring_ent *desc; - struct ioat_dma_descriptor *hw; - - if (ioat2_ring_space(ioat) < 1) { - dev_err(to_dev(&ioat->base), - "Unable to start null desc - ring full\n"); - return; - } - - dev_dbg(to_dev(&ioat->base), "%s: head: %#x tail: %#x issued: %#x\n", - __func__, ioat->head, ioat->tail, ioat->issued); - desc = ioat2_get_ring_ent(ioat, ioat->head); - - hw = desc->hw; - hw->ctl = 0; - hw->ctl_f.null = 1; - hw->ctl_f.int_en = 1; - hw->ctl_f.compl_write = 1; - /* set size to non-zero value (channel returns error when size is 0) */ - hw->size = NULL_DESC_BUFFER_SIZE; - hw->src_addr = 0; - hw->dst_addr = 0; - async_tx_ack(&desc->txd); - ioat2_set_chainaddr(ioat, desc->txd.phys); - dump_desc_dbg(ioat, desc); - wmb(); - ioat->head += 1; - __ioat2_issue_pending(ioat); -} - -static void ioat2_start_null_desc(struct ioat2_dma_chan *ioat) -{ - spin_lock_bh(&ioat->prep_lock); - __ioat2_start_null_desc(ioat); - spin_unlock_bh(&ioat->prep_lock); -} - -static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete) -{ - struct ioat_chan_common *chan = &ioat->base; - struct dma_async_tx_descriptor *tx; - struct ioat_ring_ent *desc; - bool seen_current = false; - u16 active; - int idx = ioat->tail, i; - - dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n", - __func__, ioat->head, ioat->tail, ioat->issued); - - active = ioat2_ring_active(ioat); - for (i = 0; i < active && !seen_current; i++) { - smp_read_barrier_depends(); - prefetch(ioat2_get_ring_ent(ioat, idx + i + 1)); - desc = ioat2_get_ring_ent(ioat, idx + i); - tx = &desc->txd; - dump_desc_dbg(ioat, desc); - if (tx->cookie) { - dma_descriptor_unmap(tx); - dma_cookie_complete(tx); - if (tx->callback) { - tx->callback(tx->callback_param); - tx->callback = NULL; - } - } - - if (tx->phys == phys_complete) - seen_current = true; - } - smp_mb(); /* finish all descriptor reads before incrementing tail */ - ioat->tail = idx + i; - BUG_ON(active && !seen_current); /* no active descs have written a completion? */ - - chan->last_completion = phys_complete; - if (active - i == 0) { - dev_dbg(to_dev(chan), "%s: cancel completion timeout\n", - __func__); - clear_bit(IOAT_COMPLETION_PENDING, &chan->state); - mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); - } -} - -/** - * ioat2_cleanup - clean finished descriptors (advance tail pointer) - * @chan: ioat channel to be cleaned up - */ -static void ioat2_cleanup(struct ioat2_dma_chan *ioat) -{ - struct ioat_chan_common *chan = &ioat->base; - dma_addr_t phys_complete; - - spin_lock_bh(&chan->cleanup_lock); - if (ioat_cleanup_preamble(chan, &phys_complete)) - __cleanup(ioat, phys_complete); - spin_unlock_bh(&chan->cleanup_lock); -} - -void ioat2_cleanup_event(unsigned long data) -{ - struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data); - struct ioat_chan_common *chan = &ioat->base; - - ioat2_cleanup(ioat); - if (!test_bit(IOAT_RUN, &chan->state)) - return; - writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); -} - -void __ioat2_restart_chan(struct ioat2_dma_chan *ioat) -{ - struct ioat_chan_common *chan = &ioat->base; - - /* set the tail to be re-issued */ - ioat->issued = ioat->tail; - ioat->dmacount = 0; - set_bit(IOAT_COMPLETION_PENDING, &chan->state); - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); - - dev_dbg(to_dev(chan), - "%s: head: %#x tail: %#x issued: %#x count: %#x\n", - __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount); - - if (ioat2_ring_pending(ioat)) { - struct ioat_ring_ent *desc; - - desc = ioat2_get_ring_ent(ioat, ioat->tail); - ioat2_set_chainaddr(ioat, desc->txd.phys); - __ioat2_issue_pending(ioat); - } else - __ioat2_start_null_desc(ioat); -} - -int ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo) -{ - unsigned long end = jiffies + tmo; - int err = 0; - u32 status; - - status = ioat_chansts(chan); - if (is_ioat_active(status) || is_ioat_idle(status)) - ioat_suspend(chan); - while (is_ioat_active(status) || is_ioat_idle(status)) { - if (tmo && time_after(jiffies, end)) { - err = -ETIMEDOUT; - break; - } - status = ioat_chansts(chan); - cpu_relax(); - } - - return err; -} - -int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo) -{ - unsigned long end = jiffies + tmo; - int err = 0; - - ioat_reset(chan); - while (ioat_reset_pending(chan)) { - if (end && time_after(jiffies, end)) { - err = -ETIMEDOUT; - break; - } - cpu_relax(); - } - - return err; -} - -static void ioat2_restart_channel(struct ioat2_dma_chan *ioat) -{ - struct ioat_chan_common *chan = &ioat->base; - dma_addr_t phys_complete; - - ioat2_quiesce(chan, 0); - if (ioat_cleanup_preamble(chan, &phys_complete)) - __cleanup(ioat, phys_complete); - - __ioat2_restart_chan(ioat); -} - -static void check_active(struct ioat2_dma_chan *ioat) -{ - struct ioat_chan_common *chan = &ioat->base; - - if (ioat2_ring_active(ioat)) { - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); - return; - } - - if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &chan->state)) - mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); - else if (ioat->alloc_order > ioat_get_alloc_order()) { - /* if the ring is idle, empty, and oversized try to step - * down the size - */ - reshape_ring(ioat, ioat->alloc_order - 1); - - /* keep shrinking until we get back to our minimum - * default size - */ - if (ioat->alloc_order > ioat_get_alloc_order()) - mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); - } - -} - -void ioat2_timer_event(unsigned long data) -{ - struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data); - struct ioat_chan_common *chan = &ioat->base; - dma_addr_t phys_complete; - u64 status; - - status = ioat_chansts(chan); - - /* when halted due to errors check for channel - * programming errors before advancing the completion state - */ - if (is_ioat_halted(status)) { - u32 chanerr; - - chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); - dev_err(to_dev(chan), "%s: Channel halted (%x)\n", - __func__, chanerr); - if (test_bit(IOAT_RUN, &chan->state)) - BUG_ON(is_ioat_bug(chanerr)); - else /* we never got off the ground */ - return; - } - - /* if we haven't made progress and we have already - * acknowledged a pending completion once, then be more - * forceful with a restart - */ - spin_lock_bh(&chan->cleanup_lock); - if (ioat_cleanup_preamble(chan, &phys_complete)) - __cleanup(ioat, phys_complete); - else if (test_bit(IOAT_COMPLETION_ACK, &chan->state)) { - spin_lock_bh(&ioat->prep_lock); - ioat2_restart_channel(ioat); - spin_unlock_bh(&ioat->prep_lock); - spin_unlock_bh(&chan->cleanup_lock); - return; - } else { - set_bit(IOAT_COMPLETION_ACK, &chan->state); - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); - } - - - if (ioat2_ring_active(ioat)) - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); - else { - spin_lock_bh(&ioat->prep_lock); - check_active(ioat); - spin_unlock_bh(&ioat->prep_lock); - } - spin_unlock_bh(&chan->cleanup_lock); -} - -static int ioat2_reset_hw(struct ioat_chan_common *chan) -{ - /* throw away whatever the channel was doing and get it initialized */ - u32 chanerr; - - ioat2_quiesce(chan, msecs_to_jiffies(100)); - - chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); - writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET); - - return ioat2_reset_sync(chan, msecs_to_jiffies(200)); -} - -/** - * ioat2_enumerate_channels - find and initialize the device's channels - * @device: the device to be enumerated - */ -int ioat2_enumerate_channels(struct ioatdma_device *device) -{ - struct ioat2_dma_chan *ioat; - struct device *dev = &device->pdev->dev; - struct dma_device *dma = &device->common; - u8 xfercap_log; - int i; - - INIT_LIST_HEAD(&dma->channels); - dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET); - dma->chancnt &= 0x1f; /* bits [4:0] valid */ - if (dma->chancnt > ARRAY_SIZE(device->idx)) { - dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n", - dma->chancnt, ARRAY_SIZE(device->idx)); - dma->chancnt = ARRAY_SIZE(device->idx); - } - xfercap_log = readb(device->reg_base + IOAT_XFERCAP_OFFSET); - xfercap_log &= 0x1f; /* bits [4:0] valid */ - if (xfercap_log == 0) - return 0; - dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log); - - /* FIXME which i/oat version is i7300? */ -#ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL - if (i7300_idle_platform_probe(NULL, NULL, 1) == 0) - dma->chancnt--; -#endif - for (i = 0; i < dma->chancnt; i++) { - ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL); - if (!ioat) - break; - - ioat_init_channel(device, &ioat->base, i); - ioat->xfercap_log = xfercap_log; - spin_lock_init(&ioat->prep_lock); - if (device->reset_hw(&ioat->base)) { - i = 0; - break; - } - } - dma->chancnt = i; - return i; -} - -static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx) -{ - struct dma_chan *c = tx->chan; - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - struct ioat_chan_common *chan = &ioat->base; - dma_cookie_t cookie; - - cookie = dma_cookie_assign(tx); - dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie); - - if (!test_and_set_bit(IOAT_CHAN_ACTIVE, &chan->state)) - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); - - /* make descriptor updates visible before advancing ioat->head, - * this is purposefully not smp_wmb() since we are also - * publishing the descriptor updates to a dma device - */ - wmb(); - - ioat->head += ioat->produce; - - ioat2_update_pending(ioat); - spin_unlock_bh(&ioat->prep_lock); - - return cookie; -} - -static struct ioat_ring_ent *ioat2_alloc_ring_ent(struct dma_chan *chan, gfp_t flags) -{ - struct ioat_dma_descriptor *hw; - struct ioat_ring_ent *desc; - struct ioatdma_device *dma; - dma_addr_t phys; - - dma = to_ioatdma_device(chan->device); - hw = pci_pool_alloc(dma->dma_pool, flags, &phys); - if (!hw) - return NULL; - memset(hw, 0, sizeof(*hw)); - - desc = kmem_cache_zalloc(ioat2_cache, flags); - if (!desc) { - pci_pool_free(dma->dma_pool, hw, phys); - return NULL; - } - - dma_async_tx_descriptor_init(&desc->txd, chan); - desc->txd.tx_submit = ioat2_tx_submit_unlock; - desc->hw = hw; - desc->txd.phys = phys; - return desc; -} - -static void ioat2_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan) -{ - struct ioatdma_device *dma; - - dma = to_ioatdma_device(chan->device); - pci_pool_free(dma->dma_pool, desc->hw, desc->txd.phys); - kmem_cache_free(ioat2_cache, desc); -} - -static struct ioat_ring_ent **ioat2_alloc_ring(struct dma_chan *c, int order, gfp_t flags) -{ - struct ioat_ring_ent **ring; - int descs = 1 << order; - int i; - - if (order > ioat_get_max_alloc_order()) - return NULL; - - /* allocate the array to hold the software ring */ - ring = kcalloc(descs, sizeof(*ring), flags); - if (!ring) - return NULL; - for (i = 0; i < descs; i++) { - ring[i] = ioat2_alloc_ring_ent(c, flags); - if (!ring[i]) { - while (i--) - ioat2_free_ring_ent(ring[i], c); - kfree(ring); - return NULL; - } - set_desc_id(ring[i], i); - } - - /* link descs */ - for (i = 0; i < descs-1; i++) { - struct ioat_ring_ent *next = ring[i+1]; - struct ioat_dma_descriptor *hw = ring[i]->hw; - - hw->next = next->txd.phys; - } - ring[i]->hw->next = ring[0]->txd.phys; - - return ring; -} - -void ioat2_free_chan_resources(struct dma_chan *c); - -/* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring - * @chan: channel to be initialized - */ -int ioat2_alloc_chan_resources(struct dma_chan *c) -{ - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - struct ioat_chan_common *chan = &ioat->base; - struct ioat_ring_ent **ring; - u64 status; - int order; - int i = 0; - - /* have we already been set up? */ - if (ioat->ring) - return 1 << ioat->alloc_order; - - /* Setup register to interrupt and write completion status on error */ - writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET); - - /* allocate a completion writeback area */ - /* doing 2 32bit writes to mmio since 1 64b write doesn't work */ - chan->completion = pci_pool_alloc(chan->device->completion_pool, - GFP_KERNEL, &chan->completion_dma); - if (!chan->completion) - return -ENOMEM; - - memset(chan->completion, 0, sizeof(*chan->completion)); - writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF, - chan->reg_base + IOAT_CHANCMP_OFFSET_LOW); - writel(((u64) chan->completion_dma) >> 32, - chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); - - order = ioat_get_alloc_order(); - ring = ioat2_alloc_ring(c, order, GFP_KERNEL); - if (!ring) - return -ENOMEM; - - spin_lock_bh(&chan->cleanup_lock); - spin_lock_bh(&ioat->prep_lock); - ioat->ring = ring; - ioat->head = 0; - ioat->issued = 0; - ioat->tail = 0; - ioat->alloc_order = order; - set_bit(IOAT_RUN, &chan->state); - spin_unlock_bh(&ioat->prep_lock); - spin_unlock_bh(&chan->cleanup_lock); - - ioat2_start_null_desc(ioat); - - /* check that we got off the ground */ - do { - udelay(1); - status = ioat_chansts(chan); - } while (i++ < 20 && !is_ioat_active(status) && !is_ioat_idle(status)); - - if (is_ioat_active(status) || is_ioat_idle(status)) { - return 1 << ioat->alloc_order; - } else { - u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); - - dev_WARN(to_dev(chan), - "failed to start channel chanerr: %#x\n", chanerr); - ioat2_free_chan_resources(c); - return -EFAULT; - } -} - -bool reshape_ring(struct ioat2_dma_chan *ioat, int order) -{ - /* reshape differs from normal ring allocation in that we want - * to allocate a new software ring while only - * extending/truncating the hardware ring - */ - struct ioat_chan_common *chan = &ioat->base; - struct dma_chan *c = &chan->common; - const u32 curr_size = ioat2_ring_size(ioat); - const u16 active = ioat2_ring_active(ioat); - const u32 new_size = 1 << order; - struct ioat_ring_ent **ring; - u16 i; - - if (order > ioat_get_max_alloc_order()) - return false; - - /* double check that we have at least 1 free descriptor */ - if (active == curr_size) - return false; - - /* when shrinking, verify that we can hold the current active - * set in the new ring - */ - if (active >= new_size) - return false; - - /* allocate the array to hold the software ring */ - ring = kcalloc(new_size, sizeof(*ring), GFP_NOWAIT); - if (!ring) - return false; - - /* allocate/trim descriptors as needed */ - if (new_size > curr_size) { - /* copy current descriptors to the new ring */ - for (i = 0; i < curr_size; i++) { - u16 curr_idx = (ioat->tail+i) & (curr_size-1); - u16 new_idx = (ioat->tail+i) & (new_size-1); - - ring[new_idx] = ioat->ring[curr_idx]; - set_desc_id(ring[new_idx], new_idx); - } - - /* add new descriptors to the ring */ - for (i = curr_size; i < new_size; i++) { - u16 new_idx = (ioat->tail+i) & (new_size-1); - - ring[new_idx] = ioat2_alloc_ring_ent(c, GFP_NOWAIT); - if (!ring[new_idx]) { - while (i--) { - u16 new_idx = (ioat->tail+i) & (new_size-1); - - ioat2_free_ring_ent(ring[new_idx], c); - } - kfree(ring); - return false; - } - set_desc_id(ring[new_idx], new_idx); - } - - /* hw link new descriptors */ - for (i = curr_size-1; i < new_size; i++) { - u16 new_idx = (ioat->tail+i) & (new_size-1); - struct ioat_ring_ent *next = ring[(new_idx+1) & (new_size-1)]; - struct ioat_dma_descriptor *hw = ring[new_idx]->hw; - - hw->next = next->txd.phys; - } - } else { - struct ioat_dma_descriptor *hw; - struct ioat_ring_ent *next; - - /* copy current descriptors to the new ring, dropping the - * removed descriptors - */ - for (i = 0; i < new_size; i++) { - u16 curr_idx = (ioat->tail+i) & (curr_size-1); - u16 new_idx = (ioat->tail+i) & (new_size-1); - - ring[new_idx] = ioat->ring[curr_idx]; - set_desc_id(ring[new_idx], new_idx); - } - - /* free deleted descriptors */ - for (i = new_size; i < curr_size; i++) { - struct ioat_ring_ent *ent; - - ent = ioat2_get_ring_ent(ioat, ioat->tail+i); - ioat2_free_ring_ent(ent, c); - } - - /* fix up hardware ring */ - hw = ring[(ioat->tail+new_size-1) & (new_size-1)]->hw; - next = ring[(ioat->tail+new_size) & (new_size-1)]; - hw->next = next->txd.phys; - } - - dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n", - __func__, new_size); - - kfree(ioat->ring); - ioat->ring = ring; - ioat->alloc_order = order; - - return true; -} - -/** - * ioat2_check_space_lock - verify space and grab ring producer lock - * @ioat: ioat2,3 channel (ring) to operate on - * @num_descs: allocation length - */ -int ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs) -{ - struct ioat_chan_common *chan = &ioat->base; - bool retry; - - retry: - spin_lock_bh(&ioat->prep_lock); - /* never allow the last descriptor to be consumed, we need at - * least one free at all times to allow for on-the-fly ring - * resizing. - */ - if (likely(ioat2_ring_space(ioat) > num_descs)) { - dev_dbg(to_dev(chan), "%s: num_descs: %d (%x:%x:%x)\n", - __func__, num_descs, ioat->head, ioat->tail, ioat->issued); - ioat->produce = num_descs; - return 0; /* with ioat->prep_lock held */ - } - retry = test_and_set_bit(IOAT_RESHAPE_PENDING, &chan->state); - spin_unlock_bh(&ioat->prep_lock); - - /* is another cpu already trying to expand the ring? */ - if (retry) - goto retry; - - spin_lock_bh(&chan->cleanup_lock); - spin_lock_bh(&ioat->prep_lock); - retry = reshape_ring(ioat, ioat->alloc_order + 1); - clear_bit(IOAT_RESHAPE_PENDING, &chan->state); - spin_unlock_bh(&ioat->prep_lock); - spin_unlock_bh(&chan->cleanup_lock); - - /* if we were able to expand the ring retry the allocation */ - if (retry) - goto retry; - - if (printk_ratelimit()) - dev_dbg(to_dev(chan), "%s: ring full! num_descs: %d (%x:%x:%x)\n", - __func__, num_descs, ioat->head, ioat->tail, ioat->issued); - - /* progress reclaim in the allocation failure case we may be - * called under bh_disabled so we need to trigger the timer - * event directly - */ - if (time_is_before_jiffies(chan->timer.expires) - && timer_pending(&chan->timer)) { - struct ioatdma_device *device = chan->device; - - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); - device->timer_fn((unsigned long) &chan->common); - } - - return -ENOMEM; -} - -struct dma_async_tx_descriptor * -ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, - dma_addr_t dma_src, size_t len, unsigned long flags) -{ - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - struct ioat_dma_descriptor *hw; - struct ioat_ring_ent *desc; - dma_addr_t dst = dma_dest; - dma_addr_t src = dma_src; - size_t total_len = len; - int num_descs, idx, i; - - num_descs = ioat2_xferlen_to_descs(ioat, len); - if (likely(num_descs) && ioat2_check_space_lock(ioat, num_descs) == 0) - idx = ioat->head; - else - return NULL; - i = 0; - do { - size_t copy = min_t(size_t, len, 1 << ioat->xfercap_log); - - desc = ioat2_get_ring_ent(ioat, idx + i); - hw = desc->hw; - - hw->size = copy; - hw->ctl = 0; - hw->src_addr = src; - hw->dst_addr = dst; - - len -= copy; - dst += copy; - src += copy; - dump_desc_dbg(ioat, desc); - } while (++i < num_descs); - - desc->txd.flags = flags; - desc->len = total_len; - hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); - hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE); - hw->ctl_f.compl_write = 1; - dump_desc_dbg(ioat, desc); - /* we leave the channel locked to ensure in order submission */ - - return &desc->txd; -} - -/** - * ioat2_free_chan_resources - release all the descriptors - * @chan: the channel to be cleaned - */ -void ioat2_free_chan_resources(struct dma_chan *c) -{ - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - struct ioat_chan_common *chan = &ioat->base; - struct ioatdma_device *device = chan->device; - struct ioat_ring_ent *desc; - const u16 total_descs = 1 << ioat->alloc_order; - int descs; - int i; - - /* Before freeing channel resources first check - * if they have been previously allocated for this channel. - */ - if (!ioat->ring) - return; - - ioat_stop(chan); - device->reset_hw(chan); - - spin_lock_bh(&chan->cleanup_lock); - spin_lock_bh(&ioat->prep_lock); - descs = ioat2_ring_space(ioat); - dev_dbg(to_dev(chan), "freeing %d idle descriptors\n", descs); - for (i = 0; i < descs; i++) { - desc = ioat2_get_ring_ent(ioat, ioat->head + i); - ioat2_free_ring_ent(desc, c); - } - - if (descs < total_descs) - dev_err(to_dev(chan), "Freeing %d in use descriptors!\n", - total_descs - descs); - - for (i = 0; i < total_descs - descs; i++) { - desc = ioat2_get_ring_ent(ioat, ioat->tail + i); - dump_desc_dbg(ioat, desc); - ioat2_free_ring_ent(desc, c); - } - - kfree(ioat->ring); - ioat->ring = NULL; - ioat->alloc_order = 0; - pci_pool_free(device->completion_pool, chan->completion, - chan->completion_dma); - spin_unlock_bh(&ioat->prep_lock); - spin_unlock_bh(&chan->cleanup_lock); - - chan->last_completion = 0; - chan->completion_dma = 0; - ioat->dmacount = 0; -} - -static ssize_t ring_size_show(struct dma_chan *c, char *page) -{ - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - - return sprintf(page, "%d\n", (1 << ioat->alloc_order) & ~1); -} -static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size); - -static ssize_t ring_active_show(struct dma_chan *c, char *page) -{ - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - - /* ...taken outside the lock, no need to be precise */ - return sprintf(page, "%d\n", ioat2_ring_active(ioat)); -} -static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active); - -static struct attribute *ioat2_attrs[] = { - &ring_size_attr.attr, - &ring_active_attr.attr, - &ioat_cap_attr.attr, - &ioat_version_attr.attr, - NULL, -}; - -struct kobj_type ioat2_ktype = { - .sysfs_ops = &ioat_sysfs_ops, - .default_attrs = ioat2_attrs, -}; - -int ioat2_dma_probe(struct ioatdma_device *device, int dca) -{ - struct pci_dev *pdev = device->pdev; - struct dma_device *dma; - struct dma_chan *c; - struct ioat_chan_common *chan; - int err; - - device->enumerate_channels = ioat2_enumerate_channels; - device->reset_hw = ioat2_reset_hw; - device->cleanup_fn = ioat2_cleanup_event; - device->timer_fn = ioat2_timer_event; - device->self_test = ioat_dma_self_test; - dma = &device->common; - dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock; - dma->device_issue_pending = ioat2_issue_pending; - dma->device_alloc_chan_resources = ioat2_alloc_chan_resources; - dma->device_free_chan_resources = ioat2_free_chan_resources; - dma->device_tx_status = ioat_dma_tx_status; - - err = ioat_probe(device); - if (err) - return err; - - list_for_each_entry(c, &dma->channels, device_node) { - chan = to_chan_common(c); - writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU, - chan->reg_base + IOAT_DCACTRL_OFFSET); - } - - err = ioat_register(device); - if (err) - return err; - - ioat_kobject_add(device, &ioat2_ktype); - - if (dca) - device->dca = ioat2_dca_init(pdev, device->reg_base); - - return err; -} diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h deleted file mode 100644 index bf24ebe874b0..000000000000 --- a/drivers/dma/ioat/dma_v2.h +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) - * any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * The full GNU General Public License is included in this distribution in the - * file called COPYING. - */ -#ifndef IOATDMA_V2_H -#define IOATDMA_V2_H - -#include -#include -#include "dma.h" -#include "hw.h" - - -extern int ioat_pending_level; -extern int ioat_ring_alloc_order; - -/* - * workaround for IOAT ver.3.0 null descriptor issue - * (channel returns error when size is 0) - */ -#define NULL_DESC_BUFFER_SIZE 1 - -#define IOAT_MAX_ORDER 16 -#define ioat_get_alloc_order() \ - (min(ioat_ring_alloc_order, IOAT_MAX_ORDER)) -#define ioat_get_max_alloc_order() \ - (min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER)) - -/* struct ioat2_dma_chan - ioat v2 / v3 channel attributes - * @base: common ioat channel parameters - * @xfercap_log; log2 of channel max transfer length (for fast division) - * @head: allocated index - * @issued: hardware notification point - * @tail: cleanup index - * @dmacount: identical to 'head' except for occasionally resetting to zero - * @alloc_order: log2 of the number of allocated descriptors - * @produce: number of descriptors to produce at submit time - * @ring: software ring buffer implementation of hardware ring - * @prep_lock: serializes descriptor preparation (producers) - */ -struct ioat2_dma_chan { - struct ioat_chan_common base; - size_t xfercap_log; - u16 head; - u16 issued; - u16 tail; - u16 dmacount; - u16 alloc_order; - u16 produce; - struct ioat_ring_ent **ring; - spinlock_t prep_lock; -}; - -static inline struct ioat2_dma_chan *to_ioat2_chan(struct dma_chan *c) -{ - struct ioat_chan_common *chan = to_chan_common(c); - - return container_of(chan, struct ioat2_dma_chan, base); -} - -static inline u32 ioat2_ring_size(struct ioat2_dma_chan *ioat) -{ - return 1 << ioat->alloc_order; -} - -/* count of descriptors in flight with the engine */ -static inline u16 ioat2_ring_active(struct ioat2_dma_chan *ioat) -{ - return CIRC_CNT(ioat->head, ioat->tail, ioat2_ring_size(ioat)); -} - -/* count of descriptors pending submission to hardware */ -static inline u16 ioat2_ring_pending(struct ioat2_dma_chan *ioat) -{ - return CIRC_CNT(ioat->head, ioat->issued, ioat2_ring_size(ioat)); -} - -static inline u32 ioat2_ring_space(struct ioat2_dma_chan *ioat) -{ - return ioat2_ring_size(ioat) - ioat2_ring_active(ioat); -} - -static inline u16 ioat2_xferlen_to_descs(struct ioat2_dma_chan *ioat, size_t len) -{ - u16 num_descs = len >> ioat->xfercap_log; - - num_descs += !!(len & ((1 << ioat->xfercap_log) - 1)); - return num_descs; -} - -/** - * struct ioat_ring_ent - wrapper around hardware descriptor - * @hw: hardware DMA descriptor (for memcpy) - * @fill: hardware fill descriptor - * @xor: hardware xor descriptor - * @xor_ex: hardware xor extension descriptor - * @pq: hardware pq descriptor - * @pq_ex: hardware pq extension descriptor - * @pqu: hardware pq update descriptor - * @raw: hardware raw (un-typed) descriptor - * @txd: the generic software descriptor for all engines - * @len: total transaction length for unmap - * @result: asynchronous result of validate operations - * @id: identifier for debug - */ - -struct ioat_ring_ent { - union { - struct ioat_dma_descriptor *hw; - struct ioat_xor_descriptor *xor; - struct ioat_xor_ext_descriptor *xor_ex; - struct ioat_pq_descriptor *pq; - struct ioat_pq_ext_descriptor *pq_ex; - struct ioat_pq_update_descriptor *pqu; - struct ioat_raw_descriptor *raw; - }; - size_t len; - struct dma_async_tx_descriptor txd; - enum sum_check_flags *result; - #ifdef DEBUG - int id; - #endif - struct ioat_sed_ent *sed; -}; - -static inline struct ioat_ring_ent * -ioat2_get_ring_ent(struct ioat2_dma_chan *ioat, u16 idx) -{ - return ioat->ring[idx & (ioat2_ring_size(ioat) - 1)]; -} - -static inline void ioat2_set_chainaddr(struct ioat2_dma_chan *ioat, u64 addr) -{ - struct ioat_chan_common *chan = &ioat->base; - - writel(addr & 0x00000000FFFFFFFF, - chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW); - writel(addr >> 32, - chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH); -} - -int ioat2_dma_probe(struct ioatdma_device *dev, int dca); -int ioat3_dma_probe(struct ioatdma_device *dev, int dca); -struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase); -struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase); -int ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs); -int ioat2_enumerate_channels(struct ioatdma_device *device); -struct dma_async_tx_descriptor * -ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, - dma_addr_t dma_src, size_t len, unsigned long flags); -void ioat2_issue_pending(struct dma_chan *chan); -int ioat2_alloc_chan_resources(struct dma_chan *c); -void ioat2_free_chan_resources(struct dma_chan *c); -void __ioat2_restart_chan(struct ioat2_dma_chan *ioat); -bool reshape_ring(struct ioat2_dma_chan *ioat, int order); -void __ioat2_issue_pending(struct ioat2_dma_chan *ioat); -void ioat2_cleanup_event(unsigned long data); -void ioat2_timer_event(unsigned long data); -int ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo); -int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo); -extern struct kobj_type ioat2_ktype; -extern struct kmem_cache *ioat2_cache; -#endif /* IOATDMA_V2_H */ diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c deleted file mode 100644 index 64790a45ef5d..000000000000 --- a/drivers/dma/ioat/dma_v3.c +++ /dev/null @@ -1,1717 +0,0 @@ -/* - * This file is provided under a dual BSD/GPLv2 license. When using or - * redistributing this file, you may do so under either license. - * - * GPL LICENSE SUMMARY - * - * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * The full GNU General Public License is included in this distribution in - * the file called "COPYING". - * - * BSD LICENSE - * - * Copyright(c) 2004-2009 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/* - * Support routines for v3+ hardware - */ -#include -#include -#include -#include -#include -#include -#include "../dmaengine.h" -#include "registers.h" -#include "hw.h" -#include "dma.h" -#include "dma_v2.h" - -extern struct kmem_cache *ioat3_sed_cache; - -/* ioat hardware assumes at least two sources for raid operations */ -#define src_cnt_to_sw(x) ((x) + 2) -#define src_cnt_to_hw(x) ((x) - 2) -#define ndest_to_sw(x) ((x) + 1) -#define ndest_to_hw(x) ((x) - 1) -#define src16_cnt_to_sw(x) ((x) + 9) -#define src16_cnt_to_hw(x) ((x) - 9) - -/* provide a lookup table for setting the source address in the base or - * extended descriptor of an xor or pq descriptor - */ -static const u8 xor_idx_to_desc = 0xe0; -static const u8 xor_idx_to_field[] = { 1, 4, 5, 6, 7, 0, 1, 2 }; -static const u8 pq_idx_to_desc = 0xf8; -static const u8 pq16_idx_to_desc[] = { 0, 0, 1, 1, 1, 1, 1, 1, 1, - 2, 2, 2, 2, 2, 2, 2 }; -static const u8 pq_idx_to_field[] = { 1, 4, 5, 0, 1, 2, 4, 5 }; -static const u8 pq16_idx_to_field[] = { 1, 4, 1, 2, 3, 4, 5, 6, 7, - 0, 1, 2, 3, 4, 5, 6 }; - -static void ioat3_eh(struct ioat2_dma_chan *ioat); - -static void xor_set_src(struct ioat_raw_descriptor *descs[2], - dma_addr_t addr, u32 offset, int idx) -{ - struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1]; - - raw->field[xor_idx_to_field[idx]] = addr + offset; -} - -static dma_addr_t pq_get_src(struct ioat_raw_descriptor *descs[2], int idx) -{ - struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1]; - - return raw->field[pq_idx_to_field[idx]]; -} - -static dma_addr_t pq16_get_src(struct ioat_raw_descriptor *desc[3], int idx) -{ - struct ioat_raw_descriptor *raw = desc[pq16_idx_to_desc[idx]]; - - return raw->field[pq16_idx_to_field[idx]]; -} - -static void pq_set_src(struct ioat_raw_descriptor *descs[2], - dma_addr_t addr, u32 offset, u8 coef, int idx) -{ - struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *) descs[0]; - struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1]; - - raw->field[pq_idx_to_field[idx]] = addr + offset; - pq->coef[idx] = coef; -} - -static bool is_jf_ioat(struct pci_dev *pdev) -{ - switch (pdev->device) { - case PCI_DEVICE_ID_INTEL_IOAT_JSF0: - case PCI_DEVICE_ID_INTEL_IOAT_JSF1: - case PCI_DEVICE_ID_INTEL_IOAT_JSF2: - case PCI_DEVICE_ID_INTEL_IOAT_JSF3: - case PCI_DEVICE_ID_INTEL_IOAT_JSF4: - case PCI_DEVICE_ID_INTEL_IOAT_JSF5: - case PCI_DEVICE_ID_INTEL_IOAT_JSF6: - case PCI_DEVICE_ID_INTEL_IOAT_JSF7: - case PCI_DEVICE_ID_INTEL_IOAT_JSF8: - case PCI_DEVICE_ID_INTEL_IOAT_JSF9: - return true; - default: - return false; - } -} - -static bool is_snb_ioat(struct pci_dev *pdev) -{ - switch (pdev->device) { - case PCI_DEVICE_ID_INTEL_IOAT_SNB0: - case PCI_DEVICE_ID_INTEL_IOAT_SNB1: - case PCI_DEVICE_ID_INTEL_IOAT_SNB2: - case PCI_DEVICE_ID_INTEL_IOAT_SNB3: - case PCI_DEVICE_ID_INTEL_IOAT_SNB4: - case PCI_DEVICE_ID_INTEL_IOAT_SNB5: - case PCI_DEVICE_ID_INTEL_IOAT_SNB6: - case PCI_DEVICE_ID_INTEL_IOAT_SNB7: - case PCI_DEVICE_ID_INTEL_IOAT_SNB8: - case PCI_DEVICE_ID_INTEL_IOAT_SNB9: - return true; - default: - return false; - } -} - -static bool is_ivb_ioat(struct pci_dev *pdev) -{ - switch (pdev->device) { - case PCI_DEVICE_ID_INTEL_IOAT_IVB0: - case PCI_DEVICE_ID_INTEL_IOAT_IVB1: - case PCI_DEVICE_ID_INTEL_IOAT_IVB2: - case PCI_DEVICE_ID_INTEL_IOAT_IVB3: - case PCI_DEVICE_ID_INTEL_IOAT_IVB4: - case PCI_DEVICE_ID_INTEL_IOAT_IVB5: - case PCI_DEVICE_ID_INTEL_IOAT_IVB6: - case PCI_DEVICE_ID_INTEL_IOAT_IVB7: - case PCI_DEVICE_ID_INTEL_IOAT_IVB8: - case PCI_DEVICE_ID_INTEL_IOAT_IVB9: - return true; - default: - return false; - } - -} - -static bool is_hsw_ioat(struct pci_dev *pdev) -{ - switch (pdev->device) { - case PCI_DEVICE_ID_INTEL_IOAT_HSW0: - case PCI_DEVICE_ID_INTEL_IOAT_HSW1: - case PCI_DEVICE_ID_INTEL_IOAT_HSW2: - case PCI_DEVICE_ID_INTEL_IOAT_HSW3: - case PCI_DEVICE_ID_INTEL_IOAT_HSW4: - case PCI_DEVICE_ID_INTEL_IOAT_HSW5: - case PCI_DEVICE_ID_INTEL_IOAT_HSW6: - case PCI_DEVICE_ID_INTEL_IOAT_HSW7: - case PCI_DEVICE_ID_INTEL_IOAT_HSW8: - case PCI_DEVICE_ID_INTEL_IOAT_HSW9: - return true; - default: - return false; - } - -} - -static bool is_xeon_cb32(struct pci_dev *pdev) -{ - return is_jf_ioat(pdev) || is_snb_ioat(pdev) || is_ivb_ioat(pdev) || - is_hsw_ioat(pdev); -} - -static bool is_bwd_ioat(struct pci_dev *pdev) -{ - switch (pdev->device) { - case PCI_DEVICE_ID_INTEL_IOAT_BWD0: - case PCI_DEVICE_ID_INTEL_IOAT_BWD1: - case PCI_DEVICE_ID_INTEL_IOAT_BWD2: - case PCI_DEVICE_ID_INTEL_IOAT_BWD3: - /* even though not Atom, BDX-DE has same DMA silicon */ - case PCI_DEVICE_ID_INTEL_IOAT_BDXDE0: - case PCI_DEVICE_ID_INTEL_IOAT_BDXDE1: - case PCI_DEVICE_ID_INTEL_IOAT_BDXDE2: - case PCI_DEVICE_ID_INTEL_IOAT_BDXDE3: - return true; - default: - return false; - } -} - -static bool is_bwd_noraid(struct pci_dev *pdev) -{ - switch (pdev->device) { - case PCI_DEVICE_ID_INTEL_IOAT_BWD2: - case PCI_DEVICE_ID_INTEL_IOAT_BWD3: - case PCI_DEVICE_ID_INTEL_IOAT_BDXDE0: - case PCI_DEVICE_ID_INTEL_IOAT_BDXDE1: - case PCI_DEVICE_ID_INTEL_IOAT_BDXDE2: - case PCI_DEVICE_ID_INTEL_IOAT_BDXDE3: - return true; - default: - return false; - } - -} - -static void pq16_set_src(struct ioat_raw_descriptor *desc[3], - dma_addr_t addr, u32 offset, u8 coef, unsigned idx) -{ - struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *)desc[0]; - struct ioat_pq16a_descriptor *pq16 = - (struct ioat_pq16a_descriptor *)desc[1]; - struct ioat_raw_descriptor *raw = desc[pq16_idx_to_desc[idx]]; - - raw->field[pq16_idx_to_field[idx]] = addr + offset; - - if (idx < 8) - pq->coef[idx] = coef; - else - pq16->coef[idx - 8] = coef; -} - -static struct ioat_sed_ent * -ioat3_alloc_sed(struct ioatdma_device *device, unsigned int hw_pool) -{ - struct ioat_sed_ent *sed; - gfp_t flags = __GFP_ZERO | GFP_ATOMIC; - - sed = kmem_cache_alloc(ioat3_sed_cache, flags); - if (!sed) - return NULL; - - sed->hw_pool = hw_pool; - sed->hw = dma_pool_alloc(device->sed_hw_pool[hw_pool], - flags, &sed->dma); - if (!sed->hw) { - kmem_cache_free(ioat3_sed_cache, sed); - return NULL; - } - - return sed; -} - -static void ioat3_free_sed(struct ioatdma_device *device, struct ioat_sed_ent *sed) -{ - if (!sed) - return; - - dma_pool_free(device->sed_hw_pool[sed->hw_pool], sed->hw, sed->dma); - kmem_cache_free(ioat3_sed_cache, sed); -} - -static bool desc_has_ext(struct ioat_ring_ent *desc) -{ - struct ioat_dma_descriptor *hw = desc->hw; - - if (hw->ctl_f.op == IOAT_OP_XOR || - hw->ctl_f.op == IOAT_OP_XOR_VAL) { - struct ioat_xor_descriptor *xor = desc->xor; - - if (src_cnt_to_sw(xor->ctl_f.src_cnt) > 5) - return true; - } else if (hw->ctl_f.op == IOAT_OP_PQ || - hw->ctl_f.op == IOAT_OP_PQ_VAL) { - struct ioat_pq_descriptor *pq = desc->pq; - - if (src_cnt_to_sw(pq->ctl_f.src_cnt) > 3) - return true; - } - - return false; -} - -static u64 ioat3_get_current_completion(struct ioat_chan_common *chan) -{ - u64 phys_complete; - u64 completion; - - completion = *chan->completion; - phys_complete = ioat_chansts_to_addr(completion); - - dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__, - (unsigned long long) phys_complete); - - return phys_complete; -} - -static bool ioat3_cleanup_preamble(struct ioat_chan_common *chan, - u64 *phys_complete) -{ - *phys_complete = ioat3_get_current_completion(chan); - if (*phys_complete == chan->last_completion) - return false; - - clear_bit(IOAT_COMPLETION_ACK, &chan->state); - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); - - return true; -} - -static void -desc_get_errstat(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc) -{ - struct ioat_dma_descriptor *hw = desc->hw; - - switch (hw->ctl_f.op) { - case IOAT_OP_PQ_VAL: - case IOAT_OP_PQ_VAL_16S: - { - struct ioat_pq_descriptor *pq = desc->pq; - - /* check if there's error written */ - if (!pq->dwbes_f.wbes) - return; - - /* need to set a chanerr var for checking to clear later */ - - if (pq->dwbes_f.p_val_err) - *desc->result |= SUM_CHECK_P_RESULT; - - if (pq->dwbes_f.q_val_err) - *desc->result |= SUM_CHECK_Q_RESULT; - - return; - } - default: - return; - } -} - -/** - * __cleanup - reclaim used descriptors - * @ioat: channel (ring) to clean - * - * The difference from the dma_v2.c __cleanup() is that this routine - * handles extended descriptors and dma-unmapping raid operations. - */ -static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete) -{ - struct ioat_chan_common *chan = &ioat->base; - struct ioatdma_device *device = chan->device; - struct ioat_ring_ent *desc; - bool seen_current = false; - int idx = ioat->tail, i; - u16 active; - - dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n", - __func__, ioat->head, ioat->tail, ioat->issued); - - /* - * At restart of the channel, the completion address and the - * channel status will be 0 due to starting a new chain. Since - * it's new chain and the first descriptor "fails", there is - * nothing to clean up. We do not want to reap the entire submitted - * chain due to this 0 address value and then BUG. - */ - if (!phys_complete) - return; - - active = ioat2_ring_active(ioat); - for (i = 0; i < active && !seen_current; i++) { - struct dma_async_tx_descriptor *tx; - - smp_read_barrier_depends(); - prefetch(ioat2_get_ring_ent(ioat, idx + i + 1)); - desc = ioat2_get_ring_ent(ioat, idx + i); - dump_desc_dbg(ioat, desc); - - /* set err stat if we are using dwbes */ - if (device->cap & IOAT_CAP_DWBES) - desc_get_errstat(ioat, desc); - - tx = &desc->txd; - if (tx->cookie) { - dma_cookie_complete(tx); - dma_descriptor_unmap(tx); - if (tx->callback) { - tx->callback(tx->callback_param); - tx->callback = NULL; - } - } - - if (tx->phys == phys_complete) - seen_current = true; - - /* skip extended descriptors */ - if (desc_has_ext(desc)) { - BUG_ON(i + 1 >= active); - i++; - } - - /* cleanup super extended descriptors */ - if (desc->sed) { - ioat3_free_sed(device, desc->sed); - desc->sed = NULL; - } - } - smp_mb(); /* finish all descriptor reads before incrementing tail */ - ioat->tail = idx + i; - BUG_ON(active && !seen_current); /* no active descs have written a completion? */ - chan->last_completion = phys_complete; - - if (active - i == 0) { - dev_dbg(to_dev(chan), "%s: cancel completion timeout\n", - __func__); - clear_bit(IOAT_COMPLETION_PENDING, &chan->state); - mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); - } - /* 5 microsecond delay per pending descriptor */ - writew(min((5 * (active - i)), IOAT_INTRDELAY_MASK), - chan->device->reg_base + IOAT_INTRDELAY_OFFSET); -} - -static void ioat3_cleanup(struct ioat2_dma_chan *ioat) -{ - struct ioat_chan_common *chan = &ioat->base; - u64 phys_complete; - - spin_lock_bh(&chan->cleanup_lock); - - if (ioat3_cleanup_preamble(chan, &phys_complete)) - __cleanup(ioat, phys_complete); - - if (is_ioat_halted(*chan->completion)) { - u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); - - if (chanerr & IOAT_CHANERR_HANDLE_MASK) { - mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); - ioat3_eh(ioat); - } - } - - spin_unlock_bh(&chan->cleanup_lock); -} - -static void ioat3_cleanup_event(unsigned long data) -{ - struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data); - struct ioat_chan_common *chan = &ioat->base; - - ioat3_cleanup(ioat); - if (!test_bit(IOAT_RUN, &chan->state)) - return; - writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); -} - -static void ioat3_restart_channel(struct ioat2_dma_chan *ioat) -{ - struct ioat_chan_common *chan = &ioat->base; - u64 phys_complete; - - ioat2_quiesce(chan, 0); - if (ioat3_cleanup_preamble(chan, &phys_complete)) - __cleanup(ioat, phys_complete); - - __ioat2_restart_chan(ioat); -} - -static void ioat3_eh(struct ioat2_dma_chan *ioat) -{ - struct ioat_chan_common *chan = &ioat->base; - struct pci_dev *pdev = to_pdev(chan); - struct ioat_dma_descriptor *hw; - struct dma_async_tx_descriptor *tx; - u64 phys_complete; - struct ioat_ring_ent *desc; - u32 err_handled = 0; - u32 chanerr_int; - u32 chanerr; - - /* cleanup so tail points to descriptor that caused the error */ - if (ioat3_cleanup_preamble(chan, &phys_complete)) - __cleanup(ioat, phys_complete); - - chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); - pci_read_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, &chanerr_int); - - dev_dbg(to_dev(chan), "%s: error = %x:%x\n", - __func__, chanerr, chanerr_int); - - desc = ioat2_get_ring_ent(ioat, ioat->tail); - hw = desc->hw; - dump_desc_dbg(ioat, desc); - - switch (hw->ctl_f.op) { - case IOAT_OP_XOR_VAL: - if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) { - *desc->result |= SUM_CHECK_P_RESULT; - err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR; - } - break; - case IOAT_OP_PQ_VAL: - case IOAT_OP_PQ_VAL_16S: - if (chanerr & IOAT_CHANERR_XOR_P_OR_CRC_ERR) { - *desc->result |= SUM_CHECK_P_RESULT; - err_handled |= IOAT_CHANERR_XOR_P_OR_CRC_ERR; - } - if (chanerr & IOAT_CHANERR_XOR_Q_ERR) { - *desc->result |= SUM_CHECK_Q_RESULT; - err_handled |= IOAT_CHANERR_XOR_Q_ERR; - } - break; - } - - /* fault on unhandled error or spurious halt */ - if (chanerr ^ err_handled || chanerr == 0) { - dev_err(to_dev(chan), "%s: fatal error (%x:%x)\n", - __func__, chanerr, err_handled); - BUG(); - } else { /* cleanup the faulty descriptor */ - tx = &desc->txd; - if (tx->cookie) { - dma_cookie_complete(tx); - dma_descriptor_unmap(tx); - if (tx->callback) { - tx->callback(tx->callback_param); - tx->callback = NULL; - } - } - } - - writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET); - pci_write_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, chanerr_int); - - /* mark faulting descriptor as complete */ - *chan->completion = desc->txd.phys; - - spin_lock_bh(&ioat->prep_lock); - ioat3_restart_channel(ioat); - spin_unlock_bh(&ioat->prep_lock); -} - -static void check_active(struct ioat2_dma_chan *ioat) -{ - struct ioat_chan_common *chan = &ioat->base; - - if (ioat2_ring_active(ioat)) { - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); - return; - } - - if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &chan->state)) - mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); - else if (ioat->alloc_order > ioat_get_alloc_order()) { - /* if the ring is idle, empty, and oversized try to step - * down the size - */ - reshape_ring(ioat, ioat->alloc_order - 1); - - /* keep shrinking until we get back to our minimum - * default size - */ - if (ioat->alloc_order > ioat_get_alloc_order()) - mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); - } - -} - -static void ioat3_timer_event(unsigned long data) -{ - struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data); - struct ioat_chan_common *chan = &ioat->base; - dma_addr_t phys_complete; - u64 status; - - status = ioat_chansts(chan); - - /* when halted due to errors check for channel - * programming errors before advancing the completion state - */ - if (is_ioat_halted(status)) { - u32 chanerr; - - chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); - dev_err(to_dev(chan), "%s: Channel halted (%x)\n", - __func__, chanerr); - if (test_bit(IOAT_RUN, &chan->state)) - BUG_ON(is_ioat_bug(chanerr)); - else /* we never got off the ground */ - return; - } - - /* if we haven't made progress and we have already - * acknowledged a pending completion once, then be more - * forceful with a restart - */ - spin_lock_bh(&chan->cleanup_lock); - if (ioat_cleanup_preamble(chan, &phys_complete)) - __cleanup(ioat, phys_complete); - else if (test_bit(IOAT_COMPLETION_ACK, &chan->state)) { - spin_lock_bh(&ioat->prep_lock); - ioat3_restart_channel(ioat); - spin_unlock_bh(&ioat->prep_lock); - spin_unlock_bh(&chan->cleanup_lock); - return; - } else { - set_bit(IOAT_COMPLETION_ACK, &chan->state); - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); - } - - - if (ioat2_ring_active(ioat)) - mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); - else { - spin_lock_bh(&ioat->prep_lock); - check_active(ioat); - spin_unlock_bh(&ioat->prep_lock); - } - spin_unlock_bh(&chan->cleanup_lock); -} - -static enum dma_status -ioat3_tx_status(struct dma_chan *c, dma_cookie_t cookie, - struct dma_tx_state *txstate) -{ - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - enum dma_status ret; - - ret = dma_cookie_status(c, cookie, txstate); - if (ret == DMA_COMPLETE) - return ret; - - ioat3_cleanup(ioat); - - return dma_cookie_status(c, cookie, txstate); -} - -static struct dma_async_tx_descriptor * -__ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result, - dma_addr_t dest, dma_addr_t *src, unsigned int src_cnt, - size_t len, unsigned long flags) -{ - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - struct ioat_ring_ent *compl_desc; - struct ioat_ring_ent *desc; - struct ioat_ring_ent *ext; - size_t total_len = len; - struct ioat_xor_descriptor *xor; - struct ioat_xor_ext_descriptor *xor_ex = NULL; - struct ioat_dma_descriptor *hw; - int num_descs, with_ext, idx, i; - u32 offset = 0; - u8 op = result ? IOAT_OP_XOR_VAL : IOAT_OP_XOR; - - BUG_ON(src_cnt < 2); - - num_descs = ioat2_xferlen_to_descs(ioat, len); - /* we need 2x the number of descriptors to cover greater than 5 - * sources - */ - if (src_cnt > 5) { - with_ext = 1; - num_descs *= 2; - } else - with_ext = 0; - - /* completion writes from the raid engine may pass completion - * writes from the legacy engine, so we need one extra null - * (legacy) descriptor to ensure all completion writes arrive in - * order. - */ - if (likely(num_descs) && ioat2_check_space_lock(ioat, num_descs+1) == 0) - idx = ioat->head; - else - return NULL; - i = 0; - do { - struct ioat_raw_descriptor *descs[2]; - size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log); - int s; - - desc = ioat2_get_ring_ent(ioat, idx + i); - xor = desc->xor; - - /* save a branch by unconditionally retrieving the - * extended descriptor xor_set_src() knows to not write - * to it in the single descriptor case - */ - ext = ioat2_get_ring_ent(ioat, idx + i + 1); - xor_ex = ext->xor_ex; - - descs[0] = (struct ioat_raw_descriptor *) xor; - descs[1] = (struct ioat_raw_descriptor *) xor_ex; - for (s = 0; s < src_cnt; s++) - xor_set_src(descs, src[s], offset, s); - xor->size = xfer_size; - xor->dst_addr = dest + offset; - xor->ctl = 0; - xor->ctl_f.op = op; - xor->ctl_f.src_cnt = src_cnt_to_hw(src_cnt); - - len -= xfer_size; - offset += xfer_size; - dump_desc_dbg(ioat, desc); - } while ((i += 1 + with_ext) < num_descs); - - /* last xor descriptor carries the unmap parameters and fence bit */ - desc->txd.flags = flags; - desc->len = total_len; - if (result) - desc->result = result; - xor->ctl_f.fence = !!(flags & DMA_PREP_FENCE); - - /* completion descriptor carries interrupt bit */ - compl_desc = ioat2_get_ring_ent(ioat, idx + i); - compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT; - hw = compl_desc->hw; - hw->ctl = 0; - hw->ctl_f.null = 1; - hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); - hw->ctl_f.compl_write = 1; - hw->size = NULL_DESC_BUFFER_SIZE; - dump_desc_dbg(ioat, compl_desc); - - /* we leave the channel locked to ensure in order submission */ - return &compl_desc->txd; -} - -static struct dma_async_tx_descriptor * -ioat3_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, - unsigned int src_cnt, size_t len, unsigned long flags) -{ - return __ioat3_prep_xor_lock(chan, NULL, dest, src, src_cnt, len, flags); -} - -static struct dma_async_tx_descriptor * -ioat3_prep_xor_val(struct dma_chan *chan, dma_addr_t *src, - unsigned int src_cnt, size_t len, - enum sum_check_flags *result, unsigned long flags) -{ - /* the cleanup routine only sets bits on validate failure, it - * does not clear bits on validate success... so clear it here - */ - *result = 0; - - return __ioat3_prep_xor_lock(chan, result, src[0], &src[1], - src_cnt - 1, len, flags); -} - -static void -dump_pq_desc_dbg(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc, struct ioat_ring_ent *ext) -{ - struct device *dev = to_dev(&ioat->base); - struct ioat_pq_descriptor *pq = desc->pq; - struct ioat_pq_ext_descriptor *pq_ex = ext ? ext->pq_ex : NULL; - struct ioat_raw_descriptor *descs[] = { (void *) pq, (void *) pq_ex }; - int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt); - int i; - - dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x" - " sz: %#10.8x ctl: %#x (op: %#x int: %d compl: %d pq: '%s%s'" - " src_cnt: %d)\n", - desc_id(desc), (unsigned long long) desc->txd.phys, - (unsigned long long) (pq_ex ? pq_ex->next : pq->next), - desc->txd.flags, pq->size, pq->ctl, pq->ctl_f.op, pq->ctl_f.int_en, - pq->ctl_f.compl_write, - pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q", - pq->ctl_f.src_cnt); - for (i = 0; i < src_cnt; i++) - dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i, - (unsigned long long) pq_get_src(descs, i), pq->coef[i]); - dev_dbg(dev, "\tP: %#llx\n", pq->p_addr); - dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr); - dev_dbg(dev, "\tNEXT: %#llx\n", pq->next); -} - -static void dump_pq16_desc_dbg(struct ioat2_dma_chan *ioat, - struct ioat_ring_ent *desc) -{ - struct device *dev = to_dev(&ioat->base); - struct ioat_pq_descriptor *pq = desc->pq; - struct ioat_raw_descriptor *descs[] = { (void *)pq, - (void *)pq, - (void *)pq }; - int src_cnt = src16_cnt_to_sw(pq->ctl_f.src_cnt); - int i; - - if (desc->sed) { - descs[1] = (void *)desc->sed->hw; - descs[2] = (void *)desc->sed->hw + 64; - } - - dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x" - " sz: %#x ctl: %#x (op: %#x int: %d compl: %d pq: '%s%s'" - " src_cnt: %d)\n", - desc_id(desc), (unsigned long long) desc->txd.phys, - (unsigned long long) pq->next, - desc->txd.flags, pq->size, pq->ctl, - pq->ctl_f.op, pq->ctl_f.int_en, - pq->ctl_f.compl_write, - pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q", - pq->ctl_f.src_cnt); - for (i = 0; i < src_cnt; i++) { - dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i, - (unsigned long long) pq16_get_src(descs, i), - pq->coef[i]); - } - dev_dbg(dev, "\tP: %#llx\n", pq->p_addr); - dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr); -} - -static struct dma_async_tx_descriptor * -__ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result, - const dma_addr_t *dst, const dma_addr_t *src, - unsigned int src_cnt, const unsigned char *scf, - size_t len, unsigned long flags) -{ - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - struct ioat_chan_common *chan = &ioat->base; - struct ioatdma_device *device = chan->device; - struct ioat_ring_ent *compl_desc; - struct ioat_ring_ent *desc; - struct ioat_ring_ent *ext; - size_t total_len = len; - struct ioat_pq_descriptor *pq; - struct ioat_pq_ext_descriptor *pq_ex = NULL; - struct ioat_dma_descriptor *hw; - u32 offset = 0; - u8 op = result ? IOAT_OP_PQ_VAL : IOAT_OP_PQ; - int i, s, idx, with_ext, num_descs; - int cb32 = (device->version < IOAT_VER_3_3) ? 1 : 0; - - dev_dbg(to_dev(chan), "%s\n", __func__); - /* the engine requires at least two sources (we provide - * at least 1 implied source in the DMA_PREP_CONTINUE case) - */ - BUG_ON(src_cnt + dmaf_continue(flags) < 2); - - num_descs = ioat2_xferlen_to_descs(ioat, len); - /* we need 2x the number of descriptors to cover greater than 3 - * sources (we need 1 extra source in the q-only continuation - * case and 3 extra sources in the p+q continuation case. - */ - if (src_cnt + dmaf_p_disabled_continue(flags) > 3 || - (dmaf_continue(flags) && !dmaf_p_disabled_continue(flags))) { - with_ext = 1; - num_descs *= 2; - } else - with_ext = 0; - - /* completion writes from the raid engine may pass completion - * writes from the legacy engine, so we need one extra null - * (legacy) descriptor to ensure all completion writes arrive in - * order. - */ - if (likely(num_descs) && - ioat2_check_space_lock(ioat, num_descs + cb32) == 0) - idx = ioat->head; - else - return NULL; - i = 0; - do { - struct ioat_raw_descriptor *descs[2]; - size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log); - - desc = ioat2_get_ring_ent(ioat, idx + i); - pq = desc->pq; - - /* save a branch by unconditionally retrieving the - * extended descriptor pq_set_src() knows to not write - * to it in the single descriptor case - */ - ext = ioat2_get_ring_ent(ioat, idx + i + with_ext); - pq_ex = ext->pq_ex; - - descs[0] = (struct ioat_raw_descriptor *) pq; - descs[1] = (struct ioat_raw_descriptor *) pq_ex; - - for (s = 0; s < src_cnt; s++) - pq_set_src(descs, src[s], offset, scf[s], s); - - /* see the comment for dma_maxpq in include/linux/dmaengine.h */ - if (dmaf_p_disabled_continue(flags)) - pq_set_src(descs, dst[1], offset, 1, s++); - else if (dmaf_continue(flags)) { - pq_set_src(descs, dst[0], offset, 0, s++); - pq_set_src(descs, dst[1], offset, 1, s++); - pq_set_src(descs, dst[1], offset, 0, s++); - } - pq->size = xfer_size; - pq->p_addr = dst[0] + offset; - pq->q_addr = dst[1] + offset; - pq->ctl = 0; - pq->ctl_f.op = op; - /* we turn on descriptor write back error status */ - if (device->cap & IOAT_CAP_DWBES) - pq->ctl_f.wb_en = result ? 1 : 0; - pq->ctl_f.src_cnt = src_cnt_to_hw(s); - pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P); - pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q); - - len -= xfer_size; - offset += xfer_size; - } while ((i += 1 + with_ext) < num_descs); - - /* last pq descriptor carries the unmap parameters and fence bit */ - desc->txd.flags = flags; - desc->len = total_len; - if (result) - desc->result = result; - pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE); - dump_pq_desc_dbg(ioat, desc, ext); - - if (!cb32) { - pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); - pq->ctl_f.compl_write = 1; - compl_desc = desc; - } else { - /* completion descriptor carries interrupt bit */ - compl_desc = ioat2_get_ring_ent(ioat, idx + i); - compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT; - hw = compl_desc->hw; - hw->ctl = 0; - hw->ctl_f.null = 1; - hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); - hw->ctl_f.compl_write = 1; - hw->size = NULL_DESC_BUFFER_SIZE; - dump_desc_dbg(ioat, compl_desc); - } - - - /* we leave the channel locked to ensure in order submission */ - return &compl_desc->txd; -} - -static struct dma_async_tx_descriptor * -__ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result, - const dma_addr_t *dst, const dma_addr_t *src, - unsigned int src_cnt, const unsigned char *scf, - size_t len, unsigned long flags) -{ - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - struct ioat_chan_common *chan = &ioat->base; - struct ioatdma_device *device = chan->device; - struct ioat_ring_ent *desc; - size_t total_len = len; - struct ioat_pq_descriptor *pq; - u32 offset = 0; - u8 op; - int i, s, idx, num_descs; - - /* this function is only called with 9-16 sources */ - op = result ? IOAT_OP_PQ_VAL_16S : IOAT_OP_PQ_16S; - - dev_dbg(to_dev(chan), "%s\n", __func__); - - num_descs = ioat2_xferlen_to_descs(ioat, len); - - /* - * 16 source pq is only available on cb3.3 and has no completion - * write hw bug. - */ - if (num_descs && ioat2_check_space_lock(ioat, num_descs) == 0) - idx = ioat->head; - else - return NULL; - - i = 0; - - do { - struct ioat_raw_descriptor *descs[4]; - size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log); - - desc = ioat2_get_ring_ent(ioat, idx + i); - pq = desc->pq; - - descs[0] = (struct ioat_raw_descriptor *) pq; - - desc->sed = ioat3_alloc_sed(device, (src_cnt-2) >> 3); - if (!desc->sed) { - dev_err(to_dev(chan), - "%s: no free sed entries\n", __func__); - return NULL; - } - - pq->sed_addr = desc->sed->dma; - desc->sed->parent = desc; - - descs[1] = (struct ioat_raw_descriptor *)desc->sed->hw; - descs[2] = (void *)descs[1] + 64; - - for (s = 0; s < src_cnt; s++) - pq16_set_src(descs, src[s], offset, scf[s], s); - - /* see the comment for dma_maxpq in include/linux/dmaengine.h */ - if (dmaf_p_disabled_continue(flags)) - pq16_set_src(descs, dst[1], offset, 1, s++); - else if (dmaf_continue(flags)) { - pq16_set_src(descs, dst[0], offset, 0, s++); - pq16_set_src(descs, dst[1], offset, 1, s++); - pq16_set_src(descs, dst[1], offset, 0, s++); - } - - pq->size = xfer_size; - pq->p_addr = dst[0] + offset; - pq->q_addr = dst[1] + offset; - pq->ctl = 0; - pq->ctl_f.op = op; - pq->ctl_f.src_cnt = src16_cnt_to_hw(s); - /* we turn on descriptor write back error status */ - if (device->cap & IOAT_CAP_DWBES) - pq->ctl_f.wb_en = result ? 1 : 0; - pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P); - pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q); - - len -= xfer_size; - offset += xfer_size; - } while (++i < num_descs); - - /* last pq descriptor carries the unmap parameters and fence bit */ - desc->txd.flags = flags; - desc->len = total_len; - if (result) - desc->result = result; - pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE); - - /* with cb3.3 we should be able to do completion w/o a null desc */ - pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); - pq->ctl_f.compl_write = 1; - - dump_pq16_desc_dbg(ioat, desc); - - /* we leave the channel locked to ensure in order submission */ - return &desc->txd; -} - -static int src_cnt_flags(unsigned int src_cnt, unsigned long flags) -{ - if (dmaf_p_disabled_continue(flags)) - return src_cnt + 1; - else if (dmaf_continue(flags)) - return src_cnt + 3; - else - return src_cnt; -} - -static struct dma_async_tx_descriptor * -ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, - unsigned int src_cnt, const unsigned char *scf, size_t len, - unsigned long flags) -{ - /* specify valid address for disabled result */ - if (flags & DMA_PREP_PQ_DISABLE_P) - dst[0] = dst[1]; - if (flags & DMA_PREP_PQ_DISABLE_Q) - dst[1] = dst[0]; - - /* handle the single source multiply case from the raid6 - * recovery path - */ - if ((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1) { - dma_addr_t single_source[2]; - unsigned char single_source_coef[2]; - - BUG_ON(flags & DMA_PREP_PQ_DISABLE_Q); - single_source[0] = src[0]; - single_source[1] = src[0]; - single_source_coef[0] = scf[0]; - single_source_coef[1] = 0; - - return src_cnt_flags(src_cnt, flags) > 8 ? - __ioat3_prep_pq16_lock(chan, NULL, dst, single_source, - 2, single_source_coef, len, - flags) : - __ioat3_prep_pq_lock(chan, NULL, dst, single_source, 2, - single_source_coef, len, flags); - - } else { - return src_cnt_flags(src_cnt, flags) > 8 ? - __ioat3_prep_pq16_lock(chan, NULL, dst, src, src_cnt, - scf, len, flags) : - __ioat3_prep_pq_lock(chan, NULL, dst, src, src_cnt, - scf, len, flags); - } -} - -static struct dma_async_tx_descriptor * -ioat3_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, - unsigned int src_cnt, const unsigned char *scf, size_t len, - enum sum_check_flags *pqres, unsigned long flags) -{ - /* specify valid address for disabled result */ - if (flags & DMA_PREP_PQ_DISABLE_P) - pq[0] = pq[1]; - if (flags & DMA_PREP_PQ_DISABLE_Q) - pq[1] = pq[0]; - - /* the cleanup routine only sets bits on validate failure, it - * does not clear bits on validate success... so clear it here - */ - *pqres = 0; - - return src_cnt_flags(src_cnt, flags) > 8 ? - __ioat3_prep_pq16_lock(chan, pqres, pq, src, src_cnt, scf, len, - flags) : - __ioat3_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len, - flags); -} - -static struct dma_async_tx_descriptor * -ioat3_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src, - unsigned int src_cnt, size_t len, unsigned long flags) -{ - unsigned char scf[src_cnt]; - dma_addr_t pq[2]; - - memset(scf, 0, src_cnt); - pq[0] = dst; - flags |= DMA_PREP_PQ_DISABLE_Q; - pq[1] = dst; /* specify valid address for disabled result */ - - return src_cnt_flags(src_cnt, flags) > 8 ? - __ioat3_prep_pq16_lock(chan, NULL, pq, src, src_cnt, scf, len, - flags) : - __ioat3_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len, - flags); -} - -static struct dma_async_tx_descriptor * -ioat3_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src, - unsigned int src_cnt, size_t len, - enum sum_check_flags *result, unsigned long flags) -{ - unsigned char scf[src_cnt]; - dma_addr_t pq[2]; - - /* the cleanup routine only sets bits on validate failure, it - * does not clear bits on validate success... so clear it here - */ - *result = 0; - - memset(scf, 0, src_cnt); - pq[0] = src[0]; - flags |= DMA_PREP_PQ_DISABLE_Q; - pq[1] = pq[0]; /* specify valid address for disabled result */ - - return src_cnt_flags(src_cnt, flags) > 8 ? - __ioat3_prep_pq16_lock(chan, result, pq, &src[1], src_cnt - 1, - scf, len, flags) : - __ioat3_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1, - scf, len, flags); -} - -static struct dma_async_tx_descriptor * -ioat3_prep_interrupt_lock(struct dma_chan *c, unsigned long flags) -{ - struct ioat2_dma_chan *ioat = to_ioat2_chan(c); - struct ioat_ring_ent *desc; - struct ioat_dma_descriptor *hw; - - if (ioat2_check_space_lock(ioat, 1) == 0) - desc = ioat2_get_ring_ent(ioat, ioat->head); - else - return NULL; - - hw = desc->hw; - hw->ctl = 0; - hw->ctl_f.null = 1; - hw->ctl_f.int_en = 1; - hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE); - hw->ctl_f.compl_write = 1; - hw->size = NULL_DESC_BUFFER_SIZE; - hw->src_addr = 0; - hw->dst_addr = 0; - - desc->txd.flags = flags; - desc->len = 1; - - dump_desc_dbg(ioat, desc); - - /* we leave the channel locked to ensure in order submission */ - return &desc->txd; -} - -static void ioat3_dma_test_callback(void *dma_async_param) -{ - struct completion *cmp = dma_async_param; - - complete(cmp); -} - -#define IOAT_NUM_SRC_TEST 6 /* must be <= 8 */ -static int ioat_xor_val_self_test(struct ioatdma_device *device) -{ - int i, src_idx; - struct page *dest; - struct page *xor_srcs[IOAT_NUM_SRC_TEST]; - struct page *xor_val_srcs[IOAT_NUM_SRC_TEST + 1]; - dma_addr_t dma_srcs[IOAT_NUM_SRC_TEST + 1]; - dma_addr_t dest_dma; - struct dma_async_tx_descriptor *tx; - struct dma_chan *dma_chan; - dma_cookie_t cookie; - u8 cmp_byte = 0; - u32 cmp_word; - u32 xor_val_result; - int err = 0; - struct completion cmp; - unsigned long tmo; - struct device *dev = &device->pdev->dev; - struct dma_device *dma = &device->common; - u8 op = 0; - - dev_dbg(dev, "%s\n", __func__); - - if (!dma_has_cap(DMA_XOR, dma->cap_mask)) - return 0; - - for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) { - xor_srcs[src_idx] = alloc_page(GFP_KERNEL); - if (!xor_srcs[src_idx]) { - while (src_idx--) - __free_page(xor_srcs[src_idx]); - return -ENOMEM; - } - } - - dest = alloc_page(GFP_KERNEL); - if (!dest) { - while (src_idx--) - __free_page(xor_srcs[src_idx]); - return -ENOMEM; - } - - /* Fill in src buffers */ - for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) { - u8 *ptr = page_address(xor_srcs[src_idx]); - for (i = 0; i < PAGE_SIZE; i++) - ptr[i] = (1 << src_idx); - } - - for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) - cmp_byte ^= (u8) (1 << src_idx); - - cmp_word = (cmp_byte << 24) | (cmp_byte << 16) | - (cmp_byte << 8) | cmp_byte; - - memset(page_address(dest), 0, PAGE_SIZE); - - dma_chan = container_of(dma->channels.next, struct dma_chan, - device_node); - if (dma->device_alloc_chan_resources(dma_chan) < 1) { - err = -ENODEV; - goto out; - } - - /* test xor */ - op = IOAT_OP_XOR; - - dest_dma = dma_map_page(dev, dest, 0, PAGE_SIZE, DMA_FROM_DEVICE); - if (dma_mapping_error(dev, dest_dma)) - goto dma_unmap; - - for (i = 0; i < IOAT_NUM_SRC_TEST; i++) - dma_srcs[i] = DMA_ERROR_CODE; - for (i = 0; i < IOAT_NUM_SRC_TEST; i++) { - dma_srcs[i] = dma_map_page(dev, xor_srcs[i], 0, PAGE_SIZE, - DMA_TO_DEVICE); - if (dma_mapping_error(dev, dma_srcs[i])) - goto dma_unmap; - } - tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs, - IOAT_NUM_SRC_TEST, PAGE_SIZE, - DMA_PREP_INTERRUPT); - - if (!tx) { - dev_err(dev, "Self-test xor prep failed\n"); - err = -ENODEV; - goto dma_unmap; - } - - async_tx_ack(tx); - init_completion(&cmp); - tx->callback = ioat3_dma_test_callback; - tx->callback_param = &cmp; - cookie = tx->tx_submit(tx); - if (cookie < 0) { - dev_err(dev, "Self-test xor setup failed\n"); - err = -ENODEV; - goto dma_unmap; - } - dma->device_issue_pending(dma_chan); - - tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); - - if (tmo == 0 || - dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { - dev_err(dev, "Self-test xor timed out\n"); - err = -ENODEV; - goto dma_unmap; - } - - for (i = 0; i < IOAT_NUM_SRC_TEST; i++) - dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE); - - dma_sync_single_for_cpu(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE); - for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) { - u32 *ptr = page_address(dest); - if (ptr[i] != cmp_word) { - dev_err(dev, "Self-test xor failed compare\n"); - err = -ENODEV; - goto free_resources; - } - } - dma_sync_single_for_device(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE); - - dma_unmap_page(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE); - - /* skip validate if the capability is not present */ - if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask)) - goto free_resources; - - op = IOAT_OP_XOR_VAL; - - /* validate the sources with the destintation page */ - for (i = 0; i < IOAT_NUM_SRC_TEST; i++) - xor_val_srcs[i] = xor_srcs[i]; - xor_val_srcs[i] = dest; - - xor_val_result = 1; - - for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) - dma_srcs[i] = DMA_ERROR_CODE; - for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) { - dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE, - DMA_TO_DEVICE); - if (dma_mapping_error(dev, dma_srcs[i])) - goto dma_unmap; - } - tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs, - IOAT_NUM_SRC_TEST + 1, PAGE_SIZE, - &xor_val_result, DMA_PREP_INTERRUPT); - if (!tx) { - dev_err(dev, "Self-test zero prep failed\n"); - err = -ENODEV; - goto dma_unmap; - } - - async_tx_ack(tx); - init_completion(&cmp); - tx->callback = ioat3_dma_test_callback; - tx->callback_param = &cmp; - cookie = tx->tx_submit(tx); - if (cookie < 0) { - dev_err(dev, "Self-test zero setup failed\n"); - err = -ENODEV; - goto dma_unmap; - } - dma->device_issue_pending(dma_chan); - - tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); - - if (tmo == 0 || - dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { - dev_err(dev, "Self-test validate timed out\n"); - err = -ENODEV; - goto dma_unmap; - } - - for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) - dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE); - - if (xor_val_result != 0) { - dev_err(dev, "Self-test validate failed compare\n"); - err = -ENODEV; - goto free_resources; - } - - memset(page_address(dest), 0, PAGE_SIZE); - - /* test for non-zero parity sum */ - op = IOAT_OP_XOR_VAL; - - xor_val_result = 0; - for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) - dma_srcs[i] = DMA_ERROR_CODE; - for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) { - dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE, - DMA_TO_DEVICE); - if (dma_mapping_error(dev, dma_srcs[i])) - goto dma_unmap; - } - tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs, - IOAT_NUM_SRC_TEST + 1, PAGE_SIZE, - &xor_val_result, DMA_PREP_INTERRUPT); - if (!tx) { - dev_err(dev, "Self-test 2nd zero prep failed\n"); - err = -ENODEV; - goto dma_unmap; - } - - async_tx_ack(tx); - init_completion(&cmp); - tx->callback = ioat3_dma_test_callback; - tx->callback_param = &cmp; - cookie = tx->tx_submit(tx); - if (cookie < 0) { - dev_err(dev, "Self-test 2nd zero setup failed\n"); - err = -ENODEV; - goto dma_unmap; - } - dma->device_issue_pending(dma_chan); - - tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); - - if (tmo == 0 || - dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { - dev_err(dev, "Self-test 2nd validate timed out\n"); - err = -ENODEV; - goto dma_unmap; - } - - if (xor_val_result != SUM_CHECK_P_RESULT) { - dev_err(dev, "Self-test validate failed compare\n"); - err = -ENODEV; - goto dma_unmap; - } - - for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) - dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE); - - goto free_resources; -dma_unmap: - if (op == IOAT_OP_XOR) { - if (dest_dma != DMA_ERROR_CODE) - dma_unmap_page(dev, dest_dma, PAGE_SIZE, - DMA_FROM_DEVICE); - for (i = 0; i < IOAT_NUM_SRC_TEST; i++) - if (dma_srcs[i] != DMA_ERROR_CODE) - dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, - DMA_TO_DEVICE); - } else if (op == IOAT_OP_XOR_VAL) { - for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) - if (dma_srcs[i] != DMA_ERROR_CODE) - dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, - DMA_TO_DEVICE); - } -free_resources: - dma->device_free_chan_resources(dma_chan); -out: - src_idx = IOAT_NUM_SRC_TEST; - while (src_idx--) - __free_page(xor_srcs[src_idx]); - __free_page(dest); - return err; -} - -static int ioat3_dma_self_test(struct ioatdma_device *device) -{ - int rc = ioat_dma_self_test(device); - - if (rc) - return rc; - - rc = ioat_xor_val_self_test(device); - if (rc) - return rc; - - return 0; -} - -static int ioat3_irq_reinit(struct ioatdma_device *device) -{ - struct pci_dev *pdev = device->pdev; - int irq = pdev->irq, i; - - if (!is_bwd_ioat(pdev)) - return 0; - - switch (device->irq_mode) { - case IOAT_MSIX: - for (i = 0; i < device->common.chancnt; i++) { - struct msix_entry *msix = &device->msix_entries[i]; - struct ioat_chan_common *chan; - - chan = ioat_chan_by_index(device, i); - devm_free_irq(&pdev->dev, msix->vector, chan); - } - - pci_disable_msix(pdev); - break; - case IOAT_MSI: - pci_disable_msi(pdev); - /* fall through */ - case IOAT_INTX: - devm_free_irq(&pdev->dev, irq, device); - break; - default: - return 0; - } - device->irq_mode = IOAT_NOIRQ; - - return ioat_dma_setup_interrupts(device); -} - -static int ioat3_reset_hw(struct ioat_chan_common *chan) -{ - /* throw away whatever the channel was doing and get it - * initialized, with ioat3 specific workarounds - */ - struct ioatdma_device *device = chan->device; - struct pci_dev *pdev = device->pdev; - u32 chanerr; - u16 dev_id; - int err; - - ioat2_quiesce(chan, msecs_to_jiffies(100)); - - chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); - writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET); - - if (device->version < IOAT_VER_3_3) { - /* clear any pending errors */ - err = pci_read_config_dword(pdev, - IOAT_PCI_CHANERR_INT_OFFSET, &chanerr); - if (err) { - dev_err(&pdev->dev, - "channel error register unreachable\n"); - return err; - } - pci_write_config_dword(pdev, - IOAT_PCI_CHANERR_INT_OFFSET, chanerr); - - /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit - * (workaround for spurious config parity error after restart) - */ - pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id); - if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0) { - pci_write_config_dword(pdev, - IOAT_PCI_DMAUNCERRSTS_OFFSET, - 0x10); - } - } - - err = ioat2_reset_sync(chan, msecs_to_jiffies(200)); - if (!err) - err = ioat3_irq_reinit(device); - - if (err) - dev_err(&pdev->dev, "Failed to reset: %d\n", err); - - return err; -} - -static void ioat3_intr_quirk(struct ioatdma_device *device) -{ - struct dma_device *dma; - struct dma_chan *c; - struct ioat_chan_common *chan; - u32 errmask; - - dma = &device->common; - - /* - * if we have descriptor write back error status, we mask the - * error interrupts - */ - if (device->cap & IOAT_CAP_DWBES) { - list_for_each_entry(c, &dma->channels, device_node) { - chan = to_chan_common(c); - errmask = readl(chan->reg_base + - IOAT_CHANERR_MASK_OFFSET); - errmask |= IOAT_CHANERR_XOR_P_OR_CRC_ERR | - IOAT_CHANERR_XOR_Q_ERR; - writel(errmask, chan->reg_base + - IOAT_CHANERR_MASK_OFFSET); - } - } -} - -int ioat3_dma_probe(struct ioatdma_device *device, int dca) -{ - struct pci_dev *pdev = device->pdev; - int dca_en = system_has_dca_enabled(pdev); - struct dma_device *dma; - struct dma_chan *c; - struct ioat_chan_common *chan; - bool is_raid_device = false; - int err; - - device->enumerate_channels = ioat2_enumerate_channels; - device->reset_hw = ioat3_reset_hw; - device->self_test = ioat3_dma_self_test; - device->intr_quirk = ioat3_intr_quirk; - dma = &device->common; - dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock; - dma->device_issue_pending = ioat2_issue_pending; - dma->device_alloc_chan_resources = ioat2_alloc_chan_resources; - dma->device_free_chan_resources = ioat2_free_chan_resources; - - dma_cap_set(DMA_INTERRUPT, dma->cap_mask); - dma->device_prep_dma_interrupt = ioat3_prep_interrupt_lock; - - device->cap = readl(device->reg_base + IOAT_DMA_CAP_OFFSET); - - if (is_xeon_cb32(pdev) || is_bwd_noraid(pdev)) - device->cap &= ~(IOAT_CAP_XOR | IOAT_CAP_PQ | IOAT_CAP_RAID16SS); - - /* dca is incompatible with raid operations */ - if (dca_en && (device->cap & (IOAT_CAP_XOR|IOAT_CAP_PQ))) - device->cap &= ~(IOAT_CAP_XOR|IOAT_CAP_PQ); - - if (device->cap & IOAT_CAP_XOR) { - is_raid_device = true; - dma->max_xor = 8; - - dma_cap_set(DMA_XOR, dma->cap_mask); - dma->device_prep_dma_xor = ioat3_prep_xor; - - dma_cap_set(DMA_XOR_VAL, dma->cap_mask); - dma->device_prep_dma_xor_val = ioat3_prep_xor_val; - } - - if (device->cap & IOAT_CAP_PQ) { - is_raid_device = true; - - dma->device_prep_dma_pq = ioat3_prep_pq; - dma->device_prep_dma_pq_val = ioat3_prep_pq_val; - dma_cap_set(DMA_PQ, dma->cap_mask); - dma_cap_set(DMA_PQ_VAL, dma->cap_mask); - - if (device->cap & IOAT_CAP_RAID16SS) { - dma_set_maxpq(dma, 16, 0); - } else { - dma_set_maxpq(dma, 8, 0); - } - - if (!(device->cap & IOAT_CAP_XOR)) { - dma->device_prep_dma_xor = ioat3_prep_pqxor; - dma->device_prep_dma_xor_val = ioat3_prep_pqxor_val; - dma_cap_set(DMA_XOR, dma->cap_mask); - dma_cap_set(DMA_XOR_VAL, dma->cap_mask); - - if (device->cap & IOAT_CAP_RAID16SS) { - dma->max_xor = 16; - } else { - dma->max_xor = 8; - } - } - } - - dma->device_tx_status = ioat3_tx_status; - device->cleanup_fn = ioat3_cleanup_event; - device->timer_fn = ioat3_timer_event; - - /* starting with CB3.3 super extended descriptors are supported */ - if (device->cap & IOAT_CAP_RAID16SS) { - char pool_name[14]; - int i; - - for (i = 0; i < MAX_SED_POOLS; i++) { - snprintf(pool_name, 14, "ioat_hw%d_sed", i); - - /* allocate SED DMA pool */ - device->sed_hw_pool[i] = dmam_pool_create(pool_name, - &pdev->dev, - SED_SIZE * (i + 1), 64, 0); - if (!device->sed_hw_pool[i]) - return -ENOMEM; - - } - } - - err = ioat_probe(device); - if (err) - return err; - - list_for_each_entry(c, &dma->channels, device_node) { - chan = to_chan_common(c); - writel(IOAT_DMA_DCA_ANY_CPU, - chan->reg_base + IOAT_DCACTRL_OFFSET); - } - - err = ioat_register(device); - if (err) - return err; - - ioat_kobject_add(device, &ioat2_ktype); - - if (dca) - device->dca = ioat3_dca_init(pdev, device->reg_base); - - return 0; -} diff --git a/drivers/dma/ioat/hw.h b/drivers/dma/ioat/hw.h index a3e731edce57..690e3b4f8202 100644 --- a/drivers/dma/ioat/hw.h +++ b/drivers/dma/ioat/hw.h @@ -21,11 +21,6 @@ #define IOAT_MMIO_BAR 0 /* CB device ID's */ -#define IOAT_PCI_DID_5000 0x1A38 -#define IOAT_PCI_DID_CNB 0x360B -#define IOAT_PCI_DID_SCNB 0x65FF -#define IOAT_PCI_DID_SNB 0x402F - #define PCI_DEVICE_ID_INTEL_IOAT_IVB0 0x0e20 #define PCI_DEVICE_ID_INTEL_IOAT_IVB1 0x0e21 #define PCI_DEVICE_ID_INTEL_IOAT_IVB2 0x0e22 @@ -58,6 +53,17 @@ #define PCI_DEVICE_ID_INTEL_IOAT_BDXDE2 0x6f52 #define PCI_DEVICE_ID_INTEL_IOAT_BDXDE3 0x6f53 +#define PCI_DEVICE_ID_INTEL_IOAT_BDX0 0x6f20 +#define PCI_DEVICE_ID_INTEL_IOAT_BDX1 0x6f21 +#define PCI_DEVICE_ID_INTEL_IOAT_BDX2 0x6f22 +#define PCI_DEVICE_ID_INTEL_IOAT_BDX3 0x6f23 +#define PCI_DEVICE_ID_INTEL_IOAT_BDX4 0x6f24 +#define PCI_DEVICE_ID_INTEL_IOAT_BDX5 0x6f25 +#define PCI_DEVICE_ID_INTEL_IOAT_BDX6 0x6f26 +#define PCI_DEVICE_ID_INTEL_IOAT_BDX7 0x6f27 +#define PCI_DEVICE_ID_INTEL_IOAT_BDX8 0x6f2e +#define PCI_DEVICE_ID_INTEL_IOAT_BDX9 0x6f2f + #define IOAT_VER_1_2 0x12 /* Version 1.2 */ #define IOAT_VER_2_0 0x20 /* Version 2.0 */ #define IOAT_VER_3_0 0x30 /* Version 3.0 */ diff --git a/drivers/dma/ioat/init.c b/drivers/dma/ioat/init.c new file mode 100644 index 000000000000..1c3c9b0abf4e --- /dev/null +++ b/drivers/dma/ioat/init.c @@ -0,0 +1,1314 @@ +/* + * Intel I/OAT DMA Linux driver + * Copyright(c) 2004 - 2015 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * The full GNU General Public License is included in this distribution in + * the file called "COPYING". + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "dma.h" +#include "registers.h" +#include "hw.h" + +#include "../dmaengine.h" + +MODULE_VERSION(IOAT_DMA_VERSION); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel Corporation"); + +static struct pci_device_id ioat_pci_tbl[] = { + /* I/OAT v3 platforms */ + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG2) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG3) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG4) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG5) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG6) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG7) }, + + /* I/OAT v3.2 platforms */ + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF0) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF1) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF2) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF3) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF4) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF5) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF6) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF7) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF8) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF9) }, + + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB0) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB1) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB2) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB3) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB4) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB5) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB6) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB7) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB8) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB9) }, + + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB0) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB1) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB2) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB3) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB4) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB5) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB6) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB7) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB8) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB9) }, + + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW0) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW1) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW2) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW3) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW4) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW5) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW6) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW7) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW8) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW9) }, + + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX0) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX1) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX2) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX3) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX4) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX5) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX6) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX7) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX8) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX9) }, + + /* I/OAT v3.3 platforms */ + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD0) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD1) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD2) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD3) }, + + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE0) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE1) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE2) }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE3) }, + + { 0, } +}; +MODULE_DEVICE_TABLE(pci, ioat_pci_tbl); + +static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id); +static void ioat_remove(struct pci_dev *pdev); +static void +ioat_init_channel(struct ioatdma_device *ioat_dma, + struct ioatdma_chan *ioat_chan, int idx); +static void ioat_intr_quirk(struct ioatdma_device *ioat_dma); +static int ioat_enumerate_channels(struct ioatdma_device *ioat_dma); +static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma); + +static int ioat_dca_enabled = 1; +module_param(ioat_dca_enabled, int, 0644); +MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)"); +int ioat_pending_level = 4; +module_param(ioat_pending_level, int, 0644); +MODULE_PARM_DESC(ioat_pending_level, + "high-water mark for pushing ioat descriptors (default: 4)"); +int ioat_ring_alloc_order = 8; +module_param(ioat_ring_alloc_order, int, 0644); +MODULE_PARM_DESC(ioat_ring_alloc_order, + "ioat+: allocate 2^n descriptors per channel (default: 8 max: 16)"); +int ioat_ring_max_alloc_order = IOAT_MAX_ORDER; +module_param(ioat_ring_max_alloc_order, int, 0644); +MODULE_PARM_DESC(ioat_ring_max_alloc_order, + "ioat+: upper limit for ring size (default: 16)"); +static char ioat_interrupt_style[32] = "msix"; +module_param_string(ioat_interrupt_style, ioat_interrupt_style, + sizeof(ioat_interrupt_style), 0644); +MODULE_PARM_DESC(ioat_interrupt_style, + "set ioat interrupt style: msix (default), msi, intx"); + +struct kmem_cache *ioat_cache; +struct kmem_cache *ioat_sed_cache; + +static bool is_jf_ioat(struct pci_dev *pdev) +{ + switch (pdev->device) { + case PCI_DEVICE_ID_INTEL_IOAT_JSF0: + case PCI_DEVICE_ID_INTEL_IOAT_JSF1: + case PCI_DEVICE_ID_INTEL_IOAT_JSF2: + case PCI_DEVICE_ID_INTEL_IOAT_JSF3: + case PCI_DEVICE_ID_INTEL_IOAT_JSF4: + case PCI_DEVICE_ID_INTEL_IOAT_JSF5: + case PCI_DEVICE_ID_INTEL_IOAT_JSF6: + case PCI_DEVICE_ID_INTEL_IOAT_JSF7: + case PCI_DEVICE_ID_INTEL_IOAT_JSF8: + case PCI_DEVICE_ID_INTEL_IOAT_JSF9: + return true; + default: + return false; + } +} + +static bool is_snb_ioat(struct pci_dev *pdev) +{ + switch (pdev->device) { + case PCI_DEVICE_ID_INTEL_IOAT_SNB0: + case PCI_DEVICE_ID_INTEL_IOAT_SNB1: + case PCI_DEVICE_ID_INTEL_IOAT_SNB2: + case PCI_DEVICE_ID_INTEL_IOAT_SNB3: + case PCI_DEVICE_ID_INTEL_IOAT_SNB4: + case PCI_DEVICE_ID_INTEL_IOAT_SNB5: + case PCI_DEVICE_ID_INTEL_IOAT_SNB6: + case PCI_DEVICE_ID_INTEL_IOAT_SNB7: + case PCI_DEVICE_ID_INTEL_IOAT_SNB8: + case PCI_DEVICE_ID_INTEL_IOAT_SNB9: + return true; + default: + return false; + } +} + +static bool is_ivb_ioat(struct pci_dev *pdev) +{ + switch (pdev->device) { + case PCI_DEVICE_ID_INTEL_IOAT_IVB0: + case PCI_DEVICE_ID_INTEL_IOAT_IVB1: + case PCI_DEVICE_ID_INTEL_IOAT_IVB2: + case PCI_DEVICE_ID_INTEL_IOAT_IVB3: + case PCI_DEVICE_ID_INTEL_IOAT_IVB4: + case PCI_DEVICE_ID_INTEL_IOAT_IVB5: + case PCI_DEVICE_ID_INTEL_IOAT_IVB6: + case PCI_DEVICE_ID_INTEL_IOAT_IVB7: + case PCI_DEVICE_ID_INTEL_IOAT_IVB8: + case PCI_DEVICE_ID_INTEL_IOAT_IVB9: + return true; + default: + return false; + } + +} + +static bool is_hsw_ioat(struct pci_dev *pdev) +{ + switch (pdev->device) { + case PCI_DEVICE_ID_INTEL_IOAT_HSW0: + case PCI_DEVICE_ID_INTEL_IOAT_HSW1: + case PCI_DEVICE_ID_INTEL_IOAT_HSW2: + case PCI_DEVICE_ID_INTEL_IOAT_HSW3: + case PCI_DEVICE_ID_INTEL_IOAT_HSW4: + case PCI_DEVICE_ID_INTEL_IOAT_HSW5: + case PCI_DEVICE_ID_INTEL_IOAT_HSW6: + case PCI_DEVICE_ID_INTEL_IOAT_HSW7: + case PCI_DEVICE_ID_INTEL_IOAT_HSW8: + case PCI_DEVICE_ID_INTEL_IOAT_HSW9: + return true; + default: + return false; + } + +} + +static bool is_bdx_ioat(struct pci_dev *pdev) +{ + switch (pdev->device) { + case PCI_DEVICE_ID_INTEL_IOAT_BDX0: + case PCI_DEVICE_ID_INTEL_IOAT_BDX1: + case PCI_DEVICE_ID_INTEL_IOAT_BDX2: + case PCI_DEVICE_ID_INTEL_IOAT_BDX3: + case PCI_DEVICE_ID_INTEL_IOAT_BDX4: + case PCI_DEVICE_ID_INTEL_IOAT_BDX5: + case PCI_DEVICE_ID_INTEL_IOAT_BDX6: + case PCI_DEVICE_ID_INTEL_IOAT_BDX7: + case PCI_DEVICE_ID_INTEL_IOAT_BDX8: + case PCI_DEVICE_ID_INTEL_IOAT_BDX9: + return true; + default: + return false; + } +} + +static bool is_xeon_cb32(struct pci_dev *pdev) +{ + return is_jf_ioat(pdev) || is_snb_ioat(pdev) || is_ivb_ioat(pdev) || + is_hsw_ioat(pdev) || is_bdx_ioat(pdev); +} + +bool is_bwd_ioat(struct pci_dev *pdev) +{ + switch (pdev->device) { + case PCI_DEVICE_ID_INTEL_IOAT_BWD0: + case PCI_DEVICE_ID_INTEL_IOAT_BWD1: + case PCI_DEVICE_ID_INTEL_IOAT_BWD2: + case PCI_DEVICE_ID_INTEL_IOAT_BWD3: + /* even though not Atom, BDX-DE has same DMA silicon */ + case PCI_DEVICE_ID_INTEL_IOAT_BDXDE0: + case PCI_DEVICE_ID_INTEL_IOAT_BDXDE1: + case PCI_DEVICE_ID_INTEL_IOAT_BDXDE2: + case PCI_DEVICE_ID_INTEL_IOAT_BDXDE3: + return true; + default: + return false; + } +} + +static bool is_bwd_noraid(struct pci_dev *pdev) +{ + switch (pdev->device) { + case PCI_DEVICE_ID_INTEL_IOAT_BWD2: + case PCI_DEVICE_ID_INTEL_IOAT_BWD3: + case PCI_DEVICE_ID_INTEL_IOAT_BDXDE0: + case PCI_DEVICE_ID_INTEL_IOAT_BDXDE1: + case PCI_DEVICE_ID_INTEL_IOAT_BDXDE2: + case PCI_DEVICE_ID_INTEL_IOAT_BDXDE3: + return true; + default: + return false; + } + +} + +/* + * Perform a IOAT transaction to verify the HW works. + */ +#define IOAT_TEST_SIZE 2000 + +static void ioat_dma_test_callback(void *dma_async_param) +{ + struct completion *cmp = dma_async_param; + + complete(cmp); +} + +/** + * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works. + * @ioat_dma: dma device to be tested + */ +static int ioat_dma_self_test(struct ioatdma_device *ioat_dma) +{ + int i; + u8 *src; + u8 *dest; + struct dma_device *dma = &ioat_dma->dma_dev; + struct device *dev = &ioat_dma->pdev->dev; + struct dma_chan *dma_chan; + struct dma_async_tx_descriptor *tx; + dma_addr_t dma_dest, dma_src; + dma_cookie_t cookie; + int err = 0; + struct completion cmp; + unsigned long tmo; + unsigned long flags; + + src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL); + if (!src) + return -ENOMEM; + dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL); + if (!dest) { + kfree(src); + return -ENOMEM; + } + + /* Fill in src buffer */ + for (i = 0; i < IOAT_TEST_SIZE; i++) + src[i] = (u8)i; + + /* Start copy, using first DMA channel */ + dma_chan = container_of(dma->channels.next, struct dma_chan, + device_node); + if (dma->device_alloc_chan_resources(dma_chan) < 1) { + dev_err(dev, "selftest cannot allocate chan resource\n"); + err = -ENODEV; + goto out; + } + + dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma_src)) { + dev_err(dev, "mapping src buffer failed\n"); + goto free_resources; + } + dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE); + if (dma_mapping_error(dev, dma_dest)) { + dev_err(dev, "mapping dest buffer failed\n"); + goto unmap_src; + } + flags = DMA_PREP_INTERRUPT; + tx = ioat_dma->dma_dev.device_prep_dma_memcpy(dma_chan, dma_dest, + dma_src, IOAT_TEST_SIZE, + flags); + if (!tx) { + dev_err(dev, "Self-test prep failed, disabling\n"); + err = -ENODEV; + goto unmap_dma; + } + + async_tx_ack(tx); + init_completion(&cmp); + tx->callback = ioat_dma_test_callback; + tx->callback_param = &cmp; + cookie = tx->tx_submit(tx); + if (cookie < 0) { + dev_err(dev, "Self-test setup failed, disabling\n"); + err = -ENODEV; + goto unmap_dma; + } + dma->device_issue_pending(dma_chan); + + tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); + + if (tmo == 0 || + dma->device_tx_status(dma_chan, cookie, NULL) + != DMA_COMPLETE) { + dev_err(dev, "Self-test copy timed out, disabling\n"); + err = -ENODEV; + goto unmap_dma; + } + if (memcmp(src, dest, IOAT_TEST_SIZE)) { + dev_err(dev, "Self-test copy failed compare, disabling\n"); + err = -ENODEV; + goto free_resources; + } + +unmap_dma: + dma_unmap_single(dev, dma_dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE); +unmap_src: + dma_unmap_single(dev, dma_src, IOAT_TEST_SIZE, DMA_TO_DEVICE); +free_resources: + dma->device_free_chan_resources(dma_chan); +out: + kfree(src); + kfree(dest); + return err; +} + +/** + * ioat_dma_setup_interrupts - setup interrupt handler + * @ioat_dma: ioat dma device + */ +int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma) +{ + struct ioatdma_chan *ioat_chan; + struct pci_dev *pdev = ioat_dma->pdev; + struct device *dev = &pdev->dev; + struct msix_entry *msix; + int i, j, msixcnt; + int err = -EINVAL; + u8 intrctrl = 0; + + if (!strcmp(ioat_interrupt_style, "msix")) + goto msix; + if (!strcmp(ioat_interrupt_style, "msi")) + goto msi; + if (!strcmp(ioat_interrupt_style, "intx")) + goto intx; + dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style); + goto err_no_irq; + +msix: + /* The number of MSI-X vectors should equal the number of channels */ + msixcnt = ioat_dma->dma_dev.chancnt; + for (i = 0; i < msixcnt; i++) + ioat_dma->msix_entries[i].entry = i; + + err = pci_enable_msix_exact(pdev, ioat_dma->msix_entries, msixcnt); + if (err) + goto msi; + + for (i = 0; i < msixcnt; i++) { + msix = &ioat_dma->msix_entries[i]; + ioat_chan = ioat_chan_by_index(ioat_dma, i); + err = devm_request_irq(dev, msix->vector, + ioat_dma_do_interrupt_msix, 0, + "ioat-msix", ioat_chan); + if (err) { + for (j = 0; j < i; j++) { + msix = &ioat_dma->msix_entries[j]; + ioat_chan = ioat_chan_by_index(ioat_dma, j); + devm_free_irq(dev, msix->vector, ioat_chan); + } + goto msi; + } + } + intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL; + ioat_dma->irq_mode = IOAT_MSIX; + goto done; + +msi: + err = pci_enable_msi(pdev); + if (err) + goto intx; + + err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0, + "ioat-msi", ioat_dma); + if (err) { + pci_disable_msi(pdev); + goto intx; + } + ioat_dma->irq_mode = IOAT_MSI; + goto done; + +intx: + err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, + IRQF_SHARED, "ioat-intx", ioat_dma); + if (err) + goto err_no_irq; + + ioat_dma->irq_mode = IOAT_INTX; +done: + if (is_bwd_ioat(pdev)) + ioat_intr_quirk(ioat_dma); + intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN; + writeb(intrctrl, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET); + return 0; + +err_no_irq: + /* Disable all interrupt generation */ + writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET); + ioat_dma->irq_mode = IOAT_NOIRQ; + dev_err(dev, "no usable interrupts\n"); + return err; +} + +static void ioat_disable_interrupts(struct ioatdma_device *ioat_dma) +{ + /* Disable all interrupt generation */ + writeb(0, ioat_dma->reg_base + IOAT_INTRCTRL_OFFSET); +} + +static int ioat_probe(struct ioatdma_device *ioat_dma) +{ + int err = -ENODEV; + struct dma_device *dma = &ioat_dma->dma_dev; + struct pci_dev *pdev = ioat_dma->pdev; + struct device *dev = &pdev->dev; + + /* DMA coherent memory pool for DMA descriptor allocations */ + ioat_dma->dma_pool = pci_pool_create("dma_desc_pool", pdev, + sizeof(struct ioat_dma_descriptor), + 64, 0); + if (!ioat_dma->dma_pool) { + err = -ENOMEM; + goto err_dma_pool; + } + + ioat_dma->completion_pool = pci_pool_create("completion_pool", pdev, + sizeof(u64), + SMP_CACHE_BYTES, + SMP_CACHE_BYTES); + + if (!ioat_dma->completion_pool) { + err = -ENOMEM; + goto err_completion_pool; + } + + ioat_enumerate_channels(ioat_dma); + + dma_cap_set(DMA_MEMCPY, dma->cap_mask); + dma->dev = &pdev->dev; + + if (!dma->chancnt) { + dev_err(dev, "channel enumeration error\n"); + goto err_setup_interrupts; + } + + err = ioat_dma_setup_interrupts(ioat_dma); + if (err) + goto err_setup_interrupts; + + err = ioat3_dma_self_test(ioat_dma); + if (err) + goto err_self_test; + + return 0; + +err_self_test: + ioat_disable_interrupts(ioat_dma); +err_setup_interrupts: + pci_pool_destroy(ioat_dma->completion_pool); +err_completion_pool: + pci_pool_destroy(ioat_dma->dma_pool); +err_dma_pool: + return err; +} + +static int ioat_register(struct ioatdma_device *ioat_dma) +{ + int err = dma_async_device_register(&ioat_dma->dma_dev); + + if (err) { + ioat_disable_interrupts(ioat_dma); + pci_pool_destroy(ioat_dma->completion_pool); + pci_pool_destroy(ioat_dma->dma_pool); + } + + return err; +} + +static void ioat_dma_remove(struct ioatdma_device *ioat_dma) +{ + struct dma_device *dma = &ioat_dma->dma_dev; + + ioat_disable_interrupts(ioat_dma); + + ioat_kobject_del(ioat_dma); + + dma_async_device_unregister(dma); + + pci_pool_destroy(ioat_dma->dma_pool); + pci_pool_destroy(ioat_dma->completion_pool); + + INIT_LIST_HEAD(&dma->channels); +} + +/** + * ioat_enumerate_channels - find and initialize the device's channels + * @ioat_dma: the ioat dma device to be enumerated + */ +static int ioat_enumerate_channels(struct ioatdma_device *ioat_dma) +{ + struct ioatdma_chan *ioat_chan; + struct device *dev = &ioat_dma->pdev->dev; + struct dma_device *dma = &ioat_dma->dma_dev; + u8 xfercap_log; + int i; + + INIT_LIST_HEAD(&dma->channels); + dma->chancnt = readb(ioat_dma->reg_base + IOAT_CHANCNT_OFFSET); + dma->chancnt &= 0x1f; /* bits [4:0] valid */ + if (dma->chancnt > ARRAY_SIZE(ioat_dma->idx)) { + dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n", + dma->chancnt, ARRAY_SIZE(ioat_dma->idx)); + dma->chancnt = ARRAY_SIZE(ioat_dma->idx); + } + xfercap_log = readb(ioat_dma->reg_base + IOAT_XFERCAP_OFFSET); + xfercap_log &= 0x1f; /* bits [4:0] valid */ + if (xfercap_log == 0) + return 0; + dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log); + + for (i = 0; i < dma->chancnt; i++) { + ioat_chan = devm_kzalloc(dev, sizeof(*ioat_chan), GFP_KERNEL); + if (!ioat_chan) + break; + + ioat_init_channel(ioat_dma, ioat_chan, i); + ioat_chan->xfercap_log = xfercap_log; + spin_lock_init(&ioat_chan->prep_lock); + if (ioat_reset_hw(ioat_chan)) { + i = 0; + break; + } + } + dma->chancnt = i; + return i; +} + +/** + * ioat_free_chan_resources - release all the descriptors + * @chan: the channel to be cleaned + */ +static void ioat_free_chan_resources(struct dma_chan *c) +{ + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); + struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; + struct ioat_ring_ent *desc; + const int total_descs = 1 << ioat_chan->alloc_order; + int descs; + int i; + + /* Before freeing channel resources first check + * if they have been previously allocated for this channel. + */ + if (!ioat_chan->ring) + return; + + ioat_stop(ioat_chan); + ioat_reset_hw(ioat_chan); + + spin_lock_bh(&ioat_chan->cleanup_lock); + spin_lock_bh(&ioat_chan->prep_lock); + descs = ioat_ring_space(ioat_chan); + dev_dbg(to_dev(ioat_chan), "freeing %d idle descriptors\n", descs); + for (i = 0; i < descs; i++) { + desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head + i); + ioat_free_ring_ent(desc, c); + } + + if (descs < total_descs) + dev_err(to_dev(ioat_chan), "Freeing %d in use descriptors!\n", + total_descs - descs); + + for (i = 0; i < total_descs - descs; i++) { + desc = ioat_get_ring_ent(ioat_chan, ioat_chan->tail + i); + dump_desc_dbg(ioat_chan, desc); + ioat_free_ring_ent(desc, c); + } + + kfree(ioat_chan->ring); + ioat_chan->ring = NULL; + ioat_chan->alloc_order = 0; + pci_pool_free(ioat_dma->completion_pool, ioat_chan->completion, + ioat_chan->completion_dma); + spin_unlock_bh(&ioat_chan->prep_lock); + spin_unlock_bh(&ioat_chan->cleanup_lock); + + ioat_chan->last_completion = 0; + ioat_chan->completion_dma = 0; + ioat_chan->dmacount = 0; +} + +/* ioat_alloc_chan_resources - allocate/initialize ioat descriptor ring + * @chan: channel to be initialized + */ +static int ioat_alloc_chan_resources(struct dma_chan *c) +{ + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); + struct ioat_ring_ent **ring; + u64 status; + int order; + int i = 0; + u32 chanerr; + + /* have we already been set up? */ + if (ioat_chan->ring) + return 1 << ioat_chan->alloc_order; + + /* Setup register to interrupt and write completion status on error */ + writew(IOAT_CHANCTRL_RUN, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET); + + /* allocate a completion writeback area */ + /* doing 2 32bit writes to mmio since 1 64b write doesn't work */ + ioat_chan->completion = + pci_pool_alloc(ioat_chan->ioat_dma->completion_pool, + GFP_KERNEL, &ioat_chan->completion_dma); + if (!ioat_chan->completion) + return -ENOMEM; + + memset(ioat_chan->completion, 0, sizeof(*ioat_chan->completion)); + writel(((u64)ioat_chan->completion_dma) & 0x00000000FFFFFFFF, + ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW); + writel(((u64)ioat_chan->completion_dma) >> 32, + ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); + + order = ioat_get_alloc_order(); + ring = ioat_alloc_ring(c, order, GFP_KERNEL); + if (!ring) + return -ENOMEM; + + spin_lock_bh(&ioat_chan->cleanup_lock); + spin_lock_bh(&ioat_chan->prep_lock); + ioat_chan->ring = ring; + ioat_chan->head = 0; + ioat_chan->issued = 0; + ioat_chan->tail = 0; + ioat_chan->alloc_order = order; + set_bit(IOAT_RUN, &ioat_chan->state); + spin_unlock_bh(&ioat_chan->prep_lock); + spin_unlock_bh(&ioat_chan->cleanup_lock); + + ioat_start_null_desc(ioat_chan); + + /* check that we got off the ground */ + do { + udelay(1); + status = ioat_chansts(ioat_chan); + } while (i++ < 20 && !is_ioat_active(status) && !is_ioat_idle(status)); + + if (is_ioat_active(status) || is_ioat_idle(status)) + return 1 << ioat_chan->alloc_order; + + chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); + + dev_WARN(to_dev(ioat_chan), + "failed to start channel chanerr: %#x\n", chanerr); + ioat_free_chan_resources(c); + return -EFAULT; +} + +/* common channel initialization */ +static void +ioat_init_channel(struct ioatdma_device *ioat_dma, + struct ioatdma_chan *ioat_chan, int idx) +{ + struct dma_device *dma = &ioat_dma->dma_dev; + struct dma_chan *c = &ioat_chan->dma_chan; + unsigned long data = (unsigned long) c; + + ioat_chan->ioat_dma = ioat_dma; + ioat_chan->reg_base = ioat_dma->reg_base + (0x80 * (idx + 1)); + spin_lock_init(&ioat_chan->cleanup_lock); + ioat_chan->dma_chan.device = dma; + dma_cookie_init(&ioat_chan->dma_chan); + list_add_tail(&ioat_chan->dma_chan.device_node, &dma->channels); + ioat_dma->idx[idx] = ioat_chan; + init_timer(&ioat_chan->timer); + ioat_chan->timer.function = ioat_timer_event; + ioat_chan->timer.data = data; + tasklet_init(&ioat_chan->cleanup_task, ioat_cleanup_event, data); +} + +#define IOAT_NUM_SRC_TEST 6 /* must be <= 8 */ +static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma) +{ + int i, src_idx; + struct page *dest; + struct page *xor_srcs[IOAT_NUM_SRC_TEST]; + struct page *xor_val_srcs[IOAT_NUM_SRC_TEST + 1]; + dma_addr_t dma_srcs[IOAT_NUM_SRC_TEST + 1]; + dma_addr_t dest_dma; + struct dma_async_tx_descriptor *tx; + struct dma_chan *dma_chan; + dma_cookie_t cookie; + u8 cmp_byte = 0; + u32 cmp_word; + u32 xor_val_result; + int err = 0; + struct completion cmp; + unsigned long tmo; + struct device *dev = &ioat_dma->pdev->dev; + struct dma_device *dma = &ioat_dma->dma_dev; + u8 op = 0; + + dev_dbg(dev, "%s\n", __func__); + + if (!dma_has_cap(DMA_XOR, dma->cap_mask)) + return 0; + + for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) { + xor_srcs[src_idx] = alloc_page(GFP_KERNEL); + if (!xor_srcs[src_idx]) { + while (src_idx--) + __free_page(xor_srcs[src_idx]); + return -ENOMEM; + } + } + + dest = alloc_page(GFP_KERNEL); + if (!dest) { + while (src_idx--) + __free_page(xor_srcs[src_idx]); + return -ENOMEM; + } + + /* Fill in src buffers */ + for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) { + u8 *ptr = page_address(xor_srcs[src_idx]); + + for (i = 0; i < PAGE_SIZE; i++) + ptr[i] = (1 << src_idx); + } + + for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) + cmp_byte ^= (u8) (1 << src_idx); + + cmp_word = (cmp_byte << 24) | (cmp_byte << 16) | + (cmp_byte << 8) | cmp_byte; + + memset(page_address(dest), 0, PAGE_SIZE); + + dma_chan = container_of(dma->channels.next, struct dma_chan, + device_node); + if (dma->device_alloc_chan_resources(dma_chan) < 1) { + err = -ENODEV; + goto out; + } + + /* test xor */ + op = IOAT_OP_XOR; + + dest_dma = dma_map_page(dev, dest, 0, PAGE_SIZE, DMA_FROM_DEVICE); + if (dma_mapping_error(dev, dest_dma)) + goto dma_unmap; + + for (i = 0; i < IOAT_NUM_SRC_TEST; i++) + dma_srcs[i] = DMA_ERROR_CODE; + for (i = 0; i < IOAT_NUM_SRC_TEST; i++) { + dma_srcs[i] = dma_map_page(dev, xor_srcs[i], 0, PAGE_SIZE, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma_srcs[i])) + goto dma_unmap; + } + tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs, + IOAT_NUM_SRC_TEST, PAGE_SIZE, + DMA_PREP_INTERRUPT); + + if (!tx) { + dev_err(dev, "Self-test xor prep failed\n"); + err = -ENODEV; + goto dma_unmap; + } + + async_tx_ack(tx); + init_completion(&cmp); + tx->callback = ioat_dma_test_callback; + tx->callback_param = &cmp; + cookie = tx->tx_submit(tx); + if (cookie < 0) { + dev_err(dev, "Self-test xor setup failed\n"); + err = -ENODEV; + goto dma_unmap; + } + dma->device_issue_pending(dma_chan); + + tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); + + if (tmo == 0 || + dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { + dev_err(dev, "Self-test xor timed out\n"); + err = -ENODEV; + goto dma_unmap; + } + + for (i = 0; i < IOAT_NUM_SRC_TEST; i++) + dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE); + + dma_sync_single_for_cpu(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE); + for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) { + u32 *ptr = page_address(dest); + + if (ptr[i] != cmp_word) { + dev_err(dev, "Self-test xor failed compare\n"); + err = -ENODEV; + goto free_resources; + } + } + dma_sync_single_for_device(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE); + + dma_unmap_page(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE); + + /* skip validate if the capability is not present */ + if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask)) + goto free_resources; + + op = IOAT_OP_XOR_VAL; + + /* validate the sources with the destintation page */ + for (i = 0; i < IOAT_NUM_SRC_TEST; i++) + xor_val_srcs[i] = xor_srcs[i]; + xor_val_srcs[i] = dest; + + xor_val_result = 1; + + for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) + dma_srcs[i] = DMA_ERROR_CODE; + for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) { + dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma_srcs[i])) + goto dma_unmap; + } + tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs, + IOAT_NUM_SRC_TEST + 1, PAGE_SIZE, + &xor_val_result, DMA_PREP_INTERRUPT); + if (!tx) { + dev_err(dev, "Self-test zero prep failed\n"); + err = -ENODEV; + goto dma_unmap; + } + + async_tx_ack(tx); + init_completion(&cmp); + tx->callback = ioat_dma_test_callback; + tx->callback_param = &cmp; + cookie = tx->tx_submit(tx); + if (cookie < 0) { + dev_err(dev, "Self-test zero setup failed\n"); + err = -ENODEV; + goto dma_unmap; + } + dma->device_issue_pending(dma_chan); + + tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); + + if (tmo == 0 || + dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { + dev_err(dev, "Self-test validate timed out\n"); + err = -ENODEV; + goto dma_unmap; + } + + for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) + dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE); + + if (xor_val_result != 0) { + dev_err(dev, "Self-test validate failed compare\n"); + err = -ENODEV; + goto free_resources; + } + + memset(page_address(dest), 0, PAGE_SIZE); + + /* test for non-zero parity sum */ + op = IOAT_OP_XOR_VAL; + + xor_val_result = 0; + for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) + dma_srcs[i] = DMA_ERROR_CODE; + for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) { + dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma_srcs[i])) + goto dma_unmap; + } + tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs, + IOAT_NUM_SRC_TEST + 1, PAGE_SIZE, + &xor_val_result, DMA_PREP_INTERRUPT); + if (!tx) { + dev_err(dev, "Self-test 2nd zero prep failed\n"); + err = -ENODEV; + goto dma_unmap; + } + + async_tx_ack(tx); + init_completion(&cmp); + tx->callback = ioat_dma_test_callback; + tx->callback_param = &cmp; + cookie = tx->tx_submit(tx); + if (cookie < 0) { + dev_err(dev, "Self-test 2nd zero setup failed\n"); + err = -ENODEV; + goto dma_unmap; + } + dma->device_issue_pending(dma_chan); + + tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); + + if (tmo == 0 || + dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { + dev_err(dev, "Self-test 2nd validate timed out\n"); + err = -ENODEV; + goto dma_unmap; + } + + if (xor_val_result != SUM_CHECK_P_RESULT) { + dev_err(dev, "Self-test validate failed compare\n"); + err = -ENODEV; + goto dma_unmap; + } + + for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) + dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, DMA_TO_DEVICE); + + goto free_resources; +dma_unmap: + if (op == IOAT_OP_XOR) { + if (dest_dma != DMA_ERROR_CODE) + dma_unmap_page(dev, dest_dma, PAGE_SIZE, + DMA_FROM_DEVICE); + for (i = 0; i < IOAT_NUM_SRC_TEST; i++) + if (dma_srcs[i] != DMA_ERROR_CODE) + dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, + DMA_TO_DEVICE); + } else if (op == IOAT_OP_XOR_VAL) { + for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++) + if (dma_srcs[i] != DMA_ERROR_CODE) + dma_unmap_page(dev, dma_srcs[i], PAGE_SIZE, + DMA_TO_DEVICE); + } +free_resources: + dma->device_free_chan_resources(dma_chan); +out: + src_idx = IOAT_NUM_SRC_TEST; + while (src_idx--) + __free_page(xor_srcs[src_idx]); + __free_page(dest); + return err; +} + +static int ioat3_dma_self_test(struct ioatdma_device *ioat_dma) +{ + int rc; + + rc = ioat_dma_self_test(ioat_dma); + if (rc) + return rc; + + rc = ioat_xor_val_self_test(ioat_dma); + + return rc; +} + +static void ioat_intr_quirk(struct ioatdma_device *ioat_dma) +{ + struct dma_device *dma; + struct dma_chan *c; + struct ioatdma_chan *ioat_chan; + u32 errmask; + + dma = &ioat_dma->dma_dev; + + /* + * if we have descriptor write back error status, we mask the + * error interrupts + */ + if (ioat_dma->cap & IOAT_CAP_DWBES) { + list_for_each_entry(c, &dma->channels, device_node) { + ioat_chan = to_ioat_chan(c); + errmask = readl(ioat_chan->reg_base + + IOAT_CHANERR_MASK_OFFSET); + errmask |= IOAT_CHANERR_XOR_P_OR_CRC_ERR | + IOAT_CHANERR_XOR_Q_ERR; + writel(errmask, ioat_chan->reg_base + + IOAT_CHANERR_MASK_OFFSET); + } + } +} + +static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca) +{ + struct pci_dev *pdev = ioat_dma->pdev; + int dca_en = system_has_dca_enabled(pdev); + struct dma_device *dma; + struct dma_chan *c; + struct ioatdma_chan *ioat_chan; + bool is_raid_device = false; + int err; + + dma = &ioat_dma->dma_dev; + dma->device_prep_dma_memcpy = ioat_dma_prep_memcpy_lock; + dma->device_issue_pending = ioat_issue_pending; + dma->device_alloc_chan_resources = ioat_alloc_chan_resources; + dma->device_free_chan_resources = ioat_free_chan_resources; + + dma_cap_set(DMA_INTERRUPT, dma->cap_mask); + dma->device_prep_dma_interrupt = ioat_prep_interrupt_lock; + + ioat_dma->cap = readl(ioat_dma->reg_base + IOAT_DMA_CAP_OFFSET); + + if (is_xeon_cb32(pdev) || is_bwd_noraid(pdev)) + ioat_dma->cap &= + ~(IOAT_CAP_XOR | IOAT_CAP_PQ | IOAT_CAP_RAID16SS); + + /* dca is incompatible with raid operations */ + if (dca_en && (ioat_dma->cap & (IOAT_CAP_XOR|IOAT_CAP_PQ))) + ioat_dma->cap &= ~(IOAT_CAP_XOR|IOAT_CAP_PQ); + + if (ioat_dma->cap & IOAT_CAP_XOR) { + is_raid_device = true; + dma->max_xor = 8; + + dma_cap_set(DMA_XOR, dma->cap_mask); + dma->device_prep_dma_xor = ioat_prep_xor; + + dma_cap_set(DMA_XOR_VAL, dma->cap_mask); + dma->device_prep_dma_xor_val = ioat_prep_xor_val; + } + + if (ioat_dma->cap & IOAT_CAP_PQ) { + is_raid_device = true; + + dma->device_prep_dma_pq = ioat_prep_pq; + dma->device_prep_dma_pq_val = ioat_prep_pq_val; + dma_cap_set(DMA_PQ, dma->cap_mask); + dma_cap_set(DMA_PQ_VAL, dma->cap_mask); + + if (ioat_dma->cap & IOAT_CAP_RAID16SS) + dma_set_maxpq(dma, 16, 0); + else + dma_set_maxpq(dma, 8, 0); + + if (!(ioat_dma->cap & IOAT_CAP_XOR)) { + dma->device_prep_dma_xor = ioat_prep_pqxor; + dma->device_prep_dma_xor_val = ioat_prep_pqxor_val; + dma_cap_set(DMA_XOR, dma->cap_mask); + dma_cap_set(DMA_XOR_VAL, dma->cap_mask); + + if (ioat_dma->cap & IOAT_CAP_RAID16SS) + dma->max_xor = 16; + else + dma->max_xor = 8; + } + } + + dma->device_tx_status = ioat_tx_status; + + /* starting with CB3.3 super extended descriptors are supported */ + if (ioat_dma->cap & IOAT_CAP_RAID16SS) { + char pool_name[14]; + int i; + + for (i = 0; i < MAX_SED_POOLS; i++) { + snprintf(pool_name, 14, "ioat_hw%d_sed", i); + + /* allocate SED DMA pool */ + ioat_dma->sed_hw_pool[i] = dmam_pool_create(pool_name, + &pdev->dev, + SED_SIZE * (i + 1), 64, 0); + if (!ioat_dma->sed_hw_pool[i]) + return -ENOMEM; + + } + } + + if (!(ioat_dma->cap & (IOAT_CAP_XOR | IOAT_CAP_PQ))) + dma_cap_set(DMA_PRIVATE, dma->cap_mask); + + err = ioat_probe(ioat_dma); + if (err) + return err; + + list_for_each_entry(c, &dma->channels, device_node) { + ioat_chan = to_ioat_chan(c); + writel(IOAT_DMA_DCA_ANY_CPU, + ioat_chan->reg_base + IOAT_DCACTRL_OFFSET); + } + + err = ioat_register(ioat_dma); + if (err) + return err; + + ioat_kobject_add(ioat_dma, &ioat_ktype); + + if (dca) + ioat_dma->dca = ioat_dca_init(pdev, ioat_dma->reg_base); + + return 0; +} + +#define DRV_NAME "ioatdma" + +static struct pci_driver ioat_pci_driver = { + .name = DRV_NAME, + .id_table = ioat_pci_tbl, + .probe = ioat_pci_probe, + .remove = ioat_remove, +}; + +static struct ioatdma_device * +alloc_ioatdma(struct pci_dev *pdev, void __iomem *iobase) +{ + struct device *dev = &pdev->dev; + struct ioatdma_device *d = devm_kzalloc(dev, sizeof(*d), GFP_KERNEL); + + if (!d) + return NULL; + d->pdev = pdev; + d->reg_base = iobase; + return d; +} + +static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + void __iomem * const *iomap; + struct device *dev = &pdev->dev; + struct ioatdma_device *device; + int err; + + err = pcim_enable_device(pdev); + if (err) + return err; + + err = pcim_iomap_regions(pdev, 1 << IOAT_MMIO_BAR, DRV_NAME); + if (err) + return err; + iomap = pcim_iomap_table(pdev); + if (!iomap) + return -ENOMEM; + + err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); + if (err) + err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); + if (err) + return err; + + err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); + if (err) + err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + if (err) + return err; + + device = alloc_ioatdma(pdev, iomap[IOAT_MMIO_BAR]); + if (!device) + return -ENOMEM; + pci_set_master(pdev); + pci_set_drvdata(pdev, device); + + device->version = readb(device->reg_base + IOAT_VER_OFFSET); + if (device->version >= IOAT_VER_3_0) + err = ioat3_dma_probe(device, ioat_dca_enabled); + else + return -ENODEV; + + if (err) { + dev_err(dev, "Intel(R) I/OAT DMA Engine init failed\n"); + return -ENODEV; + } + + return 0; +} + +static void ioat_remove(struct pci_dev *pdev) +{ + struct ioatdma_device *device = pci_get_drvdata(pdev); + + if (!device) + return; + + dev_err(&pdev->dev, "Removing dma and dca services\n"); + if (device->dca) { + unregister_dca_provider(device->dca, &pdev->dev); + free_dca_provider(device->dca); + device->dca = NULL; + } + ioat_dma_remove(device); +} + +static int __init ioat_init_module(void) +{ + int err = -ENOMEM; + + pr_info("%s: Intel(R) QuickData Technology Driver %s\n", + DRV_NAME, IOAT_DMA_VERSION); + + ioat_cache = kmem_cache_create("ioat", sizeof(struct ioat_ring_ent), + 0, SLAB_HWCACHE_ALIGN, NULL); + if (!ioat_cache) + return -ENOMEM; + + ioat_sed_cache = KMEM_CACHE(ioat_sed_ent, 0); + if (!ioat_sed_cache) + goto err_ioat_cache; + + err = pci_register_driver(&ioat_pci_driver); + if (err) + goto err_ioat3_cache; + + return 0; + + err_ioat3_cache: + kmem_cache_destroy(ioat_sed_cache); + + err_ioat_cache: + kmem_cache_destroy(ioat_cache); + + return err; +} +module_init(ioat_init_module); + +static void __exit ioat_exit_module(void) +{ + pci_unregister_driver(&ioat_pci_driver); + kmem_cache_destroy(ioat_cache); +} +module_exit(ioat_exit_module); diff --git a/drivers/dma/ioat/pci.c b/drivers/dma/ioat/pci.c deleted file mode 100644 index 76f0dc688a19..000000000000 --- a/drivers/dma/ioat/pci.c +++ /dev/null @@ -1,258 +0,0 @@ -/* - * Intel I/OAT DMA Linux driver - * Copyright(c) 2007 - 2009 Intel Corporation. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * The full GNU General Public License is included in this distribution in - * the file called "COPYING". - * - */ - -/* - * This driver supports an Intel I/OAT DMA engine, which does asynchronous - * copy operations. - */ - -#include -#include -#include -#include -#include -#include -#include "dma.h" -#include "dma_v2.h" -#include "registers.h" -#include "hw.h" - -MODULE_VERSION(IOAT_DMA_VERSION); -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_AUTHOR("Intel Corporation"); - -static struct pci_device_id ioat_pci_tbl[] = { - /* I/OAT v1 platforms */ - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_CNB) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SCNB) }, - { PCI_VDEVICE(UNISYS, PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR) }, - - /* I/OAT v2 platforms */ - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB) }, - - /* I/OAT v3 platforms */ - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG2) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG3) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG4) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG5) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG6) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG7) }, - - /* I/OAT v3.2 platforms */ - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF0) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF1) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF2) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF3) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF4) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF5) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF6) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF7) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF8) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_JSF9) }, - - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB0) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB1) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB2) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB3) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB4) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB5) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB6) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB7) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB8) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB9) }, - - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB0) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB1) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB2) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB3) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB4) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB5) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB6) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB7) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB8) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB9) }, - - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW0) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW1) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW2) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW3) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW4) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW5) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW6) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW7) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW8) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW9) }, - - /* I/OAT v3.3 platforms */ - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD0) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD1) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD2) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD3) }, - - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE0) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE1) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE2) }, - { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE3) }, - - { 0, } -}; -MODULE_DEVICE_TABLE(pci, ioat_pci_tbl); - -static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id); -static void ioat_remove(struct pci_dev *pdev); - -static int ioat_dca_enabled = 1; -module_param(ioat_dca_enabled, int, 0644); -MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)"); - -struct kmem_cache *ioat2_cache; -struct kmem_cache *ioat3_sed_cache; - -#define DRV_NAME "ioatdma" - -static struct pci_driver ioat_pci_driver = { - .name = DRV_NAME, - .id_table = ioat_pci_tbl, - .probe = ioat_pci_probe, - .remove = ioat_remove, -}; - -static struct ioatdma_device * -alloc_ioatdma(struct pci_dev *pdev, void __iomem *iobase) -{ - struct device *dev = &pdev->dev; - struct ioatdma_device *d = devm_kzalloc(dev, sizeof(*d), GFP_KERNEL); - - if (!d) - return NULL; - d->pdev = pdev; - d->reg_base = iobase; - return d; -} - -static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) -{ - void __iomem * const *iomap; - struct device *dev = &pdev->dev; - struct ioatdma_device *device; - int err; - - err = pcim_enable_device(pdev); - if (err) - return err; - - err = pcim_iomap_regions(pdev, 1 << IOAT_MMIO_BAR, DRV_NAME); - if (err) - return err; - iomap = pcim_iomap_table(pdev); - if (!iomap) - return -ENOMEM; - - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); - if (err) - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); - if (err) - return err; - - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); - if (err) - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); - if (err) - return err; - - device = alloc_ioatdma(pdev, iomap[IOAT_MMIO_BAR]); - if (!device) - return -ENOMEM; - pci_set_master(pdev); - pci_set_drvdata(pdev, device); - - device->version = readb(device->reg_base + IOAT_VER_OFFSET); - if (device->version == IOAT_VER_1_2) - err = ioat1_dma_probe(device, ioat_dca_enabled); - else if (device->version == IOAT_VER_2_0) - err = ioat2_dma_probe(device, ioat_dca_enabled); - else if (device->version >= IOAT_VER_3_0) - err = ioat3_dma_probe(device, ioat_dca_enabled); - else - return -ENODEV; - - if (err) { - dev_err(dev, "Intel(R) I/OAT DMA Engine init failed\n"); - return -ENODEV; - } - - return 0; -} - -static void ioat_remove(struct pci_dev *pdev) -{ - struct ioatdma_device *device = pci_get_drvdata(pdev); - - if (!device) - return; - - dev_err(&pdev->dev, "Removing dma and dca services\n"); - if (device->dca) { - unregister_dca_provider(device->dca, &pdev->dev); - free_dca_provider(device->dca); - device->dca = NULL; - } - ioat_dma_remove(device); -} - -static int __init ioat_init_module(void) -{ - int err = -ENOMEM; - - pr_info("%s: Intel(R) QuickData Technology Driver %s\n", - DRV_NAME, IOAT_DMA_VERSION); - - ioat2_cache = kmem_cache_create("ioat2", sizeof(struct ioat_ring_ent), - 0, SLAB_HWCACHE_ALIGN, NULL); - if (!ioat2_cache) - return -ENOMEM; - - ioat3_sed_cache = KMEM_CACHE(ioat_sed_ent, 0); - if (!ioat3_sed_cache) - goto err_ioat2_cache; - - err = pci_register_driver(&ioat_pci_driver); - if (err) - goto err_ioat3_cache; - - return 0; - - err_ioat3_cache: - kmem_cache_destroy(ioat3_sed_cache); - - err_ioat2_cache: - kmem_cache_destroy(ioat2_cache); - - return err; -} -module_init(ioat_init_module); - -static void __exit ioat_exit_module(void) -{ - pci_unregister_driver(&ioat_pci_driver); - kmem_cache_destroy(ioat2_cache); -} -module_exit(ioat_exit_module); diff --git a/drivers/dma/ioat/prep.c b/drivers/dma/ioat/prep.c new file mode 100644 index 000000000000..ad4fb41cd23b --- /dev/null +++ b/drivers/dma/ioat/prep.c @@ -0,0 +1,715 @@ +/* + * Intel I/OAT DMA Linux driver + * Copyright(c) 2004 - 2015 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * The full GNU General Public License is included in this distribution in + * the file called "COPYING". + * + */ +#include +#include +#include +#include +#include +#include +#include "../dmaengine.h" +#include "registers.h" +#include "hw.h" +#include "dma.h" + +#define MAX_SCF 1024 + +/* provide a lookup table for setting the source address in the base or + * extended descriptor of an xor or pq descriptor + */ +static const u8 xor_idx_to_desc = 0xe0; +static const u8 xor_idx_to_field[] = { 1, 4, 5, 6, 7, 0, 1, 2 }; +static const u8 pq_idx_to_desc = 0xf8; +static const u8 pq16_idx_to_desc[] = { 0, 0, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2 }; +static const u8 pq_idx_to_field[] = { 1, 4, 5, 0, 1, 2, 4, 5 }; +static const u8 pq16_idx_to_field[] = { 1, 4, 1, 2, 3, 4, 5, 6, 7, + 0, 1, 2, 3, 4, 5, 6 }; + +static void xor_set_src(struct ioat_raw_descriptor *descs[2], + dma_addr_t addr, u32 offset, int idx) +{ + struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1]; + + raw->field[xor_idx_to_field[idx]] = addr + offset; +} + +static dma_addr_t pq_get_src(struct ioat_raw_descriptor *descs[2], int idx) +{ + struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1]; + + return raw->field[pq_idx_to_field[idx]]; +} + +static dma_addr_t pq16_get_src(struct ioat_raw_descriptor *desc[3], int idx) +{ + struct ioat_raw_descriptor *raw = desc[pq16_idx_to_desc[idx]]; + + return raw->field[pq16_idx_to_field[idx]]; +} + +static void pq_set_src(struct ioat_raw_descriptor *descs[2], + dma_addr_t addr, u32 offset, u8 coef, int idx) +{ + struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *) descs[0]; + struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1]; + + raw->field[pq_idx_to_field[idx]] = addr + offset; + pq->coef[idx] = coef; +} + +static void pq16_set_src(struct ioat_raw_descriptor *desc[3], + dma_addr_t addr, u32 offset, u8 coef, unsigned idx) +{ + struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *)desc[0]; + struct ioat_pq16a_descriptor *pq16 = + (struct ioat_pq16a_descriptor *)desc[1]; + struct ioat_raw_descriptor *raw = desc[pq16_idx_to_desc[idx]]; + + raw->field[pq16_idx_to_field[idx]] = addr + offset; + + if (idx < 8) + pq->coef[idx] = coef; + else + pq16->coef[idx - 8] = coef; +} + +static struct ioat_sed_ent * +ioat3_alloc_sed(struct ioatdma_device *ioat_dma, unsigned int hw_pool) +{ + struct ioat_sed_ent *sed; + gfp_t flags = __GFP_ZERO | GFP_ATOMIC; + + sed = kmem_cache_alloc(ioat_sed_cache, flags); + if (!sed) + return NULL; + + sed->hw_pool = hw_pool; + sed->hw = dma_pool_alloc(ioat_dma->sed_hw_pool[hw_pool], + flags, &sed->dma); + if (!sed->hw) { + kmem_cache_free(ioat_sed_cache, sed); + return NULL; + } + + return sed; +} + +struct dma_async_tx_descriptor * +ioat_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, + dma_addr_t dma_src, size_t len, unsigned long flags) +{ + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); + struct ioat_dma_descriptor *hw; + struct ioat_ring_ent *desc; + dma_addr_t dst = dma_dest; + dma_addr_t src = dma_src; + size_t total_len = len; + int num_descs, idx, i; + + num_descs = ioat_xferlen_to_descs(ioat_chan, len); + if (likely(num_descs) && + ioat_check_space_lock(ioat_chan, num_descs) == 0) + idx = ioat_chan->head; + else + return NULL; + i = 0; + do { + size_t copy = min_t(size_t, len, 1 << ioat_chan->xfercap_log); + + desc = ioat_get_ring_ent(ioat_chan, idx + i); + hw = desc->hw; + + hw->size = copy; + hw->ctl = 0; + hw->src_addr = src; + hw->dst_addr = dst; + + len -= copy; + dst += copy; + src += copy; + dump_desc_dbg(ioat_chan, desc); + } while (++i < num_descs); + + desc->txd.flags = flags; + desc->len = total_len; + hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); + hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE); + hw->ctl_f.compl_write = 1; + dump_desc_dbg(ioat_chan, desc); + /* we leave the channel locked to ensure in order submission */ + + return &desc->txd; +} + + +static struct dma_async_tx_descriptor * +__ioat_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result, + dma_addr_t dest, dma_addr_t *src, unsigned int src_cnt, + size_t len, unsigned long flags) +{ + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); + struct ioat_ring_ent *compl_desc; + struct ioat_ring_ent *desc; + struct ioat_ring_ent *ext; + size_t total_len = len; + struct ioat_xor_descriptor *xor; + struct ioat_xor_ext_descriptor *xor_ex = NULL; + struct ioat_dma_descriptor *hw; + int num_descs, with_ext, idx, i; + u32 offset = 0; + u8 op = result ? IOAT_OP_XOR_VAL : IOAT_OP_XOR; + + BUG_ON(src_cnt < 2); + + num_descs = ioat_xferlen_to_descs(ioat_chan, len); + /* we need 2x the number of descriptors to cover greater than 5 + * sources + */ + if (src_cnt > 5) { + with_ext = 1; + num_descs *= 2; + } else + with_ext = 0; + + /* completion writes from the raid engine may pass completion + * writes from the legacy engine, so we need one extra null + * (legacy) descriptor to ensure all completion writes arrive in + * order. + */ + if (likely(num_descs) && + ioat_check_space_lock(ioat_chan, num_descs+1) == 0) + idx = ioat_chan->head; + else + return NULL; + i = 0; + do { + struct ioat_raw_descriptor *descs[2]; + size_t xfer_size = min_t(size_t, + len, 1 << ioat_chan->xfercap_log); + int s; + + desc = ioat_get_ring_ent(ioat_chan, idx + i); + xor = desc->xor; + + /* save a branch by unconditionally retrieving the + * extended descriptor xor_set_src() knows to not write + * to it in the single descriptor case + */ + ext = ioat_get_ring_ent(ioat_chan, idx + i + 1); + xor_ex = ext->xor_ex; + + descs[0] = (struct ioat_raw_descriptor *) xor; + descs[1] = (struct ioat_raw_descriptor *) xor_ex; + for (s = 0; s < src_cnt; s++) + xor_set_src(descs, src[s], offset, s); + xor->size = xfer_size; + xor->dst_addr = dest + offset; + xor->ctl = 0; + xor->ctl_f.op = op; + xor->ctl_f.src_cnt = src_cnt_to_hw(src_cnt); + + len -= xfer_size; + offset += xfer_size; + dump_desc_dbg(ioat_chan, desc); + } while ((i += 1 + with_ext) < num_descs); + + /* last xor descriptor carries the unmap parameters and fence bit */ + desc->txd.flags = flags; + desc->len = total_len; + if (result) + desc->result = result; + xor->ctl_f.fence = !!(flags & DMA_PREP_FENCE); + + /* completion descriptor carries interrupt bit */ + compl_desc = ioat_get_ring_ent(ioat_chan, idx + i); + compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT; + hw = compl_desc->hw; + hw->ctl = 0; + hw->ctl_f.null = 1; + hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); + hw->ctl_f.compl_write = 1; + hw->size = NULL_DESC_BUFFER_SIZE; + dump_desc_dbg(ioat_chan, compl_desc); + + /* we leave the channel locked to ensure in order submission */ + return &compl_desc->txd; +} + +struct dma_async_tx_descriptor * +ioat_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, + unsigned int src_cnt, size_t len, unsigned long flags) +{ + return __ioat_prep_xor_lock(chan, NULL, dest, src, src_cnt, len, flags); +} + +struct dma_async_tx_descriptor * +ioat_prep_xor_val(struct dma_chan *chan, dma_addr_t *src, + unsigned int src_cnt, size_t len, + enum sum_check_flags *result, unsigned long flags) +{ + /* the cleanup routine only sets bits on validate failure, it + * does not clear bits on validate success... so clear it here + */ + *result = 0; + + return __ioat_prep_xor_lock(chan, result, src[0], &src[1], + src_cnt - 1, len, flags); +} + +static void +dump_pq_desc_dbg(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc, + struct ioat_ring_ent *ext) +{ + struct device *dev = to_dev(ioat_chan); + struct ioat_pq_descriptor *pq = desc->pq; + struct ioat_pq_ext_descriptor *pq_ex = ext ? ext->pq_ex : NULL; + struct ioat_raw_descriptor *descs[] = { (void *) pq, (void *) pq_ex }; + int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt); + int i; + + dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x" + " sz: %#10.8x ctl: %#x (op: %#x int: %d compl: %d pq: '%s%s'" + " src_cnt: %d)\n", + desc_id(desc), (unsigned long long) desc->txd.phys, + (unsigned long long) (pq_ex ? pq_ex->next : pq->next), + desc->txd.flags, pq->size, pq->ctl, pq->ctl_f.op, + pq->ctl_f.int_en, pq->ctl_f.compl_write, + pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q", + pq->ctl_f.src_cnt); + for (i = 0; i < src_cnt; i++) + dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i, + (unsigned long long) pq_get_src(descs, i), pq->coef[i]); + dev_dbg(dev, "\tP: %#llx\n", pq->p_addr); + dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr); + dev_dbg(dev, "\tNEXT: %#llx\n", pq->next); +} + +static void dump_pq16_desc_dbg(struct ioatdma_chan *ioat_chan, + struct ioat_ring_ent *desc) +{ + struct device *dev = to_dev(ioat_chan); + struct ioat_pq_descriptor *pq = desc->pq; + struct ioat_raw_descriptor *descs[] = { (void *)pq, + (void *)pq, + (void *)pq }; + int src_cnt = src16_cnt_to_sw(pq->ctl_f.src_cnt); + int i; + + if (desc->sed) { + descs[1] = (void *)desc->sed->hw; + descs[2] = (void *)desc->sed->hw + 64; + } + + dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x" + " sz: %#x ctl: %#x (op: %#x int: %d compl: %d pq: '%s%s'" + " src_cnt: %d)\n", + desc_id(desc), (unsigned long long) desc->txd.phys, + (unsigned long long) pq->next, + desc->txd.flags, pq->size, pq->ctl, + pq->ctl_f.op, pq->ctl_f.int_en, + pq->ctl_f.compl_write, + pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q", + pq->ctl_f.src_cnt); + for (i = 0; i < src_cnt; i++) { + dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i, + (unsigned long long) pq16_get_src(descs, i), + pq->coef[i]); + } + dev_dbg(dev, "\tP: %#llx\n", pq->p_addr); + dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr); +} + +static struct dma_async_tx_descriptor * +__ioat_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result, + const dma_addr_t *dst, const dma_addr_t *src, + unsigned int src_cnt, const unsigned char *scf, + size_t len, unsigned long flags) +{ + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); + struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; + struct ioat_ring_ent *compl_desc; + struct ioat_ring_ent *desc; + struct ioat_ring_ent *ext; + size_t total_len = len; + struct ioat_pq_descriptor *pq; + struct ioat_pq_ext_descriptor *pq_ex = NULL; + struct ioat_dma_descriptor *hw; + u32 offset = 0; + u8 op = result ? IOAT_OP_PQ_VAL : IOAT_OP_PQ; + int i, s, idx, with_ext, num_descs; + int cb32 = (ioat_dma->version < IOAT_VER_3_3) ? 1 : 0; + + dev_dbg(to_dev(ioat_chan), "%s\n", __func__); + /* the engine requires at least two sources (we provide + * at least 1 implied source in the DMA_PREP_CONTINUE case) + */ + BUG_ON(src_cnt + dmaf_continue(flags) < 2); + + num_descs = ioat_xferlen_to_descs(ioat_chan, len); + /* we need 2x the number of descriptors to cover greater than 3 + * sources (we need 1 extra source in the q-only continuation + * case and 3 extra sources in the p+q continuation case. + */ + if (src_cnt + dmaf_p_disabled_continue(flags) > 3 || + (dmaf_continue(flags) && !dmaf_p_disabled_continue(flags))) { + with_ext = 1; + num_descs *= 2; + } else + with_ext = 0; + + /* completion writes from the raid engine may pass completion + * writes from the legacy engine, so we need one extra null + * (legacy) descriptor to ensure all completion writes arrive in + * order. + */ + if (likely(num_descs) && + ioat_check_space_lock(ioat_chan, num_descs + cb32) == 0) + idx = ioat_chan->head; + else + return NULL; + i = 0; + do { + struct ioat_raw_descriptor *descs[2]; + size_t xfer_size = min_t(size_t, len, + 1 << ioat_chan->xfercap_log); + + desc = ioat_get_ring_ent(ioat_chan, idx + i); + pq = desc->pq; + + /* save a branch by unconditionally retrieving the + * extended descriptor pq_set_src() knows to not write + * to it in the single descriptor case + */ + ext = ioat_get_ring_ent(ioat_chan, idx + i + with_ext); + pq_ex = ext->pq_ex; + + descs[0] = (struct ioat_raw_descriptor *) pq; + descs[1] = (struct ioat_raw_descriptor *) pq_ex; + + for (s = 0; s < src_cnt; s++) + pq_set_src(descs, src[s], offset, scf[s], s); + + /* see the comment for dma_maxpq in include/linux/dmaengine.h */ + if (dmaf_p_disabled_continue(flags)) + pq_set_src(descs, dst[1], offset, 1, s++); + else if (dmaf_continue(flags)) { + pq_set_src(descs, dst[0], offset, 0, s++); + pq_set_src(descs, dst[1], offset, 1, s++); + pq_set_src(descs, dst[1], offset, 0, s++); + } + pq->size = xfer_size; + pq->p_addr = dst[0] + offset; + pq->q_addr = dst[1] + offset; + pq->ctl = 0; + pq->ctl_f.op = op; + /* we turn on descriptor write back error status */ + if (ioat_dma->cap & IOAT_CAP_DWBES) + pq->ctl_f.wb_en = result ? 1 : 0; + pq->ctl_f.src_cnt = src_cnt_to_hw(s); + pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P); + pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q); + + len -= xfer_size; + offset += xfer_size; + } while ((i += 1 + with_ext) < num_descs); + + /* last pq descriptor carries the unmap parameters and fence bit */ + desc->txd.flags = flags; + desc->len = total_len; + if (result) + desc->result = result; + pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE); + dump_pq_desc_dbg(ioat_chan, desc, ext); + + if (!cb32) { + pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); + pq->ctl_f.compl_write = 1; + compl_desc = desc; + } else { + /* completion descriptor carries interrupt bit */ + compl_desc = ioat_get_ring_ent(ioat_chan, idx + i); + compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT; + hw = compl_desc->hw; + hw->ctl = 0; + hw->ctl_f.null = 1; + hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); + hw->ctl_f.compl_write = 1; + hw->size = NULL_DESC_BUFFER_SIZE; + dump_desc_dbg(ioat_chan, compl_desc); + } + + + /* we leave the channel locked to ensure in order submission */ + return &compl_desc->txd; +} + +static struct dma_async_tx_descriptor * +__ioat_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result, + const dma_addr_t *dst, const dma_addr_t *src, + unsigned int src_cnt, const unsigned char *scf, + size_t len, unsigned long flags) +{ + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); + struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; + struct ioat_ring_ent *desc; + size_t total_len = len; + struct ioat_pq_descriptor *pq; + u32 offset = 0; + u8 op; + int i, s, idx, num_descs; + + /* this function is only called with 9-16 sources */ + op = result ? IOAT_OP_PQ_VAL_16S : IOAT_OP_PQ_16S; + + dev_dbg(to_dev(ioat_chan), "%s\n", __func__); + + num_descs = ioat_xferlen_to_descs(ioat_chan, len); + + /* + * 16 source pq is only available on cb3.3 and has no completion + * write hw bug. + */ + if (num_descs && ioat_check_space_lock(ioat_chan, num_descs) == 0) + idx = ioat_chan->head; + else + return NULL; + + i = 0; + + do { + struct ioat_raw_descriptor *descs[4]; + size_t xfer_size = min_t(size_t, len, + 1 << ioat_chan->xfercap_log); + + desc = ioat_get_ring_ent(ioat_chan, idx + i); + pq = desc->pq; + + descs[0] = (struct ioat_raw_descriptor *) pq; + + desc->sed = ioat3_alloc_sed(ioat_dma, (src_cnt-2) >> 3); + if (!desc->sed) { + dev_err(to_dev(ioat_chan), + "%s: no free sed entries\n", __func__); + return NULL; + } + + pq->sed_addr = desc->sed->dma; + desc->sed->parent = desc; + + descs[1] = (struct ioat_raw_descriptor *)desc->sed->hw; + descs[2] = (void *)descs[1] + 64; + + for (s = 0; s < src_cnt; s++) + pq16_set_src(descs, src[s], offset, scf[s], s); + + /* see the comment for dma_maxpq in include/linux/dmaengine.h */ + if (dmaf_p_disabled_continue(flags)) + pq16_set_src(descs, dst[1], offset, 1, s++); + else if (dmaf_continue(flags)) { + pq16_set_src(descs, dst[0], offset, 0, s++); + pq16_set_src(descs, dst[1], offset, 1, s++); + pq16_set_src(descs, dst[1], offset, 0, s++); + } + + pq->size = xfer_size; + pq->p_addr = dst[0] + offset; + pq->q_addr = dst[1] + offset; + pq->ctl = 0; + pq->ctl_f.op = op; + pq->ctl_f.src_cnt = src16_cnt_to_hw(s); + /* we turn on descriptor write back error status */ + if (ioat_dma->cap & IOAT_CAP_DWBES) + pq->ctl_f.wb_en = result ? 1 : 0; + pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P); + pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q); + + len -= xfer_size; + offset += xfer_size; + } while (++i < num_descs); + + /* last pq descriptor carries the unmap parameters and fence bit */ + desc->txd.flags = flags; + desc->len = total_len; + if (result) + desc->result = result; + pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE); + + /* with cb3.3 we should be able to do completion w/o a null desc */ + pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); + pq->ctl_f.compl_write = 1; + + dump_pq16_desc_dbg(ioat_chan, desc); + + /* we leave the channel locked to ensure in order submission */ + return &desc->txd; +} + +static int src_cnt_flags(unsigned int src_cnt, unsigned long flags) +{ + if (dmaf_p_disabled_continue(flags)) + return src_cnt + 1; + else if (dmaf_continue(flags)) + return src_cnt + 3; + else + return src_cnt; +} + +struct dma_async_tx_descriptor * +ioat_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, + unsigned int src_cnt, const unsigned char *scf, size_t len, + unsigned long flags) +{ + /* specify valid address for disabled result */ + if (flags & DMA_PREP_PQ_DISABLE_P) + dst[0] = dst[1]; + if (flags & DMA_PREP_PQ_DISABLE_Q) + dst[1] = dst[0]; + + /* handle the single source multiply case from the raid6 + * recovery path + */ + if ((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1) { + dma_addr_t single_source[2]; + unsigned char single_source_coef[2]; + + BUG_ON(flags & DMA_PREP_PQ_DISABLE_Q); + single_source[0] = src[0]; + single_source[1] = src[0]; + single_source_coef[0] = scf[0]; + single_source_coef[1] = 0; + + return src_cnt_flags(src_cnt, flags) > 8 ? + __ioat_prep_pq16_lock(chan, NULL, dst, single_source, + 2, single_source_coef, len, + flags) : + __ioat_prep_pq_lock(chan, NULL, dst, single_source, 2, + single_source_coef, len, flags); + + } else { + return src_cnt_flags(src_cnt, flags) > 8 ? + __ioat_prep_pq16_lock(chan, NULL, dst, src, src_cnt, + scf, len, flags) : + __ioat_prep_pq_lock(chan, NULL, dst, src, src_cnt, + scf, len, flags); + } +} + +struct dma_async_tx_descriptor * +ioat_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, + unsigned int src_cnt, const unsigned char *scf, size_t len, + enum sum_check_flags *pqres, unsigned long flags) +{ + /* specify valid address for disabled result */ + if (flags & DMA_PREP_PQ_DISABLE_P) + pq[0] = pq[1]; + if (flags & DMA_PREP_PQ_DISABLE_Q) + pq[1] = pq[0]; + + /* the cleanup routine only sets bits on validate failure, it + * does not clear bits on validate success... so clear it here + */ + *pqres = 0; + + return src_cnt_flags(src_cnt, flags) > 8 ? + __ioat_prep_pq16_lock(chan, pqres, pq, src, src_cnt, scf, len, + flags) : + __ioat_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len, + flags); +} + +struct dma_async_tx_descriptor * +ioat_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src, + unsigned int src_cnt, size_t len, unsigned long flags) +{ + unsigned char scf[MAX_SCF]; + dma_addr_t pq[2]; + + if (src_cnt > MAX_SCF) + return NULL; + + memset(scf, 0, src_cnt); + pq[0] = dst; + flags |= DMA_PREP_PQ_DISABLE_Q; + pq[1] = dst; /* specify valid address for disabled result */ + + return src_cnt_flags(src_cnt, flags) > 8 ? + __ioat_prep_pq16_lock(chan, NULL, pq, src, src_cnt, scf, len, + flags) : + __ioat_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len, + flags); +} + +struct dma_async_tx_descriptor * +ioat_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src, + unsigned int src_cnt, size_t len, + enum sum_check_flags *result, unsigned long flags) +{ + unsigned char scf[MAX_SCF]; + dma_addr_t pq[2]; + + if (src_cnt > MAX_SCF) + return NULL; + + /* the cleanup routine only sets bits on validate failure, it + * does not clear bits on validate success... so clear it here + */ + *result = 0; + + memset(scf, 0, src_cnt); + pq[0] = src[0]; + flags |= DMA_PREP_PQ_DISABLE_Q; + pq[1] = pq[0]; /* specify valid address for disabled result */ + + return src_cnt_flags(src_cnt, flags) > 8 ? + __ioat_prep_pq16_lock(chan, result, pq, &src[1], src_cnt - 1, + scf, len, flags) : + __ioat_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1, + scf, len, flags); +} + +struct dma_async_tx_descriptor * +ioat_prep_interrupt_lock(struct dma_chan *c, unsigned long flags) +{ + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); + struct ioat_ring_ent *desc; + struct ioat_dma_descriptor *hw; + + if (ioat_check_space_lock(ioat_chan, 1) == 0) + desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head); + else + return NULL; + + hw = desc->hw; + hw->ctl = 0; + hw->ctl_f.null = 1; + hw->ctl_f.int_en = 1; + hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE); + hw->ctl_f.compl_write = 1; + hw->size = NULL_DESC_BUFFER_SIZE; + hw->src_addr = 0; + hw->dst_addr = 0; + + desc->txd.flags = flags; + desc->len = 1; + + dump_desc_dbg(ioat_chan, desc); + + /* we leave the channel locked to ensure in order submission */ + return &desc->txd; +} + diff --git a/drivers/dma/ioat/sysfs.c b/drivers/dma/ioat/sysfs.c new file mode 100644 index 000000000000..cb4a857ee21b --- /dev/null +++ b/drivers/dma/ioat/sysfs.c @@ -0,0 +1,135 @@ +/* + * Intel I/OAT DMA Linux driver + * Copyright(c) 2004 - 2015 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * The full GNU General Public License is included in this distribution in + * the file called "COPYING". + * + */ + +#include +#include +#include +#include +#include "dma.h" +#include "registers.h" +#include "hw.h" + +#include "../dmaengine.h" + +static ssize_t cap_show(struct dma_chan *c, char *page) +{ + struct dma_device *dma = c->device; + + return sprintf(page, "copy%s%s%s%s%s\n", + dma_has_cap(DMA_PQ, dma->cap_mask) ? " pq" : "", + dma_has_cap(DMA_PQ_VAL, dma->cap_mask) ? " pq_val" : "", + dma_has_cap(DMA_XOR, dma->cap_mask) ? " xor" : "", + dma_has_cap(DMA_XOR_VAL, dma->cap_mask) ? " xor_val" : "", + dma_has_cap(DMA_INTERRUPT, dma->cap_mask) ? " intr" : ""); + +} +struct ioat_sysfs_entry ioat_cap_attr = __ATTR_RO(cap); + +static ssize_t version_show(struct dma_chan *c, char *page) +{ + struct dma_device *dma = c->device; + struct ioatdma_device *ioat_dma = to_ioatdma_device(dma); + + return sprintf(page, "%d.%d\n", + ioat_dma->version >> 4, ioat_dma->version & 0xf); +} +struct ioat_sysfs_entry ioat_version_attr = __ATTR_RO(version); + +static ssize_t +ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page) +{ + struct ioat_sysfs_entry *entry; + struct ioatdma_chan *ioat_chan; + + entry = container_of(attr, struct ioat_sysfs_entry, attr); + ioat_chan = container_of(kobj, struct ioatdma_chan, kobj); + + if (!entry->show) + return -EIO; + return entry->show(&ioat_chan->dma_chan, page); +} + +const struct sysfs_ops ioat_sysfs_ops = { + .show = ioat_attr_show, +}; + +void ioat_kobject_add(struct ioatdma_device *ioat_dma, struct kobj_type *type) +{ + struct dma_device *dma = &ioat_dma->dma_dev; + struct dma_chan *c; + + list_for_each_entry(c, &dma->channels, device_node) { + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); + struct kobject *parent = &c->dev->device.kobj; + int err; + + err = kobject_init_and_add(&ioat_chan->kobj, type, + parent, "quickdata"); + if (err) { + dev_warn(to_dev(ioat_chan), + "sysfs init error (%d), continuing...\n", err); + kobject_put(&ioat_chan->kobj); + set_bit(IOAT_KOBJ_INIT_FAIL, &ioat_chan->state); + } + } +} + +void ioat_kobject_del(struct ioatdma_device *ioat_dma) +{ + struct dma_device *dma = &ioat_dma->dma_dev; + struct dma_chan *c; + + list_for_each_entry(c, &dma->channels, device_node) { + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); + + if (!test_bit(IOAT_KOBJ_INIT_FAIL, &ioat_chan->state)) { + kobject_del(&ioat_chan->kobj); + kobject_put(&ioat_chan->kobj); + } + } +} + +static ssize_t ring_size_show(struct dma_chan *c, char *page) +{ + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); + + return sprintf(page, "%d\n", (1 << ioat_chan->alloc_order) & ~1); +} +static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size); + +static ssize_t ring_active_show(struct dma_chan *c, char *page) +{ + struct ioatdma_chan *ioat_chan = to_ioat_chan(c); + + /* ...taken outside the lock, no need to be precise */ + return sprintf(page, "%d\n", ioat_ring_active(ioat_chan)); +} +static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active); + +static struct attribute *ioat_attrs[] = { + &ring_size_attr.attr, + &ring_active_attr.attr, + &ioat_cap_attr.attr, + &ioat_version_attr.attr, + NULL, +}; + +struct kobj_type ioat_ktype = { + .sysfs_ops = &ioat_sysfs_ops, + .default_attrs = ioat_attrs, +}; diff --git a/drivers/dma/ipu/ipu_irq.c b/drivers/dma/ipu/ipu_irq.c index 2e284a4438bc..4768a829253a 100644 --- a/drivers/dma/ipu/ipu_irq.c +++ b/drivers/dma/ipu/ipu_irq.c @@ -265,10 +265,10 @@ int ipu_irq_unmap(unsigned int source) return ret; } -/* Chained IRQ handler for IPU error interrupt */ -static void ipu_irq_err(unsigned int irq, struct irq_desc *desc) +/* Chained IRQ handler for IPU function and error interrupt */ +static void ipu_irq_handler(unsigned int __irq, struct irq_desc *desc) { - struct ipu *ipu = irq_get_handler_data(irq); + struct ipu *ipu = irq_desc_get_handler_data(desc); u32 status; int i, line; @@ -286,43 +286,7 @@ static void ipu_irq_err(unsigned int irq, struct irq_desc *desc) raw_spin_unlock(&bank_lock); while ((line = ffs(status))) { struct ipu_irq_map *map; - - line--; - status &= ~(1UL << line); - - raw_spin_lock(&bank_lock); - map = src2map(32 * i + line); - if (map) - irq = map->irq; - raw_spin_unlock(&bank_lock); - - if (!map) { - pr_err("IPU: Interrupt on unmapped source %u bank %d\n", - line, i); - continue; - } - generic_handle_irq(irq); - } - } -} - -/* Chained IRQ handler for IPU function interrupt */ -static void ipu_irq_fn(unsigned int irq, struct irq_desc *desc) -{ - struct ipu *ipu = irq_desc_get_handler_data(desc); - u32 status; - int i, line; - - for (i = 0; i < IPU_IRQ_NR_FN_BANKS; i++) { - struct ipu_irq_bank *bank = irq_bank + i; - - raw_spin_lock(&bank_lock); - status = ipu_read_reg(ipu, bank->status); - /* Not clearing all interrupts, see above */ - status &= ipu_read_reg(ipu, bank->control); - raw_spin_unlock(&bank_lock); - while ((line = ffs(status))) { - struct ipu_irq_map *map; + unsigned int irq = NO_IRQ; line--; status &= ~(1UL << line); @@ -377,16 +341,12 @@ int __init ipu_irq_attach_irq(struct ipu *ipu, struct platform_device *dev) irq_map[i].irq = irq; irq_map[i].source = -EINVAL; irq_set_handler(irq, handle_level_irq); -#ifdef CONFIG_ARM - set_irq_flags(irq, IRQF_VALID | IRQF_PROBE); -#endif + irq_clear_status_flags(irq, IRQ_NOREQUEST | IRQ_NOPROBE); } - irq_set_handler_data(ipu->irq_fn, ipu); - irq_set_chained_handler(ipu->irq_fn, ipu_irq_fn); + irq_set_chained_handler_and_data(ipu->irq_fn, ipu_irq_handler, ipu); - irq_set_handler_data(ipu->irq_err, ipu); - irq_set_chained_handler(ipu->irq_err, ipu_irq_err); + irq_set_chained_handler_and_data(ipu->irq_err, ipu_irq_handler, ipu); ipu->irq_base = irq_base; @@ -399,16 +359,12 @@ void ipu_irq_detach_irq(struct ipu *ipu, struct platform_device *dev) irq_base = ipu->irq_base; - irq_set_chained_handler(ipu->irq_fn, NULL); - irq_set_handler_data(ipu->irq_fn, NULL); + irq_set_chained_handler_and_data(ipu->irq_fn, NULL, NULL); - irq_set_chained_handler(ipu->irq_err, NULL); - irq_set_handler_data(ipu->irq_err, NULL); + irq_set_chained_handler_and_data(ipu->irq_err, NULL, NULL); for (irq = irq_base; irq < irq_base + CONFIG_MX3_IPU_IRQS; irq++) { -#ifdef CONFIG_ARM - set_irq_flags(irq, 0); -#endif + irq_set_status_flags(irq, IRQ_NOREQUEST); irq_set_chip(irq, NULL); irq_set_chip_data(irq, NULL); } diff --git a/drivers/dma/k3dma.c b/drivers/dma/k3dma.c index 647e362f01fd..1ba2fd73852d 100644 --- a/drivers/dma/k3dma.c +++ b/drivers/dma/k3dma.c @@ -24,7 +24,6 @@ #include "virt-dma.h" #define DRIVER_NAME "k3-dma" -#define DMA_ALIGN 3 #define DMA_MAX_SIZE 0x1ffc #define INT_STAT 0x00 @@ -732,7 +731,7 @@ static int k3_dma_probe(struct platform_device *op) d->slave.device_pause = k3_dma_transfer_pause; d->slave.device_resume = k3_dma_transfer_resume; d->slave.device_terminate_all = k3_dma_terminate_all; - d->slave.copy_align = DMA_ALIGN; + d->slave.copy_align = DMAENGINE_ALIGN_8_BYTES; /* init virtual channel */ d->chans = devm_kzalloc(&op->dev, diff --git a/drivers/dma/lpc18xx-dmamux.c b/drivers/dma/lpc18xx-dmamux.c new file mode 100644 index 000000000000..761f32687055 --- /dev/null +++ b/drivers/dma/lpc18xx-dmamux.c @@ -0,0 +1,183 @@ +/* + * DMA Router driver for LPC18xx/43xx DMA MUX + * + * Copyright (C) 2015 Joachim Eastwood + * + * Based on TI DMA Crossbar driver by: + * Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com + * Author: Peter Ujfalusi + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +/* CREG register offset and macros for mux manipulation */ +#define LPC18XX_CREG_DMAMUX 0x11c +#define LPC18XX_DMAMUX_VAL(v, n) ((v) << (n * 2)) +#define LPC18XX_DMAMUX_MASK(n) (0x3 << (n * 2)) +#define LPC18XX_DMAMUX_MAX_VAL 0x3 + +struct lpc18xx_dmamux { + u32 value; + bool busy; +}; + +struct lpc18xx_dmamux_data { + struct dma_router dmarouter; + struct lpc18xx_dmamux *muxes; + u32 dma_master_requests; + u32 dma_mux_requests; + struct regmap *reg; + spinlock_t lock; +}; + +static void lpc18xx_dmamux_free(struct device *dev, void *route_data) +{ + struct lpc18xx_dmamux_data *dmamux = dev_get_drvdata(dev); + struct lpc18xx_dmamux *mux = route_data; + unsigned long flags; + + spin_lock_irqsave(&dmamux->lock, flags); + mux->busy = false; + spin_unlock_irqrestore(&dmamux->lock, flags); +} + +static void *lpc18xx_dmamux_reserve(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct platform_device *pdev = of_find_device_by_node(ofdma->of_node); + struct lpc18xx_dmamux_data *dmamux = platform_get_drvdata(pdev); + unsigned long flags; + unsigned mux; + + if (dma_spec->args_count != 3) { + dev_err(&pdev->dev, "invalid number of dma mux args\n"); + return ERR_PTR(-EINVAL); + } + + mux = dma_spec->args[0]; + if (mux >= dmamux->dma_master_requests) { + dev_err(&pdev->dev, "invalid mux number: %d\n", + dma_spec->args[0]); + return ERR_PTR(-EINVAL); + } + + if (dma_spec->args[1] > LPC18XX_DMAMUX_MAX_VAL) { + dev_err(&pdev->dev, "invalid dma mux value: %d\n", + dma_spec->args[1]); + return ERR_PTR(-EINVAL); + } + + /* The of_node_put() will be done in the core for the node */ + dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", 0); + if (!dma_spec->np) { + dev_err(&pdev->dev, "can't get dma master\n"); + return ERR_PTR(-EINVAL); + } + + spin_lock_irqsave(&dmamux->lock, flags); + if (dmamux->muxes[mux].busy) { + spin_unlock_irqrestore(&dmamux->lock, flags); + dev_err(&pdev->dev, "dma request %u busy with %u.%u\n", + mux, mux, dmamux->muxes[mux].value); + of_node_put(dma_spec->np); + return ERR_PTR(-EBUSY); + } + + dmamux->muxes[mux].busy = true; + dmamux->muxes[mux].value = dma_spec->args[1]; + + regmap_update_bits(dmamux->reg, LPC18XX_CREG_DMAMUX, + LPC18XX_DMAMUX_MASK(mux), + LPC18XX_DMAMUX_VAL(dmamux->muxes[mux].value, mux)); + spin_unlock_irqrestore(&dmamux->lock, flags); + + dma_spec->args[1] = dma_spec->args[2]; + dma_spec->args_count = 2; + + dev_dbg(&pdev->dev, "mapping dmamux %u.%u to dma request %u\n", mux, + dmamux->muxes[mux].value, mux); + + return &dmamux->muxes[mux]; +} + +static int lpc18xx_dmamux_probe(struct platform_device *pdev) +{ + struct device_node *dma_np, *np = pdev->dev.of_node; + struct lpc18xx_dmamux_data *dmamux; + int ret; + + dmamux = devm_kzalloc(&pdev->dev, sizeof(*dmamux), GFP_KERNEL); + if (!dmamux) + return -ENOMEM; + + dmamux->reg = syscon_regmap_lookup_by_compatible("nxp,lpc1850-creg"); + if (IS_ERR(dmamux->reg)) { + dev_err(&pdev->dev, "syscon lookup failed\n"); + return PTR_ERR(dmamux->reg); + } + + ret = of_property_read_u32(np, "dma-requests", + &dmamux->dma_mux_requests); + if (ret) { + dev_err(&pdev->dev, "missing dma-requests property\n"); + return ret; + } + + dma_np = of_parse_phandle(np, "dma-masters", 0); + if (!dma_np) { + dev_err(&pdev->dev, "can't get dma master\n"); + return -ENODEV; + } + + ret = of_property_read_u32(dma_np, "dma-requests", + &dmamux->dma_master_requests); + of_node_put(dma_np); + if (ret) { + dev_err(&pdev->dev, "missing master dma-requests property\n"); + return ret; + } + + dmamux->muxes = devm_kcalloc(&pdev->dev, dmamux->dma_master_requests, + sizeof(struct lpc18xx_dmamux), + GFP_KERNEL); + if (!dmamux->muxes) + return -ENOMEM; + + spin_lock_init(&dmamux->lock); + platform_set_drvdata(pdev, dmamux); + dmamux->dmarouter.dev = &pdev->dev; + dmamux->dmarouter.route_free = lpc18xx_dmamux_free; + + return of_dma_router_register(np, lpc18xx_dmamux_reserve, + &dmamux->dmarouter); +} + +static const struct of_device_id lpc18xx_dmamux_match[] = { + { .compatible = "nxp,lpc1850-dmamux" }, + {}, +}; + +static struct platform_driver lpc18xx_dmamux_driver = { + .probe = lpc18xx_dmamux_probe, + .driver = { + .name = "lpc18xx-dmamux", + .of_match_table = lpc18xx_dmamux_match, + }, +}; + +static int __init lpc18xx_dmamux_init(void) +{ + return platform_driver_register(&lpc18xx_dmamux_driver); +} +arch_initcall(lpc18xx_dmamux_init); diff --git a/drivers/dma/mic_x100_dma.h b/drivers/dma/mic_x100_dma.h index f663b0bdd11d..d89982034e68 100644 --- a/drivers/dma/mic_x100_dma.h +++ b/drivers/dma/mic_x100_dma.h @@ -39,7 +39,7 @@ */ #define MIC_DMA_MAX_NUM_CHAN 8 #define MIC_DMA_NUM_CHAN 4 -#define MIC_DMA_ALIGN_SHIFT 6 +#define MIC_DMA_ALIGN_SHIFT DMAENGINE_ALIGN_64_BYTES #define MIC_DMA_ALIGN_BYTES (1 << MIC_DMA_ALIGN_SHIFT) #define MIC_DMA_DESC_RX_SIZE (128 * 1024 - 4) diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c index 462a0229a743..e39457f13d4d 100644 --- a/drivers/dma/mmp_pdma.c +++ b/drivers/dma/mmp_pdma.c @@ -72,7 +72,6 @@ #define DCMD_WIDTH4 (3 << 14) /* 4 byte width (Word) */ #define DCMD_LENGTH 0x01fff /* length mask (max = 8K - 1) */ -#define PDMA_ALIGNMENT 3 #define PDMA_MAX_DESC_BYTES DCMD_LENGTH struct mmp_pdma_desc_hw { @@ -1071,7 +1070,7 @@ static int mmp_pdma_probe(struct platform_device *op) pdev->device.device_issue_pending = mmp_pdma_issue_pending; pdev->device.device_config = mmp_pdma_config; pdev->device.device_terminate_all = mmp_pdma_terminate_all; - pdev->device.copy_align = PDMA_ALIGNMENT; + pdev->device.copy_align = DMAENGINE_ALIGN_8_BYTES; pdev->device.src_addr_widths = widths; pdev->device.dst_addr_widths = widths; pdev->device.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); diff --git a/drivers/dma/mmp_tdma.c b/drivers/dma/mmp_tdma.c index e683761e0f8f..3df0422607d5 100644 --- a/drivers/dma/mmp_tdma.c +++ b/drivers/dma/mmp_tdma.c @@ -100,7 +100,6 @@ enum mmp_tdma_type { PXA910_SQU, }; -#define TDMA_ALIGNMENT 3 #define TDMA_MAX_XFER_BYTES SZ_64K struct mmp_tdma_chan { @@ -695,7 +694,7 @@ static int mmp_tdma_probe(struct platform_device *pdev) tdev->device.device_pause = mmp_tdma_pause_chan; tdev->device.device_resume = mmp_tdma_resume_chan; tdev->device.device_terminate_all = mmp_tdma_terminate_all; - tdev->device.copy_align = TDMA_ALIGNMENT; + tdev->device.copy_align = DMAENGINE_ALIGN_8_BYTES; dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); platform_set_drvdata(pdev, tdev); diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c index f1325f62563e..1c2de9a834a9 100644 --- a/drivers/dma/mv_xor.c +++ b/drivers/dma/mv_xor.c @@ -13,7 +13,6 @@ */ #include -#include #include #include #include @@ -26,6 +25,7 @@ #include #include #include +#include #include #include "dmaengine.h" @@ -1126,7 +1126,8 @@ static const struct of_device_id mv_xor_dt_ids[] = { { .compatible = "marvell,armada-380-xor", .data = (void *)XOR_MODE_IN_DESC }, {}, }; -MODULE_DEVICE_TABLE(of, mv_xor_dt_ids); + +static unsigned int mv_xor_engine_count; static int mv_xor_probe(struct platform_device *pdev) { @@ -1134,6 +1135,7 @@ static int mv_xor_probe(struct platform_device *pdev) struct mv_xor_device *xordev; struct mv_xor_platform_data *pdata = dev_get_platdata(&pdev->dev); struct resource *res; + unsigned int max_engines, max_channels; int i, ret; int op_in_desc; @@ -1177,6 +1179,21 @@ static int mv_xor_probe(struct platform_device *pdev) if (!IS_ERR(xordev->clk)) clk_prepare_enable(xordev->clk); + /* + * We don't want to have more than one channel per CPU in + * order for async_tx to perform well. So we limit the number + * of engines and channels so that we take into account this + * constraint. Note that we also want to use channels from + * separate engines when possible. + */ + max_engines = num_present_cpus(); + max_channels = min_t(unsigned int, + MV_XOR_MAX_CHANNELS, + DIV_ROUND_UP(num_present_cpus(), 2)); + + if (mv_xor_engine_count >= max_engines) + return 0; + if (pdev->dev.of_node) { struct device_node *np; int i = 0; @@ -1190,13 +1207,13 @@ static int mv_xor_probe(struct platform_device *pdev) int irq; op_in_desc = (int)of_id->data; + if (i >= max_channels) + continue; + dma_cap_zero(cap_mask); - if (of_property_read_bool(np, "dmacap,memcpy")) - dma_cap_set(DMA_MEMCPY, cap_mask); - if (of_property_read_bool(np, "dmacap,xor")) - dma_cap_set(DMA_XOR, cap_mask); - if (of_property_read_bool(np, "dmacap,interrupt")) - dma_cap_set(DMA_INTERRUPT, cap_mask); + dma_cap_set(DMA_MEMCPY, cap_mask); + dma_cap_set(DMA_XOR, cap_mask); + dma_cap_set(DMA_INTERRUPT, cap_mask); irq = irq_of_parse_and_map(np, 0); if (!irq) { @@ -1216,7 +1233,7 @@ static int mv_xor_probe(struct platform_device *pdev) i++; } } else if (pdata && pdata->channels) { - for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) { + for (i = 0; i < max_channels; i++) { struct mv_xor_channel_data *cd; struct mv_xor_chan *chan; int irq; @@ -1263,27 +1280,8 @@ static int mv_xor_probe(struct platform_device *pdev) return ret; } -static int mv_xor_remove(struct platform_device *pdev) -{ - struct mv_xor_device *xordev = platform_get_drvdata(pdev); - int i; - - for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) { - if (xordev->channels[i]) - mv_xor_channel_remove(xordev->channels[i]); - } - - if (!IS_ERR(xordev->clk)) { - clk_disable_unprepare(xordev->clk); - clk_put(xordev->clk); - } - - return 0; -} - static struct platform_driver mv_xor_driver = { .probe = mv_xor_probe, - .remove = mv_xor_remove, .driver = { .name = MV_XOR_NAME, .of_match_table = of_match_ptr(mv_xor_dt_ids), @@ -1295,19 +1293,10 @@ static int __init mv_xor_init(void) { return platform_driver_register(&mv_xor_driver); } -module_init(mv_xor_init); - -/* it's currently unsafe to unload this module */ -#if 0 -static void __exit mv_xor_exit(void) -{ - platform_driver_unregister(&mv_xor_driver); - return; -} - -module_exit(mv_xor_exit); -#endif +device_initcall(mv_xor_init); +/* MODULE_AUTHOR("Saeed Bishara "); MODULE_DESCRIPTION("DMA engine driver for Marvell's XOR engine"); MODULE_LICENSE("GPL"); +*/ diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c index b859792dde95..113605f6fe20 100644 --- a/drivers/dma/pch_dma.c +++ b/drivers/dma/pch_dma.c @@ -11,10 +11,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ #include diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index ecab4ea059b4..17ee758b419f 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -1198,6 +1198,9 @@ static inline int _loop(unsigned dry_run, u8 buf[], unsigned lcnt0, lcnt1, ljmp0, ljmp1; struct _arg_LPEND lpend; + if (*bursts == 1) + return _bursts(dry_run, buf, pxs, 1); + /* Max iterations possible in DMALP is 256 */ if (*bursts >= 256*256) { lcnt1 = 256; diff --git a/drivers/dma/pxa_dma.c b/drivers/dma/pxa_dma.c index ddcbbf5cd9e9..5cb61ce01036 100644 --- a/drivers/dma/pxa_dma.c +++ b/drivers/dma/pxa_dma.c @@ -184,19 +184,18 @@ static unsigned int pxad_drcmr(unsigned int line) static int dbg_show_requester_chan(struct seq_file *s, void *p) { - int pos = 0; struct pxad_phy *phy = s->private; int i; u32 drcmr; - pos += seq_printf(s, "DMA channel %d requester :\n", phy->idx); + seq_printf(s, "DMA channel %d requester :\n", phy->idx); for (i = 0; i < 70; i++) { drcmr = readl_relaxed(phy->base + pxad_drcmr(i)); if ((drcmr & DRCMR_CHLNUM) == phy->idx) - pos += seq_printf(s, "\tRequester %d (MAPVLD=%d)\n", i, - !!(drcmr & DRCMR_MAPVLD)); + seq_printf(s, "\tRequester %d (MAPVLD=%d)\n", i, + !!(drcmr & DRCMR_MAPVLD)); } - return pos; + return 0; } static inline int dbg_burst_from_dcmd(u32 dcmd) @@ -906,21 +905,21 @@ static void pxad_get_config(struct pxad_chan *chan, enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED; *dcmd = 0; - if (chan->cfg.direction == DMA_DEV_TO_MEM) { + if (dir == DMA_DEV_TO_MEM) { maxburst = chan->cfg.src_maxburst; width = chan->cfg.src_addr_width; dev_addr = chan->cfg.src_addr; *dev_src = dev_addr; *dcmd |= PXA_DCMD_INCTRGADDR | PXA_DCMD_FLOWSRC; } - if (chan->cfg.direction == DMA_MEM_TO_DEV) { + if (dir == DMA_MEM_TO_DEV) { maxburst = chan->cfg.dst_maxburst; width = chan->cfg.dst_addr_width; dev_addr = chan->cfg.dst_addr; *dev_dst = dev_addr; *dcmd |= PXA_DCMD_INCSRCADDR | PXA_DCMD_FLOWTRG; } - if (chan->cfg.direction == DMA_MEM_TO_MEM) + if (dir == DMA_MEM_TO_MEM) *dcmd |= PXA_DCMD_BURST32 | PXA_DCMD_INCTRGADDR | PXA_DCMD_INCSRCADDR; diff --git a/drivers/dma/sh/Kconfig b/drivers/dma/sh/Kconfig index 0f371524a4d9..9fda65af841e 100644 --- a/drivers/dma/sh/Kconfig +++ b/drivers/dma/sh/Kconfig @@ -39,18 +39,6 @@ config SH_DMAE_R8A73A4 endif -config SUDMAC - tristate "Renesas SUDMAC support" - depends on SH_DMAE_BASE - help - Enable support for the Renesas SUDMAC controllers. - -config RCAR_HPB_DMAE - tristate "Renesas R-Car HPB DMAC support" - depends on SH_DMAE_BASE - help - Enable support for the Renesas R-Car series DMA controllers. - config RCAR_DMAC tristate "Renesas R-Car Gen2 DMA Controller" depends on ARCH_SHMOBILE || COMPILE_TEST @@ -59,6 +47,12 @@ config RCAR_DMAC This driver supports the general purpose DMA controller found in the Renesas R-Car second generation SoCs. +config RCAR_HPB_DMAE + tristate "Renesas R-Car HPB DMAC support" + depends on SH_DMAE_BASE + help + Enable support for the Renesas R-Car series DMA controllers. + config RENESAS_USB_DMAC tristate "Renesas USB-DMA Controller" depends on ARCH_SHMOBILE || COMPILE_TEST @@ -67,3 +61,9 @@ config RENESAS_USB_DMAC help This driver supports the USB-DMA controller found in the Renesas SoCs. + +config SUDMAC + tristate "Renesas SUDMAC support" + depends on SH_DMAE_BASE + help + Enable support for the Renesas SUDMAC controllers. diff --git a/drivers/dma/sh/Makefile b/drivers/dma/sh/Makefile index b8a598066ce2..0133e4658196 100644 --- a/drivers/dma/sh/Makefile +++ b/drivers/dma/sh/Makefile @@ -13,7 +13,7 @@ shdma-$(CONFIG_SH_DMAE_R8A73A4) += shdma-r8a73a4.o shdma-objs := $(shdma-y) obj-$(CONFIG_SH_DMAE) += shdma.o -obj-$(CONFIG_SUDMAC) += sudmac.o -obj-$(CONFIG_RCAR_HPB_DMAE) += rcar-hpbdma.o obj-$(CONFIG_RCAR_DMAC) += rcar-dmac.o +obj-$(CONFIG_RCAR_HPB_DMAE) += rcar-hpbdma.o obj-$(CONFIG_RENESAS_USB_DMAC) += usb-dmac.o +obj-$(CONFIG_SUDMAC) += sudmac.o diff --git a/drivers/dma/sirf-dma.c b/drivers/dma/sirf-dma.c index 8c5186cc9f63..7d5598d874e1 100644 --- a/drivers/dma/sirf-dma.c +++ b/drivers/dma/sirf-dma.c @@ -455,6 +455,7 @@ static int sirfsoc_dma_terminate_all(struct dma_chan *chan) switch (sdma->type) { case SIRFSOC_DMA_VER_A7V1: writel_relaxed(1 << cid, sdma->base + SIRFSOC_DMA_INT_EN_CLR); + writel_relaxed(1 << cid, sdma->base + SIRFSOC_DMA_CH_INT); writel_relaxed((1 << cid) | 1 << (cid + 16), sdma->base + SIRFSOC_DMA_CH_LOOP_CTRL_CLR_ATLAS7); @@ -462,6 +463,8 @@ static int sirfsoc_dma_terminate_all(struct dma_chan *chan) break; case SIRFSOC_DMA_VER_A7V2: writel_relaxed(0, sdma->base + SIRFSOC_DMA_INT_EN_ATLAS7); + writel_relaxed(SIRFSOC_DMA_INT_ALL_ATLAS7, + sdma->base + SIRFSOC_DMA_INT_ATLAS7); writel_relaxed(0, sdma->base + SIRFSOC_DMA_LOOP_CTRL_ATLAS7); writel_relaxed(0, sdma->base + SIRFSOC_DMA_VALID_ATLAS7); break; diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c index 3c10f034d4b9..750d1b313684 100644 --- a/drivers/dma/ste_dma40.c +++ b/drivers/dma/ste_dma40.c @@ -2853,7 +2853,7 @@ static void d40_ops_init(struct d40_base *base, struct dma_device *dev) * This controller can only access address at even * 32bit boundaries, i.e. 2^2 */ - dev->copy_align = 2; + dev->copy_align = DMAENGINE_ALIGN_4_BYTES; } if (dma_has_cap(DMA_SG, dev->cap_mask)) diff --git a/drivers/dma/sun4i-dma.c b/drivers/dma/sun4i-dma.c new file mode 100644 index 000000000000..a1a500d96ff2 --- /dev/null +++ b/drivers/dma/sun4i-dma.c @@ -0,0 +1,1288 @@ +/* + * Copyright (C) 2014 Emilio López + * Emilio López + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "virt-dma.h" + +/** Common macros to normal and dedicated DMA registers **/ + +#define SUN4I_DMA_CFG_LOADING BIT(31) +#define SUN4I_DMA_CFG_DST_DATA_WIDTH(width) ((width) << 25) +#define SUN4I_DMA_CFG_DST_BURST_LENGTH(len) ((len) << 23) +#define SUN4I_DMA_CFG_DST_ADDR_MODE(mode) ((mode) << 21) +#define SUN4I_DMA_CFG_DST_DRQ_TYPE(type) ((type) << 16) +#define SUN4I_DMA_CFG_SRC_DATA_WIDTH(width) ((width) << 9) +#define SUN4I_DMA_CFG_SRC_BURST_LENGTH(len) ((len) << 7) +#define SUN4I_DMA_CFG_SRC_ADDR_MODE(mode) ((mode) << 5) +#define SUN4I_DMA_CFG_SRC_DRQ_TYPE(type) (type) + +/** Normal DMA register values **/ + +/* Normal DMA source/destination data request type values */ +#define SUN4I_NDMA_DRQ_TYPE_SDRAM 0x16 +#define SUN4I_NDMA_DRQ_TYPE_LIMIT (0x1F + 1) + +/** Normal DMA register layout **/ + +/* Dedicated DMA source/destination address mode values */ +#define SUN4I_NDMA_ADDR_MODE_LINEAR 0 +#define SUN4I_NDMA_ADDR_MODE_IO 1 + +/* Normal DMA configuration register layout */ +#define SUN4I_NDMA_CFG_CONT_MODE BIT(30) +#define SUN4I_NDMA_CFG_WAIT_STATE(n) ((n) << 27) +#define SUN4I_NDMA_CFG_DST_NON_SECURE BIT(22) +#define SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN BIT(15) +#define SUN4I_NDMA_CFG_SRC_NON_SECURE BIT(6) + +/** Dedicated DMA register values **/ + +/* Dedicated DMA source/destination address mode values */ +#define SUN4I_DDMA_ADDR_MODE_LINEAR 0 +#define SUN4I_DDMA_ADDR_MODE_IO 1 +#define SUN4I_DDMA_ADDR_MODE_HORIZONTAL_PAGE 2 +#define SUN4I_DDMA_ADDR_MODE_VERTICAL_PAGE 3 + +/* Dedicated DMA source/destination data request type values */ +#define SUN4I_DDMA_DRQ_TYPE_SDRAM 0x1 +#define SUN4I_DDMA_DRQ_TYPE_LIMIT (0x1F + 1) + +/** Dedicated DMA register layout **/ + +/* Dedicated DMA configuration register layout */ +#define SUN4I_DDMA_CFG_BUSY BIT(30) +#define SUN4I_DDMA_CFG_CONT_MODE BIT(29) +#define SUN4I_DDMA_CFG_DST_NON_SECURE BIT(28) +#define SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN BIT(15) +#define SUN4I_DDMA_CFG_SRC_NON_SECURE BIT(12) + +/* Dedicated DMA parameter register layout */ +#define SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(n) (((n) - 1) << 24) +#define SUN4I_DDMA_PARA_DST_WAIT_CYCLES(n) (((n) - 1) << 16) +#define SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(n) (((n) - 1) << 8) +#define SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(n) (((n) - 1) << 0) + +/** DMA register offsets **/ + +/* General register offsets */ +#define SUN4I_DMA_IRQ_ENABLE_REG 0x0 +#define SUN4I_DMA_IRQ_PENDING_STATUS_REG 0x4 + +/* Normal DMA register offsets */ +#define SUN4I_NDMA_CHANNEL_REG_BASE(n) (0x100 + (n) * 0x20) +#define SUN4I_NDMA_CFG_REG 0x0 +#define SUN4I_NDMA_SRC_ADDR_REG 0x4 +#define SUN4I_NDMA_DST_ADDR_REG 0x8 +#define SUN4I_NDMA_BYTE_COUNT_REG 0xC + +/* Dedicated DMA register offsets */ +#define SUN4I_DDMA_CHANNEL_REG_BASE(n) (0x300 + (n) * 0x20) +#define SUN4I_DDMA_CFG_REG 0x0 +#define SUN4I_DDMA_SRC_ADDR_REG 0x4 +#define SUN4I_DDMA_DST_ADDR_REG 0x8 +#define SUN4I_DDMA_BYTE_COUNT_REG 0xC +#define SUN4I_DDMA_PARA_REG 0x18 + +/** DMA Driver **/ + +/* + * Normal DMA has 8 channels, and Dedicated DMA has another 8, so + * that's 16 channels. As for endpoints, there's 29 and 21 + * respectively. Given that the Normal DMA endpoints (other than + * SDRAM) can be used as tx/rx, we need 78 vchans in total + */ +#define SUN4I_NDMA_NR_MAX_CHANNELS 8 +#define SUN4I_DDMA_NR_MAX_CHANNELS 8 +#define SUN4I_DMA_NR_MAX_CHANNELS \ + (SUN4I_NDMA_NR_MAX_CHANNELS + SUN4I_DDMA_NR_MAX_CHANNELS) +#define SUN4I_NDMA_NR_MAX_VCHANS (29 * 2 - 1) +#define SUN4I_DDMA_NR_MAX_VCHANS 21 +#define SUN4I_DMA_NR_MAX_VCHANS \ + (SUN4I_NDMA_NR_MAX_VCHANS + SUN4I_DDMA_NR_MAX_VCHANS) + +/* This set of SUN4I_DDMA timing parameters were found experimentally while + * working with the SPI driver and seem to make it behave correctly */ +#define SUN4I_DDMA_MAGIC_SPI_PARAMETERS \ + (SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(1) | \ + SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(1) | \ + SUN4I_DDMA_PARA_DST_WAIT_CYCLES(2) | \ + SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(2)) + +struct sun4i_dma_pchan { + /* Register base of channel */ + void __iomem *base; + /* vchan currently being serviced */ + struct sun4i_dma_vchan *vchan; + /* Is this a dedicated pchan? */ + int is_dedicated; +}; + +struct sun4i_dma_vchan { + struct virt_dma_chan vc; + struct dma_slave_config cfg; + struct sun4i_dma_pchan *pchan; + struct sun4i_dma_promise *processing; + struct sun4i_dma_contract *contract; + u8 endpoint; + int is_dedicated; +}; + +struct sun4i_dma_promise { + u32 cfg; + u32 para; + dma_addr_t src; + dma_addr_t dst; + size_t len; + struct list_head list; +}; + +/* A contract is a set of promises */ +struct sun4i_dma_contract { + struct virt_dma_desc vd; + struct list_head demands; + struct list_head completed_demands; + int is_cyclic; +}; + +struct sun4i_dma_dev { + DECLARE_BITMAP(pchans_used, SUN4I_DMA_NR_MAX_CHANNELS); + struct dma_device slave; + struct sun4i_dma_pchan *pchans; + struct sun4i_dma_vchan *vchans; + void __iomem *base; + struct clk *clk; + int irq; + spinlock_t lock; +}; + +static struct sun4i_dma_dev *to_sun4i_dma_dev(struct dma_device *dev) +{ + return container_of(dev, struct sun4i_dma_dev, slave); +} + +static struct sun4i_dma_vchan *to_sun4i_dma_vchan(struct dma_chan *chan) +{ + return container_of(chan, struct sun4i_dma_vchan, vc.chan); +} + +static struct sun4i_dma_contract *to_sun4i_dma_contract(struct virt_dma_desc *vd) +{ + return container_of(vd, struct sun4i_dma_contract, vd); +} + +static struct device *chan2dev(struct dma_chan *chan) +{ + return &chan->dev->device; +} + +static int convert_burst(u32 maxburst) +{ + if (maxburst > 8) + return -EINVAL; + + /* 1 -> 0, 4 -> 1, 8 -> 2 */ + return (maxburst >> 2); +} + +static int convert_buswidth(enum dma_slave_buswidth addr_width) +{ + if (addr_width > DMA_SLAVE_BUSWIDTH_4_BYTES) + return -EINVAL; + + /* 8 (1 byte) -> 0, 16 (2 bytes) -> 1, 32 (4 bytes) -> 2 */ + return (addr_width >> 1); +} + +static void sun4i_dma_free_chan_resources(struct dma_chan *chan) +{ + struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); + + vchan_free_chan_resources(&vchan->vc); +} + +static struct sun4i_dma_pchan *find_and_use_pchan(struct sun4i_dma_dev *priv, + struct sun4i_dma_vchan *vchan) +{ + struct sun4i_dma_pchan *pchan = NULL, *pchans = priv->pchans; + unsigned long flags; + int i, max; + + /* + * pchans 0-SUN4I_NDMA_NR_MAX_CHANNELS are normal, and + * SUN4I_NDMA_NR_MAX_CHANNELS+ are dedicated ones + */ + if (vchan->is_dedicated) { + i = SUN4I_NDMA_NR_MAX_CHANNELS; + max = SUN4I_DMA_NR_MAX_CHANNELS; + } else { + i = 0; + max = SUN4I_NDMA_NR_MAX_CHANNELS; + } + + spin_lock_irqsave(&priv->lock, flags); + for_each_clear_bit_from(i, &priv->pchans_used, max) { + pchan = &pchans[i]; + pchan->vchan = vchan; + set_bit(i, priv->pchans_used); + break; + } + spin_unlock_irqrestore(&priv->lock, flags); + + return pchan; +} + +static void release_pchan(struct sun4i_dma_dev *priv, + struct sun4i_dma_pchan *pchan) +{ + unsigned long flags; + int nr = pchan - priv->pchans; + + spin_lock_irqsave(&priv->lock, flags); + + pchan->vchan = NULL; + clear_bit(nr, priv->pchans_used); + + spin_unlock_irqrestore(&priv->lock, flags); +} + +static void configure_pchan(struct sun4i_dma_pchan *pchan, + struct sun4i_dma_promise *d) +{ + /* + * Configure addresses and misc parameters depending on type + * SUN4I_DDMA has an extra field with timing parameters + */ + if (pchan->is_dedicated) { + writel_relaxed(d->src, pchan->base + SUN4I_DDMA_SRC_ADDR_REG); + writel_relaxed(d->dst, pchan->base + SUN4I_DDMA_DST_ADDR_REG); + writel_relaxed(d->len, pchan->base + SUN4I_DDMA_BYTE_COUNT_REG); + writel_relaxed(d->para, pchan->base + SUN4I_DDMA_PARA_REG); + writel_relaxed(d->cfg, pchan->base + SUN4I_DDMA_CFG_REG); + } else { + writel_relaxed(d->src, pchan->base + SUN4I_NDMA_SRC_ADDR_REG); + writel_relaxed(d->dst, pchan->base + SUN4I_NDMA_DST_ADDR_REG); + writel_relaxed(d->len, pchan->base + SUN4I_NDMA_BYTE_COUNT_REG); + writel_relaxed(d->cfg, pchan->base + SUN4I_NDMA_CFG_REG); + } +} + +static void set_pchan_interrupt(struct sun4i_dma_dev *priv, + struct sun4i_dma_pchan *pchan, + int half, int end) +{ + u32 reg; + int pchan_number = pchan - priv->pchans; + unsigned long flags; + + spin_lock_irqsave(&priv->lock, flags); + + reg = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG); + + if (half) + reg |= BIT(pchan_number * 2); + else + reg &= ~BIT(pchan_number * 2); + + if (end) + reg |= BIT(pchan_number * 2 + 1); + else + reg &= ~BIT(pchan_number * 2 + 1); + + writel_relaxed(reg, priv->base + SUN4I_DMA_IRQ_ENABLE_REG); + + spin_unlock_irqrestore(&priv->lock, flags); +} + +/** + * Execute pending operations on a vchan + * + * When given a vchan, this function will try to acquire a suitable + * pchan and, if successful, will configure it to fulfill a promise + * from the next pending contract. + * + * This function must be called with &vchan->vc.lock held. + */ +static int __execute_vchan_pending(struct sun4i_dma_dev *priv, + struct sun4i_dma_vchan *vchan) +{ + struct sun4i_dma_promise *promise = NULL; + struct sun4i_dma_contract *contract = NULL; + struct sun4i_dma_pchan *pchan; + struct virt_dma_desc *vd; + int ret; + + lockdep_assert_held(&vchan->vc.lock); + + /* We need a pchan to do anything, so secure one if available */ + pchan = find_and_use_pchan(priv, vchan); + if (!pchan) + return -EBUSY; + + /* + * Channel endpoints must not be repeated, so if this vchan + * has already submitted some work, we can't do anything else + */ + if (vchan->processing) { + dev_dbg(chan2dev(&vchan->vc.chan), + "processing something to this endpoint already\n"); + ret = -EBUSY; + goto release_pchan; + } + + do { + /* Figure out which contract we're working with today */ + vd = vchan_next_desc(&vchan->vc); + if (!vd) { + dev_dbg(chan2dev(&vchan->vc.chan), + "No pending contract found"); + ret = 0; + goto release_pchan; + } + + contract = to_sun4i_dma_contract(vd); + if (list_empty(&contract->demands)) { + /* The contract has been completed so mark it as such */ + list_del(&contract->vd.node); + vchan_cookie_complete(&contract->vd); + dev_dbg(chan2dev(&vchan->vc.chan), + "Empty contract found and marked complete"); + } + } while (list_empty(&contract->demands)); + + /* Now find out what we need to do */ + promise = list_first_entry(&contract->demands, + struct sun4i_dma_promise, list); + vchan->processing = promise; + + /* ... and make it reality */ + if (promise) { + vchan->contract = contract; + vchan->pchan = pchan; + set_pchan_interrupt(priv, pchan, contract->is_cyclic, 1); + configure_pchan(pchan, promise); + } + + return 0; + +release_pchan: + release_pchan(priv, pchan); + return ret; +} + +static int sanitize_config(struct dma_slave_config *sconfig, + enum dma_transfer_direction direction) +{ + switch (direction) { + case DMA_MEM_TO_DEV: + if ((sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) || + !sconfig->dst_maxburst) + return -EINVAL; + + if (sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) + sconfig->src_addr_width = sconfig->dst_addr_width; + + if (!sconfig->src_maxburst) + sconfig->src_maxburst = sconfig->dst_maxburst; + + break; + + case DMA_DEV_TO_MEM: + if ((sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) || + !sconfig->src_maxburst) + return -EINVAL; + + if (sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) + sconfig->dst_addr_width = sconfig->src_addr_width; + + if (!sconfig->dst_maxburst) + sconfig->dst_maxburst = sconfig->src_maxburst; + + break; + default: + return 0; + } + + return 0; +} + +/** + * Generate a promise, to be used in a normal DMA contract. + * + * A NDMA promise contains all the information required to program the + * normal part of the DMA Engine and get data copied. A non-executed + * promise will live in the demands list on a contract. Once it has been + * completed, it will be moved to the completed demands list for later freeing. + * All linked promises will be freed when the corresponding contract is freed + */ +static struct sun4i_dma_promise * +generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, + size_t len, struct dma_slave_config *sconfig, + enum dma_transfer_direction direction) +{ + struct sun4i_dma_promise *promise; + int ret; + + ret = sanitize_config(sconfig, direction); + if (ret) + return NULL; + + promise = kzalloc(sizeof(*promise), GFP_NOWAIT); + if (!promise) + return NULL; + + promise->src = src; + promise->dst = dest; + promise->len = len; + promise->cfg = SUN4I_DMA_CFG_LOADING | + SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN; + + dev_dbg(chan2dev(chan), + "src burst %d, dst burst %d, src buswidth %d, dst buswidth %d", + sconfig->src_maxburst, sconfig->dst_maxburst, + sconfig->src_addr_width, sconfig->dst_addr_width); + + /* Source burst */ + ret = convert_burst(sconfig->src_maxburst); + if (IS_ERR_VALUE(ret)) + goto fail; + promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret); + + /* Destination burst */ + ret = convert_burst(sconfig->dst_maxburst); + if (IS_ERR_VALUE(ret)) + goto fail; + promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret); + + /* Source bus width */ + ret = convert_buswidth(sconfig->src_addr_width); + if (IS_ERR_VALUE(ret)) + goto fail; + promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret); + + /* Destination bus width */ + ret = convert_buswidth(sconfig->dst_addr_width); + if (IS_ERR_VALUE(ret)) + goto fail; + promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret); + + return promise; + +fail: + kfree(promise); + return NULL; +} + +/** + * Generate a promise, to be used in a dedicated DMA contract. + * + * A DDMA promise contains all the information required to program the + * Dedicated part of the DMA Engine and get data copied. A non-executed + * promise will live in the demands list on a contract. Once it has been + * completed, it will be moved to the completed demands list for later freeing. + * All linked promises will be freed when the corresponding contract is freed + */ +static struct sun4i_dma_promise * +generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, + size_t len, struct dma_slave_config *sconfig) +{ + struct sun4i_dma_promise *promise; + int ret; + + promise = kzalloc(sizeof(*promise), GFP_NOWAIT); + if (!promise) + return NULL; + + promise->src = src; + promise->dst = dest; + promise->len = len; + promise->cfg = SUN4I_DMA_CFG_LOADING | + SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN; + + /* Source burst */ + ret = convert_burst(sconfig->src_maxburst); + if (IS_ERR_VALUE(ret)) + goto fail; + promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret); + + /* Destination burst */ + ret = convert_burst(sconfig->dst_maxburst); + if (IS_ERR_VALUE(ret)) + goto fail; + promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret); + + /* Source bus width */ + ret = convert_buswidth(sconfig->src_addr_width); + if (IS_ERR_VALUE(ret)) + goto fail; + promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret); + + /* Destination bus width */ + ret = convert_buswidth(sconfig->dst_addr_width); + if (IS_ERR_VALUE(ret)) + goto fail; + promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret); + + return promise; + +fail: + kfree(promise); + return NULL; +} + +/** + * Generate a contract + * + * Contracts function as DMA descriptors. As our hardware does not support + * linked lists, we need to implement SG via software. We use a contract + * to hold all the pieces of the request and process them serially one + * after another. Each piece is represented as a promise. + */ +static struct sun4i_dma_contract *generate_dma_contract(void) +{ + struct sun4i_dma_contract *contract; + + contract = kzalloc(sizeof(*contract), GFP_NOWAIT); + if (!contract) + return NULL; + + INIT_LIST_HEAD(&contract->demands); + INIT_LIST_HEAD(&contract->completed_demands); + + return contract; +} + +/** + * Get next promise on a cyclic transfer + * + * Cyclic contracts contain a series of promises which are executed on a + * loop. This function returns the next promise from a cyclic contract, + * so it can be programmed into the hardware. + */ +static struct sun4i_dma_promise * +get_next_cyclic_promise(struct sun4i_dma_contract *contract) +{ + struct sun4i_dma_promise *promise; + + promise = list_first_entry_or_null(&contract->demands, + struct sun4i_dma_promise, list); + if (!promise) { + list_splice_init(&contract->completed_demands, + &contract->demands); + promise = list_first_entry(&contract->demands, + struct sun4i_dma_promise, list); + } + + return promise; +} + +/** + * Free a contract and all its associated promises + */ +static void sun4i_dma_free_contract(struct virt_dma_desc *vd) +{ + struct sun4i_dma_contract *contract = to_sun4i_dma_contract(vd); + struct sun4i_dma_promise *promise; + + /* Free all the demands and completed demands */ + list_for_each_entry(promise, &contract->demands, list) + kfree(promise); + + list_for_each_entry(promise, &contract->completed_demands, list) + kfree(promise); + + kfree(contract); +} + +static struct dma_async_tx_descriptor * +sun4i_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, + dma_addr_t src, size_t len, unsigned long flags) +{ + struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); + struct dma_slave_config *sconfig = &vchan->cfg; + struct sun4i_dma_promise *promise; + struct sun4i_dma_contract *contract; + + contract = generate_dma_contract(); + if (!contract) + return NULL; + + /* + * We can only do the copy to bus aligned addresses, so + * choose the best one so we get decent performance. We also + * maximize the burst size for this same reason. + */ + sconfig->src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + sconfig->dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + sconfig->src_maxburst = 8; + sconfig->dst_maxburst = 8; + + if (vchan->is_dedicated) + promise = generate_ddma_promise(chan, src, dest, len, sconfig); + else + promise = generate_ndma_promise(chan, src, dest, len, sconfig, + DMA_MEM_TO_MEM); + + if (!promise) { + kfree(contract); + return NULL; + } + + /* Configure memcpy mode */ + if (vchan->is_dedicated) { + promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM) | + SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM); + } else { + promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM) | + SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM); + } + + /* Fill the contract with our only promise */ + list_add_tail(&promise->list, &contract->demands); + + /* And add it to the vchan */ + return vchan_tx_prep(&vchan->vc, &contract->vd, flags); +} + +static struct dma_async_tx_descriptor * +sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len, + size_t period_len, enum dma_transfer_direction dir, + unsigned long flags) +{ + struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); + struct dma_slave_config *sconfig = &vchan->cfg; + struct sun4i_dma_promise *promise; + struct sun4i_dma_contract *contract; + dma_addr_t src, dest; + u32 endpoints; + int nr_periods, offset, plength, i; + + if (!is_slave_direction(dir)) { + dev_err(chan2dev(chan), "Invalid DMA direction\n"); + return NULL; + } + + if (vchan->is_dedicated) { + /* + * As we are using this just for audio data, we need to use + * normal DMA. There is nothing stopping us from supporting + * dedicated DMA here as well, so if a client comes up and + * requires it, it will be simple to implement it. + */ + dev_err(chan2dev(chan), + "Cyclic transfers are only supported on Normal DMA\n"); + return NULL; + } + + contract = generate_dma_contract(); + if (!contract) + return NULL; + + contract->is_cyclic = 1; + + /* Figure out the endpoints and the address we need */ + if (dir == DMA_MEM_TO_DEV) { + src = buf; + dest = sconfig->dst_addr; + endpoints = SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM) | + SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) | + SUN4I_DMA_CFG_DST_ADDR_MODE(SUN4I_NDMA_ADDR_MODE_IO); + } else { + src = sconfig->src_addr; + dest = buf; + endpoints = SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) | + SUN4I_DMA_CFG_SRC_ADDR_MODE(SUN4I_NDMA_ADDR_MODE_IO) | + SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM); + } + + /* + * We will be using half done interrupts to make two periods + * out of a promise, so we need to program the DMA engine less + * often + */ + + /* + * The engine can interrupt on half-transfer, so we can use + * this feature to program the engine half as often as if we + * didn't use it (keep in mind the hardware doesn't support + * linked lists). + * + * Say you have a set of periods (| marks the start/end, I for + * interrupt, P for programming the engine to do a new + * transfer), the easy but slow way would be to do + * + * |---|---|---|---| (periods / promises) + * P I,P I,P I,P I + * + * Using half transfer interrupts you can do + * + * |-------|-------| (promises as configured on hw) + * |---|---|---|---| (periods) + * P I I,P I I + * + * Which requires half the engine programming for the same + * functionality. + */ + nr_periods = DIV_ROUND_UP(len / period_len, 2); + for (i = 0; i < nr_periods; i++) { + /* Calculate the offset in the buffer and the length needed */ + offset = i * period_len * 2; + plength = min((len - offset), (period_len * 2)); + if (dir == DMA_MEM_TO_DEV) + src = buf + offset; + else + dest = buf + offset; + + /* Make the promise */ + promise = generate_ndma_promise(chan, src, dest, + plength, sconfig, dir); + if (!promise) { + /* TODO: should we free everything? */ + return NULL; + } + promise->cfg |= endpoints; + + /* Then add it to the contract */ + list_add_tail(&promise->list, &contract->demands); + } + + /* And add it to the vchan */ + return vchan_tx_prep(&vchan->vc, &contract->vd, flags); +} + +static struct dma_async_tx_descriptor * +sun4i_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction dir, + unsigned long flags, void *context) +{ + struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); + struct dma_slave_config *sconfig = &vchan->cfg; + struct sun4i_dma_promise *promise; + struct sun4i_dma_contract *contract; + u8 ram_type, io_mode, linear_mode; + struct scatterlist *sg; + dma_addr_t srcaddr, dstaddr; + u32 endpoints, para; + int i; + + if (!sgl) + return NULL; + + if (!is_slave_direction(dir)) { + dev_err(chan2dev(chan), "Invalid DMA direction\n"); + return NULL; + } + + contract = generate_dma_contract(); + if (!contract) + return NULL; + + if (vchan->is_dedicated) { + io_mode = SUN4I_DDMA_ADDR_MODE_IO; + linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR; + ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM; + } else { + io_mode = SUN4I_NDMA_ADDR_MODE_IO; + linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR; + ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM; + } + + if (dir == DMA_MEM_TO_DEV) + endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) | + SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) | + SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) | + SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode); + else + endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) | + SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) | + SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) | + SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode); + + for_each_sg(sgl, sg, sg_len, i) { + /* Figure out addresses */ + if (dir == DMA_MEM_TO_DEV) { + srcaddr = sg_dma_address(sg); + dstaddr = sconfig->dst_addr; + } else { + srcaddr = sconfig->src_addr; + dstaddr = sg_dma_address(sg); + } + + /* + * These are the magic DMA engine timings that keep SPI going. + * I haven't seen any interface on DMAEngine to configure + * timings, and so far they seem to work for everything we + * support, so I've kept them here. I don't know if other + * devices need different timings because, as usual, we only + * have the "para" bitfield meanings, but no comment on what + * the values should be when doing a certain operation :| + */ + para = SUN4I_DDMA_MAGIC_SPI_PARAMETERS; + + /* And make a suitable promise */ + if (vchan->is_dedicated) + promise = generate_ddma_promise(chan, srcaddr, dstaddr, + sg_dma_len(sg), + sconfig); + else + promise = generate_ndma_promise(chan, srcaddr, dstaddr, + sg_dma_len(sg), + sconfig, dir); + + if (!promise) + return NULL; /* TODO: should we free everything? */ + + promise->cfg |= endpoints; + promise->para = para; + + /* Then add it to the contract */ + list_add_tail(&promise->list, &contract->demands); + } + + /* + * Once we've got all the promises ready, add the contract + * to the pending list on the vchan + */ + return vchan_tx_prep(&vchan->vc, &contract->vd, flags); +} + +static int sun4i_dma_terminate_all(struct dma_chan *chan) +{ + struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device); + struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); + struct sun4i_dma_pchan *pchan = vchan->pchan; + LIST_HEAD(head); + unsigned long flags; + + spin_lock_irqsave(&vchan->vc.lock, flags); + vchan_get_all_descriptors(&vchan->vc, &head); + spin_unlock_irqrestore(&vchan->vc.lock, flags); + + /* + * Clearing the configuration register will halt the pchan. Interrupts + * may still trigger, so don't forget to disable them. + */ + if (pchan) { + if (pchan->is_dedicated) + writel(0, pchan->base + SUN4I_DDMA_CFG_REG); + else + writel(0, pchan->base + SUN4I_NDMA_CFG_REG); + set_pchan_interrupt(priv, pchan, 0, 0); + release_pchan(priv, pchan); + } + + spin_lock_irqsave(&vchan->vc.lock, flags); + vchan_dma_desc_free_list(&vchan->vc, &head); + /* Clear these so the vchan is usable again */ + vchan->processing = NULL; + vchan->pchan = NULL; + spin_unlock_irqrestore(&vchan->vc.lock, flags); + + return 0; +} + +static int sun4i_dma_config(struct dma_chan *chan, + struct dma_slave_config *config) +{ + struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); + + memcpy(&vchan->cfg, config, sizeof(*config)); + + return 0; +} + +static struct dma_chan *sun4i_dma_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct sun4i_dma_dev *priv = ofdma->of_dma_data; + struct sun4i_dma_vchan *vchan; + struct dma_chan *chan; + u8 is_dedicated = dma_spec->args[0]; + u8 endpoint = dma_spec->args[1]; + + /* Check if type is Normal or Dedicated */ + if (is_dedicated != 0 && is_dedicated != 1) + return NULL; + + /* Make sure the endpoint looks sane */ + if ((is_dedicated && endpoint >= SUN4I_DDMA_DRQ_TYPE_LIMIT) || + (!is_dedicated && endpoint >= SUN4I_NDMA_DRQ_TYPE_LIMIT)) + return NULL; + + chan = dma_get_any_slave_channel(&priv->slave); + if (!chan) + return NULL; + + /* Assign the endpoint to the vchan */ + vchan = to_sun4i_dma_vchan(chan); + vchan->is_dedicated = is_dedicated; + vchan->endpoint = endpoint; + + return chan; +} + +static enum dma_status sun4i_dma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, + struct dma_tx_state *state) +{ + struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); + struct sun4i_dma_pchan *pchan = vchan->pchan; + struct sun4i_dma_contract *contract; + struct sun4i_dma_promise *promise; + struct virt_dma_desc *vd; + unsigned long flags; + enum dma_status ret; + size_t bytes = 0; + + ret = dma_cookie_status(chan, cookie, state); + if (!state || (ret == DMA_COMPLETE)) + return ret; + + spin_lock_irqsave(&vchan->vc.lock, flags); + vd = vchan_find_desc(&vchan->vc, cookie); + if (!vd) + goto exit; + contract = to_sun4i_dma_contract(vd); + + list_for_each_entry(promise, &contract->demands, list) + bytes += promise->len; + + /* + * The hardware is configured to return the remaining byte + * quantity. If possible, replace the first listed element's + * full size with the actual remaining amount + */ + promise = list_first_entry_or_null(&contract->demands, + struct sun4i_dma_promise, list); + if (promise && pchan) { + bytes -= promise->len; + if (pchan->is_dedicated) + bytes += readl(pchan->base + SUN4I_DDMA_BYTE_COUNT_REG); + else + bytes += readl(pchan->base + SUN4I_NDMA_BYTE_COUNT_REG); + } + +exit: + + dma_set_residue(state, bytes); + spin_unlock_irqrestore(&vchan->vc.lock, flags); + + return ret; +} + +static void sun4i_dma_issue_pending(struct dma_chan *chan) +{ + struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device); + struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); + unsigned long flags; + + spin_lock_irqsave(&vchan->vc.lock, flags); + + /* + * If there are pending transactions for this vchan, push one of + * them into the engine to get the ball rolling. + */ + if (vchan_issue_pending(&vchan->vc)) + __execute_vchan_pending(priv, vchan); + + spin_unlock_irqrestore(&vchan->vc.lock, flags); +} + +static irqreturn_t sun4i_dma_interrupt(int irq, void *dev_id) +{ + struct sun4i_dma_dev *priv = dev_id; + struct sun4i_dma_pchan *pchans = priv->pchans, *pchan; + struct sun4i_dma_vchan *vchan; + struct sun4i_dma_contract *contract; + struct sun4i_dma_promise *promise; + unsigned long pendirq, irqs, disableirqs; + int bit, i, free_room, allow_mitigation = 1; + + pendirq = readl_relaxed(priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG); + +handle_pending: + + disableirqs = 0; + free_room = 0; + + for_each_set_bit(bit, &pendirq, 32) { + pchan = &pchans[bit >> 1]; + vchan = pchan->vchan; + if (!vchan) /* a terminated channel may still interrupt */ + continue; + contract = vchan->contract; + + /* + * Disable the IRQ and free the pchan if it's an end + * interrupt (odd bit) + */ + if (bit & 1) { + spin_lock(&vchan->vc.lock); + + /* + * Move the promise into the completed list now that + * we're done with it + */ + list_del(&vchan->processing->list); + list_add_tail(&vchan->processing->list, + &contract->completed_demands); + + /* + * Cyclic DMA transfers are special: + * - There's always something we can dispatch + * - We need to run the callback + * - Latency is very important, as this is used by audio + * We therefore just cycle through the list and dispatch + * whatever we have here, reusing the pchan. There's + * no need to run the thread after this. + * + * For non-cyclic transfers we need to look around, + * so we can program some more work, or notify the + * client that their transfers have been completed. + */ + if (contract->is_cyclic) { + promise = get_next_cyclic_promise(contract); + vchan->processing = promise; + configure_pchan(pchan, promise); + vchan_cyclic_callback(&contract->vd); + } else { + vchan->processing = NULL; + vchan->pchan = NULL; + + free_room = 1; + disableirqs |= BIT(bit); + release_pchan(priv, pchan); + } + + spin_unlock(&vchan->vc.lock); + } else { + /* Half done interrupt */ + if (contract->is_cyclic) + vchan_cyclic_callback(&contract->vd); + else + disableirqs |= BIT(bit); + } + } + + /* Disable the IRQs for events we handled */ + spin_lock(&priv->lock); + irqs = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG); + writel_relaxed(irqs & ~disableirqs, + priv->base + SUN4I_DMA_IRQ_ENABLE_REG); + spin_unlock(&priv->lock); + + /* Writing 1 to the pending field will clear the pending interrupt */ + writel_relaxed(pendirq, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG); + + /* + * If a pchan was freed, we may be able to schedule something else, + * so have a look around + */ + if (free_room) { + for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) { + vchan = &priv->vchans[i]; + spin_lock(&vchan->vc.lock); + __execute_vchan_pending(priv, vchan); + spin_unlock(&vchan->vc.lock); + } + } + + /* + * Handle newer interrupts if some showed up, but only do it once + * to avoid a too long a loop + */ + if (allow_mitigation) { + pendirq = readl_relaxed(priv->base + + SUN4I_DMA_IRQ_PENDING_STATUS_REG); + if (pendirq) { + allow_mitigation = 0; + goto handle_pending; + } + } + + return IRQ_HANDLED; +} + +static int sun4i_dma_probe(struct platform_device *pdev) +{ + struct sun4i_dma_dev *priv; + struct resource *res; + int i, j, ret; + + priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + priv->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(priv->base)) + return PTR_ERR(priv->base); + + priv->irq = platform_get_irq(pdev, 0); + if (priv->irq < 0) { + dev_err(&pdev->dev, "Cannot claim IRQ\n"); + return priv->irq; + } + + priv->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(priv->clk)) { + dev_err(&pdev->dev, "No clock specified\n"); + return PTR_ERR(priv->clk); + } + + platform_set_drvdata(pdev, priv); + spin_lock_init(&priv->lock); + + dma_cap_zero(priv->slave.cap_mask); + dma_cap_set(DMA_PRIVATE, priv->slave.cap_mask); + dma_cap_set(DMA_MEMCPY, priv->slave.cap_mask); + dma_cap_set(DMA_CYCLIC, priv->slave.cap_mask); + dma_cap_set(DMA_SLAVE, priv->slave.cap_mask); + + INIT_LIST_HEAD(&priv->slave.channels); + priv->slave.device_free_chan_resources = sun4i_dma_free_chan_resources; + priv->slave.device_tx_status = sun4i_dma_tx_status; + priv->slave.device_issue_pending = sun4i_dma_issue_pending; + priv->slave.device_prep_slave_sg = sun4i_dma_prep_slave_sg; + priv->slave.device_prep_dma_memcpy = sun4i_dma_prep_dma_memcpy; + priv->slave.device_prep_dma_cyclic = sun4i_dma_prep_dma_cyclic; + priv->slave.device_config = sun4i_dma_config; + priv->slave.device_terminate_all = sun4i_dma_terminate_all; + priv->slave.copy_align = 2; + priv->slave.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + priv->slave.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + priv->slave.directions = BIT(DMA_DEV_TO_MEM) | + BIT(DMA_MEM_TO_DEV); + priv->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + + priv->slave.dev = &pdev->dev; + + priv->pchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_CHANNELS, + sizeof(struct sun4i_dma_pchan), GFP_KERNEL); + priv->vchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_VCHANS, + sizeof(struct sun4i_dma_vchan), GFP_KERNEL); + if (!priv->vchans || !priv->pchans) + return -ENOMEM; + + /* + * [0..SUN4I_NDMA_NR_MAX_CHANNELS) are normal pchans, and + * [SUN4I_NDMA_NR_MAX_CHANNELS..SUN4I_DMA_NR_MAX_CHANNELS) are + * dedicated ones + */ + for (i = 0; i < SUN4I_NDMA_NR_MAX_CHANNELS; i++) + priv->pchans[i].base = priv->base + + SUN4I_NDMA_CHANNEL_REG_BASE(i); + + for (j = 0; i < SUN4I_DMA_NR_MAX_CHANNELS; i++, j++) { + priv->pchans[i].base = priv->base + + SUN4I_DDMA_CHANNEL_REG_BASE(j); + priv->pchans[i].is_dedicated = 1; + } + + for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) { + struct sun4i_dma_vchan *vchan = &priv->vchans[i]; + + spin_lock_init(&vchan->vc.lock); + vchan->vc.desc_free = sun4i_dma_free_contract; + vchan_init(&vchan->vc, &priv->slave); + } + + ret = clk_prepare_enable(priv->clk); + if (ret) { + dev_err(&pdev->dev, "Couldn't enable the clock\n"); + return ret; + } + + /* + * Make sure the IRQs are all disabled and accounted for. The bootloader + * likes to leave these dirty + */ + writel(0, priv->base + SUN4I_DMA_IRQ_ENABLE_REG); + writel(0xFFFFFFFF, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG); + + ret = devm_request_irq(&pdev->dev, priv->irq, sun4i_dma_interrupt, + 0, dev_name(&pdev->dev), priv); + if (ret) { + dev_err(&pdev->dev, "Cannot request IRQ\n"); + goto err_clk_disable; + } + + ret = dma_async_device_register(&priv->slave); + if (ret) { + dev_warn(&pdev->dev, "Failed to register DMA engine device\n"); + goto err_clk_disable; + } + + ret = of_dma_controller_register(pdev->dev.of_node, sun4i_dma_of_xlate, + priv); + if (ret) { + dev_err(&pdev->dev, "of_dma_controller_register failed\n"); + goto err_dma_unregister; + } + + dev_dbg(&pdev->dev, "Successfully probed SUN4I_DMA\n"); + + return 0; + +err_dma_unregister: + dma_async_device_unregister(&priv->slave); +err_clk_disable: + clk_disable_unprepare(priv->clk); + return ret; +} + +static int sun4i_dma_remove(struct platform_device *pdev) +{ + struct sun4i_dma_dev *priv = platform_get_drvdata(pdev); + + /* Disable IRQ so no more work is scheduled */ + disable_irq(priv->irq); + + of_dma_controller_free(pdev->dev.of_node); + dma_async_device_unregister(&priv->slave); + + clk_disable_unprepare(priv->clk); + + return 0; +} + +static const struct of_device_id sun4i_dma_match[] = { + { .compatible = "allwinner,sun4i-a10-dma" }, + { /* sentinel */ }, +}; + +static struct platform_driver sun4i_dma_driver = { + .probe = sun4i_dma_probe, + .remove = sun4i_dma_remove, + .driver = { + .name = "sun4i-dma", + .of_match_table = sun4i_dma_match, + }, +}; + +module_platform_driver(sun4i_dma_driver); + +MODULE_DESCRIPTION("Allwinner A10 Dedicated DMA Controller Driver"); +MODULE_AUTHOR("Emilio López "); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c index 842ff97c2cfb..73e0be6e2100 100644 --- a/drivers/dma/sun6i-dma.c +++ b/drivers/dma/sun6i-dma.c @@ -969,7 +969,7 @@ static int sun6i_dma_probe(struct platform_device *pdev) sdc->slave.device_issue_pending = sun6i_dma_issue_pending; sdc->slave.device_prep_slave_sg = sun6i_dma_prep_slave_sg; sdc->slave.device_prep_dma_memcpy = sun6i_dma_prep_dma_memcpy; - sdc->slave.copy_align = 4; + sdc->slave.copy_align = DMAENGINE_ALIGN_4_BYTES; sdc->slave.device_config = sun6i_dma_config; sdc->slave.device_pause = sun6i_dma_pause; sdc->slave.device_resume = sun6i_dma_resume; diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c index eaf585e8286b..c8f79dcaaee8 100644 --- a/drivers/dma/tegra20-apb-dma.c +++ b/drivers/dma/tegra20-apb-dma.c @@ -155,7 +155,6 @@ struct tegra_dma_sg_req { int req_len; bool configured; bool last_sg; - bool half_done; struct list_head node; struct tegra_dma_desc *dma_desc; }; @@ -188,7 +187,7 @@ struct tegra_dma_channel { bool config_init; int id; int irq; - unsigned long chan_base_offset; + void __iomem *chan_addr; spinlock_t lock; bool busy; struct tegra_dma *tdma; @@ -203,8 +202,6 @@ struct tegra_dma_channel { /* ISR handler and tasklet for bottom half of isr handling */ dma_isr_handler isr_handler; struct tasklet_struct tasklet; - dma_async_tx_callback callback; - void *callback_param; /* Channel-slave specific configuration */ unsigned int slave_id; @@ -222,6 +219,13 @@ struct tegra_dma { void __iomem *base_addr; const struct tegra_dma_chip_data *chip_data; + /* + * Counter for managing global pausing of the DMA controller. + * Only applicable for devices that don't support individual + * channel pausing. + */ + u32 global_pause_count; + /* Some register need to be cache before suspend */ u32 reg_gen; @@ -242,12 +246,12 @@ static inline u32 tdma_read(struct tegra_dma *tdma, u32 reg) static inline void tdc_write(struct tegra_dma_channel *tdc, u32 reg, u32 val) { - writel(val, tdc->tdma->base_addr + tdc->chan_base_offset + reg); + writel(val, tdc->chan_addr + reg); } static inline u32 tdc_read(struct tegra_dma_channel *tdc, u32 reg) { - return readl(tdc->tdma->base_addr + tdc->chan_base_offset + reg); + return readl(tdc->chan_addr + reg); } static inline struct tegra_dma_channel *to_tegra_dma_chan(struct dma_chan *dc) @@ -361,16 +365,32 @@ static void tegra_dma_global_pause(struct tegra_dma_channel *tdc, struct tegra_dma *tdma = tdc->tdma; spin_lock(&tdma->global_lock); - tdma_write(tdma, TEGRA_APBDMA_GENERAL, 0); - if (wait_for_burst_complete) - udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME); + + if (tdc->tdma->global_pause_count == 0) { + tdma_write(tdma, TEGRA_APBDMA_GENERAL, 0); + if (wait_for_burst_complete) + udelay(TEGRA_APBDMA_BURST_COMPLETE_TIME); + } + + tdc->tdma->global_pause_count++; + + spin_unlock(&tdma->global_lock); } static void tegra_dma_global_resume(struct tegra_dma_channel *tdc) { struct tegra_dma *tdma = tdc->tdma; - tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE); + spin_lock(&tdma->global_lock); + + if (WARN_ON(tdc->tdma->global_pause_count == 0)) + goto out; + + if (--tdc->tdma->global_pause_count == 0) + tdma_write(tdma, TEGRA_APBDMA_GENERAL, + TEGRA_APBDMA_GENERAL_ENABLE); + +out: spin_unlock(&tdma->global_lock); } @@ -601,7 +621,6 @@ static void handle_once_dma_done(struct tegra_dma_channel *tdc, return; tdc_start_head_req(tdc); - return; } static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc, @@ -628,7 +647,6 @@ static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc, if (!st) dma_desc->dma_status = DMA_ERROR; } - return; } static void tegra_dma_tasklet(unsigned long data) @@ -720,7 +738,6 @@ static void tegra_dma_issue_pending(struct dma_chan *dc) } end: spin_unlock_irqrestore(&tdc->lock, flags); - return; } static int tegra_dma_terminate_all(struct dma_chan *dc) @@ -932,7 +949,6 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( struct tegra_dma_sg_req *sg_req = NULL; u32 burst_size; enum dma_slave_buswidth slave_bw; - int ret; if (!tdc->config_init) { dev_err(tdc2dev(tdc), "dma channel is not configured\n"); @@ -943,9 +959,8 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( return NULL; } - ret = get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr, - &burst_size, &slave_bw); - if (ret < 0) + if (get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr, + &burst_size, &slave_bw) < 0) return NULL; INIT_LIST_HEAD(&req_list); @@ -1048,7 +1063,6 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic( dma_addr_t mem = buf_addr; u32 burst_size; enum dma_slave_buswidth slave_bw; - int ret; if (!buf_len || !period_len) { dev_err(tdc2dev(tdc), "Invalid buffer/period len\n"); @@ -1087,12 +1101,10 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic( return NULL; } - ret = get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr, - &burst_size, &slave_bw); - if (ret < 0) + if (get_transfer_param(tdc, direction, &apb_ptr, &apb_seq, &csr, + &burst_size, &slave_bw) < 0) return NULL; - ahb_seq = TEGRA_APBDMA_AHBSEQ_INTR_ENB; ahb_seq |= TEGRA_APBDMA_AHBSEQ_WRAP_NONE << TEGRA_APBDMA_AHBSEQ_WRAP_SHIFT; @@ -1136,7 +1148,6 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic( sg_req->ch_regs.apb_seq = apb_seq; sg_req->ch_regs.ahb_seq = ahb_seq; sg_req->configured = false; - sg_req->half_done = false; sg_req->last_sg = false; sg_req->dma_desc = dma_desc; sg_req->req_len = len; @@ -1377,8 +1388,9 @@ static int tegra_dma_probe(struct platform_device *pdev) for (i = 0; i < cdata->nr_channels; i++) { struct tegra_dma_channel *tdc = &tdma->channels[i]; - tdc->chan_base_offset = TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET + - i * cdata->channel_reg_size; + tdc->chan_addr = tdma->base_addr + + TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET + + (i * cdata->channel_reg_size); res = platform_get_resource(pdev, IORESOURCE_IRQ, i); if (!res) { @@ -1418,6 +1430,7 @@ static int tegra_dma_probe(struct platform_device *pdev) dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask); dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask); + tdma->global_pause_count = 0; tdma->dma_dev.dev = &pdev->dev; tdma->dma_dev.device_alloc_chan_resources = tegra_dma_alloc_chan_resources; diff --git a/drivers/dma/ti-dma-crossbar.c b/drivers/dma/ti-dma-crossbar.c index 24f5ca2356bf..5cce8c9d0026 100644 --- a/drivers/dma/ti-dma-crossbar.c +++ b/drivers/dma/ti-dma-crossbar.c @@ -20,16 +20,19 @@ #define TI_XBAR_OUTPUTS 127 #define TI_XBAR_INPUTS 256 -static DEFINE_IDR(map_idr); +#define TI_XBAR_EDMA_OFFSET 0 +#define TI_XBAR_SDMA_OFFSET 1 struct ti_dma_xbar_data { void __iomem *iomem; struct dma_router dmarouter; + struct idr map_idr; u16 safe_val; /* Value to rest the crossbar lines */ u32 xbar_requests; /* number of DMA requests connected to XBAR */ u32 dma_requests; /* number of DMA requests forwarded to DMA */ + u32 dma_offset; }; struct ti_dma_xbar_map { @@ -51,7 +54,7 @@ static void ti_dma_xbar_free(struct device *dev, void *route_data) map->xbar_in, map->xbar_out); ti_dma_xbar_write(xbar->iomem, map->xbar_out, xbar->safe_val); - idr_remove(&map_idr, map->xbar_out); + idr_remove(&xbar->map_idr, map->xbar_out); kfree(map); } @@ -81,12 +84,11 @@ static void *ti_dma_xbar_route_allocate(struct of_phandle_args *dma_spec, return ERR_PTR(-ENOMEM); } - map->xbar_out = idr_alloc(&map_idr, NULL, 0, xbar->dma_requests, + map->xbar_out = idr_alloc(&xbar->map_idr, NULL, 0, xbar->dma_requests, GFP_KERNEL); map->xbar_in = (u16)dma_spec->args[0]; - /* The DMA request is 1 based in sDMA */ - dma_spec->args[0] = map->xbar_out + 1; + dma_spec->args[0] = map->xbar_out + xbar->dma_offset; dev_dbg(&pdev->dev, "Mapping XBAR%u to DMA%d\n", map->xbar_in, map->xbar_out); @@ -96,9 +98,22 @@ static void *ti_dma_xbar_route_allocate(struct of_phandle_args *dma_spec, return map; } +static const struct of_device_id ti_dma_master_match[] = { + { + .compatible = "ti,omap4430-sdma", + .data = (void *)TI_XBAR_SDMA_OFFSET, + }, + { + .compatible = "ti,edma3", + .data = (void *)TI_XBAR_EDMA_OFFSET, + }, + {}, +}; + static int ti_dma_xbar_probe(struct platform_device *pdev) { struct device_node *node = pdev->dev.of_node; + const struct of_device_id *match; struct device_node *dma_node; struct ti_dma_xbar_data *xbar; struct resource *res; @@ -113,12 +128,20 @@ static int ti_dma_xbar_probe(struct platform_device *pdev) if (!xbar) return -ENOMEM; + idr_init(&xbar->map_idr); + dma_node = of_parse_phandle(node, "dma-masters", 0); if (!dma_node) { dev_err(&pdev->dev, "Can't get DMA master node\n"); return -ENODEV; } + match = of_match_node(ti_dma_master_match, dma_node); + if (!match) { + dev_err(&pdev->dev, "DMA master is not supported\n"); + return -EINVAL; + } + if (of_property_read_u32(dma_node, "dma-requests", &xbar->dma_requests)) { dev_info(&pdev->dev, @@ -139,17 +162,15 @@ static int ti_dma_xbar_probe(struct platform_device *pdev) xbar->safe_val = (u16)safe_val; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) - return -ENODEV; - iomem = devm_ioremap_resource(&pdev->dev, res); - if (!iomem) - return -ENOMEM; + if (IS_ERR(iomem)) + return PTR_ERR(iomem); xbar->iomem = iomem; xbar->dmarouter.dev = &pdev->dev; xbar->dmarouter.route_free = ti_dma_xbar_free; + xbar->dma_offset = (u32)match->data; platform_set_drvdata(pdev, xbar); diff --git a/drivers/dma/timb_dma.c b/drivers/dma/timb_dma.c index c4c3d93fdd1b..559cd4073698 100644 --- a/drivers/dma/timb_dma.c +++ b/drivers/dma/timb_dma.c @@ -10,10 +10,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ /* Supports: diff --git a/drivers/dma/xgene-dma.c b/drivers/dma/xgene-dma.c index dff22ab01851..b23e8d52d126 100644 --- a/drivers/dma/xgene-dma.c +++ b/drivers/dma/xgene-dma.c @@ -21,6 +21,7 @@ * NOTE: PM support is currently not available. */ +#include #include #include #include @@ -151,7 +152,6 @@ #define XGENE_DMA_PQ_CHANNEL 1 #define XGENE_DMA_MAX_BYTE_CNT 0x4000 /* 16 KB */ #define XGENE_DMA_MAX_64B_DESC_BYTE_CNT 0x14000 /* 80 KB */ -#define XGENE_DMA_XOR_ALIGNMENT 6 /* 64 Bytes */ #define XGENE_DMA_MAX_XOR_SRC 5 #define XGENE_DMA_16K_BUFFER_LEN_CODE 0x0 #define XGENE_DMA_INVALID_LEN_CODE 0x7800000000000000ULL @@ -764,12 +764,17 @@ static void xgene_dma_cleanup_descriptors(struct xgene_dma_chan *chan) struct xgene_dma_ring *ring = &chan->rx_ring; struct xgene_dma_desc_sw *desc_sw, *_desc_sw; struct xgene_dma_desc_hw *desc_hw; + struct list_head ld_completed; u8 status; + INIT_LIST_HEAD(&ld_completed); + + spin_lock_bh(&chan->lock); + /* Clean already completed and acked descriptors */ xgene_dma_clean_completed_descriptor(chan); - /* Run the callback for each descriptor, in order */ + /* Move all completed descriptors to ld completed queue, in order */ list_for_each_entry_safe(desc_sw, _desc_sw, &chan->ld_running, node) { /* Get subsequent hw descriptor from DMA rx ring */ desc_hw = &ring->desc_hw[ring->head]; @@ -812,15 +817,17 @@ static void xgene_dma_cleanup_descriptors(struct xgene_dma_chan *chan) /* Mark this hw descriptor as processed */ desc_hw->m0 = cpu_to_le64(XGENE_DMA_DESC_EMPTY_SIGNATURE); - xgene_dma_run_tx_complete_actions(chan, desc_sw); - - xgene_dma_clean_running_descriptor(chan, desc_sw); - /* * Decrement the pending transaction count * as we have processed one */ chan->pending--; + + /* + * Delete this node from ld running queue and append it to + * ld completed queue for further processing + */ + list_move_tail(&desc_sw->node, &ld_completed); } /* @@ -829,6 +836,14 @@ static void xgene_dma_cleanup_descriptors(struct xgene_dma_chan *chan) * ahead and free the descriptors below. */ xgene_chan_xfer_ld_pending(chan); + + spin_unlock_bh(&chan->lock); + + /* Run the callback for each descriptor, in order */ + list_for_each_entry_safe(desc_sw, _desc_sw, &ld_completed, node) { + xgene_dma_run_tx_complete_actions(chan, desc_sw); + xgene_dma_clean_running_descriptor(chan, desc_sw); + } } static int xgene_dma_alloc_chan_resources(struct dma_chan *dchan) @@ -877,11 +892,11 @@ static void xgene_dma_free_chan_resources(struct dma_chan *dchan) if (!chan->desc_pool) return; - spin_lock_bh(&chan->lock); - /* Process all running descriptor */ xgene_dma_cleanup_descriptors(chan); + spin_lock_bh(&chan->lock); + /* Clean all link descriptor queues */ xgene_dma_free_desc_list(chan, &chan->ld_pending); xgene_dma_free_desc_list(chan, &chan->ld_running); @@ -1201,15 +1216,11 @@ static void xgene_dma_tasklet_cb(unsigned long data) { struct xgene_dma_chan *chan = (struct xgene_dma_chan *)data; - spin_lock_bh(&chan->lock); - /* Run all cleanup for descriptors which have been completed */ xgene_dma_cleanup_descriptors(chan); /* Re-enable DMA channel IRQ */ enable_irq(chan->rx_irq); - - spin_unlock_bh(&chan->lock); } static irqreturn_t xgene_dma_chan_ring_isr(int irq, void *id) @@ -1741,13 +1752,13 @@ static void xgene_dma_set_caps(struct xgene_dma_chan *chan, if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { dma_dev->device_prep_dma_xor = xgene_dma_prep_xor; dma_dev->max_xor = XGENE_DMA_MAX_XOR_SRC; - dma_dev->xor_align = XGENE_DMA_XOR_ALIGNMENT; + dma_dev->xor_align = DMAENGINE_ALIGN_64_BYTES; } if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { dma_dev->device_prep_dma_pq = xgene_dma_prep_pq; dma_dev->max_pq = XGENE_DMA_MAX_XOR_SRC; - dma_dev->pq_align = XGENE_DMA_XOR_ALIGNMENT; + dma_dev->pq_align = DMAENGINE_ALIGN_64_BYTES; } } @@ -1944,16 +1955,18 @@ static int xgene_dma_probe(struct platform_device *pdev) return ret; pdma->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(pdma->clk)) { + if (IS_ERR(pdma->clk) && !ACPI_COMPANION(&pdev->dev)) { dev_err(&pdev->dev, "Failed to get clk\n"); return PTR_ERR(pdma->clk); } /* Enable clk before accessing registers */ - ret = clk_prepare_enable(pdma->clk); - if (ret) { - dev_err(&pdev->dev, "Failed to enable clk %d\n", ret); - return ret; + if (!IS_ERR(pdma->clk)) { + ret = clk_prepare_enable(pdma->clk); + if (ret) { + dev_err(&pdev->dev, "Failed to enable clk %d\n", ret); + return ret; + } } /* Remove DMA RAM out of shutdown */ @@ -1998,7 +2011,8 @@ static int xgene_dma_probe(struct platform_device *pdev) err_dma_mask: err_clk_enable: - clk_disable_unprepare(pdma->clk); + if (!IS_ERR(pdma->clk)) + clk_disable_unprepare(pdma->clk); return ret; } @@ -2022,11 +2036,20 @@ static int xgene_dma_remove(struct platform_device *pdev) xgene_dma_delete_chan_rings(chan); } - clk_disable_unprepare(pdma->clk); + if (!IS_ERR(pdma->clk)) + clk_disable_unprepare(pdma->clk); return 0; } +#ifdef CONFIG_ACPI +static const struct acpi_device_id xgene_dma_acpi_match_ptr[] = { + {"APMC0D43", 0}, + {}, +}; +MODULE_DEVICE_TABLE(acpi, xgene_dma_acpi_match_ptr); +#endif + static const struct of_device_id xgene_dma_of_match_ptr[] = { {.compatible = "apm,xgene-storm-dma",}, {}, @@ -2039,6 +2062,7 @@ static struct platform_driver xgene_dma_driver = { .driver = { .name = "X-Gene-DMA", .of_match_table = xgene_dma_of_match_ptr, + .acpi_match_table = ACPI_PTR(xgene_dma_acpi_match_ptr), }, }; diff --git a/drivers/dma/zx296702_dma.c b/drivers/dma/zx296702_dma.c new file mode 100644 index 000000000000..39915a6b7986 --- /dev/null +++ b/drivers/dma/zx296702_dma.c @@ -0,0 +1,951 @@ +/* + * Copyright 2015 Linaro. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "virt-dma.h" + +#define DRIVER_NAME "zx-dma" +#define DMA_ALIGN 4 +#define DMA_MAX_SIZE (0x10000 - PAGE_SIZE) +#define LLI_BLOCK_SIZE (4 * PAGE_SIZE) + +#define REG_ZX_SRC_ADDR 0x00 +#define REG_ZX_DST_ADDR 0x04 +#define REG_ZX_TX_X_COUNT 0x08 +#define REG_ZX_TX_ZY_COUNT 0x0c +#define REG_ZX_SRC_ZY_STEP 0x10 +#define REG_ZX_DST_ZY_STEP 0x14 +#define REG_ZX_LLI_ADDR 0x1c +#define REG_ZX_CTRL 0x20 +#define REG_ZX_TC_IRQ 0x800 +#define REG_ZX_SRC_ERR_IRQ 0x804 +#define REG_ZX_DST_ERR_IRQ 0x808 +#define REG_ZX_CFG_ERR_IRQ 0x80c +#define REG_ZX_TC_IRQ_RAW 0x810 +#define REG_ZX_SRC_ERR_IRQ_RAW 0x814 +#define REG_ZX_DST_ERR_IRQ_RAW 0x818 +#define REG_ZX_CFG_ERR_IRQ_RAW 0x81c +#define REG_ZX_STATUS 0x820 +#define REG_ZX_DMA_GRP_PRIO 0x824 +#define REG_ZX_DMA_ARB 0x828 + +#define ZX_FORCE_CLOSE BIT(31) +#define ZX_DST_BURST_WIDTH(x) (((x) & 0x7) << 13) +#define ZX_MAX_BURST_LEN 16 +#define ZX_SRC_BURST_LEN(x) (((x) & 0xf) << 9) +#define ZX_SRC_BURST_WIDTH(x) (((x) & 0x7) << 6) +#define ZX_IRQ_ENABLE_ALL (3 << 4) +#define ZX_DST_FIFO_MODE BIT(3) +#define ZX_SRC_FIFO_MODE BIT(2) +#define ZX_SOFT_REQ BIT(1) +#define ZX_CH_ENABLE BIT(0) + +#define ZX_DMA_BUSWIDTHS \ + (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \ + BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) + +enum zx_dma_burst_width { + ZX_DMA_WIDTH_8BIT = 0, + ZX_DMA_WIDTH_16BIT = 1, + ZX_DMA_WIDTH_32BIT = 2, + ZX_DMA_WIDTH_64BIT = 3, +}; + +struct zx_desc_hw { + u32 saddr; + u32 daddr; + u32 src_x; + u32 src_zy; + u32 src_zy_step; + u32 dst_zy_step; + u32 reserved1; + u32 lli; + u32 ctr; + u32 reserved[7]; /* pack as hardware registers region size */ +} __aligned(32); + +struct zx_dma_desc_sw { + struct virt_dma_desc vd; + dma_addr_t desc_hw_lli; + size_t desc_num; + size_t size; + struct zx_desc_hw *desc_hw; +}; + +struct zx_dma_phy; + +struct zx_dma_chan { + struct dma_slave_config slave_cfg; + int id; /* Request phy chan id */ + u32 ccfg; + u32 cyclic; + struct virt_dma_chan vc; + struct zx_dma_phy *phy; + struct list_head node; + dma_addr_t dev_addr; + enum dma_status status; +}; + +struct zx_dma_phy { + u32 idx; + void __iomem *base; + struct zx_dma_chan *vchan; + struct zx_dma_desc_sw *ds_run; + struct zx_dma_desc_sw *ds_done; +}; + +struct zx_dma_dev { + struct dma_device slave; + void __iomem *base; + spinlock_t lock; /* lock for ch and phy */ + struct list_head chan_pending; + struct zx_dma_phy *phy; + struct zx_dma_chan *chans; + struct clk *clk; + struct dma_pool *pool; + u32 dma_channels; + u32 dma_requests; + int irq; +}; + +#define to_zx_dma(dmadev) container_of(dmadev, struct zx_dma_dev, slave) + +static struct zx_dma_chan *to_zx_chan(struct dma_chan *chan) +{ + return container_of(chan, struct zx_dma_chan, vc.chan); +} + +static void zx_dma_terminate_chan(struct zx_dma_phy *phy, struct zx_dma_dev *d) +{ + u32 val = 0; + + val = readl_relaxed(phy->base + REG_ZX_CTRL); + val &= ~ZX_CH_ENABLE; + val |= ZX_FORCE_CLOSE; + writel_relaxed(val, phy->base + REG_ZX_CTRL); + + val = 0x1 << phy->idx; + writel_relaxed(val, d->base + REG_ZX_TC_IRQ_RAW); + writel_relaxed(val, d->base + REG_ZX_SRC_ERR_IRQ_RAW); + writel_relaxed(val, d->base + REG_ZX_DST_ERR_IRQ_RAW); + writel_relaxed(val, d->base + REG_ZX_CFG_ERR_IRQ_RAW); +} + +static void zx_dma_set_desc(struct zx_dma_phy *phy, struct zx_desc_hw *hw) +{ + writel_relaxed(hw->saddr, phy->base + REG_ZX_SRC_ADDR); + writel_relaxed(hw->daddr, phy->base + REG_ZX_DST_ADDR); + writel_relaxed(hw->src_x, phy->base + REG_ZX_TX_X_COUNT); + writel_relaxed(0, phy->base + REG_ZX_TX_ZY_COUNT); + writel_relaxed(0, phy->base + REG_ZX_SRC_ZY_STEP); + writel_relaxed(0, phy->base + REG_ZX_DST_ZY_STEP); + writel_relaxed(hw->lli, phy->base + REG_ZX_LLI_ADDR); + writel_relaxed(hw->ctr, phy->base + REG_ZX_CTRL); +} + +static u32 zx_dma_get_curr_lli(struct zx_dma_phy *phy) +{ + return readl_relaxed(phy->base + REG_ZX_LLI_ADDR); +} + +static u32 zx_dma_get_chan_stat(struct zx_dma_dev *d) +{ + return readl_relaxed(d->base + REG_ZX_STATUS); +} + +static void zx_dma_init_state(struct zx_dma_dev *d) +{ + /* set same priority */ + writel_relaxed(0x0, d->base + REG_ZX_DMA_ARB); + /* clear all irq */ + writel_relaxed(0xffffffff, d->base + REG_ZX_TC_IRQ_RAW); + writel_relaxed(0xffffffff, d->base + REG_ZX_SRC_ERR_IRQ_RAW); + writel_relaxed(0xffffffff, d->base + REG_ZX_DST_ERR_IRQ_RAW); + writel_relaxed(0xffffffff, d->base + REG_ZX_CFG_ERR_IRQ_RAW); +} + +static int zx_dma_start_txd(struct zx_dma_chan *c) +{ + struct zx_dma_dev *d = to_zx_dma(c->vc.chan.device); + struct virt_dma_desc *vd = vchan_next_desc(&c->vc); + + if (!c->phy) + return -EAGAIN; + + if (BIT(c->phy->idx) & zx_dma_get_chan_stat(d)) + return -EAGAIN; + + if (vd) { + struct zx_dma_desc_sw *ds = + container_of(vd, struct zx_dma_desc_sw, vd); + /* + * fetch and remove request from vc->desc_issued + * so vc->desc_issued only contains desc pending + */ + list_del(&ds->vd.node); + c->phy->ds_run = ds; + c->phy->ds_done = NULL; + /* start dma */ + zx_dma_set_desc(c->phy, ds->desc_hw); + return 0; + } + c->phy->ds_done = NULL; + c->phy->ds_run = NULL; + return -EAGAIN; +} + +static void zx_dma_task(struct zx_dma_dev *d) +{ + struct zx_dma_phy *p; + struct zx_dma_chan *c, *cn; + unsigned pch, pch_alloc = 0; + unsigned long flags; + + /* check new dma request of running channel in vc->desc_issued */ + list_for_each_entry_safe(c, cn, &d->slave.channels, + vc.chan.device_node) { + spin_lock_irqsave(&c->vc.lock, flags); + p = c->phy; + if (p && p->ds_done && zx_dma_start_txd(c)) { + /* No current txd associated with this channel */ + dev_dbg(d->slave.dev, "pchan %u: free\n", p->idx); + /* Mark this channel free */ + c->phy = NULL; + p->vchan = NULL; + } + spin_unlock_irqrestore(&c->vc.lock, flags); + } + + /* check new channel request in d->chan_pending */ + spin_lock_irqsave(&d->lock, flags); + while (!list_empty(&d->chan_pending)) { + c = list_first_entry(&d->chan_pending, + struct zx_dma_chan, node); + p = &d->phy[c->id]; + if (!p->vchan) { + /* remove from d->chan_pending */ + list_del_init(&c->node); + pch_alloc |= 1 << c->id; + /* Mark this channel allocated */ + p->vchan = c; + c->phy = p; + } else { + dev_dbg(d->slave.dev, "pchan %u: busy!\n", c->id); + } + } + spin_unlock_irqrestore(&d->lock, flags); + + for (pch = 0; pch < d->dma_channels; pch++) { + if (pch_alloc & (1 << pch)) { + p = &d->phy[pch]; + c = p->vchan; + if (c) { + spin_lock_irqsave(&c->vc.lock, flags); + zx_dma_start_txd(c); + spin_unlock_irqrestore(&c->vc.lock, flags); + } + } + } +} + +static irqreturn_t zx_dma_int_handler(int irq, void *dev_id) +{ + struct zx_dma_dev *d = (struct zx_dma_dev *)dev_id; + struct zx_dma_phy *p; + struct zx_dma_chan *c; + u32 tc = readl_relaxed(d->base + REG_ZX_TC_IRQ); + u32 serr = readl_relaxed(d->base + REG_ZX_SRC_ERR_IRQ); + u32 derr = readl_relaxed(d->base + REG_ZX_DST_ERR_IRQ); + u32 cfg = readl_relaxed(d->base + REG_ZX_CFG_ERR_IRQ); + u32 i, irq_chan = 0, task = 0; + + while (tc) { + i = __ffs(tc); + tc &= ~BIT(i); + p = &d->phy[i]; + c = p->vchan; + if (c) { + unsigned long flags; + + spin_lock_irqsave(&c->vc.lock, flags); + if (c->cyclic) { + vchan_cyclic_callback(&p->ds_run->vd); + } else { + vchan_cookie_complete(&p->ds_run->vd); + p->ds_done = p->ds_run; + task = 1; + } + spin_unlock_irqrestore(&c->vc.lock, flags); + irq_chan |= BIT(i); + } + } + + if (serr || derr || cfg) + dev_warn(d->slave.dev, "DMA ERR src 0x%x, dst 0x%x, cfg 0x%x\n", + serr, derr, cfg); + + writel_relaxed(irq_chan, d->base + REG_ZX_TC_IRQ_RAW); + writel_relaxed(serr, d->base + REG_ZX_SRC_ERR_IRQ_RAW); + writel_relaxed(derr, d->base + REG_ZX_DST_ERR_IRQ_RAW); + writel_relaxed(cfg, d->base + REG_ZX_CFG_ERR_IRQ_RAW); + + if (task) + zx_dma_task(d); + return IRQ_HANDLED; +} + +static void zx_dma_free_chan_resources(struct dma_chan *chan) +{ + struct zx_dma_chan *c = to_zx_chan(chan); + struct zx_dma_dev *d = to_zx_dma(chan->device); + unsigned long flags; + + spin_lock_irqsave(&d->lock, flags); + list_del_init(&c->node); + spin_unlock_irqrestore(&d->lock, flags); + + vchan_free_chan_resources(&c->vc); + c->ccfg = 0; +} + +static enum dma_status zx_dma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, + struct dma_tx_state *state) +{ + struct zx_dma_chan *c = to_zx_chan(chan); + struct zx_dma_phy *p; + struct virt_dma_desc *vd; + unsigned long flags; + enum dma_status ret; + size_t bytes = 0; + + ret = dma_cookie_status(&c->vc.chan, cookie, state); + if (ret == DMA_COMPLETE || !state) + return ret; + + spin_lock_irqsave(&c->vc.lock, flags); + p = c->phy; + ret = c->status; + + /* + * If the cookie is on our issue queue, then the residue is + * its total size. + */ + vd = vchan_find_desc(&c->vc, cookie); + if (vd) { + bytes = container_of(vd, struct zx_dma_desc_sw, vd)->size; + } else if ((!p) || (!p->ds_run)) { + bytes = 0; + } else { + struct zx_dma_desc_sw *ds = p->ds_run; + u32 clli = 0, index = 0; + + bytes = 0; + clli = zx_dma_get_curr_lli(p); + index = (clli - ds->desc_hw_lli) / sizeof(struct zx_desc_hw); + for (; index < ds->desc_num; index++) { + bytes += ds->desc_hw[index].src_x; + /* end of lli */ + if (!ds->desc_hw[index].lli) + break; + } + } + spin_unlock_irqrestore(&c->vc.lock, flags); + dma_set_residue(state, bytes); + return ret; +} + +static void zx_dma_issue_pending(struct dma_chan *chan) +{ + struct zx_dma_chan *c = to_zx_chan(chan); + struct zx_dma_dev *d = to_zx_dma(chan->device); + unsigned long flags; + int issue = 0; + + spin_lock_irqsave(&c->vc.lock, flags); + /* add request to vc->desc_issued */ + if (vchan_issue_pending(&c->vc)) { + spin_lock(&d->lock); + if (!c->phy && list_empty(&c->node)) { + /* if new channel, add chan_pending */ + list_add_tail(&c->node, &d->chan_pending); + issue = 1; + dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc); + } + spin_unlock(&d->lock); + } else { + dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc); + } + spin_unlock_irqrestore(&c->vc.lock, flags); + + if (issue) + zx_dma_task(d); +} + +static void zx_dma_fill_desc(struct zx_dma_desc_sw *ds, dma_addr_t dst, + dma_addr_t src, size_t len, u32 num, u32 ccfg) +{ + if ((num + 1) < ds->desc_num) + ds->desc_hw[num].lli = ds->desc_hw_lli + (num + 1) * + sizeof(struct zx_desc_hw); + ds->desc_hw[num].saddr = src; + ds->desc_hw[num].daddr = dst; + ds->desc_hw[num].src_x = len; + ds->desc_hw[num].ctr = ccfg; +} + +static struct zx_dma_desc_sw *zx_alloc_desc_resource(int num, + struct dma_chan *chan) +{ + struct zx_dma_chan *c = to_zx_chan(chan); + struct zx_dma_desc_sw *ds; + struct zx_dma_dev *d = to_zx_dma(chan->device); + int lli_limit = LLI_BLOCK_SIZE / sizeof(struct zx_desc_hw); + + if (num > lli_limit) { + dev_dbg(chan->device->dev, "vch %p: sg num %d exceed max %d\n", + &c->vc, num, lli_limit); + return NULL; + } + + ds = kzalloc(sizeof(*ds), GFP_ATOMIC); + if (!ds) + return NULL; + + ds->desc_hw = dma_pool_alloc(d->pool, GFP_NOWAIT, &ds->desc_hw_lli); + if (!ds->desc_hw) { + dev_dbg(chan->device->dev, "vch %p: dma alloc fail\n", &c->vc); + kfree(ds); + return NULL; + } + memset(ds->desc_hw, sizeof(struct zx_desc_hw) * num, 0); + ds->desc_num = num; + return ds; +} + +static enum zx_dma_burst_width zx_dma_burst_width(enum dma_slave_buswidth width) +{ + switch (width) { + case DMA_SLAVE_BUSWIDTH_1_BYTE: + case DMA_SLAVE_BUSWIDTH_2_BYTES: + case DMA_SLAVE_BUSWIDTH_4_BYTES: + case DMA_SLAVE_BUSWIDTH_8_BYTES: + return ffs(width) - 1; + default: + return ZX_DMA_WIDTH_32BIT; + } +} + +static int zx_pre_config(struct zx_dma_chan *c, enum dma_transfer_direction dir) +{ + struct dma_slave_config *cfg = &c->slave_cfg; + enum zx_dma_burst_width src_width; + enum zx_dma_burst_width dst_width; + u32 maxburst = 0; + + switch (dir) { + case DMA_MEM_TO_MEM: + c->ccfg = ZX_CH_ENABLE | ZX_SOFT_REQ + | ZX_SRC_BURST_LEN(ZX_MAX_BURST_LEN - 1) + | ZX_SRC_BURST_WIDTH(ZX_DMA_WIDTH_32BIT) + | ZX_DST_BURST_WIDTH(ZX_DMA_WIDTH_32BIT); + break; + case DMA_MEM_TO_DEV: + c->dev_addr = cfg->dst_addr; + /* dst len is calculated from src width, len and dst width. + * We need make sure dst len not exceed MAX LEN. + * Trailing single transaction that does not fill a full + * burst also require identical src/dst data width. + */ + dst_width = zx_dma_burst_width(cfg->dst_addr_width); + maxburst = cfg->dst_maxburst; + maxburst = maxburst < ZX_MAX_BURST_LEN ? + maxburst : ZX_MAX_BURST_LEN; + c->ccfg = ZX_DST_FIFO_MODE | ZX_CH_ENABLE + | ZX_SRC_BURST_LEN(maxburst - 1) + | ZX_SRC_BURST_WIDTH(dst_width) + | ZX_DST_BURST_WIDTH(dst_width); + break; + case DMA_DEV_TO_MEM: + c->dev_addr = cfg->src_addr; + src_width = zx_dma_burst_width(cfg->src_addr_width); + maxburst = cfg->src_maxburst; + maxburst = maxburst < ZX_MAX_BURST_LEN ? + maxburst : ZX_MAX_BURST_LEN; + c->ccfg = ZX_SRC_FIFO_MODE | ZX_CH_ENABLE + | ZX_SRC_BURST_LEN(maxburst - 1) + | ZX_SRC_BURST_WIDTH(src_width) + | ZX_DST_BURST_WIDTH(src_width); + break; + default: + return -EINVAL; + } + return 0; +} + +static struct dma_async_tx_descriptor *zx_dma_prep_memcpy( + struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, + size_t len, unsigned long flags) +{ + struct zx_dma_chan *c = to_zx_chan(chan); + struct zx_dma_desc_sw *ds; + size_t copy = 0; + int num = 0; + + if (!len) + return NULL; + + if (zx_pre_config(c, DMA_MEM_TO_MEM)) + return NULL; + + num = DIV_ROUND_UP(len, DMA_MAX_SIZE); + + ds = zx_alloc_desc_resource(num, chan); + if (!ds) + return NULL; + + ds->size = len; + num = 0; + + do { + copy = min_t(size_t, len, DMA_MAX_SIZE); + zx_dma_fill_desc(ds, dst, src, copy, num++, c->ccfg); + + src += copy; + dst += copy; + len -= copy; + } while (len); + + c->cyclic = 0; + ds->desc_hw[num - 1].lli = 0; /* end of link */ + ds->desc_hw[num - 1].ctr |= ZX_IRQ_ENABLE_ALL; + return vchan_tx_prep(&c->vc, &ds->vd, flags); +} + +static struct dma_async_tx_descriptor *zx_dma_prep_slave_sg( + struct dma_chan *chan, struct scatterlist *sgl, unsigned int sglen, + enum dma_transfer_direction dir, unsigned long flags, void *context) +{ + struct zx_dma_chan *c = to_zx_chan(chan); + struct zx_dma_desc_sw *ds; + size_t len, avail, total = 0; + struct scatterlist *sg; + dma_addr_t addr, src = 0, dst = 0; + int num = sglen, i; + + if (!sgl) + return NULL; + + if (zx_pre_config(c, dir)) + return NULL; + + for_each_sg(sgl, sg, sglen, i) { + avail = sg_dma_len(sg); + if (avail > DMA_MAX_SIZE) + num += DIV_ROUND_UP(avail, DMA_MAX_SIZE) - 1; + } + + ds = zx_alloc_desc_resource(num, chan); + if (!ds) + return NULL; + + c->cyclic = 0; + num = 0; + for_each_sg(sgl, sg, sglen, i) { + addr = sg_dma_address(sg); + avail = sg_dma_len(sg); + total += avail; + + do { + len = min_t(size_t, avail, DMA_MAX_SIZE); + + if (dir == DMA_MEM_TO_DEV) { + src = addr; + dst = c->dev_addr; + } else if (dir == DMA_DEV_TO_MEM) { + src = c->dev_addr; + dst = addr; + } + + zx_dma_fill_desc(ds, dst, src, len, num++, c->ccfg); + + addr += len; + avail -= len; + } while (avail); + } + + ds->desc_hw[num - 1].lli = 0; /* end of link */ + ds->desc_hw[num - 1].ctr |= ZX_IRQ_ENABLE_ALL; + ds->size = total; + return vchan_tx_prep(&c->vc, &ds->vd, flags); +} + +static struct dma_async_tx_descriptor *zx_dma_prep_dma_cyclic( + struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction dir, + unsigned long flags) +{ + struct zx_dma_chan *c = to_zx_chan(chan); + struct zx_dma_desc_sw *ds; + dma_addr_t src = 0, dst = 0; + int num_periods = buf_len / period_len; + int buf = 0, num = 0; + + if (period_len > DMA_MAX_SIZE) { + dev_err(chan->device->dev, "maximum period size exceeded\n"); + return NULL; + } + + if (zx_pre_config(c, dir)) + return NULL; + + ds = zx_alloc_desc_resource(num_periods, chan); + if (!ds) + return NULL; + c->cyclic = 1; + + while (buf < buf_len) { + if (dir == DMA_MEM_TO_DEV) { + src = dma_addr; + dst = c->dev_addr; + } else if (dir == DMA_DEV_TO_MEM) { + src = c->dev_addr; + dst = dma_addr; + } + zx_dma_fill_desc(ds, dst, src, period_len, num++, + c->ccfg | ZX_IRQ_ENABLE_ALL); + dma_addr += period_len; + buf += period_len; + } + + ds->desc_hw[num - 1].lli = ds->desc_hw_lli; + ds->size = buf_len; + return vchan_tx_prep(&c->vc, &ds->vd, flags); +} + +static int zx_dma_config(struct dma_chan *chan, + struct dma_slave_config *cfg) +{ + struct zx_dma_chan *c = to_zx_chan(chan); + + if (!cfg) + return -EINVAL; + + memcpy(&c->slave_cfg, cfg, sizeof(*cfg)); + + return 0; +} + +static int zx_dma_terminate_all(struct dma_chan *chan) +{ + struct zx_dma_chan *c = to_zx_chan(chan); + struct zx_dma_dev *d = to_zx_dma(chan->device); + struct zx_dma_phy *p = c->phy; + unsigned long flags; + LIST_HEAD(head); + + dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc); + + /* Prevent this channel being scheduled */ + spin_lock(&d->lock); + list_del_init(&c->node); + spin_unlock(&d->lock); + + /* Clear the tx descriptor lists */ + spin_lock_irqsave(&c->vc.lock, flags); + vchan_get_all_descriptors(&c->vc, &head); + if (p) { + /* vchan is assigned to a pchan - stop the channel */ + zx_dma_terminate_chan(p, d); + c->phy = NULL; + p->vchan = NULL; + p->ds_run = NULL; + p->ds_done = NULL; + } + spin_unlock_irqrestore(&c->vc.lock, flags); + vchan_dma_desc_free_list(&c->vc, &head); + + return 0; +} + +static int zx_dma_transfer_pause(struct dma_chan *chan) +{ + struct zx_dma_chan *c = to_zx_chan(chan); + u32 val = 0; + + val = readl_relaxed(c->phy->base + REG_ZX_CTRL); + val &= ~ZX_CH_ENABLE; + writel_relaxed(val, c->phy->base + REG_ZX_CTRL); + + return 0; +} + +static int zx_dma_transfer_resume(struct dma_chan *chan) +{ + struct zx_dma_chan *c = to_zx_chan(chan); + u32 val = 0; + + val = readl_relaxed(c->phy->base + REG_ZX_CTRL); + val |= ZX_CH_ENABLE; + writel_relaxed(val, c->phy->base + REG_ZX_CTRL); + + return 0; +} + +static void zx_dma_free_desc(struct virt_dma_desc *vd) +{ + struct zx_dma_desc_sw *ds = + container_of(vd, struct zx_dma_desc_sw, vd); + struct zx_dma_dev *d = to_zx_dma(vd->tx.chan->device); + + dma_pool_free(d->pool, ds->desc_hw, ds->desc_hw_lli); + kfree(ds); +} + +static const struct of_device_id zx6702_dma_dt_ids[] = { + { .compatible = "zte,zx296702-dma", }, + {} +}; +MODULE_DEVICE_TABLE(of, zx6702_dma_dt_ids); + +static struct dma_chan *zx_of_dma_simple_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct zx_dma_dev *d = ofdma->of_dma_data; + unsigned int request = dma_spec->args[0]; + struct dma_chan *chan; + struct zx_dma_chan *c; + + if (request > d->dma_requests) + return NULL; + + chan = dma_get_any_slave_channel(&d->slave); + if (!chan) { + dev_err(d->slave.dev, "get channel fail in %s.\n", __func__); + return NULL; + } + c = to_zx_chan(chan); + c->id = request; + dev_info(d->slave.dev, "zx_dma: pchan %u: alloc vchan %p\n", + c->id, &c->vc); + return chan; +} + +static int zx_dma_probe(struct platform_device *op) +{ + struct zx_dma_dev *d; + struct resource *iores; + int i, ret = 0; + + iores = platform_get_resource(op, IORESOURCE_MEM, 0); + if (!iores) + return -EINVAL; + + d = devm_kzalloc(&op->dev, sizeof(*d), GFP_KERNEL); + if (!d) + return -ENOMEM; + + d->base = devm_ioremap_resource(&op->dev, iores); + if (IS_ERR(d->base)) + return PTR_ERR(d->base); + + of_property_read_u32((&op->dev)->of_node, + "dma-channels", &d->dma_channels); + of_property_read_u32((&op->dev)->of_node, + "dma-requests", &d->dma_requests); + if (!d->dma_requests || !d->dma_channels) + return -EINVAL; + + d->clk = devm_clk_get(&op->dev, NULL); + if (IS_ERR(d->clk)) { + dev_err(&op->dev, "no dma clk\n"); + return PTR_ERR(d->clk); + } + + d->irq = platform_get_irq(op, 0); + ret = devm_request_irq(&op->dev, d->irq, zx_dma_int_handler, + 0, DRIVER_NAME, d); + if (ret) + return ret; + + /* A DMA memory pool for LLIs, align on 32-byte boundary */ + d->pool = dmam_pool_create(DRIVER_NAME, &op->dev, + LLI_BLOCK_SIZE, 32, 0); + if (!d->pool) + return -ENOMEM; + + /* init phy channel */ + d->phy = devm_kzalloc(&op->dev, + d->dma_channels * sizeof(struct zx_dma_phy), GFP_KERNEL); + if (!d->phy) + return -ENOMEM; + + for (i = 0; i < d->dma_channels; i++) { + struct zx_dma_phy *p = &d->phy[i]; + + p->idx = i; + p->base = d->base + i * 0x40; + } + + INIT_LIST_HEAD(&d->slave.channels); + dma_cap_set(DMA_SLAVE, d->slave.cap_mask); + dma_cap_set(DMA_MEMCPY, d->slave.cap_mask); + dma_cap_set(DMA_PRIVATE, d->slave.cap_mask); + d->slave.dev = &op->dev; + d->slave.device_free_chan_resources = zx_dma_free_chan_resources; + d->slave.device_tx_status = zx_dma_tx_status; + d->slave.device_prep_dma_memcpy = zx_dma_prep_memcpy; + d->slave.device_prep_slave_sg = zx_dma_prep_slave_sg; + d->slave.device_prep_dma_cyclic = zx_dma_prep_dma_cyclic; + d->slave.device_issue_pending = zx_dma_issue_pending; + d->slave.device_config = zx_dma_config; + d->slave.device_terminate_all = zx_dma_terminate_all; + d->slave.device_pause = zx_dma_transfer_pause; + d->slave.device_resume = zx_dma_transfer_resume; + d->slave.copy_align = DMA_ALIGN; + d->slave.src_addr_widths = ZX_DMA_BUSWIDTHS; + d->slave.dst_addr_widths = ZX_DMA_BUSWIDTHS; + d->slave.directions = BIT(DMA_MEM_TO_MEM) | BIT(DMA_MEM_TO_DEV) + | BIT(DMA_DEV_TO_MEM); + d->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; + + /* init virtual channel */ + d->chans = devm_kzalloc(&op->dev, + d->dma_requests * sizeof(struct zx_dma_chan), GFP_KERNEL); + if (!d->chans) + return -ENOMEM; + + for (i = 0; i < d->dma_requests; i++) { + struct zx_dma_chan *c = &d->chans[i]; + + c->status = DMA_IN_PROGRESS; + INIT_LIST_HEAD(&c->node); + c->vc.desc_free = zx_dma_free_desc; + vchan_init(&c->vc, &d->slave); + } + + /* Enable clock before accessing registers */ + ret = clk_prepare_enable(d->clk); + if (ret < 0) { + dev_err(&op->dev, "clk_prepare_enable failed: %d\n", ret); + goto zx_dma_out; + } + + zx_dma_init_state(d); + + spin_lock_init(&d->lock); + INIT_LIST_HEAD(&d->chan_pending); + platform_set_drvdata(op, d); + + ret = dma_async_device_register(&d->slave); + if (ret) + goto clk_dis; + + ret = of_dma_controller_register((&op->dev)->of_node, + zx_of_dma_simple_xlate, d); + if (ret) + goto of_dma_register_fail; + + dev_info(&op->dev, "initialized\n"); + return 0; + +of_dma_register_fail: + dma_async_device_unregister(&d->slave); +clk_dis: + clk_disable_unprepare(d->clk); +zx_dma_out: + return ret; +} + +static int zx_dma_remove(struct platform_device *op) +{ + struct zx_dma_chan *c, *cn; + struct zx_dma_dev *d = platform_get_drvdata(op); + + /* explictly free the irq */ + devm_free_irq(&op->dev, d->irq, d); + + dma_async_device_unregister(&d->slave); + of_dma_controller_free((&op->dev)->of_node); + + list_for_each_entry_safe(c, cn, &d->slave.channels, + vc.chan.device_node) { + list_del(&c->vc.chan.device_node); + } + clk_disable_unprepare(d->clk); + dmam_pool_destroy(d->pool); + + return 0; +} + +#ifdef CONFIG_PM_SLEEP +static int zx_dma_suspend_dev(struct device *dev) +{ + struct zx_dma_dev *d = dev_get_drvdata(dev); + u32 stat = 0; + + stat = zx_dma_get_chan_stat(d); + if (stat) { + dev_warn(d->slave.dev, + "chan %d is running fail to suspend\n", stat); + return -1; + } + clk_disable_unprepare(d->clk); + return 0; +} + +static int zx_dma_resume_dev(struct device *dev) +{ + struct zx_dma_dev *d = dev_get_drvdata(dev); + int ret = 0; + + ret = clk_prepare_enable(d->clk); + if (ret < 0) { + dev_err(d->slave.dev, "clk_prepare_enable failed: %d\n", ret); + return ret; + } + zx_dma_init_state(d); + return 0; +} +#endif + +static SIMPLE_DEV_PM_OPS(zx_dma_pmops, zx_dma_suspend_dev, zx_dma_resume_dev); + +static struct platform_driver zx_pdma_driver = { + .driver = { + .name = DRIVER_NAME, + .pm = &zx_dma_pmops, + .of_match_table = zx6702_dma_dt_ids, + }, + .probe = zx_dma_probe, + .remove = zx_dma_remove, +}; + +module_platform_driver(zx_pdma_driver); + +MODULE_DESCRIPTION("ZTE ZX296702 DMA Driver"); +MODULE_AUTHOR("Jun Nie jun.nie@linaro.org"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/edac/mce_amd.c b/drivers/edac/mce_amd.c index 58586d59bf8e..e3a945ce374b 100644 --- a/drivers/edac/mce_amd.c +++ b/drivers/edac/mce_amd.c @@ -763,7 +763,8 @@ int amd_decode_mce(struct notifier_block *nb, unsigned long val, void *data) c->x86, c->x86_model, c->x86_mask, m->bank, ((m->status & MCI_STATUS_OVER) ? "Over" : "-"), - ((m->status & MCI_STATUS_UC) ? "UE" : "CE"), + ((m->status & MCI_STATUS_UC) ? "UE" : + (m->status & MCI_STATUS_DEFERRED) ? "-" : "CE"), ((m->status & MCI_STATUS_MISCV) ? "MiscV" : "-"), ((m->status & MCI_STATUS_PCC) ? "PCC" : "-"), ((m->status & MCI_STATUS_ADDRV) ? "AddrV" : "-")); diff --git a/drivers/edac/sb_edac.c b/drivers/edac/sb_edac.c index ca7831168298..cf1268ddef0c 100644 --- a/drivers/edac/sb_edac.c +++ b/drivers/edac/sb_edac.c @@ -280,6 +280,7 @@ struct sbridge_info { u8 max_interleave; u8 (*get_node_id)(struct sbridge_pvt *pvt); enum mem_type (*get_memory_type)(struct sbridge_pvt *pvt); + enum dev_type (*get_width)(struct sbridge_pvt *pvt, u32 mtr); struct pci_dev *pci_vtd; }; @@ -471,6 +472,9 @@ static const struct pci_id_table pci_dev_descr_ibridge_table[] = { #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD2 0x2f6c #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TAD3 0x2f6d #define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO0 0x2fbd +#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO1 0x2fbf +#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO2 0x2fb9 +#define PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO3 0x2fbb static const struct pci_id_descr pci_dev_descr_haswell[] = { /* first item must be the HA */ { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0, 0) }, @@ -488,6 +492,9 @@ static const struct pci_id_descr pci_dev_descr_haswell[] = { { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA0_TAD3, 1) }, { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO0, 1) }, + { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO1, 1) }, + { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO2, 1) }, + { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO3, 1) }, { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_TA, 1) }, { PCI_DESCR(PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1_THERMAL, 1) }, @@ -762,6 +769,49 @@ static enum mem_type haswell_get_memory_type(struct sbridge_pvt *pvt) return mtype; } +static enum dev_type sbridge_get_width(struct sbridge_pvt *pvt, u32 mtr) +{ + /* there's no way to figure out */ + return DEV_UNKNOWN; +} + +static enum dev_type __ibridge_get_width(u32 mtr) +{ + enum dev_type type; + + switch (mtr) { + case 3: + type = DEV_UNKNOWN; + break; + case 2: + type = DEV_X16; + break; + case 1: + type = DEV_X8; + break; + case 0: + type = DEV_X4; + break; + } + + return type; +} + +static enum dev_type ibridge_get_width(struct sbridge_pvt *pvt, u32 mtr) +{ + /* + * ddr3_width on the documentation but also valid for DDR4 on + * Haswell + */ + return __ibridge_get_width(GET_BITFIELD(mtr, 7, 8)); +} + +static enum dev_type broadwell_get_width(struct sbridge_pvt *pvt, u32 mtr) +{ + /* ddr3_width on the documentation but also valid for DDR4 */ + return __ibridge_get_width(GET_BITFIELD(mtr, 8, 9)); +} + static u8 get_node_id(struct sbridge_pvt *pvt) { u32 reg; @@ -966,17 +1016,7 @@ static int get_dimm_config(struct mem_ctl_info *mci) dimm->nr_pages = npages; dimm->grain = 32; - switch (banks) { - case 16: - dimm->dtype = DEV_X16; - break; - case 8: - dimm->dtype = DEV_X8; - break; - case 4: - dimm->dtype = DEV_X4; - break; - } + dimm->dtype = pvt->info.get_width(pvt, mtr); dimm->mtype = mtype; dimm->edac_mode = mode; snprintf(dimm->label, sizeof(dimm->label), @@ -1869,7 +1909,11 @@ static int haswell_mci_bind_devs(struct mem_ctl_info *mci, } break; case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO0: - pvt->pci_ddrio = pdev; + case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO1: + case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO2: + case PCI_DEVICE_ID_INTEL_HASWELL_IMC_DDRIO3: + if (!pvt->pci_ddrio) + pvt->pci_ddrio = pdev; break; case PCI_DEVICE_ID_INTEL_HASWELL_IMC_HA1: pvt->pci_ha1 = pdev; @@ -2361,6 +2405,7 @@ static int sbridge_register_mci(struct sbridge_dev *sbridge_dev, enum type type) pvt->info.interleave_list = ibridge_interleave_list; pvt->info.max_interleave = ARRAY_SIZE(ibridge_interleave_list); pvt->info.interleave_pkg = ibridge_interleave_pkg; + pvt->info.get_width = ibridge_get_width; mci->ctl_name = kasprintf(GFP_KERNEL, "Ivy Bridge Socket#%d", mci->mc_idx); /* Store pci devices at mci for faster access */ @@ -2380,6 +2425,7 @@ static int sbridge_register_mci(struct sbridge_dev *sbridge_dev, enum type type) pvt->info.interleave_list = sbridge_interleave_list; pvt->info.max_interleave = ARRAY_SIZE(sbridge_interleave_list); pvt->info.interleave_pkg = sbridge_interleave_pkg; + pvt->info.get_width = sbridge_get_width; mci->ctl_name = kasprintf(GFP_KERNEL, "Sandy Bridge Socket#%d", mci->mc_idx); /* Store pci devices at mci for faster access */ @@ -2399,6 +2445,7 @@ static int sbridge_register_mci(struct sbridge_dev *sbridge_dev, enum type type) pvt->info.interleave_list = ibridge_interleave_list; pvt->info.max_interleave = ARRAY_SIZE(ibridge_interleave_list); pvt->info.interleave_pkg = ibridge_interleave_pkg; + pvt->info.get_width = ibridge_get_width; mci->ctl_name = kasprintf(GFP_KERNEL, "Haswell Socket#%d", mci->mc_idx); /* Store pci devices at mci for faster access */ @@ -2418,6 +2465,7 @@ static int sbridge_register_mci(struct sbridge_dev *sbridge_dev, enum type type) pvt->info.interleave_list = ibridge_interleave_list; pvt->info.max_interleave = ARRAY_SIZE(ibridge_interleave_list); pvt->info.interleave_pkg = ibridge_interleave_pkg; + pvt->info.get_width = broadwell_get_width; mci->ctl_name = kasprintf(GFP_KERNEL, "Broadwell Socket#%d", mci->mc_idx); /* Store pci devices at mci for faster access */ diff --git a/drivers/edac/xgene_edac.c b/drivers/edac/xgene_edac.c index 14636e4b6a08..ba06904af2e1 100644 --- a/drivers/edac/xgene_edac.c +++ b/drivers/edac/xgene_edac.c @@ -1168,7 +1168,6 @@ static struct platform_driver xgene_edac_driver = { .remove = xgene_edac_remove, .driver = { .name = "xgene-edac", - .owner = THIS_MODULE, .of_match_table = xgene_edac_of_match, }, }; diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig index 99c69a3205c4..d8de6a8dd4de 100644 --- a/drivers/firmware/Kconfig +++ b/drivers/firmware/Kconfig @@ -5,6 +5,9 @@ menu "Firmware Drivers" +config ARM_PSCI_FW + bool + config EDD tristate "BIOS Enhanced Disk Drive calls determine boot disk" depends on X86 diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile index 4a4b897f9314..000830fc6707 100644 --- a/drivers/firmware/Makefile +++ b/drivers/firmware/Makefile @@ -1,6 +1,7 @@ # # Makefile for the linux kernel. # +obj-$(CONFIG_ARM_PSCI_FW) += psci.o obj-$(CONFIG_DMI) += dmi_scan.o obj-$(CONFIG_DMI_SYSFS) += dmi-sysfs.o obj-$(CONFIG_EDD) += edd.o diff --git a/drivers/firmware/efi/Kconfig b/drivers/firmware/efi/Kconfig index 54071c148340..84533e02fbf8 100644 --- a/drivers/firmware/efi/Kconfig +++ b/drivers/firmware/efi/Kconfig @@ -43,7 +43,7 @@ config EFI_VARS_PSTORE_DEFAULT_DISABLE config EFI_RUNTIME_MAP bool "Export efi runtime maps to sysfs" - depends on X86 && EFI && KEXEC + depends on X86 && EFI && KEXEC_CORE default y help Export efi runtime memory maps to /sys/firmware/efi/runtime-map. diff --git a/drivers/firmware/psci.c b/drivers/firmware/psci.c new file mode 100644 index 000000000000..42700f09a8c5 --- /dev/null +++ b/drivers/firmware/psci.c @@ -0,0 +1,382 @@ +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Copyright (C) 2015 ARM Limited + */ + +#define pr_fmt(fmt) "psci: " fmt + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +/* + * While a 64-bit OS can make calls with SMC32 calling conventions, for some + * calls it is necessary to use SMC64 to pass or return 64-bit values. For such + * calls PSCI_0_2_FN_NATIVE(x) will choose the appropriate (native-width) + * function ID. + */ +#ifdef CONFIG_64BIT +#define PSCI_0_2_FN_NATIVE(name) PSCI_0_2_FN64_##name +#else +#define PSCI_0_2_FN_NATIVE(name) PSCI_0_2_FN_##name +#endif + +/* + * The CPU any Trusted OS is resident on. The trusted OS may reject CPU_OFF + * calls to its resident CPU, so we must avoid issuing those. We never migrate + * a Trusted OS even if it claims to be capable of migration -- doing so will + * require cooperation with a Trusted OS driver. + */ +static int resident_cpu = -1; + +bool psci_tos_resident_on(int cpu) +{ + return cpu == resident_cpu; +} + +struct psci_operations psci_ops; + +typedef unsigned long (psci_fn)(unsigned long, unsigned long, + unsigned long, unsigned long); +asmlinkage psci_fn __invoke_psci_fn_hvc; +asmlinkage psci_fn __invoke_psci_fn_smc; +static psci_fn *invoke_psci_fn; + +enum psci_function { + PSCI_FN_CPU_SUSPEND, + PSCI_FN_CPU_ON, + PSCI_FN_CPU_OFF, + PSCI_FN_MIGRATE, + PSCI_FN_MAX, +}; + +static u32 psci_function_id[PSCI_FN_MAX]; + +static int psci_to_linux_errno(int errno) +{ + switch (errno) { + case PSCI_RET_SUCCESS: + return 0; + case PSCI_RET_NOT_SUPPORTED: + return -EOPNOTSUPP; + case PSCI_RET_INVALID_PARAMS: + return -EINVAL; + case PSCI_RET_DENIED: + return -EPERM; + }; + + return -EINVAL; +} + +static u32 psci_get_version(void) +{ + return invoke_psci_fn(PSCI_0_2_FN_PSCI_VERSION, 0, 0, 0); +} + +static int psci_cpu_suspend(u32 state, unsigned long entry_point) +{ + int err; + u32 fn; + + fn = psci_function_id[PSCI_FN_CPU_SUSPEND]; + err = invoke_psci_fn(fn, state, entry_point, 0); + return psci_to_linux_errno(err); +} + +static int psci_cpu_off(u32 state) +{ + int err; + u32 fn; + + fn = psci_function_id[PSCI_FN_CPU_OFF]; + err = invoke_psci_fn(fn, state, 0, 0); + return psci_to_linux_errno(err); +} + +static int psci_cpu_on(unsigned long cpuid, unsigned long entry_point) +{ + int err; + u32 fn; + + fn = psci_function_id[PSCI_FN_CPU_ON]; + err = invoke_psci_fn(fn, cpuid, entry_point, 0); + return psci_to_linux_errno(err); +} + +static int psci_migrate(unsigned long cpuid) +{ + int err; + u32 fn; + + fn = psci_function_id[PSCI_FN_MIGRATE]; + err = invoke_psci_fn(fn, cpuid, 0, 0); + return psci_to_linux_errno(err); +} + +static int psci_affinity_info(unsigned long target_affinity, + unsigned long lowest_affinity_level) +{ + return invoke_psci_fn(PSCI_0_2_FN_NATIVE(AFFINITY_INFO), + target_affinity, lowest_affinity_level, 0); +} + +static int psci_migrate_info_type(void) +{ + return invoke_psci_fn(PSCI_0_2_FN_MIGRATE_INFO_TYPE, 0, 0, 0); +} + +static unsigned long psci_migrate_info_up_cpu(void) +{ + return invoke_psci_fn(PSCI_0_2_FN_NATIVE(MIGRATE_INFO_UP_CPU), + 0, 0, 0); +} + +static int get_set_conduit_method(struct device_node *np) +{ + const char *method; + + pr_info("probing for conduit method from DT.\n"); + + if (of_property_read_string(np, "method", &method)) { + pr_warn("missing \"method\" property\n"); + return -ENXIO; + } + + if (!strcmp("hvc", method)) { + invoke_psci_fn = __invoke_psci_fn_hvc; + } else if (!strcmp("smc", method)) { + invoke_psci_fn = __invoke_psci_fn_smc; + } else { + pr_warn("invalid \"method\" property: %s\n", method); + return -EINVAL; + } + return 0; +} + +static void psci_sys_reset(enum reboot_mode reboot_mode, const char *cmd) +{ + invoke_psci_fn(PSCI_0_2_FN_SYSTEM_RESET, 0, 0, 0); +} + +static void psci_sys_poweroff(void) +{ + invoke_psci_fn(PSCI_0_2_FN_SYSTEM_OFF, 0, 0, 0); +} + +/* + * Detect the presence of a resident Trusted OS which may cause CPU_OFF to + * return DENIED (which would be fatal). + */ +static void __init psci_init_migrate(void) +{ + unsigned long cpuid; + int type, cpu = -1; + + type = psci_ops.migrate_info_type(); + + if (type == PSCI_0_2_TOS_MP) { + pr_info("Trusted OS migration not required\n"); + return; + } + + if (type == PSCI_RET_NOT_SUPPORTED) { + pr_info("MIGRATE_INFO_TYPE not supported.\n"); + return; + } + + if (type != PSCI_0_2_TOS_UP_MIGRATE && + type != PSCI_0_2_TOS_UP_NO_MIGRATE) { + pr_err("MIGRATE_INFO_TYPE returned unknown type (%d)\n", type); + return; + } + + cpuid = psci_migrate_info_up_cpu(); + if (cpuid & ~MPIDR_HWID_BITMASK) { + pr_warn("MIGRATE_INFO_UP_CPU reported invalid physical ID (0x%lx)\n", + cpuid); + return; + } + + cpu = get_logical_index(cpuid); + resident_cpu = cpu >= 0 ? cpu : -1; + + pr_info("Trusted OS resident on physical CPU 0x%lx\n", cpuid); +} + +static void __init psci_0_2_set_functions(void) +{ + pr_info("Using standard PSCI v0.2 function IDs\n"); + psci_function_id[PSCI_FN_CPU_SUSPEND] = PSCI_0_2_FN_NATIVE(CPU_SUSPEND); + psci_ops.cpu_suspend = psci_cpu_suspend; + + psci_function_id[PSCI_FN_CPU_OFF] = PSCI_0_2_FN_CPU_OFF; + psci_ops.cpu_off = psci_cpu_off; + + psci_function_id[PSCI_FN_CPU_ON] = PSCI_0_2_FN_NATIVE(CPU_ON); + psci_ops.cpu_on = psci_cpu_on; + + psci_function_id[PSCI_FN_MIGRATE] = PSCI_0_2_FN_NATIVE(MIGRATE); + psci_ops.migrate = psci_migrate; + + psci_ops.affinity_info = psci_affinity_info; + + psci_ops.migrate_info_type = psci_migrate_info_type; + + arm_pm_restart = psci_sys_reset; + + pm_power_off = psci_sys_poweroff; +} + +/* + * Probe function for PSCI firmware versions >= 0.2 + */ +static int __init psci_probe(void) +{ + u32 ver = psci_get_version(); + + pr_info("PSCIv%d.%d detected in firmware.\n", + PSCI_VERSION_MAJOR(ver), + PSCI_VERSION_MINOR(ver)); + + if (PSCI_VERSION_MAJOR(ver) == 0 && PSCI_VERSION_MINOR(ver) < 2) { + pr_err("Conflicting PSCI version detected.\n"); + return -EINVAL; + } + + psci_0_2_set_functions(); + + psci_init_migrate(); + + return 0; +} + +typedef int (*psci_initcall_t)(const struct device_node *); + +/* + * PSCI init function for PSCI versions >=0.2 + * + * Probe based on PSCI PSCI_VERSION function + */ +static int __init psci_0_2_init(struct device_node *np) +{ + int err; + + err = get_set_conduit_method(np); + + if (err) + goto out_put_node; + /* + * Starting with v0.2, the PSCI specification introduced a call + * (PSCI_VERSION) that allows probing the firmware version, so + * that PSCI function IDs and version specific initialization + * can be carried out according to the specific version reported + * by firmware + */ + err = psci_probe(); + +out_put_node: + of_node_put(np); + return err; +} + +/* + * PSCI < v0.2 get PSCI Function IDs via DT. + */ +static int __init psci_0_1_init(struct device_node *np) +{ + u32 id; + int err; + + err = get_set_conduit_method(np); + + if (err) + goto out_put_node; + + pr_info("Using PSCI v0.1 Function IDs from DT\n"); + + if (!of_property_read_u32(np, "cpu_suspend", &id)) { + psci_function_id[PSCI_FN_CPU_SUSPEND] = id; + psci_ops.cpu_suspend = psci_cpu_suspend; + } + + if (!of_property_read_u32(np, "cpu_off", &id)) { + psci_function_id[PSCI_FN_CPU_OFF] = id; + psci_ops.cpu_off = psci_cpu_off; + } + + if (!of_property_read_u32(np, "cpu_on", &id)) { + psci_function_id[PSCI_FN_CPU_ON] = id; + psci_ops.cpu_on = psci_cpu_on; + } + + if (!of_property_read_u32(np, "migrate", &id)) { + psci_function_id[PSCI_FN_MIGRATE] = id; + psci_ops.migrate = psci_migrate; + } + +out_put_node: + of_node_put(np); + return err; +} + +static const struct of_device_id const psci_of_match[] __initconst = { + { .compatible = "arm,psci", .data = psci_0_1_init}, + { .compatible = "arm,psci-0.2", .data = psci_0_2_init}, + {}, +}; + +int __init psci_dt_init(void) +{ + struct device_node *np; + const struct of_device_id *matched_np; + psci_initcall_t init_fn; + + np = of_find_matching_node_and_match(NULL, psci_of_match, &matched_np); + + if (!np) + return -ENODEV; + + init_fn = (psci_initcall_t)matched_np->data; + return init_fn(np); +} + +#ifdef CONFIG_ACPI +/* + * We use PSCI 0.2+ when ACPI is deployed on ARM64 and it's + * explicitly clarified in SBBR + */ +int __init psci_acpi_init(void) +{ + if (!acpi_psci_present()) { + pr_info("is not implemented in ACPI.\n"); + return -EOPNOTSUPP; + } + + pr_info("probing for conduit method from ACPI.\n"); + + if (acpi_psci_use_hvc()) + invoke_psci_fn = __invoke_psci_fn_hvc; + else + invoke_psci_fn = __invoke_psci_fn_smc; + + return psci_probe(); +} +#endif diff --git a/drivers/firmware/qcom_scm-32.c b/drivers/firmware/qcom_scm-32.c index 1bd6f9c34331..29e6850665eb 100644 --- a/drivers/firmware/qcom_scm-32.c +++ b/drivers/firmware/qcom_scm-32.c @@ -24,7 +24,6 @@ #include #include -#include #include #include "qcom_scm.h" @@ -219,8 +218,7 @@ static int __qcom_scm_call(const struct qcom_scm_command *cmd) * Flush the command buffer so that the secure world sees * the correct data. */ - __cpuc_flush_dcache_area((void *)cmd, cmd->len); - outer_flush_range(cmd_addr, cmd_addr + cmd->len); + secure_flush_area(cmd, cmd->len); ret = smc(cmd_addr); if (ret < 0) diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig index 8f1fe739c985..b4fc9e4d24c6 100644 --- a/drivers/gpio/Kconfig +++ b/drivers/gpio/Kconfig @@ -113,7 +113,6 @@ config GPIO_74XX_MMIO config GPIO_ALTERA tristate "Altera GPIO" depends on OF_GPIO - select GPIO_GENERIC select GPIOLIB_IRQCHIP help Say Y or M here to build support for the Altera PIO device. @@ -131,6 +130,7 @@ config GPIO_BRCMSTB default y if ARCH_BRCMSTB depends on OF_GPIO && (ARCH_BRCMSTB || COMPILE_TEST) select GPIO_GENERIC + select GPIOLIB_IRQCHIP help Say yes here to enable GPIO support for Broadcom STB (BCM7XXX) SoCs. @@ -172,6 +172,7 @@ config GPIO_ETRAXFS depends on CRIS || COMPILE_TEST depends on OF select GPIO_GENERIC + select GPIOLIB_IRQCHIP help Say yes here to support the GPIO controller on Axis ETRAX FS SoCs. @@ -308,7 +309,6 @@ config GPIO_MVEBU def_bool y depends on PLAT_ORION depends on OF - select GPIO_GENERIC select GENERIC_IRQ_CHIP config GPIO_MXC @@ -1005,6 +1005,12 @@ config GPIO_MC33880 SPI driver for Freescale MC33880 high-side/low-side switch. This provides GPIO interface supporting inputs and outputs. +config GPIO_ZX + bool "ZTE ZX GPIO support" + select GPIOLIB_IRQCHIP + help + Say yes here to support the GPIO device on ZTE ZX SoCs. + endmenu menu "USB GPIO expanders" diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile index f82cd678ce08..f79a7c482a99 100644 --- a/drivers/gpio/Makefile +++ b/drivers/gpio/Makefile @@ -20,6 +20,7 @@ obj-$(CONFIG_GPIO_ADP5588) += gpio-adp5588.o obj-$(CONFIG_GPIO_ALTERA) += gpio-altera.o obj-$(CONFIG_GPIO_AMD8111) += gpio-amd8111.o obj-$(CONFIG_GPIO_ARIZONA) += gpio-arizona.o +obj-$(CONFIG_ATH79) += gpio-ath79.o obj-$(CONFIG_GPIO_BCM_KONA) += gpio-bcm-kona.o obj-$(CONFIG_GPIO_BRCMSTB) += gpio-brcmstb.o obj-$(CONFIG_GPIO_BT8XX) += gpio-bt8xx.o @@ -116,3 +117,4 @@ obj-$(CONFIG_GPIO_XLP) += gpio-xlp.o obj-$(CONFIG_GPIO_XTENSA) += gpio-xtensa.o obj-$(CONFIG_GPIO_ZEVIO) += gpio-zevio.o obj-$(CONFIG_GPIO_ZYNQ) += gpio-zynq.o +obj-$(CONFIG_GPIO_ZX) += gpio-zx.o diff --git a/drivers/gpio/devres.c b/drivers/gpio/devres.c index 07ba82317ece..903fcf4d04a0 100644 --- a/drivers/gpio/devres.c +++ b/drivers/gpio/devres.c @@ -59,13 +59,13 @@ static int devm_gpiod_match_array(struct device *dev, void *res, void *data) * automatically disposed on driver detach. See gpiod_get() for detailed * information about behavior and return values. */ -struct gpio_desc *__must_check __devm_gpiod_get(struct device *dev, +struct gpio_desc *__must_check devm_gpiod_get(struct device *dev, const char *con_id, enum gpiod_flags flags) { return devm_gpiod_get_index(dev, con_id, 0, flags); } -EXPORT_SYMBOL(__devm_gpiod_get); +EXPORT_SYMBOL(devm_gpiod_get); /** * devm_gpiod_get_optional - Resource-managed gpiod_get_optional() @@ -77,13 +77,13 @@ EXPORT_SYMBOL(__devm_gpiod_get); * are automatically disposed on driver detach. See gpiod_get_optional() for * detailed information about behavior and return values. */ -struct gpio_desc *__must_check __devm_gpiod_get_optional(struct device *dev, +struct gpio_desc *__must_check devm_gpiod_get_optional(struct device *dev, const char *con_id, enum gpiod_flags flags) { return devm_gpiod_get_index_optional(dev, con_id, 0, flags); } -EXPORT_SYMBOL(__devm_gpiod_get_optional); +EXPORT_SYMBOL(devm_gpiod_get_optional); /** * devm_gpiod_get_index - Resource-managed gpiod_get_index() @@ -96,7 +96,7 @@ EXPORT_SYMBOL(__devm_gpiod_get_optional); * automatically disposed on driver detach. See gpiod_get_index() for detailed * information about behavior and return values. */ -struct gpio_desc *__must_check __devm_gpiod_get_index(struct device *dev, +struct gpio_desc *__must_check devm_gpiod_get_index(struct device *dev, const char *con_id, unsigned int idx, enum gpiod_flags flags) @@ -120,7 +120,7 @@ struct gpio_desc *__must_check __devm_gpiod_get_index(struct device *dev, return desc; } -EXPORT_SYMBOL(__devm_gpiod_get_index); +EXPORT_SYMBOL(devm_gpiod_get_index); /** * devm_get_gpiod_from_child - get a GPIO descriptor from a device's child node @@ -182,10 +182,10 @@ EXPORT_SYMBOL(devm_get_gpiod_from_child); * gpiod_get_index_optional() for detailed information about behavior and * return values. */ -struct gpio_desc *__must_check __devm_gpiod_get_index_optional(struct device *dev, +struct gpio_desc *__must_check devm_gpiod_get_index_optional(struct device *dev, const char *con_id, unsigned int index, - enum gpiod_flags flags) + enum gpiod_flags flags) { struct gpio_desc *desc; @@ -197,7 +197,7 @@ struct gpio_desc *__must_check __devm_gpiod_get_index_optional(struct device *de return desc; } -EXPORT_SYMBOL(__devm_gpiod_get_index_optional); +EXPORT_SYMBOL(devm_gpiod_get_index_optional); /** * devm_gpiod_get_array - Resource-managed gpiod_get_array() diff --git a/drivers/gpio/gpio-74xx-mmio.c b/drivers/gpio/gpio-74xx-mmio.c index 0763655cca6c..6ed7c0fb3378 100644 --- a/drivers/gpio/gpio-74xx-mmio.c +++ b/drivers/gpio/gpio-74xx-mmio.c @@ -129,7 +129,7 @@ static int mmio_74xx_gpio_probe(struct platform_device *pdev) if (IS_ERR(dat)) return PTR_ERR(dat); - priv->flags = (unsigned)of_id->data; + priv->flags = (uintptr_t) of_id->data; err = bgpio_init(&priv->bgc, &pdev->dev, DIV_ROUND_UP(MMIO_74XX_BIT_CNT(priv->flags), 8), diff --git a/drivers/gpio/gpio-adp5588.c b/drivers/gpio/gpio-adp5588.c index d3fe6a6776da..984186ee58a0 100644 --- a/drivers/gpio/gpio-adp5588.c +++ b/drivers/gpio/gpio-adp5588.c @@ -305,15 +305,7 @@ static int adp5588_irq_setup(struct adp5588_gpio *dev) irq_set_chip_and_handler(irq, &adp5588_irq_chip, handle_level_irq); irq_set_nested_thread(irq, 1); -#ifdef CONFIG_ARM - /* - * ARM needs us to explicitly flag the IRQ as VALID, - * once we do so, it will also set the noprobe. - */ - set_irq_flags(irq, IRQF_VALID); -#else - irq_set_noprobe(irq); -#endif + irq_modify_status(irq, IRQ_NOREQUEST, IRQ_NOPROBE); } ret = request_threaded_irq(client->irq, diff --git a/drivers/gpio/gpio-altera.c b/drivers/gpio/gpio-altera.c index 0f3d336d6303..9b7e0b3db387 100644 --- a/drivers/gpio/gpio-altera.c +++ b/drivers/gpio/gpio-altera.c @@ -338,9 +338,9 @@ static int altera_gpio_remove(struct platform_device *pdev) { struct altera_gpio_chip *altera_gc = platform_get_drvdata(pdev); - gpiochip_remove(&altera_gc->mmchip.gc); + of_mm_gpiochip_remove(&altera_gc->mmchip); - return -EIO; + return 0; } static const struct of_device_id altera_gpio_of_match[] = { diff --git a/arch/mips/ath79/gpio.c b/drivers/gpio/gpio-ath79.c similarity index 78% rename from arch/mips/ath79/gpio.c rename to drivers/gpio/gpio-ath79.c index f59ccb26520a..03b995304ad6 100644 --- a/arch/mips/ath79/gpio.c +++ b/drivers/gpio/gpio-ath79.c @@ -24,8 +24,6 @@ #include #include -#include -#include "common.h" static void __iomem *ath79_gpio_base; static u32 ath79_gpio_count; @@ -139,47 +137,6 @@ static struct gpio_chip ath79_gpio_chip = { .base = 0, }; -static void __iomem *ath79_gpio_get_function_reg(void) -{ - u32 reg = 0; - - if (soc_is_ar71xx() || - soc_is_ar724x() || - soc_is_ar913x() || - soc_is_ar933x()) - reg = AR71XX_GPIO_REG_FUNC; - else if (soc_is_ar934x()) - reg = AR934X_GPIO_REG_FUNC; - else - BUG(); - - return ath79_gpio_base + reg; -} - -void ath79_gpio_function_setup(u32 set, u32 clear) -{ - void __iomem *reg = ath79_gpio_get_function_reg(); - unsigned long flags; - - spin_lock_irqsave(&ath79_gpio_lock, flags); - - __raw_writel((__raw_readl(reg) & ~clear) | set, reg); - /* flush write */ - __raw_readl(reg); - - spin_unlock_irqrestore(&ath79_gpio_lock, flags); -} - -void ath79_gpio_function_enable(u32 mask) -{ - ath79_gpio_function_setup(mask, 0); -} - -void ath79_gpio_function_disable(u32 mask) -{ - ath79_gpio_function_setup(0, mask); -} - static const struct of_device_id ath79_gpio_of_match[] = { { .compatible = "qca,ar7100-gpio" }, { .compatible = "qca,ar9340-gpio" }, @@ -245,35 +202,3 @@ static struct platform_driver ath79_gpio_driver = { }; module_platform_driver(ath79_gpio_driver); - -int gpio_get_value(unsigned gpio) -{ - if (gpio < ath79_gpio_count) - return __ath79_gpio_get_value(gpio); - - return __gpio_get_value(gpio); -} -EXPORT_SYMBOL(gpio_get_value); - -void gpio_set_value(unsigned gpio, int value) -{ - if (gpio < ath79_gpio_count) - __ath79_gpio_set_value(gpio, value); - else - __gpio_set_value(gpio, value); -} -EXPORT_SYMBOL(gpio_set_value); - -int gpio_to_irq(unsigned gpio) -{ - /* FIXME */ - return -EINVAL; -} -EXPORT_SYMBOL(gpio_to_irq); - -int irq_to_gpio(unsigned irq) -{ - /* FIXME */ - return -EINVAL; -} -EXPORT_SYMBOL(irq_to_gpio); diff --git a/drivers/gpio/gpio-bcm-kona.c b/drivers/gpio/gpio-bcm-kona.c index 40343fa92c7b..31b90ac15204 100644 --- a/drivers/gpio/gpio-bcm-kona.c +++ b/drivers/gpio/gpio-bcm-kona.c @@ -438,7 +438,7 @@ static void bcm_kona_gpio_irq_handler(unsigned int irq, struct irq_desc *desc) void __iomem *reg_base; int bit, bank_id; unsigned long sta; - struct bcm_kona_gpio_bank *bank = irq_get_handler_data(irq); + struct bcm_kona_gpio_bank *bank = irq_desc_get_handler_data(desc); struct irq_chip *chip = irq_desc_get_chip(desc); chained_irq_enter(chip, desc); @@ -525,11 +525,7 @@ static int bcm_kona_gpio_irq_map(struct irq_domain *d, unsigned int irq, return ret; irq_set_lockdep_class(irq, &gpio_lock_class); irq_set_chip_and_handler(irq, &bcm_gpio_irq_chip, handle_simple_irq); -#ifdef CONFIG_ARM - set_irq_flags(irq, IRQF_VALID); -#else irq_set_noprobe(irq); -#endif return 0; } @@ -644,17 +640,6 @@ static int bcm_kona_gpio_probe(struct platform_device *pdev) dev_err(dev, "Couldn't add GPIO chip -- %d\n", ret); goto err_irq_domain; } - for (i = 0; i < chip->ngpio; i++) { - int irq = bcm_kona_gpio_to_irq(chip, i); - irq_set_lockdep_class(irq, &gpio_lock_class); - irq_set_chip_and_handler(irq, &bcm_gpio_irq_chip, - handle_simple_irq); -#ifdef CONFIG_ARM - set_irq_flags(irq, IRQF_VALID); -#else - irq_set_noprobe(irq); -#endif - } for (i = 0; i < kona_gpio->num_bank; i++) { bank = &kona_gpio->banks[i]; irq_set_chained_handler_and_data(bank->irq, diff --git a/drivers/gpio/gpio-brcmstb.c b/drivers/gpio/gpio-brcmstb.c index 4630a8133ea6..9ea86d2ac054 100644 --- a/drivers/gpio/gpio-brcmstb.c +++ b/drivers/gpio/gpio-brcmstb.c @@ -17,6 +17,10 @@ #include #include #include +#include +#include +#include +#include #define GIO_BANK_SIZE 0x20 #define GIO_ODEN(bank) (((bank) * GIO_BANK_SIZE) + 0x00) @@ -34,14 +38,18 @@ struct brcmstb_gpio_bank { struct bgpio_chip bgc; struct brcmstb_gpio_priv *parent_priv; u32 width; + struct irq_chip irq_chip; }; struct brcmstb_gpio_priv { struct list_head bank_list; void __iomem *reg_base; - int num_banks; struct platform_device *pdev; + int parent_irq; int gpio_base; + bool can_wake; + int parent_wake_irq; + struct notifier_block reboot_notifier; }; #define MAX_GPIO_PER_BANK 32 @@ -63,6 +71,203 @@ brcmstb_gpio_gc_to_priv(struct gpio_chip *gc) return bank->parent_priv; } +static void brcmstb_gpio_set_imask(struct brcmstb_gpio_bank *bank, + unsigned int offset, bool enable) +{ + struct bgpio_chip *bgc = &bank->bgc; + struct brcmstb_gpio_priv *priv = bank->parent_priv; + u32 mask = bgc->pin2mask(bgc, offset); + u32 imask; + unsigned long flags; + + spin_lock_irqsave(&bgc->lock, flags); + imask = bgc->read_reg(priv->reg_base + GIO_MASK(bank->id)); + if (enable) + imask |= mask; + else + imask &= ~mask; + bgc->write_reg(priv->reg_base + GIO_MASK(bank->id), imask); + spin_unlock_irqrestore(&bgc->lock, flags); +} + +/* -------------------- IRQ chip functions -------------------- */ + +static void brcmstb_gpio_irq_mask(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct brcmstb_gpio_bank *bank = brcmstb_gpio_gc_to_bank(gc); + + brcmstb_gpio_set_imask(bank, d->hwirq, false); +} + +static void brcmstb_gpio_irq_unmask(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct brcmstb_gpio_bank *bank = brcmstb_gpio_gc_to_bank(gc); + + brcmstb_gpio_set_imask(bank, d->hwirq, true); +} + +static int brcmstb_gpio_irq_set_type(struct irq_data *d, unsigned int type) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct brcmstb_gpio_bank *bank = brcmstb_gpio_gc_to_bank(gc); + struct brcmstb_gpio_priv *priv = bank->parent_priv; + u32 mask = BIT(d->hwirq); + u32 edge_insensitive, iedge_insensitive; + u32 edge_config, iedge_config; + u32 level, ilevel; + unsigned long flags; + + switch (type) { + case IRQ_TYPE_LEVEL_LOW: + level = 0; + edge_config = 0; + edge_insensitive = 0; + break; + case IRQ_TYPE_LEVEL_HIGH: + level = mask; + edge_config = 0; + edge_insensitive = 0; + break; + case IRQ_TYPE_EDGE_FALLING: + level = 0; + edge_config = 0; + edge_insensitive = 0; + break; + case IRQ_TYPE_EDGE_RISING: + level = 0; + edge_config = mask; + edge_insensitive = 0; + break; + case IRQ_TYPE_EDGE_BOTH: + level = 0; + edge_config = 0; /* don't care, but want known value */ + edge_insensitive = mask; + break; + default: + return -EINVAL; + } + + spin_lock_irqsave(&bank->bgc.lock, flags); + + iedge_config = bank->bgc.read_reg(priv->reg_base + + GIO_EC(bank->id)) & ~mask; + iedge_insensitive = bank->bgc.read_reg(priv->reg_base + + GIO_EI(bank->id)) & ~mask; + ilevel = bank->bgc.read_reg(priv->reg_base + + GIO_LEVEL(bank->id)) & ~mask; + + bank->bgc.write_reg(priv->reg_base + GIO_EC(bank->id), + iedge_config | edge_config); + bank->bgc.write_reg(priv->reg_base + GIO_EI(bank->id), + iedge_insensitive | edge_insensitive); + bank->bgc.write_reg(priv->reg_base + GIO_LEVEL(bank->id), + ilevel | level); + + spin_unlock_irqrestore(&bank->bgc.lock, flags); + return 0; +} + +static int brcmstb_gpio_priv_set_wake(struct brcmstb_gpio_priv *priv, + unsigned int enable) +{ + int ret = 0; + + /* + * Only enable wake IRQ once for however many hwirqs can wake + * since they all use the same wake IRQ. Mask will be set + * up appropriately thanks to IRQCHIP_MASK_ON_SUSPEND flag. + */ + if (enable) + ret = enable_irq_wake(priv->parent_wake_irq); + else + ret = disable_irq_wake(priv->parent_wake_irq); + if (ret) + dev_err(&priv->pdev->dev, "failed to %s wake-up interrupt\n", + enable ? "enable" : "disable"); + return ret; +} + +static int brcmstb_gpio_irq_set_wake(struct irq_data *d, unsigned int enable) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct brcmstb_gpio_priv *priv = brcmstb_gpio_gc_to_priv(gc); + + return brcmstb_gpio_priv_set_wake(priv, enable); +} + +static irqreturn_t brcmstb_gpio_wake_irq_handler(int irq, void *data) +{ + struct brcmstb_gpio_priv *priv = data; + + if (!priv || irq != priv->parent_wake_irq) + return IRQ_NONE; + pm_wakeup_event(&priv->pdev->dev, 0); + return IRQ_HANDLED; +} + +static void brcmstb_gpio_irq_bank_handler(struct brcmstb_gpio_bank *bank) +{ + struct brcmstb_gpio_priv *priv = bank->parent_priv; + struct irq_domain *irq_domain = bank->bgc.gc.irqdomain; + void __iomem *reg_base = priv->reg_base; + unsigned long status; + unsigned long flags; + + spin_lock_irqsave(&bank->bgc.lock, flags); + while ((status = bank->bgc.read_reg(reg_base + GIO_STAT(bank->id)) & + bank->bgc.read_reg(reg_base + GIO_MASK(bank->id)))) { + int bit; + + for_each_set_bit(bit, &status, 32) { + u32 stat = bank->bgc.read_reg(reg_base + + GIO_STAT(bank->id)); + if (bit >= bank->width) + dev_warn(&priv->pdev->dev, + "IRQ for invalid GPIO (bank=%d, offset=%d)\n", + bank->id, bit); + bank->bgc.write_reg(reg_base + GIO_STAT(bank->id), + stat | BIT(bit)); + generic_handle_irq(irq_find_mapping(irq_domain, bit)); + } + } + spin_unlock_irqrestore(&bank->bgc.lock, flags); +} + +/* Each UPG GIO block has one IRQ for all banks */ +static void brcmstb_gpio_irq_handler(unsigned int irq, struct irq_desc *desc) +{ + struct gpio_chip *gc = irq_desc_get_handler_data(desc); + struct brcmstb_gpio_priv *priv = brcmstb_gpio_gc_to_priv(gc); + struct irq_chip *chip = irq_desc_get_chip(desc); + struct list_head *pos; + + /* Interrupts weren't properly cleared during probe */ + BUG_ON(!priv || !chip); + + chained_irq_enter(chip, desc); + list_for_each(pos, &priv->bank_list) { + struct brcmstb_gpio_bank *bank = + list_entry(pos, struct brcmstb_gpio_bank, node); + brcmstb_gpio_irq_bank_handler(bank); + } + chained_irq_exit(chip, desc); +} + +static int brcmstb_gpio_reboot(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct brcmstb_gpio_priv *priv = + container_of(nb, struct brcmstb_gpio_priv, reboot_notifier); + + /* Enable GPIO for S5 cold boot */ + if (action == SYS_POWER_OFF) + brcmstb_gpio_priv_set_wake(priv, 1); + + return NOTIFY_DONE; +} + /* Make sure that the number of banks matches up between properties */ static int brcmstb_gpio_sanity_check_banks(struct device *dev, struct device_node *np, struct resource *res) @@ -100,7 +305,13 @@ static int brcmstb_gpio_remove(struct platform_device *pdev) bank = list_entry(pos, struct brcmstb_gpio_bank, node); ret = bgpio_remove(&bank->bgc); if (ret) - dev_err(&pdev->dev, "gpiochip_remove fail in cleanup"); + dev_err(&pdev->dev, "gpiochip_remove fail in cleanup\n"); + } + if (priv->reboot_notifier.notifier_call) { + ret = unregister_reboot_notifier(&priv->reboot_notifier); + if (ret) + dev_err(&pdev->dev, + "failed to unregister reboot notifier\n"); } return ret; } @@ -121,7 +332,7 @@ static int brcmstb_gpio_of_xlate(struct gpio_chip *gc, return -EINVAL; offset = gpiospec->args[0] - (gc->base - priv->gpio_base); - if (offset >= gc->ngpio) + if (offset >= gc->ngpio || offset < 0) return -EINVAL; if (unlikely(offset >= bank->width)) { @@ -136,6 +347,65 @@ static int brcmstb_gpio_of_xlate(struct gpio_chip *gc, return offset; } +/* Before calling, must have bank->parent_irq set and gpiochip registered */ +static int brcmstb_gpio_irq_setup(struct platform_device *pdev, + struct brcmstb_gpio_bank *bank) +{ + struct brcmstb_gpio_priv *priv = bank->parent_priv; + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + + bank->irq_chip.name = dev_name(dev); + bank->irq_chip.irq_mask = brcmstb_gpio_irq_mask; + bank->irq_chip.irq_unmask = brcmstb_gpio_irq_unmask; + bank->irq_chip.irq_set_type = brcmstb_gpio_irq_set_type; + + /* Ensures that all non-wakeup IRQs are disabled at suspend */ + bank->irq_chip.flags = IRQCHIP_MASK_ON_SUSPEND; + + if (IS_ENABLED(CONFIG_PM_SLEEP) && !priv->can_wake && + of_property_read_bool(np, "wakeup-source")) { + priv->parent_wake_irq = platform_get_irq(pdev, 1); + if (priv->parent_wake_irq < 0) { + dev_warn(dev, + "Couldn't get wake IRQ - GPIOs will not be able to wake from sleep"); + } else { + int err; + + /* + * Set wakeup capability before requesting wakeup + * interrupt, so we can process boot-time "wakeups" + * (e.g., from S5 cold boot) + */ + device_set_wakeup_capable(dev, true); + device_wakeup_enable(dev); + err = devm_request_irq(dev, priv->parent_wake_irq, + brcmstb_gpio_wake_irq_handler, 0, + "brcmstb-gpio-wake", priv); + + if (err < 0) { + dev_err(dev, "Couldn't request wake IRQ"); + return err; + } + + priv->reboot_notifier.notifier_call = + brcmstb_gpio_reboot; + register_reboot_notifier(&priv->reboot_notifier); + priv->can_wake = true; + } + } + + if (priv->can_wake) + bank->irq_chip.irq_set_wake = brcmstb_gpio_irq_set_wake; + + gpiochip_irqchip_add(&bank->bgc.gc, &bank->irq_chip, 0, + handle_simple_irq, IRQ_TYPE_NONE); + gpiochip_set_chained_irqchip(&bank->bgc.gc, &bank->irq_chip, + priv->parent_irq, brcmstb_gpio_irq_handler); + + return 0; +} + static int brcmstb_gpio_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -146,6 +416,7 @@ static int brcmstb_gpio_probe(struct platform_device *pdev) struct property *prop; const __be32 *p; u32 bank_width; + int num_banks = 0; int err; static int gpio_base; @@ -164,6 +435,16 @@ static int brcmstb_gpio_probe(struct platform_device *pdev) priv->reg_base = reg_base; priv->pdev = pdev; + if (of_property_read_bool(np, "interrupt-controller")) { + priv->parent_irq = platform_get_irq(pdev, 0); + if (priv->parent_irq <= 0) { + dev_err(dev, "Couldn't get IRQ"); + return -ENOENT; + } + } else { + priv->parent_irq = -ENOENT; + } + if (brcmstb_gpio_sanity_check_banks(dev, np, res)) return -EINVAL; @@ -180,7 +461,7 @@ static int brcmstb_gpio_probe(struct platform_device *pdev) } bank->parent_priv = priv; - bank->id = priv->num_banks; + bank->id = num_banks; if (bank_width <= 0 || bank_width > MAX_GPIO_PER_BANK) { dev_err(dev, "Invalid bank width %d\n", bank_width); goto fail; @@ -212,6 +493,12 @@ static int brcmstb_gpio_probe(struct platform_device *pdev) /* not all ngpio lines are valid, will use bank width later */ gc->ngpio = MAX_GPIO_PER_BANK; + /* + * Mask all interrupts by default, since wakeup interrupts may + * be retained from S5 cold boot + */ + bank->bgc.write_reg(reg_base + GIO_MASK(bank->id), 0); + err = gpiochip_add(gc); if (err) { dev_err(dev, "Could not add gpiochip for bank %d\n", @@ -219,17 +506,24 @@ static int brcmstb_gpio_probe(struct platform_device *pdev) goto fail; } gpio_base += gc->ngpio; + + if (priv->parent_irq > 0) { + err = brcmstb_gpio_irq_setup(pdev, bank); + if (err) + goto fail; + } + dev_dbg(dev, "bank=%d, base=%d, ngpio=%d, width=%d\n", bank->id, gc->base, gc->ngpio, bank->width); /* Everything looks good, so add bank to list */ list_add(&bank->node, &priv->bank_list); - priv->num_banks++; + num_banks++; } dev_info(dev, "Registered %d banks (GPIO(s): %d-%d)\n", - priv->num_banks, priv->gpio_base, gpio_base - 1); + num_banks, priv->gpio_base, gpio_base - 1); return 0; diff --git a/drivers/gpio/gpio-davinci.c b/drivers/gpio/gpio-davinci.c index c246ac3dda7c..94b0ab709721 100644 --- a/drivers/gpio/gpio-davinci.c +++ b/drivers/gpio/gpio-davinci.c @@ -65,11 +65,11 @@ static struct davinci_gpio_regs __iomem *gpio2regs(unsigned gpio) return ptr; } -static inline struct davinci_gpio_regs __iomem *irq2regs(int irq) +static inline struct davinci_gpio_regs __iomem *irq2regs(struct irq_data *d) { struct davinci_gpio_regs __iomem *g; - g = (__force struct davinci_gpio_regs __iomem *)irq_get_chip_data(irq); + g = (__force struct davinci_gpio_regs __iomem *)irq_data_get_irq_chip_data(d); return g; } @@ -287,7 +287,7 @@ static int davinci_gpio_probe(struct platform_device *pdev) static void gpio_irq_disable(struct irq_data *d) { - struct davinci_gpio_regs __iomem *g = irq2regs(d->irq); + struct davinci_gpio_regs __iomem *g = irq2regs(d); u32 mask = (u32) irq_data_get_irq_handler_data(d); writel_relaxed(mask, &g->clr_falling); @@ -296,7 +296,7 @@ static void gpio_irq_disable(struct irq_data *d) static void gpio_irq_enable(struct irq_data *d) { - struct davinci_gpio_regs __iomem *g = irq2regs(d->irq); + struct davinci_gpio_regs __iomem *g = irq2regs(d); u32 mask = (u32) irq_data_get_irq_handler_data(d); unsigned status = irqd_get_trigger_type(d); @@ -327,8 +327,9 @@ static struct irq_chip gpio_irqchip = { }; static void -gpio_irq_handler(unsigned irq, struct irq_desc *desc) +gpio_irq_handler(unsigned __irq, struct irq_desc *desc) { + unsigned int irq = irq_desc_get_irq(desc); struct davinci_gpio_regs __iomem *g; u32 mask = 0xffff; struct davinci_gpio_controller *d; @@ -396,7 +397,7 @@ static int gpio_irq_type_unbanked(struct irq_data *data, unsigned trigger) struct davinci_gpio_regs __iomem *g; u32 mask; - d = (struct davinci_gpio_controller *)data->handler_data; + d = (struct davinci_gpio_controller *)irq_data_get_irq_handler_data(data); g = (struct davinci_gpio_regs __iomem *)d->regs; mask = __gpio_mask(data->irq - d->gpio_irq); @@ -422,7 +423,6 @@ davinci_gpio_irq_map(struct irq_domain *d, unsigned int irq, irq_set_irq_type(irq, IRQ_TYPE_NONE); irq_set_chip_data(irq, (__force void *)g); irq_set_handler_data(irq, (void *)__gpio_mask(hw)); - set_irq_flags(irq, IRQF_VALID); return 0; } @@ -545,7 +545,7 @@ static int davinci_gpio_irq_setup(struct platform_device *pdev) chips[0].chip.to_irq = gpio_to_irq_unbanked; chips[0].gpio_irq = bank_irq; chips[0].gpio_unbanked = pdata->gpio_unbanked; - binten = BIT(0); + binten = GENMASK(pdata->gpio_unbanked / 16, 0); /* AINTC handles mask/unmask; GPIO handles triggering */ irq = bank_irq; diff --git a/drivers/gpio/gpio-dwapb.c b/drivers/gpio/gpio-dwapb.c index 55fa9853a7f2..c5be4b9b8baf 100644 --- a/drivers/gpio/gpio-dwapb.c +++ b/drivers/gpio/gpio-dwapb.c @@ -149,7 +149,7 @@ static u32 dwapb_do_irq(struct dwapb_gpio *gpio) static void dwapb_irq_handler(u32 irq, struct irq_desc *desc) { - struct dwapb_gpio *gpio = irq_get_handler_data(irq); + struct dwapb_gpio *gpio = irq_desc_get_handler_data(desc); struct irq_chip *chip = irq_desc_get_chip(desc); dwapb_do_irq(gpio); diff --git a/drivers/gpio/gpio-em.c b/drivers/gpio/gpio-em.c index fbf287307c4c..6bca1e125e12 100644 --- a/drivers/gpio/gpio-em.c +++ b/drivers/gpio/gpio-em.c @@ -31,7 +31,6 @@ #include #include #include -#include struct em_gio_priv { void __iomem *base0; @@ -262,7 +261,6 @@ static int em_gio_irq_domain_map(struct irq_domain *h, unsigned int irq, irq_set_chip_data(irq, h->host_data); irq_set_chip_and_handler(irq, &p->irq_chip, handle_level_irq); - set_irq_flags(irq, IRQF_VALID); /* kill me now */ return 0; } @@ -273,13 +271,12 @@ static const struct irq_domain_ops em_gio_irq_domain_ops = { static int em_gio_probe(struct platform_device *pdev) { - struct gpio_em_config pdata_dt; - struct gpio_em_config *pdata = dev_get_platdata(&pdev->dev); struct em_gio_priv *p; struct resource *io[2], *irq[2]; struct gpio_chip *gpio_chip; struct irq_chip *irq_chip; const char *name = dev_name(&pdev->dev); + unsigned int ngpios; int ret; p = devm_kzalloc(&pdev->dev, sizeof(*p), GFP_KERNEL); @@ -319,18 +316,10 @@ static int em_gio_probe(struct platform_device *pdev) goto err0; } - if (!pdata) { - memset(&pdata_dt, 0, sizeof(pdata_dt)); - pdata = &pdata_dt; - - if (of_property_read_u32(pdev->dev.of_node, "ngpios", - &pdata->number_of_pins)) { - dev_err(&pdev->dev, "Missing ngpios OF property\n"); - ret = -EINVAL; - goto err0; - } - - pdata->gpio_base = -1; + if (of_property_read_u32(pdev->dev.of_node, "ngpios", &ngpios)) { + dev_err(&pdev->dev, "Missing ngpios OF property\n"); + ret = -EINVAL; + goto err0; } gpio_chip = &p->gpio_chip; @@ -345,8 +334,8 @@ static int em_gio_probe(struct platform_device *pdev) gpio_chip->label = name; gpio_chip->dev = &pdev->dev; gpio_chip->owner = THIS_MODULE; - gpio_chip->base = pdata->gpio_base; - gpio_chip->ngpio = pdata->number_of_pins; + gpio_chip->base = -1; + gpio_chip->ngpio = ngpios; irq_chip = &p->irq_chip; irq_chip->name = name; @@ -357,9 +346,7 @@ static int em_gio_probe(struct platform_device *pdev) irq_chip->irq_release_resources = em_gio_irq_relres; irq_chip->flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND; - p->irq_domain = irq_domain_add_simple(pdev->dev.of_node, - pdata->number_of_pins, - pdata->irq_base, + p->irq_domain = irq_domain_add_simple(pdev->dev.of_node, ngpios, 0, &em_gio_irq_domain_ops, p); if (!p->irq_domain) { ret = -ENXIO; @@ -387,12 +374,6 @@ static int em_gio_probe(struct platform_device *pdev) goto err1; } - if (pdata->pctl_name) { - ret = gpiochip_add_pin_range(gpio_chip, pdata->pctl_name, 0, - gpio_chip->base, gpio_chip->ngpio); - if (ret < 0) - dev_warn(&pdev->dev, "failed to add pin range\n"); - } return 0; err1: diff --git a/drivers/gpio/gpio-ep93xx.c b/drivers/gpio/gpio-ep93xx.c index 45684f36ddb1..9d90366ea259 100644 --- a/drivers/gpio/gpio-ep93xx.c +++ b/drivers/gpio/gpio-ep93xx.c @@ -100,13 +100,15 @@ static void ep93xx_gpio_ab_irq_handler(unsigned int irq, struct irq_desc *desc) } } -static void ep93xx_gpio_f_irq_handler(unsigned int irq, struct irq_desc *desc) +static void ep93xx_gpio_f_irq_handler(unsigned int __irq, + struct irq_desc *desc) { /* * map discontiguous hw irq range to continuous sw irq range: * * IRQ_EP93XX_GPIO{0..7}MUX -> gpio_to_irq(EP93XX_GPIO_LINE_F({0..7}) */ + unsigned int irq = irq_desc_get_irq(desc); int port_f_idx = ((irq + 1) & 7) ^ 4; /* {19..22,47..50} -> {0..7} */ int gpio_irq = gpio_to_irq(EP93XX_GPIO_LINE_F(0)) + port_f_idx; @@ -208,7 +210,7 @@ static int ep93xx_gpio_irq_type(struct irq_data *d, unsigned int type) return -EINVAL; } - __irq_set_handler_locked(d->irq, handler); + irq_set_handler_locked(d, handler); gpio_int_enabled[port] |= port_mask; @@ -234,7 +236,7 @@ static void ep93xx_gpio_init_irq(void) gpio_irq <= gpio_to_irq(EP93XX_GPIO_LINE_MAX_IRQ); ++gpio_irq) { irq_set_chip_and_handler(gpio_irq, &ep93xx_gpio_irq_chip, handle_level_irq); - set_irq_flags(gpio_irq, IRQF_VALID); + irq_clear_status_flags(gpio_irq, IRQ_NOREQUEST); } irq_set_chained_handler(IRQ_EP93XX_GPIO_AB, diff --git a/drivers/gpio/gpio-etraxfs.c b/drivers/gpio/gpio-etraxfs.c index 28071f4a5672..2ffcd9fdd1f2 100644 --- a/drivers/gpio/gpio-etraxfs.c +++ b/drivers/gpio/gpio-etraxfs.c @@ -1,8 +1,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -13,6 +15,7 @@ #define ETRAX_FS_rw_intr_mask 16 #define ETRAX_FS_rw_ack_intr 20 #define ETRAX_FS_r_intr 24 +#define ETRAX_FS_r_masked_intr 28 #define ETRAX_FS_rw_pb_dout 32 #define ETRAX_FS_r_pb_din 36 #define ETRAX_FS_rw_pb_oe 40 @@ -26,6 +29,48 @@ #define ETRAX_FS_r_pe_din 84 #define ETRAX_FS_rw_pe_oe 88 +#define ARTPEC3_r_pa_din 0 +#define ARTPEC3_rw_pa_dout 4 +#define ARTPEC3_rw_pa_oe 8 +#define ARTPEC3_r_pb_din 44 +#define ARTPEC3_rw_pb_dout 48 +#define ARTPEC3_rw_pb_oe 52 +#define ARTPEC3_r_pc_din 88 +#define ARTPEC3_rw_pc_dout 92 +#define ARTPEC3_rw_pc_oe 96 +#define ARTPEC3_r_pd_din 116 +#define ARTPEC3_rw_intr_cfg 120 +#define ARTPEC3_rw_intr_pins 124 +#define ARTPEC3_rw_intr_mask 128 +#define ARTPEC3_rw_ack_intr 132 +#define ARTPEC3_r_masked_intr 140 + +#define GIO_CFG_OFF 0 +#define GIO_CFG_HI 1 +#define GIO_CFG_LO 2 +#define GIO_CFG_SET 3 +#define GIO_CFG_POSEDGE 5 +#define GIO_CFG_NEGEDGE 6 +#define GIO_CFG_ANYEDGE 7 + +struct etraxfs_gpio_info; + +struct etraxfs_gpio_block { + spinlock_t lock; + u32 mask; + u32 cfg; + u32 pins; + unsigned int group[8]; + + void __iomem *regs; + const struct etraxfs_gpio_info *info; +}; + +struct etraxfs_gpio_chip { + struct bgpio_chip bgc; + struct etraxfs_gpio_block *block; +}; + struct etraxfs_gpio_port { const char *label; unsigned int oe; @@ -37,6 +82,12 @@ struct etraxfs_gpio_port { struct etraxfs_gpio_info { unsigned int num_ports; const struct etraxfs_gpio_port *ports; + + unsigned int rw_ack_intr; + unsigned int rw_intr_mask; + unsigned int rw_intr_cfg; + unsigned int rw_intr_pins; + unsigned int r_masked_intr; }; static const struct etraxfs_gpio_port etraxfs_gpio_etraxfs_ports[] = { @@ -80,8 +131,56 @@ static const struct etraxfs_gpio_port etraxfs_gpio_etraxfs_ports[] = { static const struct etraxfs_gpio_info etraxfs_gpio_etraxfs = { .num_ports = ARRAY_SIZE(etraxfs_gpio_etraxfs_ports), .ports = etraxfs_gpio_etraxfs_ports, + .rw_ack_intr = ETRAX_FS_rw_ack_intr, + .rw_intr_mask = ETRAX_FS_rw_intr_mask, + .rw_intr_cfg = ETRAX_FS_rw_intr_cfg, + .r_masked_intr = ETRAX_FS_r_masked_intr, }; +static const struct etraxfs_gpio_port etraxfs_gpio_artpec3_ports[] = { + { + .label = "A", + .ngpio = 32, + .oe = ARTPEC3_rw_pa_oe, + .dout = ARTPEC3_rw_pa_dout, + .din = ARTPEC3_r_pa_din, + }, + { + .label = "B", + .ngpio = 32, + .oe = ARTPEC3_rw_pb_oe, + .dout = ARTPEC3_rw_pb_dout, + .din = ARTPEC3_r_pb_din, + }, + { + .label = "C", + .ngpio = 16, + .oe = ARTPEC3_rw_pc_oe, + .dout = ARTPEC3_rw_pc_dout, + .din = ARTPEC3_r_pc_din, + }, + { + .label = "D", + .ngpio = 32, + .din = ARTPEC3_r_pd_din, + }, +}; + +static const struct etraxfs_gpio_info etraxfs_gpio_artpec3 = { + .num_ports = ARRAY_SIZE(etraxfs_gpio_artpec3_ports), + .ports = etraxfs_gpio_artpec3_ports, + .rw_ack_intr = ARTPEC3_rw_ack_intr, + .rw_intr_mask = ARTPEC3_rw_intr_mask, + .rw_intr_cfg = ARTPEC3_rw_intr_cfg, + .r_masked_intr = ARTPEC3_r_masked_intr, + .rw_intr_pins = ARTPEC3_rw_intr_pins, +}; + +static unsigned int etraxfs_gpio_chip_to_port(struct gpio_chip *gc) +{ + return gc->label[0] - 'A'; +} + static int etraxfs_gpio_of_xlate(struct gpio_chip *gc, const struct of_phandle_args *gpiospec, u32 *flags) @@ -90,7 +189,7 @@ static int etraxfs_gpio_of_xlate(struct gpio_chip *gc, * Port numbers are A to E, and the properties are integers, so we * specify them as 0xA - 0xE. */ - if (gc->label[0] - 'A' + 0xA != gpiospec->args[2]) + if (etraxfs_gpio_chip_to_port(gc) + 0xA != gpiospec->args[2]) return -EINVAL; return of_gpio_simple_xlate(gc, gpiospec, flags); @@ -101,24 +200,174 @@ static const struct of_device_id etraxfs_gpio_of_table[] = { .compatible = "axis,etraxfs-gio", .data = &etraxfs_gpio_etraxfs, }, + { + .compatible = "axis,artpec3-gio", + .data = &etraxfs_gpio_artpec3, + }, {}, }; +static unsigned int etraxfs_gpio_to_group_irq(unsigned int gpio) +{ + return gpio % 8; +} + +static unsigned int etraxfs_gpio_to_group_pin(struct etraxfs_gpio_chip *chip, + unsigned int gpio) +{ + return 4 * etraxfs_gpio_chip_to_port(&chip->bgc.gc) + gpio / 8; +} + +static void etraxfs_gpio_irq_ack(struct irq_data *d) +{ + struct etraxfs_gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct etraxfs_gpio_block *block = chip->block; + unsigned int grpirq = etraxfs_gpio_to_group_irq(d->hwirq); + + writel(BIT(grpirq), block->regs + block->info->rw_ack_intr); +} + +static void etraxfs_gpio_irq_mask(struct irq_data *d) +{ + struct etraxfs_gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct etraxfs_gpio_block *block = chip->block; + unsigned int grpirq = etraxfs_gpio_to_group_irq(d->hwirq); + + spin_lock(&block->lock); + block->mask &= ~BIT(grpirq); + writel(block->mask, block->regs + block->info->rw_intr_mask); + spin_unlock(&block->lock); +} + +static void etraxfs_gpio_irq_unmask(struct irq_data *d) +{ + struct etraxfs_gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct etraxfs_gpio_block *block = chip->block; + unsigned int grpirq = etraxfs_gpio_to_group_irq(d->hwirq); + + spin_lock(&block->lock); + block->mask |= BIT(grpirq); + writel(block->mask, block->regs + block->info->rw_intr_mask); + spin_unlock(&block->lock); +} + +static int etraxfs_gpio_irq_set_type(struct irq_data *d, u32 type) +{ + struct etraxfs_gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct etraxfs_gpio_block *block = chip->block; + unsigned int grpirq = etraxfs_gpio_to_group_irq(d->hwirq); + u32 cfg; + + switch (type) { + case IRQ_TYPE_EDGE_RISING: + cfg = GIO_CFG_POSEDGE; + break; + case IRQ_TYPE_EDGE_FALLING: + cfg = GIO_CFG_NEGEDGE; + break; + case IRQ_TYPE_EDGE_BOTH: + cfg = GIO_CFG_ANYEDGE; + break; + case IRQ_TYPE_LEVEL_LOW: + cfg = GIO_CFG_LO; + break; + case IRQ_TYPE_LEVEL_HIGH: + cfg = GIO_CFG_HI; + break; + default: + return -EINVAL; + } + + spin_lock(&block->lock); + block->cfg &= ~(0x7 << (grpirq * 3)); + block->cfg |= (cfg << (grpirq * 3)); + writel(block->cfg, block->regs + block->info->rw_intr_cfg); + spin_unlock(&block->lock); + + return 0; +} + +static int etraxfs_gpio_irq_request_resources(struct irq_data *d) +{ + struct etraxfs_gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct etraxfs_gpio_block *block = chip->block; + unsigned int grpirq = etraxfs_gpio_to_group_irq(d->hwirq); + int ret = -EBUSY; + + spin_lock(&block->lock); + if (block->group[grpirq]) + goto out; + + ret = gpiochip_lock_as_irq(&chip->bgc.gc, d->hwirq); + if (ret) + goto out; + + block->group[grpirq] = d->irq; + if (block->info->rw_intr_pins) { + unsigned int pin = etraxfs_gpio_to_group_pin(chip, d->hwirq); + + block->pins &= ~(0xf << (grpirq * 4)); + block->pins |= (pin << (grpirq * 4)); + + writel(block->pins, block->regs + block->info->rw_intr_pins); + } + +out: + spin_unlock(&block->lock); + return ret; +} + +static void etraxfs_gpio_irq_release_resources(struct irq_data *d) +{ + struct etraxfs_gpio_chip *chip = irq_data_get_irq_chip_data(d); + struct etraxfs_gpio_block *block = chip->block; + unsigned int grpirq = etraxfs_gpio_to_group_irq(d->hwirq); + + spin_lock(&block->lock); + block->group[grpirq] = 0; + gpiochip_unlock_as_irq(&chip->bgc.gc, d->hwirq); + spin_unlock(&block->lock); +} + +static struct irq_chip etraxfs_gpio_irq_chip = { + .name = "gpio-etraxfs", + .irq_ack = etraxfs_gpio_irq_ack, + .irq_mask = etraxfs_gpio_irq_mask, + .irq_unmask = etraxfs_gpio_irq_unmask, + .irq_set_type = etraxfs_gpio_irq_set_type, + .irq_request_resources = etraxfs_gpio_irq_request_resources, + .irq_release_resources = etraxfs_gpio_irq_release_resources, +}; + +static irqreturn_t etraxfs_gpio_interrupt(int irq, void *dev_id) +{ + struct etraxfs_gpio_block *block = dev_id; + unsigned long intr = readl(block->regs + block->info->r_masked_intr); + int bit; + + for_each_set_bit(bit, &intr, 8) + generic_handle_irq(block->group[bit]); + + return IRQ_RETVAL(intr & 0xff); +} + static int etraxfs_gpio_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; const struct etraxfs_gpio_info *info; const struct of_device_id *match; - struct bgpio_chip *chips; - struct resource *res; + struct etraxfs_gpio_block *block; + struct etraxfs_gpio_chip *chips; + struct resource *res, *irq; + bool allportsirq = false; void __iomem *regs; int ret; int i; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); regs = devm_ioremap_resource(dev, res); - if (!regs) - return -ENOMEM; + if (IS_ERR(regs)) + return PTR_ERR(regs); match = of_match_node(etraxfs_gpio_of_table, dev->of_node); if (!match) @@ -130,19 +379,57 @@ static int etraxfs_gpio_probe(struct platform_device *pdev) if (!chips) return -ENOMEM; + irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0); + if (!irq) + return -EINVAL; + + block = devm_kzalloc(dev, sizeof(*block), GFP_KERNEL); + if (!block) + return -ENOMEM; + + spin_lock_init(&block->lock); + + block->regs = regs; + block->info = info; + + writel(0, block->regs + info->rw_intr_mask); + writel(0, block->regs + info->rw_intr_cfg); + if (info->rw_intr_pins) { + allportsirq = true; + writel(0, block->regs + info->rw_intr_pins); + } + + ret = devm_request_irq(dev, irq->start, etraxfs_gpio_interrupt, + IRQF_SHARED, dev_name(dev), block); + if (ret) { + dev_err(dev, "Unable to request irq %d\n", ret); + return ret; + } + for (i = 0; i < info->num_ports; i++) { - struct bgpio_chip *bgc = &chips[i]; + struct etraxfs_gpio_chip *chip = &chips[i]; + struct bgpio_chip *bgc = &chip->bgc; const struct etraxfs_gpio_port *port = &info->ports[i]; + unsigned long flags = BGPIOF_READ_OUTPUT_REG_SET; + void __iomem *dat = regs + port->din; + void __iomem *set = regs + port->dout; + void __iomem *dirout = regs + port->oe; + + chip->block = block; + + if (dirout == set) { + dirout = set = NULL; + flags = BGPIOF_NO_OUTPUT; + } ret = bgpio_init(bgc, dev, 4, - regs + port->din, /* dat */ - regs + port->dout, /* set */ - NULL, /* clr */ - regs + port->oe, /* dirout */ - NULL, /* dirin */ - BGPIOF_UNREADABLE_REG_SET); - if (ret) - return ret; + dat, set, NULL, dirout, NULL, + flags); + if (ret) { + dev_err(dev, "Unable to init port %s\n", + port->label); + continue; + } bgc->gc.ngpio = port->ngpio; bgc->gc.label = port->label; @@ -152,9 +439,21 @@ static int etraxfs_gpio_probe(struct platform_device *pdev) bgc->gc.of_xlate = etraxfs_gpio_of_xlate; ret = gpiochip_add(&bgc->gc); - if (ret) + if (ret) { dev_err(dev, "Unable to register port %s\n", bgc->gc.label); + continue; + } + + if (i > 0 && !allportsirq) + continue; + + ret = gpiochip_irqchip_add(&bgc->gc, &etraxfs_gpio_irq_chip, 0, + handle_level_irq, IRQ_TYPE_NONE); + if (ret) { + dev_err(dev, "Unable to add irqchip to port %s\n", + bgc->gc.label); + } } return 0; diff --git a/drivers/gpio/gpio-generic.c b/drivers/gpio/gpio-generic.c index 9bda3727fac1..a3f07537fe62 100644 --- a/drivers/gpio/gpio-generic.c +++ b/drivers/gpio/gpio-generic.c @@ -153,6 +153,10 @@ static int bgpio_get(struct gpio_chip *gc, unsigned int gpio) return !!(bgc->read_reg(bgc->reg_dat) & bgc->pin2mask(bgc, gpio)); } +static void bgpio_set_none(struct gpio_chip *gc, unsigned int gpio, int val) +{ +} + static void bgpio_set(struct gpio_chip *gc, unsigned int gpio, int val) { struct bgpio_chip *bgc = to_bgpio_chip(gc); @@ -279,6 +283,12 @@ static int bgpio_simple_dir_in(struct gpio_chip *gc, unsigned int gpio) return 0; } +static int bgpio_dir_out_err(struct gpio_chip *gc, unsigned int gpio, + int val) +{ + return -EINVAL; +} + static int bgpio_simple_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) { @@ -302,6 +312,14 @@ static int bgpio_dir_in(struct gpio_chip *gc, unsigned int gpio) return 0; } +static int bgpio_get_dir(struct gpio_chip *gc, unsigned int gpio) +{ + struct bgpio_chip *bgc = to_bgpio_chip(gc); + + return (bgc->read_reg(bgc->reg_dir) & bgc->pin2mask(bgc, gpio)) ? + GPIOF_DIR_OUT : GPIOF_DIR_IN; +} + static int bgpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) { struct bgpio_chip *bgc = to_bgpio_chip(gc); @@ -351,6 +369,14 @@ static int bgpio_dir_out_inv(struct gpio_chip *gc, unsigned int gpio, int val) return 0; } +static int bgpio_get_dir_inv(struct gpio_chip *gc, unsigned int gpio) +{ + struct bgpio_chip *bgc = to_bgpio_chip(gc); + + return (bgc->read_reg(bgc->reg_dir) & bgc->pin2mask(bgc, gpio)) ? + GPIOF_DIR_IN : GPIOF_DIR_OUT; +} + static int bgpio_setup_accessors(struct device *dev, struct bgpio_chip *bgc, bool bit_be, @@ -444,6 +470,9 @@ static int bgpio_setup_io(struct bgpio_chip *bgc, bgc->reg_set = set; bgc->gc.set = bgpio_set_set; bgc->gc.set_multiple = bgpio_set_multiple_set; + } else if (flags & BGPIOF_NO_OUTPUT) { + bgc->gc.set = bgpio_set_none; + bgc->gc.set_multiple = NULL; } else { bgc->gc.set = bgpio_set; bgc->gc.set_multiple = bgpio_set_multiple; @@ -460,7 +489,8 @@ static int bgpio_setup_io(struct bgpio_chip *bgc, static int bgpio_setup_direction(struct bgpio_chip *bgc, void __iomem *dirout, - void __iomem *dirin) + void __iomem *dirin, + unsigned long flags) { if (dirout && dirin) { return -EINVAL; @@ -468,12 +498,17 @@ static int bgpio_setup_direction(struct bgpio_chip *bgc, bgc->reg_dir = dirout; bgc->gc.direction_output = bgpio_dir_out; bgc->gc.direction_input = bgpio_dir_in; + bgc->gc.get_direction = bgpio_get_dir; } else if (dirin) { bgc->reg_dir = dirin; bgc->gc.direction_output = bgpio_dir_out_inv; bgc->gc.direction_input = bgpio_dir_in_inv; + bgc->gc.get_direction = bgpio_get_dir_inv; } else { - bgc->gc.direction_output = bgpio_simple_dir_out; + if (flags & BGPIOF_NO_OUTPUT) + bgc->gc.direction_output = bgpio_dir_out_err; + else + bgc->gc.direction_output = bgpio_simple_dir_out; bgc->gc.direction_input = bgpio_simple_dir_in; } @@ -525,7 +560,7 @@ int bgpio_init(struct bgpio_chip *bgc, struct device *dev, if (ret) return ret; - ret = bgpio_setup_direction(bgc, dirout, dirin); + ret = bgpio_setup_direction(bgc, dirout, dirin, flags); if (ret) return ret; diff --git a/drivers/gpio/gpio-grgpio.c b/drivers/gpio/gpio-grgpio.c index 0a8f7617e72e..801423fe8143 100644 --- a/drivers/gpio/gpio-grgpio.c +++ b/drivers/gpio/gpio-grgpio.c @@ -104,17 +104,12 @@ static void grgpio_set_imask(struct grgpio_priv *priv, unsigned int offset, { struct bgpio_chip *bgc = &priv->bgc; unsigned long mask = bgc->pin2mask(bgc, offset); - unsigned long flags; - - spin_lock_irqsave(&bgc->lock, flags); if (val) priv->imask |= mask; else priv->imask &= ~mask; bgc->write_reg(priv->regs + GRGPIO_IMASK, priv->imask); - - spin_unlock_irqrestore(&bgc->lock, flags); } static int grgpio_to_irq(struct gpio_chip *gc, unsigned offset) @@ -180,16 +175,26 @@ static void grgpio_irq_mask(struct irq_data *d) { struct grgpio_priv *priv = irq_data_get_irq_chip_data(d); int offset = d->hwirq; + unsigned long flags; + + spin_lock_irqsave(&priv->bgc.lock, flags); grgpio_set_imask(priv, offset, 0); + + spin_unlock_irqrestore(&priv->bgc.lock, flags); } static void grgpio_irq_unmask(struct irq_data *d) { struct grgpio_priv *priv = irq_data_get_irq_chip_data(d); int offset = d->hwirq; + unsigned long flags; + + spin_lock_irqsave(&priv->bgc.lock, flags); grgpio_set_imask(priv, offset, 1); + + spin_unlock_irqrestore(&priv->bgc.lock, flags); } static struct irq_chip grgpio_irq_chip = { @@ -281,12 +286,7 @@ static int grgpio_irq_map(struct irq_domain *d, unsigned int irq, irq_set_chip_data(irq, priv); irq_set_chip_and_handler(irq, &grgpio_irq_chip, handle_simple_irq); - irq_clear_status_flags(irq, IRQ_NOREQUEST); -#ifdef CONFIG_ARM - set_irq_flags(irq, IRQF_VALID); -#else irq_set_noprobe(irq); -#endif return ret; } @@ -301,9 +301,6 @@ static void grgpio_irq_unmap(struct irq_domain *d, unsigned int irq) int ngpio = priv->bgc.gc.ngpio; int i; -#ifdef CONFIG_ARM - set_irq_flags(irq, 0); -#endif irq_set_chip_and_handler(irq, NULL, NULL); irq_set_chip_data(irq, NULL); diff --git a/drivers/gpio/gpio-max732x.c b/drivers/gpio/gpio-max732x.c index 7d3c90e9da71..8c5252c6c327 100644 --- a/drivers/gpio/gpio-max732x.c +++ b/drivers/gpio/gpio-max732x.c @@ -685,9 +685,14 @@ static int max732x_probe(struct i2c_client *client, mutex_init(&chip->lock); - max732x_readb(chip, is_group_a(chip, 0), &chip->reg_out[0]); - if (nr_port > 8) - max732x_readb(chip, is_group_a(chip, 8), &chip->reg_out[1]); + ret = max732x_readb(chip, is_group_a(chip, 0), &chip->reg_out[0]); + if (ret) + goto out_failed; + if (nr_port > 8) { + ret = max732x_readb(chip, is_group_a(chip, 8), &chip->reg_out[1]); + if (ret) + goto out_failed; + } ret = gpiochip_add(&chip->gpio_chip); if (ret) diff --git a/drivers/gpio/gpio-mcp23s08.c b/drivers/gpio/gpio-mcp23s08.c index 2fc7ff852d16..73db7ecd7ffd 100644 --- a/drivers/gpio/gpio-mcp23s08.c +++ b/drivers/gpio/gpio-mcp23s08.c @@ -507,11 +507,7 @@ static int mcp23s08_irq_setup(struct mcp23s08 *mcp) irq_set_chip_data(irq, mcp); irq_set_chip(irq, &mcp23s08_irq_chip); irq_set_nested_thread(irq, true); -#ifdef CONFIG_ARM - set_irq_flags(irq, IRQF_VALID); -#else irq_set_noprobe(irq); -#endif } return 0; } diff --git a/drivers/gpio/gpio-mpc8xxx.c b/drivers/gpio/gpio-mpc8xxx.c index 20aa66f34f6e..8ef7a12de983 100644 --- a/drivers/gpio/gpio-mpc8xxx.c +++ b/drivers/gpio/gpio-mpc8xxx.c @@ -32,7 +32,7 @@ struct mpc8xxx_gpio_chip { struct of_mm_gpio_chip mm_gc; - spinlock_t lock; + raw_spinlock_t lock; /* * shadowed data register to be able to clear/set output pins in @@ -95,7 +95,7 @@ static void mpc8xxx_gpio_set(struct gpio_chip *gc, unsigned int gpio, int val) struct mpc8xxx_gpio_chip *mpc8xxx_gc = to_mpc8xxx_gpio_chip(mm); unsigned long flags; - spin_lock_irqsave(&mpc8xxx_gc->lock, flags); + raw_spin_lock_irqsave(&mpc8xxx_gc->lock, flags); if (val) mpc8xxx_gc->data |= mpc8xxx_gpio2mask(gpio); @@ -104,7 +104,7 @@ static void mpc8xxx_gpio_set(struct gpio_chip *gc, unsigned int gpio, int val) out_be32(mm->regs + GPIO_DAT, mpc8xxx_gc->data); - spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); + raw_spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); } static void mpc8xxx_gpio_set_multiple(struct gpio_chip *gc, @@ -115,7 +115,7 @@ static void mpc8xxx_gpio_set_multiple(struct gpio_chip *gc, unsigned long flags; int i; - spin_lock_irqsave(&mpc8xxx_gc->lock, flags); + raw_spin_lock_irqsave(&mpc8xxx_gc->lock, flags); for (i = 0; i < gc->ngpio; i++) { if (*mask == 0) @@ -130,7 +130,7 @@ static void mpc8xxx_gpio_set_multiple(struct gpio_chip *gc, out_be32(mm->regs + GPIO_DAT, mpc8xxx_gc->data); - spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); + raw_spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); } static int mpc8xxx_gpio_dir_in(struct gpio_chip *gc, unsigned int gpio) @@ -139,11 +139,11 @@ static int mpc8xxx_gpio_dir_in(struct gpio_chip *gc, unsigned int gpio) struct mpc8xxx_gpio_chip *mpc8xxx_gc = to_mpc8xxx_gpio_chip(mm); unsigned long flags; - spin_lock_irqsave(&mpc8xxx_gc->lock, flags); + raw_spin_lock_irqsave(&mpc8xxx_gc->lock, flags); clrbits32(mm->regs + GPIO_DIR, mpc8xxx_gpio2mask(gpio)); - spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); + raw_spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); return 0; } @@ -156,11 +156,11 @@ static int mpc8xxx_gpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val mpc8xxx_gpio_set(gc, gpio, val); - spin_lock_irqsave(&mpc8xxx_gc->lock, flags); + raw_spin_lock_irqsave(&mpc8xxx_gc->lock, flags); setbits32(mm->regs + GPIO_DIR, mpc8xxx_gpio2mask(gpio)); - spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); + raw_spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); return 0; } @@ -174,6 +174,15 @@ static int mpc5121_gpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val return mpc8xxx_gpio_dir_out(gc, gpio, val); } +static int mpc5125_gpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) +{ + /* GPIO 0..3 are input only on MPC5125 */ + if (gpio <= 3) + return -EINVAL; + + return mpc8xxx_gpio_dir_out(gc, gpio, val); +} + static int mpc8xxx_gpio_to_irq(struct gpio_chip *gc, unsigned offset) { struct of_mm_gpio_chip *mm = to_of_mm_gpio_chip(gc); @@ -206,11 +215,11 @@ static void mpc8xxx_irq_unmask(struct irq_data *d) struct of_mm_gpio_chip *mm = &mpc8xxx_gc->mm_gc; unsigned long flags; - spin_lock_irqsave(&mpc8xxx_gc->lock, flags); + raw_spin_lock_irqsave(&mpc8xxx_gc->lock, flags); setbits32(mm->regs + GPIO_IMR, mpc8xxx_gpio2mask(irqd_to_hwirq(d))); - spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); + raw_spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); } static void mpc8xxx_irq_mask(struct irq_data *d) @@ -219,11 +228,11 @@ static void mpc8xxx_irq_mask(struct irq_data *d) struct of_mm_gpio_chip *mm = &mpc8xxx_gc->mm_gc; unsigned long flags; - spin_lock_irqsave(&mpc8xxx_gc->lock, flags); + raw_spin_lock_irqsave(&mpc8xxx_gc->lock, flags); clrbits32(mm->regs + GPIO_IMR, mpc8xxx_gpio2mask(irqd_to_hwirq(d))); - spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); + raw_spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); } static void mpc8xxx_irq_ack(struct irq_data *d) @@ -242,17 +251,17 @@ static int mpc8xxx_irq_set_type(struct irq_data *d, unsigned int flow_type) switch (flow_type) { case IRQ_TYPE_EDGE_FALLING: - spin_lock_irqsave(&mpc8xxx_gc->lock, flags); + raw_spin_lock_irqsave(&mpc8xxx_gc->lock, flags); setbits32(mm->regs + GPIO_ICR, mpc8xxx_gpio2mask(irqd_to_hwirq(d))); - spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); + raw_spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); break; case IRQ_TYPE_EDGE_BOTH: - spin_lock_irqsave(&mpc8xxx_gc->lock, flags); + raw_spin_lock_irqsave(&mpc8xxx_gc->lock, flags); clrbits32(mm->regs + GPIO_ICR, mpc8xxx_gpio2mask(irqd_to_hwirq(d))); - spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); + raw_spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); break; default: @@ -282,22 +291,22 @@ static int mpc512x_irq_set_type(struct irq_data *d, unsigned int flow_type) switch (flow_type) { case IRQ_TYPE_EDGE_FALLING: case IRQ_TYPE_LEVEL_LOW: - spin_lock_irqsave(&mpc8xxx_gc->lock, flags); + raw_spin_lock_irqsave(&mpc8xxx_gc->lock, flags); clrsetbits_be32(reg, 3 << shift, 2 << shift); - spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); + raw_spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); break; case IRQ_TYPE_EDGE_RISING: case IRQ_TYPE_LEVEL_HIGH: - spin_lock_irqsave(&mpc8xxx_gc->lock, flags); + raw_spin_lock_irqsave(&mpc8xxx_gc->lock, flags); clrsetbits_be32(reg, 3 << shift, 1 << shift); - spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); + raw_spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); break; case IRQ_TYPE_EDGE_BOTH: - spin_lock_irqsave(&mpc8xxx_gc->lock, flags); + raw_spin_lock_irqsave(&mpc8xxx_gc->lock, flags); clrbits32(reg, 3 << shift); - spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); + raw_spin_unlock_irqrestore(&mpc8xxx_gc->lock, flags); break; default: @@ -312,17 +321,13 @@ static struct irq_chip mpc8xxx_irq_chip = { .irq_unmask = mpc8xxx_irq_unmask, .irq_mask = mpc8xxx_irq_mask, .irq_ack = mpc8xxx_irq_ack, + /* this might get overwritten in mpc8xxx_probe() */ .irq_set_type = mpc8xxx_irq_set_type, }; static int mpc8xxx_gpio_irq_map(struct irq_domain *h, unsigned int irq, irq_hw_number_t hwirq) { - struct mpc8xxx_gpio_chip *mpc8xxx_gc = h->host_data; - - if (mpc8xxx_gc->of_dev_id_data) - mpc8xxx_irq_chip.irq_set_type = mpc8xxx_gc->of_dev_id_data; - irq_set_chip_data(irq, h->host_data); irq_set_chip_and_handler(irq, &mpc8xxx_irq_chip, handle_level_irq); @@ -334,11 +339,38 @@ static const struct irq_domain_ops mpc8xxx_gpio_irq_ops = { .xlate = irq_domain_xlate_twocell, }; -static struct of_device_id mpc8xxx_gpio_ids[] = { +struct mpc8xxx_gpio_devtype { + int (*gpio_dir_out)(struct gpio_chip *, unsigned int, int); + int (*gpio_get)(struct gpio_chip *, unsigned int); + int (*irq_set_type)(struct irq_data *, unsigned int); +}; + +static const struct mpc8xxx_gpio_devtype mpc512x_gpio_devtype = { + .gpio_dir_out = mpc5121_gpio_dir_out, + .irq_set_type = mpc512x_irq_set_type, +}; + +static const struct mpc8xxx_gpio_devtype mpc5125_gpio_devtype = { + .gpio_dir_out = mpc5125_gpio_dir_out, + .irq_set_type = mpc512x_irq_set_type, +}; + +static const struct mpc8xxx_gpio_devtype mpc8572_gpio_devtype = { + .gpio_get = mpc8572_gpio_get, +}; + +static const struct mpc8xxx_gpio_devtype mpc8xxx_gpio_devtype_default = { + .gpio_dir_out = mpc8xxx_gpio_dir_out, + .gpio_get = mpc8xxx_gpio_get, + .irq_set_type = mpc8xxx_irq_set_type, +}; + +static const struct of_device_id mpc8xxx_gpio_ids[] = { { .compatible = "fsl,mpc8349-gpio", }, - { .compatible = "fsl,mpc8572-gpio", }, + { .compatible = "fsl,mpc8572-gpio", .data = &mpc8572_gpio_devtype, }, { .compatible = "fsl,mpc8610-gpio", }, - { .compatible = "fsl,mpc5121-gpio", .data = mpc512x_irq_set_type, }, + { .compatible = "fsl,mpc5121-gpio", .data = &mpc512x_gpio_devtype, }, + { .compatible = "fsl,mpc5125-gpio", .data = &mpc5125_gpio_devtype, }, { .compatible = "fsl,pq3-gpio", }, { .compatible = "fsl,qoriq-gpio", }, {} @@ -351,6 +383,8 @@ static int mpc8xxx_probe(struct platform_device *pdev) struct of_mm_gpio_chip *mm_gc; struct gpio_chip *gc; const struct of_device_id *id; + const struct mpc8xxx_gpio_devtype *devtype = + of_device_get_match_data(&pdev->dev); int ret; mpc8xxx_gc = devm_kzalloc(&pdev->dev, sizeof(*mpc8xxx_gc), GFP_KERNEL); @@ -359,7 +393,7 @@ static int mpc8xxx_probe(struct platform_device *pdev) platform_set_drvdata(pdev, mpc8xxx_gc); - spin_lock_init(&mpc8xxx_gc->lock); + raw_spin_lock_init(&mpc8xxx_gc->lock); mm_gc = &mpc8xxx_gc->mm_gc; gc = &mm_gc->gc; @@ -367,10 +401,18 @@ static int mpc8xxx_probe(struct platform_device *pdev) mm_gc->save_regs = mpc8xxx_gpio_save_regs; gc->ngpio = MPC8XXX_GPIO_PINS; gc->direction_input = mpc8xxx_gpio_dir_in; - gc->direction_output = of_device_is_compatible(np, "fsl,mpc5121-gpio") ? - mpc5121_gpio_dir_out : mpc8xxx_gpio_dir_out; - gc->get = of_device_is_compatible(np, "fsl,mpc8572-gpio") ? - mpc8572_gpio_get : mpc8xxx_gpio_get; + + if (!devtype) + devtype = &mpc8xxx_gpio_devtype_default; + + /* + * It's assumed that only a single type of gpio controller is available + * on the current machine, so overwriting global data is fine. + */ + mpc8xxx_irq_chip.irq_set_type = devtype->irq_set_type; + + gc->direction_output = devtype->gpio_dir_out ?: mpc8xxx_gpio_dir_out; + gc->get = devtype->gpio_get ?: mpc8xxx_gpio_get; gc->set = mpc8xxx_gpio_set; gc->set_multiple = mpc8xxx_gpio_set_multiple; gc->to_irq = mpc8xxx_gpio_to_irq; @@ -396,8 +438,8 @@ static int mpc8xxx_probe(struct platform_device *pdev) out_be32(mm_gc->regs + GPIO_IER, 0xffffffff); out_be32(mm_gc->regs + GPIO_IMR, 0); - irq_set_handler_data(mpc8xxx_gc->irqn, mpc8xxx_gc); - irq_set_chained_handler(mpc8xxx_gc->irqn, mpc8xxx_gpio_irq_cascade); + irq_set_chained_handler_and_data(mpc8xxx_gc->irqn, + mpc8xxx_gpio_irq_cascade, mpc8xxx_gc); return 0; } @@ -407,8 +449,7 @@ static int mpc8xxx_remove(struct platform_device *pdev) struct mpc8xxx_gpio_chip *mpc8xxx_gc = platform_get_drvdata(pdev); if (mpc8xxx_gc->irq) { - irq_set_handler_data(mpc8xxx_gc->irqn, NULL); - irq_set_chained_handler(mpc8xxx_gc->irqn, NULL); + irq_set_chained_handler_and_data(mpc8xxx_gc->irqn, NULL, NULL); irq_domain_remove(mpc8xxx_gc->irq); } diff --git a/drivers/gpio/gpio-msm-v2.c b/drivers/gpio/gpio-msm-v2.c index 52ff18229fdc..d2012cfb5571 100644 --- a/drivers/gpio/gpio-msm-v2.c +++ b/drivers/gpio/gpio-msm-v2.c @@ -187,14 +187,6 @@ static int msm_gpio_to_irq(struct gpio_chip *chip, unsigned offset) return irq_create_mapping(domain, offset); } -static inline int msm_irq_to_gpio(struct gpio_chip *chip, unsigned irq) -{ - struct irq_data *irq_data = irq_get_irq_data(irq); - - return irq_data->hwirq; -} - - /* For dual-edge interrupts in software, since the hardware has no * such support: * @@ -238,7 +230,7 @@ static void msm_gpio_update_dual_edge_pos(unsigned gpio) static void msm_gpio_irq_ack(struct irq_data *d) { - int gpio = msm_irq_to_gpio(&msm_gpio.gpio_chip, d->irq); + int gpio = d->hwirq; writel(BIT(INTR_STATUS), GPIO_INTR_STATUS(gpio)); if (test_bit(gpio, msm_gpio.dual_edge_irqs)) @@ -247,8 +239,8 @@ static void msm_gpio_irq_ack(struct irq_data *d) static void msm_gpio_irq_mask(struct irq_data *d) { - int gpio = msm_irq_to_gpio(&msm_gpio.gpio_chip, d->irq); unsigned long irq_flags; + int gpio = d->hwirq; spin_lock_irqsave(&tlmm_lock, irq_flags); writel(TARGET_PROC_NONE, GPIO_INTR_CFG_SU(gpio)); @@ -259,8 +251,8 @@ static void msm_gpio_irq_mask(struct irq_data *d) static void msm_gpio_irq_unmask(struct irq_data *d) { - int gpio = msm_irq_to_gpio(&msm_gpio.gpio_chip, d->irq); unsigned long irq_flags; + int gpio = d->hwirq; spin_lock_irqsave(&tlmm_lock, irq_flags); __set_bit(gpio, msm_gpio.enabled_irqs); @@ -271,8 +263,8 @@ static void msm_gpio_irq_unmask(struct irq_data *d) static int msm_gpio_irq_set_type(struct irq_data *d, unsigned int flow_type) { - int gpio = msm_irq_to_gpio(&msm_gpio.gpio_chip, d->irq); unsigned long irq_flags; + int gpio = d->hwirq; uint32_t bits; spin_lock_irqsave(&tlmm_lock, irq_flags); @@ -281,14 +273,14 @@ static int msm_gpio_irq_set_type(struct irq_data *d, unsigned int flow_type) if (flow_type & IRQ_TYPE_EDGE_BOTH) { bits |= BIT(INTR_DECT_CTL); - __irq_set_handler_locked(d->irq, handle_edge_irq); + irq_set_handler_locked(d, handle_edge_irq); if ((flow_type & IRQ_TYPE_EDGE_BOTH) == IRQ_TYPE_EDGE_BOTH) __set_bit(gpio, msm_gpio.dual_edge_irqs); else __clear_bit(gpio, msm_gpio.dual_edge_irqs); } else { bits &= ~BIT(INTR_DECT_CTL); - __irq_set_handler_locked(d->irq, handle_level_irq); + irq_set_handler_locked(d, handle_level_irq); __clear_bit(gpio, msm_gpio.dual_edge_irqs); } @@ -331,7 +323,7 @@ static void msm_summary_irq_handler(unsigned int irq, struct irq_desc *desc) static int msm_gpio_irq_set_wake(struct irq_data *d, unsigned int on) { - int gpio = msm_irq_to_gpio(&msm_gpio.gpio_chip, d->irq); + int gpio = d->hwirq; if (on) { if (bitmap_empty(msm_gpio.wake_irqs, MAX_NR_GPIO)) @@ -363,7 +355,6 @@ static int msm_gpio_irq_domain_map(struct irq_domain *d, unsigned int irq, irq_set_lockdep_class(irq, &msm_gpio_lock_class); irq_set_chip_and_handler(irq, &msm_gpio_irq_chip, handle_level_irq); - set_irq_flags(irq, IRQF_VALID); return 0; } diff --git a/drivers/gpio/gpio-mvebu.c b/drivers/gpio/gpio-mvebu.c index 1a54205860f5..b396bf3bf294 100644 --- a/drivers/gpio/gpio-mvebu.c +++ b/drivers/gpio/gpio-mvebu.c @@ -458,9 +458,9 @@ static int mvebu_gpio_irq_set_type(struct irq_data *d, unsigned int type) return 0; } -static void mvebu_gpio_irq_handler(unsigned int irq, struct irq_desc *desc) +static void mvebu_gpio_irq_handler(unsigned int __irq, struct irq_desc *desc) { - struct mvebu_gpio_chip *mvchip = irq_get_handler_data(irq); + struct mvebu_gpio_chip *mvchip = irq_desc_get_handler_data(desc); struct irq_chip *chip = irq_desc_get_chip(desc); u32 cause, type; int i; @@ -787,8 +787,8 @@ static int mvebu_gpio_probe(struct platform_device *pdev) if (irq < 0) continue; - irq_set_handler_data(irq, mvchip); - irq_set_chained_handler(irq, mvebu_gpio_irq_handler); + irq_set_chained_handler_and_data(irq, mvebu_gpio_irq_handler, + mvchip); } mvchip->irqbase = irq_alloc_descs(-1, 0, ngpios, -1); diff --git a/drivers/gpio/gpio-mxc.c b/drivers/gpio/gpio-mxc.c index ec1eb1b7250f..b752b560126e 100644 --- a/drivers/gpio/gpio-mxc.c +++ b/drivers/gpio/gpio-mxc.c @@ -275,8 +275,8 @@ static void mxc_gpio_irq_handler(struct mxc_gpio_port *port, u32 irq_stat) static void mx3_gpio_irq_handler(u32 irq, struct irq_desc *desc) { u32 irq_stat; - struct mxc_gpio_port *port = irq_get_handler_data(irq); - struct irq_chip *chip = irq_get_chip(irq); + struct mxc_gpio_port *port = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); chained_irq_enter(chip, desc); @@ -292,7 +292,7 @@ static void mx2_gpio_irq_handler(u32 irq, struct irq_desc *desc) { u32 irq_msk, irq_stat; struct mxc_gpio_port *port; - struct irq_chip *chip = irq_get_chip(irq); + struct irq_chip *chip = irq_desc_get_chip(desc); chained_irq_enter(chip, desc); @@ -339,7 +339,7 @@ static int gpio_set_wake_irq(struct irq_data *d, u32 enable) return 0; } -static void __init mxc_gpio_init_gc(struct mxc_gpio_port *port, int irq_base) +static void mxc_gpio_init_gc(struct mxc_gpio_port *port, int irq_base) { struct irq_chip_generic *gc; struct irq_chip_type *ct; @@ -354,6 +354,7 @@ static void __init mxc_gpio_init_gc(struct mxc_gpio_port *port, int irq_base) ct->chip.irq_unmask = irq_gc_mask_set_bit; ct->chip.irq_set_type = gpio_set_irq_type; ct->chip.irq_set_wake = gpio_set_wake_irq; + ct->chip.flags = IRQCHIP_MASK_ON_SUSPEND; ct->regs.ack = GPIO_ISR; ct->regs.mask = GPIO_IMR; diff --git a/drivers/gpio/gpio-mxs.c b/drivers/gpio/gpio-mxs.c index 551d15d7c369..b7f383eb18d9 100644 --- a/drivers/gpio/gpio-mxs.c +++ b/drivers/gpio/gpio-mxs.c @@ -157,7 +157,7 @@ static void mxs_flip_edge(struct mxs_gpio_port *port, u32 gpio) static void mxs_gpio_irq_handler(u32 irq, struct irq_desc *desc) { u32 irq_stat; - struct mxs_gpio_port *port = irq_get_handler_data(irq); + struct mxs_gpio_port *port = irq_desc_get_handler_data(desc); desc->irq_data.chip->irq_ack(&desc->irq_data); diff --git a/drivers/gpio/gpio-omap.c b/drivers/gpio/gpio-omap.c index 61a731ff9a07..2ae0d47e9554 100644 --- a/drivers/gpio/gpio-omap.c +++ b/drivers/gpio/gpio-omap.c @@ -29,6 +29,7 @@ #include #define OFF_MODE 1 +#define OMAP4_GPIO_DEBOUNCINGTIME_MASK 0xFF static LIST_HEAD(omap_gpio_list); @@ -57,7 +58,7 @@ struct gpio_bank { u32 saved_datain; u32 level_mask; u32 toggle_mask; - spinlock_t lock; + raw_spinlock_t lock; struct gpio_chip chip; struct clk *dbck; u32 mod_usage; @@ -175,7 +176,7 @@ static inline void omap_gpio_rmw(void __iomem *base, u32 reg, u32 mask, bool set static inline void omap_gpio_dbck_enable(struct gpio_bank *bank) { if (bank->dbck_enable_mask && !bank->dbck_enabled) { - clk_prepare_enable(bank->dbck); + clk_enable(bank->dbck); bank->dbck_enabled = true; writel_relaxed(bank->dbck_enable_mask, @@ -193,7 +194,7 @@ static inline void omap_gpio_dbck_disable(struct gpio_bank *bank) */ writel_relaxed(0, bank->base + bank->regs->debounce_en); - clk_disable_unprepare(bank->dbck); + clk_disable(bank->dbck); bank->dbck_enabled = false; } } @@ -204,8 +205,9 @@ static inline void omap_gpio_dbck_disable(struct gpio_bank *bank) * @offset: the gpio number on this @bank * @debounce: debounce time to use * - * OMAP's debounce time is in 31us steps so we need - * to convert and round up to the closest unit. + * OMAP's debounce time is in 31us steps + * = (GPIO_DEBOUNCINGTIME[7:0].DEBOUNCETIME + 1) x 31 + * so we need to convert and round up to the closest unit. */ static void omap2_set_gpio_debounce(struct gpio_bank *bank, unsigned offset, unsigned debounce) @@ -213,34 +215,33 @@ static void omap2_set_gpio_debounce(struct gpio_bank *bank, unsigned offset, void __iomem *reg; u32 val; u32 l; + bool enable = !!debounce; if (!bank->dbck_flag) return; - if (debounce < 32) - debounce = 0x01; - else if (debounce > 7936) - debounce = 0xff; - else - debounce = (debounce / 0x1f) - 1; + if (enable) { + debounce = DIV_ROUND_UP(debounce, 31) - 1; + debounce &= OMAP4_GPIO_DEBOUNCINGTIME_MASK; + } l = BIT(offset); - clk_prepare_enable(bank->dbck); + clk_enable(bank->dbck); reg = bank->base + bank->regs->debounce; writel_relaxed(debounce, reg); reg = bank->base + bank->regs->debounce_en; val = readl_relaxed(reg); - if (debounce) + if (enable) val |= l; else val &= ~l; bank->dbck_enable_mask = val; writel_relaxed(val, reg); - clk_disable_unprepare(bank->dbck); + clk_disable(bank->dbck); /* * Enable debounce clock per module. * This call is mandatory because in omap_gpio_request() when @@ -285,7 +286,7 @@ static void omap_clear_gpio_debounce(struct gpio_bank *bank, unsigned offset) bank->context.debounce = 0; writel_relaxed(bank->context.debounce, bank->base + bank->regs->debounce); - clk_disable_unprepare(bank->dbck); + clk_disable(bank->dbck); bank->dbck_enabled = false; } } @@ -498,24 +499,24 @@ static int omap_gpio_irq_type(struct irq_data *d, unsigned type) if (!BANK_USED(bank)) pm_runtime_get_sync(bank->dev); - spin_lock_irqsave(&bank->lock, flags); + raw_spin_lock_irqsave(&bank->lock, flags); retval = omap_set_gpio_triggering(bank, offset, type); if (retval) { - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); goto error; } omap_gpio_init_irq(bank, offset); if (!omap_gpio_is_input(bank, offset)) { - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); retval = -EINVAL; goto error; } - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); if (type & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_LEVEL_HIGH)) - __irq_set_handler_locked(d->irq, handle_level_irq); + irq_set_handler_locked(d, handle_level_irq); else if (type & (IRQ_TYPE_EDGE_FALLING | IRQ_TYPE_EDGE_RISING)) - __irq_set_handler_locked(d->irq, handle_edge_irq); + irq_set_handler_locked(d, handle_edge_irq); return 0; @@ -636,14 +637,14 @@ static int omap_set_gpio_wakeup(struct gpio_bank *bank, unsigned offset, return -EINVAL; } - spin_lock_irqsave(&bank->lock, flags); + raw_spin_lock_irqsave(&bank->lock, flags); if (enable) bank->context.wake_en |= gpio_bit; else bank->context.wake_en &= ~gpio_bit; writel_relaxed(bank->context.wake_en, bank->base + bank->regs->wkup_en); - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); return 0; } @@ -669,10 +670,10 @@ static int omap_gpio_request(struct gpio_chip *chip, unsigned offset) if (!BANK_USED(bank)) pm_runtime_get_sync(bank->dev); - spin_lock_irqsave(&bank->lock, flags); + raw_spin_lock_irqsave(&bank->lock, flags); omap_enable_gpio_module(bank, offset); bank->mod_usage |= BIT(offset); - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); return 0; } @@ -682,14 +683,14 @@ static void omap_gpio_free(struct gpio_chip *chip, unsigned offset) struct gpio_bank *bank = container_of(chip, struct gpio_bank, chip); unsigned long flags; - spin_lock_irqsave(&bank->lock, flags); + raw_spin_lock_irqsave(&bank->lock, flags); bank->mod_usage &= ~(BIT(offset)); if (!LINE_USED(bank->irq_usage, offset)) { omap_set_gpio_direction(bank, offset, 1); omap_clear_gpio_debounce(bank, offset); } omap_disable_gpio_module(bank, offset); - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); /* * If this is the last gpio to be freed in the bank, @@ -716,7 +717,8 @@ static void omap_gpio_irq_handler(unsigned int irq, struct irq_desc *desc) struct gpio_bank *bank; int unmasked = 0; struct irq_chip *irqchip = irq_desc_get_chip(desc); - struct gpio_chip *chip = irq_get_handler_data(irq); + struct gpio_chip *chip = irq_desc_get_handler_data(desc); + unsigned long lock_flags; chained_irq_enter(irqchip, desc); @@ -731,6 +733,8 @@ static void omap_gpio_irq_handler(unsigned int irq, struct irq_desc *desc) u32 isr_saved, level_mask = 0; u32 enabled; + raw_spin_lock_irqsave(&bank->lock, lock_flags); + enabled = omap_get_gpio_irqbank_mask(bank); isr_saved = isr = readl_relaxed(isr_reg) & enabled; @@ -744,6 +748,8 @@ static void omap_gpio_irq_handler(unsigned int irq, struct irq_desc *desc) omap_clear_gpio_irqbank(bank, isr_saved & ~level_mask); omap_enable_gpio_irqbank(bank, isr_saved & ~level_mask); + raw_spin_unlock_irqrestore(&bank->lock, lock_flags); + /* if there is only edge sensitive GPIO pin interrupts configured, we could unmask GPIO bank interrupt immediately */ if (!level_mask && !unmasked) { @@ -758,6 +764,7 @@ static void omap_gpio_irq_handler(unsigned int irq, struct irq_desc *desc) bit = __ffs(isr); isr &= ~(BIT(bit)); + raw_spin_lock_irqsave(&bank->lock, lock_flags); /* * Some chips can't respond to both rising and falling * at the same time. If this irq was requested with @@ -768,6 +775,8 @@ static void omap_gpio_irq_handler(unsigned int irq, struct irq_desc *desc) if (bank->toggle_mask & (BIT(bit))) omap_toggle_gpio_edge_triggering(bank, bit); + raw_spin_unlock_irqrestore(&bank->lock, lock_flags); + generic_handle_irq(irq_find_mapping(bank->chip.irqdomain, bit)); } @@ -791,7 +800,7 @@ static unsigned int omap_gpio_irq_startup(struct irq_data *d) if (!BANK_USED(bank)) pm_runtime_get_sync(bank->dev); - spin_lock_irqsave(&bank->lock, flags); + raw_spin_lock_irqsave(&bank->lock, flags); if (!LINE_USED(bank->mod_usage, offset)) omap_set_gpio_direction(bank, offset, 1); @@ -800,12 +809,12 @@ static unsigned int omap_gpio_irq_startup(struct irq_data *d) omap_enable_gpio_module(bank, offset); bank->irq_usage |= BIT(offset); - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); omap_gpio_unmask_irq(d); return 0; err: - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); if (!BANK_USED(bank)) pm_runtime_put(bank->dev); return -EINVAL; @@ -817,7 +826,7 @@ static void omap_gpio_irq_shutdown(struct irq_data *d) unsigned long flags; unsigned offset = d->hwirq; - spin_lock_irqsave(&bank->lock, flags); + raw_spin_lock_irqsave(&bank->lock, flags); bank->irq_usage &= ~(BIT(offset)); omap_set_gpio_irqenable(bank, offset, 0); omap_clear_gpio_irqstatus(bank, offset); @@ -825,7 +834,7 @@ static void omap_gpio_irq_shutdown(struct irq_data *d) if (!LINE_USED(bank->mod_usage, offset)) omap_clear_gpio_debounce(bank, offset); omap_disable_gpio_module(bank, offset); - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); /* * If this is the last IRQ to be freed in the bank, @@ -849,10 +858,10 @@ static void omap_gpio_mask_irq(struct irq_data *d) unsigned offset = d->hwirq; unsigned long flags; - spin_lock_irqsave(&bank->lock, flags); + raw_spin_lock_irqsave(&bank->lock, flags); omap_set_gpio_irqenable(bank, offset, 0); omap_set_gpio_triggering(bank, offset, IRQ_TYPE_NONE); - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); } static void omap_gpio_unmask_irq(struct irq_data *d) @@ -862,7 +871,7 @@ static void omap_gpio_unmask_irq(struct irq_data *d) u32 trigger = irqd_get_trigger_type(d); unsigned long flags; - spin_lock_irqsave(&bank->lock, flags); + raw_spin_lock_irqsave(&bank->lock, flags); if (trigger) omap_set_gpio_triggering(bank, offset, trigger); @@ -874,7 +883,7 @@ static void omap_gpio_unmask_irq(struct irq_data *d) } omap_set_gpio_irqenable(bank, offset, 1); - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); } /*---------------------------------------------------------------------*/ @@ -887,9 +896,9 @@ static int omap_mpuio_suspend_noirq(struct device *dev) OMAP_MPUIO_GPIO_MASKIT / bank->stride; unsigned long flags; - spin_lock_irqsave(&bank->lock, flags); + raw_spin_lock_irqsave(&bank->lock, flags); writel_relaxed(0xffff & ~bank->context.wake_en, mask_reg); - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); return 0; } @@ -902,9 +911,9 @@ static int omap_mpuio_resume_noirq(struct device *dev) OMAP_MPUIO_GPIO_MASKIT / bank->stride; unsigned long flags; - spin_lock_irqsave(&bank->lock, flags); + raw_spin_lock_irqsave(&bank->lock, flags); writel_relaxed(bank->context.wake_en, mask_reg); - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); return 0; } @@ -950,9 +959,9 @@ static int omap_gpio_get_direction(struct gpio_chip *chip, unsigned offset) bank = container_of(chip, struct gpio_bank, chip); reg = bank->base + bank->regs->direction; - spin_lock_irqsave(&bank->lock, flags); + raw_spin_lock_irqsave(&bank->lock, flags); dir = !!(readl_relaxed(reg) & BIT(offset)); - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); return dir; } @@ -962,9 +971,9 @@ static int omap_gpio_input(struct gpio_chip *chip, unsigned offset) unsigned long flags; bank = container_of(chip, struct gpio_bank, chip); - spin_lock_irqsave(&bank->lock, flags); + raw_spin_lock_irqsave(&bank->lock, flags); omap_set_gpio_direction(bank, offset, 1); - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); return 0; } @@ -986,10 +995,10 @@ static int omap_gpio_output(struct gpio_chip *chip, unsigned offset, int value) unsigned long flags; bank = container_of(chip, struct gpio_bank, chip); - spin_lock_irqsave(&bank->lock, flags); + raw_spin_lock_irqsave(&bank->lock, flags); bank->set_dataout(bank, offset, value); omap_set_gpio_direction(bank, offset, 0); - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); return 0; } @@ -1001,9 +1010,9 @@ static int omap_gpio_debounce(struct gpio_chip *chip, unsigned offset, bank = container_of(chip, struct gpio_bank, chip); - spin_lock_irqsave(&bank->lock, flags); + raw_spin_lock_irqsave(&bank->lock, flags); omap2_set_gpio_debounce(bank, offset, debounce); - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); return 0; } @@ -1014,9 +1023,9 @@ static void omap_gpio_set(struct gpio_chip *chip, unsigned offset, int value) unsigned long flags; bank = container_of(chip, struct gpio_bank, chip); - spin_lock_irqsave(&bank->lock, flags); + raw_spin_lock_irqsave(&bank->lock, flags); bank->set_dataout(bank, offset, value); - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); } /*---------------------------------------------------------------------*/ @@ -1061,10 +1070,6 @@ static void omap_gpio_mod_init(struct gpio_bank *bank) /* Initialize interface clk ungated, module enabled */ if (bank->regs->ctrl) writel_relaxed(0, base + bank->regs->ctrl); - - bank->dbck = clk_get(bank->dev, "dbclk"); - if (IS_ERR(bank->dbck)) - dev_err(bank->dev, "Could not get gpio dbck\n"); } static int omap_gpio_chip_init(struct gpio_bank *bank, struct irq_chip *irqc) @@ -1178,13 +1183,16 @@ static int omap_gpio_probe(struct platform_device *pdev) irqc->irq_set_wake = omap_gpio_wake_enable, irqc->name = dev_name(&pdev->dev); - res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (unlikely(!res)) { - dev_err(dev, "Invalid IRQ resource\n"); - return -ENODEV; + bank->irq = platform_get_irq(pdev, 0); + if (bank->irq <= 0) { + if (!bank->irq) + bank->irq = -ENXIO; + if (bank->irq != -EPROBE_DEFER) + dev_err(dev, + "can't get irq resource ret=%d\n", bank->irq); + return bank->irq; } - bank->irq = res->start; bank->dev = dev; bank->chip.dev = dev; bank->chip.owner = THIS_MODULE; @@ -1213,16 +1221,26 @@ static int omap_gpio_probe(struct platform_device *pdev) else bank->set_dataout = omap_set_gpio_dataout_mask; - spin_lock_init(&bank->lock); + raw_spin_lock_init(&bank->lock); /* Static mapping, never released */ res = platform_get_resource(pdev, IORESOURCE_MEM, 0); bank->base = devm_ioremap_resource(dev, res); if (IS_ERR(bank->base)) { - irq_domain_remove(bank->chip.irqdomain); return PTR_ERR(bank->base); } + if (bank->dbck_flag) { + bank->dbck = devm_clk_get(bank->dev, "dbclk"); + if (IS_ERR(bank->dbck)) { + dev_err(bank->dev, + "Could not get gpio dbck. Disable debounce\n"); + bank->dbck_flag = false; + } else { + clk_prepare(bank->dbck); + } + } + platform_set_drvdata(pdev, bank); pm_runtime_enable(bank->dev); @@ -1254,6 +1272,8 @@ static int omap_gpio_remove(struct platform_device *pdev) list_del(&bank->node); gpiochip_remove(&bank->chip); pm_runtime_disable(bank->dev); + if (bank->dbck_flag) + clk_unprepare(bank->dbck); return 0; } @@ -1271,7 +1291,7 @@ static int omap_gpio_runtime_suspend(struct device *dev) unsigned long flags; u32 wake_low, wake_hi; - spin_lock_irqsave(&bank->lock, flags); + raw_spin_lock_irqsave(&bank->lock, flags); /* * Only edges can generate a wakeup event to the PRCM. @@ -1324,7 +1344,7 @@ static int omap_gpio_runtime_suspend(struct device *dev) bank->get_context_loss_count(bank->dev); omap_gpio_dbck_disable(bank); - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); return 0; } @@ -1339,7 +1359,7 @@ static int omap_gpio_runtime_resume(struct device *dev) unsigned long flags; int c; - spin_lock_irqsave(&bank->lock, flags); + raw_spin_lock_irqsave(&bank->lock, flags); /* * On the first resume during the probe, the context has not @@ -1375,14 +1395,14 @@ static int omap_gpio_runtime_resume(struct device *dev) if (c != bank->context_loss_count) { omap_gpio_restore_context(bank); } else { - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); return 0; } } } if (!bank->workaround_enabled) { - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); return 0; } @@ -1437,7 +1457,7 @@ static int omap_gpio_runtime_resume(struct device *dev) } bank->workaround_enabled = false; - spin_unlock_irqrestore(&bank->lock, flags); + raw_spin_unlock_irqrestore(&bank->lock, flags); return 0; } diff --git a/drivers/gpio/gpio-pcf857x.c b/drivers/gpio/gpio-pcf857x.c index 404f3c61ef9b..1d4d9bc8b69d 100644 --- a/drivers/gpio/gpio-pcf857x.c +++ b/drivers/gpio/gpio-pcf857x.c @@ -88,7 +88,6 @@ struct pcf857x { struct gpio_chip chip; struct i2c_client *client; struct mutex lock; /* protect 'out' */ - spinlock_t slock; /* protect irq demux */ unsigned out; /* software latch */ unsigned status; /* current status */ unsigned int irq_parent; @@ -185,23 +184,21 @@ static void pcf857x_set(struct gpio_chip *chip, unsigned offset, int value) static irqreturn_t pcf857x_irq(int irq, void *data) { struct pcf857x *gpio = data; - unsigned long change, i, status, flags; + unsigned long change, i, status; status = gpio->read(gpio->client); - spin_lock_irqsave(&gpio->slock, flags); - /* * call the interrupt handler iff gpio is used as * interrupt source, just to avoid bad irqs */ - + mutex_lock(&gpio->lock); change = (gpio->status ^ status) & gpio->irq_enabled; + gpio->status = status; + mutex_unlock(&gpio->lock); + for_each_set_bit(i, &change, gpio->chip.ngpio) handle_nested_irq(irq_find_mapping(gpio->chip.irqdomain, i)); - gpio->status = status; - - spin_unlock_irqrestore(&gpio->slock, flags); return IRQ_HANDLED; } @@ -293,7 +290,6 @@ static int pcf857x_probe(struct i2c_client *client, return -ENOMEM; mutex_init(&gpio->lock); - spin_lock_init(&gpio->slock); gpio->chip.base = pdata ? pdata->gpio_base : -1; gpio->chip.can_sleep = true; diff --git a/drivers/gpio/gpio-pch.c b/drivers/gpio/gpio-pch.c index 2d9a950ca2d4..34ed176df15a 100644 --- a/drivers/gpio/gpio-pch.c +++ b/drivers/gpio/gpio-pch.c @@ -281,9 +281,9 @@ static int pch_irq_type(struct irq_data *d, unsigned int type) /* And the handler */ if (type & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_LEVEL_HIGH)) - __irq_set_handler_locked(d->irq, handle_level_irq); + irq_set_handler_locked(d, handle_level_irq); else if (type & (IRQ_TYPE_EDGE_FALLING | IRQ_TYPE_EDGE_RISING)) - __irq_set_handler_locked(d->irq, handle_edge_irq); + irq_set_handler_locked(d, handle_edge_irq); unlock: spin_unlock_irqrestore(&chip->spinlock, flags); diff --git a/drivers/gpio/gpio-pxa.c b/drivers/gpio/gpio-pxa.c index cdbbcf0faf9d..55a11de3d5b7 100644 --- a/drivers/gpio/gpio-pxa.c +++ b/drivers/gpio/gpio-pxa.c @@ -524,7 +524,7 @@ static int pxa_irq_domain_map(struct irq_domain *d, unsigned int irq, { irq_set_chip_and_handler(irq, &pxa_muxed_gpio_chip, handle_edge_irq); - set_irq_flags(irq, IRQF_VALID | IRQF_PROBE); + irq_set_noprobe(irq); return 0; } @@ -643,20 +643,20 @@ static int pxa_gpio_probe(struct platform_device *pdev) irq = gpio_to_irq(0); irq_set_chip_and_handler(irq, &pxa_muxed_gpio_chip, handle_edge_irq); - set_irq_flags(irq, IRQF_VALID | IRQF_PROBE); + irq_clear_status_flags(irq, IRQ_NOREQUEST | IRQ_NOPROBE); } if (irq1 > 0) { irq = gpio_to_irq(1); irq_set_chip_and_handler(irq, &pxa_muxed_gpio_chip, handle_edge_irq); - set_irq_flags(irq, IRQF_VALID | IRQF_PROBE); + irq_clear_status_flags(irq, IRQ_NOREQUEST | IRQ_NOPROBE); } for (irq = gpio_to_irq(gpio_offset); irq <= gpio_to_irq(pxa_last_gpio); irq++) { irq_set_chip_and_handler(irq, &pxa_muxed_gpio_chip, handle_edge_irq); - set_irq_flags(irq, IRQF_VALID | IRQF_PROBE); + irq_clear_status_flags(irq, IRQ_NOREQUEST | IRQ_NOPROBE); } } diff --git a/drivers/gpio/gpio-rcar.c b/drivers/gpio/gpio-rcar.c index 1e14a6c74ed1..2a8122444614 100644 --- a/drivers/gpio/gpio-rcar.c +++ b/drivers/gpio/gpio-rcar.c @@ -251,17 +251,32 @@ static void gpio_rcar_config_general_input_output_mode(struct gpio_chip *chip, static int gpio_rcar_request(struct gpio_chip *chip, unsigned offset) { - return pinctrl_request_gpio(chip->base + offset); + struct gpio_rcar_priv *p = gpio_to_priv(chip); + int error; + + error = pm_runtime_get_sync(&p->pdev->dev); + if (error < 0) + return error; + + error = pinctrl_request_gpio(chip->base + offset); + if (error) + pm_runtime_put(&p->pdev->dev); + + return error; } static void gpio_rcar_free(struct gpio_chip *chip, unsigned offset) { + struct gpio_rcar_priv *p = gpio_to_priv(chip); + pinctrl_free_gpio(chip->base + offset); /* Set the GPIO as an input to ensure that the next GPIO request won't * drive the GPIO pin as an output. */ gpio_rcar_config_general_input_output_mode(chip, offset, false); + + pm_runtime_put(&p->pdev->dev); } static int gpio_rcar_direction_input(struct gpio_chip *chip, unsigned offset) @@ -326,6 +341,10 @@ static const struct of_device_id gpio_rcar_of_table[] = { }, { .compatible = "renesas,gpio-r8a7794", .data = &gpio_rcar_info_gen2, + }, { + .compatible = "renesas,gpio-r8a7795", + /* Gen3 GPIO is identical to Gen2. */ + .data = &gpio_rcar_info_gen2, }, { .compatible = "renesas,gpio-rcar", .data = &gpio_rcar_info_gen1, @@ -405,7 +424,6 @@ static int gpio_rcar_probe(struct platform_device *pdev) } pm_runtime_enable(dev); - pm_runtime_get_sync(dev); io = platform_get_resource(pdev, IORESOURCE_MEM, 0); irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0); @@ -487,7 +505,6 @@ static int gpio_rcar_probe(struct platform_device *pdev) err1: gpiochip_remove(gpio_chip); err0: - pm_runtime_put(dev); pm_runtime_disable(dev); return ret; } @@ -498,7 +515,6 @@ static int gpio_rcar_remove(struct platform_device *pdev) gpiochip_remove(&p->gpio_chip); - pm_runtime_put(&pdev->dev); pm_runtime_disable(&pdev->dev); return 0; } diff --git a/drivers/gpio/gpio-sa1100.c b/drivers/gpio/gpio-sa1100.c index 3fa22dade243..67bd2f5d89e8 100644 --- a/drivers/gpio/gpio-sa1100.c +++ b/drivers/gpio/gpio-sa1100.c @@ -155,7 +155,7 @@ static int sa1100_gpio_irqdomain_map(struct irq_domain *d, { irq_set_chip_and_handler(irq, &sa1100_gpio_irq_chip, handle_edge_irq); - set_irq_flags(irq, IRQF_VALID | IRQF_PROBE); + irq_set_noprobe(irq); return 0; } @@ -173,9 +173,9 @@ static struct irq_domain *sa1100_gpio_irqdomain; * and call the handler. */ static void -sa1100_gpio_handler(unsigned int irq, struct irq_desc *desc) +sa1100_gpio_handler(unsigned int __irq, struct irq_desc *desc) { - unsigned int mask; + unsigned int irq, mask; mask = GEDR; do { diff --git a/drivers/gpio/gpio-sta2x11.c b/drivers/gpio/gpio-sta2x11.c index 18579ac65b2b..55e47828ddfc 100644 --- a/drivers/gpio/gpio-sta2x11.c +++ b/drivers/gpio/gpio-sta2x11.c @@ -346,7 +346,7 @@ static void gsta_alloc_irq_chip(struct gsta_gpio *chip) i = chip->irq_base + j; irq_set_chip_and_handler(i, &ct->chip, ct->handler); irq_set_chip_data(i, gc); - irq_modify_status(i, IRQ_NOREQUEST | IRQ_NOPROBE, 0); + irq_clear_status_flags(i, IRQ_NOREQUEST | IRQ_NOPROBE); } gc->irq_cnt = i - gc->irq_base; } diff --git a/drivers/gpio/gpio-tc3589x.c b/drivers/gpio/gpio-tc3589x.c index 31b244cffabb..d1d585ddb9ab 100644 --- a/drivers/gpio/gpio-tc3589x.c +++ b/drivers/gpio/gpio-tc3589x.c @@ -102,7 +102,7 @@ static struct gpio_chip template_chip = { static int tc3589x_gpio_irq_set_type(struct irq_data *d, unsigned int type) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct tc3589x_gpio *tc3589x_gpio = container_of(gc, struct tc3589x_gpio, chip); + struct tc3589x_gpio *tc3589x_gpio = to_tc3589x_gpio(gc); int offset = d->hwirq; int regoffset = offset / 8; int mask = 1 << (offset % 8); @@ -130,7 +130,7 @@ static int tc3589x_gpio_irq_set_type(struct irq_data *d, unsigned int type) static void tc3589x_gpio_irq_lock(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct tc3589x_gpio *tc3589x_gpio = container_of(gc, struct tc3589x_gpio, chip); + struct tc3589x_gpio *tc3589x_gpio = to_tc3589x_gpio(gc); mutex_lock(&tc3589x_gpio->irq_lock); } @@ -138,7 +138,7 @@ static void tc3589x_gpio_irq_lock(struct irq_data *d) static void tc3589x_gpio_irq_sync_unlock(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct tc3589x_gpio *tc3589x_gpio = container_of(gc, struct tc3589x_gpio, chip); + struct tc3589x_gpio *tc3589x_gpio = to_tc3589x_gpio(gc); struct tc3589x *tc3589x = tc3589x_gpio->tc3589x; static const u8 regmap[] = { [REG_IBE] = TC3589x_GPIOIBE0, @@ -167,7 +167,7 @@ static void tc3589x_gpio_irq_sync_unlock(struct irq_data *d) static void tc3589x_gpio_irq_mask(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct tc3589x_gpio *tc3589x_gpio = container_of(gc, struct tc3589x_gpio, chip); + struct tc3589x_gpio *tc3589x_gpio = to_tc3589x_gpio(gc); int offset = d->hwirq; int regoffset = offset / 8; int mask = 1 << (offset % 8); @@ -178,7 +178,7 @@ static void tc3589x_gpio_irq_mask(struct irq_data *d) static void tc3589x_gpio_irq_unmask(struct irq_data *d) { struct gpio_chip *gc = irq_data_get_irq_chip_data(d); - struct tc3589x_gpio *tc3589x_gpio = container_of(gc, struct tc3589x_gpio, chip); + struct tc3589x_gpio *tc3589x_gpio = to_tc3589x_gpio(gc); int offset = d->hwirq; int regoffset = offset / 8; int mask = 1 << (offset % 8); diff --git a/drivers/gpio/gpio-tegra.c b/drivers/gpio/gpio-tegra.c index 9b25c90f725c..9b14aafb576d 100644 --- a/drivers/gpio/gpio-tegra.c +++ b/drivers/gpio/gpio-tegra.c @@ -252,9 +252,9 @@ static int tegra_gpio_irq_set_type(struct irq_data *d, unsigned int type) tegra_gpio_enable(gpio); if (type & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_LEVEL_HIGH)) - __irq_set_handler_locked(d->irq, handle_level_irq); + irq_set_handler_locked(d, handle_level_irq); else if (type & (IRQ_TYPE_EDGE_FALLING | IRQ_TYPE_EDGE_RISING)) - __irq_set_handler_locked(d->irq, handle_edge_irq); + irq_set_handler_locked(d, handle_edge_irq); return 0; } @@ -268,16 +268,14 @@ static void tegra_gpio_irq_shutdown(struct irq_data *d) static void tegra_gpio_irq_handler(unsigned int irq, struct irq_desc *desc) { - struct tegra_gpio_bank *bank; int port; int pin; int unmasked = 0; struct irq_chip *chip = irq_desc_get_chip(desc); + struct tegra_gpio_bank *bank = irq_desc_get_handler_data(desc); chained_irq_enter(chip, desc); - bank = irq_get_handler_data(irq); - for (port = 0; port < 4; port++) { int gpio = tegra_gpio_compose(bank->bank, port, 0); unsigned long sta = tegra_gpio_readl(GPIO_INT_STA(gpio)) & @@ -509,7 +507,6 @@ static int tegra_gpio_probe(struct platform_device *pdev) irq_set_chip_data(irq, bank); irq_set_chip_and_handler(irq, &tegra_gpio_irq_chip, handle_simple_irq); - set_irq_flags(irq, IRQF_VALID); } for (i = 0; i < tegra_gpio_bank_count; i++) { diff --git a/drivers/gpio/gpio-timberdale.c b/drivers/gpio/gpio-timberdale.c index e8f97e03c9bb..5a492054589f 100644 --- a/drivers/gpio/gpio-timberdale.c +++ b/drivers/gpio/gpio-timberdale.c @@ -194,11 +194,12 @@ static int timbgpio_irq_type(struct irq_data *d, unsigned trigger) static void timbgpio_irq(unsigned int irq, struct irq_desc *desc) { - struct timbgpio *tgpio = irq_get_handler_data(irq); + struct timbgpio *tgpio = irq_desc_get_handler_data(desc); + struct irq_data *data = irq_desc_get_irq_data(desc); unsigned long ipr; int offset; - desc->irq_data.chip->irq_ack(irq_get_irq_data(irq)); + data->chip->irq_ack(data); ipr = ioread32(tgpio->membase + TGPIO_IPR); iowrite32(ipr, tgpio->membase + TGPIO_ICR); @@ -294,13 +295,10 @@ static int timbgpio_probe(struct platform_device *pdev) irq_set_chip_and_handler(tgpio->irq_base + i, &timbgpio_irqchip, handle_simple_irq); irq_set_chip_data(tgpio->irq_base + i, tgpio); -#ifdef CONFIG_ARM - set_irq_flags(tgpio->irq_base + i, IRQF_VALID | IRQF_PROBE); -#endif + irq_clear_status_flags(tgpio->irq_base + i, IRQ_NOREQUEST | IRQ_NOPROBE); } - irq_set_handler_data(irq, tgpio); - irq_set_chained_handler(irq, timbgpio_irq); + irq_set_chained_handler_and_data(irq, timbgpio_irq, tgpio); return 0; } diff --git a/drivers/gpio/gpio-tz1090.c b/drivers/gpio/gpio-tz1090.c index 445660adc898..bbac92ae4c32 100644 --- a/drivers/gpio/gpio-tz1090.c +++ b/drivers/gpio/gpio-tz1090.c @@ -510,8 +510,8 @@ static int tz1090_gpio_bank_probe(struct tz1090_gpio_bank_info *info) gc->chip_types[1].chip.flags = IRQCHIP_MASK_ON_SUSPEND; /* Setup chained handler for this GPIO bank */ - irq_set_handler_data(bank->irq, bank); - irq_set_chained_handler(bank->irq, tz1090_gpio_irq_handler); + irq_set_chained_handler_and_data(bank->irq, tz1090_gpio_irq_handler, + bank); return 0; } diff --git a/drivers/gpio/gpio-vf610.c b/drivers/gpio/gpio-vf610.c index 7bd9f209ffa8..3d5714d4f405 100644 --- a/drivers/gpio/gpio-vf610.c +++ b/drivers/gpio/gpio-vf610.c @@ -60,6 +60,8 @@ struct vf610_gpio_port { #define PORT_INT_EITHER_EDGE 0xb #define PORT_INT_LOGIC_ONE 0xc +static struct irq_chip vf610_gpio_irq_chip; + static const struct of_device_id vf610_gpio_dt_ids[] = { { .compatible = "fsl,vf610-gpio" }, { /* sentinel */ } @@ -120,7 +122,7 @@ static int vf610_gpio_direction_output(struct gpio_chip *chip, unsigned gpio, static void vf610_gpio_irq_handler(u32 irq, struct irq_desc *desc) { - struct vf610_gpio_port *port = irq_get_handler_data(irq); + struct vf610_gpio_port *port = irq_desc_get_handler_data(desc); struct irq_chip *chip = irq_desc_get_chip(desc); int pin; unsigned long irq_isfr; @@ -173,6 +175,11 @@ static int vf610_gpio_irq_set_type(struct irq_data *d, u32 type) port->irqc[d->hwirq] = irqc; + if (type & IRQ_TYPE_LEVEL_MASK) + __irq_set_handler_locked(d->irq, handle_level_irq); + else + __irq_set_handler_locked(d->irq, handle_edge_irq); + return 0; } @@ -263,7 +270,7 @@ static int vf610_gpio_probe(struct platform_device *pdev) vf610_gpio_writel(~0, port->base + PORT_ISFR); ret = gpiochip_irqchip_add(gc, &vf610_gpio_irq_chip, 0, - handle_simple_irq, IRQ_TYPE_NONE); + handle_edge_irq, IRQ_TYPE_NONE); if (ret) { dev_err(dev, "failed to add irqchip\n"); gpiochip_remove(gc); diff --git a/drivers/gpio/gpio-xlp.c b/drivers/gpio/gpio-xlp.c index 9bdab7203d65..e02499a15e72 100644 --- a/drivers/gpio/gpio-xlp.c +++ b/drivers/gpio/gpio-xlp.c @@ -387,7 +387,7 @@ static int xlp_gpio_probe(struct platform_device *pdev) irq_base = irq_alloc_descs(-1, XLP_GPIO_IRQ_BASE, gc->ngpio, 0); if (irq_base < 0) { dev_err(&pdev->dev, "Failed to allocate IRQ numbers\n"); - return err; + return -ENODEV; } err = gpiochip_add(gc); diff --git a/drivers/gpio/gpio-zx.c b/drivers/gpio/gpio-zx.c new file mode 100644 index 000000000000..12ee1969298c --- /dev/null +++ b/drivers/gpio/gpio-zx.c @@ -0,0 +1,324 @@ +/* + * Copyright (C) 2015 Linaro Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ZX_GPIO_DIR 0x00 +#define ZX_GPIO_IVE 0x04 +#define ZX_GPIO_IV 0x08 +#define ZX_GPIO_IEP 0x0C +#define ZX_GPIO_IEN 0x10 +#define ZX_GPIO_DI 0x14 +#define ZX_GPIO_DO1 0x18 +#define ZX_GPIO_DO0 0x1C +#define ZX_GPIO_DO 0x20 + +#define ZX_GPIO_IM 0x28 +#define ZX_GPIO_IE 0x2C + +#define ZX_GPIO_MIS 0x30 +#define ZX_GPIO_IC 0x34 + +#define ZX_GPIO_NR 16 + +struct zx_gpio { + spinlock_t lock; + + void __iomem *base; + struct gpio_chip gc; + bool uses_pinctrl; +}; + +static inline struct zx_gpio *to_zx(struct gpio_chip *gc) +{ + return container_of(gc, struct zx_gpio, gc); +} + +static int zx_gpio_request(struct gpio_chip *gc, unsigned offset) +{ + struct zx_gpio *chip = to_zx(gc); + int gpio = gc->base + offset; + + if (chip->uses_pinctrl) + return pinctrl_request_gpio(gpio); + return 0; +} + +static void zx_gpio_free(struct gpio_chip *gc, unsigned offset) +{ + struct zx_gpio *chip = to_zx(gc); + int gpio = gc->base + offset; + + if (chip->uses_pinctrl) + pinctrl_free_gpio(gpio); +} + +static int zx_direction_input(struct gpio_chip *gc, unsigned offset) +{ + struct zx_gpio *chip = to_zx(gc); + unsigned long flags; + u16 gpiodir; + + if (offset >= gc->ngpio) + return -EINVAL; + + spin_lock_irqsave(&chip->lock, flags); + gpiodir = readw_relaxed(chip->base + ZX_GPIO_DIR); + gpiodir &= ~BIT(offset); + writew_relaxed(gpiodir, chip->base + ZX_GPIO_DIR); + spin_unlock_irqrestore(&chip->lock, flags); + + return 0; +} + +static int zx_direction_output(struct gpio_chip *gc, unsigned offset, + int value) +{ + struct zx_gpio *chip = to_zx(gc); + unsigned long flags; + u16 gpiodir; + + if (offset >= gc->ngpio) + return -EINVAL; + + spin_lock_irqsave(&chip->lock, flags); + gpiodir = readw_relaxed(chip->base + ZX_GPIO_DIR); + gpiodir |= BIT(offset); + writew_relaxed(gpiodir, chip->base + ZX_GPIO_DIR); + + if (value) + writew_relaxed(BIT(offset), chip->base + ZX_GPIO_DO1); + else + writew_relaxed(BIT(offset), chip->base + ZX_GPIO_DO0); + spin_unlock_irqrestore(&chip->lock, flags); + + return 0; +} + +static int zx_get_value(struct gpio_chip *gc, unsigned offset) +{ + struct zx_gpio *chip = to_zx(gc); + + return !!(readw_relaxed(chip->base + ZX_GPIO_DI) & BIT(offset)); +} + +static void zx_set_value(struct gpio_chip *gc, unsigned offset, int value) +{ + struct zx_gpio *chip = to_zx(gc); + + if (value) + writew_relaxed(BIT(offset), chip->base + ZX_GPIO_DO1); + else + writew_relaxed(BIT(offset), chip->base + ZX_GPIO_DO0); +} + +static int zx_irq_type(struct irq_data *d, unsigned trigger) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct zx_gpio *chip = to_zx(gc); + int offset = irqd_to_hwirq(d); + unsigned long flags; + u16 gpiois, gpioi_epos, gpioi_eneg, gpioiev; + u16 bit = BIT(offset); + + if (offset < 0 || offset >= ZX_GPIO_NR) + return -EINVAL; + + spin_lock_irqsave(&chip->lock, flags); + + gpioiev = readw_relaxed(chip->base + ZX_GPIO_IV); + gpiois = readw_relaxed(chip->base + ZX_GPIO_IVE); + gpioi_epos = readw_relaxed(chip->base + ZX_GPIO_IEP); + gpioi_eneg = readw_relaxed(chip->base + ZX_GPIO_IEN); + + if (trigger & (IRQ_TYPE_LEVEL_HIGH | IRQ_TYPE_LEVEL_LOW)) { + gpiois |= bit; + if (trigger & IRQ_TYPE_LEVEL_HIGH) + gpioiev |= bit; + else + gpioiev &= ~bit; + } else + gpiois &= ~bit; + + if ((trigger & IRQ_TYPE_EDGE_BOTH) == IRQ_TYPE_EDGE_BOTH) { + gpioi_epos |= bit; + gpioi_eneg |= bit; + } else { + if (trigger & IRQ_TYPE_EDGE_RISING) { + gpioi_epos |= bit; + gpioi_eneg &= ~bit; + } else if (trigger & IRQ_TYPE_EDGE_FALLING) { + gpioi_eneg |= bit; + gpioi_epos &= ~bit; + } + } + + writew_relaxed(gpiois, chip->base + ZX_GPIO_IVE); + writew_relaxed(gpioi_epos, chip->base + ZX_GPIO_IEP); + writew_relaxed(gpioi_eneg, chip->base + ZX_GPIO_IEN); + writew_relaxed(gpioiev, chip->base + ZX_GPIO_IV); + spin_unlock_irqrestore(&chip->lock, flags); + + return 0; +} + +static void zx_irq_handler(unsigned irq, struct irq_desc *desc) +{ + unsigned long pending; + int offset; + struct gpio_chip *gc = irq_desc_get_handler_data(desc); + struct zx_gpio *chip = to_zx(gc); + struct irq_chip *irqchip = irq_desc_get_chip(desc); + + chained_irq_enter(irqchip, desc); + + pending = readw_relaxed(chip->base + ZX_GPIO_MIS); + writew_relaxed(pending, chip->base + ZX_GPIO_IC); + if (pending) { + for_each_set_bit(offset, &pending, ZX_GPIO_NR) + generic_handle_irq(irq_find_mapping(gc->irqdomain, + offset)); + } + + chained_irq_exit(irqchip, desc); +} + +static void zx_irq_mask(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct zx_gpio *chip = to_zx(gc); + u16 mask = BIT(irqd_to_hwirq(d) % ZX_GPIO_NR); + u16 gpioie; + + spin_lock(&chip->lock); + gpioie = readw_relaxed(chip->base + ZX_GPIO_IM) | mask; + writew_relaxed(gpioie, chip->base + ZX_GPIO_IM); + gpioie = readw_relaxed(chip->base + ZX_GPIO_IE) & ~mask; + writew_relaxed(gpioie, chip->base + ZX_GPIO_IE); + spin_unlock(&chip->lock); +} + +static void zx_irq_unmask(struct irq_data *d) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(d); + struct zx_gpio *chip = to_zx(gc); + u16 mask = BIT(irqd_to_hwirq(d) % ZX_GPIO_NR); + u16 gpioie; + + spin_lock(&chip->lock); + gpioie = readw_relaxed(chip->base + ZX_GPIO_IM) & ~mask; + writew_relaxed(gpioie, chip->base + ZX_GPIO_IM); + gpioie = readw_relaxed(chip->base + ZX_GPIO_IE) | mask; + writew_relaxed(gpioie, chip->base + ZX_GPIO_IE); + spin_unlock(&chip->lock); +} + +static struct irq_chip zx_irqchip = { + .name = "zx-gpio", + .irq_mask = zx_irq_mask, + .irq_unmask = zx_irq_unmask, + .irq_set_type = zx_irq_type, +}; + +static int zx_gpio_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct zx_gpio *chip; + struct resource *res; + int irq, id, ret; + + chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL); + if (!chip) + return -ENOMEM; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + chip->base = devm_ioremap_resource(dev, res); + if (IS_ERR(chip->base)) + return PTR_ERR(chip->base); + + spin_lock_init(&chip->lock); + if (of_property_read_bool(dev->of_node, "gpio-ranges")) + chip->uses_pinctrl = true; + + id = of_alias_get_id(dev->of_node, "gpio"); + chip->gc.request = zx_gpio_request; + chip->gc.free = zx_gpio_free; + chip->gc.direction_input = zx_direction_input; + chip->gc.direction_output = zx_direction_output; + chip->gc.get = zx_get_value; + chip->gc.set = zx_set_value; + chip->gc.base = ZX_GPIO_NR * id; + chip->gc.ngpio = ZX_GPIO_NR; + chip->gc.label = dev_name(dev); + chip->gc.dev = dev; + chip->gc.owner = THIS_MODULE; + + ret = gpiochip_add(&chip->gc); + if (ret) + return ret; + + /* + * irq_chip support + */ + writew_relaxed(0xffff, chip->base + ZX_GPIO_IM); + writew_relaxed(0, chip->base + ZX_GPIO_IE); + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + dev_err(dev, "invalid IRQ\n"); + gpiochip_remove(&chip->gc); + return -ENODEV; + } + + ret = gpiochip_irqchip_add(&chip->gc, &zx_irqchip, + 0, handle_simple_irq, + IRQ_TYPE_NONE); + if (ret) { + dev_err(dev, "could not add irqchip\n"); + gpiochip_remove(&chip->gc); + return ret; + } + gpiochip_set_chained_irqchip(&chip->gc, &zx_irqchip, + irq, zx_irq_handler); + + platform_set_drvdata(pdev, chip); + dev_info(dev, "ZX GPIO chip registered\n"); + + return 0; +} + +static const struct of_device_id zx_gpio_match[] = { + { + .compatible = "zte,zx296702-gpio", + }, + { }, +}; +MODULE_DEVICE_TABLE(of, zx_gpio_match); + +static struct platform_driver zx_gpio_driver = { + .probe = zx_gpio_probe, + .driver = { + .name = "zx_gpio", + .of_match_table = of_match_ptr(zx_gpio_match), + }, +}; + +module_platform_driver(zx_gpio_driver) + +MODULE_AUTHOR("Jun Nie "); +MODULE_DESCRIPTION("ZTE ZX296702 GPIO driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/gpio/gpio-zynq.c b/drivers/gpio/gpio-zynq.c index a78882389836..27348e7cb705 100644 --- a/drivers/gpio/gpio-zynq.c +++ b/drivers/gpio/gpio-zynq.c @@ -441,10 +441,10 @@ static int zynq_gpio_set_irq_type(struct irq_data *irq_data, unsigned int type) gpio->base_addr + ZYNQ_GPIO_INTANY_OFFSET(bank_num)); if (type & IRQ_TYPE_LEVEL_MASK) { - __irq_set_chip_handler_name_locked(irq_data->irq, + irq_set_chip_handler_name_locked(irq_data, &zynq_gpio_level_irqchip, handle_fasteoi_irq, NULL); } else { - __irq_set_chip_handler_name_locked(irq_data->irq, + irq_set_chip_handler_name_locked(irq_data, &zynq_gpio_edge_irqchip, handle_level_irq, NULL); } @@ -518,7 +518,7 @@ static void zynq_gpio_irqhandler(unsigned int irq, struct irq_desc *desc) { u32 int_sts, int_enb; unsigned int bank_num; - struct zynq_gpio *gpio = irq_get_handler_data(irq); + struct zynq_gpio *gpio = irq_desc_get_handler_data(desc); struct irq_chip *irqchip = irq_desc_get_chip(desc); chained_irq_enter(irqchip, desc); @@ -782,6 +782,12 @@ static int __init zynq_gpio_init(void) } postcore_initcall(zynq_gpio_init); +static void __exit zynq_gpio_exit(void) +{ + platform_driver_unregister(&zynq_gpio_driver); +} +module_exit(zynq_gpio_exit); + MODULE_AUTHOR("Xilinx Inc."); MODULE_DESCRIPTION("Zynq GPIO driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/gpio/gpiolib-acpi.c b/drivers/gpio/gpiolib-acpi.c index 533fe5dbe6f8..143a9bdbaa53 100644 --- a/drivers/gpio/gpiolib-acpi.c +++ b/drivers/gpio/gpiolib-acpi.c @@ -68,7 +68,7 @@ static int acpi_gpiochip_find(struct gpio_chip *gc, void *data) * GPIO controller driver. * * Typically the returned offset is same as @pin, but if the GPIO - * controller uses pin controller and the mapping is not contigous the + * controller uses pin controller and the mapping is not contiguous the * offset might be different. */ static int acpi_gpiochip_pin_to_gpio_offset(struct gpio_chip *chip, int pin) diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c index 9a0ec48a4737..fa6e3c8823d6 100644 --- a/drivers/gpio/gpiolib-of.c +++ b/drivers/gpio/gpiolib-of.c @@ -136,7 +136,6 @@ static struct gpio_desc *of_get_gpio_hog(struct device_node *np, { struct device_node *chip_np; enum of_gpio_flags xlate_flags; - struct gpio_desc *desc; struct gg_data gg_data = { .flags = &xlate_flags, }; @@ -193,9 +192,7 @@ static struct gpio_desc *of_get_gpio_hog(struct device_node *np, if (name && of_property_read_string(np, "line-name", name)) *name = np->name; - desc = gg_data.out_gpio; - - return desc; + return gg_data.out_gpio; } /** @@ -338,7 +335,7 @@ void of_mm_gpiochip_remove(struct of_mm_gpio_chip *mm_gc) EXPORT_SYMBOL(of_mm_gpiochip_remove); #ifdef CONFIG_PINCTRL -static void of_gpiochip_add_pin_range(struct gpio_chip *chip) +static int of_gpiochip_add_pin_range(struct gpio_chip *chip) { struct device_node *np = chip->of_node; struct of_phandle_args pinspec; @@ -349,7 +346,7 @@ static void of_gpiochip_add_pin_range(struct gpio_chip *chip) struct property *group_names; if (!np) - return; + return 0; group_names = of_find_property(np, group_names_propname, NULL); @@ -361,11 +358,11 @@ static void of_gpiochip_add_pin_range(struct gpio_chip *chip) pctldev = of_pinctrl_get(pinspec.np); if (!pctldev) - break; + return -EPROBE_DEFER; if (pinspec.args[2]) { if (group_names) { - ret = of_property_read_string_index(np, + of_property_read_string_index(np, group_names_propname, index, &name); if (strlen(name)) { @@ -381,7 +378,7 @@ static void of_gpiochip_add_pin_range(struct gpio_chip *chip) pinspec.args[1], pinspec.args[2]); if (ret) - break; + return ret; } else { /* npins == 0: special range */ if (pinspec.args[1]) { @@ -411,32 +408,41 @@ static void of_gpiochip_add_pin_range(struct gpio_chip *chip) ret = gpiochip_add_pingroup_range(chip, pctldev, pinspec.args[0], name); if (ret) - break; + return ret; } } + + return 0; } #else -static void of_gpiochip_add_pin_range(struct gpio_chip *chip) {} +static int of_gpiochip_add_pin_range(struct gpio_chip *chip) { return 0; } #endif -void of_gpiochip_add(struct gpio_chip *chip) +int of_gpiochip_add(struct gpio_chip *chip) { + int status; + if ((!chip->of_node) && (chip->dev)) chip->of_node = chip->dev->of_node; if (!chip->of_node) - return; + return 0; if (!chip->of_xlate) { chip->of_gpio_n_cells = 2; chip->of_xlate = of_gpio_simple_xlate; } - of_gpiochip_add_pin_range(chip); + status = of_gpiochip_add_pin_range(chip); + if (status) + return status; + of_node_get(chip->of_node); of_gpiochip_scan_hogs(chip); + + return 0; } void of_gpiochip_remove(struct gpio_chip *chip) diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index bf4bd1d120c3..980c1f87866a 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -190,7 +190,7 @@ EXPORT_SYMBOL_GPL(gpiod_get_direction); */ static int gpiochip_add_to_list(struct gpio_chip *chip) { - struct list_head *pos = &gpio_chips; + struct list_head *pos; struct gpio_chip *_chip; int err = 0; @@ -287,7 +287,13 @@ int gpiochip_add(struct gpio_chip *chip) INIT_LIST_HEAD(&chip->pin_ranges); #endif - of_gpiochip_add(chip); + if (!chip->owner && chip->dev && chip->dev->driver) + chip->owner = chip->dev->driver->owner; + + status = of_gpiochip_add(chip); + if (status) + goto err_remove_chip; + acpi_gpiochip_add(chip); status = gpiochip_sysfs_register(chip); @@ -443,8 +449,8 @@ void gpiochip_set_chained_irqchip(struct gpio_chip *gpiochip, * The parent irqchip is already using the chip_data for this * irqchip, so our callbacks simply use the handler_data. */ - irq_set_handler_data(parent_irq, gpiochip); - irq_set_chained_handler(parent_irq, parent_handler); + irq_set_chained_handler_and_data(parent_irq, parent_handler, + gpiochip); gpiochip->irq_parent = parent_irq; } @@ -456,12 +462,6 @@ void gpiochip_set_chained_irqchip(struct gpio_chip *gpiochip, } EXPORT_SYMBOL_GPL(gpiochip_set_chained_irqchip); -/* - * This lock class tells lockdep that GPIO irqs are in a different - * category than their parents, so it won't report false recursion. - */ -static struct lock_class_key gpiochip_irq_lock_class; - /** * gpiochip_irq_map() - maps an IRQ into a GPIO irqchip * @d: the irqdomain used by this irqchip @@ -478,16 +478,17 @@ static int gpiochip_irq_map(struct irq_domain *d, unsigned int irq, struct gpio_chip *chip = d->host_data; irq_set_chip_data(irq, chip); - irq_set_lockdep_class(irq, &gpiochip_irq_lock_class); + /* + * This lock class tells lockdep that GPIO irqs are in a different + * category than their parents, so it won't report false recursion. + */ + irq_set_lockdep_class(irq, chip->lock_key); irq_set_chip_and_handler(irq, chip->irqchip, chip->irq_handler); /* Chips that can sleep need nested thread handlers */ if (chip->can_sleep && !chip->irq_not_threaded) irq_set_nested_thread(irq, 1); -#ifdef CONFIG_ARM - set_irq_flags(irq, IRQF_VALID); -#else irq_set_noprobe(irq); -#endif + /* * No set-up of the hardware will happen if IRQ_TYPE_NONE * is passed as default type. @@ -502,9 +503,6 @@ static void gpiochip_irq_unmap(struct irq_domain *d, unsigned int irq) { struct gpio_chip *chip = d->host_data; -#ifdef CONFIG_ARM - set_irq_flags(irq, 0); -#endif if (chip->can_sleep) irq_set_nested_thread(irq, 0); irq_set_chip_and_handler(irq, NULL, NULL); @@ -522,10 +520,14 @@ static int gpiochip_irq_reqres(struct irq_data *d) { struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + if (!try_module_get(chip->owner)) + return -ENODEV; + if (gpiochip_lock_as_irq(chip, d->hwirq)) { chip_err(chip, "unable to lock HW IRQ %lu for IRQ\n", d->hwirq); + module_put(chip->owner); return -EINVAL; } return 0; @@ -536,6 +538,7 @@ static void gpiochip_irq_relres(struct irq_data *d) struct gpio_chip *chip = irq_data_get_irq_chip_data(d); gpiochip_unlock_as_irq(chip, d->hwirq); + module_put(chip->owner); } static int gpiochip_to_irq(struct gpio_chip *chip, unsigned offset) @@ -584,6 +587,7 @@ static void gpiochip_irqchip_remove(struct gpio_chip *gpiochip) * @handler: the irq handler to use (often a predefined irq core function) * @type: the default type for IRQs on this irqchip, pass IRQ_TYPE_NONE * to have the core avoid setting up any default type in the hardware. + * @lock_key: lockdep class * * This function closely associates a certain irqchip with a certain * gpiochip, providing an irq domain to translate the local IRQs to @@ -599,11 +603,12 @@ static void gpiochip_irqchip_remove(struct gpio_chip *gpiochip) * the pins on the gpiochip can generate a unique IRQ. Everything else * need to be open coded. */ -int gpiochip_irqchip_add(struct gpio_chip *gpiochip, - struct irq_chip *irqchip, - unsigned int first_irq, - irq_flow_handler_t handler, - unsigned int type) +int _gpiochip_irqchip_add(struct gpio_chip *gpiochip, + struct irq_chip *irqchip, + unsigned int first_irq, + irq_flow_handler_t handler, + unsigned int type, + struct lock_class_key *lock_key) { struct device_node *of_node; unsigned int offset; @@ -629,6 +634,7 @@ int gpiochip_irqchip_add(struct gpio_chip *gpiochip, gpiochip->irq_handler = handler; gpiochip->irq_default_type = type; gpiochip->to_irq = gpiochip_to_irq; + gpiochip->lock_key = lock_key; gpiochip->irqdomain = irq_domain_add_simple(of_node, gpiochip->ngpio, first_irq, &gpiochip_domain_ops, gpiochip); @@ -636,8 +642,16 @@ int gpiochip_irqchip_add(struct gpio_chip *gpiochip, gpiochip->irqchip = NULL; return -EINVAL; } - irqchip->irq_request_resources = gpiochip_irq_reqres; - irqchip->irq_release_resources = gpiochip_irq_relres; + + /* + * It is possible for a driver to override this, but only if the + * alternative functions are both implemented. + */ + if (!irqchip->irq_request_resources && + !irqchip->irq_release_resources) { + irqchip->irq_request_resources = gpiochip_irq_reqres; + irqchip->irq_release_resources = gpiochip_irq_relres; + } /* * Prepare the mapping since the irqchip shall be orthogonal to @@ -658,7 +672,7 @@ int gpiochip_irqchip_add(struct gpio_chip *gpiochip, return 0; } -EXPORT_SYMBOL_GPL(gpiochip_irqchip_add); +EXPORT_SYMBOL_GPL(_gpiochip_irqchip_add); #else /* CONFIG_GPIOLIB_IRQCHIP */ @@ -671,7 +685,7 @@ static void gpiochip_irqchip_remove(struct gpio_chip *gpiochip) {} /** * gpiochip_add_pingroup_range() - add a range for GPIO <-> pin mapping * @chip: the gpiochip to add the range for - * @pinctrl: the dev_name() of the pin controller to map to + * @pctldev: the pin controller to map to * @gpio_offset: the start offset in the current gpio_chip number space * @pin_group: name of the pin group inside the pin controller */ @@ -1672,6 +1686,19 @@ void gpiod_add_lookup_table(struct gpiod_lookup_table *table) mutex_unlock(&gpio_lookup_lock); } +/** + * gpiod_remove_lookup_table() - unregister GPIO device consumers + * @table: table of consumers to unregister + */ +void gpiod_remove_lookup_table(struct gpiod_lookup_table *table) +{ + mutex_lock(&gpio_lookup_lock); + + list_del(&table->list); + + mutex_unlock(&gpio_lookup_lock); +} + static struct gpio_desc *of_find_gpio(struct device *dev, const char *con_id, unsigned int idx, enum gpio_lookup_flags *flags) @@ -1894,12 +1921,12 @@ EXPORT_SYMBOL_GPL(gpiod_count); * dev, -ENOENT if no GPIO has been assigned to the requested function, or * another IS_ERR() code if an error occurred while trying to acquire the GPIO. */ -struct gpio_desc *__must_check __gpiod_get(struct device *dev, const char *con_id, +struct gpio_desc *__must_check gpiod_get(struct device *dev, const char *con_id, enum gpiod_flags flags) { return gpiod_get_index(dev, con_id, 0, flags); } -EXPORT_SYMBOL_GPL(__gpiod_get); +EXPORT_SYMBOL_GPL(gpiod_get); /** * gpiod_get_optional - obtain an optional GPIO for a given GPIO function @@ -1911,13 +1938,13 @@ EXPORT_SYMBOL_GPL(__gpiod_get); * the requested function it will return NULL. This is convenient for drivers * that need to handle optional GPIOs. */ -struct gpio_desc *__must_check __gpiod_get_optional(struct device *dev, +struct gpio_desc *__must_check gpiod_get_optional(struct device *dev, const char *con_id, enum gpiod_flags flags) { return gpiod_get_index_optional(dev, con_id, 0, flags); } -EXPORT_SYMBOL_GPL(__gpiod_get_optional); +EXPORT_SYMBOL_GPL(gpiod_get_optional); /** @@ -1974,7 +2001,7 @@ static int gpiod_configure_flags(struct gpio_desc *desc, const char *con_id, * requested function and/or index, or another IS_ERR() code if an error * occurred while trying to acquire the GPIO. */ -struct gpio_desc *__must_check __gpiod_get_index(struct device *dev, +struct gpio_desc *__must_check gpiod_get_index(struct device *dev, const char *con_id, unsigned int idx, enum gpiod_flags flags) @@ -2023,7 +2050,7 @@ struct gpio_desc *__must_check __gpiod_get_index(struct device *dev, return desc; } -EXPORT_SYMBOL_GPL(__gpiod_get_index); +EXPORT_SYMBOL_GPL(gpiod_get_index); /** * fwnode_get_named_gpiod - obtain a GPIO from firmware node @@ -2092,7 +2119,7 @@ EXPORT_SYMBOL_GPL(fwnode_get_named_gpiod); * specified index was assigned to the requested function it will return NULL. * This is convenient for drivers that need to handle optional GPIOs. */ -struct gpio_desc *__must_check __gpiod_get_index_optional(struct device *dev, +struct gpio_desc *__must_check gpiod_get_index_optional(struct device *dev, const char *con_id, unsigned int index, enum gpiod_flags flags) @@ -2107,7 +2134,7 @@ struct gpio_desc *__must_check __gpiod_get_index_optional(struct device *dev, return desc; } -EXPORT_SYMBOL_GPL(__gpiod_get_index_optional); +EXPORT_SYMBOL_GPL(gpiod_get_index_optional); /** * gpiod_hog - Hog the specified GPIO desc given the provided flags diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index c46ca311d8c3..1a0a8df2eed8 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -37,9 +37,29 @@ config DRM_KMS_FB_HELPER select FB select FRAMEBUFFER_CONSOLE if !EXPERT select FRAMEBUFFER_CONSOLE_DETECT_PRIMARY if FRAMEBUFFER_CONSOLE + select FB_SYS_FOPS + select FB_SYS_FILLRECT + select FB_SYS_COPYAREA + select FB_SYS_IMAGEBLIT + select FB_CFB_FILLRECT + select FB_CFB_COPYAREA + select FB_CFB_IMAGEBLIT help FBDEV helpers for KMS drivers. +config DRM_FBDEV_EMULATION + bool "Enable legacy fbdev support for your modesetting driver" + depends on DRM + select DRM_KMS_HELPER + select DRM_KMS_FB_HELPER + default y + help + Choose this option if you have a need for the legacy fbdev + support. Note that this support also provides the linux console + support on top of your modesetting driver. + + If in doubt, say "Y". + config DRM_LOAD_EDID_FIRMWARE bool "Allow to specify an EDID data set instead of probing for it" depends on DRM_KMS_HELPER @@ -79,8 +99,6 @@ config DRM_KMS_CMA_HELPER source "drivers/gpu/drm/i2c/Kconfig" -source "drivers/gpu/drm/bridge/Kconfig" - config DRM_TDFX tristate "3dfx Banshee/Voodoo3+" depends on DRM && PCI @@ -110,6 +128,7 @@ config DRM_RADEON select POWER_SUPPLY select HWMON select BACKLIGHT_CLASS_DEVICE + select BACKLIGHT_LCD_SUPPORT select INTERVAL_TREE help Choose this option if you have an ATI Radeon graphics card. There @@ -133,6 +152,7 @@ config DRM_AMDGPU select POWER_SUPPLY select HWMON select BACKLIGHT_CLASS_DEVICE + select BACKLIGHT_LCD_SUPPORT select INTERVAL_TREE help Choose this option if you have a recent AMD Radeon graphics card. @@ -231,10 +251,14 @@ source "drivers/gpu/drm/virtio/Kconfig" source "drivers/gpu/drm/msm/Kconfig" +source "drivers/gpu/drm/fsl-dcu/Kconfig" + source "drivers/gpu/drm/tegra/Kconfig" source "drivers/gpu/drm/panel/Kconfig" +source "drivers/gpu/drm/bridge/Kconfig" + source "drivers/gpu/drm/sti/Kconfig" source "drivers/gpu/drm/amd/amdkfd/Kconfig" diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index 5713d0534504..45e7719846b1 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -23,7 +23,7 @@ drm-$(CONFIG_OF) += drm_of.o drm_kms_helper-y := drm_crtc_helper.o drm_dp_helper.o drm_probe_helper.o \ drm_plane_helper.o drm_dp_mst_topology.o drm_atomic_helper.o drm_kms_helper-$(CONFIG_DRM_LOAD_EDID_FIRMWARE) += drm_edid_load.o -drm_kms_helper-$(CONFIG_DRM_KMS_FB_HELPER) += drm_fb_helper.o +drm_kms_helper-$(CONFIG_DRM_FBDEV_EMULATION) += drm_fb_helper.o drm_kms_helper-$(CONFIG_DRM_KMS_CMA_HELPER) += drm_fb_cma_helper.o obj-$(CONFIG_DRM_KMS_HELPER) += drm_kms_helper.o @@ -70,3 +70,4 @@ obj-$(CONFIG_DRM_IMX) += imx/ obj-y += i2c/ obj-y += panel/ obj-y += bridge/ +obj-$(CONFIG_DRM_FSL_DCU) += fsl-dcu/ diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile index 616dfd4a1398..04c270757030 100644 --- a/drivers/gpu/drm/amd/amdgpu/Makefile +++ b/drivers/gpu/drm/amd/amdgpu/Makefile @@ -3,7 +3,9 @@ # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher. ccflags-y := -Iinclude/drm -Idrivers/gpu/drm/amd/include/asic_reg \ - -Idrivers/gpu/drm/amd/include + -Idrivers/gpu/drm/amd/include \ + -Idrivers/gpu/drm/amd/amdgpu \ + -Idrivers/gpu/drm/amd/scheduler amdgpu-y := amdgpu_drv.o @@ -21,7 +23,8 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \ # add asic specific block amdgpu-$(CONFIG_DRM_AMDGPU_CIK)+= cik.o gmc_v7_0.o cik_ih.o kv_smc.o kv_dpm.o \ - ci_smc.o ci_dpm.o dce_v8_0.o gfx_v7_0.o cik_sdma.o uvd_v4_2.o vce_v2_0.o + ci_smc.o ci_dpm.o dce_v8_0.o gfx_v7_0.o cik_sdma.o uvd_v4_2.o vce_v2_0.o \ + amdgpu_amdkfd_gfx_v7.o amdgpu-y += \ vi.o @@ -43,6 +46,7 @@ amdgpu-y += \ amdgpu_dpm.o \ cz_smc.o cz_dpm.o \ tonga_smc.o tonga_dpm.o \ + fiji_smc.o fiji_dpm.o \ iceland_smc.o iceland_dpm.o # add DCE block @@ -71,6 +75,20 @@ amdgpu-y += \ amdgpu_vce.o \ vce_v3_0.o +# add amdkfd interfaces +amdgpu-y += \ + amdgpu_amdkfd.o \ + amdgpu_amdkfd_gfx_v8.o + +# add cgs +amdgpu-y += amdgpu_cgs.o + +# GPU scheduler +amdgpu-y += \ + ../scheduler/gpu_scheduler.o \ + ../scheduler/sched_fence.o \ + amdgpu_sched.o + amdgpu-$(CONFIG_COMPAT) += amdgpu_ioc32.o amdgpu-$(CONFIG_VGA_SWITCHEROO) += amdgpu_atpx_handler.o amdgpu-$(CONFIG_ACPI) += amdgpu_acpi.o diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index f7b49d5ce4b8..668939a14206 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -42,17 +42,19 @@ #include #include +#include #include #include #include "amd_shared.h" -#include "amdgpu_family.h" #include "amdgpu_mode.h" #include "amdgpu_ih.h" #include "amdgpu_irq.h" #include "amdgpu_ucode.h" #include "amdgpu_gds.h" +#include "gpu_scheduler.h" + /* * Modules parameters. */ @@ -77,7 +79,11 @@ extern int amdgpu_bapm; extern int amdgpu_deep_color; extern int amdgpu_vm_size; extern int amdgpu_vm_block_size; +extern int amdgpu_enable_scheduler; +extern int amdgpu_sched_jobs; +extern int amdgpu_sched_hw_submission; +#define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS 3000 #define AMDGPU_MAX_USEC_TIMEOUT 100000 /* 100 ms */ #define AMDGPU_FENCE_JIFFIES_TIMEOUT (HZ / 2) /* AMDGPU_IB_POOL_SIZE must be a power of 2 */ @@ -92,6 +98,9 @@ extern int amdgpu_vm_block_size; #define AMDGPU_MAX_COMPUTE_RINGS 8 #define AMDGPU_MAX_VCE_RINGS 2 +/* max number of IP instances */ +#define AMDGPU_MAX_SDMA_INSTANCES 2 + /* number of hw syncs before falling back on blocking */ #define AMDGPU_NUM_SYNCS 4 @@ -177,7 +186,9 @@ struct amdgpu_vm; struct amdgpu_ring; struct amdgpu_semaphore; struct amdgpu_cs_parser; +struct amdgpu_job; struct amdgpu_irq_src; +struct amdgpu_fpriv; enum amdgpu_cp_irq { AMDGPU_CP_IRQ_GFX_EOP = 0, @@ -239,7 +250,7 @@ struct amdgpu_buffer_funcs { unsigned copy_num_dw; /* used for buffer migration */ - void (*emit_copy_buffer)(struct amdgpu_ring *ring, + void (*emit_copy_buffer)(struct amdgpu_ib *ib, /* src addr in bytes */ uint64_t src_offset, /* dst addr in bytes */ @@ -254,7 +265,7 @@ struct amdgpu_buffer_funcs { unsigned fill_num_dw; /* used for buffer clearing */ - void (*emit_fill_buffer)(struct amdgpu_ring *ring, + void (*emit_fill_buffer)(struct amdgpu_ib *ib, /* value to write to memory */ uint32_t src_data, /* dst addr in bytes */ @@ -332,6 +343,8 @@ struct amdgpu_ring_funcs { int (*test_ring)(struct amdgpu_ring *ring); int (*test_ib)(struct amdgpu_ring *ring); bool (*is_lockup)(struct amdgpu_ring *ring); + /* insert NOP packets */ + void (*insert_nop)(struct amdgpu_ring *ring, uint32_t count); }; /* @@ -381,10 +394,10 @@ struct amdgpu_fence_driver { uint64_t sync_seq[AMDGPU_MAX_RINGS]; atomic64_t last_seq; bool initialized; - bool delayed_irq; struct amdgpu_irq_src *irq_src; unsigned irq_type; struct delayed_work lockup_work; + wait_queue_head_t fence_queue; }; /* some special values for the owner field */ @@ -423,20 +436,20 @@ void amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring); int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring, struct amdgpu_irq_src *irq_src, unsigned irq_type); +void amdgpu_fence_driver_suspend(struct amdgpu_device *adev); +void amdgpu_fence_driver_resume(struct amdgpu_device *adev); int amdgpu_fence_emit(struct amdgpu_ring *ring, void *owner, struct amdgpu_fence **fence); -int amdgpu_fence_recreate(struct amdgpu_ring *ring, void *owner, - uint64_t seq, struct amdgpu_fence **fence); void amdgpu_fence_process(struct amdgpu_ring *ring); int amdgpu_fence_wait_next(struct amdgpu_ring *ring); int amdgpu_fence_wait_empty(struct amdgpu_ring *ring); unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring); -bool amdgpu_fence_signaled(struct amdgpu_fence *fence); -int amdgpu_fence_wait(struct amdgpu_fence *fence, bool interruptible); -int amdgpu_fence_wait_any(struct amdgpu_device *adev, - struct amdgpu_fence **fences, - bool intr); +signed long amdgpu_fence_wait_any(struct amdgpu_device *adev, + struct fence **array, + uint32_t count, + bool intr, + signed long t); struct amdgpu_fence *amdgpu_fence_ref(struct amdgpu_fence *fence); void amdgpu_fence_unref(struct amdgpu_fence **fence); @@ -481,7 +494,7 @@ static inline bool amdgpu_fence_is_earlier(struct amdgpu_fence *a, return a->seq < b->seq; } -int amdgpu_user_fence_emit(struct amdgpu_ring *ring, struct amdgpu_user_fence *user, +int amdgpu_user_fence_emit(struct amdgpu_ring *ring, struct amdgpu_user_fence *user, void *owner, struct amdgpu_fence **fence); /* @@ -509,7 +522,7 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t dst_offset, uint32_t byte_count, struct reservation_object *resv, - struct amdgpu_fence **fence); + struct fence **fence); int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma); struct amdgpu_bo_list_entry { @@ -532,14 +545,16 @@ struct amdgpu_bo_va_mapping { struct amdgpu_bo_va { /* protected by bo being reserved */ struct list_head bo_list; - uint64_t addr; - struct amdgpu_fence *last_pt_update; + struct fence *last_pt_update; unsigned ref_count; - /* protected by vm mutex */ - struct list_head mappings; + /* protected by vm mutex and spinlock */ struct list_head vm_status; + /* mappings for this bo_va */ + struct list_head invalids; + struct list_head valids; + /* constant after initialization */ struct amdgpu_vm *vm; struct amdgpu_bo *bo; @@ -643,7 +658,7 @@ struct amdgpu_sa_bo { struct amdgpu_sa_manager *manager; unsigned soffset; unsigned eoffset; - struct amdgpu_fence *fence; + struct fence *fence; }; /* @@ -685,7 +700,7 @@ bool amdgpu_semaphore_emit_wait(struct amdgpu_ring *ring, struct amdgpu_semaphore *semaphore); void amdgpu_semaphore_free(struct amdgpu_device *adev, struct amdgpu_semaphore **semaphore, - struct amdgpu_fence *fence); + struct fence *fence); /* * Synchronization @@ -693,20 +708,23 @@ void amdgpu_semaphore_free(struct amdgpu_device *adev, struct amdgpu_sync { struct amdgpu_semaphore *semaphores[AMDGPU_NUM_SYNCS]; struct amdgpu_fence *sync_to[AMDGPU_MAX_RINGS]; - struct amdgpu_fence *last_vm_update; + DECLARE_HASHTABLE(fences, 4); + struct fence *last_vm_update; }; void amdgpu_sync_create(struct amdgpu_sync *sync); -void amdgpu_sync_fence(struct amdgpu_sync *sync, - struct amdgpu_fence *fence); +int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync, + struct fence *f); int amdgpu_sync_resv(struct amdgpu_device *adev, struct amdgpu_sync *sync, struct reservation_object *resv, void *owner); int amdgpu_sync_rings(struct amdgpu_sync *sync, struct amdgpu_ring *ring); +struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync); +int amdgpu_sync_wait(struct amdgpu_sync *sync); void amdgpu_sync_free(struct amdgpu_device *adev, struct amdgpu_sync *sync, - struct amdgpu_fence *fence); + struct fence *fence); /* * GART structures, functions & helpers @@ -821,7 +839,9 @@ struct amdgpu_flip_work { uint64_t base; struct drm_pending_vblank_event *event; struct amdgpu_bo *old_rbo; - struct fence *fence; + struct fence *excl; + unsigned shared_count; + struct fence **shared; }; @@ -844,6 +864,8 @@ struct amdgpu_ib { uint32_t gws_base, gws_size; uint32_t oa_base, oa_size; uint32_t flags; + /* resulting sequence number */ + uint64_t sequence; }; enum amdgpu_ring_type { @@ -854,11 +876,23 @@ enum amdgpu_ring_type { AMDGPU_RING_TYPE_VCE }; +extern struct amd_sched_backend_ops amdgpu_sched_ops; + +int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, + struct amdgpu_ring *ring, + struct amdgpu_ib *ibs, + unsigned num_ibs, + int (*free_job)(struct amdgpu_job *), + void *owner, + struct fence **fence); + struct amdgpu_ring { struct amdgpu_device *adev; const struct amdgpu_ring_funcs *funcs; struct amdgpu_fence_driver fence_drv; + struct amd_gpu_scheduler *scheduler; + spinlock_t fence_lock; struct mutex *ring_lock; struct amdgpu_bo *ring_obj; volatile uint32_t *ring; @@ -892,6 +926,7 @@ struct amdgpu_ring { struct amdgpu_ctx *current_ctx; enum amdgpu_ring_type type; char name[16]; + bool is_pte_ring; }; /* @@ -933,7 +968,7 @@ struct amdgpu_vm_id { unsigned id; uint64_t pd_gpu_addr; /* last flushed PD/PT update */ - struct amdgpu_fence *flushed_updates; + struct fence *flushed_updates; /* last use of vmid */ struct amdgpu_fence *last_id_use; }; @@ -943,18 +978,22 @@ struct amdgpu_vm { struct rb_root va; - /* protecting invalidated and freed */ + /* protecting invalidated */ spinlock_t status_lock; /* BOs moved, but not yet updated in the PT */ struct list_head invalidated; - /* BOs freed, but not yet updated in the PT */ + /* BOs cleared in the PT because of a move */ + struct list_head cleared; + + /* BO mappings freed, but not yet updated in the PT */ struct list_head freed; /* contains the page directory */ struct amdgpu_bo *page_directory; unsigned max_pde_used; + struct fence *page_directory_fence; /* array of page tables, one for each page directory entry */ struct amdgpu_vm_pt *page_tables; @@ -983,27 +1022,47 @@ struct amdgpu_vm_manager { * context related structures */ -struct amdgpu_ctx_state { - uint64_t flags; - uint32_t hangs; +#define AMDGPU_CTX_MAX_CS_PENDING 16 + +struct amdgpu_ctx_ring { + uint64_t sequence; + struct fence *fences[AMDGPU_CTX_MAX_CS_PENDING]; + struct amd_sched_entity entity; }; struct amdgpu_ctx { - /* call kref_get()before CS start and kref_put() after CS fence signaled */ - struct kref refcount; - struct amdgpu_fpriv *fpriv; - struct amdgpu_ctx_state state; - uint32_t id; - unsigned reset_counter; + struct kref refcount; + struct amdgpu_device *adev; + unsigned reset_counter; + spinlock_t ring_lock; + struct amdgpu_ctx_ring rings[AMDGPU_MAX_RINGS]; }; struct amdgpu_ctx_mgr { - struct amdgpu_device *adev; - struct idr ctx_handles; - /* lock for IDR system */ - struct mutex lock; + struct amdgpu_device *adev; + struct mutex lock; + /* protected by lock */ + struct idr ctx_handles; }; +int amdgpu_ctx_init(struct amdgpu_device *adev, bool kernel, + struct amdgpu_ctx *ctx); +void amdgpu_ctx_fini(struct amdgpu_ctx *ctx); + +struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id); +int amdgpu_ctx_put(struct amdgpu_ctx *ctx); + +uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring, + struct fence *fence); +struct fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx, + struct amdgpu_ring *ring, uint64_t seq); + +int amdgpu_ctx_ioctl(struct drm_device *dev, void *data, + struct drm_file *filp); + +void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr); +void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr); + /* * file private structure */ @@ -1012,7 +1071,7 @@ struct amdgpu_fpriv { struct amdgpu_vm vm; struct mutex bo_list_lock; struct idr bo_list_handles; - struct amdgpu_ctx_mgr ctx_mgr; + struct amdgpu_ctx_mgr ctx_mgr; }; /* @@ -1160,6 +1219,7 @@ int amdgpu_ib_ring_tests(struct amdgpu_device *adev); void amdgpu_ring_free_size(struct amdgpu_ring *ring); int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned ndw); int amdgpu_ring_lock(struct amdgpu_ring *ring, unsigned ndw); +void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count); void amdgpu_ring_commit(struct amdgpu_ring *ring); void amdgpu_ring_unlock_commit(struct amdgpu_ring *ring); void amdgpu_ring_undo(struct amdgpu_ring *ring); @@ -1207,6 +1267,16 @@ struct amdgpu_cs_parser { struct amdgpu_user_fence uf; }; +struct amdgpu_job { + struct amd_sched_job base; + struct amdgpu_device *adev; + struct amdgpu_ib *ibs; + uint32_t num_ibs; + struct mutex job_lock; + struct amdgpu_user_fence uf; + int (*free_job)(struct amdgpu_job *sched_job); +}; + static inline u32 amdgpu_get_ib_value(struct amdgpu_cs_parser *p, uint32_t ib_idx, int idx) { return p->ibs[ib_idx].ptr[idx]; @@ -1601,7 +1671,6 @@ struct amdgpu_uvd { struct amdgpu_bo *vcpu_bo; void *cpu_addr; uint64_t gpu_addr; - void *saved_bo; atomic_t handles[AMDGPU_MAX_UVD_HANDLES]; struct drm_file *filp[AMDGPU_MAX_UVD_HANDLES]; struct delayed_work idle_work; @@ -1645,6 +1714,7 @@ struct amdgpu_sdma { uint32_t feature_version; struct amdgpu_ring ring; + bool burst_nop; }; /* @@ -1849,17 +1919,12 @@ struct amdgpu_atcs { struct amdgpu_atcs_functions functions; }; -int amdgpu_ctx_alloc(struct amdgpu_device *adev,struct amdgpu_fpriv *fpriv, - uint32_t *id,uint32_t flags); -int amdgpu_ctx_free(struct amdgpu_device *adev, struct amdgpu_fpriv *fpriv, - uint32_t id); +/* + * CGS + */ +void *amdgpu_cgs_create_device(struct amdgpu_device *adev); +void amdgpu_cgs_destroy_device(void *cgs_device); -void amdgpu_ctx_fini(struct amdgpu_fpriv *fpriv); -struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id); -int amdgpu_ctx_put(struct amdgpu_ctx *ctx); - -extern int amdgpu_ctx_ioctl(struct drm_device *dev, void *data, - struct drm_file *filp); /* * Core structure, functions and helpers. @@ -1883,7 +1948,7 @@ struct amdgpu_device { struct rw_semaphore exclusive_lock; /* ASIC */ - enum amdgpu_asic_type asic_type; + enum amd_asic_type asic_type; uint32_t family; uint32_t rev_id; uint32_t external_rev_id; @@ -1976,7 +2041,6 @@ struct amdgpu_device { struct amdgpu_irq_src hpd_irq; /* rings */ - wait_queue_head_t fence_queue; unsigned fence_context; struct mutex ring_lock; unsigned num_rings; @@ -1999,7 +2063,7 @@ struct amdgpu_device { struct amdgpu_gfx gfx; /* sdma */ - struct amdgpu_sdma sdma[2]; + struct amdgpu_sdma sdma[AMDGPU_MAX_SDMA_INSTANCES]; struct amdgpu_irq_src sdma_trap_irq; struct amdgpu_irq_src sdma_illegal_inst_irq; @@ -2025,6 +2089,12 @@ struct amdgpu_device { /* tracking pinned memory */ u64 vram_pin_size; u64 gart_pin_size; + + /* amdkfd interface */ + struct kfd_dev *kfd; + + /* kernel conext for IB submission */ + struct amdgpu_ctx kernel_ctx; }; bool amdgpu_device_is_px(struct drm_device *dev); @@ -2132,6 +2202,21 @@ static inline void amdgpu_ring_write(struct amdgpu_ring *ring, uint32_t v) ring->ring_free_dw--; } +static inline struct amdgpu_sdma * amdgpu_get_sdma_instance(struct amdgpu_ring *ring) +{ + struct amdgpu_device *adev = ring->adev; + int i; + + for (i = 0; i < AMDGPU_MAX_SDMA_INSTANCES; i++) + if (&adev->sdma[i].ring == ring) + break; + + if (i < AMDGPU_MAX_SDMA_INSTANCES) + return &adev->sdma[i]; + else + return NULL; +} + /* * ASICs macro. */ @@ -2183,8 +2268,8 @@ static inline void amdgpu_ring_write(struct amdgpu_ring *ring, uint32_t v) #define amdgpu_display_add_connector(adev, ci, sd, ct, ib, coi, h, r) (adev)->mode_info.funcs->add_connector((adev), (ci), (sd), (ct), (ib), (coi), (h), (r)) #define amdgpu_display_stop_mc_access(adev, s) (adev)->mode_info.funcs->stop_mc_access((adev), (s)) #define amdgpu_display_resume_mc_access(adev, s) (adev)->mode_info.funcs->resume_mc_access((adev), (s)) -#define amdgpu_emit_copy_buffer(adev, r, s, d, b) (adev)->mman.buffer_funcs->emit_copy_buffer((r), (s), (d), (b)) -#define amdgpu_emit_fill_buffer(adev, r, s, d, b) (adev)->mman.buffer_funcs->emit_fill_buffer((r), (s), (d), (b)) +#define amdgpu_emit_copy_buffer(adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_copy_buffer((ib), (s), (d), (b)) +#define amdgpu_emit_fill_buffer(adev, ib, s, d, b) (adev)->mman.buffer_funcs->emit_fill_buffer((ib), (s), (d), (b)) #define amdgpu_dpm_get_temperature(adev) (adev)->pm.funcs->get_temperature((adev)) #define amdgpu_dpm_pre_set_power_state(adev) (adev)->pm.funcs->pre_set_power_state((adev)) #define amdgpu_dpm_set_power_state(adev) (adev)->pm.funcs->set_power_state((adev)) @@ -2212,6 +2297,12 @@ void amdgpu_pci_config_reset(struct amdgpu_device *adev); bool amdgpu_card_posted(struct amdgpu_device *adev); void amdgpu_update_display_priority(struct amdgpu_device *adev); bool amdgpu_boot_test_post_card(struct amdgpu_device *adev); +struct amdgpu_cs_parser *amdgpu_cs_parser_create(struct amdgpu_device *adev, + struct drm_file *filp, + struct amdgpu_ctx *ctx, + struct amdgpu_ib *ibs, + uint32_t num_ibs); + int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data); int amdgpu_cs_get_ring(struct amdgpu_device *adev, u32 ip_type, u32 ip_instance, u32 ring, @@ -2275,11 +2366,11 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm); struct amdgpu_bo_list_entry *amdgpu_vm_get_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct list_head *head); -struct amdgpu_fence *amdgpu_vm_grab_id(struct amdgpu_ring *ring, - struct amdgpu_vm *vm); +int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, + struct amdgpu_sync *sync); void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_vm *vm, - struct amdgpu_fence *updates); + struct fence *updates); void amdgpu_vm_fence(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_fence *fence); @@ -2309,7 +2400,7 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, uint64_t addr); void amdgpu_vm_bo_rmv(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va); - +int amdgpu_vm_free_job(struct amdgpu_job *job); /* * functions used by amdgpu_encoder.c */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c new file mode 100644 index 000000000000..496ed2192eba --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -0,0 +1,269 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "amdgpu_amdkfd.h" +#include "amd_shared.h" +#include +#include "amdgpu.h" +#include + +const struct kfd2kgd_calls *kfd2kgd; +const struct kgd2kfd_calls *kgd2kfd; +bool (*kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls**); + +bool amdgpu_amdkfd_init(void) +{ +#if defined(CONFIG_HSA_AMD_MODULE) + bool (*kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls**); + + kgd2kfd_init_p = symbol_request(kgd2kfd_init); + + if (kgd2kfd_init_p == NULL) + return false; +#endif + return true; +} + +bool amdgpu_amdkfd_load_interface(struct amdgpu_device *rdev) +{ +#if defined(CONFIG_HSA_AMD_MODULE) + bool (*kgd2kfd_init_p)(unsigned, const struct kgd2kfd_calls**); +#endif + + switch (rdev->asic_type) { +#ifdef CONFIG_DRM_AMDGPU_CIK + case CHIP_KAVERI: + kfd2kgd = amdgpu_amdkfd_gfx_7_get_functions(); + break; +#endif + case CHIP_CARRIZO: + kfd2kgd = amdgpu_amdkfd_gfx_8_0_get_functions(); + break; + default: + return false; + } + +#if defined(CONFIG_HSA_AMD_MODULE) + kgd2kfd_init_p = symbol_request(kgd2kfd_init); + + if (kgd2kfd_init_p == NULL) { + kfd2kgd = NULL; + return false; + } + + if (!kgd2kfd_init_p(KFD_INTERFACE_VERSION, &kgd2kfd)) { + symbol_put(kgd2kfd_init); + kfd2kgd = NULL; + kgd2kfd = NULL; + + return false; + } + + return true; +#elif defined(CONFIG_HSA_AMD) + if (!kgd2kfd_init(KFD_INTERFACE_VERSION, &kgd2kfd)) { + kfd2kgd = NULL; + kgd2kfd = NULL; + return false; + } + + return true; +#else + kfd2kgd = NULL; + return false; +#endif +} + +void amdgpu_amdkfd_fini(void) +{ + if (kgd2kfd) { + kgd2kfd->exit(); + symbol_put(kgd2kfd_init); + } +} + +void amdgpu_amdkfd_device_probe(struct amdgpu_device *rdev) +{ + if (kgd2kfd) + rdev->kfd = kgd2kfd->probe((struct kgd_dev *)rdev, + rdev->pdev, kfd2kgd); +} + +void amdgpu_amdkfd_device_init(struct amdgpu_device *rdev) +{ + if (rdev->kfd) { + struct kgd2kfd_shared_resources gpu_resources = { + .compute_vmid_bitmap = 0xFF00, + + .first_compute_pipe = 1, + .compute_pipe_count = 4 - 1, + }; + + amdgpu_doorbell_get_kfd_info(rdev, + &gpu_resources.doorbell_physical_address, + &gpu_resources.doorbell_aperture_size, + &gpu_resources.doorbell_start_offset); + + kgd2kfd->device_init(rdev->kfd, &gpu_resources); + } +} + +void amdgpu_amdkfd_device_fini(struct amdgpu_device *rdev) +{ + if (rdev->kfd) { + kgd2kfd->device_exit(rdev->kfd); + rdev->kfd = NULL; + } +} + +void amdgpu_amdkfd_interrupt(struct amdgpu_device *rdev, + const void *ih_ring_entry) +{ + if (rdev->kfd) + kgd2kfd->interrupt(rdev->kfd, ih_ring_entry); +} + +void amdgpu_amdkfd_suspend(struct amdgpu_device *rdev) +{ + if (rdev->kfd) + kgd2kfd->suspend(rdev->kfd); +} + +int amdgpu_amdkfd_resume(struct amdgpu_device *rdev) +{ + int r = 0; + + if (rdev->kfd) + r = kgd2kfd->resume(rdev->kfd); + + return r; +} + +u32 pool_to_domain(enum kgd_memory_pool p) +{ + switch (p) { + case KGD_POOL_FRAMEBUFFER: return AMDGPU_GEM_DOMAIN_VRAM; + default: return AMDGPU_GEM_DOMAIN_GTT; + } +} + +int alloc_gtt_mem(struct kgd_dev *kgd, size_t size, + void **mem_obj, uint64_t *gpu_addr, + void **cpu_ptr) +{ + struct amdgpu_device *rdev = (struct amdgpu_device *)kgd; + struct kgd_mem **mem = (struct kgd_mem **) mem_obj; + int r; + + BUG_ON(kgd == NULL); + BUG_ON(gpu_addr == NULL); + BUG_ON(cpu_ptr == NULL); + + *mem = kmalloc(sizeof(struct kgd_mem), GFP_KERNEL); + if ((*mem) == NULL) + return -ENOMEM; + + r = amdgpu_bo_create(rdev, size, PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_GTT, + AMDGPU_GEM_CREATE_CPU_GTT_USWC, NULL, &(*mem)->bo); + if (r) { + dev_err(rdev->dev, + "failed to allocate BO for amdkfd (%d)\n", r); + return r; + } + + /* map the buffer */ + r = amdgpu_bo_reserve((*mem)->bo, true); + if (r) { + dev_err(rdev->dev, "(%d) failed to reserve bo for amdkfd\n", r); + goto allocate_mem_reserve_bo_failed; + } + + r = amdgpu_bo_pin((*mem)->bo, AMDGPU_GEM_DOMAIN_GTT, + &(*mem)->gpu_addr); + if (r) { + dev_err(rdev->dev, "(%d) failed to pin bo for amdkfd\n", r); + goto allocate_mem_pin_bo_failed; + } + *gpu_addr = (*mem)->gpu_addr; + + r = amdgpu_bo_kmap((*mem)->bo, &(*mem)->cpu_ptr); + if (r) { + dev_err(rdev->dev, + "(%d) failed to map bo to kernel for amdkfd\n", r); + goto allocate_mem_kmap_bo_failed; + } + *cpu_ptr = (*mem)->cpu_ptr; + + amdgpu_bo_unreserve((*mem)->bo); + + return 0; + +allocate_mem_kmap_bo_failed: + amdgpu_bo_unpin((*mem)->bo); +allocate_mem_pin_bo_failed: + amdgpu_bo_unreserve((*mem)->bo); +allocate_mem_reserve_bo_failed: + amdgpu_bo_unref(&(*mem)->bo); + + return r; +} + +void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj) +{ + struct kgd_mem *mem = (struct kgd_mem *) mem_obj; + + BUG_ON(mem == NULL); + + amdgpu_bo_reserve(mem->bo, true); + amdgpu_bo_kunmap(mem->bo); + amdgpu_bo_unpin(mem->bo); + amdgpu_bo_unreserve(mem->bo); + amdgpu_bo_unref(&(mem->bo)); + kfree(mem); +} + +uint64_t get_vmem_size(struct kgd_dev *kgd) +{ + struct amdgpu_device *rdev = + (struct amdgpu_device *)kgd; + + BUG_ON(kgd == NULL); + + return rdev->mc.real_vram_size; +} + +uint64_t get_gpu_clock_counter(struct kgd_dev *kgd) +{ + struct amdgpu_device *rdev = (struct amdgpu_device *)kgd; + + if (rdev->asic_funcs->get_gpu_clock_counter) + return rdev->asic_funcs->get_gpu_clock_counter(rdev); + return 0; +} + +uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd) +{ + struct amdgpu_device *rdev = (struct amdgpu_device *)kgd; + + /* The sclk is in quantas of 10kHz */ + return rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk / 100; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h new file mode 100644 index 000000000000..a8be765542e6 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -0,0 +1,65 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/* amdgpu_amdkfd.h defines the private interface between amdgpu and amdkfd. */ + +#ifndef AMDGPU_AMDKFD_H_INCLUDED +#define AMDGPU_AMDKFD_H_INCLUDED + +#include +#include + +struct amdgpu_device; + +struct kgd_mem { + struct amdgpu_bo *bo; + uint64_t gpu_addr; + void *cpu_ptr; +}; + +bool amdgpu_amdkfd_init(void); +void amdgpu_amdkfd_fini(void); + +bool amdgpu_amdkfd_load_interface(struct amdgpu_device *rdev); + +void amdgpu_amdkfd_suspend(struct amdgpu_device *rdev); +int amdgpu_amdkfd_resume(struct amdgpu_device *rdev); +void amdgpu_amdkfd_interrupt(struct amdgpu_device *rdev, + const void *ih_ring_entry); +void amdgpu_amdkfd_device_probe(struct amdgpu_device *rdev); +void amdgpu_amdkfd_device_init(struct amdgpu_device *rdev); +void amdgpu_amdkfd_device_fini(struct amdgpu_device *rdev); + +struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void); +struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void); + +/* Shared API */ +int alloc_gtt_mem(struct kgd_dev *kgd, size_t size, + void **mem_obj, uint64_t *gpu_addr, + void **cpu_ptr); +void free_gtt_mem(struct kgd_dev *kgd, void *mem_obj); +uint64_t get_vmem_size(struct kgd_dev *kgd); +uint64_t get_gpu_clock_counter(struct kgd_dev *kgd); + +uint32_t get_max_engine_clock_in_mhz(struct kgd_dev *kgd); + +#endif /* AMDGPU_AMDKFD_H_INCLUDED */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c new file mode 100644 index 000000000000..dd2037bc0b4a --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c @@ -0,0 +1,670 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include "amdgpu.h" +#include "amdgpu_amdkfd.h" +#include "cikd.h" +#include "cik_sdma.h" +#include "amdgpu_ucode.h" +#include "gca/gfx_7_2_d.h" +#include "gca/gfx_7_2_enum.h" +#include "gca/gfx_7_2_sh_mask.h" +#include "oss/oss_2_0_d.h" +#include "oss/oss_2_0_sh_mask.h" +#include "gmc/gmc_7_1_d.h" +#include "gmc/gmc_7_1_sh_mask.h" +#include "cik_structs.h" + +#define CIK_PIPE_PER_MEC (4) + +enum { + MAX_TRAPID = 8, /* 3 bits in the bitfield. */ + MAX_WATCH_ADDRESSES = 4 +}; + +enum { + ADDRESS_WATCH_REG_ADDR_HI = 0, + ADDRESS_WATCH_REG_ADDR_LO, + ADDRESS_WATCH_REG_CNTL, + ADDRESS_WATCH_REG_MAX +}; + +/* not defined in the CI/KV reg file */ +enum { + ADDRESS_WATCH_REG_CNTL_ATC_BIT = 0x10000000UL, + ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK = 0x00FFFFFF, + ADDRESS_WATCH_REG_ADDLOW_MASK_EXTENSION = 0x03000000, + /* extend the mask to 26 bits to match the low address field */ + ADDRESS_WATCH_REG_ADDLOW_SHIFT = 6, + ADDRESS_WATCH_REG_ADDHIGH_MASK = 0xFFFF +}; + +static const uint32_t watchRegs[MAX_WATCH_ADDRESSES * ADDRESS_WATCH_REG_MAX] = { + mmTCP_WATCH0_ADDR_H, mmTCP_WATCH0_ADDR_L, mmTCP_WATCH0_CNTL, + mmTCP_WATCH1_ADDR_H, mmTCP_WATCH1_ADDR_L, mmTCP_WATCH1_CNTL, + mmTCP_WATCH2_ADDR_H, mmTCP_WATCH2_ADDR_L, mmTCP_WATCH2_CNTL, + mmTCP_WATCH3_ADDR_H, mmTCP_WATCH3_ADDR_L, mmTCP_WATCH3_CNTL +}; + +union TCP_WATCH_CNTL_BITS { + struct { + uint32_t mask:24; + uint32_t vmid:4; + uint32_t atc:1; + uint32_t mode:2; + uint32_t valid:1; + } bitfields, bits; + uint32_t u32All; + signed int i32All; + float f32All; +}; + +/* + * Register access functions + */ + +static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, + uint32_t sh_mem_config, uint32_t sh_mem_ape1_base, + uint32_t sh_mem_ape1_limit, uint32_t sh_mem_bases); + +static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, + unsigned int vmid); + +static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id, + uint32_t hpd_size, uint64_t hpd_gpu_addr); +static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id); +static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, + uint32_t queue_id, uint32_t __user *wptr); +static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd); +static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address, + uint32_t pipe_id, uint32_t queue_id); + +static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type, + unsigned int timeout, uint32_t pipe_id, + uint32_t queue_id); +static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd); +static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, + unsigned int timeout); +static int kgd_address_watch_disable(struct kgd_dev *kgd); +static int kgd_address_watch_execute(struct kgd_dev *kgd, + unsigned int watch_point_id, + uint32_t cntl_val, + uint32_t addr_hi, + uint32_t addr_lo); +static int kgd_wave_control_execute(struct kgd_dev *kgd, + uint32_t gfx_index_val, + uint32_t sq_cmd); +static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd, + unsigned int watch_point_id, + unsigned int reg_offset); + +static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, uint8_t vmid); +static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd, + uint8_t vmid); +static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid); + +static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type); + +static const struct kfd2kgd_calls kfd2kgd = { + .init_gtt_mem_allocation = alloc_gtt_mem, + .free_gtt_mem = free_gtt_mem, + .get_vmem_size = get_vmem_size, + .get_gpu_clock_counter = get_gpu_clock_counter, + .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz, + .program_sh_mem_settings = kgd_program_sh_mem_settings, + .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping, + .init_pipeline = kgd_init_pipeline, + .init_interrupts = kgd_init_interrupts, + .hqd_load = kgd_hqd_load, + .hqd_sdma_load = kgd_hqd_sdma_load, + .hqd_is_occupied = kgd_hqd_is_occupied, + .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied, + .hqd_destroy = kgd_hqd_destroy, + .hqd_sdma_destroy = kgd_hqd_sdma_destroy, + .address_watch_disable = kgd_address_watch_disable, + .address_watch_execute = kgd_address_watch_execute, + .wave_control_execute = kgd_wave_control_execute, + .address_watch_get_offset = kgd_address_watch_get_offset, + .get_atc_vmid_pasid_mapping_pasid = get_atc_vmid_pasid_mapping_pasid, + .get_atc_vmid_pasid_mapping_valid = get_atc_vmid_pasid_mapping_valid, + .write_vmid_invalidate_request = write_vmid_invalidate_request, + .get_fw_version = get_fw_version +}; + +struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions() +{ + return (struct kfd2kgd_calls *)&kfd2kgd; +} + +static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd) +{ + return (struct amdgpu_device *)kgd; +} + +static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe, + uint32_t queue, uint32_t vmid) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue); + + mutex_lock(&adev->srbm_mutex); + WREG32(mmSRBM_GFX_CNTL, value); +} + +static void unlock_srbm(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + WREG32(mmSRBM_GFX_CNTL, 0); + mutex_unlock(&adev->srbm_mutex); +} + +static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id, + uint32_t queue_id) +{ + uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1; + uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC); + + lock_srbm(kgd, mec, pipe, queue_id, 0); +} + +static void release_queue(struct kgd_dev *kgd) +{ + unlock_srbm(kgd); +} + +static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, + uint32_t sh_mem_config, + uint32_t sh_mem_ape1_base, + uint32_t sh_mem_ape1_limit, + uint32_t sh_mem_bases) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + lock_srbm(kgd, 0, 0, 0, vmid); + + WREG32(mmSH_MEM_CONFIG, sh_mem_config); + WREG32(mmSH_MEM_APE1_BASE, sh_mem_ape1_base); + WREG32(mmSH_MEM_APE1_LIMIT, sh_mem_ape1_limit); + WREG32(mmSH_MEM_BASES, sh_mem_bases); + + unlock_srbm(kgd); +} + +static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, + unsigned int vmid) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + /* + * We have to assume that there is no outstanding mapping. + * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because + * a mapping is in progress or because a mapping finished and the + * SW cleared it. So the protocol is to always wait & clear. + */ + uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid | + ATC_VMID0_PASID_MAPPING__VALID_MASK; + + WREG32(mmATC_VMID0_PASID_MAPPING + vmid, pasid_mapping); + + while (!(RREG32(mmATC_VMID_PASID_MAPPING_UPDATE_STATUS) & (1U << vmid))) + cpu_relax(); + WREG32(mmATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid); + + /* Mapping vmid to pasid also for IH block */ + WREG32(mmIH_VMID_0_LUT + vmid, pasid_mapping); + + return 0; +} + +static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id, + uint32_t hpd_size, uint64_t hpd_gpu_addr) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + uint32_t mec = (++pipe_id / CIK_PIPE_PER_MEC) + 1; + uint32_t pipe = (pipe_id % CIK_PIPE_PER_MEC); + + lock_srbm(kgd, mec, pipe, 0, 0); + WREG32(mmCP_HPD_EOP_BASE_ADDR, lower_32_bits(hpd_gpu_addr >> 8)); + WREG32(mmCP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(hpd_gpu_addr >> 8)); + WREG32(mmCP_HPD_EOP_VMID, 0); + WREG32(mmCP_HPD_EOP_CONTROL, hpd_size); + unlock_srbm(kgd); + + return 0; +} + +static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t mec; + uint32_t pipe; + + mec = (pipe_id / CIK_PIPE_PER_MEC) + 1; + pipe = (pipe_id % CIK_PIPE_PER_MEC); + + lock_srbm(kgd, mec, pipe, 0, 0); + + WREG32(mmCPC_INT_CNTL, CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK | + CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK); + + unlock_srbm(kgd); + + return 0; +} + +static inline uint32_t get_sdma_base_addr(struct cik_sdma_rlc_registers *m) +{ + uint32_t retval; + + retval = m->sdma_engine_id * SDMA1_REGISTER_OFFSET + + m->sdma_queue_id * KFD_CIK_SDMA_QUEUE_OFFSET; + + pr_debug("kfd: sdma base address: 0x%x\n", retval); + + return retval; +} + +static inline struct cik_mqd *get_mqd(void *mqd) +{ + return (struct cik_mqd *)mqd; +} + +static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd) +{ + return (struct cik_sdma_rlc_registers *)mqd; +} + +static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, + uint32_t queue_id, uint32_t __user *wptr) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t wptr_shadow, is_wptr_shadow_valid; + struct cik_mqd *m; + + m = get_mqd(mqd); + + is_wptr_shadow_valid = !get_user(wptr_shadow, wptr); + + acquire_queue(kgd, pipe_id, queue_id); + WREG32(mmCP_MQD_BASE_ADDR, m->cp_mqd_base_addr_lo); + WREG32(mmCP_MQD_BASE_ADDR_HI, m->cp_mqd_base_addr_hi); + WREG32(mmCP_MQD_CONTROL, m->cp_mqd_control); + + WREG32(mmCP_HQD_PQ_BASE, m->cp_hqd_pq_base_lo); + WREG32(mmCP_HQD_PQ_BASE_HI, m->cp_hqd_pq_base_hi); + WREG32(mmCP_HQD_PQ_CONTROL, m->cp_hqd_pq_control); + + WREG32(mmCP_HQD_IB_CONTROL, m->cp_hqd_ib_control); + WREG32(mmCP_HQD_IB_BASE_ADDR, m->cp_hqd_ib_base_addr_lo); + WREG32(mmCP_HQD_IB_BASE_ADDR_HI, m->cp_hqd_ib_base_addr_hi); + + WREG32(mmCP_HQD_IB_RPTR, m->cp_hqd_ib_rptr); + + WREG32(mmCP_HQD_PERSISTENT_STATE, m->cp_hqd_persistent_state); + WREG32(mmCP_HQD_SEMA_CMD, m->cp_hqd_sema_cmd); + WREG32(mmCP_HQD_MSG_TYPE, m->cp_hqd_msg_type); + + WREG32(mmCP_HQD_ATOMIC0_PREOP_LO, m->cp_hqd_atomic0_preop_lo); + WREG32(mmCP_HQD_ATOMIC0_PREOP_HI, m->cp_hqd_atomic0_preop_hi); + WREG32(mmCP_HQD_ATOMIC1_PREOP_LO, m->cp_hqd_atomic1_preop_lo); + WREG32(mmCP_HQD_ATOMIC1_PREOP_HI, m->cp_hqd_atomic1_preop_hi); + + WREG32(mmCP_HQD_PQ_RPTR_REPORT_ADDR, m->cp_hqd_pq_rptr_report_addr_lo); + WREG32(mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI, + m->cp_hqd_pq_rptr_report_addr_hi); + + WREG32(mmCP_HQD_PQ_RPTR, m->cp_hqd_pq_rptr); + + WREG32(mmCP_HQD_PQ_WPTR_POLL_ADDR, m->cp_hqd_pq_wptr_poll_addr_lo); + WREG32(mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, m->cp_hqd_pq_wptr_poll_addr_hi); + + WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, m->cp_hqd_pq_doorbell_control); + + WREG32(mmCP_HQD_VMID, m->cp_hqd_vmid); + + WREG32(mmCP_HQD_QUANTUM, m->cp_hqd_quantum); + + WREG32(mmCP_HQD_PIPE_PRIORITY, m->cp_hqd_pipe_priority); + WREG32(mmCP_HQD_QUEUE_PRIORITY, m->cp_hqd_queue_priority); + + WREG32(mmCP_HQD_IQ_RPTR, m->cp_hqd_iq_rptr); + + if (is_wptr_shadow_valid) + WREG32(mmCP_HQD_PQ_WPTR, wptr_shadow); + + WREG32(mmCP_HQD_ACTIVE, m->cp_hqd_active); + release_queue(kgd); + + return 0; +} + +static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct cik_sdma_rlc_registers *m; + uint32_t sdma_base_addr; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(m); + + WREG32(sdma_base_addr + mmSDMA0_RLC0_VIRTUAL_ADDR, + m->sdma_rlc_virtual_addr); + + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, + m->sdma_rlc_rb_base); + + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE_HI, + m->sdma_rlc_rb_base_hi); + + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_LO, + m->sdma_rlc_rb_rptr_addr_lo); + + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR_ADDR_HI, + m->sdma_rlc_rb_rptr_addr_hi); + + WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, + m->sdma_rlc_doorbell); + + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, + m->sdma_rlc_rb_cntl); + + return 0; +} + +static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address, + uint32_t pipe_id, uint32_t queue_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t act; + bool retval = false; + uint32_t low, high; + + acquire_queue(kgd, pipe_id, queue_id); + act = RREG32(mmCP_HQD_ACTIVE); + if (act) { + low = lower_32_bits(queue_address >> 8); + high = upper_32_bits(queue_address >> 8); + + if (low == RREG32(mmCP_HQD_PQ_BASE) && + high == RREG32(mmCP_HQD_PQ_BASE_HI)) + retval = true; + } + release_queue(kgd); + return retval; +} + +static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct cik_sdma_rlc_registers *m; + uint32_t sdma_base_addr; + uint32_t sdma_rlc_rb_cntl; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(m); + + sdma_rlc_rb_cntl = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL); + + if (sdma_rlc_rb_cntl & SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK) + return true; + + return false; +} + +static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type, + unsigned int timeout, uint32_t pipe_id, + uint32_t queue_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t temp; + + acquire_queue(kgd, pipe_id, queue_id); + WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, 0); + + WREG32(mmCP_HQD_DEQUEUE_REQUEST, reset_type); + + while (true) { + temp = RREG32(mmCP_HQD_ACTIVE); + if (temp & CP_HQD_ACTIVE__ACTIVE_MASK) + break; + if (timeout == 0) { + pr_err("kfd: cp queue preemption time out (%dms)\n", + temp); + release_queue(kgd); + return -ETIME; + } + msleep(20); + timeout -= 20; + } + + release_queue(kgd); + return 0; +} + +static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, + unsigned int timeout) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct cik_sdma_rlc_registers *m; + uint32_t sdma_base_addr; + uint32_t temp; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(m); + + temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL); + temp = temp & ~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK; + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, temp); + + while (true) { + temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS); + if (temp & SDMA0_STATUS_REG__RB_CMD_IDLE__SHIFT) + break; + if (timeout == 0) + return -ETIME; + msleep(20); + timeout -= 20; + } + + WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, 0); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, 0); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, 0); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, 0); + + return 0; +} + +static int kgd_address_watch_disable(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + union TCP_WATCH_CNTL_BITS cntl; + unsigned int i; + + cntl.u32All = 0; + + cntl.bitfields.valid = 0; + cntl.bitfields.mask = ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK; + cntl.bitfields.atc = 1; + + /* Turning off this address until we set all the registers */ + for (i = 0; i < MAX_WATCH_ADDRESSES; i++) + WREG32(watchRegs[i * ADDRESS_WATCH_REG_MAX + + ADDRESS_WATCH_REG_CNTL], cntl.u32All); + + return 0; +} + +static int kgd_address_watch_execute(struct kgd_dev *kgd, + unsigned int watch_point_id, + uint32_t cntl_val, + uint32_t addr_hi, + uint32_t addr_lo) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + union TCP_WATCH_CNTL_BITS cntl; + + cntl.u32All = cntl_val; + + /* Turning off this watch point until we set all the registers */ + cntl.bitfields.valid = 0; + WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + + ADDRESS_WATCH_REG_CNTL], cntl.u32All); + + WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + + ADDRESS_WATCH_REG_ADDR_HI], addr_hi); + + WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + + ADDRESS_WATCH_REG_ADDR_LO], addr_lo); + + /* Enable the watch point */ + cntl.bitfields.valid = 1; + + WREG32(watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + + ADDRESS_WATCH_REG_CNTL], cntl.u32All); + + return 0; +} + +static int kgd_wave_control_execute(struct kgd_dev *kgd, + uint32_t gfx_index_val, + uint32_t sq_cmd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t data; + + mutex_lock(&adev->grbm_idx_mutex); + + WREG32(mmGRBM_GFX_INDEX, gfx_index_val); + WREG32(mmSQ_CMD, sq_cmd); + + /* Restore the GRBM_GFX_INDEX register */ + + data = GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK | + GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK | + GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK; + + WREG32(mmGRBM_GFX_INDEX, data); + + mutex_unlock(&adev->grbm_idx_mutex); + + return 0; +} + +static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd, + unsigned int watch_point_id, + unsigned int reg_offset) +{ + return watchRegs[watch_point_id * ADDRESS_WATCH_REG_MAX + reg_offset]; +} + +static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, + uint8_t vmid) +{ + uint32_t reg; + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + reg = RREG32(mmATC_VMID0_PASID_MAPPING + vmid); + return reg & ATC_VMID0_PASID_MAPPING__VALID_MASK; +} + +static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd, + uint8_t vmid) +{ + uint32_t reg; + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + reg = RREG32(mmATC_VMID0_PASID_MAPPING + vmid); + return reg & ATC_VMID0_PASID_MAPPING__VALID_MASK; +} + +static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid) +{ + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid); +} + +static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type) +{ + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + const union amdgpu_firmware_header *hdr; + + BUG_ON(kgd == NULL); + + switch (type) { + case KGD_ENGINE_PFP: + hdr = (const union amdgpu_firmware_header *) + adev->gfx.pfp_fw->data; + break; + + case KGD_ENGINE_ME: + hdr = (const union amdgpu_firmware_header *) + adev->gfx.me_fw->data; + break; + + case KGD_ENGINE_CE: + hdr = (const union amdgpu_firmware_header *) + adev->gfx.ce_fw->data; + break; + + case KGD_ENGINE_MEC1: + hdr = (const union amdgpu_firmware_header *) + adev->gfx.mec_fw->data; + break; + + case KGD_ENGINE_MEC2: + hdr = (const union amdgpu_firmware_header *) + adev->gfx.mec2_fw->data; + break; + + case KGD_ENGINE_RLC: + hdr = (const union amdgpu_firmware_header *) + adev->gfx.rlc_fw->data; + break; + + case KGD_ENGINE_SDMA1: + hdr = (const union amdgpu_firmware_header *) + adev->sdma[0].fw->data; + break; + + case KGD_ENGINE_SDMA2: + hdr = (const union amdgpu_firmware_header *) + adev->sdma[1].fw->data; + break; + + default: + return 0; + } + + if (hdr == NULL) + return 0; + + /* Only 12 bit in use*/ + return hdr->common.ucode_version; +} + diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c new file mode 100644 index 000000000000..dfd1d503bccf --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c @@ -0,0 +1,543 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include +#include +#include +#include +#include +#include "amdgpu.h" +#include "amdgpu_amdkfd.h" +#include "amdgpu_ucode.h" +#include "gca/gfx_8_0_sh_mask.h" +#include "gca/gfx_8_0_d.h" +#include "gca/gfx_8_0_enum.h" +#include "oss/oss_3_0_sh_mask.h" +#include "oss/oss_3_0_d.h" +#include "gmc/gmc_8_1_sh_mask.h" +#include "gmc/gmc_8_1_d.h" +#include "vi_structs.h" +#include "vid.h" + +#define VI_PIPE_PER_MEC (4) + +struct cik_sdma_rlc_registers; + +/* + * Register access functions + */ + +static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, + uint32_t sh_mem_config, + uint32_t sh_mem_ape1_base, uint32_t sh_mem_ape1_limit, + uint32_t sh_mem_bases); +static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, + unsigned int vmid); +static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id, + uint32_t hpd_size, uint64_t hpd_gpu_addr); +static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id); +static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, + uint32_t queue_id, uint32_t __user *wptr); +static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd); +static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address, + uint32_t pipe_id, uint32_t queue_id); +static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd); +static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type, + unsigned int timeout, uint32_t pipe_id, + uint32_t queue_id); +static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, + unsigned int timeout); +static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid); +static int kgd_address_watch_disable(struct kgd_dev *kgd); +static int kgd_address_watch_execute(struct kgd_dev *kgd, + unsigned int watch_point_id, + uint32_t cntl_val, + uint32_t addr_hi, + uint32_t addr_lo); +static int kgd_wave_control_execute(struct kgd_dev *kgd, + uint32_t gfx_index_val, + uint32_t sq_cmd); +static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd, + unsigned int watch_point_id, + unsigned int reg_offset); + +static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, + uint8_t vmid); +static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd, + uint8_t vmid); +static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid); +static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type); + +static const struct kfd2kgd_calls kfd2kgd = { + .init_gtt_mem_allocation = alloc_gtt_mem, + .free_gtt_mem = free_gtt_mem, + .get_vmem_size = get_vmem_size, + .get_gpu_clock_counter = get_gpu_clock_counter, + .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz, + .program_sh_mem_settings = kgd_program_sh_mem_settings, + .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping, + .init_pipeline = kgd_init_pipeline, + .init_interrupts = kgd_init_interrupts, + .hqd_load = kgd_hqd_load, + .hqd_sdma_load = kgd_hqd_sdma_load, + .hqd_is_occupied = kgd_hqd_is_occupied, + .hqd_sdma_is_occupied = kgd_hqd_sdma_is_occupied, + .hqd_destroy = kgd_hqd_destroy, + .hqd_sdma_destroy = kgd_hqd_sdma_destroy, + .address_watch_disable = kgd_address_watch_disable, + .address_watch_execute = kgd_address_watch_execute, + .wave_control_execute = kgd_wave_control_execute, + .address_watch_get_offset = kgd_address_watch_get_offset, + .get_atc_vmid_pasid_mapping_pasid = + get_atc_vmid_pasid_mapping_pasid, + .get_atc_vmid_pasid_mapping_valid = + get_atc_vmid_pasid_mapping_valid, + .write_vmid_invalidate_request = write_vmid_invalidate_request, + .get_fw_version = get_fw_version +}; + +struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions() +{ + return (struct kfd2kgd_calls *)&kfd2kgd; +} + +static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd) +{ + return (struct amdgpu_device *)kgd; +} + +static void lock_srbm(struct kgd_dev *kgd, uint32_t mec, uint32_t pipe, + uint32_t queue, uint32_t vmid) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t value = PIPEID(pipe) | MEID(mec) | VMID(vmid) | QUEUEID(queue); + + mutex_lock(&adev->srbm_mutex); + WREG32(mmSRBM_GFX_CNTL, value); +} + +static void unlock_srbm(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + WREG32(mmSRBM_GFX_CNTL, 0); + mutex_unlock(&adev->srbm_mutex); +} + +static void acquire_queue(struct kgd_dev *kgd, uint32_t pipe_id, + uint32_t queue_id) +{ + uint32_t mec = (++pipe_id / VI_PIPE_PER_MEC) + 1; + uint32_t pipe = (pipe_id % VI_PIPE_PER_MEC); + + lock_srbm(kgd, mec, pipe, queue_id, 0); +} + +static void release_queue(struct kgd_dev *kgd) +{ + unlock_srbm(kgd); +} + +static void kgd_program_sh_mem_settings(struct kgd_dev *kgd, uint32_t vmid, + uint32_t sh_mem_config, + uint32_t sh_mem_ape1_base, + uint32_t sh_mem_ape1_limit, + uint32_t sh_mem_bases) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + lock_srbm(kgd, 0, 0, 0, vmid); + + WREG32(mmSH_MEM_CONFIG, sh_mem_config); + WREG32(mmSH_MEM_APE1_BASE, sh_mem_ape1_base); + WREG32(mmSH_MEM_APE1_LIMIT, sh_mem_ape1_limit); + WREG32(mmSH_MEM_BASES, sh_mem_bases); + + unlock_srbm(kgd); +} + +static int kgd_set_pasid_vmid_mapping(struct kgd_dev *kgd, unsigned int pasid, + unsigned int vmid) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + /* + * We have to assume that there is no outstanding mapping. + * The ATC_VMID_PASID_MAPPING_UPDATE_STATUS bit could be 0 because + * a mapping is in progress or because a mapping finished + * and the SW cleared it. + * So the protocol is to always wait & clear. + */ + uint32_t pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid | + ATC_VMID0_PASID_MAPPING__VALID_MASK; + + WREG32(mmATC_VMID0_PASID_MAPPING + vmid, pasid_mapping); + + while (!(RREG32(mmATC_VMID_PASID_MAPPING_UPDATE_STATUS) & (1U << vmid))) + cpu_relax(); + WREG32(mmATC_VMID_PASID_MAPPING_UPDATE_STATUS, 1U << vmid); + + /* Mapping vmid to pasid also for IH block */ + WREG32(mmIH_VMID_0_LUT + vmid, pasid_mapping); + + return 0; +} + +static int kgd_init_pipeline(struct kgd_dev *kgd, uint32_t pipe_id, + uint32_t hpd_size, uint64_t hpd_gpu_addr) +{ + return 0; +} + +static int kgd_init_interrupts(struct kgd_dev *kgd, uint32_t pipe_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t mec; + uint32_t pipe; + + mec = (++pipe_id / VI_PIPE_PER_MEC) + 1; + pipe = (pipe_id % VI_PIPE_PER_MEC); + + lock_srbm(kgd, mec, pipe, 0, 0); + + WREG32(mmCPC_INT_CNTL, CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK); + + unlock_srbm(kgd); + + return 0; +} + +static inline uint32_t get_sdma_base_addr(struct cik_sdma_rlc_registers *m) +{ + return 0; +} + +static inline struct vi_mqd *get_mqd(void *mqd) +{ + return (struct vi_mqd *)mqd; +} + +static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd) +{ + return (struct cik_sdma_rlc_registers *)mqd; +} + +static int kgd_hqd_load(struct kgd_dev *kgd, void *mqd, uint32_t pipe_id, + uint32_t queue_id, uint32_t __user *wptr) +{ + struct vi_mqd *m; + uint32_t shadow_wptr, valid_wptr; + struct amdgpu_device *adev = get_amdgpu_device(kgd); + + m = get_mqd(mqd); + + valid_wptr = copy_from_user(&shadow_wptr, wptr, sizeof(shadow_wptr)); + acquire_queue(kgd, pipe_id, queue_id); + + WREG32(mmCP_MQD_CONTROL, m->cp_mqd_control); + WREG32(mmCP_MQD_BASE_ADDR, m->cp_mqd_base_addr_lo); + WREG32(mmCP_MQD_BASE_ADDR_HI, m->cp_mqd_base_addr_hi); + + WREG32(mmCP_HQD_VMID, m->cp_hqd_vmid); + WREG32(mmCP_HQD_PERSISTENT_STATE, m->cp_hqd_persistent_state); + WREG32(mmCP_HQD_PIPE_PRIORITY, m->cp_hqd_pipe_priority); + WREG32(mmCP_HQD_QUEUE_PRIORITY, m->cp_hqd_queue_priority); + WREG32(mmCP_HQD_QUANTUM, m->cp_hqd_quantum); + WREG32(mmCP_HQD_PQ_BASE, m->cp_hqd_pq_base_lo); + WREG32(mmCP_HQD_PQ_BASE_HI, m->cp_hqd_pq_base_hi); + WREG32(mmCP_HQD_PQ_RPTR_REPORT_ADDR, m->cp_hqd_pq_rptr_report_addr_lo); + WREG32(mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI, + m->cp_hqd_pq_rptr_report_addr_hi); + + if (valid_wptr > 0) + WREG32(mmCP_HQD_PQ_WPTR, shadow_wptr); + + WREG32(mmCP_HQD_PQ_CONTROL, m->cp_hqd_pq_control); + WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, m->cp_hqd_pq_doorbell_control); + + WREG32(mmCP_HQD_EOP_BASE_ADDR, m->cp_hqd_eop_base_addr_lo); + WREG32(mmCP_HQD_EOP_BASE_ADDR_HI, m->cp_hqd_eop_base_addr_hi); + WREG32(mmCP_HQD_EOP_CONTROL, m->cp_hqd_eop_control); + WREG32(mmCP_HQD_EOP_RPTR, m->cp_hqd_eop_rptr); + WREG32(mmCP_HQD_EOP_WPTR, m->cp_hqd_eop_wptr); + WREG32(mmCP_HQD_EOP_EVENTS, m->cp_hqd_eop_done_events); + + WREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_LO, m->cp_hqd_ctx_save_base_addr_lo); + WREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_HI, m->cp_hqd_ctx_save_base_addr_hi); + WREG32(mmCP_HQD_CTX_SAVE_CONTROL, m->cp_hqd_ctx_save_control); + WREG32(mmCP_HQD_CNTL_STACK_OFFSET, m->cp_hqd_cntl_stack_offset); + WREG32(mmCP_HQD_CNTL_STACK_SIZE, m->cp_hqd_cntl_stack_size); + WREG32(mmCP_HQD_WG_STATE_OFFSET, m->cp_hqd_wg_state_offset); + WREG32(mmCP_HQD_CTX_SAVE_SIZE, m->cp_hqd_ctx_save_size); + + WREG32(mmCP_HQD_IB_CONTROL, m->cp_hqd_ib_control); + + WREG32(mmCP_HQD_DEQUEUE_REQUEST, m->cp_hqd_dequeue_request); + WREG32(mmCP_HQD_ERROR, m->cp_hqd_error); + WREG32(mmCP_HQD_EOP_WPTR_MEM, m->cp_hqd_eop_wptr_mem); + WREG32(mmCP_HQD_EOP_DONES, m->cp_hqd_eop_dones); + + WREG32(mmCP_HQD_ACTIVE, m->cp_hqd_active); + + release_queue(kgd); + + return 0; +} + +static int kgd_hqd_sdma_load(struct kgd_dev *kgd, void *mqd) +{ + return 0; +} + +static bool kgd_hqd_is_occupied(struct kgd_dev *kgd, uint64_t queue_address, + uint32_t pipe_id, uint32_t queue_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t act; + bool retval = false; + uint32_t low, high; + + acquire_queue(kgd, pipe_id, queue_id); + act = RREG32(mmCP_HQD_ACTIVE); + if (act) { + low = lower_32_bits(queue_address >> 8); + high = upper_32_bits(queue_address >> 8); + + if (low == RREG32(mmCP_HQD_PQ_BASE) && + high == RREG32(mmCP_HQD_PQ_BASE_HI)) + retval = true; + } + release_queue(kgd); + return retval; +} + +static bool kgd_hqd_sdma_is_occupied(struct kgd_dev *kgd, void *mqd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct cik_sdma_rlc_registers *m; + uint32_t sdma_base_addr; + uint32_t sdma_rlc_rb_cntl; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(m); + + sdma_rlc_rb_cntl = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL); + + if (sdma_rlc_rb_cntl & SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK) + return true; + + return false; +} + +static int kgd_hqd_destroy(struct kgd_dev *kgd, uint32_t reset_type, + unsigned int timeout, uint32_t pipe_id, + uint32_t queue_id) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t temp; + + acquire_queue(kgd, pipe_id, queue_id); + + WREG32(mmCP_HQD_DEQUEUE_REQUEST, reset_type); + + while (true) { + temp = RREG32(mmCP_HQD_ACTIVE); + if (temp & CP_HQD_ACTIVE__ACTIVE_MASK) + break; + if (timeout == 0) { + pr_err("kfd: cp queue preemption time out (%dms)\n", + temp); + release_queue(kgd); + return -ETIME; + } + msleep(20); + timeout -= 20; + } + + release_queue(kgd); + return 0; +} + +static int kgd_hqd_sdma_destroy(struct kgd_dev *kgd, void *mqd, + unsigned int timeout) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + struct cik_sdma_rlc_registers *m; + uint32_t sdma_base_addr; + uint32_t temp; + + m = get_sdma_mqd(mqd); + sdma_base_addr = get_sdma_base_addr(m); + + temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL); + temp = temp & ~SDMA0_RLC0_RB_CNTL__RB_ENABLE_MASK; + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_CNTL, temp); + + while (true) { + temp = RREG32(sdma_base_addr + mmSDMA0_RLC0_CONTEXT_STATUS); + if (temp & SDMA0_STATUS_REG__RB_CMD_IDLE__SHIFT) + break; + if (timeout == 0) + return -ETIME; + msleep(20); + timeout -= 20; + } + + WREG32(sdma_base_addr + mmSDMA0_RLC0_DOORBELL, 0); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_RPTR, 0); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_WPTR, 0); + WREG32(sdma_base_addr + mmSDMA0_RLC0_RB_BASE, 0); + + return 0; +} + +static bool get_atc_vmid_pasid_mapping_valid(struct kgd_dev *kgd, + uint8_t vmid) +{ + uint32_t reg; + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + reg = RREG32(mmATC_VMID0_PASID_MAPPING + vmid); + return reg & ATC_VMID0_PASID_MAPPING__VALID_MASK; +} + +static uint16_t get_atc_vmid_pasid_mapping_pasid(struct kgd_dev *kgd, + uint8_t vmid) +{ + uint32_t reg; + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + reg = RREG32(mmATC_VMID0_PASID_MAPPING + vmid); + return reg & ATC_VMID0_PASID_MAPPING__VALID_MASK; +} + +static void write_vmid_invalidate_request(struct kgd_dev *kgd, uint8_t vmid) +{ + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + + WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid); +} + +static int kgd_address_watch_disable(struct kgd_dev *kgd) +{ + return 0; +} + +static int kgd_address_watch_execute(struct kgd_dev *kgd, + unsigned int watch_point_id, + uint32_t cntl_val, + uint32_t addr_hi, + uint32_t addr_lo) +{ + return 0; +} + +static int kgd_wave_control_execute(struct kgd_dev *kgd, + uint32_t gfx_index_val, + uint32_t sq_cmd) +{ + struct amdgpu_device *adev = get_amdgpu_device(kgd); + uint32_t data = 0; + + mutex_lock(&adev->grbm_idx_mutex); + + WREG32(mmGRBM_GFX_INDEX, gfx_index_val); + WREG32(mmSQ_CMD, sq_cmd); + + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + INSTANCE_BROADCAST_WRITES, 1); + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + SH_BROADCAST_WRITES, 1); + data = REG_SET_FIELD(data, GRBM_GFX_INDEX, + SE_BROADCAST_WRITES, 1); + + WREG32(mmGRBM_GFX_INDEX, data); + mutex_unlock(&adev->grbm_idx_mutex); + + return 0; +} + +static uint32_t kgd_address_watch_get_offset(struct kgd_dev *kgd, + unsigned int watch_point_id, + unsigned int reg_offset) +{ + return 0; +} + +static uint16_t get_fw_version(struct kgd_dev *kgd, enum kgd_engine_type type) +{ + struct amdgpu_device *adev = (struct amdgpu_device *) kgd; + const union amdgpu_firmware_header *hdr; + + BUG_ON(kgd == NULL); + + switch (type) { + case KGD_ENGINE_PFP: + hdr = (const union amdgpu_firmware_header *) + adev->gfx.pfp_fw->data; + break; + + case KGD_ENGINE_ME: + hdr = (const union amdgpu_firmware_header *) + adev->gfx.me_fw->data; + break; + + case KGD_ENGINE_CE: + hdr = (const union amdgpu_firmware_header *) + adev->gfx.ce_fw->data; + break; + + case KGD_ENGINE_MEC1: + hdr = (const union amdgpu_firmware_header *) + adev->gfx.mec_fw->data; + break; + + case KGD_ENGINE_MEC2: + hdr = (const union amdgpu_firmware_header *) + adev->gfx.mec2_fw->data; + break; + + case KGD_ENGINE_RLC: + hdr = (const union amdgpu_firmware_header *) + adev->gfx.rlc_fw->data; + break; + + case KGD_ENGINE_SDMA1: + hdr = (const union amdgpu_firmware_header *) + adev->sdma[0].fw->data; + break; + + case KGD_ENGINE_SDMA2: + hdr = (const union amdgpu_firmware_header *) + adev->sdma[1].fw->data; + break; + + default: + return 0; + } + + if (hdr == NULL) + return 0; + + /* Only 12 bit in use*/ + return hdr->common.ucode_version; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c index 6a588371d54a..77f1d7c6ea3a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c @@ -897,7 +897,7 @@ bool amdgpu_atombios_get_asic_ss_info(struct amdgpu_device *adev, if ((id == ASIC_INTERNAL_ENGINE_SS) || (id == ASIC_INTERNAL_MEMORY_SS)) ss->rate /= 100; - if (adev->flags & AMDGPU_IS_APU) + if (adev->flags & AMD_IS_APU) amdgpu_atombios_get_igp_ss_overrides(adev, ss, id); return true; } @@ -1058,7 +1058,7 @@ void amdgpu_atombios_set_memory_clock(struct amdgpu_device *adev, SET_MEMORY_CLOCK_PS_ALLOCATION args; int index = GetIndexIntoMasterTable(COMMAND, SetMemoryClock); - if (adev->flags & AMDGPU_IS_APU) + if (adev->flags & AMD_IS_APU) return; args.ulTargetMemoryClock = cpu_to_le32(mem_clock); /* 10 khz */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c index 2742b9a35cbc..98d59ee640ce 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_benchmark.c @@ -33,7 +33,7 @@ static int amdgpu_benchmark_do_move(struct amdgpu_device *adev, unsigned size, { unsigned long start_jiffies; unsigned long end_jiffies; - struct amdgpu_fence *fence = NULL; + struct fence *fence = NULL; int i, r; start_jiffies = jiffies; @@ -42,17 +42,17 @@ static int amdgpu_benchmark_do_move(struct amdgpu_device *adev, unsigned size, r = amdgpu_copy_buffer(ring, saddr, daddr, size, NULL, &fence); if (r) goto exit_do_move; - r = amdgpu_fence_wait(fence, false); + r = fence_wait(fence, false); if (r) goto exit_do_move; - amdgpu_fence_unref(&fence); + fence_put(fence); } end_jiffies = jiffies; r = jiffies_to_msecs(end_jiffies - start_jiffies); exit_do_move: if (fence) - amdgpu_fence_unref(&fence); + fence_put(fence); return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c index ceb444f6d418..02add0a508cb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c @@ -48,7 +48,7 @@ static bool igp_read_bios_from_vram(struct amdgpu_device *adev) resource_size_t vram_base; resource_size_t size = 256 * 1024; /* ??? */ - if (!(adev->flags & AMDGPU_IS_APU)) + if (!(adev->flags & AMD_IS_APU)) if (!amdgpu_card_posted(adev)) return false; @@ -184,7 +184,7 @@ static bool amdgpu_atrm_get_bios(struct amdgpu_device *adev) bool found = false; /* ATRM is for the discrete card only */ - if (adev->flags & AMDGPU_IS_APU) + if (adev->flags & AMD_IS_APU) return false; while ((pdev = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, pdev)) != NULL) { @@ -246,7 +246,7 @@ static inline bool amdgpu_atrm_get_bios(struct amdgpu_device *adev) static bool amdgpu_read_disabled_bios(struct amdgpu_device *adev) { - if (adev->flags & AMDGPU_IS_APU) + if (adev->flags & AMD_IS_APU) return igp_read_bios_from_vram(adev); else return amdgpu_asic_read_disabled_bios(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c new file mode 100644 index 000000000000..6b1243f9f86d --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c @@ -0,0 +1,838 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#include +#include +#include +#include +#include +#include +#include "amdgpu.h" +#include "cgs_linux.h" +#include "atom.h" +#include "amdgpu_ucode.h" + + +struct amdgpu_cgs_device { + struct cgs_device base; + struct amdgpu_device *adev; +}; + +#define CGS_FUNC_ADEV \ + struct amdgpu_device *adev = \ + ((struct amdgpu_cgs_device *)cgs_device)->adev + +static int amdgpu_cgs_gpu_mem_info(void *cgs_device, enum cgs_gpu_mem_type type, + uint64_t *mc_start, uint64_t *mc_size, + uint64_t *mem_size) +{ + CGS_FUNC_ADEV; + switch(type) { + case CGS_GPU_MEM_TYPE__VISIBLE_CONTIG_FB: + case CGS_GPU_MEM_TYPE__VISIBLE_FB: + *mc_start = 0; + *mc_size = adev->mc.visible_vram_size; + *mem_size = adev->mc.visible_vram_size - adev->vram_pin_size; + break; + case CGS_GPU_MEM_TYPE__INVISIBLE_CONTIG_FB: + case CGS_GPU_MEM_TYPE__INVISIBLE_FB: + *mc_start = adev->mc.visible_vram_size; + *mc_size = adev->mc.real_vram_size - adev->mc.visible_vram_size; + *mem_size = *mc_size; + break; + case CGS_GPU_MEM_TYPE__GART_CACHEABLE: + case CGS_GPU_MEM_TYPE__GART_WRITECOMBINE: + *mc_start = adev->mc.gtt_start; + *mc_size = adev->mc.gtt_size; + *mem_size = adev->mc.gtt_size - adev->gart_pin_size; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int amdgpu_cgs_gmap_kmem(void *cgs_device, void *kmem, + uint64_t size, + uint64_t min_offset, uint64_t max_offset, + cgs_handle_t *kmem_handle, uint64_t *mcaddr) +{ + CGS_FUNC_ADEV; + int ret; + struct amdgpu_bo *bo; + struct page *kmem_page = vmalloc_to_page(kmem); + int npages = ALIGN(size, PAGE_SIZE) >> PAGE_SHIFT; + + struct sg_table *sg = drm_prime_pages_to_sg(&kmem_page, npages); + ret = amdgpu_bo_create(adev, size, PAGE_SIZE, false, + AMDGPU_GEM_DOMAIN_GTT, 0, sg, &bo); + if (ret) + return ret; + ret = amdgpu_bo_reserve(bo, false); + if (unlikely(ret != 0)) + return ret; + + /* pin buffer into GTT */ + ret = amdgpu_bo_pin_restricted(bo, AMDGPU_GEM_DOMAIN_GTT, + min_offset, max_offset, mcaddr); + amdgpu_bo_unreserve(bo); + + *kmem_handle = (cgs_handle_t)bo; + return ret; +} + +static int amdgpu_cgs_gunmap_kmem(void *cgs_device, cgs_handle_t kmem_handle) +{ + struct amdgpu_bo *obj = (struct amdgpu_bo *)kmem_handle; + + if (obj) { + int r = amdgpu_bo_reserve(obj, false); + if (likely(r == 0)) { + amdgpu_bo_unpin(obj); + amdgpu_bo_unreserve(obj); + } + amdgpu_bo_unref(&obj); + + } + return 0; +} + +static int amdgpu_cgs_alloc_gpu_mem(void *cgs_device, + enum cgs_gpu_mem_type type, + uint64_t size, uint64_t align, + uint64_t min_offset, uint64_t max_offset, + cgs_handle_t *handle) +{ + CGS_FUNC_ADEV; + uint16_t flags = 0; + int ret = 0; + uint32_t domain = 0; + struct amdgpu_bo *obj; + struct ttm_placement placement; + struct ttm_place place; + + if (min_offset > max_offset) { + BUG_ON(1); + return -EINVAL; + } + + /* fail if the alignment is not a power of 2 */ + if (((align != 1) && (align & (align - 1))) + || size == 0 || align == 0) + return -EINVAL; + + + switch(type) { + case CGS_GPU_MEM_TYPE__VISIBLE_CONTIG_FB: + case CGS_GPU_MEM_TYPE__VISIBLE_FB: + flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; + domain = AMDGPU_GEM_DOMAIN_VRAM; + if (max_offset > adev->mc.real_vram_size) + return -EINVAL; + place.fpfn = min_offset >> PAGE_SHIFT; + place.lpfn = max_offset >> PAGE_SHIFT; + place.flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED | + TTM_PL_FLAG_VRAM; + break; + case CGS_GPU_MEM_TYPE__INVISIBLE_CONTIG_FB: + case CGS_GPU_MEM_TYPE__INVISIBLE_FB: + flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS; + domain = AMDGPU_GEM_DOMAIN_VRAM; + if (adev->mc.visible_vram_size < adev->mc.real_vram_size) { + place.fpfn = + max(min_offset, adev->mc.visible_vram_size) >> PAGE_SHIFT; + place.lpfn = + min(max_offset, adev->mc.real_vram_size) >> PAGE_SHIFT; + place.flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED | + TTM_PL_FLAG_VRAM; + } + + break; + case CGS_GPU_MEM_TYPE__GART_CACHEABLE: + domain = AMDGPU_GEM_DOMAIN_GTT; + place.fpfn = min_offset >> PAGE_SHIFT; + place.lpfn = max_offset >> PAGE_SHIFT; + place.flags = TTM_PL_FLAG_CACHED | TTM_PL_FLAG_TT; + break; + case CGS_GPU_MEM_TYPE__GART_WRITECOMBINE: + flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC; + domain = AMDGPU_GEM_DOMAIN_GTT; + place.fpfn = min_offset >> PAGE_SHIFT; + place.lpfn = max_offset >> PAGE_SHIFT; + place.flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_TT | + TTM_PL_FLAG_UNCACHED; + break; + default: + return -EINVAL; + } + + + *handle = 0; + + placement.placement = &place; + placement.num_placement = 1; + placement.busy_placement = &place; + placement.num_busy_placement = 1; + + ret = amdgpu_bo_create_restricted(adev, size, PAGE_SIZE, + true, domain, flags, + NULL, &placement, &obj); + if (ret) { + DRM_ERROR("(%d) bo create failed\n", ret); + return ret; + } + *handle = (cgs_handle_t)obj; + + return ret; +} + +static int amdgpu_cgs_import_gpu_mem(void *cgs_device, int dmabuf_fd, + cgs_handle_t *handle) +{ + CGS_FUNC_ADEV; + int r; + uint32_t dma_handle; + struct drm_gem_object *obj; + struct amdgpu_bo *bo; + struct drm_device *dev = adev->ddev; + struct drm_file *file_priv = NULL, *priv; + + mutex_lock(&dev->struct_mutex); + list_for_each_entry(priv, &dev->filelist, lhead) { + rcu_read_lock(); + if (priv->pid == get_pid(task_pid(current))) + file_priv = priv; + rcu_read_unlock(); + if (file_priv) + break; + } + mutex_unlock(&dev->struct_mutex); + r = dev->driver->prime_fd_to_handle(dev, + file_priv, dmabuf_fd, + &dma_handle); + spin_lock(&file_priv->table_lock); + + /* Check if we currently have a reference on the object */ + obj = idr_find(&file_priv->object_idr, dma_handle); + if (obj == NULL) { + spin_unlock(&file_priv->table_lock); + return -EINVAL; + } + spin_unlock(&file_priv->table_lock); + bo = gem_to_amdgpu_bo(obj); + *handle = (cgs_handle_t)bo; + return 0; +} + +static int amdgpu_cgs_free_gpu_mem(void *cgs_device, cgs_handle_t handle) +{ + struct amdgpu_bo *obj = (struct amdgpu_bo *)handle; + + if (obj) { + int r = amdgpu_bo_reserve(obj, false); + if (likely(r == 0)) { + amdgpu_bo_kunmap(obj); + amdgpu_bo_unpin(obj); + amdgpu_bo_unreserve(obj); + } + amdgpu_bo_unref(&obj); + + } + return 0; +} + +static int amdgpu_cgs_gmap_gpu_mem(void *cgs_device, cgs_handle_t handle, + uint64_t *mcaddr) +{ + int r; + u64 min_offset, max_offset; + struct amdgpu_bo *obj = (struct amdgpu_bo *)handle; + + WARN_ON_ONCE(obj->placement.num_placement > 1); + + min_offset = obj->placements[0].fpfn << PAGE_SHIFT; + max_offset = obj->placements[0].lpfn << PAGE_SHIFT; + + r = amdgpu_bo_reserve(obj, false); + if (unlikely(r != 0)) + return r; + r = amdgpu_bo_pin_restricted(obj, AMDGPU_GEM_DOMAIN_GTT, + min_offset, max_offset, mcaddr); + amdgpu_bo_unreserve(obj); + return r; +} + +static int amdgpu_cgs_gunmap_gpu_mem(void *cgs_device, cgs_handle_t handle) +{ + int r; + struct amdgpu_bo *obj = (struct amdgpu_bo *)handle; + r = amdgpu_bo_reserve(obj, false); + if (unlikely(r != 0)) + return r; + r = amdgpu_bo_unpin(obj); + amdgpu_bo_unreserve(obj); + return r; +} + +static int amdgpu_cgs_kmap_gpu_mem(void *cgs_device, cgs_handle_t handle, + void **map) +{ + int r; + struct amdgpu_bo *obj = (struct amdgpu_bo *)handle; + r = amdgpu_bo_reserve(obj, false); + if (unlikely(r != 0)) + return r; + r = amdgpu_bo_kmap(obj, map); + amdgpu_bo_unreserve(obj); + return r; +} + +static int amdgpu_cgs_kunmap_gpu_mem(void *cgs_device, cgs_handle_t handle) +{ + int r; + struct amdgpu_bo *obj = (struct amdgpu_bo *)handle; + r = amdgpu_bo_reserve(obj, false); + if (unlikely(r != 0)) + return r; + amdgpu_bo_kunmap(obj); + amdgpu_bo_unreserve(obj); + return r; +} + +static uint32_t amdgpu_cgs_read_register(void *cgs_device, unsigned offset) +{ + CGS_FUNC_ADEV; + return RREG32(offset); +} + +static void amdgpu_cgs_write_register(void *cgs_device, unsigned offset, + uint32_t value) +{ + CGS_FUNC_ADEV; + WREG32(offset, value); +} + +static uint32_t amdgpu_cgs_read_ind_register(void *cgs_device, + enum cgs_ind_reg space, + unsigned index) +{ + CGS_FUNC_ADEV; + switch (space) { + case CGS_IND_REG__MMIO: + return RREG32_IDX(index); + case CGS_IND_REG__PCIE: + return RREG32_PCIE(index); + case CGS_IND_REG__SMC: + return RREG32_SMC(index); + case CGS_IND_REG__UVD_CTX: + return RREG32_UVD_CTX(index); + case CGS_IND_REG__DIDT: + return RREG32_DIDT(index); + case CGS_IND_REG__AUDIO_ENDPT: + DRM_ERROR("audio endpt register access not implemented.\n"); + return 0; + } + WARN(1, "Invalid indirect register space"); + return 0; +} + +static void amdgpu_cgs_write_ind_register(void *cgs_device, + enum cgs_ind_reg space, + unsigned index, uint32_t value) +{ + CGS_FUNC_ADEV; + switch (space) { + case CGS_IND_REG__MMIO: + return WREG32_IDX(index, value); + case CGS_IND_REG__PCIE: + return WREG32_PCIE(index, value); + case CGS_IND_REG__SMC: + return WREG32_SMC(index, value); + case CGS_IND_REG__UVD_CTX: + return WREG32_UVD_CTX(index, value); + case CGS_IND_REG__DIDT: + return WREG32_DIDT(index, value); + case CGS_IND_REG__AUDIO_ENDPT: + DRM_ERROR("audio endpt register access not implemented.\n"); + return; + } + WARN(1, "Invalid indirect register space"); +} + +static uint8_t amdgpu_cgs_read_pci_config_byte(void *cgs_device, unsigned addr) +{ + CGS_FUNC_ADEV; + uint8_t val; + int ret = pci_read_config_byte(adev->pdev, addr, &val); + if (WARN(ret, "pci_read_config_byte error")) + return 0; + return val; +} + +static uint16_t amdgpu_cgs_read_pci_config_word(void *cgs_device, unsigned addr) +{ + CGS_FUNC_ADEV; + uint16_t val; + int ret = pci_read_config_word(adev->pdev, addr, &val); + if (WARN(ret, "pci_read_config_word error")) + return 0; + return val; +} + +static uint32_t amdgpu_cgs_read_pci_config_dword(void *cgs_device, + unsigned addr) +{ + CGS_FUNC_ADEV; + uint32_t val; + int ret = pci_read_config_dword(adev->pdev, addr, &val); + if (WARN(ret, "pci_read_config_dword error")) + return 0; + return val; +} + +static void amdgpu_cgs_write_pci_config_byte(void *cgs_device, unsigned addr, + uint8_t value) +{ + CGS_FUNC_ADEV; + int ret = pci_write_config_byte(adev->pdev, addr, value); + WARN(ret, "pci_write_config_byte error"); +} + +static void amdgpu_cgs_write_pci_config_word(void *cgs_device, unsigned addr, + uint16_t value) +{ + CGS_FUNC_ADEV; + int ret = pci_write_config_word(adev->pdev, addr, value); + WARN(ret, "pci_write_config_word error"); +} + +static void amdgpu_cgs_write_pci_config_dword(void *cgs_device, unsigned addr, + uint32_t value) +{ + CGS_FUNC_ADEV; + int ret = pci_write_config_dword(adev->pdev, addr, value); + WARN(ret, "pci_write_config_dword error"); +} + +static const void *amdgpu_cgs_atom_get_data_table(void *cgs_device, + unsigned table, uint16_t *size, + uint8_t *frev, uint8_t *crev) +{ + CGS_FUNC_ADEV; + uint16_t data_start; + + if (amdgpu_atom_parse_data_header( + adev->mode_info.atom_context, table, size, + frev, crev, &data_start)) + return (uint8_t*)adev->mode_info.atom_context->bios + + data_start; + + return NULL; +} + +static int amdgpu_cgs_atom_get_cmd_table_revs(void *cgs_device, unsigned table, + uint8_t *frev, uint8_t *crev) +{ + CGS_FUNC_ADEV; + + if (amdgpu_atom_parse_cmd_header( + adev->mode_info.atom_context, table, + frev, crev)) + return 0; + + return -EINVAL; +} + +static int amdgpu_cgs_atom_exec_cmd_table(void *cgs_device, unsigned table, + void *args) +{ + CGS_FUNC_ADEV; + + return amdgpu_atom_execute_table( + adev->mode_info.atom_context, table, args); +} + +static int amdgpu_cgs_create_pm_request(void *cgs_device, cgs_handle_t *request) +{ + /* TODO */ + return 0; +} + +static int amdgpu_cgs_destroy_pm_request(void *cgs_device, cgs_handle_t request) +{ + /* TODO */ + return 0; +} + +static int amdgpu_cgs_set_pm_request(void *cgs_device, cgs_handle_t request, + int active) +{ + /* TODO */ + return 0; +} + +static int amdgpu_cgs_pm_request_clock(void *cgs_device, cgs_handle_t request, + enum cgs_clock clock, unsigned freq) +{ + /* TODO */ + return 0; +} + +static int amdgpu_cgs_pm_request_engine(void *cgs_device, cgs_handle_t request, + enum cgs_engine engine, int powered) +{ + /* TODO */ + return 0; +} + + + +static int amdgpu_cgs_pm_query_clock_limits(void *cgs_device, + enum cgs_clock clock, + struct cgs_clock_limits *limits) +{ + /* TODO */ + return 0; +} + +static int amdgpu_cgs_set_camera_voltages(void *cgs_device, uint32_t mask, + const uint32_t *voltages) +{ + DRM_ERROR("not implemented"); + return -EPERM; +} + +struct cgs_irq_params { + unsigned src_id; + cgs_irq_source_set_func_t set; + cgs_irq_handler_func_t handler; + void *private_data; +}; + +static int cgs_set_irq_state(struct amdgpu_device *adev, + struct amdgpu_irq_src *src, + unsigned type, + enum amdgpu_interrupt_state state) +{ + struct cgs_irq_params *irq_params = + (struct cgs_irq_params *)src->data; + if (!irq_params) + return -EINVAL; + if (!irq_params->set) + return -EINVAL; + return irq_params->set(irq_params->private_data, + irq_params->src_id, + type, + (int)state); +} + +static int cgs_process_irq(struct amdgpu_device *adev, + struct amdgpu_irq_src *source, + struct amdgpu_iv_entry *entry) +{ + struct cgs_irq_params *irq_params = + (struct cgs_irq_params *)source->data; + if (!irq_params) + return -EINVAL; + if (!irq_params->handler) + return -EINVAL; + return irq_params->handler(irq_params->private_data, + irq_params->src_id, + entry->iv_entry); +} + +static const struct amdgpu_irq_src_funcs cgs_irq_funcs = { + .set = cgs_set_irq_state, + .process = cgs_process_irq, +}; + +static int amdgpu_cgs_add_irq_source(void *cgs_device, unsigned src_id, + unsigned num_types, + cgs_irq_source_set_func_t set, + cgs_irq_handler_func_t handler, + void *private_data) +{ + CGS_FUNC_ADEV; + int ret = 0; + struct cgs_irq_params *irq_params; + struct amdgpu_irq_src *source = + kzalloc(sizeof(struct amdgpu_irq_src), GFP_KERNEL); + if (!source) + return -ENOMEM; + irq_params = + kzalloc(sizeof(struct cgs_irq_params), GFP_KERNEL); + if (!irq_params) { + kfree(source); + return -ENOMEM; + } + source->num_types = num_types; + source->funcs = &cgs_irq_funcs; + irq_params->src_id = src_id; + irq_params->set = set; + irq_params->handler = handler; + irq_params->private_data = private_data; + source->data = (void *)irq_params; + ret = amdgpu_irq_add_id(adev, src_id, source); + if (ret) { + kfree(irq_params); + kfree(source); + } + + return ret; +} + +static int amdgpu_cgs_irq_get(void *cgs_device, unsigned src_id, unsigned type) +{ + CGS_FUNC_ADEV; + return amdgpu_irq_get(adev, adev->irq.sources[src_id], type); +} + +static int amdgpu_cgs_irq_put(void *cgs_device, unsigned src_id, unsigned type) +{ + CGS_FUNC_ADEV; + return amdgpu_irq_put(adev, adev->irq.sources[src_id], type); +} + +int amdgpu_cgs_set_clockgating_state(void *cgs_device, + enum amd_ip_block_type block_type, + enum amd_clockgating_state state) +{ + CGS_FUNC_ADEV; + int i, r = -1; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_block_status[i].valid) + continue; + + if (adev->ip_blocks[i].type == block_type) { + r = adev->ip_blocks[i].funcs->set_clockgating_state( + (void *)adev, + state); + break; + } + } + return r; +} + +int amdgpu_cgs_set_powergating_state(void *cgs_device, + enum amd_ip_block_type block_type, + enum amd_powergating_state state) +{ + CGS_FUNC_ADEV; + int i, r = -1; + + for (i = 0; i < adev->num_ip_blocks; i++) { + if (!adev->ip_block_status[i].valid) + continue; + + if (adev->ip_blocks[i].type == block_type) { + r = adev->ip_blocks[i].funcs->set_powergating_state( + (void *)adev, + state); + break; + } + } + return r; +} + + +static uint32_t fw_type_convert(void *cgs_device, uint32_t fw_type) +{ + CGS_FUNC_ADEV; + enum AMDGPU_UCODE_ID result = AMDGPU_UCODE_ID_MAXIMUM; + + switch (fw_type) { + case CGS_UCODE_ID_SDMA0: + result = AMDGPU_UCODE_ID_SDMA0; + break; + case CGS_UCODE_ID_SDMA1: + result = AMDGPU_UCODE_ID_SDMA1; + break; + case CGS_UCODE_ID_CP_CE: + result = AMDGPU_UCODE_ID_CP_CE; + break; + case CGS_UCODE_ID_CP_PFP: + result = AMDGPU_UCODE_ID_CP_PFP; + break; + case CGS_UCODE_ID_CP_ME: + result = AMDGPU_UCODE_ID_CP_ME; + break; + case CGS_UCODE_ID_CP_MEC: + case CGS_UCODE_ID_CP_MEC_JT1: + result = AMDGPU_UCODE_ID_CP_MEC1; + break; + case CGS_UCODE_ID_CP_MEC_JT2: + if (adev->asic_type == CHIP_TONGA) + result = AMDGPU_UCODE_ID_CP_MEC2; + else if (adev->asic_type == CHIP_CARRIZO) + result = AMDGPU_UCODE_ID_CP_MEC1; + break; + case CGS_UCODE_ID_RLC_G: + result = AMDGPU_UCODE_ID_RLC_G; + break; + default: + DRM_ERROR("Firmware type not supported\n"); + } + return result; +} + +static int amdgpu_cgs_get_firmware_info(void *cgs_device, + enum cgs_ucode_id type, + struct cgs_firmware_info *info) +{ + CGS_FUNC_ADEV; + + if (CGS_UCODE_ID_SMU != type) { + uint64_t gpu_addr; + uint32_t data_size; + const struct gfx_firmware_header_v1_0 *header; + enum AMDGPU_UCODE_ID id; + struct amdgpu_firmware_info *ucode; + + id = fw_type_convert(cgs_device, type); + ucode = &adev->firmware.ucode[id]; + if (ucode->fw == NULL) + return -EINVAL; + + gpu_addr = ucode->mc_addr; + header = (const struct gfx_firmware_header_v1_0 *)ucode->fw->data; + data_size = le32_to_cpu(header->header.ucode_size_bytes); + + if ((type == CGS_UCODE_ID_CP_MEC_JT1) || + (type == CGS_UCODE_ID_CP_MEC_JT2)) { + gpu_addr += le32_to_cpu(header->jt_offset) << 2; + data_size = le32_to_cpu(header->jt_size) << 2; + } + info->mc_addr = gpu_addr; + info->image_size = data_size; + info->version = (uint16_t)le32_to_cpu(header->header.ucode_version); + info->feature_version = (uint16_t)le32_to_cpu(header->ucode_feature_version); + } else { + char fw_name[30] = {0}; + int err = 0; + uint32_t ucode_size; + uint32_t ucode_start_address; + const uint8_t *src; + const struct smc_firmware_header_v1_0 *hdr; + + switch (adev->asic_type) { + case CHIP_TONGA: + strcpy(fw_name, "amdgpu/tonga_smc.bin"); + break; + default: + DRM_ERROR("SMC firmware not supported\n"); + return -EINVAL; + } + + err = request_firmware(&adev->pm.fw, fw_name, adev->dev); + if (err) { + DRM_ERROR("Failed to request firmware\n"); + return err; + } + + err = amdgpu_ucode_validate(adev->pm.fw); + if (err) { + DRM_ERROR("Failed to load firmware \"%s\"", fw_name); + release_firmware(adev->pm.fw); + adev->pm.fw = NULL; + return err; + } + + hdr = (const struct smc_firmware_header_v1_0 *) adev->pm.fw->data; + adev->pm.fw_version = le32_to_cpu(hdr->header.ucode_version); + ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes); + ucode_start_address = le32_to_cpu(hdr->ucode_start_addr); + src = (const uint8_t *)(adev->pm.fw->data + + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + + info->version = adev->pm.fw_version; + info->image_size = ucode_size; + info->kptr = (void *)src; + } + return 0; +} + +static const struct cgs_ops amdgpu_cgs_ops = { + amdgpu_cgs_gpu_mem_info, + amdgpu_cgs_gmap_kmem, + amdgpu_cgs_gunmap_kmem, + amdgpu_cgs_alloc_gpu_mem, + amdgpu_cgs_free_gpu_mem, + amdgpu_cgs_gmap_gpu_mem, + amdgpu_cgs_gunmap_gpu_mem, + amdgpu_cgs_kmap_gpu_mem, + amdgpu_cgs_kunmap_gpu_mem, + amdgpu_cgs_read_register, + amdgpu_cgs_write_register, + amdgpu_cgs_read_ind_register, + amdgpu_cgs_write_ind_register, + amdgpu_cgs_read_pci_config_byte, + amdgpu_cgs_read_pci_config_word, + amdgpu_cgs_read_pci_config_dword, + amdgpu_cgs_write_pci_config_byte, + amdgpu_cgs_write_pci_config_word, + amdgpu_cgs_write_pci_config_dword, + amdgpu_cgs_atom_get_data_table, + amdgpu_cgs_atom_get_cmd_table_revs, + amdgpu_cgs_atom_exec_cmd_table, + amdgpu_cgs_create_pm_request, + amdgpu_cgs_destroy_pm_request, + amdgpu_cgs_set_pm_request, + amdgpu_cgs_pm_request_clock, + amdgpu_cgs_pm_request_engine, + amdgpu_cgs_pm_query_clock_limits, + amdgpu_cgs_set_camera_voltages, + amdgpu_cgs_get_firmware_info, + amdgpu_cgs_set_powergating_state, + amdgpu_cgs_set_clockgating_state +}; + +static const struct cgs_os_ops amdgpu_cgs_os_ops = { + amdgpu_cgs_import_gpu_mem, + amdgpu_cgs_add_irq_source, + amdgpu_cgs_irq_get, + amdgpu_cgs_irq_put +}; + +void *amdgpu_cgs_create_device(struct amdgpu_device *adev) +{ + struct amdgpu_cgs_device *cgs_device = + kmalloc(sizeof(*cgs_device), GFP_KERNEL); + + if (!cgs_device) { + DRM_ERROR("Couldn't allocate CGS device structure\n"); + return NULL; + } + + cgs_device->base.ops = &amdgpu_cgs_ops; + cgs_device->base.os_ops = &amdgpu_cgs_os_ops; + cgs_device->adev = adev; + + return cgs_device; +} + +void amdgpu_cgs_destroy_device(void *cgs_device) +{ + kfree(cgs_device); +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c index 27df17a0e620..89c3dd62ba21 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c @@ -75,6 +75,11 @@ void amdgpu_connector_hotplug(struct drm_connector *connector) if (!amdgpu_display_hpd_sense(adev, amdgpu_connector->hpd.hpd)) { drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF); } else if (amdgpu_atombios_dp_needs_link_train(amdgpu_connector)) { + /* Don't try to start link training before we + * have the dpcd */ + if (!amdgpu_atombios_dp_get_dpcd(amdgpu_connector)) + return; + /* set it to OFF so that drm_helper_connector_dpms() * won't return immediately since the current state * is ON at this point. diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index 1f040d85ac47..3b355aeb62fd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -126,6 +126,30 @@ int amdgpu_cs_get_ring(struct amdgpu_device *adev, u32 ip_type, return 0; } +struct amdgpu_cs_parser *amdgpu_cs_parser_create(struct amdgpu_device *adev, + struct drm_file *filp, + struct amdgpu_ctx *ctx, + struct amdgpu_ib *ibs, + uint32_t num_ibs) +{ + struct amdgpu_cs_parser *parser; + int i; + + parser = kzalloc(sizeof(struct amdgpu_cs_parser), GFP_KERNEL); + if (!parser) + return NULL; + + parser->adev = adev; + parser->filp = filp; + parser->ctx = ctx; + parser->ibs = ibs; + parser->num_ibs = num_ibs; + for (i = 0; i < num_ibs; i++) + ibs[i].ctx = ctx; + + return parser; +} + int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data) { union drm_amdgpu_cs *cs = data; @@ -147,13 +171,13 @@ int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data) /* get chunks */ INIT_LIST_HEAD(&p->validated); - chunk_array = kcalloc(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL); + chunk_array = kmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL); if (chunk_array == NULL) { r = -ENOMEM; goto out; } - chunk_array_user = (uint64_t *)(unsigned long)(cs->in.chunks); + chunk_array_user = (uint64_t __user *)(cs->in.chunks); if (copy_from_user(chunk_array, chunk_array_user, sizeof(uint64_t)*cs->in.num_chunks)) { r = -EFAULT; @@ -161,7 +185,7 @@ int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data) } p->nchunks = cs->in.num_chunks; - p->chunks = kcalloc(p->nchunks, sizeof(struct amdgpu_cs_chunk), + p->chunks = kmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk), GFP_KERNEL); if (p->chunks == NULL) { r = -ENOMEM; @@ -173,7 +197,7 @@ int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data) struct drm_amdgpu_cs_chunk user_chunk; uint32_t __user *cdata; - chunk_ptr = (void __user *)(unsigned long)chunk_array[i]; + chunk_ptr = (void __user *)chunk_array[i]; if (copy_from_user(&user_chunk, chunk_ptr, sizeof(struct drm_amdgpu_cs_chunk))) { r = -EFAULT; @@ -183,7 +207,7 @@ int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data) p->chunks[i].length_dw = user_chunk.length_dw; size = p->chunks[i].length_dw; - cdata = (void __user *)(unsigned long)user_chunk.chunk_data; + cdata = (void __user *)user_chunk.chunk_data; p->chunks[i].user_ptr = cdata; p->chunks[i].kdata = drm_malloc_ab(size, sizeof(uint32_t)); @@ -235,11 +259,10 @@ int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data) } } + p->ibs = kcalloc(p->num_ibs, sizeof(struct amdgpu_ib), GFP_KERNEL); - if (!p->ibs) { + if (!p->ibs) r = -ENOMEM; - goto out; - } out: kfree(chunk_array); @@ -331,7 +354,7 @@ int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p) * into account. We don't want to disallow buffer moves * completely. */ - if (current_domain != AMDGPU_GEM_DOMAIN_CPU && + if ((lobj->allowed_domains & current_domain) != 0 && (domain & current_domain) == 0 && /* will be moved */ bytes_moved > bytes_moved_threshold) { /* don't move it */ @@ -415,18 +438,8 @@ static int cmp_size_smaller_first(void *priv, struct list_head *a, return (int)la->robj->tbo.num_pages - (int)lb->robj->tbo.num_pages; } -/** - * cs_parser_fini() - clean parser states - * @parser: parser structure holding parsing context. - * @error: error number - * - * If error is set than unvalidate buffer, otherwise just free memory - * used by parsing context. - **/ -static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bool backoff) +static void amdgpu_cs_parser_fini_early(struct amdgpu_cs_parser *parser, int error, bool backoff) { - unsigned i; - if (!error) { /* Sort the buffer list from the smallest to largest buffer, * which affects the order of buffers in the LRU list. @@ -447,21 +460,45 @@ static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bo ttm_eu_backoff_reservation(&parser->ticket, &parser->validated); } +} +static void amdgpu_cs_parser_fini_late(struct amdgpu_cs_parser *parser) +{ + unsigned i; if (parser->ctx) amdgpu_ctx_put(parser->ctx); if (parser->bo_list) amdgpu_bo_list_put(parser->bo_list); + drm_free_large(parser->vm_bos); for (i = 0; i < parser->nchunks; i++) drm_free_large(parser->chunks[i].kdata); kfree(parser->chunks); - if (parser->ibs) - for (i = 0; i < parser->num_ibs; i++) - amdgpu_ib_free(parser->adev, &parser->ibs[i]); - kfree(parser->ibs); - if (parser->uf.bo) - drm_gem_object_unreference_unlocked(&parser->uf.bo->gem_base); + if (!amdgpu_enable_scheduler) + { + if (parser->ibs) + for (i = 0; i < parser->num_ibs; i++) + amdgpu_ib_free(parser->adev, &parser->ibs[i]); + kfree(parser->ibs); + if (parser->uf.bo) + drm_gem_object_unreference_unlocked(&parser->uf.bo->gem_base); + } + + kfree(parser); +} + +/** + * cs_parser_fini() - clean parser states + * @parser: parser structure holding parsing context. + * @error: error number + * + * If error is set than unvalidate buffer, otherwise just free memory + * used by parsing context. + **/ +static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bool backoff) +{ + amdgpu_cs_parser_fini_early(parser, error, backoff); + amdgpu_cs_parser_fini_late(parser); } static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p, @@ -476,12 +513,18 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p, if (r) return r; + r = amdgpu_sync_fence(adev, &p->ibs[0].sync, vm->page_directory_fence); + if (r) + return r; + r = amdgpu_vm_clear_freed(adev, vm); if (r) return r; if (p->bo_list) { for (i = 0; i < p->bo_list->num_entries; i++) { + struct fence *f; + /* ignore duplicates */ bo = p->bo_list->array[i].robj; if (!bo) @@ -495,7 +538,10 @@ static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p, if (r) return r; - amdgpu_sync_fence(&p->ibs[0].sync, bo_va->last_pt_update); + f = bo_va->last_pt_update; + r = amdgpu_sync_fence(adev, &p->ibs[0].sync, f); + if (r) + return r; } } @@ -529,9 +575,9 @@ static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev, goto out; } amdgpu_cs_sync_rings(parser); - - r = amdgpu_ib_schedule(adev, parser->num_ibs, parser->ibs, - parser->filp); + if (!amdgpu_enable_scheduler) + r = amdgpu_ib_schedule(adev, parser->num_ibs, parser->ibs, + parser->filp); out: mutex_unlock(&vm->mutex); @@ -650,7 +696,6 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev, ib->oa_size = amdgpu_bo_size(oa); } } - /* wrap the last IB with user fence */ if (parser->uf.bo) { struct amdgpu_ib *ib = &parser->ibs[parser->num_ibs - 1]; @@ -693,9 +738,9 @@ static int amdgpu_cs_dependencies(struct amdgpu_device *adev, sizeof(struct drm_amdgpu_cs_chunk_dep); for (j = 0; j < num_deps; ++j) { - struct amdgpu_fence *fence; struct amdgpu_ring *ring; struct amdgpu_ctx *ctx; + struct fence *fence; r = amdgpu_cs_get_ring(adev, deps[j].ip_type, deps[j].ip_instance, @@ -707,85 +752,137 @@ static int amdgpu_cs_dependencies(struct amdgpu_device *adev, if (ctx == NULL) return -EINVAL; - r = amdgpu_fence_recreate(ring, p->filp, - deps[j].handle, - &fence); - if (r) { + fence = amdgpu_ctx_get_fence(ctx, ring, + deps[j].handle); + if (IS_ERR(fence)) { + r = PTR_ERR(fence); amdgpu_ctx_put(ctx); return r; - } - amdgpu_sync_fence(&ib->sync, fence); - amdgpu_fence_unref(&fence); - amdgpu_ctx_put(ctx); + } else if (fence) { + r = amdgpu_sync_fence(adev, &ib->sync, fence); + fence_put(fence); + amdgpu_ctx_put(ctx); + if (r) + return r; + } } } return 0; } +static int amdgpu_cs_free_job(struct amdgpu_job *sched_job) +{ + int i; + if (sched_job->ibs) + for (i = 0; i < sched_job->num_ibs; i++) + amdgpu_ib_free(sched_job->adev, &sched_job->ibs[i]); + kfree(sched_job->ibs); + if (sched_job->uf.bo) + drm_gem_object_unreference_unlocked(&sched_job->uf.bo->gem_base); + return 0; +} + int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { struct amdgpu_device *adev = dev->dev_private; union drm_amdgpu_cs *cs = data; - struct amdgpu_cs_parser parser; - int r, i; + struct amdgpu_cs_parser *parser; bool reserved_buffers = false; + int i, r; down_read(&adev->exclusive_lock); if (!adev->accel_working) { up_read(&adev->exclusive_lock); return -EBUSY; } - /* initialize parser */ - memset(&parser, 0, sizeof(struct amdgpu_cs_parser)); - parser.filp = filp; - parser.adev = adev; - r = amdgpu_cs_parser_init(&parser, data); + + parser = amdgpu_cs_parser_create(adev, filp, NULL, NULL, 0); + if (!parser) + return -ENOMEM; + r = amdgpu_cs_parser_init(parser, data); if (r) { DRM_ERROR("Failed to initialize parser !\n"); - amdgpu_cs_parser_fini(&parser, r, false); + amdgpu_cs_parser_fini(parser, r, false); up_read(&adev->exclusive_lock); r = amdgpu_cs_handle_lockup(adev, r); return r; } - r = amdgpu_cs_parser_relocs(&parser); - if (r) { - if (r != -ERESTARTSYS) { - if (r == -ENOMEM) - DRM_ERROR("Not enough memory for command submission!\n"); - else - DRM_ERROR("Failed to process the buffer list %d!\n", r); - } + r = amdgpu_cs_parser_relocs(parser); + if (r == -ENOMEM) + DRM_ERROR("Not enough memory for command submission!\n"); + else if (r && r != -ERESTARTSYS) + DRM_ERROR("Failed to process the buffer list %d!\n", r); + else if (!r) { + reserved_buffers = true; + r = amdgpu_cs_ib_fill(adev, parser); } if (!r) { - reserved_buffers = true; - r = amdgpu_cs_ib_fill(adev, &parser); + r = amdgpu_cs_dependencies(adev, parser); + if (r) + DRM_ERROR("Failed in the dependencies handling %d!\n", r); } - if (!r) - r = amdgpu_cs_dependencies(adev, &parser); - - if (r) { - amdgpu_cs_parser_fini(&parser, r, reserved_buffers); - up_read(&adev->exclusive_lock); - r = amdgpu_cs_handle_lockup(adev, r); - return r; - } - - for (i = 0; i < parser.num_ibs; i++) - trace_amdgpu_cs(&parser, i); - - r = amdgpu_cs_ib_vm_chunk(adev, &parser); - if (r) { + if (r) goto out; + + for (i = 0; i < parser->num_ibs; i++) + trace_amdgpu_cs(parser, i); + + r = amdgpu_cs_ib_vm_chunk(adev, parser); + if (r) + goto out; + + if (amdgpu_enable_scheduler && parser->num_ibs) { + struct amdgpu_job *job; + struct amdgpu_ring * ring = parser->ibs->ring; + job = kzalloc(sizeof(struct amdgpu_job), GFP_KERNEL); + if (!job) + return -ENOMEM; + job->base.sched = ring->scheduler; + job->base.s_entity = &parser->ctx->rings[ring->idx].entity; + job->adev = parser->adev; + job->ibs = parser->ibs; + job->num_ibs = parser->num_ibs; + job->base.owner = parser->filp; + mutex_init(&job->job_lock); + if (job->ibs[job->num_ibs - 1].user) { + memcpy(&job->uf, &parser->uf, + sizeof(struct amdgpu_user_fence)); + job->ibs[job->num_ibs - 1].user = &job->uf; + } + + job->free_job = amdgpu_cs_free_job; + mutex_lock(&job->job_lock); + r = amd_sched_entity_push_job((struct amd_sched_job *)job); + if (r) { + mutex_unlock(&job->job_lock); + amdgpu_cs_free_job(job); + kfree(job); + goto out; + } + cs->out.handle = + amdgpu_ctx_add_fence(parser->ctx, ring, + &job->base.s_fence->base); + parser->ibs[parser->num_ibs - 1].sequence = cs->out.handle; + + list_sort(NULL, &parser->validated, cmp_size_smaller_first); + ttm_eu_fence_buffer_objects(&parser->ticket, + &parser->validated, + &job->base.s_fence->base); + + mutex_unlock(&job->job_lock); + amdgpu_cs_parser_fini_late(parser); + up_read(&adev->exclusive_lock); + return 0; } - cs->out.handle = parser.ibs[parser.num_ibs - 1].fence->seq; + cs->out.handle = parser->ibs[parser->num_ibs - 1].sequence; out: - amdgpu_cs_parser_fini(&parser, r, true); + amdgpu_cs_parser_fini(parser, r, reserved_buffers); up_read(&adev->exclusive_lock); r = amdgpu_cs_handle_lockup(adev, r); return r; @@ -806,30 +903,29 @@ int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, union drm_amdgpu_wait_cs *wait = data; struct amdgpu_device *adev = dev->dev_private; unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout); - struct amdgpu_fence *fence = NULL; struct amdgpu_ring *ring = NULL; struct amdgpu_ctx *ctx; + struct fence *fence; long r; + r = amdgpu_cs_get_ring(adev, wait->in.ip_type, wait->in.ip_instance, + wait->in.ring, &ring); + if (r) + return r; + ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id); if (ctx == NULL) return -EINVAL; - r = amdgpu_cs_get_ring(adev, wait->in.ip_type, wait->in.ip_instance, - wait->in.ring, &ring); - if (r) { - amdgpu_ctx_put(ctx); - return r; - } + fence = amdgpu_ctx_get_fence(ctx, ring, wait->in.handle); + if (IS_ERR(fence)) + r = PTR_ERR(fence); + else if (fence) { + r = fence_wait_timeout(fence, true, timeout); + fence_put(fence); + } else + r = 1; - r = amdgpu_fence_recreate(ring, filp, wait->in.handle, &fence); - if (r) { - amdgpu_ctx_put(ctx); - return r; - } - - r = fence_wait_timeout(&fence->base, true, timeout); - amdgpu_fence_unref(&fence); amdgpu_ctx_put(ctx); if (r < 0) return r; @@ -864,7 +960,16 @@ amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser, if (!reloc->bo_va) continue; - list_for_each_entry(mapping, &reloc->bo_va->mappings, list) { + list_for_each_entry(mapping, &reloc->bo_va->valids, list) { + if (mapping->it.start > addr || + addr > mapping->it.last) + continue; + + *bo = reloc->bo_va->bo; + return mapping; + } + + list_for_each_entry(mapping, &reloc->bo_va->invalids, list) { if (mapping->it.start > addr || addr > mapping->it.last) continue; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c index 6c66ac8a1891..20cbc4eb5a6f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c @@ -25,54 +25,107 @@ #include #include "amdgpu.h" -static void amdgpu_ctx_do_release(struct kref *ref) +int amdgpu_ctx_init(struct amdgpu_device *adev, bool kernel, + struct amdgpu_ctx *ctx) { - struct amdgpu_ctx *ctx; - struct amdgpu_ctx_mgr *mgr; + unsigned i, j; + int r; - ctx = container_of(ref, struct amdgpu_ctx, refcount); - mgr = &ctx->fpriv->ctx_mgr; + memset(ctx, 0, sizeof(*ctx)); + ctx->adev = adev; + kref_init(&ctx->refcount); + spin_lock_init(&ctx->ring_lock); + for (i = 0; i < AMDGPU_MAX_RINGS; ++i) + ctx->rings[i].sequence = 1; - idr_remove(&mgr->ctx_handles, ctx->id); - kfree(ctx); + if (amdgpu_enable_scheduler) { + /* create context entity for each ring */ + for (i = 0; i < adev->num_rings; i++) { + struct amd_sched_rq *rq; + if (kernel) + rq = &adev->rings[i]->scheduler->kernel_rq; + else + rq = &adev->rings[i]->scheduler->sched_rq; + r = amd_sched_entity_init(adev->rings[i]->scheduler, + &ctx->rings[i].entity, + rq, amdgpu_sched_jobs); + if (r) + break; + } + + if (i < adev->num_rings) { + for (j = 0; j < i; j++) + amd_sched_entity_fini(adev->rings[j]->scheduler, + &ctx->rings[j].entity); + kfree(ctx); + return r; + } + } + return 0; } -int amdgpu_ctx_alloc(struct amdgpu_device *adev, struct amdgpu_fpriv *fpriv, uint32_t *id, uint32_t flags) +void amdgpu_ctx_fini(struct amdgpu_ctx *ctx) +{ + struct amdgpu_device *adev = ctx->adev; + unsigned i, j; + + for (i = 0; i < AMDGPU_MAX_RINGS; ++i) + for (j = 0; j < AMDGPU_CTX_MAX_CS_PENDING; ++j) + fence_put(ctx->rings[i].fences[j]); + + if (amdgpu_enable_scheduler) { + for (i = 0; i < adev->num_rings; i++) + amd_sched_entity_fini(adev->rings[i]->scheduler, + &ctx->rings[i].entity); + } +} + +static int amdgpu_ctx_alloc(struct amdgpu_device *adev, + struct amdgpu_fpriv *fpriv, + uint32_t *id) { - int r; - struct amdgpu_ctx *ctx; struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; + struct amdgpu_ctx *ctx; + int r; ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); if (!ctx) return -ENOMEM; mutex_lock(&mgr->lock); - r = idr_alloc(&mgr->ctx_handles, ctx, 0, 0, GFP_KERNEL); + r = idr_alloc(&mgr->ctx_handles, ctx, 1, 0, GFP_KERNEL); if (r < 0) { mutex_unlock(&mgr->lock); kfree(ctx); return r; } *id = (uint32_t)r; - - memset(ctx, 0, sizeof(*ctx)); - ctx->id = *id; - ctx->fpriv = fpriv; - kref_init(&ctx->refcount); + r = amdgpu_ctx_init(adev, false, ctx); mutex_unlock(&mgr->lock); - return 0; + return r; } -int amdgpu_ctx_free(struct amdgpu_device *adev, struct amdgpu_fpriv *fpriv, uint32_t id) +static void amdgpu_ctx_do_release(struct kref *ref) { struct amdgpu_ctx *ctx; + + ctx = container_of(ref, struct amdgpu_ctx, refcount); + + amdgpu_ctx_fini(ctx); + + kfree(ctx); +} + +static int amdgpu_ctx_free(struct amdgpu_fpriv *fpriv, uint32_t id) +{ struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; + struct amdgpu_ctx *ctx; mutex_lock(&mgr->lock); ctx = idr_find(&mgr->ctx_handles, id); if (ctx) { + idr_remove(&mgr->ctx_handles, id); kref_put(&ctx->refcount, amdgpu_ctx_do_release); mutex_unlock(&mgr->lock); return 0; @@ -86,9 +139,13 @@ static int amdgpu_ctx_query(struct amdgpu_device *adev, union drm_amdgpu_ctx_out *out) { struct amdgpu_ctx *ctx; - struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; + struct amdgpu_ctx_mgr *mgr; unsigned reset_counter; + if (!fpriv) + return -EINVAL; + + mgr = &fpriv->ctx_mgr; mutex_lock(&mgr->lock); ctx = idr_find(&mgr->ctx_handles, id); if (!ctx) { @@ -97,8 +154,8 @@ static int amdgpu_ctx_query(struct amdgpu_device *adev, } /* TODO: these two are always zero */ - out->state.flags = ctx->state.flags; - out->state.hangs = ctx->state.hangs; + out->state.flags = 0x0; + out->state.hangs = 0x0; /* determine if a GPU reset has occured since the last call */ reset_counter = atomic_read(&adev->gpu_reset_counter); @@ -113,28 +170,11 @@ static int amdgpu_ctx_query(struct amdgpu_device *adev, return 0; } -void amdgpu_ctx_fini(struct amdgpu_fpriv *fpriv) -{ - struct idr *idp; - struct amdgpu_ctx *ctx; - uint32_t id; - struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; - idp = &mgr->ctx_handles; - - idr_for_each_entry(idp,ctx,id) { - if (kref_put(&ctx->refcount, amdgpu_ctx_do_release) != 1) - DRM_ERROR("ctx (id=%ul) is still alive\n",ctx->id); - } - - mutex_destroy(&mgr->lock); -} - int amdgpu_ctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) { int r; uint32_t id; - uint32_t flags; union drm_amdgpu_ctx *args = data; struct amdgpu_device *adev = dev->dev_private; @@ -142,15 +182,14 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data, r = 0; id = args->in.ctx_id; - flags = args->in.flags; switch (args->in.op) { case AMDGPU_CTX_OP_ALLOC_CTX: - r = amdgpu_ctx_alloc(adev, fpriv, &id, flags); + r = amdgpu_ctx_alloc(adev, fpriv, &id); args->out.alloc.ctx_id = id; break; case AMDGPU_CTX_OP_FREE_CTX: - r = amdgpu_ctx_free(adev, fpriv, id); + r = amdgpu_ctx_free(fpriv, id); break; case AMDGPU_CTX_OP_QUERY_STATE: r = amdgpu_ctx_query(adev, fpriv, id, &args->out); @@ -165,7 +204,12 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data, struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id) { struct amdgpu_ctx *ctx; - struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; + struct amdgpu_ctx_mgr *mgr; + + if (!fpriv) + return NULL; + + mgr = &fpriv->ctx_mgr; mutex_lock(&mgr->lock); ctx = idr_find(&mgr->ctx_handles, id); @@ -177,17 +221,86 @@ struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id) int amdgpu_ctx_put(struct amdgpu_ctx *ctx) { - struct amdgpu_fpriv *fpriv; - struct amdgpu_ctx_mgr *mgr; - if (ctx == NULL) return -EINVAL; - fpriv = ctx->fpriv; - mgr = &fpriv->ctx_mgr; - mutex_lock(&mgr->lock); kref_put(&ctx->refcount, amdgpu_ctx_do_release); - mutex_unlock(&mgr->lock); - return 0; } + +uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring, + struct fence *fence) +{ + struct amdgpu_ctx_ring *cring = & ctx->rings[ring->idx]; + uint64_t seq = cring->sequence; + unsigned idx = 0; + struct fence *other = NULL; + + idx = seq % AMDGPU_CTX_MAX_CS_PENDING; + other = cring->fences[idx]; + if (other) { + signed long r; + r = fence_wait_timeout(other, false, MAX_SCHEDULE_TIMEOUT); + if (r < 0) + DRM_ERROR("Error (%ld) waiting for fence!\n", r); + } + + fence_get(fence); + + spin_lock(&ctx->ring_lock); + cring->fences[idx] = fence; + cring->sequence++; + spin_unlock(&ctx->ring_lock); + + fence_put(other); + + return seq; +} + +struct fence *amdgpu_ctx_get_fence(struct amdgpu_ctx *ctx, + struct amdgpu_ring *ring, uint64_t seq) +{ + struct amdgpu_ctx_ring *cring = & ctx->rings[ring->idx]; + struct fence *fence; + + spin_lock(&ctx->ring_lock); + + if (seq >= cring->sequence) { + spin_unlock(&ctx->ring_lock); + return ERR_PTR(-EINVAL); + } + + + if (seq + AMDGPU_CTX_MAX_CS_PENDING < cring->sequence) { + spin_unlock(&ctx->ring_lock); + return NULL; + } + + fence = fence_get(cring->fences[seq % AMDGPU_CTX_MAX_CS_PENDING]); + spin_unlock(&ctx->ring_lock); + + return fence; +} + +void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr) +{ + mutex_init(&mgr->lock); + idr_init(&mgr->ctx_handles); +} + +void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr) +{ + struct amdgpu_ctx *ctx; + struct idr *idp; + uint32_t id; + + idp = &mgr->ctx_handles; + + idr_for_each_entry(idp, ctx, id) { + if (kref_put(&ctx->refcount, amdgpu_ctx_do_release) != 1) + DRM_ERROR("ctx %p is still alive\n", ctx); + } + + idr_destroy(&mgr->ctx_handles); + mutex_destroy(&mgr->lock); +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 99f158e1baff..6ff6ae945794 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -55,6 +55,7 @@ static const char *amdgpu_asic_name[] = { "MULLINS", "TOPAZ", "TONGA", + "FIJI", "CARRIZO", "LAST", }; @@ -63,7 +64,7 @@ bool amdgpu_device_is_px(struct drm_device *dev) { struct amdgpu_device *adev = dev->dev_private; - if (adev->flags & AMDGPU_IS_PX) + if (adev->flags & AMD_IS_PX) return true; return false; } @@ -243,7 +244,8 @@ static int amdgpu_vram_scratch_init(struct amdgpu_device *adev) if (adev->vram_scratch.robj == NULL) { r = amdgpu_bo_create(adev, AMDGPU_GPU_PAGE_SIZE, - PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM, 0, + PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, NULL, &adev->vram_scratch.robj); if (r) { return r; @@ -1160,6 +1162,7 @@ static int amdgpu_early_init(struct amdgpu_device *adev) switch (adev->asic_type) { case CHIP_TOPAZ: case CHIP_TONGA: + case CHIP_FIJI: case CHIP_CARRIZO: if (adev->asic_type == CHIP_CARRIZO) adev->family = AMDGPU_FAMILY_CZ; @@ -1377,7 +1380,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, adev->ddev = ddev; adev->pdev = pdev; adev->flags = flags; - adev->asic_type = flags & AMDGPU_ASIC_MASK; + adev->asic_type = flags & AMD_ASIC_MASK; adev->is_atom_bios = false; adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT; adev->mc.gtt_size = 512 * 1024 * 1024; @@ -1523,6 +1526,11 @@ int amdgpu_device_init(struct amdgpu_device *adev, return r; } + r = amdgpu_ctx_init(adev, true, &adev->kernel_ctx); + if (r) { + dev_err(adev->dev, "failed to create kernel context (%d).\n", r); + return r; + } r = amdgpu_ib_ring_tests(adev); if (r) DRM_ERROR("ib ring test failed (%d).\n", r); @@ -1584,6 +1592,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev) adev->shutdown = true; /* evict vram memory */ amdgpu_bo_evict_vram(adev); + amdgpu_ctx_fini(&adev->kernel_ctx); amdgpu_ib_pool_fini(adev); amdgpu_fence_driver_fini(adev); amdgpu_fbdev_fini(adev); @@ -1627,8 +1636,7 @@ int amdgpu_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon) struct amdgpu_device *adev; struct drm_crtc *crtc; struct drm_connector *connector; - int i, r; - bool force_completion = false; + int r; if (dev == NULL || dev->dev_private == NULL) { return -ENODEV; @@ -1667,21 +1675,7 @@ int amdgpu_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon) /* evict vram memory */ amdgpu_bo_evict_vram(adev); - /* wait for gpu to finish processing current batch */ - for (i = 0; i < AMDGPU_MAX_RINGS; i++) { - struct amdgpu_ring *ring = adev->rings[i]; - if (!ring) - continue; - - r = amdgpu_fence_wait_empty(ring); - if (r) { - /* delay GPU reset to resume */ - force_completion = true; - } - } - if (force_completion) { - amdgpu_fence_driver_force_completion(adev); - } + amdgpu_fence_driver_suspend(adev); r = amdgpu_suspend(adev); @@ -1739,6 +1733,8 @@ int amdgpu_resume_kms(struct drm_device *dev, bool resume, bool fbcon) r = amdgpu_resume(adev); + amdgpu_fence_driver_resume(adev); + r = amdgpu_ib_ring_tests(adev); if (r) DRM_ERROR("ib ring test failed (%d).\n", r); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c index b16b9256883e..e3d70772b531 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c @@ -35,6 +35,36 @@ #include #include +static void amdgpu_flip_wait_fence(struct amdgpu_device *adev, + struct fence **f) +{ + struct amdgpu_fence *fence; + long r; + + if (*f == NULL) + return; + + fence = to_amdgpu_fence(*f); + if (fence) { + r = fence_wait(&fence->base, false); + if (r == -EDEADLK) { + up_read(&adev->exclusive_lock); + r = amdgpu_gpu_reset(adev); + down_read(&adev->exclusive_lock); + } + } else + r = fence_wait(*f, false); + + if (r) + DRM_ERROR("failed to wait on page flip fence (%ld)!\n", r); + + /* We continue with the page flip even if we failed to wait on + * the fence, otherwise the DRM core and userspace will be + * confused about which BO the CRTC is scanning out + */ + fence_put(*f); + *f = NULL; +} static void amdgpu_flip_work_func(struct work_struct *__work) { @@ -44,34 +74,13 @@ static void amdgpu_flip_work_func(struct work_struct *__work) struct amdgpu_crtc *amdgpuCrtc = adev->mode_info.crtcs[work->crtc_id]; struct drm_crtc *crtc = &amdgpuCrtc->base; - struct amdgpu_fence *fence; unsigned long flags; - int r; + unsigned i; down_read(&adev->exclusive_lock); - if (work->fence) { - fence = to_amdgpu_fence(work->fence); - if (fence) { - r = amdgpu_fence_wait(fence, false); - if (r == -EDEADLK) { - up_read(&adev->exclusive_lock); - r = amdgpu_gpu_reset(adev); - down_read(&adev->exclusive_lock); - } - } else - r = fence_wait(work->fence, false); - - if (r) - DRM_ERROR("failed to wait on page flip fence (%d)!\n", r); - - /* We continue with the page flip even if we failed to wait on - * the fence, otherwise the DRM core and userspace will be - * confused about which BO the CRTC is scanning out - */ - - fence_put(work->fence); - work->fence = NULL; - } + amdgpu_flip_wait_fence(adev, &work->excl); + for (i = 0; i < work->shared_count; ++i) + amdgpu_flip_wait_fence(adev, &work->shared[i]); /* We borrow the event spin lock for protecting flip_status */ spin_lock_irqsave(&crtc->dev->event_lock, flags); @@ -108,6 +117,7 @@ static void amdgpu_unpin_work_func(struct work_struct *__work) DRM_ERROR("failed to reserve buffer after flip\n"); drm_gem_object_unreference_unlocked(&work->old_rbo->gem_base); + kfree(work->shared); kfree(work); } @@ -127,7 +137,7 @@ int amdgpu_crtc_page_flip(struct drm_crtc *crtc, unsigned long flags; u64 tiling_flags; u64 base; - int r; + int i, r; work = kzalloc(sizeof *work, GFP_KERNEL); if (work == NULL) @@ -167,7 +177,19 @@ int amdgpu_crtc_page_flip(struct drm_crtc *crtc, goto cleanup; } - work->fence = fence_get(reservation_object_get_excl(new_rbo->tbo.resv)); + r = reservation_object_get_fences_rcu(new_rbo->tbo.resv, &work->excl, + &work->shared_count, + &work->shared); + if (unlikely(r != 0)) { + amdgpu_bo_unreserve(new_rbo); + DRM_ERROR("failed to get fences for buffer\n"); + goto cleanup; + } + + fence_get(work->excl); + for (i = 0; i < work->shared_count; ++i) + fence_get(work->shared[i]); + amdgpu_bo_get_tiling_flags(new_rbo, &tiling_flags); amdgpu_bo_unreserve(new_rbo); @@ -212,7 +234,10 @@ int amdgpu_crtc_page_flip(struct drm_crtc *crtc, cleanup: drm_gem_object_unreference_unlocked(&work->old_rbo->gem_base); - fence_put(work->fence); + fence_put(work->excl); + for (i = 0; i < work->shared_count; ++i) + fence_put(work->shared[i]); + kfree(work->shared); kfree(work); return r; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 56da962231fc..0fcc0bd1622c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -44,12 +44,15 @@ #include "amdgpu.h" #include "amdgpu_irq.h" +#include "amdgpu_amdkfd.h" + /* * KMS wrapper. * - 3.0.0 - initial driver + * - 3.1.0 - allow reading more status registers (GRBM, SRBM, SDMA, CP) */ #define KMS_DRIVER_MAJOR 3 -#define KMS_DRIVER_MINOR 0 +#define KMS_DRIVER_MINOR 1 #define KMS_DRIVER_PATCHLEVEL 0 int amdgpu_vram_limit = 0; @@ -61,7 +64,7 @@ int amdgpu_disp_priority = 0; int amdgpu_hw_i2c = 0; int amdgpu_pcie_gen2 = -1; int amdgpu_msi = -1; -int amdgpu_lockup_timeout = 10000; +int amdgpu_lockup_timeout = 0; int amdgpu_dpm = -1; int amdgpu_smc_load_fw = 1; int amdgpu_aspm = -1; @@ -73,6 +76,9 @@ int amdgpu_deep_color = 0; int amdgpu_vm_size = 8; int amdgpu_vm_block_size = -1; int amdgpu_exp_hw_support = 0; +int amdgpu_enable_scheduler = 0; +int amdgpu_sched_jobs = 16; +int amdgpu_sched_hw_submission = 2; MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes"); module_param_named(vramlimit, amdgpu_vram_limit, int, 0600); @@ -101,7 +107,7 @@ module_param_named(pcie_gen2, amdgpu_pcie_gen2, int, 0444); MODULE_PARM_DESC(msi, "MSI support (1 = enable, 0 = disable, -1 = auto)"); module_param_named(msi, amdgpu_msi, int, 0444); -MODULE_PARM_DESC(lockup_timeout, "GPU lockup timeout in ms (defaul 10000 = 10 seconds, 0 = disable)"); +MODULE_PARM_DESC(lockup_timeout, "GPU lockup timeout in ms (default 0 = disable)"); module_param_named(lockup_timeout, amdgpu_lockup_timeout, int, 0444); MODULE_PARM_DESC(dpm, "DPM support (1 = enable, 0 = disable, -1 = auto)"); @@ -137,36 +143,45 @@ module_param_named(vm_block_size, amdgpu_vm_block_size, int, 0444); MODULE_PARM_DESC(exp_hw_support, "experimental hw support (1 = enable, 0 = disable (default))"); module_param_named(exp_hw_support, amdgpu_exp_hw_support, int, 0444); +MODULE_PARM_DESC(enable_scheduler, "enable SW GPU scheduler (1 = enable, 0 = disable ((default))"); +module_param_named(enable_scheduler, amdgpu_enable_scheduler, int, 0444); + +MODULE_PARM_DESC(sched_jobs, "the max number of jobs supported in the sw queue (default 16)"); +module_param_named(sched_jobs, amdgpu_sched_jobs, int, 0444); + +MODULE_PARM_DESC(sched_hw_submission, "the max number of HW submissions (default 2)"); +module_param_named(sched_hw_submission, amdgpu_sched_hw_submission, int, 0444); + static struct pci_device_id pciidlist[] = { #ifdef CONFIG_DRM_AMDGPU_CIK /* Kaveri */ - {0x1002, 0x1304, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x1305, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU}, - {0x1002, 0x1306, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x1307, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU}, - {0x1002, 0x1309, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x130A, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x130B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x130C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x130D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x130E, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x130F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU}, - {0x1002, 0x1310, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU}, - {0x1002, 0x1311, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU}, - {0x1002, 0x1312, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU}, - {0x1002, 0x1313, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU}, - {0x1002, 0x1315, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU}, - {0x1002, 0x1316, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU}, - {0x1002, 0x1317, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x1318, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x131B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU}, - {0x1002, 0x131C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU}, - {0x1002, 0x131D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMDGPU_IS_APU}, + {0x1002, 0x1304, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x1305, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU}, + {0x1002, 0x1306, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x1307, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU}, + {0x1002, 0x1309, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x130A, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x130B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x130C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x130D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x130E, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x130F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU}, + {0x1002, 0x1310, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU}, + {0x1002, 0x1311, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU}, + {0x1002, 0x1312, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU}, + {0x1002, 0x1313, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU}, + {0x1002, 0x1315, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU}, + {0x1002, 0x1316, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU}, + {0x1002, 0x1317, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x1318, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x131B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU}, + {0x1002, 0x131C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU}, + {0x1002, 0x131D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KAVERI|AMD_IS_APU}, /* Bonaire */ - {0x1002, 0x6640, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|AMDGPU_IS_MOBILITY}, - {0x1002, 0x6641, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|AMDGPU_IS_MOBILITY}, - {0x1002, 0x6646, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|AMDGPU_IS_MOBILITY}, - {0x1002, 0x6647, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|AMDGPU_IS_MOBILITY}, + {0x1002, 0x6640, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|AMD_IS_MOBILITY}, + {0x1002, 0x6641, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|AMD_IS_MOBILITY}, + {0x1002, 0x6646, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|AMD_IS_MOBILITY}, + {0x1002, 0x6647, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|AMD_IS_MOBILITY}, {0x1002, 0x6649, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE}, {0x1002, 0x6650, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE}, {0x1002, 0x6651, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE}, @@ -188,39 +203,39 @@ static struct pci_device_id pciidlist[] = { {0x1002, 0x67BA, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAWAII}, {0x1002, 0x67BE, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_HAWAII}, /* Kabini */ - {0x1002, 0x9830, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x9831, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_APU}, - {0x1002, 0x9832, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x9833, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_APU}, - {0x1002, 0x9834, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x9835, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_APU}, - {0x1002, 0x9836, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x9837, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_APU}, - {0x1002, 0x9838, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x9839, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x983a, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_APU}, - {0x1002, 0x983b, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x983c, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_APU}, - {0x1002, 0x983d, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_APU}, - {0x1002, 0x983e, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_APU}, - {0x1002, 0x983f, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMDGPU_IS_APU}, + {0x1002, 0x9830, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x9831, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_APU}, + {0x1002, 0x9832, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x9833, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_APU}, + {0x1002, 0x9834, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x9835, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_APU}, + {0x1002, 0x9836, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x9837, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_APU}, + {0x1002, 0x9838, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x9839, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x983a, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_APU}, + {0x1002, 0x983b, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x983c, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_APU}, + {0x1002, 0x983d, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_APU}, + {0x1002, 0x983e, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_APU}, + {0x1002, 0x983f, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_KABINI|AMD_IS_APU}, /* mullins */ - {0x1002, 0x9850, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x9851, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x9852, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x9853, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x9854, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x9855, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x9856, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x9857, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x9858, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x9859, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x985A, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x985B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x985C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x985D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x985E, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, - {0x1002, 0x985F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMDGPU_IS_MOBILITY|AMDGPU_IS_APU}, + {0x1002, 0x9850, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x9851, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x9852, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x9853, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x9854, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x9855, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x9856, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x9857, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x9858, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x9859, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x985A, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x985B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x985C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x985D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x985E, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU}, + {0x1002, 0x985F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_MULLINS|AMD_IS_MOBILITY|AMD_IS_APU}, #endif /* topaz */ {0x1002, 0x6900, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TOPAZ}, @@ -238,12 +253,14 @@ static struct pci_device_id pciidlist[] = { {0x1002, 0x6930, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TONGA}, {0x1002, 0x6938, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TONGA}, {0x1002, 0x6939, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TONGA}, + /* fiji */ + {0x1002, 0x7300, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_FIJI}, /* carrizo */ - {0x1002, 0x9870, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMDGPU_IS_APU}, - {0x1002, 0x9874, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMDGPU_IS_APU}, - {0x1002, 0x9875, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMDGPU_IS_APU}, - {0x1002, 0x9876, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMDGPU_IS_APU}, - {0x1002, 0x9877, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMDGPU_IS_APU}, + {0x1002, 0x9870, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMD_IS_APU}, + {0x1002, 0x9874, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMD_IS_APU}, + {0x1002, 0x9875, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMD_IS_APU}, + {0x1002, 0x9876, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMD_IS_APU}, + {0x1002, 0x9877, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_CARRIZO|AMD_IS_APU}, {0, 0, 0} }; @@ -279,7 +296,7 @@ static int amdgpu_pci_probe(struct pci_dev *pdev, unsigned long flags = ent->driver_data; int ret; - if ((flags & AMDGPU_EXP_HW_SUPPORT) && !amdgpu_exp_hw_support) { + if ((flags & AMD_EXP_HW_SUPPORT) && !amdgpu_exp_hw_support) { DRM_INFO("This hardware requires experimental hardware support.\n" "See modparam exp_hw_support\n"); return -ENODEV; @@ -527,12 +544,15 @@ static int __init amdgpu_init(void) driver->num_ioctls = amdgpu_max_kms_ioctl; amdgpu_register_atpx_handler(); + amdgpu_amdkfd_init(); + /* let modprobe override vga console setting */ return drm_pci_init(driver, pdriver); } static void __exit amdgpu_exit(void) { + amdgpu_amdkfd_fini(); drm_pci_exit(driver, pdriver); amdgpu_unregister_atpx_handler(); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.h index cceeb33c447a..e3a4f7048042 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.h @@ -31,7 +31,7 @@ #include #include -#include "amdgpu_family.h" +#include "amd_shared.h" /* General customization: */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c index c1645d21f8e2..8a122b1b7786 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fb.c @@ -53,9 +53,9 @@ static struct fb_ops amdgpufb_ops = { .owner = THIS_MODULE, .fb_check_var = drm_fb_helper_check_var, .fb_set_par = drm_fb_helper_set_par, - .fb_fillrect = cfb_fillrect, - .fb_copyarea = cfb_copyarea, - .fb_imageblit = cfb_imageblit, + .fb_fillrect = drm_fb_helper_cfb_fillrect, + .fb_copyarea = drm_fb_helper_cfb_copyarea, + .fb_imageblit = drm_fb_helper_cfb_imageblit, .fb_pan_display = drm_fb_helper_pan_display, .fb_blank = drm_fb_helper_blank, .fb_setcmap = drm_fb_helper_setcmap, @@ -126,8 +126,8 @@ static int amdgpufb_create_pinned_object(struct amdgpu_fbdev *rfbdev, aligned_size = ALIGN(size, PAGE_SIZE); ret = amdgpu_gem_object_create(adev, aligned_size, 0, AMDGPU_GEM_DOMAIN_VRAM, - 0, true, - &gobj); + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, + true, &gobj); if (ret) { printk(KERN_ERR "failed to allocate framebuffer (%d)\n", aligned_size); @@ -179,7 +179,6 @@ static int amdgpufb_create(struct drm_fb_helper *helper, struct drm_mode_fb_cmd2 mode_cmd; struct drm_gem_object *gobj = NULL; struct amdgpu_bo *rbo = NULL; - struct device *device = &adev->pdev->dev; int ret; unsigned long tmp; @@ -201,9 +200,9 @@ static int amdgpufb_create(struct drm_fb_helper *helper, rbo = gem_to_amdgpu_bo(gobj); /* okay we have an object now allocate the framebuffer */ - info = framebuffer_alloc(0, device); - if (info == NULL) { - ret = -ENOMEM; + info = drm_fb_helper_alloc_fbi(helper); + if (IS_ERR(info)) { + ret = PTR_ERR(info); goto out_unref; } @@ -212,14 +211,13 @@ static int amdgpufb_create(struct drm_fb_helper *helper, ret = amdgpu_framebuffer_init(adev->ddev, &rfbdev->rfb, &mode_cmd, gobj); if (ret) { DRM_ERROR("failed to initialize framebuffer %d\n", ret); - goto out_unref; + goto out_destroy_fbi; } fb = &rfbdev->rfb.base; /* setup helper */ rfbdev->helper.fb = fb; - rfbdev->helper.fbdev = info; memset_io(rbo->kptr, 0x0, amdgpu_bo_size(rbo)); @@ -239,11 +237,6 @@ static int amdgpufb_create(struct drm_fb_helper *helper, drm_fb_helper_fill_var(info, &rfbdev->helper, sizes->fb_width, sizes->fb_height); /* setup aperture base/size for vesafb takeover */ - info->apertures = alloc_apertures(1); - if (!info->apertures) { - ret = -ENOMEM; - goto out_unref; - } info->apertures->ranges[0].base = adev->ddev->mode_config.fb_base; info->apertures->ranges[0].size = adev->mc.aper_size; @@ -251,13 +244,7 @@ static int amdgpufb_create(struct drm_fb_helper *helper, if (info->screen_base == NULL) { ret = -ENOSPC; - goto out_unref; - } - - ret = fb_alloc_cmap(&info->cmap, 256, 0); - if (ret) { - ret = -ENOMEM; - goto out_unref; + goto out_destroy_fbi; } DRM_INFO("fb mappable at 0x%lX\n", info->fix.smem_start); @@ -269,6 +256,8 @@ static int amdgpufb_create(struct drm_fb_helper *helper, vga_switcheroo_client_fb_set(adev->ddev->pdev, info); return 0; +out_destroy_fbi: + drm_fb_helper_release_fbi(helper); out_unref: if (rbo) { @@ -290,17 +279,10 @@ void amdgpu_fb_output_poll_changed(struct amdgpu_device *adev) static int amdgpu_fbdev_destroy(struct drm_device *dev, struct amdgpu_fbdev *rfbdev) { - struct fb_info *info; struct amdgpu_framebuffer *rfb = &rfbdev->rfb; - if (rfbdev->helper.fbdev) { - info = rfbdev->helper.fbdev; - - unregister_framebuffer(info); - if (info->cmap.len) - fb_dealloc_cmap(&info->cmap); - framebuffer_release(info); - } + drm_fb_helper_unregister_fbi(&rfbdev->helper); + drm_fb_helper_release_fbi(&rfbdev->helper); if (rfb->obj) { amdgpufb_destroy_pinned_object(rfb->obj); @@ -395,7 +377,8 @@ void amdgpu_fbdev_fini(struct amdgpu_device *adev) void amdgpu_fbdev_set_suspend(struct amdgpu_device *adev, int state) { if (adev->mode_info.rfbdev) - fb_set_suspend(adev->mode_info.rfbdev->helper.fbdev, state); + drm_fb_helper_set_suspend(&adev->mode_info.rfbdev->helper, + state); } int amdgpu_fbdev_total_size(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index a7189a1fa6a1..1be2bd6d07ea 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -126,7 +126,8 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, void *owner, (*fence)->ring = ring; (*fence)->owner = owner; fence_init(&(*fence)->base, &amdgpu_fence_ops, - &adev->fence_queue.lock, adev->fence_context + ring->idx, + &ring->fence_drv.fence_queue.lock, + adev->fence_context + ring->idx, (*fence)->seq); amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr, (*fence)->seq, @@ -135,38 +136,6 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, void *owner, return 0; } -/** - * amdgpu_fence_recreate - recreate a fence from an user fence - * - * @ring: ring the fence is associated with - * @owner: creator of the fence - * @seq: user fence sequence number - * @fence: resulting amdgpu fence object - * - * Recreates a fence command from the user fence sequence number (all asics). - * Returns 0 on success, -ENOMEM on failure. - */ -int amdgpu_fence_recreate(struct amdgpu_ring *ring, void *owner, - uint64_t seq, struct amdgpu_fence **fence) -{ - struct amdgpu_device *adev = ring->adev; - - if (seq > ring->fence_drv.sync_seq[ring->idx]) - return -EINVAL; - - *fence = kmalloc(sizeof(struct amdgpu_fence), GFP_KERNEL); - if ((*fence) == NULL) - return -ENOMEM; - - (*fence)->seq = seq; - (*fence)->ring = ring; - (*fence)->owner = owner; - fence_init(&(*fence)->base, &amdgpu_fence_ops, - &adev->fence_queue.lock, adev->fence_context + ring->idx, - (*fence)->seq); - return 0; -} - /** * amdgpu_fence_check_signaled - callback from fence_queue * @@ -196,9 +165,7 @@ static int amdgpu_fence_check_signaled(wait_queue_t *wait, unsigned mode, int fl else FENCE_TRACE(&fence->base, "was already signaled\n"); - amdgpu_irq_put(adev, fence->ring->fence_drv.irq_src, - fence->ring->fence_drv.irq_type); - __remove_wait_queue(&adev->fence_queue, &fence->fence_wake); + __remove_wait_queue(&fence->ring->fence_drv.fence_queue, &fence->fence_wake); fence_put(&fence->base); } else FENCE_TRACE(&fence->base, "pending\n"); @@ -299,14 +266,9 @@ static void amdgpu_fence_check_lockup(struct work_struct *work) return; } - if (fence_drv->delayed_irq && ring->adev->ddev->irq_enabled) { - fence_drv->delayed_irq = false; - amdgpu_irq_update(ring->adev, fence_drv->irq_src, - fence_drv->irq_type); + if (amdgpu_fence_activity(ring)) { + wake_up_all(&ring->fence_drv.fence_queue); } - - if (amdgpu_fence_activity(ring)) - wake_up_all(&ring->adev->fence_queue); else if (amdgpu_ring_is_lockup(ring)) { /* good news we believe it's a lockup */ dev_warn(ring->adev->dev, "GPU lockup (current fence id " @@ -316,7 +278,7 @@ static void amdgpu_fence_check_lockup(struct work_struct *work) /* remember that we need an reset */ ring->adev->needs_reset = true; - wake_up_all(&ring->adev->fence_queue); + wake_up_all(&ring->fence_drv.fence_queue); } up_read(&ring->adev->exclusive_lock); } @@ -332,62 +294,8 @@ static void amdgpu_fence_check_lockup(struct work_struct *work) */ void amdgpu_fence_process(struct amdgpu_ring *ring) { - uint64_t seq, last_seq, last_emitted; - unsigned count_loop = 0; - bool wake = false; - - /* Note there is a scenario here for an infinite loop but it's - * very unlikely to happen. For it to happen, the current polling - * process need to be interrupted by another process and another - * process needs to update the last_seq btw the atomic read and - * xchg of the current process. - * - * More over for this to go in infinite loop there need to be - * continuously new fence signaled ie amdgpu_fence_read needs - * to return a different value each time for both the currently - * polling process and the other process that xchg the last_seq - * btw atomic read and xchg of the current process. And the - * value the other process set as last seq must be higher than - * the seq value we just read. Which means that current process - * need to be interrupted after amdgpu_fence_read and before - * atomic xchg. - * - * To be even more safe we count the number of time we loop and - * we bail after 10 loop just accepting the fact that we might - * have temporarly set the last_seq not to the true real last - * seq but to an older one. - */ - last_seq = atomic64_read(&ring->fence_drv.last_seq); - do { - last_emitted = ring->fence_drv.sync_seq[ring->idx]; - seq = amdgpu_fence_read(ring); - seq |= last_seq & 0xffffffff00000000LL; - if (seq < last_seq) { - seq &= 0xffffffff; - seq |= last_emitted & 0xffffffff00000000LL; - } - - if (seq <= last_seq || seq > last_emitted) { - break; - } - /* If we loop over we don't want to return without - * checking if a fence is signaled as it means that the - * seq we just read is different from the previous on. - */ - wake = true; - last_seq = seq; - if ((count_loop++) > 10) { - /* We looped over too many time leave with the - * fact that we might have set an older fence - * seq then the current real last seq as signaled - * by the hw. - */ - break; - } - } while (atomic64_xchg(&ring->fence_drv.last_seq, seq) > seq); - - if (wake) - wake_up_all(&ring->adev->fence_queue); + if (amdgpu_fence_activity(ring)) + wake_up_all(&ring->fence_drv.fence_queue); } /** @@ -447,284 +355,49 @@ static bool amdgpu_fence_enable_signaling(struct fence *f) { struct amdgpu_fence *fence = to_amdgpu_fence(f); struct amdgpu_ring *ring = fence->ring; - struct amdgpu_device *adev = ring->adev; if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq) return false; - if (down_read_trylock(&adev->exclusive_lock)) { - amdgpu_irq_get(adev, ring->fence_drv.irq_src, - ring->fence_drv.irq_type); - if (amdgpu_fence_activity(ring)) - wake_up_all_locked(&adev->fence_queue); - - /* did fence get signaled after we enabled the sw irq? */ - if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq) { - amdgpu_irq_put(adev, ring->fence_drv.irq_src, - ring->fence_drv.irq_type); - up_read(&adev->exclusive_lock); - return false; - } - - up_read(&adev->exclusive_lock); - } else { - /* we're probably in a lockup, lets not fiddle too much */ - if (amdgpu_irq_get_delayed(adev, ring->fence_drv.irq_src, - ring->fence_drv.irq_type)) - ring->fence_drv.delayed_irq = true; - amdgpu_fence_schedule_check(ring); - } - fence->fence_wake.flags = 0; fence->fence_wake.private = NULL; fence->fence_wake.func = amdgpu_fence_check_signaled; - __add_wait_queue(&adev->fence_queue, &fence->fence_wake); + __add_wait_queue(&ring->fence_drv.fence_queue, &fence->fence_wake); fence_get(f); FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx); return true; } -/** - * amdgpu_fence_signaled - check if a fence has signaled +/* + * amdgpu_ring_wait_seq_timeout - wait for seq of the specific ring to signal + * @ring: ring to wait on for the seq number + * @seq: seq number wait for * - * @fence: amdgpu fence object - * - * Check if the requested fence has signaled (all asics). - * Returns true if the fence has signaled or false if it has not. + * return value: + * 0: seq signaled, and gpu not hang + * -EDEADL: GPU hang detected + * -EINVAL: some paramter is not valid */ -bool amdgpu_fence_signaled(struct amdgpu_fence *fence) +static int amdgpu_fence_ring_wait_seq(struct amdgpu_ring *ring, uint64_t seq) { - if (!fence) - return true; + struct amdgpu_device *adev = ring->adev; + bool signaled = false; - if (amdgpu_fence_seq_signaled(fence->ring, fence->seq)) { - if (!fence_signal(&fence->base)) - FENCE_TRACE(&fence->base, "signaled from amdgpu_fence_signaled\n"); - return true; - } + BUG_ON(!ring); + if (seq > ring->fence_drv.sync_seq[ring->idx]) + return -EINVAL; - return false; -} + if (atomic64_read(&ring->fence_drv.last_seq) >= seq) + return 0; -/** - * amdgpu_fence_any_seq_signaled - check if any sequence number is signaled - * - * @adev: amdgpu device pointer - * @seq: sequence numbers - * - * Check if the last signaled fence sequnce number is >= the requested - * sequence number (all asics). - * Returns true if any has signaled (current value is >= requested value) - * or false if it has not. Helper function for amdgpu_fence_wait_seq. - */ -static bool amdgpu_fence_any_seq_signaled(struct amdgpu_device *adev, u64 *seq) -{ - unsigned i; + wait_event(ring->fence_drv.fence_queue, ( + (signaled = amdgpu_fence_seq_signaled(ring, seq)) + || adev->needs_reset)); - for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { - if (!adev->rings[i] || !seq[i]) - continue; - - if (amdgpu_fence_seq_signaled(adev->rings[i], seq[i])) - return true; - } - - return false; -} - -/** - * amdgpu_fence_wait_seq_timeout - wait for a specific sequence numbers - * - * @adev: amdgpu device pointer - * @target_seq: sequence number(s) we want to wait for - * @intr: use interruptable sleep - * @timeout: maximum time to wait, or MAX_SCHEDULE_TIMEOUT for infinite wait - * - * Wait for the requested sequence number(s) to be written by any ring - * (all asics). Sequnce number array is indexed by ring id. - * @intr selects whether to use interruptable (true) or non-interruptable - * (false) sleep when waiting for the sequence number. Helper function - * for amdgpu_fence_wait_*(). - * Returns remaining time if the sequence number has passed, 0 when - * the wait timeout, or an error for all other cases. - * -EDEADLK is returned when a GPU lockup has been detected. - */ -static long amdgpu_fence_wait_seq_timeout(struct amdgpu_device *adev, - u64 *target_seq, bool intr, - long timeout) -{ - uint64_t last_seq[AMDGPU_MAX_RINGS]; - bool signaled; - int i; - long r; - - if (timeout == 0) { - return amdgpu_fence_any_seq_signaled(adev, target_seq); - } - - while (!amdgpu_fence_any_seq_signaled(adev, target_seq)) { - - /* Save current sequence values, used to check for GPU lockups */ - for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { - struct amdgpu_ring *ring = adev->rings[i]; - - if (!ring || !target_seq[i]) - continue; - - last_seq[i] = atomic64_read(&ring->fence_drv.last_seq); - trace_amdgpu_fence_wait_begin(adev->ddev, i, target_seq[i]); - amdgpu_irq_get(adev, ring->fence_drv.irq_src, - ring->fence_drv.irq_type); - } - - if (intr) { - r = wait_event_interruptible_timeout(adev->fence_queue, ( - (signaled = amdgpu_fence_any_seq_signaled(adev, target_seq)) - || adev->needs_reset), AMDGPU_FENCE_JIFFIES_TIMEOUT); - } else { - r = wait_event_timeout(adev->fence_queue, ( - (signaled = amdgpu_fence_any_seq_signaled(adev, target_seq)) - || adev->needs_reset), AMDGPU_FENCE_JIFFIES_TIMEOUT); - } - - for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { - struct amdgpu_ring *ring = adev->rings[i]; - - if (!ring || !target_seq[i]) - continue; - - amdgpu_irq_put(adev, ring->fence_drv.irq_src, - ring->fence_drv.irq_type); - trace_amdgpu_fence_wait_end(adev->ddev, i, target_seq[i]); - } - - if (unlikely(r < 0)) - return r; - - if (unlikely(!signaled)) { - - if (adev->needs_reset) - return -EDEADLK; - - /* we were interrupted for some reason and fence - * isn't signaled yet, resume waiting */ - if (r) - continue; - - for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { - struct amdgpu_ring *ring = adev->rings[i]; - - if (!ring || !target_seq[i]) - continue; - - if (last_seq[i] != atomic64_read(&ring->fence_drv.last_seq)) - break; - } - - if (i != AMDGPU_MAX_RINGS) - continue; - - for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { - if (!adev->rings[i] || !target_seq[i]) - continue; - - if (amdgpu_ring_is_lockup(adev->rings[i])) - break; - } - - if (i < AMDGPU_MAX_RINGS) { - /* good news we believe it's a lockup */ - dev_warn(adev->dev, "GPU lockup (waiting for " - "0x%016llx last fence id 0x%016llx on" - " ring %d)\n", - target_seq[i], last_seq[i], i); - - /* remember that we need an reset */ - adev->needs_reset = true; - wake_up_all(&adev->fence_queue); - return -EDEADLK; - } - - if (timeout < MAX_SCHEDULE_TIMEOUT) { - timeout -= AMDGPU_FENCE_JIFFIES_TIMEOUT; - if (timeout <= 0) { - return 0; - } - } - } - } - return timeout; -} - -/** - * amdgpu_fence_wait - wait for a fence to signal - * - * @fence: amdgpu fence object - * @intr: use interruptable sleep - * - * Wait for the requested fence to signal (all asics). - * @intr selects whether to use interruptable (true) or non-interruptable - * (false) sleep when waiting for the fence. - * Returns 0 if the fence has passed, error for all other cases. - */ -int amdgpu_fence_wait(struct amdgpu_fence *fence, bool intr) -{ - uint64_t seq[AMDGPU_MAX_RINGS] = {}; - long r; - - seq[fence->ring->idx] = fence->seq; - r = amdgpu_fence_wait_seq_timeout(fence->ring->adev, seq, intr, MAX_SCHEDULE_TIMEOUT); - if (r < 0) { - return r; - } - - r = fence_signal(&fence->base); - if (!r) - FENCE_TRACE(&fence->base, "signaled from fence_wait\n"); - return 0; -} - -/** - * amdgpu_fence_wait_any - wait for a fence to signal on any ring - * - * @adev: amdgpu device pointer - * @fences: amdgpu fence object(s) - * @intr: use interruptable sleep - * - * Wait for any requested fence to signal (all asics). Fence - * array is indexed by ring id. @intr selects whether to use - * interruptable (true) or non-interruptable (false) sleep when - * waiting for the fences. Used by the suballocator. - * Returns 0 if any fence has passed, error for all other cases. - */ -int amdgpu_fence_wait_any(struct amdgpu_device *adev, - struct amdgpu_fence **fences, - bool intr) -{ - uint64_t seq[AMDGPU_MAX_RINGS]; - unsigned i, num_rings = 0; - long r; - - for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { - seq[i] = 0; - - if (!fences[i]) { - continue; - } - - seq[i] = fences[i]->seq; - ++num_rings; - } - - /* nothing to wait for ? */ - if (num_rings == 0) - return -ENOENT; - - r = amdgpu_fence_wait_seq_timeout(adev, seq, intr, MAX_SCHEDULE_TIMEOUT); - if (r < 0) { - return r; - } - return 0; + if (signaled) + return 0; + else + return -EDEADLK; } /** @@ -739,19 +412,12 @@ int amdgpu_fence_wait_any(struct amdgpu_device *adev, */ int amdgpu_fence_wait_next(struct amdgpu_ring *ring) { - uint64_t seq[AMDGPU_MAX_RINGS] = {}; - long r; + uint64_t seq = atomic64_read(&ring->fence_drv.last_seq) + 1ULL; - seq[ring->idx] = atomic64_read(&ring->fence_drv.last_seq) + 1ULL; - if (seq[ring->idx] >= ring->fence_drv.sync_seq[ring->idx]) { - /* nothing to wait for, last_seq is - already the last emited fence */ + if (seq >= ring->fence_drv.sync_seq[ring->idx]) return -ENOENT; - } - r = amdgpu_fence_wait_seq_timeout(ring->adev, seq, false, MAX_SCHEDULE_TIMEOUT); - if (r < 0) - return r; - return 0; + + return amdgpu_fence_ring_wait_seq(ring, seq); } /** @@ -766,23 +432,12 @@ int amdgpu_fence_wait_next(struct amdgpu_ring *ring) */ int amdgpu_fence_wait_empty(struct amdgpu_ring *ring) { - struct amdgpu_device *adev = ring->adev; - uint64_t seq[AMDGPU_MAX_RINGS] = {}; - long r; + uint64_t seq = ring->fence_drv.sync_seq[ring->idx]; - seq[ring->idx] = ring->fence_drv.sync_seq[ring->idx]; - if (!seq[ring->idx]) + if (!seq) return 0; - r = amdgpu_fence_wait_seq_timeout(adev, seq, false, MAX_SCHEDULE_TIMEOUT); - if (r < 0) { - if (r == -EDEADLK) - return -EDEADLK; - - dev_err(adev->dev, "error waiting for ring[%d] to become idle (%ld)\n", - ring->idx, r); - } - return 0; + return amdgpu_fence_ring_wait_seq(ring, seq); } /** @@ -933,9 +588,12 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring, ring->fence_drv.gpu_addr = adev->uvd.gpu_addr + index; } amdgpu_fence_write(ring, atomic64_read(&ring->fence_drv.last_seq)); - ring->fence_drv.initialized = true; + amdgpu_irq_get(adev, irq_src, irq_type); + ring->fence_drv.irq_src = irq_src; ring->fence_drv.irq_type = irq_type; + ring->fence_drv.initialized = true; + dev_info(adev->dev, "fence driver on ring %d use gpu addr 0x%016llx, " "cpu addr 0x%p\n", ring->idx, ring->fence_drv.gpu_addr, ring->fence_drv.cpu_addr); @@ -966,6 +624,16 @@ void amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring) INIT_DELAYED_WORK(&ring->fence_drv.lockup_work, amdgpu_fence_check_lockup); ring->fence_drv.ring = ring; + + if (amdgpu_enable_scheduler) { + ring->scheduler = amd_sched_create(&amdgpu_sched_ops, + ring->idx, + amdgpu_sched_hw_submission, + (void *)ring->adev); + if (!ring->scheduler) + DRM_ERROR("Failed to create scheduler on ring %d.\n", + ring->idx); + } } /** @@ -982,7 +650,6 @@ void amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring) */ int amdgpu_fence_driver_init(struct amdgpu_device *adev) { - init_waitqueue_head(&adev->fence_queue); if (amdgpu_debugfs_fence_init(adev)) dev_err(adev->dev, "fence debugfs file creation failed\n"); @@ -1011,12 +678,77 @@ void amdgpu_fence_driver_fini(struct amdgpu_device *adev) /* no need to trigger GPU reset as we are unloading */ amdgpu_fence_driver_force_completion(adev); } - wake_up_all(&adev->fence_queue); + wake_up_all(&ring->fence_drv.fence_queue); + amdgpu_irq_put(adev, ring->fence_drv.irq_src, + ring->fence_drv.irq_type); + if (ring->scheduler) + amd_sched_destroy(ring->scheduler); ring->fence_drv.initialized = false; } mutex_unlock(&adev->ring_lock); } +/** + * amdgpu_fence_driver_suspend - suspend the fence driver + * for all possible rings. + * + * @adev: amdgpu device pointer + * + * Suspend the fence driver for all possible rings (all asics). + */ +void amdgpu_fence_driver_suspend(struct amdgpu_device *adev) +{ + int i, r; + + mutex_lock(&adev->ring_lock); + for (i = 0; i < AMDGPU_MAX_RINGS; i++) { + struct amdgpu_ring *ring = adev->rings[i]; + if (!ring || !ring->fence_drv.initialized) + continue; + + /* wait for gpu to finish processing current batch */ + r = amdgpu_fence_wait_empty(ring); + if (r) { + /* delay GPU reset to resume */ + amdgpu_fence_driver_force_completion(adev); + } + + /* disable the interrupt */ + amdgpu_irq_put(adev, ring->fence_drv.irq_src, + ring->fence_drv.irq_type); + } + mutex_unlock(&adev->ring_lock); +} + +/** + * amdgpu_fence_driver_resume - resume the fence driver + * for all possible rings. + * + * @adev: amdgpu device pointer + * + * Resume the fence driver for all possible rings (all asics). + * Not all asics have all rings, so each asic will only + * start the fence driver on the rings it has using + * amdgpu_fence_driver_start_ring(). + * Returns 0 for success. + */ +void amdgpu_fence_driver_resume(struct amdgpu_device *adev) +{ + int i; + + mutex_lock(&adev->ring_lock); + for (i = 0; i < AMDGPU_MAX_RINGS; i++) { + struct amdgpu_ring *ring = adev->rings[i]; + if (!ring || !ring->fence_drv.initialized) + continue; + + /* enable the interrupt */ + amdgpu_irq_get(adev, ring->fence_drv.irq_src, + ring->fence_drv.irq_type); + } + mutex_unlock(&adev->ring_lock); +} + /** * amdgpu_fence_driver_force_completion - force all fence waiter to complete * @@ -1104,6 +836,21 @@ static inline bool amdgpu_test_signaled(struct amdgpu_fence *fence) return test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags); } +static bool amdgpu_test_signaled_any(struct fence **fences, uint32_t count) +{ + int idx; + struct fence *fence; + + for (idx = 0; idx < count; ++idx) { + fence = fences[idx]; + if (fence) { + if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) + return true; + } + } + return false; +} + struct amdgpu_wait_cb { struct fence_cb base; struct task_struct *task; @@ -1121,12 +868,48 @@ static signed long amdgpu_fence_default_wait(struct fence *f, bool intr, { struct amdgpu_fence *fence = to_amdgpu_fence(f); struct amdgpu_device *adev = fence->ring->adev; - struct amdgpu_wait_cb cb; - cb.task = current; + return amdgpu_fence_wait_any(adev, &f, 1, intr, t); +} - if (fence_add_callback(f, &cb.base, amdgpu_fence_wait_cb)) - return t; +/** + * Wait the fence array with timeout + * + * @adev: amdgpu device + * @array: the fence array with amdgpu fence pointer + * @count: the number of the fence array + * @intr: when sleep, set the current task interruptable or not + * @t: timeout to wait + * + * It will return when any fence is signaled or timeout. + */ +signed long amdgpu_fence_wait_any(struct amdgpu_device *adev, + struct fence **array, uint32_t count, + bool intr, signed long t) +{ + struct amdgpu_wait_cb *cb; + struct fence *fence; + unsigned idx; + + BUG_ON(!array); + + cb = kcalloc(count, sizeof(struct amdgpu_wait_cb), GFP_KERNEL); + if (cb == NULL) { + t = -ENOMEM; + goto err_free_cb; + } + + for (idx = 0; idx < count; ++idx) { + fence = array[idx]; + if (fence) { + cb[idx].task = current; + if (fence_add_callback(fence, + &cb[idx].base, amdgpu_fence_wait_cb)) { + /* The fence is already signaled */ + goto fence_rm_cb; + } + } + } while (t > 0) { if (intr) @@ -1135,10 +918,10 @@ static signed long amdgpu_fence_default_wait(struct fence *f, bool intr, set_current_state(TASK_UNINTERRUPTIBLE); /* - * amdgpu_test_signaled must be called after + * amdgpu_test_signaled_any must be called after * set_current_state to prevent a race with wake_up_process */ - if (amdgpu_test_signaled(fence)) + if (amdgpu_test_signaled_any(array, count)) break; if (adev->needs_reset) { @@ -1153,7 +936,16 @@ static signed long amdgpu_fence_default_wait(struct fence *f, bool intr, } __set_current_state(TASK_RUNNING); - fence_remove_callback(f, &cb.base); + +fence_rm_cb: + for (idx = 0; idx < count; ++idx) { + fence = array[idx]; + if (fence && cb[idx].base.func) + fence_remove_callback(fence, &cb[idx].base); + } + +err_free_cb: + kfree(cb); return t; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c index e02db0b2e839..cbd3a486c5c2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c @@ -125,7 +125,8 @@ int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev) if (adev->gart.robj == NULL) { r = amdgpu_bo_create(adev, adev->gart.table_size, - PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM, 0, + PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, NULL, &adev->gart.robj); if (r) { return r; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index 4afc507820c0..5839fab374bf 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -615,6 +615,7 @@ int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data, info.alignment = robj->tbo.mem.page_alignment << PAGE_SHIFT; info.domains = robj->initial_domain; info.domain_flags = robj->flags; + amdgpu_bo_unreserve(robj); if (copy_to_user(out, &info, sizeof(info))) r = -EFAULT; break; @@ -622,17 +623,19 @@ int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data, case AMDGPU_GEM_OP_SET_PLACEMENT: if (amdgpu_ttm_tt_has_userptr(robj->tbo.ttm)) { r = -EPERM; + amdgpu_bo_unreserve(robj); break; } robj->initial_domain = args->value & (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT | AMDGPU_GEM_DOMAIN_CPU); + amdgpu_bo_unreserve(robj); break; default: + amdgpu_bo_unreserve(robj); r = -EINVAL; } - amdgpu_bo_unreserve(robj); out: drm_gem_object_unreference_unlocked(gobj); return r; @@ -653,7 +656,8 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv, r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_VRAM, - 0, ttm_bo_type_device, + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, + ttm_bo_type_device, &gobj); if (r) return -ENOMEM; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index bc0fac618a3f..c439735ee670 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -73,28 +73,12 @@ int amdgpu_ib_get(struct amdgpu_ring *ring, struct amdgpu_vm *vm, if (!vm) ib->gpu_addr = amdgpu_sa_bo_gpu_addr(ib->sa_bo); - else - ib->gpu_addr = 0; - - } else { - ib->sa_bo = NULL; - ib->ptr = NULL; - ib->gpu_addr = 0; } amdgpu_sync_create(&ib->sync); ib->ring = ring; - ib->fence = NULL; - ib->user = NULL; ib->vm = vm; - ib->gds_base = 0; - ib->gds_size = 0; - ib->gws_base = 0; - ib->gws_size = 0; - ib->oa_base = 0; - ib->oa_size = 0; - ib->flags = 0; return 0; } @@ -109,8 +93,8 @@ int amdgpu_ib_get(struct amdgpu_ring *ring, struct amdgpu_vm *vm, */ void amdgpu_ib_free(struct amdgpu_device *adev, struct amdgpu_ib *ib) { - amdgpu_sync_free(adev, &ib->sync, ib->fence); - amdgpu_sa_bo_free(adev, &ib->sa_bo, ib->fence); + amdgpu_sync_free(adev, &ib->sync, &ib->fence->base); + amdgpu_sa_bo_free(adev, &ib->sa_bo, &ib->fence->base); amdgpu_fence_unref(&ib->fence); } @@ -156,7 +140,11 @@ int amdgpu_ib_schedule(struct amdgpu_device *adev, unsigned num_ibs, dev_err(adev->dev, "couldn't schedule ib\n"); return -EINVAL; } - + r = amdgpu_sync_wait(&ibs->sync); + if (r) { + dev_err(adev->dev, "IB sync failed (%d).\n", r); + return r; + } r = amdgpu_ring_lock(ring, (256 + AMDGPU_NUM_SYNCS * 8) * num_ibs); if (r) { dev_err(adev->dev, "scheduling IB failed (%d).\n", r); @@ -165,9 +153,11 @@ int amdgpu_ib_schedule(struct amdgpu_device *adev, unsigned num_ibs, if (vm) { /* grab a vm id if necessary */ - struct amdgpu_fence *vm_id_fence = NULL; - vm_id_fence = amdgpu_vm_grab_id(ibs->ring, ibs->vm); - amdgpu_sync_fence(&ibs->sync, vm_id_fence); + r = amdgpu_vm_grab_id(ibs->vm, ibs->ring, &ibs->sync); + if (r) { + amdgpu_ring_unlock_undo(ring); + return r; + } } r = amdgpu_sync_rings(&ibs->sync, ring); @@ -212,11 +202,15 @@ int amdgpu_ib_schedule(struct amdgpu_device *adev, unsigned num_ibs, return r; } + if (!amdgpu_enable_scheduler && ib->ctx) + ib->sequence = amdgpu_ctx_add_fence(ib->ctx, ring, + &ib->fence->base); + /* wrap the last IB with fence */ if (ib->user) { uint64_t addr = amdgpu_bo_gpu_offset(ib->user->bo); addr += ib->user->offset; - amdgpu_ring_emit_fence(ring, addr, ib->fence->seq, + amdgpu_ring_emit_fence(ring, addr, ib->sequence, AMDGPU_FENCE_FLAG_64BIT); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c index db5422e65ec5..5c8a803acedc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c @@ -24,6 +24,7 @@ #include #include "amdgpu.h" #include "amdgpu_ih.h" +#include "amdgpu_amdkfd.h" /** * amdgpu_ih_ring_alloc - allocate memory for the IH ring @@ -97,18 +98,12 @@ int amdgpu_ih_ring_init(struct amdgpu_device *adev, unsigned ring_size, /* add 8 bytes for the rptr/wptr shadows and * add them to the end of the ring allocation. */ - adev->irq.ih.ring = kzalloc(adev->irq.ih.ring_size + 8, GFP_KERNEL); + adev->irq.ih.ring = pci_alloc_consistent(adev->pdev, + adev->irq.ih.ring_size + 8, + &adev->irq.ih.rb_dma_addr); if (adev->irq.ih.ring == NULL) return -ENOMEM; - adev->irq.ih.rb_dma_addr = pci_map_single(adev->pdev, - (void *)adev->irq.ih.ring, - adev->irq.ih.ring_size, - PCI_DMA_BIDIRECTIONAL); - if (pci_dma_mapping_error(adev->pdev, adev->irq.ih.rb_dma_addr)) { - dev_err(&adev->pdev->dev, "Failed to DMA MAP the IH RB page\n"); - kfree((void *)adev->irq.ih.ring); - return -ENOMEM; - } + memset((void *)adev->irq.ih.ring, 0, adev->irq.ih.ring_size + 8); adev->irq.ih.wptr_offs = (adev->irq.ih.ring_size / 4) + 0; adev->irq.ih.rptr_offs = (adev->irq.ih.ring_size / 4) + 1; } @@ -148,9 +143,9 @@ void amdgpu_ih_ring_fini(struct amdgpu_device *adev) /* add 8 bytes for the rptr/wptr shadows and * add them to the end of the ring allocation. */ - pci_unmap_single(adev->pdev, adev->irq.ih.rb_dma_addr, - adev->irq.ih.ring_size + 8, PCI_DMA_BIDIRECTIONAL); - kfree((void *)adev->irq.ih.ring); + pci_free_consistent(adev->pdev, adev->irq.ih.ring_size + 8, + (void *)adev->irq.ih.ring, + adev->irq.ih.rb_dma_addr); adev->irq.ih.ring = NULL; } } else { @@ -199,6 +194,14 @@ int amdgpu_ih_process(struct amdgpu_device *adev) rmb(); while (adev->irq.ih.rptr != wptr) { + u32 ring_index = adev->irq.ih.rptr >> 2; + + /* Before dispatching irq to IP blocks, send it to amdkfd */ + amdgpu_amdkfd_interrupt(adev, + (const void *) &adev->irq.ih.ring[ring_index]); + + entry.iv_entry = (const uint32_t *) + &adev->irq.ih.ring[ring_index]; amdgpu_ih_decode_iv(adev, &entry); adev->irq.ih.rptr &= adev->irq.ih.ptr_mask; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h index c62b09e555d6..ba38ae6a1463 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h @@ -52,6 +52,7 @@ struct amdgpu_iv_entry { unsigned ring_id; unsigned vm_id; unsigned pas_id; + const uint32_t *iv_entry; }; int amdgpu_ih_ring_init(struct amdgpu_device *adev, unsigned ring_size, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c index b4d36f0f2153..0aba8e9bc8a0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c @@ -272,6 +272,11 @@ void amdgpu_irq_fini(struct amdgpu_device *adev) kfree(src->enabled_types); src->enabled_types = NULL; + if (src->data) { + kfree(src->data); + kfree(src); + adev->irq.sources[i] = NULL; + } } } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h index 8299795f2b2d..17b01aef4278 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h @@ -40,6 +40,7 @@ struct amdgpu_irq_src { unsigned num_types; atomic_t *enabled_types; const struct amdgpu_irq_src_funcs *funcs; + void *data; }; /* provided by interrupt generating IP blocks */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index 3bfe67de8349..22367939ebf1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -34,6 +34,7 @@ #include #include #include +#include "amdgpu_amdkfd.h" #if defined(CONFIG_VGA_SWITCHEROO) bool amdgpu_has_atpx(void); @@ -61,6 +62,8 @@ int amdgpu_driver_unload_kms(struct drm_device *dev) pm_runtime_get_sync(dev->dev); + amdgpu_amdkfd_device_fini(adev); + amdgpu_acpi_fini(adev); amdgpu_device_fini(adev); @@ -93,8 +96,8 @@ int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags) if ((amdgpu_runtime_pm != 0) && amdgpu_has_atpx() && - ((flags & AMDGPU_IS_APU) == 0)) - flags |= AMDGPU_IS_PX; + ((flags & AMD_IS_APU) == 0)) + flags |= AMD_IS_PX; /* amdgpu_device_init should report only fatal error * like memory allocation failure or iomapping failure, @@ -118,6 +121,10 @@ int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags) "Error during ACPI methods call\n"); } + amdgpu_amdkfd_load_interface(adev); + amdgpu_amdkfd_device_probe(adev); + amdgpu_amdkfd_device_init(adev); + if (amdgpu_device_is_px(dev)) { pm_runtime_use_autosuspend(dev->dev); pm_runtime_set_autosuspend_delay(dev->dev, 5000); @@ -444,11 +451,11 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file dev_info.num_hw_gfx_contexts = adev->gfx.config.max_hw_contexts; dev_info._pad = 0; dev_info.ids_flags = 0; - if (adev->flags & AMDGPU_IS_APU) + if (adev->flags & AMD_IS_APU) dev_info.ids_flags |= AMDGPU_IDS_FLAGS_FUSION; dev_info.virtual_address_offset = AMDGPU_VA_RESERVED_SIZE; dev_info.virtual_address_max = (uint64_t)adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE; - dev_info.virtual_address_alignment = max(PAGE_SIZE, 0x10000UL); + dev_info.virtual_address_alignment = max((int)PAGE_SIZE, AMDGPU_GPU_PAGE_SIZE); dev_info.pte_fragment_size = (1 << AMDGPU_LOG2_PAGES_PER_FRAG) * AMDGPU_GPU_PAGE_SIZE; dev_info.gart_page_size = AMDGPU_GPU_PAGE_SIZE; @@ -520,10 +527,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv) mutex_init(&fpriv->bo_list_lock); idr_init(&fpriv->bo_list_handles); - /* init context manager */ - mutex_init(&fpriv->ctx_mgr.lock); - idr_init(&fpriv->ctx_mgr.ctx_handles); - fpriv->ctx_mgr.adev = adev; + amdgpu_ctx_mgr_init(&fpriv->ctx_mgr); file_priv->driver_priv = fpriv; @@ -556,6 +560,8 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev, if (!fpriv) return; + amdgpu_ctx_mgr_fini(&fpriv->ctx_mgr); + amdgpu_vm_fini(adev, &fpriv->vm); idr_for_each_entry(&fpriv->bo_list_handles, list, handle) @@ -564,9 +570,6 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev, idr_destroy(&fpriv->bo_list_handles); mutex_destroy(&fpriv->bo_list_lock); - /* release context */ - amdgpu_ctx_fini(fpriv); - kfree(fpriv); file_priv->driver_priv = NULL; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index 8da64245b31b..08b09d55b96f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -127,7 +127,7 @@ static void amdgpu_ttm_placement_init(struct amdgpu_device *adev, placements[c].fpfn = adev->mc.visible_vram_size >> PAGE_SHIFT; placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED | - TTM_PL_FLAG_VRAM; + TTM_PL_FLAG_VRAM | TTM_PL_FLAG_TOPDOWN; } placements[c].fpfn = 0; placements[c++].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED | @@ -223,18 +223,6 @@ int amdgpu_bo_create_restricted(struct amdgpu_device *adev, size_t acc_size; int r; - /* VI has a hw bug where VM PTEs have to be allocated in groups of 8. - * do this as a temporary workaround - */ - if (!(domain & (AMDGPU_GEM_DOMAIN_GDS | AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA))) { - if (adev->asic_type >= CHIP_TOPAZ) { - if (byte_align & 0x7fff) - byte_align = ALIGN(byte_align, 0x8000); - if (size & 0x7fff) - size = ALIGN(size, 0x8000); - } - } - page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT; size = ALIGN(size, PAGE_SIZE); @@ -462,7 +450,7 @@ int amdgpu_bo_unpin(struct amdgpu_bo *bo) int amdgpu_bo_evict_vram(struct amdgpu_device *adev) { /* late 2.6.33 fix IGP hibernate - we need pm ops to do this correct */ - if (0 && (adev->flags & AMDGPU_IS_APU)) { + if (0 && (adev->flags & AMD_IS_APU)) { /* Useless to evict on IGP chips */ return 0; } @@ -478,7 +466,6 @@ void amdgpu_bo_force_delete(struct amdgpu_device *adev) } dev_err(adev->dev, "Userspace still has active objects !\n"); list_for_each_entry_safe(bo, n, &adev->gem.objects, list) { - mutex_lock(&adev->ddev->struct_mutex); dev_err(adev->dev, "%p %p %lu %lu force free\n", &bo->gem_base, bo, (unsigned long)bo->gem_base.size, *((unsigned long *)&bo->gem_base.refcount)); @@ -486,8 +473,7 @@ void amdgpu_bo_force_delete(struct amdgpu_device *adev) list_del_init(&bo->list); mutex_unlock(&bo->adev->gem.mutex); /* this should unref the ttm bo */ - drm_gem_object_unreference(&bo->gem_base); - mutex_unlock(&adev->ddev->struct_mutex); + drm_gem_object_unreference_unlocked(&bo->gem_base); } } @@ -658,13 +644,13 @@ int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo) * @shared: true if fence should be added shared * */ -void amdgpu_bo_fence(struct amdgpu_bo *bo, struct amdgpu_fence *fence, +void amdgpu_bo_fence(struct amdgpu_bo *bo, struct fence *fence, bool shared) { struct reservation_object *resv = bo->tbo.resv; if (shared) - reservation_object_add_shared_fence(resv, &fence->base); + reservation_object_add_shared_fence(resv, fence); else - reservation_object_add_excl_fence(resv, &fence->base); + reservation_object_add_excl_fence(resv, fence); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h index 675bdc30e41d..6ea18dcec561 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h @@ -161,7 +161,7 @@ int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer, void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem); int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo); -void amdgpu_bo_fence(struct amdgpu_bo *bo, struct amdgpu_fence *fence, +void amdgpu_bo_fence(struct amdgpu_bo *bo, struct fence *fence, bool shared); /* @@ -193,7 +193,7 @@ int amdgpu_sa_bo_new(struct amdgpu_device *adev, unsigned size, unsigned align); void amdgpu_sa_bo_free(struct amdgpu_device *adev, struct amdgpu_sa_bo **sa_bo, - struct amdgpu_fence *fence); + struct fence *fence); #if defined(CONFIG_DEBUG_FS) void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager, struct seq_file *m); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c index ed13baa7c976..efed11509f4a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c @@ -82,7 +82,7 @@ static ssize_t amdgpu_set_dpm_state(struct device *dev, mutex_unlock(&adev->pm.mutex); /* Can't set dpm state when the card is off */ - if (!(adev->flags & AMDGPU_IS_PX) || + if (!(adev->flags & AMD_IS_PX) || (ddev->switch_power_state == DRM_SWITCH_POWER_ON)) amdgpu_pm_compute_clocks(adev); fail: @@ -538,7 +538,7 @@ static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev) /* vce just modifies an existing state so force a change */ if (ps->vce_active != adev->pm.dpm.vce_active) goto force; - if (adev->flags & AMDGPU_IS_APU) { + if (adev->flags & AMD_IS_APU) { /* for APUs if the num crtcs changed but state is the same, * all we need to do is update the display configuration. */ @@ -580,7 +580,6 @@ static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev) amdgpu_dpm_print_power_state(adev, adev->pm.dpm.requested_ps); } - mutex_lock(&adev->ddev->struct_mutex); mutex_lock(&adev->ring_lock); /* update whether vce is active */ @@ -628,7 +627,6 @@ static void amdgpu_dpm_change_power_state_locked(struct amdgpu_device *adev) done: mutex_unlock(&adev->ring_lock); - mutex_unlock(&adev->ddev->struct_mutex); } void amdgpu_dpm_enable_uvd(struct amdgpu_device *adev, bool enable) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c index 855e2196657a..9bec91484c24 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c @@ -131,6 +131,21 @@ int amdgpu_ring_lock(struct amdgpu_ring *ring, unsigned ndw) return 0; } +/** amdgpu_ring_insert_nop - insert NOP packets + * + * @ring: amdgpu_ring structure holding ring information + * @count: the number of NOP packets to insert + * + * This is the generic insert_nop function for rings except SDMA + */ +void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) +{ + int i; + + for (i = 0; i < count; i++) + amdgpu_ring_write(ring, ring->nop); +} + /** * amdgpu_ring_commit - tell the GPU to execute the new * commands on the ring buffer @@ -143,10 +158,13 @@ int amdgpu_ring_lock(struct amdgpu_ring *ring, unsigned ndw) */ void amdgpu_ring_commit(struct amdgpu_ring *ring) { + uint32_t count; + /* We pad to match fetch size */ - while (ring->wptr & ring->align_mask) { - amdgpu_ring_write(ring, ring->nop); - } + count = ring->align_mask + 1 - (ring->wptr & ring->align_mask); + count %= ring->align_mask + 1; + ring->funcs->insert_nop(ring, count); + mb(); amdgpu_ring_set_wptr(ring); } @@ -342,6 +360,8 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, amdgpu_fence_driver_init_ring(ring); } + init_waitqueue_head(&ring->fence_drv.fence_queue); + r = amdgpu_wb_get(adev, &ring->rptr_offs); if (r) { dev_err(adev->dev, "(%d) ring rptr_offs wb alloc failed\n", r); @@ -367,7 +387,7 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring, } ring->next_rptr_gpu_addr = adev->wb.gpu_addr + (ring->next_rptr_offs * 4); ring->next_rptr_cpu_addr = &adev->wb.wb[ring->next_rptr_offs]; - + spin_lock_init(&ring->fence_lock); r = amdgpu_fence_driver_start_ring(ring, irq_src, irq_type); if (r) { dev_err(adev->dev, "failed initializing fences (%d).\n", r); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c index eb20987ce18d..74dad270362c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c @@ -139,6 +139,20 @@ int amdgpu_sa_bo_manager_suspend(struct amdgpu_device *adev, return r; } +static uint32_t amdgpu_sa_get_ring_from_fence(struct fence *f) +{ + struct amdgpu_fence *a_fence; + struct amd_sched_fence *s_fence; + + s_fence = to_amd_sched_fence(f); + if (s_fence) + return s_fence->scheduler->ring_id; + a_fence = to_amdgpu_fence(f); + if (a_fence) + return a_fence->ring->idx; + return 0; +} + static void amdgpu_sa_bo_remove_locked(struct amdgpu_sa_bo *sa_bo) { struct amdgpu_sa_manager *sa_manager = sa_bo->manager; @@ -147,7 +161,7 @@ static void amdgpu_sa_bo_remove_locked(struct amdgpu_sa_bo *sa_bo) } list_del_init(&sa_bo->olist); list_del_init(&sa_bo->flist); - amdgpu_fence_unref(&sa_bo->fence); + fence_put(sa_bo->fence); kfree(sa_bo); } @@ -160,7 +174,8 @@ static void amdgpu_sa_bo_try_free(struct amdgpu_sa_manager *sa_manager) sa_bo = list_entry(sa_manager->hole->next, struct amdgpu_sa_bo, olist); list_for_each_entry_safe_from(sa_bo, tmp, &sa_manager->olist, olist) { - if (sa_bo->fence == NULL || !amdgpu_fence_signaled(sa_bo->fence)) { + if (sa_bo->fence == NULL || + !fence_is_signaled(sa_bo->fence)) { return; } amdgpu_sa_bo_remove_locked(sa_bo); @@ -245,7 +260,7 @@ static bool amdgpu_sa_event(struct amdgpu_sa_manager *sa_manager, } static bool amdgpu_sa_bo_next_hole(struct amdgpu_sa_manager *sa_manager, - struct amdgpu_fence **fences, + struct fence **fences, unsigned *tries) { struct amdgpu_sa_bo *best_bo = NULL; @@ -274,7 +289,7 @@ static bool amdgpu_sa_bo_next_hole(struct amdgpu_sa_manager *sa_manager, sa_bo = list_first_entry(&sa_manager->flist[i], struct amdgpu_sa_bo, flist); - if (!amdgpu_fence_signaled(sa_bo->fence)) { + if (!fence_is_signaled(sa_bo->fence)) { fences[i] = sa_bo->fence; continue; } @@ -298,7 +313,8 @@ static bool amdgpu_sa_bo_next_hole(struct amdgpu_sa_manager *sa_manager, } if (best_bo) { - ++tries[best_bo->fence->ring->idx]; + uint32_t idx = amdgpu_sa_get_ring_from_fence(best_bo->fence); + ++tries[idx]; sa_manager->hole = best_bo->olist.prev; /* we knew that this one is signaled, @@ -314,9 +330,10 @@ int amdgpu_sa_bo_new(struct amdgpu_device *adev, struct amdgpu_sa_bo **sa_bo, unsigned size, unsigned align) { - struct amdgpu_fence *fences[AMDGPU_MAX_RINGS]; + struct fence *fences[AMDGPU_MAX_RINGS]; unsigned tries[AMDGPU_MAX_RINGS]; int i, r; + signed long t; BUG_ON(align > sa_manager->align); BUG_ON(size > sa_manager->size); @@ -350,7 +367,9 @@ int amdgpu_sa_bo_new(struct amdgpu_device *adev, } while (amdgpu_sa_bo_next_hole(sa_manager, fences, tries)); spin_unlock(&sa_manager->wq.lock); - r = amdgpu_fence_wait_any(adev, fences, false); + t = amdgpu_fence_wait_any(adev, fences, AMDGPU_MAX_RINGS, + false, MAX_SCHEDULE_TIMEOUT); + r = (t > 0) ? 0 : t; spin_lock(&sa_manager->wq.lock); /* if we have nothing to wait for block */ if (r == -ENOENT) { @@ -369,7 +388,7 @@ int amdgpu_sa_bo_new(struct amdgpu_device *adev, } void amdgpu_sa_bo_free(struct amdgpu_device *adev, struct amdgpu_sa_bo **sa_bo, - struct amdgpu_fence *fence) + struct fence *fence) { struct amdgpu_sa_manager *sa_manager; @@ -379,10 +398,11 @@ void amdgpu_sa_bo_free(struct amdgpu_device *adev, struct amdgpu_sa_bo **sa_bo, sa_manager = (*sa_bo)->manager; spin_lock(&sa_manager->wq.lock); - if (fence && !amdgpu_fence_signaled(fence)) { - (*sa_bo)->fence = amdgpu_fence_ref(fence); - list_add_tail(&(*sa_bo)->flist, - &sa_manager->flist[fence->ring->idx]); + if (fence && !fence_is_signaled(fence)) { + uint32_t idx; + (*sa_bo)->fence = fence_get(fence); + idx = amdgpu_sa_get_ring_from_fence(fence); + list_add_tail(&(*sa_bo)->flist, &sa_manager->flist[idx]); } else { amdgpu_sa_bo_remove_locked(*sa_bo); } @@ -409,8 +429,16 @@ void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager, seq_printf(m, "[0x%010llx 0x%010llx] size %8lld", soffset, eoffset, eoffset - soffset); if (i->fence) { - seq_printf(m, " protected by 0x%016llx on ring %d", - i->fence->seq, i->fence->ring->idx); + struct amdgpu_fence *a_fence = to_amdgpu_fence(i->fence); + struct amd_sched_fence *s_fence = to_amd_sched_fence(i->fence); + if (a_fence) + seq_printf(m, " protected by 0x%016llx on ring %d", + a_fence->seq, a_fence->ring->idx); + if (s_fence) + seq_printf(m, " protected by 0x%016x on ring %d", + s_fence->base.seqno, + s_fence->scheduler->ring_id); + } seq_printf(m, "\n"); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c new file mode 100644 index 000000000000..de98fbd2971e --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sched.c @@ -0,0 +1,128 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#include +#include +#include +#include +#include "amdgpu.h" + +static struct fence *amdgpu_sched_dependency(struct amd_sched_job *job) +{ + struct amdgpu_job *sched_job = (struct amdgpu_job *)job; + return amdgpu_sync_get_fence(&sched_job->ibs->sync); +} + +static struct fence *amdgpu_sched_run_job(struct amd_sched_job *job) +{ + struct amdgpu_job *sched_job; + struct amdgpu_fence *fence; + int r; + + if (!job) { + DRM_ERROR("job is null\n"); + return NULL; + } + sched_job = (struct amdgpu_job *)job; + mutex_lock(&sched_job->job_lock); + r = amdgpu_ib_schedule(sched_job->adev, + sched_job->num_ibs, + sched_job->ibs, + sched_job->base.owner); + if (r) + goto err; + fence = amdgpu_fence_ref(sched_job->ibs[sched_job->num_ibs - 1].fence); + + if (sched_job->free_job) + sched_job->free_job(sched_job); + + mutex_unlock(&sched_job->job_lock); + return &fence->base; + +err: + DRM_ERROR("Run job error\n"); + mutex_unlock(&sched_job->job_lock); + job->sched->ops->process_job(job); + return NULL; +} + +static void amdgpu_sched_process_job(struct amd_sched_job *job) +{ + struct amdgpu_job *sched_job; + + if (!job) { + DRM_ERROR("job is null\n"); + return; + } + sched_job = (struct amdgpu_job *)job; + /* after processing job, free memory */ + fence_put(&sched_job->base.s_fence->base); + kfree(sched_job); +} + +struct amd_sched_backend_ops amdgpu_sched_ops = { + .dependency = amdgpu_sched_dependency, + .run_job = amdgpu_sched_run_job, + .process_job = amdgpu_sched_process_job +}; + +int amdgpu_sched_ib_submit_kernel_helper(struct amdgpu_device *adev, + struct amdgpu_ring *ring, + struct amdgpu_ib *ibs, + unsigned num_ibs, + int (*free_job)(struct amdgpu_job *), + void *owner, + struct fence **f) +{ + int r = 0; + if (amdgpu_enable_scheduler) { + struct amdgpu_job *job = + kzalloc(sizeof(struct amdgpu_job), GFP_KERNEL); + if (!job) + return -ENOMEM; + job->base.sched = ring->scheduler; + job->base.s_entity = &adev->kernel_ctx.rings[ring->idx].entity; + job->adev = adev; + job->ibs = ibs; + job->num_ibs = num_ibs; + job->base.owner = owner; + mutex_init(&job->job_lock); + job->free_job = free_job; + mutex_lock(&job->job_lock); + r = amd_sched_entity_push_job((struct amd_sched_job *)job); + if (r) { + mutex_unlock(&job->job_lock); + kfree(job); + return r; + } + *f = fence_get(&job->base.s_fence->base); + mutex_unlock(&job->job_lock); + } else { + r = amdgpu_ib_schedule(adev, num_ibs, ibs, owner); + if (r) + return r; + *f = fence_get(&ibs[num_ibs - 1].fence->base); + } + + return 0; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_semaphore.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_semaphore.c index d6d41a42ab65..ff3ca52ec6fe 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_semaphore.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_semaphore.c @@ -87,7 +87,7 @@ bool amdgpu_semaphore_emit_wait(struct amdgpu_ring *ring, void amdgpu_semaphore_free(struct amdgpu_device *adev, struct amdgpu_semaphore **semaphore, - struct amdgpu_fence *fence) + struct fence *fence) { if (semaphore == NULL || *semaphore == NULL) { return; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c index 21accbdd0a1a..068aeaff7183 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c @@ -32,6 +32,11 @@ #include "amdgpu.h" #include "amdgpu_trace.h" +struct amdgpu_sync_entry { + struct hlist_node node; + struct fence *fence; +}; + /** * amdgpu_sync_create - zero init sync object * @@ -49,36 +54,104 @@ void amdgpu_sync_create(struct amdgpu_sync *sync) for (i = 0; i < AMDGPU_MAX_RINGS; ++i) sync->sync_to[i] = NULL; + hash_init(sync->fences); sync->last_vm_update = NULL; } +static bool amdgpu_sync_same_dev(struct amdgpu_device *adev, struct fence *f) +{ + struct amdgpu_fence *a_fence = to_amdgpu_fence(f); + struct amd_sched_fence *s_fence = to_amd_sched_fence(f); + + if (a_fence) + return a_fence->ring->adev == adev; + if (s_fence) + return (struct amdgpu_device *)s_fence->scheduler->priv == adev; + return false; +} + +static bool amdgpu_sync_test_owner(struct fence *f, void *owner) +{ + struct amdgpu_fence *a_fence = to_amdgpu_fence(f); + struct amd_sched_fence *s_fence = to_amd_sched_fence(f); + if (s_fence) + return s_fence->owner == owner; + if (a_fence) + return a_fence->owner == owner; + return false; +} + /** - * amdgpu_sync_fence - use the semaphore to sync to a fence + * amdgpu_sync_fence - remember to sync to this fence * * @sync: sync object to add fence to * @fence: fence to sync to * - * Sync to the fence using the semaphore objects */ -void amdgpu_sync_fence(struct amdgpu_sync *sync, - struct amdgpu_fence *fence) +int amdgpu_sync_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync, + struct fence *f) { + struct amdgpu_sync_entry *e; + struct amdgpu_fence *fence; struct amdgpu_fence *other; + struct fence *tmp, *later; - if (!fence) - return; + if (!f) + return 0; + + if (amdgpu_sync_same_dev(adev, f) && + amdgpu_sync_test_owner(f, AMDGPU_FENCE_OWNER_VM)) { + if (sync->last_vm_update) { + tmp = sync->last_vm_update; + BUG_ON(f->context != tmp->context); + later = (f->seqno - tmp->seqno <= INT_MAX) ? f : tmp; + sync->last_vm_update = fence_get(later); + fence_put(tmp); + } else + sync->last_vm_update = fence_get(f); + } + + fence = to_amdgpu_fence(f); + if (!fence || fence->ring->adev != adev) { + hash_for_each_possible(sync->fences, e, node, f->context) { + struct fence *new; + if (unlikely(e->fence->context != f->context)) + continue; + new = fence_get(fence_later(e->fence, f)); + if (new) { + fence_put(e->fence); + e->fence = new; + } + return 0; + } + + e = kmalloc(sizeof(struct amdgpu_sync_entry), GFP_KERNEL); + if (!e) + return -ENOMEM; + + hash_add(sync->fences, &e->node, f->context); + e->fence = fence_get(f); + return 0; + } other = sync->sync_to[fence->ring->idx]; sync->sync_to[fence->ring->idx] = amdgpu_fence_ref( amdgpu_fence_later(fence, other)); amdgpu_fence_unref(&other); - if (fence->owner == AMDGPU_FENCE_OWNER_VM) { - other = sync->last_vm_update; - sync->last_vm_update = amdgpu_fence_ref( - amdgpu_fence_later(fence, other)); - amdgpu_fence_unref(&other); - } + return 0; +} + +static void *amdgpu_sync_get_owner(struct fence *f) +{ + struct amdgpu_fence *a_fence = to_amdgpu_fence(f); + struct amd_sched_fence *s_fence = to_amd_sched_fence(f); + + if (s_fence) + return s_fence->owner; + else if (a_fence) + return a_fence->owner; + return AMDGPU_FENCE_OWNER_UNDEFINED; } /** @@ -97,7 +170,7 @@ int amdgpu_sync_resv(struct amdgpu_device *adev, { struct reservation_object_list *flist; struct fence *f; - struct amdgpu_fence *fence; + void *fence_owner; unsigned i; int r = 0; @@ -106,11 +179,7 @@ int amdgpu_sync_resv(struct amdgpu_device *adev, /* always sync to the exclusive fence */ f = reservation_object_get_excl(resv); - fence = f ? to_amdgpu_fence(f) : NULL; - if (fence && fence->ring->adev == adev) - amdgpu_sync_fence(sync, fence); - else if (f) - r = fence_wait(f, true); + r = amdgpu_sync_fence(adev, sync, f); flist = reservation_object_get_list(resv); if (!flist || r) @@ -119,20 +188,72 @@ int amdgpu_sync_resv(struct amdgpu_device *adev, for (i = 0; i < flist->shared_count; ++i) { f = rcu_dereference_protected(flist->shared[i], reservation_object_held(resv)); - fence = f ? to_amdgpu_fence(f) : NULL; - if (fence && fence->ring->adev == adev) { - if (fence->owner != owner || - fence->owner == AMDGPU_FENCE_OWNER_UNDEFINED) - amdgpu_sync_fence(sync, fence); - } else if (f) { - r = fence_wait(f, true); - if (r) - break; + if (amdgpu_sync_same_dev(adev, f)) { + /* VM updates are only interesting + * for other VM updates and moves. + */ + fence_owner = amdgpu_sync_get_owner(f); + if ((owner != AMDGPU_FENCE_OWNER_MOVE) && + (fence_owner != AMDGPU_FENCE_OWNER_MOVE) && + ((owner == AMDGPU_FENCE_OWNER_VM) != + (fence_owner == AMDGPU_FENCE_OWNER_VM))) + continue; + + /* Ignore fence from the same owner as + * long as it isn't undefined. + */ + if (owner != AMDGPU_FENCE_OWNER_UNDEFINED && + fence_owner == owner) + continue; } + + r = amdgpu_sync_fence(adev, sync, f); + if (r) + break; } return r; } +struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync) +{ + struct amdgpu_sync_entry *e; + struct hlist_node *tmp; + struct fence *f; + int i; + + hash_for_each_safe(sync->fences, i, tmp, e, node) { + + f = e->fence; + + hash_del(&e->node); + kfree(e); + + if (!fence_is_signaled(f)) + return f; + + fence_put(f); + } + return NULL; +} + +int amdgpu_sync_wait(struct amdgpu_sync *sync) +{ + struct amdgpu_sync_entry *e; + struct hlist_node *tmp; + int i, r; + + hash_for_each_safe(sync->fences, i, tmp, e, node) { + r = fence_wait(e->fence, false); + if (r) + return r; + + hash_del(&e->node); + fence_put(e->fence); + kfree(e); + } + return 0; +} + /** * amdgpu_sync_rings - sync ring to all registered fences * @@ -164,9 +285,9 @@ int amdgpu_sync_rings(struct amdgpu_sync *sync, return -EINVAL; } - if (count >= AMDGPU_NUM_SYNCS) { + if (amdgpu_enable_scheduler || (count >= AMDGPU_NUM_SYNCS)) { /* not enough room, wait manually */ - r = amdgpu_fence_wait(fence, false); + r = fence_wait(&fence->base, false); if (r) return r; continue; @@ -186,7 +307,7 @@ int amdgpu_sync_rings(struct amdgpu_sync *sync, if (!amdgpu_semaphore_emit_signal(other, semaphore)) { /* signaling wasn't successful wait manually */ amdgpu_ring_undo(other); - r = amdgpu_fence_wait(fence, false); + r = fence_wait(&fence->base, false); if (r) return r; continue; @@ -196,7 +317,7 @@ int amdgpu_sync_rings(struct amdgpu_sync *sync, if (!amdgpu_semaphore_emit_wait(ring, semaphore)) { /* waiting wasn't successful wait manually */ amdgpu_ring_undo(other); - r = amdgpu_fence_wait(fence, false); + r = fence_wait(&fence->base, false); if (r) return r; continue; @@ -220,15 +341,23 @@ int amdgpu_sync_rings(struct amdgpu_sync *sync, */ void amdgpu_sync_free(struct amdgpu_device *adev, struct amdgpu_sync *sync, - struct amdgpu_fence *fence) + struct fence *fence) { + struct amdgpu_sync_entry *e; + struct hlist_node *tmp; unsigned i; + hash_for_each_safe(sync->fences, i, tmp, e, node) { + hash_del(&e->node); + fence_put(e->fence); + kfree(e); + } + for (i = 0; i < AMDGPU_NUM_SYNCS; ++i) amdgpu_semaphore_free(adev, &sync->semaphores[i], fence); for (i = 0; i < AMDGPU_MAX_RINGS; ++i) amdgpu_fence_unref(&sync->sync_to[i]); - amdgpu_fence_unref(&sync->last_vm_update); + fence_put(sync->last_vm_update); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c index df202999fbfe..f80b1a43be8a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_test.c @@ -77,7 +77,7 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev) void *gtt_map, *vram_map; void **gtt_start, **gtt_end; void **vram_start, **vram_end; - struct amdgpu_fence *fence = NULL; + struct fence *fence = NULL; r = amdgpu_bo_create(adev, size, PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_GTT, 0, NULL, gtt_obj + i); @@ -116,13 +116,13 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev) goto out_lclean_unpin; } - r = amdgpu_fence_wait(fence, false); + r = fence_wait(fence, false); if (r) { DRM_ERROR("Failed to wait for GTT->VRAM fence %d\n", i); goto out_lclean_unpin; } - amdgpu_fence_unref(&fence); + fence_put(fence); r = amdgpu_bo_kmap(vram_obj, &vram_map); if (r) { @@ -161,13 +161,13 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev) goto out_lclean_unpin; } - r = amdgpu_fence_wait(fence, false); + r = fence_wait(fence, false); if (r) { DRM_ERROR("Failed to wait for VRAM->GTT fence %d\n", i); goto out_lclean_unpin; } - amdgpu_fence_unref(&fence); + fence_put(fence); r = amdgpu_bo_kmap(gtt_obj[i], >t_map); if (r) { @@ -214,7 +214,7 @@ static void amdgpu_do_test_moves(struct amdgpu_device *adev) amdgpu_bo_unref(>t_obj[i]); } if (fence) - amdgpu_fence_unref(&fence); + fence_put(fence); break; } @@ -238,7 +238,7 @@ void amdgpu_test_moves(struct amdgpu_device *adev) static int amdgpu_test_create_and_emit_fence(struct amdgpu_device *adev, struct amdgpu_ring *ring, - struct amdgpu_fence **fence) + struct fence **fence) { uint32_t handle = ring->idx ^ 0xdeafbeef; int r; @@ -269,15 +269,16 @@ static int amdgpu_test_create_and_emit_fence(struct amdgpu_device *adev, DRM_ERROR("Failed to get dummy destroy msg\n"); return r; } - } else { + struct amdgpu_fence *a_fence = NULL; r = amdgpu_ring_lock(ring, 64); if (r) { DRM_ERROR("Failed to lock ring A %d\n", ring->idx); return r; } - amdgpu_fence_emit(ring, AMDGPU_FENCE_OWNER_UNDEFINED, fence); + amdgpu_fence_emit(ring, AMDGPU_FENCE_OWNER_UNDEFINED, &a_fence); amdgpu_ring_unlock_commit(ring); + *fence = &a_fence->base; } return 0; } @@ -286,7 +287,7 @@ void amdgpu_test_ring_sync(struct amdgpu_device *adev, struct amdgpu_ring *ringA, struct amdgpu_ring *ringB) { - struct amdgpu_fence *fence1 = NULL, *fence2 = NULL; + struct fence *fence1 = NULL, *fence2 = NULL; struct amdgpu_semaphore *semaphore = NULL; int r; @@ -322,7 +323,7 @@ void amdgpu_test_ring_sync(struct amdgpu_device *adev, mdelay(1000); - if (amdgpu_fence_signaled(fence1)) { + if (fence_is_signaled(fence1)) { DRM_ERROR("Fence 1 signaled without waiting for semaphore.\n"); goto out_cleanup; } @@ -335,7 +336,7 @@ void amdgpu_test_ring_sync(struct amdgpu_device *adev, amdgpu_semaphore_emit_signal(ringB, semaphore); amdgpu_ring_unlock_commit(ringB); - r = amdgpu_fence_wait(fence1, false); + r = fence_wait(fence1, false); if (r) { DRM_ERROR("Failed to wait for sync fence 1\n"); goto out_cleanup; @@ -343,7 +344,7 @@ void amdgpu_test_ring_sync(struct amdgpu_device *adev, mdelay(1000); - if (amdgpu_fence_signaled(fence2)) { + if (fence_is_signaled(fence2)) { DRM_ERROR("Fence 2 signaled without waiting for semaphore.\n"); goto out_cleanup; } @@ -356,7 +357,7 @@ void amdgpu_test_ring_sync(struct amdgpu_device *adev, amdgpu_semaphore_emit_signal(ringB, semaphore); amdgpu_ring_unlock_commit(ringB); - r = amdgpu_fence_wait(fence2, false); + r = fence_wait(fence2, false); if (r) { DRM_ERROR("Failed to wait for sync fence 1\n"); goto out_cleanup; @@ -366,10 +367,10 @@ void amdgpu_test_ring_sync(struct amdgpu_device *adev, amdgpu_semaphore_free(adev, &semaphore, NULL); if (fence1) - amdgpu_fence_unref(&fence1); + fence_put(fence1); if (fence2) - amdgpu_fence_unref(&fence2); + fence_put(fence2); if (r) printk(KERN_WARNING "Error while testing ring sync (%d).\n", r); @@ -380,7 +381,7 @@ static void amdgpu_test_ring_sync2(struct amdgpu_device *adev, struct amdgpu_ring *ringB, struct amdgpu_ring *ringC) { - struct amdgpu_fence *fenceA = NULL, *fenceB = NULL; + struct fence *fenceA = NULL, *fenceB = NULL; struct amdgpu_semaphore *semaphore = NULL; bool sigA, sigB; int i, r; @@ -416,11 +417,11 @@ static void amdgpu_test_ring_sync2(struct amdgpu_device *adev, mdelay(1000); - if (amdgpu_fence_signaled(fenceA)) { + if (fence_is_signaled(fenceA)) { DRM_ERROR("Fence A signaled without waiting for semaphore.\n"); goto out_cleanup; } - if (amdgpu_fence_signaled(fenceB)) { + if (fence_is_signaled(fenceB)) { DRM_ERROR("Fence B signaled without waiting for semaphore.\n"); goto out_cleanup; } @@ -435,8 +436,8 @@ static void amdgpu_test_ring_sync2(struct amdgpu_device *adev, for (i = 0; i < 30; ++i) { mdelay(100); - sigA = amdgpu_fence_signaled(fenceA); - sigB = amdgpu_fence_signaled(fenceB); + sigA = fence_is_signaled(fenceA); + sigB = fence_is_signaled(fenceB); if (sigA || sigB) break; } @@ -461,12 +462,12 @@ static void amdgpu_test_ring_sync2(struct amdgpu_device *adev, mdelay(1000); - r = amdgpu_fence_wait(fenceA, false); + r = fence_wait(fenceA, false); if (r) { DRM_ERROR("Failed to wait for sync fence A\n"); goto out_cleanup; } - r = amdgpu_fence_wait(fenceB, false); + r = fence_wait(fenceB, false); if (r) { DRM_ERROR("Failed to wait for sync fence B\n"); goto out_cleanup; @@ -476,10 +477,10 @@ static void amdgpu_test_ring_sync2(struct amdgpu_device *adev, amdgpu_semaphore_free(adev, &semaphore, NULL); if (fenceA) - amdgpu_fence_unref(&fenceA); + fence_put(fenceA); if (fenceB) - amdgpu_fence_unref(&fenceB); + fence_put(fenceB); if (r) printk(KERN_WARNING "Error while testing ring sync (%d).\n", r); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index dd3415d2e45d..b5abd5cde413 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -228,7 +228,7 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo, struct amdgpu_device *adev; struct amdgpu_ring *ring; uint64_t old_start, new_start; - struct amdgpu_fence *fence; + struct fence *fence; int r; adev = amdgpu_get_adev(bo->bdev); @@ -269,9 +269,9 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo, new_mem->num_pages * PAGE_SIZE, /* bytes */ bo->resv, &fence); /* FIXME: handle copy error */ - r = ttm_bo_move_accel_cleanup(bo, &fence->base, + r = ttm_bo_move_accel_cleanup(bo, fence, evict, no_wait_gpu, new_mem); - amdgpu_fence_unref(&fence); + fence_put(fence); return r; } @@ -859,7 +859,8 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) amdgpu_ttm_set_active_vram_size(adev, adev->mc.visible_vram_size); r = amdgpu_bo_create(adev, 256 * 1024, PAGE_SIZE, true, - AMDGPU_GEM_DOMAIN_VRAM, 0, + AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, NULL, &adev->stollen_vga_memory); if (r) { return r; @@ -987,46 +988,48 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t dst_offset, uint32_t byte_count, struct reservation_object *resv, - struct amdgpu_fence **fence) + struct fence **fence) { struct amdgpu_device *adev = ring->adev; - struct amdgpu_sync sync; uint32_t max_bytes; unsigned num_loops, num_dw; + struct amdgpu_ib *ib; unsigned i; int r; - /* sync other rings */ - amdgpu_sync_create(&sync); - if (resv) { - r = amdgpu_sync_resv(adev, &sync, resv, false); - if (r) { - DRM_ERROR("sync failed (%d).\n", r); - amdgpu_sync_free(adev, &sync, NULL); - return r; - } - } - max_bytes = adev->mman.buffer_funcs->copy_max_bytes; num_loops = DIV_ROUND_UP(byte_count, max_bytes); num_dw = num_loops * adev->mman.buffer_funcs->copy_num_dw; - /* for fence and sync */ - num_dw += 64 + AMDGPU_NUM_SYNCS * 8; + /* for IB padding */ + while (num_dw & 0x7) + num_dw++; - r = amdgpu_ring_lock(ring, num_dw); + ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); + if (!ib) + return -ENOMEM; + + r = amdgpu_ib_get(ring, NULL, num_dw * 4, ib); if (r) { - DRM_ERROR("ring lock failed (%d).\n", r); - amdgpu_sync_free(adev, &sync, NULL); + kfree(ib); return r; } - amdgpu_sync_rings(&sync, ring); + ib->length_dw = 0; + + if (resv) { + r = amdgpu_sync_resv(adev, &ib->sync, resv, + AMDGPU_FENCE_OWNER_UNDEFINED); + if (r) { + DRM_ERROR("sync failed (%d).\n", r); + goto error_free; + } + } for (i = 0; i < num_loops; i++) { uint32_t cur_size_in_bytes = min(byte_count, max_bytes); - amdgpu_emit_copy_buffer(adev, ring, src_offset, dst_offset, + amdgpu_emit_copy_buffer(adev, ib, src_offset, dst_offset, cur_size_in_bytes); src_offset += cur_size_in_bytes; @@ -1034,17 +1037,24 @@ int amdgpu_copy_buffer(struct amdgpu_ring *ring, byte_count -= cur_size_in_bytes; } - r = amdgpu_fence_emit(ring, AMDGPU_FENCE_OWNER_MOVE, fence); - if (r) { - amdgpu_ring_unlock_undo(ring); - amdgpu_sync_free(adev, &sync, NULL); - return r; + amdgpu_vm_pad_ib(adev, ib); + WARN_ON(ib->length_dw > num_dw); + r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, + &amdgpu_vm_free_job, + AMDGPU_FENCE_OWNER_MOVE, + fence); + if (r) + goto error_free; + + if (!amdgpu_enable_scheduler) { + amdgpu_ib_free(adev, ib); + kfree(ib); } - - amdgpu_ring_unlock_commit(ring); - amdgpu_sync_free(adev, &sync, *fence); - return 0; +error_free: + amdgpu_ib_free(adev, ib); + kfree(ib); + return r; } #if defined(CONFIG_DEBUG_FS) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index f5c22556ec2c..2cf6c6b06e3b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c @@ -52,6 +52,7 @@ #endif #define FIRMWARE_TONGA "amdgpu/tonga_uvd.bin" #define FIRMWARE_CARRIZO "amdgpu/carrizo_uvd.bin" +#define FIRMWARE_FIJI "amdgpu/fiji_uvd.bin" /** * amdgpu_uvd_cs_ctx - Command submission parser context @@ -81,6 +82,7 @@ MODULE_FIRMWARE(FIRMWARE_MULLINS); #endif MODULE_FIRMWARE(FIRMWARE_TONGA); MODULE_FIRMWARE(FIRMWARE_CARRIZO); +MODULE_FIRMWARE(FIRMWARE_FIJI); static void amdgpu_uvd_note_usage(struct amdgpu_device *adev); static void amdgpu_uvd_idle_work_handler(struct work_struct *work); @@ -116,6 +118,9 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev) case CHIP_TONGA: fw_name = FIRMWARE_TONGA; break; + case CHIP_FIJI: + fw_name = FIRMWARE_FIJI; + break; case CHIP_CARRIZO: fw_name = FIRMWARE_CARRIZO; break; @@ -149,7 +154,9 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev) bo_size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8) + AMDGPU_UVD_STACK_SIZE + AMDGPU_UVD_HEAP_SIZE; r = amdgpu_bo_create(adev, bo_size, PAGE_SIZE, true, - AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, &adev->uvd.vcpu_bo); + AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, + NULL, &adev->uvd.vcpu_bo); if (r) { dev_err(adev->dev, "(%d) failed to allocate UVD bo\n", r); return r; @@ -216,31 +223,32 @@ int amdgpu_uvd_sw_fini(struct amdgpu_device *adev) int amdgpu_uvd_suspend(struct amdgpu_device *adev) { - unsigned size; - void *ptr; - const struct common_firmware_header *hdr; - int i; + struct amdgpu_ring *ring = &adev->uvd.ring; + int i, r; if (adev->uvd.vcpu_bo == NULL) return 0; - for (i = 0; i < AMDGPU_MAX_UVD_HANDLES; ++i) - if (atomic_read(&adev->uvd.handles[i])) - break; + for (i = 0; i < AMDGPU_MAX_UVD_HANDLES; ++i) { + uint32_t handle = atomic_read(&adev->uvd.handles[i]); + if (handle != 0) { + struct fence *fence; - if (i == AMDGPU_MAX_UVD_HANDLES) - return 0; + amdgpu_uvd_note_usage(adev); - hdr = (const struct common_firmware_header *)adev->uvd.fw->data; + r = amdgpu_uvd_get_destroy_msg(ring, handle, &fence); + if (r) { + DRM_ERROR("Error destroying UVD (%d)!\n", r); + continue; + } - size = amdgpu_bo_size(adev->uvd.vcpu_bo); - size -= le32_to_cpu(hdr->ucode_size_bytes); + fence_wait(fence, false); + fence_put(fence); - ptr = adev->uvd.cpu_addr; - ptr += le32_to_cpu(hdr->ucode_size_bytes); - - adev->uvd.saved_bo = kmalloc(size, GFP_KERNEL); - memcpy(adev->uvd.saved_bo, ptr, size); + adev->uvd.filp[i] = NULL; + atomic_set(&adev->uvd.handles[i], 0); + } + } return 0; } @@ -265,12 +273,7 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev) ptr = adev->uvd.cpu_addr; ptr += le32_to_cpu(hdr->ucode_size_bytes); - if (adev->uvd.saved_bo != NULL) { - memcpy(ptr, adev->uvd.saved_bo, size); - kfree(adev->uvd.saved_bo); - adev->uvd.saved_bo = NULL; - } else - memset(ptr, 0, size); + memset(ptr, 0, size); return 0; } @@ -283,7 +286,7 @@ void amdgpu_uvd_free_handles(struct amdgpu_device *adev, struct drm_file *filp) for (i = 0; i < AMDGPU_MAX_UVD_HANDLES; ++i) { uint32_t handle = atomic_read(&adev->uvd.handles[i]); if (handle != 0 && adev->uvd.filp[i] == filp) { - struct amdgpu_fence *fence; + struct fence *fence; amdgpu_uvd_note_usage(adev); @@ -293,8 +296,8 @@ void amdgpu_uvd_free_handles(struct amdgpu_device *adev, struct drm_file *filp) continue; } - amdgpu_fence_wait(fence, false); - amdgpu_fence_unref(&fence); + fence_wait(fence, false); + fence_put(fence); adev->uvd.filp[i] = NULL; atomic_set(&adev->uvd.handles[i], 0); @@ -374,7 +377,8 @@ static int amdgpu_uvd_cs_msg_decode(uint32_t *msg, unsigned buf_sizes[]) unsigned height_in_mb = ALIGN(height / 16, 2); unsigned fs_in_mb = width_in_mb * height_in_mb; - unsigned image_size, tmp, min_dpb_size, num_dpb_buffer, min_ctx_size; + unsigned image_size, tmp, min_dpb_size, num_dpb_buffer; + unsigned min_ctx_size = 0; image_size = width * height; image_size += image_size / 2; @@ -507,28 +511,25 @@ static int amdgpu_uvd_cs_msg(struct amdgpu_uvd_cs_ctx *ctx, { struct amdgpu_device *adev = ctx->parser->adev; int32_t *msg, msg_type, handle; - struct fence *f; void *ptr; - - int i, r; + long r; + int i; if (offset & 0x3F) { DRM_ERROR("UVD messages must be 64 byte aligned!\n"); return -EINVAL; } - f = reservation_object_get_excl(bo->tbo.resv); - if (f) { - r = amdgpu_fence_wait((struct amdgpu_fence *)f, false); - if (r) { - DRM_ERROR("Failed waiting for UVD message (%d)!\n", r); - return r; - } + r = reservation_object_wait_timeout_rcu(bo->tbo.resv, true, false, + MAX_SCHEDULE_TIMEOUT); + if (r < 0) { + DRM_ERROR("Failed waiting for UVD message (%ld)!\n", r); + return r; } r = amdgpu_bo_kmap(bo, &ptr); if (r) { - DRM_ERROR("Failed mapping the UVD message (%d)!\n", r); + DRM_ERROR("Failed mapping the UVD message (%ld)!\n", r); return r; } @@ -803,14 +804,24 @@ int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx) return 0; } +static int amdgpu_uvd_free_job( + struct amdgpu_job *sched_job) +{ + amdgpu_ib_free(sched_job->adev, sched_job->ibs); + kfree(sched_job->ibs); + return 0; +} + static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo, - struct amdgpu_fence **fence) + struct fence **fence) { struct ttm_validate_buffer tv; struct ww_acquire_ctx ticket; struct list_head head; - struct amdgpu_ib ib; + struct amdgpu_ib *ib = NULL; + struct fence *f = NULL; + struct amdgpu_device *adev = ring->adev; uint64_t addr; int i, r; @@ -832,34 +843,49 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false); if (r) goto err; - - r = amdgpu_ib_get(ring, NULL, 64, &ib); - if (r) + ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); + if (!ib) { + r = -ENOMEM; goto err; + } + r = amdgpu_ib_get(ring, NULL, 64, ib); + if (r) + goto err1; addr = amdgpu_bo_gpu_offset(bo); - ib.ptr[0] = PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0); - ib.ptr[1] = addr; - ib.ptr[2] = PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0); - ib.ptr[3] = addr >> 32; - ib.ptr[4] = PACKET0(mmUVD_GPCOM_VCPU_CMD, 0); - ib.ptr[5] = 0; + ib->ptr[0] = PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0); + ib->ptr[1] = addr; + ib->ptr[2] = PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0); + ib->ptr[3] = addr >> 32; + ib->ptr[4] = PACKET0(mmUVD_GPCOM_VCPU_CMD, 0); + ib->ptr[5] = 0; for (i = 6; i < 16; ++i) - ib.ptr[i] = PACKET2(0); - ib.length_dw = 16; + ib->ptr[i] = PACKET2(0); + ib->length_dw = 16; - r = amdgpu_ib_schedule(ring->adev, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED); + r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, + &amdgpu_uvd_free_job, + AMDGPU_FENCE_OWNER_UNDEFINED, + &f); if (r) - goto err; - ttm_eu_fence_buffer_objects(&ticket, &head, &ib.fence->base); + goto err2; + + ttm_eu_fence_buffer_objects(&ticket, &head, f); if (fence) - *fence = amdgpu_fence_ref(ib.fence); - - amdgpu_ib_free(ring->adev, &ib); + *fence = fence_get(f); amdgpu_bo_unref(&bo); - return 0; + fence_put(f); + if (amdgpu_enable_scheduler) + return 0; + amdgpu_ib_free(ring->adev, ib); + kfree(ib); + return 0; +err2: + amdgpu_ib_free(ring->adev, ib); +err1: + kfree(ib); err: ttm_eu_backoff_reservation(&ticket, &head); return r; @@ -869,7 +895,7 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, crash the vcpu so just try to emmit a dummy create/destroy msg to avoid this */ int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, - struct amdgpu_fence **fence) + struct fence **fence) { struct amdgpu_device *adev = ring->adev; struct amdgpu_bo *bo; @@ -877,7 +903,9 @@ int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, int r, i; r = amdgpu_bo_create(adev, 1024, PAGE_SIZE, true, - AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, &bo); + AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, + NULL, &bo); if (r) return r; @@ -916,7 +944,7 @@ int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, } int amdgpu_uvd_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, - struct amdgpu_fence **fence) + struct fence **fence) { struct amdgpu_device *adev = ring->adev; struct amdgpu_bo *bo; @@ -924,7 +952,9 @@ int amdgpu_uvd_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, int r, i; r = amdgpu_bo_create(adev, 1024, PAGE_SIZE, true, - AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, &bo); + AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, + NULL, &bo); if (r) return r; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h index 2255aa710e33..1724c2c86151 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h @@ -29,9 +29,9 @@ int amdgpu_uvd_sw_fini(struct amdgpu_device *adev); int amdgpu_uvd_suspend(struct amdgpu_device *adev); int amdgpu_uvd_resume(struct amdgpu_device *adev); int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, - struct amdgpu_fence **fence); + struct fence **fence); int amdgpu_uvd_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, - struct amdgpu_fence **fence); + struct fence **fence); void amdgpu_uvd_free_handles(struct amdgpu_device *adev, struct drm_file *filp); int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c index d3ca73090e39..3cab96c42aa8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c @@ -48,6 +48,7 @@ #endif #define FIRMWARE_TONGA "amdgpu/tonga_vce.bin" #define FIRMWARE_CARRIZO "amdgpu/carrizo_vce.bin" +#define FIRMWARE_FIJI "amdgpu/fiji_vce.bin" #ifdef CONFIG_DRM_AMDGPU_CIK MODULE_FIRMWARE(FIRMWARE_BONAIRE); @@ -58,6 +59,7 @@ MODULE_FIRMWARE(FIRMWARE_MULLINS); #endif MODULE_FIRMWARE(FIRMWARE_TONGA); MODULE_FIRMWARE(FIRMWARE_CARRIZO); +MODULE_FIRMWARE(FIRMWARE_FIJI); static void amdgpu_vce_idle_work_handler(struct work_struct *work); @@ -101,6 +103,9 @@ int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size) case CHIP_CARRIZO: fw_name = FIRMWARE_CARRIZO; break; + case CHIP_FIJI: + fw_name = FIRMWARE_FIJI; + break; default: return -EINVAL; @@ -136,7 +141,9 @@ int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size) /* allocate firmware, stack and heap BO */ r = amdgpu_bo_create(adev, size, PAGE_SIZE, true, - AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, &adev->vce.vcpu_bo); + AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, + NULL, &adev->vce.vcpu_bo); if (r) { dev_err(adev->dev, "(%d) failed to allocate VCE bo\n", r); return r; @@ -334,6 +341,14 @@ void amdgpu_vce_free_handles(struct amdgpu_device *adev, struct drm_file *filp) } } +static int amdgpu_vce_free_job( + struct amdgpu_job *sched_job) +{ + amdgpu_ib_free(sched_job->adev, sched_job->ibs); + kfree(sched_job->ibs); + return 0; +} + /** * amdgpu_vce_get_create_msg - generate a VCE create msg * @@ -345,59 +360,69 @@ void amdgpu_vce_free_handles(struct amdgpu_device *adev, struct drm_file *filp) * Open up a stream for HW test */ int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, - struct amdgpu_fence **fence) + struct fence **fence) { const unsigned ib_size_dw = 1024; - struct amdgpu_ib ib; + struct amdgpu_ib *ib = NULL; + struct fence *f = NULL; + struct amdgpu_device *adev = ring->adev; uint64_t dummy; int i, r; - r = amdgpu_ib_get(ring, NULL, ib_size_dw * 4, &ib); + ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); + if (!ib) + return -ENOMEM; + r = amdgpu_ib_get(ring, NULL, ib_size_dw * 4, ib); if (r) { DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); + kfree(ib); return r; } - dummy = ib.gpu_addr + 1024; + dummy = ib->gpu_addr + 1024; /* stitch together an VCE create msg */ - ib.length_dw = 0; - ib.ptr[ib.length_dw++] = 0x0000000c; /* len */ - ib.ptr[ib.length_dw++] = 0x00000001; /* session cmd */ - ib.ptr[ib.length_dw++] = handle; + ib->length_dw = 0; + ib->ptr[ib->length_dw++] = 0x0000000c; /* len */ + ib->ptr[ib->length_dw++] = 0x00000001; /* session cmd */ + ib->ptr[ib->length_dw++] = handle; - ib.ptr[ib.length_dw++] = 0x00000030; /* len */ - ib.ptr[ib.length_dw++] = 0x01000001; /* create cmd */ - ib.ptr[ib.length_dw++] = 0x00000000; - ib.ptr[ib.length_dw++] = 0x00000042; - ib.ptr[ib.length_dw++] = 0x0000000a; - ib.ptr[ib.length_dw++] = 0x00000001; - ib.ptr[ib.length_dw++] = 0x00000080; - ib.ptr[ib.length_dw++] = 0x00000060; - ib.ptr[ib.length_dw++] = 0x00000100; - ib.ptr[ib.length_dw++] = 0x00000100; - ib.ptr[ib.length_dw++] = 0x0000000c; - ib.ptr[ib.length_dw++] = 0x00000000; + ib->ptr[ib->length_dw++] = 0x00000030; /* len */ + ib->ptr[ib->length_dw++] = 0x01000001; /* create cmd */ + ib->ptr[ib->length_dw++] = 0x00000000; + ib->ptr[ib->length_dw++] = 0x00000042; + ib->ptr[ib->length_dw++] = 0x0000000a; + ib->ptr[ib->length_dw++] = 0x00000001; + ib->ptr[ib->length_dw++] = 0x00000080; + ib->ptr[ib->length_dw++] = 0x00000060; + ib->ptr[ib->length_dw++] = 0x00000100; + ib->ptr[ib->length_dw++] = 0x00000100; + ib->ptr[ib->length_dw++] = 0x0000000c; + ib->ptr[ib->length_dw++] = 0x00000000; - ib.ptr[ib.length_dw++] = 0x00000014; /* len */ - ib.ptr[ib.length_dw++] = 0x05000005; /* feedback buffer */ - ib.ptr[ib.length_dw++] = upper_32_bits(dummy); - ib.ptr[ib.length_dw++] = dummy; - ib.ptr[ib.length_dw++] = 0x00000001; + ib->ptr[ib->length_dw++] = 0x00000014; /* len */ + ib->ptr[ib->length_dw++] = 0x05000005; /* feedback buffer */ + ib->ptr[ib->length_dw++] = upper_32_bits(dummy); + ib->ptr[ib->length_dw++] = dummy; + ib->ptr[ib->length_dw++] = 0x00000001; - for (i = ib.length_dw; i < ib_size_dw; ++i) - ib.ptr[i] = 0x0; - - r = amdgpu_ib_schedule(ring->adev, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED); - if (r) { - DRM_ERROR("amdgpu: failed to schedule ib (%d).\n", r); - } + for (i = ib->length_dw; i < ib_size_dw; ++i) + ib->ptr[i] = 0x0; + r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, + &amdgpu_vce_free_job, + AMDGPU_FENCE_OWNER_UNDEFINED, + &f); + if (r) + goto err; if (fence) - *fence = amdgpu_fence_ref(ib.fence); - - amdgpu_ib_free(ring->adev, &ib); - + *fence = fence_get(f); + fence_put(f); + if (amdgpu_enable_scheduler) + return 0; +err: + amdgpu_ib_free(adev, ib); + kfree(ib); return r; } @@ -412,49 +437,59 @@ int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, * Close up a stream for HW test or if userspace failed to do so */ int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, - struct amdgpu_fence **fence) + struct fence **fence) { const unsigned ib_size_dw = 1024; - struct amdgpu_ib ib; + struct amdgpu_ib *ib = NULL; + struct fence *f = NULL; + struct amdgpu_device *adev = ring->adev; uint64_t dummy; int i, r; - r = amdgpu_ib_get(ring, NULL, ib_size_dw * 4, &ib); + ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); + if (!ib) + return -ENOMEM; + + r = amdgpu_ib_get(ring, NULL, ib_size_dw * 4, ib); if (r) { + kfree(ib); DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); return r; } - dummy = ib.gpu_addr + 1024; + dummy = ib->gpu_addr + 1024; /* stitch together an VCE destroy msg */ - ib.length_dw = 0; - ib.ptr[ib.length_dw++] = 0x0000000c; /* len */ - ib.ptr[ib.length_dw++] = 0x00000001; /* session cmd */ - ib.ptr[ib.length_dw++] = handle; + ib->length_dw = 0; + ib->ptr[ib->length_dw++] = 0x0000000c; /* len */ + ib->ptr[ib->length_dw++] = 0x00000001; /* session cmd */ + ib->ptr[ib->length_dw++] = handle; - ib.ptr[ib.length_dw++] = 0x00000014; /* len */ - ib.ptr[ib.length_dw++] = 0x05000005; /* feedback buffer */ - ib.ptr[ib.length_dw++] = upper_32_bits(dummy); - ib.ptr[ib.length_dw++] = dummy; - ib.ptr[ib.length_dw++] = 0x00000001; + ib->ptr[ib->length_dw++] = 0x00000014; /* len */ + ib->ptr[ib->length_dw++] = 0x05000005; /* feedback buffer */ + ib->ptr[ib->length_dw++] = upper_32_bits(dummy); + ib->ptr[ib->length_dw++] = dummy; + ib->ptr[ib->length_dw++] = 0x00000001; - ib.ptr[ib.length_dw++] = 0x00000008; /* len */ - ib.ptr[ib.length_dw++] = 0x02000001; /* destroy cmd */ - - for (i = ib.length_dw; i < ib_size_dw; ++i) - ib.ptr[i] = 0x0; - - r = amdgpu_ib_schedule(ring->adev, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED); - if (r) { - DRM_ERROR("amdgpu: failed to schedule ib (%d).\n", r); - } + ib->ptr[ib->length_dw++] = 0x00000008; /* len */ + ib->ptr[ib->length_dw++] = 0x02000001; /* destroy cmd */ + for (i = ib->length_dw; i < ib_size_dw; ++i) + ib->ptr[i] = 0x0; + r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, + &amdgpu_vce_free_job, + AMDGPU_FENCE_OWNER_UNDEFINED, + &f); + if (r) + goto err; if (fence) - *fence = amdgpu_fence_ref(ib.fence); - - amdgpu_ib_free(ring->adev, &ib); - + *fence = fence_get(f); + fence_put(f); + if (amdgpu_enable_scheduler) + return 0; +err: + amdgpu_ib_free(adev, ib); + kfree(ib); return r; } @@ -800,9 +835,13 @@ int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring) */ int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring) { - struct amdgpu_fence *fence = NULL; + struct fence *fence = NULL; int r; + /* skip vce ring1 ib test for now, since it's not reliable */ + if (ring == &ring->adev->vce.ring[1]) + return 0; + r = amdgpu_vce_get_create_msg(ring, 1, NULL); if (r) { DRM_ERROR("amdgpu: failed to get create msg (%d).\n", r); @@ -815,13 +854,13 @@ int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring) goto error; } - r = amdgpu_fence_wait(fence, false); + r = fence_wait(fence, false); if (r) { DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); } else { DRM_INFO("ib test on ring %d succeeded\n", ring->idx); } error: - amdgpu_fence_unref(&fence); + fence_put(fence); return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h index 7ccdb5927da5..ba2da8ee5906 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h @@ -29,9 +29,9 @@ int amdgpu_vce_sw_fini(struct amdgpu_device *adev); int amdgpu_vce_suspend(struct amdgpu_device *adev); int amdgpu_vce_resume(struct amdgpu_device *adev); int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, - struct amdgpu_fence **fence); + struct fence **fence); int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle, - struct amdgpu_fence **fence); + struct fence **fence); void amdgpu_vce_free_handles(struct amdgpu_device *adev, struct drm_file *filp); int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx); bool amdgpu_vce_ring_emit_semaphore(struct amdgpu_ring *ring, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 9a4e3b63f1cb..f68b7cdc370a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -127,16 +127,16 @@ struct amdgpu_bo_list_entry *amdgpu_vm_get_bos(struct amdgpu_device *adev, /** * amdgpu_vm_grab_id - allocate the next free VMID * - * @ring: ring we want to submit job to * @vm: vm to allocate id for + * @ring: ring we want to submit job to + * @sync: sync object where we add dependencies * - * Allocate an id for the vm (cayman+). - * Returns the fence we need to sync to (if any). + * Allocate an id for the vm, adding fences to the sync obj as necessary. * - * Global and local mutex must be locked! + * Global mutex must be locked! */ -struct amdgpu_fence *amdgpu_vm_grab_id(struct amdgpu_ring *ring, - struct amdgpu_vm *vm) +int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring, + struct amdgpu_sync *sync) { struct amdgpu_fence *best[AMDGPU_MAX_RINGS] = {}; struct amdgpu_vm_id *vm_id = &vm->ids[ring->idx]; @@ -148,7 +148,7 @@ struct amdgpu_fence *amdgpu_vm_grab_id(struct amdgpu_ring *ring, /* check if the id is still valid */ if (vm_id->id && vm_id->last_id_use && vm_id->last_id_use == adev->vm_manager.active[vm_id->id]) - return NULL; + return 0; /* we definately need to flush */ vm_id->pd_gpu_addr = ~0ll; @@ -161,7 +161,7 @@ struct amdgpu_fence *amdgpu_vm_grab_id(struct amdgpu_ring *ring, /* found a free one */ vm_id->id = i; trace_amdgpu_vm_grab_id(i, ring->idx); - return NULL; + return 0; } if (amdgpu_fence_is_earlier(fence, best[fence->ring->idx])) { @@ -172,15 +172,19 @@ struct amdgpu_fence *amdgpu_vm_grab_id(struct amdgpu_ring *ring, for (i = 0; i < 2; ++i) { if (choices[i]) { + struct amdgpu_fence *fence; + + fence = adev->vm_manager.active[choices[i]]; vm_id->id = choices[i]; + trace_amdgpu_vm_grab_id(choices[i], ring->idx); - return adev->vm_manager.active[choices[i]]; + return amdgpu_sync_fence(ring->adev, sync, &fence->base); } } /* should never happen */ BUG(); - return NULL; + return -EINVAL; } /** @@ -196,17 +200,29 @@ struct amdgpu_fence *amdgpu_vm_grab_id(struct amdgpu_ring *ring, */ void amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_vm *vm, - struct amdgpu_fence *updates) + struct fence *updates) { uint64_t pd_addr = amdgpu_bo_gpu_offset(vm->page_directory); struct amdgpu_vm_id *vm_id = &vm->ids[ring->idx]; + struct fence *flushed_updates = vm_id->flushed_updates; + bool is_earlier = false; - if (pd_addr != vm_id->pd_gpu_addr || !vm_id->flushed_updates || - amdgpu_fence_is_earlier(vm_id->flushed_updates, updates)) { + if (flushed_updates && updates) { + BUG_ON(flushed_updates->context != updates->context); + is_earlier = (updates->seqno - flushed_updates->seqno <= + INT_MAX) ? true : false; + } + + if (pd_addr != vm_id->pd_gpu_addr || !flushed_updates || + is_earlier) { trace_amdgpu_vm_flush(pd_addr, ring->idx, vm_id->id); - amdgpu_fence_unref(&vm_id->flushed_updates); - vm_id->flushed_updates = amdgpu_fence_ref(updates); + if (is_earlier) { + vm_id->flushed_updates = fence_get(updates); + fence_put(flushed_updates); + } + if (!flushed_updates) + vm_id->flushed_updates = fence_get(updates); vm_id->pd_gpu_addr = pd_addr; amdgpu_ring_emit_vm_flush(ring, vm_id->id, vm_id->pd_gpu_addr); } @@ -300,6 +316,15 @@ static void amdgpu_vm_update_pages(struct amdgpu_device *adev, } } +int amdgpu_vm_free_job(struct amdgpu_job *sched_job) +{ + int i; + for (i = 0; i < sched_job->num_ibs; i++) + amdgpu_ib_free(sched_job->adev, &sched_job->ibs[i]); + kfree(sched_job->ibs); + return 0; +} + /** * amdgpu_vm_clear_bo - initially clear the page dir/table * @@ -310,7 +335,8 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, struct amdgpu_bo *bo) { struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring; - struct amdgpu_ib ib; + struct fence *fence = NULL; + struct amdgpu_ib *ib; unsigned entries; uint64_t addr; int r; @@ -330,24 +356,33 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, addr = amdgpu_bo_gpu_offset(bo); entries = amdgpu_bo_size(bo) / 8; - r = amdgpu_ib_get(ring, NULL, entries * 2 + 64, &ib); - if (r) + ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); + if (!ib) goto error_unreserve; - ib.length_dw = 0; - - amdgpu_vm_update_pages(adev, &ib, addr, 0, entries, 0, 0, 0); - amdgpu_vm_pad_ib(adev, &ib); - WARN_ON(ib.length_dw > 64); - - r = amdgpu_ib_schedule(adev, 1, &ib, AMDGPU_FENCE_OWNER_VM); + r = amdgpu_ib_get(ring, NULL, entries * 2 + 64, ib); if (r) goto error_free; - amdgpu_bo_fence(bo, ib.fence, true); + ib->length_dw = 0; + amdgpu_vm_update_pages(adev, ib, addr, 0, entries, 0, 0, 0); + amdgpu_vm_pad_ib(adev, ib); + WARN_ON(ib->length_dw > 64); + r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, + &amdgpu_vm_free_job, + AMDGPU_FENCE_OWNER_VM, + &fence); + if (!r) + amdgpu_bo_fence(bo, fence, true); + fence_put(fence); + if (amdgpu_enable_scheduler) { + amdgpu_bo_unreserve(bo); + return 0; + } error_free: - amdgpu_ib_free(adev, &ib); + amdgpu_ib_free(adev, ib); + kfree(ib); error_unreserve: amdgpu_bo_unreserve(bo); @@ -400,7 +435,9 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, uint32_t incr = AMDGPU_VM_PTE_COUNT * 8; uint64_t last_pde = ~0, last_pt = ~0; unsigned count = 0, pt_idx, ndw; - struct amdgpu_ib ib; + struct amdgpu_ib *ib; + struct fence *fence = NULL; + int r; /* padding, etc. */ @@ -413,10 +450,14 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, if (ndw > 0xfffff) return -ENOMEM; - r = amdgpu_ib_get(ring, NULL, ndw * 4, &ib); + ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); + if (!ib) + return -ENOMEM; + + r = amdgpu_ib_get(ring, NULL, ndw * 4, ib); if (r) return r; - ib.length_dw = 0; + ib->length_dw = 0; /* walk over the address space and update the page directory */ for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) { @@ -436,7 +477,7 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, ((last_pt + incr * count) != pt)) { if (count) { - amdgpu_vm_update_pages(adev, &ib, last_pde, + amdgpu_vm_update_pages(adev, ib, last_pde, last_pt, count, incr, AMDGPU_PTE_VALID, 0); } @@ -450,23 +491,37 @@ int amdgpu_vm_update_page_directory(struct amdgpu_device *adev, } if (count) - amdgpu_vm_update_pages(adev, &ib, last_pde, last_pt, count, + amdgpu_vm_update_pages(adev, ib, last_pde, last_pt, count, incr, AMDGPU_PTE_VALID, 0); - if (ib.length_dw != 0) { - amdgpu_vm_pad_ib(adev, &ib); - amdgpu_sync_resv(adev, &ib.sync, pd->tbo.resv, AMDGPU_FENCE_OWNER_VM); - WARN_ON(ib.length_dw > ndw); - r = amdgpu_ib_schedule(adev, 1, &ib, AMDGPU_FENCE_OWNER_VM); - if (r) { - amdgpu_ib_free(adev, &ib); - return r; - } - amdgpu_bo_fence(pd, ib.fence, true); + if (ib->length_dw != 0) { + amdgpu_vm_pad_ib(adev, ib); + amdgpu_sync_resv(adev, &ib->sync, pd->tbo.resv, AMDGPU_FENCE_OWNER_VM); + WARN_ON(ib->length_dw > ndw); + r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, + &amdgpu_vm_free_job, + AMDGPU_FENCE_OWNER_VM, + &fence); + if (r) + goto error_free; + + amdgpu_bo_fence(pd, fence, true); + fence_put(vm->page_directory_fence); + vm->page_directory_fence = fence_get(fence); + fence_put(fence); + } + + if (!amdgpu_enable_scheduler || ib->length_dw == 0) { + amdgpu_ib_free(adev, ib); + kfree(ib); } - amdgpu_ib_free(adev, &ib); return 0; + +error_free: + amdgpu_ib_free(adev, ib); + kfree(ib); + return r; } /** @@ -572,9 +627,14 @@ static int amdgpu_vm_update_ptes(struct amdgpu_device *adev, { uint64_t mask = AMDGPU_VM_PTE_COUNT - 1; uint64_t last_pte = ~0, last_dst = ~0; + void *owner = AMDGPU_FENCE_OWNER_VM; unsigned count = 0; uint64_t addr; + /* sync to everything on unmapping */ + if (!(flags & AMDGPU_PTE_VALID)) + owner = AMDGPU_FENCE_OWNER_UNDEFINED; + /* walk over the address space and update the page tables */ for (addr = start; addr < end; ) { uint64_t pt_idx = addr >> amdgpu_vm_block_size; @@ -583,8 +643,7 @@ static int amdgpu_vm_update_ptes(struct amdgpu_device *adev, uint64_t pte; int r; - amdgpu_sync_resv(adev, &ib->sync, pt->tbo.resv, - AMDGPU_FENCE_OWNER_VM); + amdgpu_sync_resv(adev, &ib->sync, pt->tbo.resv, owner); r = reservation_object_reserve_shared(pt->tbo.resv); if (r) return r; @@ -640,7 +699,7 @@ static int amdgpu_vm_update_ptes(struct amdgpu_device *adev, */ static void amdgpu_vm_fence_pts(struct amdgpu_vm *vm, uint64_t start, uint64_t end, - struct amdgpu_fence *fence) + struct fence *fence) { unsigned i; @@ -670,12 +729,13 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_bo_va_mapping *mapping, uint64_t addr, uint32_t gtt_flags, - struct amdgpu_fence **fence) + struct fence **fence) { struct amdgpu_ring *ring = adev->vm_manager.vm_pte_funcs_ring; unsigned nptes, ncmds, ndw; uint32_t flags = gtt_flags; - struct amdgpu_ib ib; + struct amdgpu_ib *ib; + struct fence *f = NULL; int r; /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here @@ -722,46 +782,54 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, if (ndw > 0xfffff) return -ENOMEM; - r = amdgpu_ib_get(ring, NULL, ndw * 4, &ib); - if (r) + ib = kzalloc(sizeof(struct amdgpu_ib), GFP_KERNEL); + if (!ib) + return -ENOMEM; + + r = amdgpu_ib_get(ring, NULL, ndw * 4, ib); + if (r) { + kfree(ib); return r; - ib.length_dw = 0; - - if (!(flags & AMDGPU_PTE_VALID)) { - unsigned i; - - for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { - struct amdgpu_fence *f = vm->ids[i].last_id_use; - amdgpu_sync_fence(&ib.sync, f); - } } - r = amdgpu_vm_update_ptes(adev, vm, &ib, mapping->it.start, + ib->length_dw = 0; + + r = amdgpu_vm_update_ptes(adev, vm, ib, mapping->it.start, mapping->it.last + 1, addr + mapping->offset, flags, gtt_flags); if (r) { - amdgpu_ib_free(adev, &ib); + amdgpu_ib_free(adev, ib); + kfree(ib); return r; } - amdgpu_vm_pad_ib(adev, &ib); - WARN_ON(ib.length_dw > ndw); + amdgpu_vm_pad_ib(adev, ib); + WARN_ON(ib->length_dw > ndw); + r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, ib, 1, + &amdgpu_vm_free_job, + AMDGPU_FENCE_OWNER_VM, + &f); + if (r) + goto error_free; - r = amdgpu_ib_schedule(adev, 1, &ib, AMDGPU_FENCE_OWNER_VM); - if (r) { - amdgpu_ib_free(adev, &ib); - return r; - } amdgpu_vm_fence_pts(vm, mapping->it.start, - mapping->it.last + 1, ib.fence); + mapping->it.last + 1, f); if (fence) { - amdgpu_fence_unref(fence); - *fence = amdgpu_fence_ref(ib.fence); + fence_put(*fence); + *fence = fence_get(f); + } + fence_put(f); + if (!amdgpu_enable_scheduler) { + amdgpu_ib_free(adev, ib); + kfree(ib); } - amdgpu_ib_free(adev, &ib); - return 0; + +error_free: + amdgpu_ib_free(adev, ib); + kfree(ib); + return r; } /** @@ -794,21 +862,25 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, addr = 0; } - if (addr == bo_va->addr) - return 0; - flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem); - list_for_each_entry(mapping, &bo_va->mappings, list) { + spin_lock(&vm->status_lock); + if (!list_empty(&bo_va->vm_status)) + list_splice_init(&bo_va->valids, &bo_va->invalids); + spin_unlock(&vm->status_lock); + + list_for_each_entry(mapping, &bo_va->invalids, list) { r = amdgpu_vm_bo_update_mapping(adev, vm, mapping, addr, flags, &bo_va->last_pt_update); if (r) return r; } - bo_va->addr = addr; spin_lock(&vm->status_lock); + list_splice_init(&bo_va->invalids, &bo_va->valids); list_del_init(&bo_va->vm_status); + if (!mem) + list_add(&bo_va->vm_status, &vm->cleared); spin_unlock(&vm->status_lock); return 0; @@ -861,7 +933,7 @@ int amdgpu_vm_clear_invalids(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_sync *sync) { struct amdgpu_bo_va *bo_va = NULL; - int r; + int r = 0; spin_lock(&vm->status_lock); while (!list_empty(&vm->invalidated)) { @@ -878,8 +950,9 @@ int amdgpu_vm_clear_invalids(struct amdgpu_device *adev, spin_unlock(&vm->status_lock); if (bo_va) - amdgpu_sync_fence(sync, bo_va->last_pt_update); - return 0; + r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update); + + return r; } /** @@ -907,10 +980,10 @@ struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev, } bo_va->vm = vm; bo_va->bo = bo; - bo_va->addr = 0; bo_va->ref_count = 1; INIT_LIST_HEAD(&bo_va->bo_list); - INIT_LIST_HEAD(&bo_va->mappings); + INIT_LIST_HEAD(&bo_va->valids); + INIT_LIST_HEAD(&bo_va->invalids); INIT_LIST_HEAD(&bo_va->vm_status); mutex_lock(&vm->mutex); @@ -999,12 +1072,10 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev, mapping->offset = offset; mapping->flags = flags; - list_add(&mapping->list, &bo_va->mappings); + list_add(&mapping->list, &bo_va->invalids); interval_tree_insert(&mapping->it, &vm->va); trace_amdgpu_vm_bo_map(bo_va, mapping); - bo_va->addr = 0; - /* Make sure the page tables are allocated */ saddr >>= amdgpu_vm_block_size; eaddr >>= amdgpu_vm_block_size; @@ -1028,7 +1099,9 @@ int amdgpu_vm_bo_map(struct amdgpu_device *adev, r = amdgpu_bo_create(adev, AMDGPU_VM_PTE_COUNT * 8, AMDGPU_GPU_PAGE_SIZE, true, - AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, &pt); + AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_CREATE_NO_CPU_ACCESS, + NULL, &pt); if (r) goto error_free; @@ -1085,17 +1158,27 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, { struct amdgpu_bo_va_mapping *mapping; struct amdgpu_vm *vm = bo_va->vm; + bool valid = true; saddr /= AMDGPU_GPU_PAGE_SIZE; - list_for_each_entry(mapping, &bo_va->mappings, list) { + list_for_each_entry(mapping, &bo_va->valids, list) { if (mapping->it.start == saddr) break; } - if (&mapping->list == &bo_va->mappings) { - amdgpu_bo_unreserve(bo_va->bo); - return -ENOENT; + if (&mapping->list == &bo_va->valids) { + valid = false; + + list_for_each_entry(mapping, &bo_va->invalids, list) { + if (mapping->it.start == saddr) + break; + } + + if (&mapping->list == &bo_va->invalids) { + amdgpu_bo_unreserve(bo_va->bo); + return -ENOENT; + } } mutex_lock(&vm->mutex); @@ -1103,12 +1186,10 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, interval_tree_remove(&mapping->it, &vm->va); trace_amdgpu_vm_bo_unmap(bo_va, mapping); - if (bo_va->addr) { - /* clear the old address */ + if (valid) list_add(&mapping->list, &vm->freed); - } else { + else kfree(mapping); - } mutex_unlock(&vm->mutex); amdgpu_bo_unreserve(bo_va->bo); @@ -1139,16 +1220,19 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev, list_del(&bo_va->vm_status); spin_unlock(&vm->status_lock); - list_for_each_entry_safe(mapping, next, &bo_va->mappings, list) { + list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { list_del(&mapping->list); interval_tree_remove(&mapping->it, &vm->va); trace_amdgpu_vm_bo_unmap(bo_va, mapping); - if (bo_va->addr) - list_add(&mapping->list, &vm->freed); - else - kfree(mapping); + list_add(&mapping->list, &vm->freed); } - amdgpu_fence_unref(&bo_va->last_pt_update); + list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) { + list_del(&mapping->list); + interval_tree_remove(&mapping->it, &vm->va); + kfree(mapping); + } + + fence_put(bo_va->last_pt_update); kfree(bo_va); mutex_unlock(&vm->mutex); @@ -1169,12 +1253,10 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va; list_for_each_entry(bo_va, &bo->va, bo_list) { - if (bo_va->addr) { - spin_lock(&bo_va->vm->status_lock); - list_del(&bo_va->vm_status); + spin_lock(&bo_va->vm->status_lock); + if (list_empty(&bo_va->vm_status)) list_add(&bo_va->vm_status, &bo_va->vm->invalidated); - spin_unlock(&bo_va->vm->status_lock); - } + spin_unlock(&bo_va->vm->status_lock); } } @@ -1202,6 +1284,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm) vm->va = RB_ROOT; spin_lock_init(&vm->status_lock); INIT_LIST_HEAD(&vm->invalidated); + INIT_LIST_HEAD(&vm->cleared); INIT_LIST_HEAD(&vm->freed); pd_size = amdgpu_vm_directory_size(adev); @@ -1215,8 +1298,11 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm) return -ENOMEM; } + vm->page_directory_fence = NULL; + r = amdgpu_bo_create(adev, pd_size, align, true, - AMDGPU_GEM_DOMAIN_VRAM, 0, + AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_CREATE_NO_CPU_ACCESS, NULL, &vm->page_directory); if (r) return r; @@ -1263,9 +1349,10 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) kfree(vm->page_tables); amdgpu_bo_unref(&vm->page_directory); + fence_put(vm->page_directory_fence); for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { - amdgpu_fence_unref(&vm->ids[i].flushed_updates); + fence_put(vm->ids[i].flushed_updates); amdgpu_fence_unref(&vm->ids[i].last_id_use); } diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_dp.c b/drivers/gpu/drm/amd/amdgpu/atombios_dp.c index 9ba0a7d5bc8e..92b6acadfc52 100644 --- a/drivers/gpu/drm/amd/amdgpu/atombios_dp.c +++ b/drivers/gpu/drm/amd/amdgpu/atombios_dp.c @@ -139,7 +139,8 @@ amdgpu_atombios_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *m tx_buf[0] = msg->address & 0xff; tx_buf[1] = msg->address >> 8; - tx_buf[2] = msg->request << 4; + tx_buf[2] = (msg->request << 4) | + ((msg->address >> 16) & 0xf); tx_buf[3] = msg->size ? (msg->size - 1) : 0; switch (msg->request & ~DP_AUX_I2C_MOT) { diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c b/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c index ae8caca61e04..cd6edc40c9cd 100644 --- a/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c +++ b/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c @@ -812,7 +812,7 @@ amdgpu_atombios_encoder_setup_dig_transmitter(struct drm_encoder *encoder, int a else args.v1.ucConfig |= ATOM_TRANSMITTER_CONFIG_DIG1_ENCODER; - if ((adev->flags & AMDGPU_IS_APU) && + if ((adev->flags & AMD_IS_APU) && (amdgpu_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_UNIPHY)) { if (is_dp || !amdgpu_dig_monitor_is_duallink(encoder, amdgpu_encoder->pixel_clock)) { diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c index 341c56681841..4b6ce74753cd 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik.c +++ b/drivers/gpu/drm/amd/amdgpu/cik.c @@ -64,6 +64,8 @@ #include "oss/oss_2_0_d.h" #include "oss/oss_2_0_sh_mask.h" +#include "amdgpu_amdkfd.h" + /* * Indirect registers accessor */ @@ -836,7 +838,7 @@ static u32 cik_get_xclk(struct amdgpu_device *adev) { u32 reference_clock = adev->clock.spll.reference_freq; - if (adev->flags & AMDGPU_IS_APU) { + if (adev->flags & AMD_IS_APU) { if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK) return reference_clock / 2; } else { @@ -1233,7 +1235,7 @@ static void cik_gpu_soft_reset(struct amdgpu_device *adev, u32 reset_mask) if (reset_mask & AMDGPU_RESET_VMC) srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_VMC_MASK; - if (!(adev->flags & AMDGPU_IS_APU)) { + if (!(adev->flags & AMD_IS_APU)) { if (reset_mask & AMDGPU_RESET_MC) srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_MC_MASK; } @@ -1409,7 +1411,7 @@ static void cik_gpu_pci_config_reset(struct amdgpu_device *adev) dev_warn(adev->dev, "Wait for MC idle timed out !\n"); } - if (adev->flags & AMDGPU_IS_APU) + if (adev->flags & AMD_IS_APU) kv_save_regs_for_reset(adev, &kv_save); /* disable BM */ @@ -1427,7 +1429,7 @@ static void cik_gpu_pci_config_reset(struct amdgpu_device *adev) } /* does asic init need to be run first??? */ - if (adev->flags & AMDGPU_IS_APU) + if (adev->flags & AMD_IS_APU) kv_restore_regs_for_reset(adev, &kv_save); } @@ -1568,7 +1570,7 @@ static void cik_pcie_gen3_enable(struct amdgpu_device *adev) if (amdgpu_pcie_gen2 == 0) return; - if (adev->flags & AMDGPU_IS_APU) + if (adev->flags & AMD_IS_APU) return; ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask); @@ -1728,7 +1730,7 @@ static void cik_program_aspm(struct amdgpu_device *adev) return; /* XXX double check APUs */ - if (adev->flags & AMDGPU_IS_APU) + if (adev->flags & AMD_IS_APU) return; orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL); @@ -2448,14 +2450,21 @@ static int cik_common_suspend(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + amdgpu_amdkfd_suspend(adev); + return cik_common_hw_fini(adev); } static int cik_common_resume(void *handle) { + int r; struct amdgpu_device *adev = (struct amdgpu_device *)handle; - return cik_common_hw_init(adev); + r = cik_common_hw_init(adev); + if (r) + return r; + + return amdgpu_amdkfd_resume(adev); } static bool cik_common_is_idle(void *handle) diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c index 15df46c93f0a..9ea9de457da3 100644 --- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c @@ -188,6 +188,19 @@ static void cik_sdma_ring_set_wptr(struct amdgpu_ring *ring) WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[me], (ring->wptr << 2) & 0x3fffc); } +static void cik_sdma_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) +{ + struct amdgpu_sdma *sdma = amdgpu_get_sdma_instance(ring); + int i; + + for (i = 0; i < count; i++) + if (sdma && sdma->burst_nop && (i == 0)) + amdgpu_ring_write(ring, ring->nop | + SDMA_NOP_COUNT(count - 1)); + else + amdgpu_ring_write(ring, ring->nop); +} + /** * cik_sdma_ring_emit_ib - Schedule an IB on the DMA engine * @@ -213,8 +226,8 @@ static void cik_sdma_ring_emit_ib(struct amdgpu_ring *ring, amdgpu_ring_write(ring, next_rptr); /* IB packet must end on a 8 DW boundary */ - while ((ring->wptr & 7) != 4) - amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0)); + cik_sdma_ring_insert_nop(ring, (12 - (ring->wptr & 7)) % 8); + amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_INDIRECT_BUFFER, 0, extra_bits)); amdgpu_ring_write(ring, ib->gpu_addr & 0xffffffe0); /* base must be 32 byte aligned */ amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xffffffff); @@ -501,6 +514,8 @@ static int cik_sdma_load_microcode(struct amdgpu_device *adev) fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; adev->sdma[i].fw_version = le32_to_cpu(hdr->header.ucode_version); adev->sdma[i].feature_version = le32_to_cpu(hdr->ucode_feature_version); + if (adev->sdma[i].feature_version >= 20) + adev->sdma[i].burst_nop = true; fw_data = (const __le32 *) (adev->sdma[i].fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); WREG32(mmSDMA0_UCODE_ADDR + sdma_offsets[i], 0); @@ -614,6 +629,7 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring) { struct amdgpu_device *adev = ring->adev; struct amdgpu_ib ib; + struct fence *f = NULL; unsigned i; unsigned index; int r; @@ -629,12 +645,11 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring) gpu_addr = adev->wb.gpu_addr + (index * 4); tmp = 0xCAFEDEAD; adev->wb.wb[index] = cpu_to_le32(tmp); - + memset(&ib, 0, sizeof(ib)); r = amdgpu_ib_get(ring, NULL, 256, &ib); if (r) { - amdgpu_wb_free(adev, index); DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); - return r; + goto err0; } ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0); @@ -643,20 +658,16 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring) ib.ptr[3] = 1; ib.ptr[4] = 0xDEADBEEF; ib.length_dw = 5; + r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, &ib, 1, NULL, + AMDGPU_FENCE_OWNER_UNDEFINED, + &f); + if (r) + goto err1; - r = amdgpu_ib_schedule(adev, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED); + r = fence_wait(f, false); if (r) { - amdgpu_ib_free(adev, &ib); - amdgpu_wb_free(adev, index); - DRM_ERROR("amdgpu: failed to schedule ib (%d).\n", r); - return r; - } - r = amdgpu_fence_wait(ib.fence, false); - if (r) { - amdgpu_ib_free(adev, &ib); - amdgpu_wb_free(adev, index); DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); - return r; + goto err1; } for (i = 0; i < adev->usec_timeout; i++) { tmp = le32_to_cpu(adev->wb.wb[index]); @@ -666,12 +677,17 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring) } if (i < adev->usec_timeout) { DRM_INFO("ib test on ring %d succeeded in %u usecs\n", - ib.fence->ring->idx, i); + ring->idx, i); + goto err1; } else { DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp); r = -EINVAL; } + +err1: + fence_put(f); amdgpu_ib_free(adev, &ib); +err0: amdgpu_wb_free(adev, index); return r; } @@ -814,8 +830,19 @@ static void cik_sdma_vm_set_pte_pde(struct amdgpu_ib *ib, */ static void cik_sdma_vm_pad_ib(struct amdgpu_ib *ib) { - while (ib->length_dw & 0x7) - ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0); + struct amdgpu_sdma *sdma = amdgpu_get_sdma_instance(ib->ring); + u32 pad_count; + int i; + + pad_count = (8 - (ib->length_dw & 0x7)) % 8; + for (i = 0; i < pad_count; i++) + if (sdma && sdma->burst_nop && (i == 0)) + ib->ptr[ib->length_dw++] = + SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0) | + SDMA_NOP_COUNT(pad_count - 1); + else + ib->ptr[ib->length_dw++] = + SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0); } /** @@ -1302,6 +1329,7 @@ static const struct amdgpu_ring_funcs cik_sdma_ring_funcs = { .test_ring = cik_sdma_ring_test_ring, .test_ib = cik_sdma_ring_test_ib, .is_lockup = cik_sdma_ring_is_lockup, + .insert_nop = cik_sdma_ring_insert_nop, }; static void cik_sdma_set_ring_funcs(struct amdgpu_device *adev) @@ -1338,18 +1366,18 @@ static void cik_sdma_set_irq_funcs(struct amdgpu_device *adev) * Used by the amdgpu ttm implementation to move pages if * registered as the asic copy callback. */ -static void cik_sdma_emit_copy_buffer(struct amdgpu_ring *ring, +static void cik_sdma_emit_copy_buffer(struct amdgpu_ib *ib, uint64_t src_offset, uint64_t dst_offset, uint32_t byte_count) { - amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0)); - amdgpu_ring_write(ring, byte_count); - amdgpu_ring_write(ring, 0); /* src/dst endian swap */ - amdgpu_ring_write(ring, lower_32_bits(src_offset)); - amdgpu_ring_write(ring, upper_32_bits(src_offset)); - amdgpu_ring_write(ring, lower_32_bits(dst_offset)); - amdgpu_ring_write(ring, upper_32_bits(dst_offset)); + ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_COPY, SDMA_COPY_SUB_OPCODE_LINEAR, 0); + ib->ptr[ib->length_dw++] = byte_count; + ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */ + ib->ptr[ib->length_dw++] = lower_32_bits(src_offset); + ib->ptr[ib->length_dw++] = upper_32_bits(src_offset); + ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); + ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset); } /** @@ -1362,16 +1390,16 @@ static void cik_sdma_emit_copy_buffer(struct amdgpu_ring *ring, * * Fill GPU buffers using the DMA engine (CIK). */ -static void cik_sdma_emit_fill_buffer(struct amdgpu_ring *ring, +static void cik_sdma_emit_fill_buffer(struct amdgpu_ib *ib, uint32_t src_data, uint64_t dst_offset, uint32_t byte_count) { - amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_CONSTANT_FILL, 0, 0)); - amdgpu_ring_write(ring, lower_32_bits(dst_offset)); - amdgpu_ring_write(ring, upper_32_bits(dst_offset)); - amdgpu_ring_write(ring, src_data); - amdgpu_ring_write(ring, byte_count); + ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_CONSTANT_FILL, 0, 0); + ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); + ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset); + ib->ptr[ib->length_dw++] = src_data; + ib->ptr[ib->length_dw++] = byte_count; } static const struct amdgpu_buffer_funcs cik_sdma_buffer_funcs = { @@ -1404,5 +1432,6 @@ static void cik_sdma_set_vm_pte_funcs(struct amdgpu_device *adev) if (adev->vm_manager.vm_pte_funcs == NULL) { adev->vm_manager.vm_pte_funcs = &cik_sdma_vm_pte_funcs; adev->vm_manager.vm_pte_funcs_ring = &adev->sdma[0].ring; + adev->vm_manager.vm_pte_funcs_ring->is_pte_ring = true; } } diff --git a/drivers/gpu/drm/amd/amdgpu/cikd.h b/drivers/gpu/drm/amd/amdgpu/cikd.h index d19085a97064..7f6d457f250a 100644 --- a/drivers/gpu/drm/amd/amdgpu/cikd.h +++ b/drivers/gpu/drm/amd/amdgpu/cikd.h @@ -487,6 +487,7 @@ (((op) & 0xFF) << 0)) /* sDMA opcodes */ #define SDMA_OPCODE_NOP 0 +# define SDMA_NOP_COUNT(x) (((x) & 0x3FFF) << 16) #define SDMA_OPCODE_COPY 1 # define SDMA_COPY_SUB_OPCODE_LINEAR 0 # define SDMA_COPY_SUB_OPCODE_TILED 1 @@ -552,6 +553,12 @@ #define VCE_CMD_IB_AUTO 0x00000005 #define VCE_CMD_SEMAPHORE 0x00000006 +/* if PTR32, these are the bases for scratch and lds */ +#define PRIVATE_BASE(x) ((x) << 0) /* scratch */ +#define SHARED_BASE(x) ((x) << 16) /* LDS */ + +#define KFD_CIK_SDMA_QUEUE_OFFSET 0x200 + /* valid for both DEFAULT_MTYPE and APE1_MTYPE */ enum { MTYPE_CACHED = 0, diff --git a/drivers/gpu/drm/amd/amdgpu/cz_dpm.c b/drivers/gpu/drm/amd/amdgpu/cz_dpm.c index ace870afc7d4..44fa96ad4709 100644 --- a/drivers/gpu/drm/amd/amdgpu/cz_dpm.c +++ b/drivers/gpu/drm/amd/amdgpu/cz_dpm.c @@ -1596,9 +1596,9 @@ static int cz_dpm_update_low_memory_pstate(struct amdgpu_device *adev) if (pi->sys_info.nb_dpm_enable) { if (ps->force_high) - cz_dpm_nbdpm_lm_pstate_enable(adev, true); - else cz_dpm_nbdpm_lm_pstate_enable(adev, false); + else + cz_dpm_nbdpm_lm_pstate_enable(adev, true); } return ret; diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c index e70a26f587a0..e4d101b1252a 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c @@ -126,9 +126,31 @@ static const u32 tonga_mgcg_cgcg_init[] = mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000, }; +static const u32 golden_settings_fiji_a10[] = +{ + mmDCI_CLK_CNTL, 0x00000080, 0x00000000, + mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070, + mmFBC_MISC, 0x1f311fff, 0x12300000, + mmHDMI_CONTROL, 0x31000111, 0x00000011, +}; + +static const u32 fiji_mgcg_cgcg_init[] = +{ + mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100, + mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000, +}; + static void dce_v10_0_init_golden_registers(struct amdgpu_device *adev) { switch (adev->asic_type) { + case CHIP_FIJI: + amdgpu_program_register_sequence(adev, + fiji_mgcg_cgcg_init, + (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init)); + amdgpu_program_register_sequence(adev, + golden_settings_fiji_a10, + (const u32)ARRAY_SIZE(golden_settings_fiji_a10)); + break; case CHIP_TONGA: amdgpu_program_register_sequence(adev, tonga_mgcg_cgcg_init, @@ -803,11 +825,11 @@ static u32 dce_v10_0_line_buffer_adjust(struct amdgpu_device *adev, buffer_alloc = 2; } else if (mode->crtc_hdisplay < 4096) { mem_cfg = 0; - buffer_alloc = (adev->flags & AMDGPU_IS_APU) ? 2 : 4; + buffer_alloc = (adev->flags & AMD_IS_APU) ? 2 : 4; } else { DRM_DEBUG_KMS("Mode too big for LB!\n"); mem_cfg = 0; - buffer_alloc = (adev->flags & AMDGPU_IS_APU) ? 2 : 4; + buffer_alloc = (adev->flags & AMD_IS_APU) ? 2 : 4; } } else { mem_cfg = 1; @@ -1331,7 +1353,7 @@ static void dce_v10_0_program_watermarks(struct amdgpu_device *adev, tmp = REG_SET_FIELD(wm_mask, DPG_WATERMARK_MASK_CONTROL, URGENCY_WATERMARK_MASK, 2); WREG32(mmDPG_WATERMARK_MASK_CONTROL + amdgpu_crtc->crtc_offset, tmp); tmp = RREG32(mmDPG_PIPE_URGENCY_CONTROL + amdgpu_crtc->crtc_offset); - tmp = REG_SET_FIELD(tmp, DPG_PIPE_URGENCY_CONTROL, URGENCY_LOW_WATERMARK, latency_watermark_a); + tmp = REG_SET_FIELD(tmp, DPG_PIPE_URGENCY_CONTROL, URGENCY_LOW_WATERMARK, latency_watermark_b); tmp = REG_SET_FIELD(tmp, DPG_PIPE_URGENCY_CONTROL, URGENCY_HIGH_WATERMARK, line_time); WREG32(mmDPG_PIPE_URGENCY_CONTROL + amdgpu_crtc->crtc_offset, tmp); /* restore original selection */ @@ -2888,6 +2910,7 @@ static int dce_v10_0_early_init(void *handle) dce_v10_0_set_irq_funcs(adev); switch (adev->asic_type) { + case CHIP_FIJI: case CHIP_TONGA: adev->mode_info.num_crtc = 6; /* XXX 7??? */ adev->mode_info.num_hpd = 6; diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c index dcb402ee048a..6411e8244671 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c @@ -801,11 +801,11 @@ static u32 dce_v11_0_line_buffer_adjust(struct amdgpu_device *adev, buffer_alloc = 2; } else if (mode->crtc_hdisplay < 4096) { mem_cfg = 0; - buffer_alloc = (adev->flags & AMDGPU_IS_APU) ? 2 : 4; + buffer_alloc = (adev->flags & AMD_IS_APU) ? 2 : 4; } else { DRM_DEBUG_KMS("Mode too big for LB!\n"); mem_cfg = 0; - buffer_alloc = (adev->flags & AMDGPU_IS_APU) ? 2 : 4; + buffer_alloc = (adev->flags & AMD_IS_APU) ? 2 : 4; } } else { mem_cfg = 1; @@ -1329,7 +1329,7 @@ static void dce_v11_0_program_watermarks(struct amdgpu_device *adev, tmp = REG_SET_FIELD(wm_mask, DPG_WATERMARK_MASK_CONTROL, URGENCY_WATERMARK_MASK, 2); WREG32(mmDPG_WATERMARK_MASK_CONTROL + amdgpu_crtc->crtc_offset, tmp); tmp = RREG32(mmDPG_PIPE_URGENCY_CONTROL + amdgpu_crtc->crtc_offset); - tmp = REG_SET_FIELD(tmp, DPG_PIPE_URGENCY_CONTROL, URGENCY_LOW_WATERMARK, latency_watermark_a); + tmp = REG_SET_FIELD(tmp, DPG_PIPE_URGENCY_CONTROL, URGENCY_LOW_WATERMARK, latency_watermark_b); tmp = REG_SET_FIELD(tmp, DPG_PIPE_URGENCY_CONTROL, URGENCY_HIGH_WATERMARK, line_time); WREG32(mmDPG_PIPE_URGENCY_CONTROL + amdgpu_crtc->crtc_offset, tmp); /* restore original selection */ diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c index cc050a329c49..c86911c2ea2a 100644 --- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c @@ -770,11 +770,11 @@ static u32 dce_v8_0_line_buffer_adjust(struct amdgpu_device *adev, buffer_alloc = 2; } else if (mode->crtc_hdisplay < 4096) { tmp = 0; - buffer_alloc = (adev->flags & AMDGPU_IS_APU) ? 2 : 4; + buffer_alloc = (adev->flags & AMD_IS_APU) ? 2 : 4; } else { DRM_DEBUG_KMS("Mode too big for LB!\n"); tmp = 0; - buffer_alloc = (adev->flags & AMDGPU_IS_APU) ? 2 : 4; + buffer_alloc = (adev->flags & AMD_IS_APU) ? 2 : 4; } } else { tmp = 1; diff --git a/drivers/gpu/drm/amd/amdgpu/fiji_dpm.c b/drivers/gpu/drm/amd/amdgpu/fiji_dpm.c new file mode 100644 index 000000000000..8f9845d9a986 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/fiji_dpm.c @@ -0,0 +1,181 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include "drmP.h" +#include "amdgpu.h" +#include "fiji_smumgr.h" + +MODULE_FIRMWARE("amdgpu/fiji_smc.bin"); + +static void fiji_dpm_set_funcs(struct amdgpu_device *adev); + +static int fiji_dpm_early_init(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + fiji_dpm_set_funcs(adev); + + return 0; +} + +static int fiji_dpm_init_microcode(struct amdgpu_device *adev) +{ + char fw_name[30] = "amdgpu/fiji_smc.bin"; + int err; + + err = request_firmware(&adev->pm.fw, fw_name, adev->dev); + if (err) + goto out; + err = amdgpu_ucode_validate(adev->pm.fw); + +out: + if (err) { + DRM_ERROR("Failed to load firmware \"%s\"", fw_name); + release_firmware(adev->pm.fw); + adev->pm.fw = NULL; + } + return err; +} + +static int fiji_dpm_sw_init(void *handle) +{ + int ret; + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + ret = fiji_dpm_init_microcode(adev); + if (ret) + return ret; + + return 0; +} + +static int fiji_dpm_sw_fini(void *handle) +{ + return 0; +} + +static int fiji_dpm_hw_init(void *handle) +{ + int ret; + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + mutex_lock(&adev->pm.mutex); + + ret = fiji_smu_init(adev); + if (ret) { + DRM_ERROR("SMU initialization failed\n"); + goto fail; + } + + ret = fiji_smu_start(adev); + if (ret) { + DRM_ERROR("SMU start failed\n"); + goto fail; + } + + mutex_unlock(&adev->pm.mutex); + return 0; + +fail: + adev->firmware.smu_load = false; + mutex_unlock(&adev->pm.mutex); + return -EINVAL; +} + +static int fiji_dpm_hw_fini(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + mutex_lock(&adev->pm.mutex); + fiji_smu_fini(adev); + mutex_unlock(&adev->pm.mutex); + return 0; +} + +static int fiji_dpm_suspend(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + fiji_dpm_hw_fini(adev); + + return 0; +} + +static int fiji_dpm_resume(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + fiji_dpm_hw_init(adev); + + return 0; +} + +static int fiji_dpm_set_clockgating_state(void *handle, + enum amd_clockgating_state state) +{ + return 0; +} + +static int fiji_dpm_set_powergating_state(void *handle, + enum amd_powergating_state state) +{ + return 0; +} + +const struct amd_ip_funcs fiji_dpm_ip_funcs = { + .early_init = fiji_dpm_early_init, + .late_init = NULL, + .sw_init = fiji_dpm_sw_init, + .sw_fini = fiji_dpm_sw_fini, + .hw_init = fiji_dpm_hw_init, + .hw_fini = fiji_dpm_hw_fini, + .suspend = fiji_dpm_suspend, + .resume = fiji_dpm_resume, + .is_idle = NULL, + .wait_for_idle = NULL, + .soft_reset = NULL, + .print_status = NULL, + .set_clockgating_state = fiji_dpm_set_clockgating_state, + .set_powergating_state = fiji_dpm_set_powergating_state, +}; + +static const struct amdgpu_dpm_funcs fiji_dpm_funcs = { + .get_temperature = NULL, + .pre_set_power_state = NULL, + .set_power_state = NULL, + .post_set_power_state = NULL, + .display_configuration_changed = NULL, + .get_sclk = NULL, + .get_mclk = NULL, + .print_power_state = NULL, + .debugfs_print_current_performance_level = NULL, + .force_performance_level = NULL, + .vblank_too_short = NULL, + .powergate_uvd = NULL, +}; + +static void fiji_dpm_set_funcs(struct amdgpu_device *adev) +{ + if (NULL == adev->pm.funcs) + adev->pm.funcs = &fiji_dpm_funcs; +} diff --git a/drivers/gpu/drm/amd/amdgpu/fiji_ppsmc.h b/drivers/gpu/drm/amd/amdgpu/fiji_ppsmc.h new file mode 100644 index 000000000000..3c4824082990 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/fiji_ppsmc.h @@ -0,0 +1,182 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef FIJI_PP_SMC_H +#define FIJI_PP_SMC_H + +#pragma pack(push, 1) + +#define PPSMC_SWSTATE_FLAG_DC 0x01 +#define PPSMC_SWSTATE_FLAG_UVD 0x02 +#define PPSMC_SWSTATE_FLAG_VCE 0x04 + +#define PPSMC_THERMAL_PROTECT_TYPE_INTERNAL 0x00 +#define PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL 0x01 +#define PPSMC_THERMAL_PROTECT_TYPE_NONE 0xff + +#define PPSMC_SYSTEMFLAG_GPIO_DC 0x01 +#define PPSMC_SYSTEMFLAG_STEPVDDC 0x02 +#define PPSMC_SYSTEMFLAG_GDDR5 0x04 + +#define PPSMC_SYSTEMFLAG_DISABLE_BABYSTEP 0x08 + +#define PPSMC_SYSTEMFLAG_REGULATOR_HOT 0x10 +#define PPSMC_SYSTEMFLAG_REGULATOR_HOT_ANALOG 0x20 + +#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_MASK 0x07 +#define PPSMC_EXTRAFLAGS_AC2DC_DONT_WAIT_FOR_VBLANK 0x08 + +#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTODPMLOWSTATE 0x00 +#define PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTOINITIALSTATE 0x01 + +#define PPSMC_DPM2FLAGS_TDPCLMP 0x01 +#define PPSMC_DPM2FLAGS_PWRSHFT 0x02 +#define PPSMC_DPM2FLAGS_OCP 0x04 + +#define PPSMC_DISPLAY_WATERMARK_LOW 0 +#define PPSMC_DISPLAY_WATERMARK_HIGH 1 + +#define PPSMC_STATEFLAG_AUTO_PULSE_SKIP 0x01 +#define PPSMC_STATEFLAG_POWERBOOST 0x02 +#define PPSMC_STATEFLAG_PSKIP_ON_TDP_FAULT 0x04 +#define PPSMC_STATEFLAG_POWERSHIFT 0x08 +#define PPSMC_STATEFLAG_SLOW_READ_MARGIN 0x10 +#define PPSMC_STATEFLAG_DEEPSLEEP_THROTTLE 0x20 +#define PPSMC_STATEFLAG_DEEPSLEEP_BYPASS 0x40 + +#define FDO_MODE_HARDWARE 0 +#define FDO_MODE_PIECE_WISE_LINEAR 1 + +enum FAN_CONTROL { + FAN_CONTROL_FUZZY, + FAN_CONTROL_TABLE +}; + +//Gemini Modes +#define PPSMC_GeminiModeNone 0 //Single GPU board +#define PPSMC_GeminiModeMaster 1 //Master GPU on a Gemini board +#define PPSMC_GeminiModeSlave 2 //Slave GPU on a Gemini board + +#define PPSMC_Result_OK ((uint16_t)0x01) +#define PPSMC_Result_NoMore ((uint16_t)0x02) +#define PPSMC_Result_NotNow ((uint16_t)0x03) +#define PPSMC_Result_Failed ((uint16_t)0xFF) +#define PPSMC_Result_UnknownCmd ((uint16_t)0xFE) +#define PPSMC_Result_UnknownVT ((uint16_t)0xFD) + +typedef uint16_t PPSMC_Result; + +#define PPSMC_isERROR(x) ((uint16_t)0x80 & (x)) + +#define PPSMC_MSG_Halt ((uint16_t)0x10) +#define PPSMC_MSG_Resume ((uint16_t)0x11) +#define PPSMC_MSG_EnableDPMLevel ((uint16_t)0x12) +#define PPSMC_MSG_ZeroLevelsDisabled ((uint16_t)0x13) +#define PPSMC_MSG_OneLevelsDisabled ((uint16_t)0x14) +#define PPSMC_MSG_TwoLevelsDisabled ((uint16_t)0x15) +#define PPSMC_MSG_EnableThermalInterrupt ((uint16_t)0x16) +#define PPSMC_MSG_RunningOnAC ((uint16_t)0x17) +#define PPSMC_MSG_LevelUp ((uint16_t)0x18) +#define PPSMC_MSG_LevelDown ((uint16_t)0x19) +#define PPSMC_MSG_ResetDPMCounters ((uint16_t)0x1a) +#define PPSMC_MSG_SwitchToSwState ((uint16_t)0x20) +#define PPSMC_MSG_SwitchToSwStateLast ((uint16_t)0x3f) +#define PPSMC_MSG_SwitchToInitialState ((uint16_t)0x40) +#define PPSMC_MSG_NoForcedLevel ((uint16_t)0x41) +#define PPSMC_MSG_ForceHigh ((uint16_t)0x42) +#define PPSMC_MSG_ForceMediumOrHigh ((uint16_t)0x43) +#define PPSMC_MSG_SwitchToMinimumPower ((uint16_t)0x51) +#define PPSMC_MSG_ResumeFromMinimumPower ((uint16_t)0x52) +#define PPSMC_MSG_EnableCac ((uint16_t)0x53) +#define PPSMC_MSG_DisableCac ((uint16_t)0x54) +#define PPSMC_DPMStateHistoryStart ((uint16_t)0x55) +#define PPSMC_DPMStateHistoryStop ((uint16_t)0x56) +#define PPSMC_CACHistoryStart ((uint16_t)0x57) +#define PPSMC_CACHistoryStop ((uint16_t)0x58) +#define PPSMC_TDPClampingActive ((uint16_t)0x59) +#define PPSMC_TDPClampingInactive ((uint16_t)0x5A) +#define PPSMC_StartFanControl ((uint16_t)0x5B) +#define PPSMC_StopFanControl ((uint16_t)0x5C) +#define PPSMC_NoDisplay ((uint16_t)0x5D) +#define PPSMC_HasDisplay ((uint16_t)0x5E) +#define PPSMC_MSG_UVDPowerOFF ((uint16_t)0x60) +#define PPSMC_MSG_UVDPowerON ((uint16_t)0x61) +#define PPSMC_MSG_EnableULV ((uint16_t)0x62) +#define PPSMC_MSG_DisableULV ((uint16_t)0x63) +#define PPSMC_MSG_EnterULV ((uint16_t)0x64) +#define PPSMC_MSG_ExitULV ((uint16_t)0x65) +#define PPSMC_PowerShiftActive ((uint16_t)0x6A) +#define PPSMC_PowerShiftInactive ((uint16_t)0x6B) +#define PPSMC_OCPActive ((uint16_t)0x6C) +#define PPSMC_OCPInactive ((uint16_t)0x6D) +#define PPSMC_CACLongTermAvgEnable ((uint16_t)0x6E) +#define PPSMC_CACLongTermAvgDisable ((uint16_t)0x6F) +#define PPSMC_MSG_InferredStateSweep_Start ((uint16_t)0x70) +#define PPSMC_MSG_InferredStateSweep_Stop ((uint16_t)0x71) +#define PPSMC_MSG_SwitchToLowestInfState ((uint16_t)0x72) +#define PPSMC_MSG_SwitchToNonInfState ((uint16_t)0x73) +#define PPSMC_MSG_AllStateSweep_Start ((uint16_t)0x74) +#define PPSMC_MSG_AllStateSweep_Stop ((uint16_t)0x75) +#define PPSMC_MSG_SwitchNextLowerInfState ((uint16_t)0x76) +#define PPSMC_MSG_SwitchNextHigherInfState ((uint16_t)0x77) +#define PPSMC_MSG_MclkRetrainingTest ((uint16_t)0x78) +#define PPSMC_MSG_ForceTDPClamping ((uint16_t)0x79) +#define PPSMC_MSG_CollectCAC_PowerCorreln ((uint16_t)0x7A) +#define PPSMC_MSG_CollectCAC_WeightCalib ((uint16_t)0x7B) +#define PPSMC_MSG_CollectCAC_SQonly ((uint16_t)0x7C) +#define PPSMC_MSG_CollectCAC_TemperaturePwr ((uint16_t)0x7D) +#define PPSMC_MSG_ExtremitiesTest_Start ((uint16_t)0x7E) +#define PPSMC_MSG_ExtremitiesTest_Stop ((uint16_t)0x7F) +#define PPSMC_FlushDataCache ((uint16_t)0x80) +#define PPSMC_FlushInstrCache ((uint16_t)0x81) +#define PPSMC_MSG_SetEnabledLevels ((uint16_t)0x82) +#define PPSMC_MSG_SetForcedLevels ((uint16_t)0x83) +#define PPSMC_MSG_ResetToDefaults ((uint16_t)0x84) +#define PPSMC_MSG_SetForcedLevelsAndJump ((uint16_t)0x85) +#define PPSMC_MSG_SetCACHistoryMode ((uint16_t)0x86) +#define PPSMC_MSG_EnableDTE ((uint16_t)0x87) +#define PPSMC_MSG_DisableDTE ((uint16_t)0x88) +#define PPSMC_MSG_SmcSpaceSetAddress ((uint16_t)0x89) +#define PPSMC_MSG_SmcSpaceWriteDWordInc ((uint16_t)0x8A) +#define PPSMC_MSG_SmcSpaceWriteWordInc ((uint16_t)0x8B) +#define PPSMC_MSG_SmcSpaceWriteByteInc ((uint16_t)0x8C) + +#define PPSMC_MSG_BREAK ((uint16_t)0xF8) + +#define PPSMC_MSG_Test ((uint16_t)0x100) +#define PPSMC_MSG_DRV_DRAM_ADDR_HI ((uint16_t)0x250) +#define PPSMC_MSG_DRV_DRAM_ADDR_LO ((uint16_t)0x251) +#define PPSMC_MSG_SMU_DRAM_ADDR_HI ((uint16_t)0x252) +#define PPSMC_MSG_SMU_DRAM_ADDR_LO ((uint16_t)0x253) +#define PPSMC_MSG_LoadUcodes ((uint16_t)0x254) + +typedef uint16_t PPSMC_Msg; + +#define PPSMC_EVENT_STATUS_THERMAL 0x00000001 +#define PPSMC_EVENT_STATUS_REGULATORHOT 0x00000002 +#define PPSMC_EVENT_STATUS_DC 0x00000004 +#define PPSMC_EVENT_STATUS_GPIO17 0x00000008 + +#pragma pack(pop) + +#endif diff --git a/drivers/gpu/drm/amd/amdgpu/fiji_smc.c b/drivers/gpu/drm/amd/amdgpu/fiji_smc.c new file mode 100644 index 000000000000..322edea65857 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/fiji_smc.c @@ -0,0 +1,857 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#include +#include "drmP.h" +#include "amdgpu.h" +#include "fiji_ppsmc.h" +#include "fiji_smumgr.h" +#include "smu_ucode_xfer_vi.h" +#include "amdgpu_ucode.h" + +#include "smu/smu_7_1_3_d.h" +#include "smu/smu_7_1_3_sh_mask.h" + +#define FIJI_SMC_SIZE 0x20000 + +static int fiji_set_smc_sram_address(struct amdgpu_device *adev, uint32_t smc_address, uint32_t limit) +{ + uint32_t val; + + if (smc_address & 3) + return -EINVAL; + + if ((smc_address + 3) > limit) + return -EINVAL; + + WREG32(mmSMC_IND_INDEX_0, smc_address); + + val = RREG32(mmSMC_IND_ACCESS_CNTL); + val = REG_SET_FIELD(val, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 0); + WREG32(mmSMC_IND_ACCESS_CNTL, val); + + return 0; +} + +static int fiji_copy_bytes_to_smc(struct amdgpu_device *adev, uint32_t smc_start_address, const uint8_t *src, uint32_t byte_count, uint32_t limit) +{ + uint32_t addr; + uint32_t data, orig_data; + int result = 0; + uint32_t extra_shift; + unsigned long flags; + + if (smc_start_address & 3) + return -EINVAL; + + if ((smc_start_address + byte_count) > limit) + return -EINVAL; + + addr = smc_start_address; + + spin_lock_irqsave(&adev->smc_idx_lock, flags); + while (byte_count >= 4) { + /* Bytes are written into the SMC addres space with the MSB first */ + data = (src[0] << 24) + (src[1] << 16) + (src[2] << 8) + src[3]; + + result = fiji_set_smc_sram_address(adev, addr, limit); + + if (result) + goto out; + + WREG32(mmSMC_IND_DATA_0, data); + + src += 4; + byte_count -= 4; + addr += 4; + } + + if (0 != byte_count) { + /* Now write odd bytes left, do a read modify write cycle */ + data = 0; + + result = fiji_set_smc_sram_address(adev, addr, limit); + if (result) + goto out; + + orig_data = RREG32(mmSMC_IND_DATA_0); + extra_shift = 8 * (4 - byte_count); + + while (byte_count > 0) { + data = (data << 8) + *src++; + byte_count--; + } + + data <<= extra_shift; + data |= (orig_data & ~((~0UL) << extra_shift)); + + result = fiji_set_smc_sram_address(adev, addr, limit); + if (result) + goto out; + + WREG32(mmSMC_IND_DATA_0, data); + } + +out: + spin_unlock_irqrestore(&adev->smc_idx_lock, flags); + return result; +} + +static int fiji_program_jump_on_start(struct amdgpu_device *adev) +{ + static unsigned char data[] = {0xE0, 0x00, 0x80, 0x40}; + fiji_copy_bytes_to_smc(adev, 0x0, data, 4, sizeof(data)+1); + + return 0; +} + +static bool fiji_is_smc_ram_running(struct amdgpu_device *adev) +{ + uint32_t val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0); + val = REG_GET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, ck_disable); + + return ((0 == val) && (0x20100 <= RREG32_SMC(ixSMC_PC_C))); +} + +static int wait_smu_response(struct amdgpu_device *adev) +{ + int i; + uint32_t val; + + for (i = 0; i < adev->usec_timeout; i++) { + val = RREG32(mmSMC_RESP_0); + if (REG_GET_FIELD(val, SMC_RESP_0, SMC_RESP)) + break; + udelay(1); + } + + if (i == adev->usec_timeout) + return -EINVAL; + + return 0; +} + +static int fiji_send_msg_to_smc_offset(struct amdgpu_device *adev) +{ + if (wait_smu_response(adev)) { + DRM_ERROR("Failed to send previous message\n"); + return -EINVAL; + } + + WREG32(mmSMC_MSG_ARG_0, 0x20000); + WREG32(mmSMC_MESSAGE_0, PPSMC_MSG_Test); + + if (wait_smu_response(adev)) { + DRM_ERROR("Failed to send message\n"); + return -EINVAL; + } + + return 0; +} + +static int fiji_send_msg_to_smc(struct amdgpu_device *adev, PPSMC_Msg msg) +{ + if (!fiji_is_smc_ram_running(adev)) + { + return -EINVAL;; + } + + if (wait_smu_response(adev)) { + DRM_ERROR("Failed to send previous message\n"); + return -EINVAL; + } + + WREG32(mmSMC_MESSAGE_0, msg); + + if (wait_smu_response(adev)) { + DRM_ERROR("Failed to send message\n"); + return -EINVAL; + } + + return 0; +} + +static int fiji_send_msg_to_smc_without_waiting(struct amdgpu_device *adev, + PPSMC_Msg msg) +{ + if (wait_smu_response(adev)) { + DRM_ERROR("Failed to send previous message\n"); + return -EINVAL; + } + + WREG32(mmSMC_MESSAGE_0, msg); + + return 0; +} + +static int fiji_send_msg_to_smc_with_parameter(struct amdgpu_device *adev, + PPSMC_Msg msg, + uint32_t parameter) +{ + if (!fiji_is_smc_ram_running(adev)) + return -EINVAL; + + if (wait_smu_response(adev)) { + DRM_ERROR("Failed to send previous message\n"); + return -EINVAL; + } + + WREG32(mmSMC_MSG_ARG_0, parameter); + + return fiji_send_msg_to_smc(adev, msg); +} + +static int fiji_send_msg_to_smc_with_parameter_without_waiting( + struct amdgpu_device *adev, + PPSMC_Msg msg, uint32_t parameter) +{ + if (wait_smu_response(adev)) { + DRM_ERROR("Failed to send previous message\n"); + return -EINVAL; + } + + WREG32(mmSMC_MSG_ARG_0, parameter); + + return fiji_send_msg_to_smc_without_waiting(adev, msg); +} + +#if 0 /* not used yet */ +static int fiji_wait_for_smc_inactive(struct amdgpu_device *adev) +{ + int i; + uint32_t val; + + if (!fiji_is_smc_ram_running(adev)) + return -EINVAL; + + for (i = 0; i < adev->usec_timeout; i++) { + val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0); + if (REG_GET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, cken) == 0) + break; + udelay(1); + } + + if (i == adev->usec_timeout) + return -EINVAL; + + return 0; +} +#endif + +static int fiji_smu_upload_firmware_image(struct amdgpu_device *adev) +{ + const struct smc_firmware_header_v1_0 *hdr; + uint32_t ucode_size; + uint32_t ucode_start_address; + const uint8_t *src; + uint32_t val; + uint32_t byte_count; + uint32_t *data; + unsigned long flags; + + if (!adev->pm.fw) + return -EINVAL; + + hdr = (const struct smc_firmware_header_v1_0 *)adev->pm.fw->data; + amdgpu_ucode_print_smc_hdr(&hdr->header); + + adev->pm.fw_version = le32_to_cpu(hdr->header.ucode_version); + ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes); + ucode_start_address = le32_to_cpu(hdr->ucode_start_addr); + src = (const uint8_t *) + (adev->pm.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); + + if (ucode_size & 3) { + DRM_ERROR("SMC ucode is not 4 bytes aligned\n"); + return -EINVAL; + } + + if (ucode_size > FIJI_SMC_SIZE) { + DRM_ERROR("SMC address is beyond the SMC RAM area\n"); + return -EINVAL; + } + + spin_lock_irqsave(&adev->smc_idx_lock, flags); + WREG32(mmSMC_IND_INDEX_0, ucode_start_address); + + val = RREG32(mmSMC_IND_ACCESS_CNTL); + val = REG_SET_FIELD(val, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 1); + WREG32(mmSMC_IND_ACCESS_CNTL, val); + + byte_count = ucode_size; + data = (uint32_t *)src; + for (; byte_count >= 4; data++, byte_count -= 4) + WREG32(mmSMC_IND_DATA_0, data[0]); + + val = RREG32(mmSMC_IND_ACCESS_CNTL); + val = REG_SET_FIELD(val, SMC_IND_ACCESS_CNTL, AUTO_INCREMENT_IND_0, 0); + WREG32(mmSMC_IND_ACCESS_CNTL, val); + spin_unlock_irqrestore(&adev->smc_idx_lock, flags); + + return 0; +} + +#if 0 /* not used yet */ +static int fiji_read_smc_sram_dword(struct amdgpu_device *adev, + uint32_t smc_address, + uint32_t *value, + uint32_t limit) +{ + int result; + unsigned long flags; + + spin_lock_irqsave(&adev->smc_idx_lock, flags); + result = fiji_set_smc_sram_address(adev, smc_address, limit); + if (result == 0) + *value = RREG32(mmSMC_IND_DATA_0); + spin_unlock_irqrestore(&adev->smc_idx_lock, flags); + return result; +} + +static int fiji_write_smc_sram_dword(struct amdgpu_device *adev, + uint32_t smc_address, + uint32_t value, + uint32_t limit) +{ + int result; + unsigned long flags; + + spin_lock_irqsave(&adev->smc_idx_lock, flags); + result = fiji_set_smc_sram_address(adev, smc_address, limit); + if (result == 0) + WREG32(mmSMC_IND_DATA_0, value); + spin_unlock_irqrestore(&adev->smc_idx_lock, flags); + return result; +} + +static int fiji_smu_stop_smc(struct amdgpu_device *adev) +{ + uint32_t val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL); + val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 1); + WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val); + + val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0); + val = REG_SET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 1); + WREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0, val); + + return 0; +} +#endif + +static enum AMDGPU_UCODE_ID fiji_convert_fw_type(uint32_t fw_type) +{ + switch (fw_type) { + case UCODE_ID_SDMA0: + return AMDGPU_UCODE_ID_SDMA0; + case UCODE_ID_SDMA1: + return AMDGPU_UCODE_ID_SDMA1; + case UCODE_ID_CP_CE: + return AMDGPU_UCODE_ID_CP_CE; + case UCODE_ID_CP_PFP: + return AMDGPU_UCODE_ID_CP_PFP; + case UCODE_ID_CP_ME: + return AMDGPU_UCODE_ID_CP_ME; + case UCODE_ID_CP_MEC: + case UCODE_ID_CP_MEC_JT1: + case UCODE_ID_CP_MEC_JT2: + return AMDGPU_UCODE_ID_CP_MEC1; + case UCODE_ID_RLC_G: + return AMDGPU_UCODE_ID_RLC_G; + default: + DRM_ERROR("ucode type is out of range!\n"); + return AMDGPU_UCODE_ID_MAXIMUM; + } +} + +static int fiji_smu_populate_single_firmware_entry(struct amdgpu_device *adev, + uint32_t fw_type, + struct SMU_Entry *entry) +{ + enum AMDGPU_UCODE_ID id = fiji_convert_fw_type(fw_type); + struct amdgpu_firmware_info *ucode = &adev->firmware.ucode[id]; + const struct gfx_firmware_header_v1_0 *header = NULL; + uint64_t gpu_addr; + uint32_t data_size; + + if (ucode->fw == NULL) + return -EINVAL; + gpu_addr = ucode->mc_addr; + header = (const struct gfx_firmware_header_v1_0 *)ucode->fw->data; + data_size = le32_to_cpu(header->header.ucode_size_bytes); + + if ((fw_type == UCODE_ID_CP_MEC_JT1) || + (fw_type == UCODE_ID_CP_MEC_JT2)) { + gpu_addr += le32_to_cpu(header->jt_offset) << 2; + data_size = le32_to_cpu(header->jt_size) << 2; + } + + entry->version = (uint16_t)le32_to_cpu(header->header.ucode_version); + entry->id = (uint16_t)fw_type; + entry->image_addr_high = upper_32_bits(gpu_addr); + entry->image_addr_low = lower_32_bits(gpu_addr); + entry->meta_data_addr_high = 0; + entry->meta_data_addr_low = 0; + entry->data_size_byte = data_size; + entry->num_register_entries = 0; + + if (fw_type == UCODE_ID_RLC_G) + entry->flags = 1; + else + entry->flags = 0; + + return 0; +} + +static int fiji_smu_request_load_fw(struct amdgpu_device *adev) +{ + struct fiji_smu_private_data *private = (struct fiji_smu_private_data *)adev->smu.priv; + struct SMU_DRAMData_TOC *toc; + uint32_t fw_to_load; + + WREG32_SMC(ixSOFT_REGISTERS_TABLE_28, 0); + + fiji_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SMU_DRAM_ADDR_HI, private->smu_buffer_addr_high); + fiji_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_SMU_DRAM_ADDR_LO, private->smu_buffer_addr_low); + + toc = (struct SMU_DRAMData_TOC *)private->header; + toc->num_entries = 0; + toc->structure_version = 1; + + if (!adev->firmware.smu_load) + return 0; + + if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_RLC_G, + &toc->entry[toc->num_entries++])) { + DRM_ERROR("Failed to get firmware entry for RLC\n"); + return -EINVAL; + } + + if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_CE, + &toc->entry[toc->num_entries++])) { + DRM_ERROR("Failed to get firmware entry for CE\n"); + return -EINVAL; + } + + if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_PFP, + &toc->entry[toc->num_entries++])) { + DRM_ERROR("Failed to get firmware entry for PFP\n"); + return -EINVAL; + } + + if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_ME, + &toc->entry[toc->num_entries++])) { + DRM_ERROR("Failed to get firmware entry for ME\n"); + return -EINVAL; + } + + if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_MEC, + &toc->entry[toc->num_entries++])) { + DRM_ERROR("Failed to get firmware entry for MEC\n"); + return -EINVAL; + } + + if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_MEC_JT1, + &toc->entry[toc->num_entries++])) { + DRM_ERROR("Failed to get firmware entry for MEC_JT1\n"); + return -EINVAL; + } + + if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_CP_MEC_JT2, + &toc->entry[toc->num_entries++])) { + DRM_ERROR("Failed to get firmware entry for MEC_JT2\n"); + return -EINVAL; + } + + if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_SDMA0, + &toc->entry[toc->num_entries++])) { + DRM_ERROR("Failed to get firmware entry for SDMA0\n"); + return -EINVAL; + } + + if (fiji_smu_populate_single_firmware_entry(adev, UCODE_ID_SDMA1, + &toc->entry[toc->num_entries++])) { + DRM_ERROR("Failed to get firmware entry for SDMA1\n"); + return -EINVAL; + } + + fiji_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_DRV_DRAM_ADDR_HI, private->header_addr_high); + fiji_send_msg_to_smc_with_parameter(adev, PPSMC_MSG_DRV_DRAM_ADDR_LO, private->header_addr_low); + + fw_to_load = UCODE_ID_RLC_G_MASK | + UCODE_ID_SDMA0_MASK | + UCODE_ID_SDMA1_MASK | + UCODE_ID_CP_CE_MASK | + UCODE_ID_CP_ME_MASK | + UCODE_ID_CP_PFP_MASK | + UCODE_ID_CP_MEC_MASK; + + if (fiji_send_msg_to_smc_with_parameter_without_waiting(adev, PPSMC_MSG_LoadUcodes, fw_to_load)) { + DRM_ERROR("Fail to request SMU load ucode\n"); + return -EINVAL; + } + + return 0; +} + +static uint32_t fiji_smu_get_mask_for_fw_type(uint32_t fw_type) +{ + switch (fw_type) { + case AMDGPU_UCODE_ID_SDMA0: + return UCODE_ID_SDMA0_MASK; + case AMDGPU_UCODE_ID_SDMA1: + return UCODE_ID_SDMA1_MASK; + case AMDGPU_UCODE_ID_CP_CE: + return UCODE_ID_CP_CE_MASK; + case AMDGPU_UCODE_ID_CP_PFP: + return UCODE_ID_CP_PFP_MASK; + case AMDGPU_UCODE_ID_CP_ME: + return UCODE_ID_CP_ME_MASK; + case AMDGPU_UCODE_ID_CP_MEC1: + return UCODE_ID_CP_MEC_MASK; + case AMDGPU_UCODE_ID_CP_MEC2: + return UCODE_ID_CP_MEC_MASK; + case AMDGPU_UCODE_ID_RLC_G: + return UCODE_ID_RLC_G_MASK; + default: + DRM_ERROR("ucode type is out of range!\n"); + return 0; + } +} + +static int fiji_smu_check_fw_load_finish(struct amdgpu_device *adev, + uint32_t fw_type) +{ + uint32_t fw_mask = fiji_smu_get_mask_for_fw_type(fw_type); + int i; + + for (i = 0; i < adev->usec_timeout; i++) { + if (fw_mask == (RREG32_SMC(ixSOFT_REGISTERS_TABLE_28) & fw_mask)) + break; + udelay(1); + } + + if (i == adev->usec_timeout) { + DRM_ERROR("check firmware loading failed\n"); + return -EINVAL; + } + + return 0; +} + +static int fiji_smu_start_in_protection_mode(struct amdgpu_device *adev) +{ + int result; + uint32_t val; + int i; + + /* Assert reset */ + val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL); + val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 1); + WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val); + + result = fiji_smu_upload_firmware_image(adev); + if (result) + return result; + + /* Clear status */ + WREG32_SMC(ixSMU_STATUS, 0); + + /* Enable clock */ + val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0); + val = REG_SET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 0); + WREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0, val); + + /* De-assert reset */ + val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL); + val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 0); + WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val); + + /* Set SMU Auto Start */ + val = RREG32_SMC(ixSMU_INPUT_DATA); + val = REG_SET_FIELD(val, SMU_INPUT_DATA, AUTO_START, 1); + WREG32_SMC(ixSMU_INPUT_DATA, val); + + /* Clear firmware interrupt enable flag */ + WREG32_SMC(ixFIRMWARE_FLAGS, 0); + + for (i = 0; i < adev->usec_timeout; i++) { + val = RREG32_SMC(ixRCU_UC_EVENTS); + if (REG_GET_FIELD(val, RCU_UC_EVENTS, INTERRUPTS_ENABLED)) + break; + udelay(1); + } + + if (i == adev->usec_timeout) { + DRM_ERROR("Interrupt is not enabled by firmware\n"); + return -EINVAL; + } + + /* Call Test SMU message with 0x20000 offset + * to trigger SMU start + */ + fiji_send_msg_to_smc_offset(adev); + DRM_INFO("[FM]try triger smu start\n"); + /* Wait for done bit to be set */ + for (i = 0; i < adev->usec_timeout; i++) { + val = RREG32_SMC(ixSMU_STATUS); + if (REG_GET_FIELD(val, SMU_STATUS, SMU_DONE)) + break; + udelay(1); + } + + if (i == adev->usec_timeout) { + DRM_ERROR("Timeout for SMU start\n"); + return -EINVAL; + } + + /* Check pass/failed indicator */ + val = RREG32_SMC(ixSMU_STATUS); + if (!REG_GET_FIELD(val, SMU_STATUS, SMU_PASS)) { + DRM_ERROR("SMU Firmware start failed\n"); + return -EINVAL; + } + DRM_INFO("[FM]smu started\n"); + /* Wait for firmware to initialize */ + for (i = 0; i < adev->usec_timeout; i++) { + val = RREG32_SMC(ixFIRMWARE_FLAGS); + if(REG_GET_FIELD(val, FIRMWARE_FLAGS, INTERRUPTS_ENABLED)) + break; + udelay(1); + } + + if (i == adev->usec_timeout) { + DRM_ERROR("SMU firmware initialization failed\n"); + return -EINVAL; + } + DRM_INFO("[FM]smu initialized\n"); + + return 0; +} + +static int fiji_smu_start_in_non_protection_mode(struct amdgpu_device *adev) +{ + int i, result; + uint32_t val; + + /* wait for smc boot up */ + for (i = 0; i < adev->usec_timeout; i++) { + val = RREG32_SMC(ixRCU_UC_EVENTS); + val = REG_GET_FIELD(val, RCU_UC_EVENTS, boot_seq_done); + if (val) + break; + udelay(1); + } + + if (i == adev->usec_timeout) { + DRM_ERROR("SMC boot sequence is not completed\n"); + return -EINVAL; + } + + /* Clear firmware interrupt enable flag */ + WREG32_SMC(ixFIRMWARE_FLAGS, 0); + + /* Assert reset */ + val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL); + val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 1); + WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val); + + result = fiji_smu_upload_firmware_image(adev); + if (result) + return result; + + /* Set smc instruct start point at 0x0 */ + fiji_program_jump_on_start(adev); + + /* Enable clock */ + val = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0); + val = REG_SET_FIELD(val, SMC_SYSCON_CLOCK_CNTL_0, ck_disable, 0); + WREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0, val); + + /* De-assert reset */ + val = RREG32_SMC(ixSMC_SYSCON_RESET_CNTL); + val = REG_SET_FIELD(val, SMC_SYSCON_RESET_CNTL, rst_reg, 0); + WREG32_SMC(ixSMC_SYSCON_RESET_CNTL, val); + + /* Wait for firmware to initialize */ + for (i = 0; i < adev->usec_timeout; i++) { + val = RREG32_SMC(ixFIRMWARE_FLAGS); + if (REG_GET_FIELD(val, FIRMWARE_FLAGS, INTERRUPTS_ENABLED)) + break; + udelay(1); + } + + if (i == adev->usec_timeout) { + DRM_ERROR("Timeout for SMC firmware initialization\n"); + return -EINVAL; + } + + return 0; +} + +int fiji_smu_start(struct amdgpu_device *adev) +{ + int result; + uint32_t val; + + if (!fiji_is_smc_ram_running(adev)) { + val = RREG32_SMC(ixSMU_FIRMWARE); + if (!REG_GET_FIELD(val, SMU_FIRMWARE, SMU_MODE)) { + DRM_INFO("[FM]start smu in nonprotection mode\n"); + result = fiji_smu_start_in_non_protection_mode(adev); + if (result) + return result; + } else { + DRM_INFO("[FM]start smu in protection mode\n"); + result = fiji_smu_start_in_protection_mode(adev); + if (result) + return result; + } + } + + return fiji_smu_request_load_fw(adev); +} + +static const struct amdgpu_smumgr_funcs fiji_smumgr_funcs = { + .check_fw_load_finish = fiji_smu_check_fw_load_finish, + .request_smu_load_fw = NULL, + .request_smu_specific_fw = NULL, +}; + +int fiji_smu_init(struct amdgpu_device *adev) +{ + struct fiji_smu_private_data *private; + uint32_t image_size = ((sizeof(struct SMU_DRAMData_TOC) / 4096) + 1) * 4096; + uint32_t smu_internal_buffer_size = 200*4096; + struct amdgpu_bo **toc_buf = &adev->smu.toc_buf; + struct amdgpu_bo **smu_buf = &adev->smu.smu_buf; + uint64_t mc_addr; + void *toc_buf_ptr; + void *smu_buf_ptr; + int ret; + + private = kzalloc(sizeof(struct fiji_smu_private_data), GFP_KERNEL); + if (NULL == private) + return -ENOMEM; + + /* allocate firmware buffers */ + if (adev->firmware.smu_load) + amdgpu_ucode_init_bo(adev); + + adev->smu.priv = private; + adev->smu.fw_flags = 0; + + /* Allocate FW image data structure and header buffer */ + ret = amdgpu_bo_create(adev, image_size, PAGE_SIZE, + true, AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, + NULL, toc_buf); + if (ret) { + DRM_ERROR("Failed to allocate memory for TOC buffer\n"); + return -ENOMEM; + } + + /* Allocate buffer for SMU internal buffer */ + ret = amdgpu_bo_create(adev, smu_internal_buffer_size, PAGE_SIZE, + true, AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, + NULL, smu_buf); + if (ret) { + DRM_ERROR("Failed to allocate memory for SMU internal buffer\n"); + return -ENOMEM; + } + + /* Retrieve GPU address for header buffer and internal buffer */ + ret = amdgpu_bo_reserve(adev->smu.toc_buf, false); + if (ret) { + amdgpu_bo_unref(&adev->smu.toc_buf); + DRM_ERROR("Failed to reserve the TOC buffer\n"); + return -EINVAL; + } + + ret = amdgpu_bo_pin(adev->smu.toc_buf, AMDGPU_GEM_DOMAIN_VRAM, &mc_addr); + if (ret) { + amdgpu_bo_unreserve(adev->smu.toc_buf); + amdgpu_bo_unref(&adev->smu.toc_buf); + DRM_ERROR("Failed to pin the TOC buffer\n"); + return -EINVAL; + } + + ret = amdgpu_bo_kmap(*toc_buf, &toc_buf_ptr); + if (ret) { + amdgpu_bo_unreserve(adev->smu.toc_buf); + amdgpu_bo_unref(&adev->smu.toc_buf); + DRM_ERROR("Failed to map the TOC buffer\n"); + return -EINVAL; + } + + amdgpu_bo_unreserve(adev->smu.toc_buf); + private->header_addr_low = lower_32_bits(mc_addr); + private->header_addr_high = upper_32_bits(mc_addr); + private->header = toc_buf_ptr; + + ret = amdgpu_bo_reserve(adev->smu.smu_buf, false); + if (ret) { + amdgpu_bo_unref(&adev->smu.smu_buf); + amdgpu_bo_unref(&adev->smu.toc_buf); + DRM_ERROR("Failed to reserve the SMU internal buffer\n"); + return -EINVAL; + } + + ret = amdgpu_bo_pin(adev->smu.smu_buf, AMDGPU_GEM_DOMAIN_VRAM, &mc_addr); + if (ret) { + amdgpu_bo_unreserve(adev->smu.smu_buf); + amdgpu_bo_unref(&adev->smu.smu_buf); + amdgpu_bo_unref(&adev->smu.toc_buf); + DRM_ERROR("Failed to pin the SMU internal buffer\n"); + return -EINVAL; + } + + ret = amdgpu_bo_kmap(*smu_buf, &smu_buf_ptr); + if (ret) { + amdgpu_bo_unreserve(adev->smu.smu_buf); + amdgpu_bo_unref(&adev->smu.smu_buf); + amdgpu_bo_unref(&adev->smu.toc_buf); + DRM_ERROR("Failed to map the SMU internal buffer\n"); + return -EINVAL; + } + + amdgpu_bo_unreserve(adev->smu.smu_buf); + private->smu_buffer_addr_low = lower_32_bits(mc_addr); + private->smu_buffer_addr_high = upper_32_bits(mc_addr); + + adev->smu.smumgr_funcs = &fiji_smumgr_funcs; + + return 0; +} + +int fiji_smu_fini(struct amdgpu_device *adev) +{ + amdgpu_bo_unref(&adev->smu.toc_buf); + amdgpu_bo_unref(&adev->smu.smu_buf); + kfree(adev->smu.priv); + adev->smu.priv = NULL; + if (adev->firmware.fw_buf) + amdgpu_ucode_fini_bo(adev); + + return 0; +} diff --git a/drivers/gpu/drm/amd/amdgpu/fiji_smumgr.h b/drivers/gpu/drm/amd/amdgpu/fiji_smumgr.h new file mode 100644 index 000000000000..1cef03deeac3 --- /dev/null +++ b/drivers/gpu/drm/amd/amdgpu/fiji_smumgr.h @@ -0,0 +1,42 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef FIJI_SMUMGR_H +#define FIJI_SMUMGR_H + +#include "fiji_ppsmc.h" + +int fiji_smu_init(struct amdgpu_device *adev); +int fiji_smu_fini(struct amdgpu_device *adev); +int fiji_smu_start(struct amdgpu_device *adev); + +struct fiji_smu_private_data +{ + uint8_t *header; + uint32_t smu_buffer_addr_high; + uint32_t smu_buffer_addr_low; + uint32_t header_addr_high; + uint32_t header_addr_low; +}; + +#endif diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c index 0d8bf2cb1956..4bd1e5cf65ca 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c @@ -2173,7 +2173,7 @@ static void gfx_v7_0_gpu_init(struct amdgpu_device *adev) adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes; adev->gfx.config.mem_max_burst_length_bytes = 256; - if (adev->flags & AMDGPU_IS_APU) { + if (adev->flags & AMD_IS_APU) { /* Get memory bank mapping mode. */ tmp = RREG32(mmMC_FUS_DRAM0_BANK_ADDR_MAPPING); dimm00_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM0ADDRMAP); @@ -2648,6 +2648,7 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring) { struct amdgpu_device *adev = ring->adev; struct amdgpu_ib ib; + struct fence *f = NULL; uint32_t scratch; uint32_t tmp = 0; unsigned i; @@ -2659,29 +2660,27 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring) return r; } WREG32(scratch, 0xCAFEDEAD); + memset(&ib, 0, sizeof(ib)); r = amdgpu_ib_get(ring, NULL, 256, &ib); if (r) { DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); - amdgpu_gfx_scratch_free(adev, scratch); - return r; + goto err1; } ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1); ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START)); ib.ptr[2] = 0xDEADBEEF; ib.length_dw = 3; - r = amdgpu_ib_schedule(adev, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED); - if (r) { - amdgpu_gfx_scratch_free(adev, scratch); - amdgpu_ib_free(adev, &ib); - DRM_ERROR("amdgpu: failed to schedule ib (%d).\n", r); - return r; - } - r = amdgpu_fence_wait(ib.fence, false); + + r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, &ib, 1, NULL, + AMDGPU_FENCE_OWNER_UNDEFINED, + &f); + if (r) + goto err2; + + r = fence_wait(f, false); if (r) { DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); - amdgpu_gfx_scratch_free(adev, scratch); - amdgpu_ib_free(adev, &ib); - return r; + goto err2; } for (i = 0; i < adev->usec_timeout; i++) { tmp = RREG32(scratch); @@ -2691,14 +2690,19 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring) } if (i < adev->usec_timeout) { DRM_INFO("ib test on ring %d succeeded in %u usecs\n", - ib.fence->ring->idx, i); + ring->idx, i); + goto err2; } else { DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n", scratch, tmp); r = -EINVAL; } - amdgpu_gfx_scratch_free(adev, scratch); + +err2: + fence_put(f); amdgpu_ib_free(adev, &ib); +err1: + amdgpu_gfx_scratch_free(adev, scratch); return r; } @@ -3758,7 +3762,7 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev) int r; /* allocate rlc buffers */ - if (adev->flags & AMDGPU_IS_APU) { + if (adev->flags & AMD_IS_APU) { if (adev->asic_type == CHIP_KAVERI) { adev->gfx.rlc.reg_list = spectre_rlc_save_restore_register_list; adev->gfx.rlc.reg_list_size = @@ -3782,7 +3786,9 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev) /* save restore block */ if (adev->gfx.rlc.save_restore_obj == NULL) { r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true, - AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, &adev->gfx.rlc.save_restore_obj); + AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, + NULL, &adev->gfx.rlc.save_restore_obj); if (r) { dev_warn(adev->dev, "(%d) create RLC sr bo failed\n", r); return r; @@ -3823,7 +3829,9 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev) if (adev->gfx.rlc.clear_state_obj == NULL) { r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true, - AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, &adev->gfx.rlc.clear_state_obj); + AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, + NULL, &adev->gfx.rlc.clear_state_obj); if (r) { dev_warn(adev->dev, "(%d) create RLC c bo failed\n", r); gfx_v7_0_rlc_fini(adev); @@ -3860,7 +3868,9 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev) if (adev->gfx.rlc.cp_table_size) { if (adev->gfx.rlc.cp_table_obj == NULL) { r = amdgpu_bo_create(adev, adev->gfx.rlc.cp_table_size, PAGE_SIZE, true, - AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, &adev->gfx.rlc.cp_table_obj); + AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, + NULL, &adev->gfx.rlc.cp_table_obj); if (r) { dev_warn(adev->dev, "(%d) create RLC cp table bo failed\n", r); gfx_v7_0_rlc_fini(adev); @@ -5594,6 +5604,7 @@ static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_gfx = { .test_ring = gfx_v7_0_ring_test_ring, .test_ib = gfx_v7_0_ring_test_ib, .is_lockup = gfx_v7_0_ring_is_lockup, + .insert_nop = amdgpu_ring_insert_nop, }; static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_compute = { @@ -5610,6 +5621,7 @@ static const struct amdgpu_ring_funcs gfx_v7_0_ring_funcs_compute = { .test_ring = gfx_v7_0_ring_test_ring, .test_ib = gfx_v7_0_ring_test_ib, .is_lockup = gfx_v7_0_ring_is_lockup, + .insert_nop = amdgpu_ring_insert_nop, }; static void gfx_v7_0_set_ring_funcs(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index 20e2cfd521d5..53f07439a512 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -87,6 +87,13 @@ MODULE_FIRMWARE("amdgpu/topaz_mec.bin"); MODULE_FIRMWARE("amdgpu/topaz_mec2.bin"); MODULE_FIRMWARE("amdgpu/topaz_rlc.bin"); +MODULE_FIRMWARE("amdgpu/fiji_ce.bin"); +MODULE_FIRMWARE("amdgpu/fiji_pfp.bin"); +MODULE_FIRMWARE("amdgpu/fiji_me.bin"); +MODULE_FIRMWARE("amdgpu/fiji_mec.bin"); +MODULE_FIRMWARE("amdgpu/fiji_mec2.bin"); +MODULE_FIRMWARE("amdgpu/fiji_rlc.bin"); + static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] = { {mmGDS_VMID0_BASE, mmGDS_VMID0_SIZE, mmGDS_GWS_VMID0, mmGDS_OA_VMID0}, @@ -217,6 +224,71 @@ static const u32 tonga_mgcg_cgcg_init[] = mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001, }; +static const u32 fiji_golden_common_all[] = +{ + mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, + mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a, + mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e, + mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003, + mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, + mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, + mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF, + mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF +}; + +static const u32 golden_settings_fiji_a10[] = +{ + mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040, + mmDB_DEBUG2, 0xf00fffff, 0x00000400, + mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, + mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x00000100, + mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, + mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, + mmTCC_CTRL, 0x00100000, 0xf30fff7f, + mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff, + mmTCP_CHAN_STEER_HI, 0xffffffff, 0x7d6cf5e4, + mmTCP_CHAN_STEER_LO, 0xffffffff, 0x3928b1a0, +}; + +static const u32 fiji_mgcg_cgcg_init[] = +{ + mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffc0, + mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, + mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, + mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100, + mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100, + mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100, + mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100, + mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100, + mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100, + mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100, + mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100, + mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100, + mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100, + mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100, + mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100, + mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100, + mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100, + mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100, + mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100, + mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100, + mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100, + mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100, + mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100, + mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100, + mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100, + mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100, + mmTA_CGTT_CTRL, 0xffffffff, 0x00000100, + mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, + mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, + mmTD_CGTT_CTRL, 0xffffffff, 0x00000100, + mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, + mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200, + mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100, + mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c, + mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001, +}; + static const u32 golden_settings_iceland_a11[] = { mmCB_HW_CONTROL_3, 0x00000040, 0x00000040, @@ -439,6 +511,18 @@ static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev) iceland_golden_common_all, (const u32)ARRAY_SIZE(iceland_golden_common_all)); break; + case CHIP_FIJI: + amdgpu_program_register_sequence(adev, + fiji_mgcg_cgcg_init, + (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init)); + amdgpu_program_register_sequence(adev, + golden_settings_fiji_a10, + (const u32)ARRAY_SIZE(golden_settings_fiji_a10)); + amdgpu_program_register_sequence(adev, + fiji_golden_common_all, + (const u32)ARRAY_SIZE(fiji_golden_common_all)); + break; + case CHIP_TONGA: amdgpu_program_register_sequence(adev, tonga_mgcg_cgcg_init, @@ -526,6 +610,7 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring) { struct amdgpu_device *adev = ring->adev; struct amdgpu_ib ib; + struct fence *f = NULL; uint32_t scratch; uint32_t tmp = 0; unsigned i; @@ -537,29 +622,27 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring) return r; } WREG32(scratch, 0xCAFEDEAD); + memset(&ib, 0, sizeof(ib)); r = amdgpu_ib_get(ring, NULL, 256, &ib); if (r) { DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); - amdgpu_gfx_scratch_free(adev, scratch); - return r; + goto err1; } ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1); ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START)); ib.ptr[2] = 0xDEADBEEF; ib.length_dw = 3; - r = amdgpu_ib_schedule(adev, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED); - if (r) { - amdgpu_gfx_scratch_free(adev, scratch); - amdgpu_ib_free(adev, &ib); - DRM_ERROR("amdgpu: failed to schedule ib (%d).\n", r); - return r; - } - r = amdgpu_fence_wait(ib.fence, false); + + r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, &ib, 1, NULL, + AMDGPU_FENCE_OWNER_UNDEFINED, + &f); + if (r) + goto err2; + + r = fence_wait(f, false); if (r) { DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); - amdgpu_gfx_scratch_free(adev, scratch); - amdgpu_ib_free(adev, &ib); - return r; + goto err2; } for (i = 0; i < adev->usec_timeout; i++) { tmp = RREG32(scratch); @@ -569,14 +652,18 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring) } if (i < adev->usec_timeout) { DRM_INFO("ib test on ring %d succeeded in %u usecs\n", - ib.fence->ring->idx, i); + ring->idx, i); + goto err2; } else { DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n", scratch, tmp); r = -EINVAL; } - amdgpu_gfx_scratch_free(adev, scratch); +err2: + fence_put(f); amdgpu_ib_free(adev, &ib); +err1: + amdgpu_gfx_scratch_free(adev, scratch); return r; } @@ -601,6 +688,9 @@ static int gfx_v8_0_init_microcode(struct amdgpu_device *adev) case CHIP_CARRIZO: chip_name = "carrizo"; break; + case CHIP_FIJI: + chip_name = "fiji"; + break; default: BUG(); } @@ -1236,6 +1326,7 @@ static void gfx_v8_0_tiling_mode_table_init(struct amdgpu_device *adev) adev->gfx.config.macrotile_mode_array[reg_offset] = gb_tile_moden; WREG32(mmGB_MACROTILE_MODE0 + reg_offset, gb_tile_moden); } + case CHIP_FIJI: case CHIP_TONGA: for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { switch (reg_offset) { @@ -1914,7 +2005,7 @@ static void gfx_v8_0_setup_rb(struct amdgpu_device *adev, } /** - * gmc_v8_0_init_compute_vmid - gart enable + * gfx_v8_0_init_compute_vmid - gart enable * * @rdev: amdgpu_device pointer * @@ -1924,7 +2015,7 @@ static void gfx_v8_0_setup_rb(struct amdgpu_device *adev, #define DEFAULT_SH_MEM_BASES (0x6000) #define FIRST_COMPUTE_VMID (8) #define LAST_COMPUTE_VMID (16) -static void gmc_v8_0_init_compute_vmid(struct amdgpu_device *adev) +static void gfx_v8_0_init_compute_vmid(struct amdgpu_device *adev) { int i; uint32_t sh_mem_config; @@ -1984,6 +2075,23 @@ static void gfx_v8_0_gpu_init(struct amdgpu_device *adev) adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; gb_addr_config = TOPAZ_GB_ADDR_CONFIG_GOLDEN; break; + case CHIP_FIJI: + adev->gfx.config.max_shader_engines = 4; + adev->gfx.config.max_tile_pipes = 16; + adev->gfx.config.max_cu_per_sh = 16; + adev->gfx.config.max_sh_per_se = 1; + adev->gfx.config.max_backends_per_se = 4; + adev->gfx.config.max_texture_channel_caches = 8; + adev->gfx.config.max_gprs = 256; + adev->gfx.config.max_gs_threads = 32; + adev->gfx.config.max_hw_contexts = 8; + + adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; + adev->gfx.config.sc_prim_fifo_size_backend = 0x100; + adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; + adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; + gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN; + break; case CHIP_TONGA: adev->gfx.config.max_shader_engines = 4; adev->gfx.config.max_tile_pipes = 8; @@ -2078,7 +2186,7 @@ static void gfx_v8_0_gpu_init(struct amdgpu_device *adev) adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes; adev->gfx.config.mem_max_burst_length_bytes = 256; - if (adev->flags & AMDGPU_IS_APU) { + if (adev->flags & AMD_IS_APU) { /* Get memory bank mapping mode. */ tmp = RREG32(mmMC_FUS_DRAM0_BANK_ADDR_MAPPING); dimm00_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM0ADDRMAP); @@ -2174,7 +2282,7 @@ static void gfx_v8_0_gpu_init(struct amdgpu_device *adev) vi_srbm_select(adev, 0, 0, 0, 0); mutex_unlock(&adev->srbm_mutex); - gmc_v8_0_init_compute_vmid(adev); + gfx_v8_0_init_compute_vmid(adev); mutex_lock(&adev->grbm_idx_mutex); /* @@ -2490,6 +2598,7 @@ static int gfx_v8_0_cp_gfx_start(struct amdgpu_device *adev) amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); switch (adev->asic_type) { case CHIP_TONGA: + case CHIP_FIJI: amdgpu_ring_write(ring, 0x16000012); amdgpu_ring_write(ring, 0x0000002A); break; @@ -3131,7 +3240,8 @@ static int gfx_v8_0_cp_compute_resume(struct amdgpu_device *adev) /* enable the doorbell if requested */ if (use_doorbell) { - if (adev->asic_type == CHIP_CARRIZO) { + if ((adev->asic_type == CHIP_CARRIZO) || + (adev->asic_type == CHIP_FIJI)) { WREG32(mmCP_MEC_DOORBELL_RANGE_LOWER, AMDGPU_DOORBELL_KIQ << 2); WREG32(mmCP_MEC_DOORBELL_RANGE_UPPER, @@ -3875,7 +3985,8 @@ static bool gfx_v8_0_ring_emit_semaphore(struct amdgpu_ring *ring, unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL; if (ring->adev->asic_type == CHIP_TOPAZ || - ring->adev->asic_type == CHIP_TONGA) + ring->adev->asic_type == CHIP_TONGA || + ring->adev->asic_type == CHIP_FIJI) /* we got a hw semaphore bug in VI TONGA, return false to switch back to sw fence wait */ return false; else { @@ -4268,6 +4379,7 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = { .test_ring = gfx_v8_0_ring_test_ring, .test_ib = gfx_v8_0_ring_test_ib, .is_lockup = gfx_v8_0_ring_is_lockup, + .insert_nop = amdgpu_ring_insert_nop, }; static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = { @@ -4284,6 +4396,7 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = { .test_ring = gfx_v8_0_ring_test_ring, .test_ib = gfx_v8_0_ring_test_ib, .is_lockup = gfx_v8_0_ring_is_lockup, + .insert_nop = amdgpu_ring_insert_nop, }; static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c index ae37fce36520..774528ab8704 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c @@ -523,17 +523,11 @@ static int gmc_v7_0_gart_enable(struct amdgpu_device *adev) tmp = RREG32(mmVM_CONTEXT1_CNTL); tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1); tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 1); - tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT, 1); tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); - tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT, 1); tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); - tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT, 1); tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1); - tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, VALID_PROTECTION_FAULT_ENABLE_INTERRUPT, 1); tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1); - tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, READ_PROTECTION_FAULT_ENABLE_INTERRUPT, 1); tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1); - tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT, 1); tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE, amdgpu_vm_block_size - 9); @@ -636,7 +630,7 @@ static int gmc_v7_0_vm_init(struct amdgpu_device *adev) adev->vm_manager.nvm = AMDGPU_NUM_OF_VMIDS; /* base offset of vram pages */ - if (adev->flags & AMDGPU_IS_APU) { + if (adev->flags & AMD_IS_APU) { u64 tmp = RREG32(mmMC_VM_FB_OFFSET); tmp <<= 22; adev->vm_manager.vram_base_offset = tmp; @@ -841,7 +835,7 @@ static int gmc_v7_0_early_init(void *handle) gmc_v7_0_set_gart_funcs(adev); gmc_v7_0_set_irq_funcs(adev); - if (adev->flags & AMDGPU_IS_APU) { + if (adev->flags & AMD_IS_APU) { adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN; } else { u32 tmp = RREG32(mmMC_SEQ_MISC0); @@ -852,6 +846,13 @@ static int gmc_v7_0_early_init(void *handle) return 0; } +static int gmc_v7_0_late_init(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0); +} + static int gmc_v7_0_sw_init(void *handle) { int r; @@ -957,7 +958,7 @@ static int gmc_v7_0_hw_init(void *handle) gmc_v7_0_mc_program(adev); - if (!(adev->flags & AMDGPU_IS_APU)) { + if (!(adev->flags & AMD_IS_APU)) { r = gmc_v7_0_mc_load_microcode(adev); if (r) { DRM_ERROR("Failed to load MC firmware!\n"); @@ -976,6 +977,7 @@ static int gmc_v7_0_hw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + amdgpu_irq_put(adev, &adev->mc.vm_fault, 0); gmc_v7_0_gart_disable(adev); return 0; @@ -1172,7 +1174,7 @@ static int gmc_v7_0_soft_reset(void *handle) if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK | SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) { - if (!(adev->flags & AMDGPU_IS_APU)) + if (!(adev->flags & AMD_IS_APU)) srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_MC, 1); } @@ -1282,7 +1284,7 @@ static int gmc_v7_0_set_clockgating_state(void *handle, if (state == AMD_CG_STATE_GATE) gate = true; - if (!(adev->flags & AMDGPU_IS_APU)) { + if (!(adev->flags & AMD_IS_APU)) { gmc_v7_0_enable_mc_mgcg(adev, gate); gmc_v7_0_enable_mc_ls(adev, gate); } @@ -1301,7 +1303,7 @@ static int gmc_v7_0_set_powergating_state(void *handle, const struct amd_ip_funcs gmc_v7_0_ip_funcs = { .early_init = gmc_v7_0_early_init, - .late_init = NULL, + .late_init = gmc_v7_0_late_init, .sw_init = gmc_v7_0_sw_init, .sw_fini = gmc_v7_0_sw_fini, .hw_init = gmc_v7_0_hw_init, diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c index 8135963a66be..9a07742620d0 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c @@ -44,6 +44,7 @@ static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev); MODULE_FIRMWARE("amdgpu/topaz_mc.bin"); MODULE_FIRMWARE("amdgpu/tonga_mc.bin"); +MODULE_FIRMWARE("amdgpu/fiji_mc.bin"); static const u32 golden_settings_tonga_a11[] = { @@ -61,6 +62,19 @@ static const u32 tonga_mgcg_cgcg_init[] = mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104 }; +static const u32 golden_settings_fiji_a10[] = +{ + mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff, + mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff, + mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff, + mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff, +}; + +static const u32 fiji_mgcg_cgcg_init[] = +{ + mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104 +}; + static const u32 golden_settings_iceland_a11[] = { mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff, @@ -90,6 +104,14 @@ static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev) golden_settings_iceland_a11, (const u32)ARRAY_SIZE(golden_settings_iceland_a11)); break; + case CHIP_FIJI: + amdgpu_program_register_sequence(adev, + fiji_mgcg_cgcg_init, + (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init)); + amdgpu_program_register_sequence(adev, + golden_settings_fiji_a10, + (const u32)ARRAY_SIZE(golden_settings_fiji_a10)); + break; case CHIP_TONGA: amdgpu_program_register_sequence(adev, tonga_mgcg_cgcg_init, @@ -202,6 +224,9 @@ static int gmc_v8_0_init_microcode(struct amdgpu_device *adev) case CHIP_TONGA: chip_name = "tonga"; break; + case CHIP_FIJI: + chip_name = "fiji"; + break; case CHIP_CARRIZO: return 0; default: BUG(); @@ -628,19 +653,12 @@ static int gmc_v8_0_gart_enable(struct amdgpu_device *adev) tmp = RREG32(mmVM_CONTEXT1_CNTL); tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1); tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 1); - tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT, 1); tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); - tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT, 1); tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); - tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT, 1); tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1); - tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, VALID_PROTECTION_FAULT_ENABLE_INTERRUPT, 1); tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1); - tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, READ_PROTECTION_FAULT_ENABLE_INTERRUPT, 1); tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1); - tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT, 1); tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); - tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT, 1); tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE, amdgpu_vm_block_size - 9); @@ -737,7 +755,7 @@ static int gmc_v8_0_vm_init(struct amdgpu_device *adev) adev->vm_manager.nvm = AMDGPU_NUM_OF_VMIDS; /* base offset of vram pages */ - if (adev->flags & AMDGPU_IS_APU) { + if (adev->flags & AMD_IS_APU) { u64 tmp = RREG32(mmMC_VM_FB_OFFSET); tmp <<= 22; adev->vm_manager.vram_base_offset = tmp; @@ -816,7 +834,7 @@ static int gmc_v8_0_early_init(void *handle) gmc_v8_0_set_gart_funcs(adev); gmc_v8_0_set_irq_funcs(adev); - if (adev->flags & AMDGPU_IS_APU) { + if (adev->flags & AMD_IS_APU) { adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN; } else { u32 tmp = RREG32(mmMC_SEQ_MISC0); @@ -827,6 +845,13 @@ static int gmc_v8_0_early_init(void *handle) return 0; } +static int gmc_v8_0_late_init(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + + return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0); +} + static int gmc_v8_0_sw_init(void *handle) { int r; @@ -934,7 +959,7 @@ static int gmc_v8_0_hw_init(void *handle) gmc_v8_0_mc_program(adev); - if (!(adev->flags & AMDGPU_IS_APU)) { + if (!(adev->flags & AMD_IS_APU)) { r = gmc_v8_0_mc_load_microcode(adev); if (r) { DRM_ERROR("Failed to load MC firmware!\n"); @@ -953,6 +978,7 @@ static int gmc_v8_0_hw_fini(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + amdgpu_irq_put(adev, &adev->mc.vm_fault, 0); gmc_v8_0_gart_disable(adev); return 0; @@ -1147,7 +1173,7 @@ static int gmc_v8_0_soft_reset(void *handle) if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK | SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) { - if (!(adev->flags & AMDGPU_IS_APU)) + if (!(adev->flags & AMD_IS_APU)) srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_MC, 1); } @@ -1263,7 +1289,7 @@ static int gmc_v8_0_set_powergating_state(void *handle, const struct amd_ip_funcs gmc_v8_0_ip_funcs = { .early_init = gmc_v8_0_early_init, - .late_init = NULL, + .late_init = gmc_v8_0_late_init, .sw_init = gmc_v8_0_sw_init, .sw_fini = gmc_v8_0_sw_fini, .hw_init = gmc_v8_0_hw_init, diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_sdma_pkt_open.h b/drivers/gpu/drm/amd/amdgpu/iceland_sdma_pkt_open.h index c723602c7b0c..ee6a041cb288 100644 --- a/drivers/gpu/drm/amd/amdgpu/iceland_sdma_pkt_open.h +++ b/drivers/gpu/drm/amd/amdgpu/iceland_sdma_pkt_open.h @@ -2163,5 +2163,10 @@ #define SDMA_PKT_NOP_HEADER_sub_op_shift 8 #define SDMA_PKT_NOP_HEADER_SUB_OP(x) (((x) & SDMA_PKT_NOP_HEADER_sub_op_mask) << SDMA_PKT_NOP_HEADER_sub_op_shift) +/*define for count field*/ +#define SDMA_PKT_NOP_HEADER_count_offset 0 +#define SDMA_PKT_NOP_HEADER_count_mask 0x00003FFF +#define SDMA_PKT_NOP_HEADER_count_shift 16 +#define SDMA_PKT_NOP_HEADER_COUNT(x) (((x) & SDMA_PKT_NOP_HEADER_count_mask) << SDMA_PKT_NOP_HEADER_count_shift) #endif /* __ICELAND_SDMA_PKT_OPEN_H_ */ diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_smc.c b/drivers/gpu/drm/amd/amdgpu/iceland_smc.c index c6f1e2f12b5f..c900aa942ade 100644 --- a/drivers/gpu/drm/amd/amdgpu/iceland_smc.c +++ b/drivers/gpu/drm/amd/amdgpu/iceland_smc.c @@ -623,7 +623,9 @@ int iceland_smu_init(struct amdgpu_device *adev) /* Allocate FW image data structure and header buffer */ ret = amdgpu_bo_create(adev, image_size, PAGE_SIZE, - true, AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, toc_buf); + true, AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, + NULL, toc_buf); if (ret) { DRM_ERROR("Failed to allocate memory for TOC buffer\n"); return -ENOMEM; diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c index a988dfb1d394..14e87234171a 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c @@ -146,6 +146,8 @@ static int sdma_v2_4_init_microcode(struct amdgpu_device *adev) hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma[i].fw->data; adev->sdma[i].fw_version = le32_to_cpu(hdr->header.ucode_version); adev->sdma[i].feature_version = le32_to_cpu(hdr->ucode_feature_version); + if (adev->sdma[i].feature_version >= 20) + adev->sdma[i].burst_nop = true; if (adev->firmware.smu_load) { info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i]; @@ -218,6 +220,19 @@ static void sdma_v2_4_ring_set_wptr(struct amdgpu_ring *ring) WREG32(mmSDMA0_GFX_RB_WPTR + sdma_offsets[me], ring->wptr << 2); } +static void sdma_v2_4_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) +{ + struct amdgpu_sdma *sdma = amdgpu_get_sdma_instance(ring); + int i; + + for (i = 0; i < count; i++) + if (sdma && sdma->burst_nop && (i == 0)) + amdgpu_ring_write(ring, ring->nop | + SDMA_PKT_NOP_HEADER_COUNT(count - 1)); + else + amdgpu_ring_write(ring, ring->nop); +} + /** * sdma_v2_4_ring_emit_ib - Schedule an IB on the DMA engine * @@ -245,8 +260,8 @@ static void sdma_v2_4_ring_emit_ib(struct amdgpu_ring *ring, amdgpu_ring_write(ring, next_rptr); /* IB packet must end on a 8 DW boundary */ - while ((ring->wptr & 7) != 2) - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_NOP)); + sdma_v2_4_ring_insert_nop(ring, (10 - (ring->wptr & 7)) % 8); + amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) | SDMA_PKT_INDIRECT_HEADER_VMID(vmid)); /* base must be 32 byte aligned */ @@ -673,6 +688,7 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring) { struct amdgpu_device *adev = ring->adev; struct amdgpu_ib ib; + struct fence *f = NULL; unsigned i; unsigned index; int r; @@ -688,12 +704,11 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring) gpu_addr = adev->wb.gpu_addr + (index * 4); tmp = 0xCAFEDEAD; adev->wb.wb[index] = cpu_to_le32(tmp); - + memset(&ib, 0, sizeof(ib)); r = amdgpu_ib_get(ring, NULL, 256, &ib); if (r) { - amdgpu_wb_free(adev, index); DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); - return r; + goto err0; } ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) | @@ -707,19 +722,16 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring) ib.ptr[7] = SDMA_PKT_HEADER_OP(SDMA_OP_NOP); ib.length_dw = 8; - r = amdgpu_ib_schedule(adev, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED); + r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, &ib, 1, NULL, + AMDGPU_FENCE_OWNER_UNDEFINED, + &f); + if (r) + goto err1; + + r = fence_wait(f, false); if (r) { - amdgpu_ib_free(adev, &ib); - amdgpu_wb_free(adev, index); - DRM_ERROR("amdgpu: failed to schedule ib (%d).\n", r); - return r; - } - r = amdgpu_fence_wait(ib.fence, false); - if (r) { - amdgpu_ib_free(adev, &ib); - amdgpu_wb_free(adev, index); DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); - return r; + goto err1; } for (i = 0; i < adev->usec_timeout; i++) { tmp = le32_to_cpu(adev->wb.wb[index]); @@ -729,12 +741,17 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring) } if (i < adev->usec_timeout) { DRM_INFO("ib test on ring %d succeeded in %u usecs\n", - ib.fence->ring->idx, i); + ring->idx, i); + goto err1; } else { DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp); r = -EINVAL; } + +err1: + fence_put(f); amdgpu_ib_free(adev, &ib); +err0: amdgpu_wb_free(adev, index); return r; } @@ -877,8 +894,19 @@ static void sdma_v2_4_vm_set_pte_pde(struct amdgpu_ib *ib, */ static void sdma_v2_4_vm_pad_ib(struct amdgpu_ib *ib) { - while (ib->length_dw & 0x7) - ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_NOP); + struct amdgpu_sdma *sdma = amdgpu_get_sdma_instance(ib->ring); + u32 pad_count; + int i; + + pad_count = (8 - (ib->length_dw & 0x7)) % 8; + for (i = 0; i < pad_count; i++) + if (sdma && sdma->burst_nop && (i == 0)) + ib->ptr[ib->length_dw++] = + SDMA_PKT_HEADER_OP(SDMA_OP_NOP) | + SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1); + else + ib->ptr[ib->length_dw++] = + SDMA_PKT_HEADER_OP(SDMA_OP_NOP); } /** @@ -1312,6 +1340,7 @@ static const struct amdgpu_ring_funcs sdma_v2_4_ring_funcs = { .test_ring = sdma_v2_4_ring_test_ring, .test_ib = sdma_v2_4_ring_test_ib, .is_lockup = sdma_v2_4_ring_is_lockup, + .insert_nop = sdma_v2_4_ring_insert_nop, }; static void sdma_v2_4_set_ring_funcs(struct amdgpu_device *adev) @@ -1348,19 +1377,19 @@ static void sdma_v2_4_set_irq_funcs(struct amdgpu_device *adev) * Used by the amdgpu ttm implementation to move pages if * registered as the asic copy callback. */ -static void sdma_v2_4_emit_copy_buffer(struct amdgpu_ring *ring, +static void sdma_v2_4_emit_copy_buffer(struct amdgpu_ib *ib, uint64_t src_offset, uint64_t dst_offset, uint32_t byte_count) { - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_COPY) | - SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR)); - amdgpu_ring_write(ring, byte_count); - amdgpu_ring_write(ring, 0); /* src/dst endian swap */ - amdgpu_ring_write(ring, lower_32_bits(src_offset)); - amdgpu_ring_write(ring, upper_32_bits(src_offset)); - amdgpu_ring_write(ring, lower_32_bits(dst_offset)); - amdgpu_ring_write(ring, upper_32_bits(dst_offset)); + ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) | + SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR); + ib->ptr[ib->length_dw++] = byte_count; + ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */ + ib->ptr[ib->length_dw++] = lower_32_bits(src_offset); + ib->ptr[ib->length_dw++] = upper_32_bits(src_offset); + ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); + ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset); } /** @@ -1373,16 +1402,16 @@ static void sdma_v2_4_emit_copy_buffer(struct amdgpu_ring *ring, * * Fill GPU buffers using the DMA engine (VI). */ -static void sdma_v2_4_emit_fill_buffer(struct amdgpu_ring *ring, +static void sdma_v2_4_emit_fill_buffer(struct amdgpu_ib *ib, uint32_t src_data, uint64_t dst_offset, uint32_t byte_count) { - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL)); - amdgpu_ring_write(ring, lower_32_bits(dst_offset)); - amdgpu_ring_write(ring, upper_32_bits(dst_offset)); - amdgpu_ring_write(ring, src_data); - amdgpu_ring_write(ring, byte_count); + ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL); + ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); + ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset); + ib->ptr[ib->length_dw++] = src_data; + ib->ptr[ib->length_dw++] = byte_count; } static const struct amdgpu_buffer_funcs sdma_v2_4_buffer_funcs = { @@ -1415,5 +1444,6 @@ static void sdma_v2_4_set_vm_pte_funcs(struct amdgpu_device *adev) if (adev->vm_manager.vm_pte_funcs == NULL) { adev->vm_manager.vm_pte_funcs = &sdma_v2_4_vm_pte_funcs; adev->vm_manager.vm_pte_funcs_ring = &adev->sdma[0].ring; + adev->vm_manager.vm_pte_funcs_ring->is_pte_ring = true; } } diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c index 2b86569b18d3..9bfe92df15f7 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c @@ -53,6 +53,8 @@ MODULE_FIRMWARE("amdgpu/tonga_sdma.bin"); MODULE_FIRMWARE("amdgpu/tonga_sdma1.bin"); MODULE_FIRMWARE("amdgpu/carrizo_sdma.bin"); MODULE_FIRMWARE("amdgpu/carrizo_sdma1.bin"); +MODULE_FIRMWARE("amdgpu/fiji_sdma.bin"); +MODULE_FIRMWARE("amdgpu/fiji_sdma1.bin"); static const u32 sdma_offsets[SDMA_MAX_INSTANCE] = { @@ -80,6 +82,24 @@ static const u32 tonga_mgcg_cgcg_init[] = mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100 }; +static const u32 golden_settings_fiji_a10[] = +{ + mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007, + mmSDMA0_GFX_IB_CNTL, 0x800f0111, 0x00000100, + mmSDMA0_RLC0_IB_CNTL, 0x800f0111, 0x00000100, + mmSDMA0_RLC1_IB_CNTL, 0x800f0111, 0x00000100, + mmSDMA1_CHICKEN_BITS, 0xfc910007, 0x00810007, + mmSDMA1_GFX_IB_CNTL, 0x800f0111, 0x00000100, + mmSDMA1_RLC0_IB_CNTL, 0x800f0111, 0x00000100, + mmSDMA1_RLC1_IB_CNTL, 0x800f0111, 0x00000100, +}; + +static const u32 fiji_mgcg_cgcg_init[] = +{ + mmSDMA0_CLK_CTRL, 0xff000ff0, 0x00000100, + mmSDMA1_CLK_CTRL, 0xff000ff0, 0x00000100 +}; + static const u32 cz_golden_settings_a11[] = { mmSDMA0_CHICKEN_BITS, 0xfc910007, 0x00810007, @@ -122,6 +142,14 @@ static const u32 cz_mgcg_cgcg_init[] = static void sdma_v3_0_init_golden_registers(struct amdgpu_device *adev) { switch (adev->asic_type) { + case CHIP_FIJI: + amdgpu_program_register_sequence(adev, + fiji_mgcg_cgcg_init, + (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init)); + amdgpu_program_register_sequence(adev, + golden_settings_fiji_a10, + (const u32)ARRAY_SIZE(golden_settings_fiji_a10)); + break; case CHIP_TONGA: amdgpu_program_register_sequence(adev, tonga_mgcg_cgcg_init, @@ -167,6 +195,9 @@ static int sdma_v3_0_init_microcode(struct amdgpu_device *adev) case CHIP_TONGA: chip_name = "tonga"; break; + case CHIP_FIJI: + chip_name = "fiji"; + break; case CHIP_CARRIZO: chip_name = "carrizo"; break; @@ -187,6 +218,8 @@ static int sdma_v3_0_init_microcode(struct amdgpu_device *adev) hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma[i].fw->data; adev->sdma[i].fw_version = le32_to_cpu(hdr->header.ucode_version); adev->sdma[i].feature_version = le32_to_cpu(hdr->ucode_feature_version); + if (adev->sdma[i].feature_version >= 20) + adev->sdma[i].burst_nop = true; if (adev->firmware.smu_load) { info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i]; @@ -273,6 +306,19 @@ static void sdma_v3_0_ring_set_wptr(struct amdgpu_ring *ring) } } +static void sdma_v3_0_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) +{ + struct amdgpu_sdma *sdma = amdgpu_get_sdma_instance(ring); + int i; + + for (i = 0; i < count; i++) + if (sdma && sdma->burst_nop && (i == 0)) + amdgpu_ring_write(ring, ring->nop | + SDMA_PKT_NOP_HEADER_COUNT(count - 1)); + else + amdgpu_ring_write(ring, ring->nop); +} + /** * sdma_v3_0_ring_emit_ib - Schedule an IB on the DMA engine * @@ -299,8 +345,7 @@ static void sdma_v3_0_ring_emit_ib(struct amdgpu_ring *ring, amdgpu_ring_write(ring, next_rptr); /* IB packet must end on a 8 DW boundary */ - while ((ring->wptr & 7) != 2) - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_NOP)); + sdma_v3_0_ring_insert_nop(ring, (10 - (ring->wptr & 7)) % 8); amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_INDIRECT) | SDMA_PKT_INDIRECT_HEADER_VMID(vmid)); @@ -763,6 +808,7 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring) { struct amdgpu_device *adev = ring->adev; struct amdgpu_ib ib; + struct fence *f = NULL; unsigned i; unsigned index; int r; @@ -778,12 +824,11 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring) gpu_addr = adev->wb.gpu_addr + (index * 4); tmp = 0xCAFEDEAD; adev->wb.wb[index] = cpu_to_le32(tmp); - + memset(&ib, 0, sizeof(ib)); r = amdgpu_ib_get(ring, NULL, 256, &ib); if (r) { - amdgpu_wb_free(adev, index); DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); - return r; + goto err0; } ib.ptr[0] = SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) | @@ -797,19 +842,16 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring) ib.ptr[7] = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP); ib.length_dw = 8; - r = amdgpu_ib_schedule(adev, 1, &ib, AMDGPU_FENCE_OWNER_UNDEFINED); + r = amdgpu_sched_ib_submit_kernel_helper(adev, ring, &ib, 1, NULL, + AMDGPU_FENCE_OWNER_UNDEFINED, + &f); + if (r) + goto err1; + + r = fence_wait(f, false); if (r) { - amdgpu_ib_free(adev, &ib); - amdgpu_wb_free(adev, index); - DRM_ERROR("amdgpu: failed to schedule ib (%d).\n", r); - return r; - } - r = amdgpu_fence_wait(ib.fence, false); - if (r) { - amdgpu_ib_free(adev, &ib); - amdgpu_wb_free(adev, index); DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); - return r; + goto err1; } for (i = 0; i < adev->usec_timeout; i++) { tmp = le32_to_cpu(adev->wb.wb[index]); @@ -819,12 +861,16 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring) } if (i < adev->usec_timeout) { DRM_INFO("ib test on ring %d succeeded in %u usecs\n", - ib.fence->ring->idx, i); + ring->idx, i); + goto err1; } else { DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp); r = -EINVAL; } +err1: + fence_put(f); amdgpu_ib_free(adev, &ib); +err0: amdgpu_wb_free(adev, index); return r; } @@ -967,8 +1013,19 @@ static void sdma_v3_0_vm_set_pte_pde(struct amdgpu_ib *ib, */ static void sdma_v3_0_vm_pad_ib(struct amdgpu_ib *ib) { - while (ib->length_dw & 0x7) - ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_NOP); + struct amdgpu_sdma *sdma = amdgpu_get_sdma_instance(ib->ring); + u32 pad_count; + int i; + + pad_count = (8 - (ib->length_dw & 0x7)) % 8; + for (i = 0; i < pad_count; i++) + if (sdma && sdma->burst_nop && (i == 0)) + ib->ptr[ib->length_dw++] = + SDMA_PKT_HEADER_OP(SDMA_OP_NOP) | + SDMA_PKT_NOP_HEADER_COUNT(pad_count - 1); + else + ib->ptr[ib->length_dw++] = + SDMA_PKT_HEADER_OP(SDMA_OP_NOP); } /** @@ -1406,6 +1463,7 @@ static const struct amdgpu_ring_funcs sdma_v3_0_ring_funcs = { .test_ring = sdma_v3_0_ring_test_ring, .test_ib = sdma_v3_0_ring_test_ib, .is_lockup = sdma_v3_0_ring_is_lockup, + .insert_nop = sdma_v3_0_ring_insert_nop, }; static void sdma_v3_0_set_ring_funcs(struct amdgpu_device *adev) @@ -1442,19 +1500,19 @@ static void sdma_v3_0_set_irq_funcs(struct amdgpu_device *adev) * Used by the amdgpu ttm implementation to move pages if * registered as the asic copy callback. */ -static void sdma_v3_0_emit_copy_buffer(struct amdgpu_ring *ring, +static void sdma_v3_0_emit_copy_buffer(struct amdgpu_ib *ib, uint64_t src_offset, uint64_t dst_offset, uint32_t byte_count) { - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_COPY) | - SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR)); - amdgpu_ring_write(ring, byte_count); - amdgpu_ring_write(ring, 0); /* src/dst endian swap */ - amdgpu_ring_write(ring, lower_32_bits(src_offset)); - amdgpu_ring_write(ring, upper_32_bits(src_offset)); - amdgpu_ring_write(ring, lower_32_bits(dst_offset)); - amdgpu_ring_write(ring, upper_32_bits(dst_offset)); + ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_COPY) | + SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_COPY_LINEAR); + ib->ptr[ib->length_dw++] = byte_count; + ib->ptr[ib->length_dw++] = 0; /* src/dst endian swap */ + ib->ptr[ib->length_dw++] = lower_32_bits(src_offset); + ib->ptr[ib->length_dw++] = upper_32_bits(src_offset); + ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); + ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset); } /** @@ -1467,16 +1525,16 @@ static void sdma_v3_0_emit_copy_buffer(struct amdgpu_ring *ring, * * Fill GPU buffers using the DMA engine (VI). */ -static void sdma_v3_0_emit_fill_buffer(struct amdgpu_ring *ring, +static void sdma_v3_0_emit_fill_buffer(struct amdgpu_ib *ib, uint32_t src_data, uint64_t dst_offset, uint32_t byte_count) { - amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL)); - amdgpu_ring_write(ring, lower_32_bits(dst_offset)); - amdgpu_ring_write(ring, upper_32_bits(dst_offset)); - amdgpu_ring_write(ring, src_data); - amdgpu_ring_write(ring, byte_count); + ib->ptr[ib->length_dw++] = SDMA_PKT_HEADER_OP(SDMA_OP_CONST_FILL); + ib->ptr[ib->length_dw++] = lower_32_bits(dst_offset); + ib->ptr[ib->length_dw++] = upper_32_bits(dst_offset); + ib->ptr[ib->length_dw++] = src_data; + ib->ptr[ib->length_dw++] = byte_count; } static const struct amdgpu_buffer_funcs sdma_v3_0_buffer_funcs = { @@ -1509,5 +1567,6 @@ static void sdma_v3_0_set_vm_pte_funcs(struct amdgpu_device *adev) if (adev->vm_manager.vm_pte_funcs == NULL) { adev->vm_manager.vm_pte_funcs = &sdma_v3_0_vm_pte_funcs; adev->vm_manager.vm_pte_funcs_ring = &adev->sdma[0].ring; + adev->vm_manager.vm_pte_funcs_ring->is_pte_ring = true; } } diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_sdma_pkt_open.h b/drivers/gpu/drm/amd/amdgpu/tonga_sdma_pkt_open.h index 099b7b56113c..e5ebd084288d 100644 --- a/drivers/gpu/drm/amd/amdgpu/tonga_sdma_pkt_open.h +++ b/drivers/gpu/drm/amd/amdgpu/tonga_sdma_pkt_open.h @@ -2236,5 +2236,10 @@ #define SDMA_PKT_NOP_HEADER_sub_op_shift 8 #define SDMA_PKT_NOP_HEADER_SUB_OP(x) (((x) & SDMA_PKT_NOP_HEADER_sub_op_mask) << SDMA_PKT_NOP_HEADER_sub_op_shift) +/*define for count field*/ +#define SDMA_PKT_NOP_HEADER_count_offset 0 +#define SDMA_PKT_NOP_HEADER_count_mask 0x00003FFF +#define SDMA_PKT_NOP_HEADER_count_shift 16 +#define SDMA_PKT_NOP_HEADER_COUNT(x) (((x) & SDMA_PKT_NOP_HEADER_count_mask) << SDMA_PKT_NOP_HEADER_count_shift) #endif /* __TONGA_SDMA_PKT_OPEN_H_ */ diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_smc.c b/drivers/gpu/drm/amd/amdgpu/tonga_smc.c index 5fc53a40c7ac..1f5ac941a610 100644 --- a/drivers/gpu/drm/amd/amdgpu/tonga_smc.c +++ b/drivers/gpu/drm/amd/amdgpu/tonga_smc.c @@ -761,7 +761,9 @@ int tonga_smu_init(struct amdgpu_device *adev) /* Allocate FW image data structure and header buffer */ ret = amdgpu_bo_create(adev, image_size, PAGE_SIZE, - true, AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, toc_buf); + true, AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, + NULL, toc_buf); if (ret) { DRM_ERROR("Failed to allocate memory for TOC buffer\n"); return -ENOMEM; @@ -769,7 +771,9 @@ int tonga_smu_init(struct amdgpu_device *adev) /* Allocate buffer for SMU internal buffer */ ret = amdgpu_bo_create(adev, smu_internal_buffer_size, PAGE_SIZE, - true, AMDGPU_GEM_DOMAIN_VRAM, 0, NULL, smu_buf); + true, AMDGPU_GEM_DOMAIN_VRAM, + AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED, + NULL, smu_buf); if (ret) { DRM_ERROR("Failed to allocate memory for SMU internal buffer\n"); return -ENOMEM; diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c index 4efd671d7a9b..5fac5da694f0 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c @@ -534,7 +534,7 @@ static void uvd_v4_2_ring_emit_ib(struct amdgpu_ring *ring, static int uvd_v4_2_ring_test_ib(struct amdgpu_ring *ring) { struct amdgpu_device *adev = ring->adev; - struct amdgpu_fence *fence = NULL; + struct fence *fence = NULL; int r; r = amdgpu_asic_set_uvd_clocks(adev, 53300, 40000); @@ -555,14 +555,14 @@ static int uvd_v4_2_ring_test_ib(struct amdgpu_ring *ring) goto error; } - r = amdgpu_fence_wait(fence, false); + r = fence_wait(fence, false); if (r) { DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); goto error; } DRM_INFO("ib test on ring %d succeeded\n", ring->idx); error: - amdgpu_fence_unref(&fence); + fence_put(fence); amdgpu_asic_set_uvd_clocks(adev, 0, 0); return r; } @@ -886,6 +886,7 @@ static const struct amdgpu_ring_funcs uvd_v4_2_ring_funcs = { .test_ring = uvd_v4_2_ring_test_ring, .test_ib = uvd_v4_2_ring_test_ib, .is_lockup = amdgpu_ring_test_lockup, + .insert_nop = amdgpu_ring_insert_nop, }; static void uvd_v4_2_set_ring_funcs(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c index b756bd99c0fd..2d5c59c318af 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c @@ -580,7 +580,7 @@ static void uvd_v5_0_ring_emit_ib(struct amdgpu_ring *ring, static int uvd_v5_0_ring_test_ib(struct amdgpu_ring *ring) { struct amdgpu_device *adev = ring->adev; - struct amdgpu_fence *fence = NULL; + struct fence *fence = NULL; int r; r = amdgpu_asic_set_uvd_clocks(adev, 53300, 40000); @@ -601,14 +601,14 @@ static int uvd_v5_0_ring_test_ib(struct amdgpu_ring *ring) goto error; } - r = amdgpu_fence_wait(fence, false); + r = fence_wait(fence, false); if (r) { DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); goto error; } DRM_INFO("ib test on ring %d succeeded\n", ring->idx); error: - amdgpu_fence_unref(&fence); + fence_put(fence); amdgpu_asic_set_uvd_clocks(adev, 0, 0); return r; } @@ -825,6 +825,7 @@ static const struct amdgpu_ring_funcs uvd_v5_0_ring_funcs = { .test_ring = uvd_v5_0_ring_test_ring, .test_ib = uvd_v5_0_ring_test_ib, .is_lockup = amdgpu_ring_test_lockup, + .insert_nop = amdgpu_ring_insert_nop, }; static void uvd_v5_0_set_ring_funcs(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c index 49aa931b2cb4..d9f553fce531 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c @@ -575,7 +575,7 @@ static void uvd_v6_0_ring_emit_ib(struct amdgpu_ring *ring, */ static int uvd_v6_0_ring_test_ib(struct amdgpu_ring *ring) { - struct amdgpu_fence *fence = NULL; + struct fence *fence = NULL; int r; r = amdgpu_uvd_get_create_msg(ring, 1, NULL); @@ -590,14 +590,14 @@ static int uvd_v6_0_ring_test_ib(struct amdgpu_ring *ring) goto error; } - r = amdgpu_fence_wait(fence, false); + r = fence_wait(fence, false); if (r) { DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); goto error; } DRM_INFO("ib test on ring %d succeeded\n", ring->idx); error: - amdgpu_fence_unref(&fence); + fence_put(fence); return r; } @@ -805,6 +805,7 @@ static const struct amdgpu_ring_funcs uvd_v6_0_ring_funcs = { .test_ring = uvd_v6_0_ring_test_ring, .test_ib = uvd_v6_0_ring_test_ib, .is_lockup = amdgpu_ring_test_lockup, + .insert_nop = amdgpu_ring_insert_nop, }; static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c index 303d961d57bd..cd16df543f64 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c @@ -643,6 +643,7 @@ static const struct amdgpu_ring_funcs vce_v2_0_ring_funcs = { .test_ring = amdgpu_vce_ring_test_ring, .test_ib = amdgpu_vce_ring_test_ib, .is_lockup = amdgpu_ring_test_lockup, + .insert_nop = amdgpu_ring_insert_nop, }; static void vce_v2_0_set_ring_funcs(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c index d1064ca3670e..f0656dfb53f3 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c @@ -32,8 +32,8 @@ #include "vid.h" #include "vce/vce_3_0_d.h" #include "vce/vce_3_0_sh_mask.h" -#include "oss/oss_2_0_d.h" -#include "oss/oss_2_0_sh_mask.h" +#include "oss/oss_3_0_d.h" +#include "oss/oss_3_0_sh_mask.h" #include "gca/gfx_8_0_d.h" #include "smu/smu_7_1_2_d.h" #include "smu/smu_7_1_2_sh_mask.h" @@ -205,7 +205,14 @@ static unsigned vce_v3_0_get_harvest_config(struct amdgpu_device *adev) u32 tmp; unsigned ret; - if (adev->flags & AMDGPU_IS_APU) + /* Fiji is single pipe */ + if (adev->asic_type == CHIP_FIJI) { + ret = AMDGPU_VCE_HARVEST_VCE1; + return ret; + } + + /* Tonga and CZ are dual or single pipe */ + if (adev->flags & AMD_IS_APU) tmp = (RREG32_SMC(ixVCE_HARVEST_FUSE_MACRO__ADDRESS) & VCE_HARVEST_FUSE_MACRO__MASK) >> VCE_HARVEST_FUSE_MACRO__SHIFT; @@ -419,17 +426,41 @@ static void vce_v3_0_mc_resume(struct amdgpu_device *adev, int idx) static bool vce_v3_0_is_idle(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + u32 mask = 0; + int idx; - return !(RREG32(mmSRBM_STATUS2) & SRBM_STATUS2__VCE_BUSY_MASK); + for (idx = 0; idx < 2; ++idx) { + if (adev->vce.harvest_config & (1 << idx)) + continue; + + if (idx == 0) + mask |= SRBM_STATUS2__VCE0_BUSY_MASK; + else + mask |= SRBM_STATUS2__VCE1_BUSY_MASK; + } + + return !(RREG32(mmSRBM_STATUS2) & mask); } static int vce_v3_0_wait_for_idle(void *handle) { unsigned i; struct amdgpu_device *adev = (struct amdgpu_device *)handle; + u32 mask = 0; + int idx; + + for (idx = 0; idx < 2; ++idx) { + if (adev->vce.harvest_config & (1 << idx)) + continue; + + if (idx == 0) + mask |= SRBM_STATUS2__VCE0_BUSY_MASK; + else + mask |= SRBM_STATUS2__VCE1_BUSY_MASK; + } for (i = 0; i < adev->usec_timeout; i++) { - if (!(RREG32(mmSRBM_STATUS2) & SRBM_STATUS2__VCE_BUSY_MASK)) + if (!(RREG32(mmSRBM_STATUS2) & mask)) return 0; } return -ETIMEDOUT; @@ -438,9 +469,21 @@ static int vce_v3_0_wait_for_idle(void *handle) static int vce_v3_0_soft_reset(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + u32 mask = 0; + int idx; - WREG32_P(mmSRBM_SOFT_RESET, SRBM_SOFT_RESET__SOFT_RESET_VCE_MASK, - ~SRBM_SOFT_RESET__SOFT_RESET_VCE_MASK); + for (idx = 0; idx < 2; ++idx) { + if (adev->vce.harvest_config & (1 << idx)) + continue; + + if (idx == 0) + mask |= SRBM_SOFT_RESET__SOFT_RESET_VCE0_MASK; + else + mask |= SRBM_SOFT_RESET__SOFT_RESET_VCE1_MASK; + } + WREG32_P(mmSRBM_SOFT_RESET, mask, + ~(SRBM_SOFT_RESET__SOFT_RESET_VCE0_MASK | + SRBM_SOFT_RESET__SOFT_RESET_VCE1_MASK)); mdelay(5); return vce_v3_0_start(adev); @@ -601,6 +644,7 @@ static const struct amdgpu_ring_funcs vce_v3_0_ring_funcs = { .test_ring = amdgpu_vce_ring_test_ring, .test_ib = amdgpu_vce_ring_test_ib, .is_lockup = amdgpu_ring_test_lockup, + .insert_nop = amdgpu_ring_insert_nop, }; static void vce_v3_0_set_ring_funcs(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c index 68552da40287..552d9e75ad1b 100644 --- a/drivers/gpu/drm/amd/amdgpu/vi.c +++ b/drivers/gpu/drm/amd/amdgpu/vi.c @@ -203,6 +203,17 @@ static const u32 tonga_mgcg_cgcg_init[] = mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104, }; +static const u32 fiji_mgcg_cgcg_init[] = +{ + mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100, + mmPCIE_INDEX, 0xffffffff, 0x0140001c, + mmPCIE_DATA, 0x000f0000, 0x00000000, + mmSMC_IND_INDEX_4, 0xffffffff, 0xC060000C, + mmSMC_IND_DATA_4, 0xc0000fff, 0x00000100, + mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100, + mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104, +}; + static const u32 iceland_mgcg_cgcg_init[] = { mmPCIE_INDEX, 0xffffffff, ixPCIE_CNTL2, @@ -232,6 +243,11 @@ static void vi_init_golden_registers(struct amdgpu_device *adev) iceland_mgcg_cgcg_init, (const u32)ARRAY_SIZE(iceland_mgcg_cgcg_init)); break; + case CHIP_FIJI: + amdgpu_program_register_sequence(adev, + fiji_mgcg_cgcg_init, + (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init)); + break; case CHIP_TONGA: amdgpu_program_register_sequence(adev, tonga_mgcg_cgcg_init, @@ -261,7 +277,7 @@ static u32 vi_get_xclk(struct amdgpu_device *adev) u32 reference_clock = adev->clock.spll.reference_freq; u32 tmp; - if (adev->flags & AMDGPU_IS_APU) + if (adev->flags & AMD_IS_APU) return reference_clock; tmp = RREG32_SMC(ixCG_CLKPIN_CNTL_2); @@ -362,6 +378,26 @@ static struct amdgpu_allowed_register_entry cz_allowed_read_registers[] = { static struct amdgpu_allowed_register_entry vi_allowed_read_registers[] = { {mmGRBM_STATUS, false}, + {mmGRBM_STATUS2, false}, + {mmGRBM_STATUS_SE0, false}, + {mmGRBM_STATUS_SE1, false}, + {mmGRBM_STATUS_SE2, false}, + {mmGRBM_STATUS_SE3, false}, + {mmSRBM_STATUS, false}, + {mmSRBM_STATUS2, false}, + {mmSRBM_STATUS3, false}, + {mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET, false}, + {mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET, false}, + {mmCP_STAT, false}, + {mmCP_STALLED_STAT1, false}, + {mmCP_STALLED_STAT2, false}, + {mmCP_STALLED_STAT3, false}, + {mmCP_CPF_BUSY_STAT, false}, + {mmCP_CPF_STALLED_STAT1, false}, + {mmCP_CPF_STATUS, false}, + {mmCP_CPC_BUSY_STAT, false}, + {mmCP_CPC_STALLED_STAT1, false}, + {mmCP_CPC_STATUS, false}, {mmGB_ADDR_CONFIG, false}, {mmMC_ARB_RAMCFG, false}, {mmGB_TILE_MODE0, false}, @@ -449,6 +485,7 @@ static int vi_read_register(struct amdgpu_device *adev, u32 se_num, asic_register_table = tonga_allowed_read_registers; size = ARRAY_SIZE(tonga_allowed_read_registers); break; + case CHIP_FIJI: case CHIP_TONGA: case CHIP_CARRIZO: asic_register_table = cz_allowed_read_registers; @@ -751,7 +788,7 @@ static void vi_gpu_soft_reset(struct amdgpu_device *adev, u32 reset_mask) srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE1, 1); - if (!(adev->flags & AMDGPU_IS_APU)) { + if (!(adev->flags & AMD_IS_APU)) { if (reset_mask & AMDGPU_RESET_MC) srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_MC, 1); @@ -971,7 +1008,7 @@ static void vi_pcie_gen3_enable(struct amdgpu_device *adev) if (amdgpu_pcie_gen2 == 0) return; - if (adev->flags & AMDGPU_IS_APU) + if (adev->flags & AMD_IS_APU) return; ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask); @@ -999,7 +1036,7 @@ static void vi_enable_doorbell_aperture(struct amdgpu_device *adev, u32 tmp; /* not necessary on CZ */ - if (adev->flags & AMDGPU_IS_APU) + if (adev->flags & AMD_IS_APU) return; tmp = RREG32(mmBIF_DOORBELL_APER_EN); @@ -1127,6 +1164,74 @@ static const struct amdgpu_ip_block_version tonga_ip_blocks[] = }, }; +static const struct amdgpu_ip_block_version fiji_ip_blocks[] = +{ + /* ORDER MATTERS! */ + { + .type = AMD_IP_BLOCK_TYPE_COMMON, + .major = 2, + .minor = 0, + .rev = 0, + .funcs = &vi_common_ip_funcs, + }, + { + .type = AMD_IP_BLOCK_TYPE_GMC, + .major = 8, + .minor = 5, + .rev = 0, + .funcs = &gmc_v8_0_ip_funcs, + }, + { + .type = AMD_IP_BLOCK_TYPE_IH, + .major = 3, + .minor = 0, + .rev = 0, + .funcs = &tonga_ih_ip_funcs, + }, + { + .type = AMD_IP_BLOCK_TYPE_SMC, + .major = 7, + .minor = 1, + .rev = 0, + .funcs = &fiji_dpm_ip_funcs, + }, + { + .type = AMD_IP_BLOCK_TYPE_DCE, + .major = 10, + .minor = 1, + .rev = 0, + .funcs = &dce_v10_0_ip_funcs, + }, + { + .type = AMD_IP_BLOCK_TYPE_GFX, + .major = 8, + .minor = 0, + .rev = 0, + .funcs = &gfx_v8_0_ip_funcs, + }, + { + .type = AMD_IP_BLOCK_TYPE_SDMA, + .major = 3, + .minor = 0, + .rev = 0, + .funcs = &sdma_v3_0_ip_funcs, + }, + { + .type = AMD_IP_BLOCK_TYPE_UVD, + .major = 6, + .minor = 0, + .rev = 0, + .funcs = &uvd_v6_0_ip_funcs, + }, + { + .type = AMD_IP_BLOCK_TYPE_VCE, + .major = 3, + .minor = 0, + .rev = 0, + .funcs = &vce_v3_0_ip_funcs, + }, +}; + static const struct amdgpu_ip_block_version cz_ip_blocks[] = { /* ORDER MATTERS! */ @@ -1202,6 +1307,10 @@ int vi_set_ip_blocks(struct amdgpu_device *adev) adev->ip_blocks = topaz_ip_blocks; adev->num_ip_blocks = ARRAY_SIZE(topaz_ip_blocks); break; + case CHIP_FIJI: + adev->ip_blocks = fiji_ip_blocks; + adev->num_ip_blocks = ARRAY_SIZE(fiji_ip_blocks); + break; case CHIP_TONGA: adev->ip_blocks = tonga_ip_blocks; adev->num_ip_blocks = ARRAY_SIZE(tonga_ip_blocks); @@ -1248,7 +1357,7 @@ static int vi_common_early_init(void *handle) bool smc_enabled = false; struct amdgpu_device *adev = (struct amdgpu_device *)handle; - if (adev->flags & AMDGPU_IS_APU) { + if (adev->flags & AMD_IS_APU) { adev->smc_rreg = &cz_smc_rreg; adev->smc_wreg = &cz_smc_wreg; } else { @@ -1279,6 +1388,7 @@ static int vi_common_early_init(void *handle) if (amdgpu_smc_load_fw && smc_enabled) adev->firmware.smu_load = true; break; + case CHIP_FIJI: case CHIP_TONGA: adev->has_uvd = true; adev->cg_flags = 0; diff --git a/drivers/gpu/drm/amd/amdgpu/vi_dpm.h b/drivers/gpu/drm/amd/amdgpu/vi_dpm.h index 3b45332f5df4..fc120ba18aad 100644 --- a/drivers/gpu/drm/amd/amdgpu/vi_dpm.h +++ b/drivers/gpu/drm/amd/amdgpu/vi_dpm.h @@ -30,7 +30,7 @@ int cz_smu_start(struct amdgpu_device *adev); int cz_smu_fini(struct amdgpu_device *adev); extern const struct amd_ip_funcs tonga_dpm_ip_funcs; - +extern const struct amd_ip_funcs fiji_dpm_ip_funcs; extern const struct amd_ip_funcs iceland_dpm_ip_funcs; #endif diff --git a/drivers/gpu/drm/amd/amdgpu/vid.h b/drivers/gpu/drm/amd/amdgpu/vid.h index 31bb89452e12..d98aa9d82fa1 100644 --- a/drivers/gpu/drm/amd/amdgpu/vid.h +++ b/drivers/gpu/drm/amd/amdgpu/vid.h @@ -66,6 +66,11 @@ #define AMDGPU_NUM_OF_VMIDS 8 +#define PIPEID(x) ((x) << 0) +#define MEID(x) ((x) << 2) +#define VMID(x) ((x) << 4) +#define QUEUEID(x) ((x) << 8) + #define RB_BITMAP_WIDTH_PER_SH 2 #define MC_SEQ_MISC0__MT__MASK 0xf0000000 diff --git a/drivers/gpu/drm/amd/amdkfd/Kconfig b/drivers/gpu/drm/amd/amdkfd/Kconfig index 8dfac37ff327..e13c67c8d2c0 100644 --- a/drivers/gpu/drm/amd/amdkfd/Kconfig +++ b/drivers/gpu/drm/amd/amdkfd/Kconfig @@ -4,6 +4,6 @@ config HSA_AMD tristate "HSA kernel driver for AMD GPU devices" - depends on DRM_RADEON && AMD_IOMMU_V2 && X86_64 + depends on (DRM_RADEON || DRM_AMDGPU) && AMD_IOMMU_V2 && X86_64 help Enable this if you want to use HSA features on AMD GPU devices. diff --git a/drivers/gpu/drm/amd/amdkfd/Makefile b/drivers/gpu/drm/amd/amdkfd/Makefile index 28551153ec6d..7fc9b0f444cb 100644 --- a/drivers/gpu/drm/amd/amdkfd/Makefile +++ b/drivers/gpu/drm/amd/amdkfd/Makefile @@ -2,7 +2,8 @@ # Makefile for Heterogenous System Architecture support for AMD GPU devices # -ccflags-y := -Iinclude/drm -Idrivers/gpu/drm/amd/include/ +ccflags-y := -Iinclude/drm -Idrivers/gpu/drm/amd/include/ \ + -Idrivers/gpu/drm/amd/include/asic_reg amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \ kfd_pasid.o kfd_doorbell.o kfd_flat_memory.o \ diff --git a/drivers/gpu/drm/amd/amdkfd/cik_regs.h b/drivers/gpu/drm/amd/amdkfd/cik_regs.h index 183be5b8414f..48769d12dd7b 100644 --- a/drivers/gpu/drm/amd/amdkfd/cik_regs.h +++ b/drivers/gpu/drm/amd/amdkfd/cik_regs.h @@ -65,17 +65,6 @@ #define AQL_ENABLE 1 -#define SDMA_RB_VMID(x) (x << 24) -#define SDMA_RB_ENABLE (1 << 0) -#define SDMA_RB_SIZE(x) ((x) << 1) /* log2 */ -#define SDMA_RPTR_WRITEBACK_ENABLE (1 << 12) -#define SDMA_RPTR_WRITEBACK_TIMER(x) ((x) << 16) /* log2 */ -#define SDMA_OFFSET(x) (x << 0) -#define SDMA_DB_ENABLE (1 << 28) -#define SDMA_ATC (1 << 0) -#define SDMA_VA_PTR32 (1 << 4) -#define SDMA_VA_SHARED_BASE(x) (x << 8) - #define GRBM_GFX_INDEX 0x30800 #define ATC_VMID_PASID_MAPPING_VALID (1U << 31) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index c991973019d0..c6a1b4cc6458 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -31,7 +31,7 @@ #include #include #include -#include +#include #include #include "kfd_priv.h" #include "kfd_device_queue_manager.h" diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c index 75312c82969f..3f95f7cb4019 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c @@ -80,7 +80,12 @@ static const struct kfd_deviceid supported_devices[] = { { 0x1318, &kaveri_device_info }, /* Kaveri */ { 0x131B, &kaveri_device_info }, /* Kaveri */ { 0x131C, &kaveri_device_info }, /* Kaveri */ - { 0x131D, &kaveri_device_info } /* Kaveri */ + { 0x131D, &kaveri_device_info }, /* Kaveri */ + { 0x9870, &carrizo_device_info }, /* Carrizo */ + { 0x9874, &carrizo_device_info }, /* Carrizo */ + { 0x9875, &carrizo_device_info }, /* Carrizo */ + { 0x9876, &carrizo_device_info }, /* Carrizo */ + { 0x9877, &carrizo_device_info } /* Carrizo */ }; static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c index 9ce8a20a7aff..c6f435aa803f 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c @@ -23,6 +23,7 @@ #include "kfd_device_queue_manager.h" #include "cik_regs.h" +#include "oss/oss_2_4_sh_mask.h" static bool set_cache_memory_policy_cik(struct device_queue_manager *dqm, struct qcm_process_device *qpd, @@ -135,13 +136,16 @@ static int register_process_cik(struct device_queue_manager *dqm, static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q, struct qcm_process_device *qpd) { - uint32_t value = SDMA_ATC; + uint32_t value = (1 << SDMA0_RLC0_VIRTUAL_ADDR__ATC__SHIFT); if (q->process->is_32bit_user_mode) - value |= SDMA_VA_PTR32 | get_sh_mem_bases_32(qpd_to_pdd(qpd)); + value |= (1 << SDMA0_RLC0_VIRTUAL_ADDR__PTR32__SHIFT) | + get_sh_mem_bases_32(qpd_to_pdd(qpd)); else - value |= SDMA_VA_SHARED_BASE(get_sh_mem_bases_nybble_64( - qpd_to_pdd(qpd))); + value |= ((get_sh_mem_bases_nybble_64(qpd_to_pdd(qpd))) << + SDMA0_RLC0_VIRTUAL_ADDR__SHARED_BASE__SHIFT) & + SDMA0_RLC0_VIRTUAL_ADDR__SHARED_BASE_MASK; + q->properties.sdma_vm_addr = value; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c index 4c15212a3899..7e9cae9d349b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c @@ -22,6 +22,10 @@ */ #include "kfd_device_queue_manager.h" +#include "gca/gfx_8_0_enum.h" +#include "gca/gfx_8_0_sh_mask.h" +#include "gca/gfx_8_0_enum.h" +#include "oss/oss_3_0_sh_mask.h" static bool set_cache_memory_policy_vi(struct device_queue_manager *dqm, struct qcm_process_device *qpd, @@ -37,14 +41,40 @@ static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q, void device_queue_manager_init_vi(struct device_queue_manager_asic_ops *ops) { - pr_warn("amdkfd: VI DQM is not currently supported\n"); - ops->set_cache_memory_policy = set_cache_memory_policy_vi; ops->register_process = register_process_vi; ops->initialize = initialize_cpsch_vi; ops->init_sdma_vm = init_sdma_vm; } +static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble) +{ + /* In 64-bit mode, we can only control the top 3 bits of the LDS, + * scratch and GPUVM apertures. + * The hardware fills in the remaining 59 bits according to the + * following pattern: + * LDS: X0000000'00000000 - X0000001'00000000 (4GB) + * Scratch: X0000001'00000000 - X0000002'00000000 (4GB) + * GPUVM: Y0010000'00000000 - Y0020000'00000000 (1TB) + * + * (where X/Y is the configurable nybble with the low-bit 0) + * + * LDS and scratch will have the same top nybble programmed in the + * top 3 bits of SH_MEM_BASES.PRIVATE_BASE. + * GPUVM can have a different top nybble programmed in the + * top 3 bits of SH_MEM_BASES.SHARED_BASE. + * We don't bother to support different top nybbles + * for LDS/Scratch and GPUVM. + */ + + BUG_ON((top_address_nybble & 1) || top_address_nybble > 0xE || + top_address_nybble == 0); + + return top_address_nybble << 12 | + (top_address_nybble << 12) << + SH_MEM_BASES__SHARED_BASE__SHIFT; +} + static bool set_cache_memory_policy_vi(struct device_queue_manager *dqm, struct qcm_process_device *qpd, enum cache_policy default_policy, @@ -52,18 +82,83 @@ static bool set_cache_memory_policy_vi(struct device_queue_manager *dqm, void __user *alternate_aperture_base, uint64_t alternate_aperture_size) { - return false; + uint32_t default_mtype; + uint32_t ape1_mtype; + + default_mtype = (default_policy == cache_policy_coherent) ? + MTYPE_CC : + MTYPE_NC; + + ape1_mtype = (alternate_policy == cache_policy_coherent) ? + MTYPE_CC : + MTYPE_NC; + + qpd->sh_mem_config = (qpd->sh_mem_config & + SH_MEM_CONFIG__ADDRESS_MODE_MASK) | + SH_MEM_ALIGNMENT_MODE_UNALIGNED << + SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT | + default_mtype << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT | + ape1_mtype << SH_MEM_CONFIG__APE1_MTYPE__SHIFT | + SH_MEM_CONFIG__PRIVATE_ATC_MASK; + + return true; } static int register_process_vi(struct device_queue_manager *dqm, struct qcm_process_device *qpd) { - return -1; + struct kfd_process_device *pdd; + unsigned int temp; + + BUG_ON(!dqm || !qpd); + + pdd = qpd_to_pdd(qpd); + + /* check if sh_mem_config register already configured */ + if (qpd->sh_mem_config == 0) { + qpd->sh_mem_config = + SH_MEM_ALIGNMENT_MODE_UNALIGNED << + SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT | + MTYPE_CC << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT | + MTYPE_CC << SH_MEM_CONFIG__APE1_MTYPE__SHIFT | + SH_MEM_CONFIG__PRIVATE_ATC_MASK; + + qpd->sh_mem_ape1_limit = 0; + qpd->sh_mem_ape1_base = 0; + } + + if (qpd->pqm->process->is_32bit_user_mode) { + temp = get_sh_mem_bases_32(pdd); + qpd->sh_mem_bases = temp << SH_MEM_BASES__SHARED_BASE__SHIFT; + qpd->sh_mem_config |= SH_MEM_ADDRESS_MODE_HSA32 << + SH_MEM_CONFIG__ADDRESS_MODE__SHIFT; + } else { + temp = get_sh_mem_bases_nybble_64(pdd); + qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp); + qpd->sh_mem_config |= SH_MEM_ADDRESS_MODE_HSA64 << + SH_MEM_CONFIG__ADDRESS_MODE__SHIFT; + } + + pr_debug("kfd: is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n", + qpd->pqm->process->is_32bit_user_mode, temp, qpd->sh_mem_bases); + + return 0; } static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q, struct qcm_process_device *qpd) { + uint32_t value = (1 << SDMA0_RLC0_VIRTUAL_ADDR__ATC__SHIFT); + + if (q->process->is_32bit_user_mode) + value |= (1 << SDMA0_RLC0_VIRTUAL_ADDR__PTR32__SHIFT) | + get_sh_mem_bases_32(qpd_to_pdd(qpd)); + else + value |= ((get_sh_mem_bases_nybble_64(qpd_to_pdd(qpd))) << + SDMA0_RLC0_VIRTUAL_ADDR__SHARED_BASE__SHIFT) & + SDMA0_RLC0_VIRTUAL_ADDR__SHARED_BASE_MASK; + + q->properties.sdma_vm_addr = value; } static int initialize_cpsch_vi(struct device_queue_manager *dqm) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c index 35b987574633..2b655103ba79 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c @@ -33,7 +33,7 @@ #include #include "kfd_priv.h" #include -#include +#include #include /* diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c index 434979428fc0..d83de985e88c 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c @@ -27,6 +27,7 @@ #include "kfd_mqd_manager.h" #include "cik_regs.h" #include "cik_structs.h" +#include "oss/oss_2_4_sh_mask.h" static inline struct cik_mqd *get_mqd(void *mqd) { @@ -214,17 +215,20 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd, BUG_ON(!mm || !mqd || !q); m = get_sdma_mqd(mqd); - m->sdma_rlc_rb_cntl = - SDMA_RB_SIZE((ffs(q->queue_size / sizeof(unsigned int)))) | - SDMA_RB_VMID(q->vmid) | - SDMA_RPTR_WRITEBACK_ENABLE | - SDMA_RPTR_WRITEBACK_TIMER(6); + m->sdma_rlc_rb_cntl = ffs(q->queue_size / sizeof(unsigned int)) << + SDMA0_RLC0_RB_CNTL__RB_SIZE__SHIFT | + q->vmid << SDMA0_RLC0_RB_CNTL__RB_VMID__SHIFT | + 1 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT | + 6 << SDMA0_RLC0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT; m->sdma_rlc_rb_base = lower_32_bits(q->queue_address >> 8); m->sdma_rlc_rb_base_hi = upper_32_bits(q->queue_address >> 8); m->sdma_rlc_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr); m->sdma_rlc_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr); - m->sdma_rlc_doorbell = SDMA_OFFSET(q->doorbell_off) | SDMA_DB_ENABLE; + m->sdma_rlc_doorbell = q->doorbell_off << + SDMA0_RLC0_DOORBELL__OFFSET__SHIFT | + 1 << SDMA0_RLC0_DOORBELL__ENABLE__SHIFT; + m->sdma_rlc_virtual_addr = q->sdma_vm_addr; m->sdma_engine_id = q->sdma_engine_id; @@ -234,7 +238,9 @@ static int update_mqd_sdma(struct mqd_manager *mm, void *mqd, if (q->queue_size > 0 && q->queue_address != 0 && q->queue_percent > 0) { - m->sdma_rlc_rb_cntl |= SDMA_RB_ENABLE; + m->sdma_rlc_rb_cntl |= + 1 << SDMA0_RLC0_RB_CNTL__RB_ENABLE__SHIFT; + q->is_active = true; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c index b3a7e3ba1e38..fa32c32fa1c2 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c @@ -22,12 +22,255 @@ */ #include +#include #include "kfd_priv.h" #include "kfd_mqd_manager.h" +#include "vi_structs.h" +#include "gca/gfx_8_0_sh_mask.h" +#include "gca/gfx_8_0_enum.h" + +#define CP_MQD_CONTROL__PRIV_STATE__SHIFT 0x8 + +static inline struct vi_mqd *get_mqd(void *mqd) +{ + return (struct vi_mqd *)mqd; +} + +static int init_mqd(struct mqd_manager *mm, void **mqd, + struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, + struct queue_properties *q) +{ + int retval; + uint64_t addr; + struct vi_mqd *m; + + retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct vi_mqd), + mqd_mem_obj); + if (retval != 0) + return -ENOMEM; + + m = (struct vi_mqd *) (*mqd_mem_obj)->cpu_ptr; + addr = (*mqd_mem_obj)->gpu_addr; + + memset(m, 0, sizeof(struct vi_mqd)); + + m->header = 0xC0310800; + m->compute_pipelinestat_enable = 1; + m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; + m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; + m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; + m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; + + m->cp_hqd_persistent_state = CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK | + 0x53 << CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT; + + m->cp_mqd_control = 1 << CP_MQD_CONTROL__PRIV_STATE__SHIFT | + MTYPE_UC << CP_MQD_CONTROL__MTYPE__SHIFT; + + m->cp_mqd_base_addr_lo = lower_32_bits(addr); + m->cp_mqd_base_addr_hi = upper_32_bits(addr); + + m->cp_hqd_quantum = 1 << CP_HQD_QUANTUM__QUANTUM_EN__SHIFT | + 1 << CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT | + 10 << CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT; + + m->cp_hqd_pipe_priority = 1; + m->cp_hqd_queue_priority = 15; + + m->cp_hqd_eop_rptr = 1 << CP_HQD_EOP_RPTR__INIT_FETCHER__SHIFT; + + if (q->format == KFD_QUEUE_FORMAT_AQL) + m->cp_hqd_iq_rptr = 1; + + *mqd = m; + if (gart_addr != NULL) + *gart_addr = addr; + retval = mm->update_mqd(mm, m, q); + + return retval; +} + +static int load_mqd(struct mqd_manager *mm, void *mqd, + uint32_t pipe_id, uint32_t queue_id, + uint32_t __user *wptr) +{ + return mm->dev->kfd2kgd->hqd_load + (mm->dev->kgd, mqd, pipe_id, queue_id, wptr); +} + +static int __update_mqd(struct mqd_manager *mm, void *mqd, + struct queue_properties *q, unsigned int mtype, + unsigned int atc_bit) +{ + struct vi_mqd *m; + + BUG_ON(!mm || !q || !mqd); + + pr_debug("kfd: In func %s\n", __func__); + + m = get_mqd(mqd); + + m->cp_hqd_pq_control = 5 << CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT | + atc_bit << CP_HQD_PQ_CONTROL__PQ_ATC__SHIFT | + mtype << CP_HQD_PQ_CONTROL__MTYPE__SHIFT; + m->cp_hqd_pq_control |= + ffs(q->queue_size / sizeof(unsigned int)) - 1 - 1; + pr_debug("kfd: cp_hqd_pq_control 0x%x\n", m->cp_hqd_pq_control); + + m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); + m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); + + m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); + m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); + + m->cp_hqd_pq_doorbell_control = + 1 << CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN__SHIFT | + q->doorbell_off << + CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT; + pr_debug("kfd: cp_hqd_pq_doorbell_control 0x%x\n", + m->cp_hqd_pq_doorbell_control); + + m->cp_hqd_eop_control = atc_bit << CP_HQD_EOP_CONTROL__EOP_ATC__SHIFT | + mtype << CP_HQD_EOP_CONTROL__MTYPE__SHIFT; + + m->cp_hqd_ib_control = atc_bit << CP_HQD_IB_CONTROL__IB_ATC__SHIFT | + 3 << CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT | + mtype << CP_HQD_IB_CONTROL__MTYPE__SHIFT; + + m->cp_hqd_eop_control |= + ffs(q->eop_ring_buffer_size / sizeof(unsigned int)) - 1 - 1; + m->cp_hqd_eop_base_addr_lo = + lower_32_bits(q->eop_ring_buffer_address >> 8); + m->cp_hqd_eop_base_addr_hi = + upper_32_bits(q->eop_ring_buffer_address >> 8); + + m->cp_hqd_iq_timer = atc_bit << CP_HQD_IQ_TIMER__IQ_ATC__SHIFT | + mtype << CP_HQD_IQ_TIMER__MTYPE__SHIFT; + + m->cp_hqd_vmid = q->vmid; + + if (q->format == KFD_QUEUE_FORMAT_AQL) { + m->cp_hqd_pq_control |= CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK | + 2 << CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT; + } + + m->cp_hqd_active = 0; + q->is_active = false; + if (q->queue_size > 0 && + q->queue_address != 0 && + q->queue_percent > 0) { + m->cp_hqd_active = 1; + q->is_active = true; + } + + return 0; +} + + +static int update_mqd(struct mqd_manager *mm, void *mqd, + struct queue_properties *q) +{ + return __update_mqd(mm, mqd, q, MTYPE_CC, 1); +} + +static int destroy_mqd(struct mqd_manager *mm, void *mqd, + enum kfd_preempt_type type, + unsigned int timeout, uint32_t pipe_id, + uint32_t queue_id) +{ + return mm->dev->kfd2kgd->hqd_destroy + (mm->dev->kgd, type, timeout, + pipe_id, queue_id); +} + +static void uninit_mqd(struct mqd_manager *mm, void *mqd, + struct kfd_mem_obj *mqd_mem_obj) +{ + BUG_ON(!mm || !mqd); + kfd_gtt_sa_free(mm->dev, mqd_mem_obj); +} + +static bool is_occupied(struct mqd_manager *mm, void *mqd, + uint64_t queue_address, uint32_t pipe_id, + uint32_t queue_id) +{ + return mm->dev->kfd2kgd->hqd_is_occupied( + mm->dev->kgd, queue_address, + pipe_id, queue_id); +} + +static int init_mqd_hiq(struct mqd_manager *mm, void **mqd, + struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, + struct queue_properties *q) +{ + struct vi_mqd *m; + int retval = init_mqd(mm, mqd, mqd_mem_obj, gart_addr, q); + + if (retval != 0) + return retval; + + m = get_mqd(*mqd); + + m->cp_hqd_pq_control |= 1 << CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT | + 1 << CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT; + + return retval; +} + +static int update_mqd_hiq(struct mqd_manager *mm, void *mqd, + struct queue_properties *q) +{ + struct vi_mqd *m; + int retval = __update_mqd(mm, mqd, q, MTYPE_UC, 0); + + if (retval != 0) + return retval; + + m = get_mqd(mqd); + m->cp_hqd_vmid = q->vmid; + return retval; +} struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type, - struct kfd_dev *dev) + struct kfd_dev *dev) { - pr_warn("amdkfd: VI MQD is not currently supported\n"); - return NULL; + struct mqd_manager *mqd; + + BUG_ON(!dev); + BUG_ON(type >= KFD_MQD_TYPE_MAX); + + pr_debug("kfd: In func %s\n", __func__); + + mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL); + if (!mqd) + return NULL; + + mqd->dev = dev; + + switch (type) { + case KFD_MQD_TYPE_CP: + case KFD_MQD_TYPE_COMPUTE: + mqd->init_mqd = init_mqd; + mqd->uninit_mqd = uninit_mqd; + mqd->load_mqd = load_mqd; + mqd->update_mqd = update_mqd; + mqd->destroy_mqd = destroy_mqd; + mqd->is_occupied = is_occupied; + break; + case KFD_MQD_TYPE_HIQ: + mqd->init_mqd = init_mqd_hiq; + mqd->uninit_mqd = uninit_mqd; + mqd->load_mqd = load_mqd; + mqd->update_mqd = update_mqd_hiq; + mqd->destroy_mqd = destroy_mqd; + mqd->is_occupied = is_occupied; + break; + case KFD_MQD_TYPE_SDMA: + break; + default: + kfree(mqd); + return NULL; + } + + return mqd; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c index 99b6d28a11c3..90f391434fa3 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c @@ -27,6 +27,7 @@ #include "kfd_kernel_queue.h" #include "kfd_priv.h" #include "kfd_pm4_headers.h" +#include "kfd_pm4_headers_vi.h" #include "kfd_pm4_opcodes.h" static inline void inc_wptr(unsigned int *wptr, unsigned int increment_bytes, @@ -55,6 +56,7 @@ static void pm_calc_rlib_size(struct packet_manager *pm, bool *over_subscription) { unsigned int process_count, queue_count; + unsigned int map_queue_size; BUG_ON(!pm || !rlib_size || !over_subscription); @@ -69,9 +71,13 @@ static void pm_calc_rlib_size(struct packet_manager *pm, pr_debug("kfd: over subscribed runlist\n"); } + map_queue_size = + (pm->dqm->dev->device_info->asic_family == CHIP_CARRIZO) ? + sizeof(struct pm4_mes_map_queues) : + sizeof(struct pm4_map_queues); /* calculate run list ib allocation size */ *rlib_size = process_count * sizeof(struct pm4_map_process) + - queue_count * sizeof(struct pm4_map_queues); + queue_count * map_queue_size; /* * Increase the allocation size in case we need a chained run list @@ -176,6 +182,71 @@ static int pm_create_map_process(struct packet_manager *pm, uint32_t *buffer, return 0; } +static int pm_create_map_queue_vi(struct packet_manager *pm, uint32_t *buffer, + struct queue *q, bool is_static) +{ + struct pm4_mes_map_queues *packet; + bool use_static = is_static; + + BUG_ON(!pm || !buffer || !q); + + pr_debug("kfd: In func %s\n", __func__); + + packet = (struct pm4_mes_map_queues *)buffer; + memset(buffer, 0, sizeof(struct pm4_map_queues)); + + packet->header.u32all = build_pm4_header(IT_MAP_QUEUES, + sizeof(struct pm4_map_queues)); + packet->bitfields2.alloc_format = + alloc_format__mes_map_queues__one_per_pipe_vi; + packet->bitfields2.num_queues = 1; + packet->bitfields2.queue_sel = + queue_sel__mes_map_queues__map_to_hws_determined_queue_slots_vi; + + packet->bitfields2.engine_sel = + engine_sel__mes_map_queues__compute_vi; + packet->bitfields2.queue_type = + queue_type__mes_map_queues__normal_compute_vi; + + switch (q->properties.type) { + case KFD_QUEUE_TYPE_COMPUTE: + if (use_static) + packet->bitfields2.queue_type = + queue_type__mes_map_queues__normal_latency_static_queue_vi; + break; + case KFD_QUEUE_TYPE_DIQ: + packet->bitfields2.queue_type = + queue_type__mes_map_queues__debug_interface_queue_vi; + break; + case KFD_QUEUE_TYPE_SDMA: + packet->bitfields2.engine_sel = + engine_sel__mes_map_queues__sdma0_vi; + use_static = false; /* no static queues under SDMA */ + break; + default: + pr_err("kfd: in %s queue type %d\n", __func__, + q->properties.type); + BUG(); + break; + } + packet->bitfields3.doorbell_offset = + q->properties.doorbell_off; + + packet->mqd_addr_lo = + lower_32_bits(q->gart_mqd_addr); + + packet->mqd_addr_hi = + upper_32_bits(q->gart_mqd_addr); + + packet->wptr_addr_lo = + lower_32_bits((uint64_t)q->properties.write_ptr); + + packet->wptr_addr_hi = + upper_32_bits((uint64_t)q->properties.write_ptr); + + return 0; +} + static int pm_create_map_queue(struct packet_manager *pm, uint32_t *buffer, struct queue *q, bool is_static) { @@ -292,8 +363,17 @@ static int pm_create_runlist_ib(struct packet_manager *pm, pr_debug("kfd: static_queue, mapping kernel q %d, is debug status %d\n", kq->queue->queue, qpd->is_debug); - retval = pm_create_map_queue(pm, &rl_buffer[rl_wptr], - kq->queue, qpd->is_debug); + if (pm->dqm->dev->device_info->asic_family == + CHIP_CARRIZO) + retval = pm_create_map_queue_vi(pm, + &rl_buffer[rl_wptr], + kq->queue, + qpd->is_debug); + else + retval = pm_create_map_queue(pm, + &rl_buffer[rl_wptr], + kq->queue, + qpd->is_debug); if (retval != 0) return retval; @@ -309,8 +389,17 @@ static int pm_create_runlist_ib(struct packet_manager *pm, pr_debug("kfd: static_queue, mapping user queue %d, is debug status %d\n", q->queue, qpd->is_debug); - retval = pm_create_map_queue(pm, &rl_buffer[rl_wptr], - q, qpd->is_debug); + if (pm->dqm->dev->device_info->asic_family == + CHIP_CARRIZO) + retval = pm_create_map_queue_vi(pm, + &rl_buffer[rl_wptr], + q, + qpd->is_debug); + else + retval = pm_create_map_queue(pm, + &rl_buffer[rl_wptr], + q, + qpd->is_debug); if (retval != 0) return retval; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h new file mode 100644 index 000000000000..08c721922812 --- /dev/null +++ b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_vi.h @@ -0,0 +1,398 @@ +/* + * Copyright 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef F32_MES_PM4_PACKETS_H +#define F32_MES_PM4_PACKETS_H + +#ifndef PM4_MES_HEADER_DEFINED +#define PM4_MES_HEADER_DEFINED +union PM4_MES_TYPE_3_HEADER { + struct { + uint32_t reserved1 : 8; /* < reserved */ + uint32_t opcode : 8; /* < IT opcode */ + uint32_t count : 14;/* < number of DWORDs - 1 in the + information body. */ + uint32_t type : 2; /* < packet identifier. + It should be 3 for type 3 packets */ + }; + uint32_t u32All; +}; +#endif /* PM4_MES_HEADER_DEFINED */ + +/*--------------------MES_SET_RESOURCES--------------------*/ + +#ifndef PM4_MES_SET_RESOURCES_DEFINED +#define PM4_MES_SET_RESOURCES_DEFINED +enum mes_set_resources_queue_type_enum { + queue_type__mes_set_resources__kernel_interface_queue_kiq = 0, + queue_type__mes_set_resources__hsa_interface_queue_hiq = 1, + queue_type__mes_set_resources__hsa_debug_interface_queue = 4 +}; + + +struct pm4_mes_set_resources { + union { + union PM4_MES_TYPE_3_HEADER header; /* header */ + uint32_t ordinal1; + }; + + union { + struct { + uint32_t vmid_mask:16; + uint32_t unmap_latency:8; + uint32_t reserved1:5; + enum mes_set_resources_queue_type_enum queue_type:3; + } bitfields2; + uint32_t ordinal2; + }; + + uint32_t queue_mask_lo; + uint32_t queue_mask_hi; + uint32_t gws_mask_lo; + uint32_t gws_mask_hi; + + union { + struct { + uint32_t oac_mask:16; + uint32_t reserved2:16; + } bitfields7; + uint32_t ordinal7; + }; + + union { + struct { + uint32_t gds_heap_base:6; + uint32_t reserved3:5; + uint32_t gds_heap_size:6; + uint32_t reserved4:15; + } bitfields8; + uint32_t ordinal8; + }; + +}; +#endif + +/*--------------------MES_RUN_LIST--------------------*/ + +#ifndef PM4_MES_RUN_LIST_DEFINED +#define PM4_MES_RUN_LIST_DEFINED + +struct pm4_mes_runlist { + union { + union PM4_MES_TYPE_3_HEADER header; /* header */ + uint32_t ordinal1; + }; + + union { + struct { + uint32_t reserved1:2; + uint32_t ib_base_lo:30; + } bitfields2; + uint32_t ordinal2; + }; + + union { + struct { + uint32_t ib_base_hi:16; + uint32_t reserved2:16; + } bitfields3; + uint32_t ordinal3; + }; + + union { + struct { + uint32_t ib_size:20; + uint32_t chain:1; + uint32_t offload_polling:1; + uint32_t reserved3:1; + uint32_t valid:1; + uint32_t reserved4:8; + } bitfields4; + uint32_t ordinal4; + }; + +}; +#endif + +/*--------------------MES_MAP_PROCESS--------------------*/ + +#ifndef PM4_MES_MAP_PROCESS_DEFINED +#define PM4_MES_MAP_PROCESS_DEFINED + +struct pm4_mes_map_process { + union { + union PM4_MES_TYPE_3_HEADER header; /* header */ + uint32_t ordinal1; + }; + + union { + struct { + uint32_t pasid:16; + uint32_t reserved1:8; + uint32_t diq_enable:1; + uint32_t process_quantum:7; + } bitfields2; + uint32_t ordinal2; +}; + + union { + struct { + uint32_t page_table_base:28; + uint32_t reserved2:4; + } bitfields3; + uint32_t ordinal3; + }; + + uint32_t sh_mem_bases; + uint32_t sh_mem_ape1_base; + uint32_t sh_mem_ape1_limit; + uint32_t sh_mem_config; + uint32_t gds_addr_lo; + uint32_t gds_addr_hi; + + union { + struct { + uint32_t num_gws:6; + uint32_t reserved3:2; + uint32_t num_oac:4; + uint32_t reserved4:4; + uint32_t gds_size:6; + uint32_t num_queues:10; + } bitfields10; + uint32_t ordinal10; + }; + +}; +#endif + +/*--------------------MES_MAP_QUEUES--------------------*/ + +#ifndef PM4_MES_MAP_QUEUES_VI_DEFINED +#define PM4_MES_MAP_QUEUES_VI_DEFINED +enum mes_map_queues_queue_sel_vi_enum { + queue_sel__mes_map_queues__map_to_specified_queue_slots_vi = 0, +queue_sel__mes_map_queues__map_to_hws_determined_queue_slots_vi = 1 +}; + +enum mes_map_queues_queue_type_vi_enum { + queue_type__mes_map_queues__normal_compute_vi = 0, + queue_type__mes_map_queues__debug_interface_queue_vi = 1, + queue_type__mes_map_queues__normal_latency_static_queue_vi = 2, +queue_type__mes_map_queues__low_latency_static_queue_vi = 3 +}; + +enum mes_map_queues_alloc_format_vi_enum { + alloc_format__mes_map_queues__one_per_pipe_vi = 0, +alloc_format__mes_map_queues__all_on_one_pipe_vi = 1 +}; + +enum mes_map_queues_engine_sel_vi_enum { + engine_sel__mes_map_queues__compute_vi = 0, + engine_sel__mes_map_queues__sdma0_vi = 2, + engine_sel__mes_map_queues__sdma1_vi = 3 +}; + + +struct pm4_mes_map_queues { + union { + union PM4_MES_TYPE_3_HEADER header; /* header */ + uint32_t ordinal1; + }; + + union { + struct { + uint32_t reserved1:4; + enum mes_map_queues_queue_sel_vi_enum queue_sel:2; + uint32_t reserved2:15; + enum mes_map_queues_queue_type_vi_enum queue_type:3; + enum mes_map_queues_alloc_format_vi_enum alloc_format:2; + enum mes_map_queues_engine_sel_vi_enum engine_sel:3; + uint32_t num_queues:3; + } bitfields2; + uint32_t ordinal2; + }; + + union { + struct { + uint32_t reserved3:1; + uint32_t check_disable:1; + uint32_t doorbell_offset:21; + uint32_t reserved4:3; + uint32_t queue:6; + } bitfields3; + uint32_t ordinal3; + }; + + uint32_t mqd_addr_lo; + uint32_t mqd_addr_hi; + uint32_t wptr_addr_lo; + uint32_t wptr_addr_hi; +}; +#endif + +/*--------------------MES_QUERY_STATUS--------------------*/ + +#ifndef PM4_MES_QUERY_STATUS_DEFINED +#define PM4_MES_QUERY_STATUS_DEFINED +enum mes_query_status_interrupt_sel_enum { + interrupt_sel__mes_query_status__completion_status = 0, + interrupt_sel__mes_query_status__process_status = 1, + interrupt_sel__mes_query_status__queue_status = 2 +}; + +enum mes_query_status_command_enum { + command__mes_query_status__interrupt_only = 0, + command__mes_query_status__fence_only_immediate = 1, + command__mes_query_status__fence_only_after_write_ack = 2, + command__mes_query_status__fence_wait_for_write_ack_send_interrupt = 3 +}; + +enum mes_query_status_engine_sel_enum { + engine_sel__mes_query_status__compute = 0, + engine_sel__mes_query_status__sdma0_queue = 2, + engine_sel__mes_query_status__sdma1_queue = 3 +}; + +struct pm4_mes_query_status { + union { + union PM4_MES_TYPE_3_HEADER header; /* header */ + uint32_t ordinal1; + }; + + union { + struct { + uint32_t context_id:28; + enum mes_query_status_interrupt_sel_enum + interrupt_sel:2; + enum mes_query_status_command_enum command:2; + } bitfields2; + uint32_t ordinal2; + }; + + union { + struct { + uint32_t pasid:16; + uint32_t reserved1:16; + } bitfields3a; + struct { + uint32_t reserved2:2; + uint32_t doorbell_offset:21; + uint32_t reserved3:2; + enum mes_query_status_engine_sel_enum engine_sel:3; + uint32_t reserved4:4; + } bitfields3b; + uint32_t ordinal3; + }; + + uint32_t addr_lo; + uint32_t addr_hi; + uint32_t data_lo; + uint32_t data_hi; +}; +#endif + +/*--------------------MES_UNMAP_QUEUES--------------------*/ + +#ifndef PM4_MES_UNMAP_QUEUES_DEFINED +#define PM4_MES_UNMAP_QUEUES_DEFINED +enum mes_unmap_queues_action_enum { + action__mes_unmap_queues__preempt_queues = 0, + action__mes_unmap_queues__reset_queues = 1, + action__mes_unmap_queues__disable_process_queues = 2, + action__mes_unmap_queues__reserved = 3 +}; + +enum mes_unmap_queues_queue_sel_enum { + queue_sel__mes_unmap_queues__perform_request_on_specified_queues = 0, + queue_sel__mes_unmap_queues__perform_request_on_pasid_queues = 1, + queue_sel__mes_unmap_queues__unmap_all_queues = 2, + queue_sel__mes_unmap_queues__unmap_all_non_static_queues = 3 +}; + +enum mes_unmap_queues_engine_sel_enum { + engine_sel__mes_unmap_queues__compute = 0, + engine_sel__mes_unmap_queues__sdma0 = 2, + engine_sel__mes_unmap_queues__sdmal = 3 +}; + +struct PM4_MES_UNMAP_QUEUES { + union { + union PM4_MES_TYPE_3_HEADER header; /* header */ + uint32_t ordinal1; + }; + + union { + struct { + enum mes_unmap_queues_action_enum action:2; + uint32_t reserved1:2; + enum mes_unmap_queues_queue_sel_enum queue_sel:2; + uint32_t reserved2:20; + enum mes_unmap_queues_engine_sel_enum engine_sel:3; + uint32_t num_queues:3; + } bitfields2; + uint32_t ordinal2; + }; + + union { + struct { + uint32_t pasid:16; + uint32_t reserved3:16; + } bitfields3a; + struct { + uint32_t reserved4:2; + uint32_t doorbell_offset0:21; + uint32_t reserved5:9; + } bitfields3b; + uint32_t ordinal3; + }; + + union { + struct { + uint32_t reserved6:2; + uint32_t doorbell_offset1:21; + uint32_t reserved7:9; + } bitfields4; + uint32_t ordinal4; + }; + + union { + struct { + uint32_t reserved8:2; + uint32_t doorbell_offset2:21; + uint32_t reserved9:9; + } bitfields5; + uint32_t ordinal5; + }; + + union { + struct { + uint32_t reserved10:2; + uint32_t doorbell_offset3:21; + uint32_t reserved11:9; + } bitfields6; + uint32_t ordinal6; + }; +}; +#endif + +#endif diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c index c25728bc388a..74909e72a009 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c @@ -1186,6 +1186,11 @@ int kfd_topology_add_device(struct kfd_dev *gpu) * TODO: Retrieve max engine clock values from KGD */ + if (dev->gpu->device_info->asic_family == CHIP_CARRIZO) { + dev->node_props.capability |= HSA_CAP_DOORBELL_PACKET_TYPE; + pr_info("amdkfd: adding doorbell packet type capability\n"); + } + res = 0; err: diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h index 989624b3cd14..c3ddb9b95ff8 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h @@ -40,6 +40,7 @@ #define HSA_CAP_WATCH_POINTS_TOTALBITS_MASK 0x00000f00 #define HSA_CAP_WATCH_POINTS_TOTALBITS_SHIFT 8 #define HSA_CAP_RESERVED 0xfffff000 +#define HSA_CAP_DOORBELL_PACKET_TYPE 0x00001000 struct kfd_node_properties { uint32_t cpu_cores_count; diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h index 5bdf1b4397a0..68a8eaa1b7d0 100644 --- a/drivers/gpu/drm/amd/include/amd_shared.h +++ b/drivers/gpu/drm/amd/include/amd_shared.h @@ -23,6 +23,45 @@ #ifndef __AMD_SHARED_H__ #define __AMD_SHARED_H__ +#define AMD_MAX_USEC_TIMEOUT 100000 /* 100 ms */ + +/* +* Supported GPU families (aligned with amdgpu_drm.h) +*/ +#define AMD_FAMILY_UNKNOWN 0 +#define AMD_FAMILY_CI 120 /* Bonaire, Hawaii */ +#define AMD_FAMILY_KV 125 /* Kaveri, Kabini, Mullins */ +#define AMD_FAMILY_VI 130 /* Iceland, Tonga */ +#define AMD_FAMILY_CZ 135 /* Carrizo */ + +/* + * Supported ASIC types + */ +enum amd_asic_type { + CHIP_BONAIRE = 0, + CHIP_KAVERI, + CHIP_KABINI, + CHIP_HAWAII, + CHIP_MULLINS, + CHIP_TOPAZ, + CHIP_TONGA, + CHIP_FIJI, + CHIP_CARRIZO, + CHIP_LAST, +}; + +/* + * Chip flags + */ +enum amd_chip_flags { + AMD_ASIC_MASK = 0x0000ffffUL, + AMD_FLAGS_MASK = 0xffff0000UL, + AMD_IS_MOBILITY = 0x00010000UL, + AMD_IS_APU = 0x00020000UL, + AMD_IS_PX = 0x00040000UL, + AMD_EXP_HW_SUPPORT = 0x00080000UL, +}; + enum amd_ip_block_type { AMD_IP_BLOCK_TYPE_COMMON, AMD_IP_BLOCK_TYPE_GMC, diff --git a/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_d.h b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_d.h new file mode 100644 index 000000000000..44b1855cb8df --- /dev/null +++ b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_d.h @@ -0,0 +1,1246 @@ +/* + * SMU_7_1_3 Register documentation + * + * Copyright (C) 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef SMU_7_1_3_D_H +#define SMU_7_1_3_D_H + +#define mmGCK_SMC_IND_INDEX 0x80 +#define mmGCK0_GCK_SMC_IND_INDEX 0x80 +#define mmGCK1_GCK_SMC_IND_INDEX 0x82 +#define mmGCK2_GCK_SMC_IND_INDEX 0x84 +#define mmGCK3_GCK_SMC_IND_INDEX 0x86 +#define mmGCK_SMC_IND_DATA 0x81 +#define mmGCK0_GCK_SMC_IND_DATA 0x81 +#define mmGCK1_GCK_SMC_IND_DATA 0x83 +#define mmGCK2_GCK_SMC_IND_DATA 0x85 +#define mmGCK3_GCK_SMC_IND_DATA 0x87 +#define ixGCK_MCLK_FUSES 0xc0500008 +#define ixCG_DCLK_CNTL 0xc050009c +#define ixCG_DCLK_STATUS 0xc05000a0 +#define ixCG_VCLK_CNTL 0xc05000a4 +#define ixCG_VCLK_STATUS 0xc05000a8 +#define ixCG_ECLK_CNTL 0xc05000ac +#define ixCG_ECLK_STATUS 0xc05000b0 +#define ixCG_ACLK_CNTL 0xc05000dc +#define ixCG_MCLK_CNTL 0xc0500120 +#define ixCG_MCLK_STATUS 0xc0500124 +#define ixGCK_DFS_BYPASS_CNTL 0xc0500118 +#define ixCG_SPLL_FUNC_CNTL 0xc0500140 +#define ixCG_SPLL_FUNC_CNTL_2 0xc0500144 +#define ixCG_SPLL_FUNC_CNTL_3 0xc0500148 +#define ixCG_SPLL_FUNC_CNTL_4 0xc050014c +#define ixCG_SPLL_FUNC_CNTL_5 0xc0500150 +#define ixCG_SPLL_FUNC_CNTL_6 0xc0500154 +#define ixCG_SPLL_FUNC_CNTL_7 0xc0500158 +#define ixSPLL_CNTL_MODE 0xc0500160 +#define ixCG_SPLL_SPREAD_SPECTRUM 0xc0500164 +#define ixCG_SPLL_SPREAD_SPECTRUM_2 0xc0500168 +#define ixMPLL_BYPASSCLK_SEL 0xc050019c +#define ixCG_CLKPIN_CNTL 0xc05001a0 +#define ixCG_CLKPIN_CNTL_2 0xc05001a4 +#define ixCG_CLKPIN_CNTL_DC 0xc0500204 +#define ixTHM_CLK_CNTL 0xc05001a8 +#define ixMISC_CLK_CTRL 0xc05001ac +#define ixGCK_PLL_TEST_CNTL 0xc05001c0 +#define ixGCK_PLL_TEST_CNTL_2 0xc05001c4 +#define ixGCK_ADFS_CLK_BYPASS_CNTL1 0xc05001c8 +#define mmSMC_IND_INDEX 0x80 +#define mmSMC0_SMC_IND_INDEX 0x80 +#define mmSMC1_SMC_IND_INDEX 0x82 +#define mmSMC2_SMC_IND_INDEX 0x84 +#define mmSMC3_SMC_IND_INDEX 0x86 +#define mmSMC_IND_DATA 0x81 +#define mmSMC0_SMC_IND_DATA 0x81 +#define mmSMC1_SMC_IND_DATA 0x83 +#define mmSMC2_SMC_IND_DATA 0x85 +#define mmSMC3_SMC_IND_DATA 0x87 +#define mmSMC_IND_INDEX_0 0x80 +#define mmSMC_IND_DATA_0 0x81 +#define mmSMC_IND_INDEX_1 0x82 +#define mmSMC_IND_DATA_1 0x83 +#define mmSMC_IND_INDEX_2 0x84 +#define mmSMC_IND_DATA_2 0x85 +#define mmSMC_IND_INDEX_3 0x86 +#define mmSMC_IND_DATA_3 0x87 +#define mmSMC_IND_INDEX_4 0x88 +#define mmSMC_IND_DATA_4 0x89 +#define mmSMC_IND_INDEX_5 0x8a +#define mmSMC_IND_DATA_5 0x8b +#define mmSMC_IND_INDEX_6 0x8c +#define mmSMC_IND_DATA_6 0x8d +#define mmSMC_IND_INDEX_7 0x8e +#define mmSMC_IND_DATA_7 0x8f +#define mmSMC_IND_ACCESS_CNTL 0x92 +#define mmSMC_MESSAGE_0 0x94 +#define mmSMC_RESP_0 0x95 +#define mmSMC_MESSAGE_1 0x96 +#define mmSMC_RESP_1 0x97 +#define mmSMC_MESSAGE_2 0x98 +#define mmSMC_RESP_2 0x99 +#define mmSMC_MESSAGE_3 0x9a +#define mmSMC_RESP_3 0x9b +#define mmSMC_MESSAGE_4 0x9c +#define mmSMC_RESP_4 0x9d +#define mmSMC_MESSAGE_5 0x9e +#define mmSMC_RESP_5 0x9f +#define mmSMC_MESSAGE_6 0xa0 +#define mmSMC_RESP_6 0xa1 +#define mmSMC_MESSAGE_7 0xa2 +#define mmSMC_RESP_7 0xa3 +#define mmSMC_MSG_ARG_0 0xa4 +#define mmSMC_MSG_ARG_1 0xa5 +#define mmSMC_MSG_ARG_2 0xa6 +#define mmSMC_MSG_ARG_3 0xa7 +#define mmSMC_MSG_ARG_4 0xa8 +#define mmSMC_MSG_ARG_5 0xa9 +#define mmSMC_MSG_ARG_6 0xaa +#define mmSMC_MSG_ARG_7 0xab +#define mmSMC_MESSAGE_8 0xb5 +#define mmSMC_RESP_8 0xb6 +#define mmSMC_MESSAGE_9 0xb7 +#define mmSMC_RESP_9 0xb8 +#define mmSMC_MESSAGE_10 0xb9 +#define mmSMC_RESP_10 0xba +#define mmSMC_MESSAGE_11 0xbb +#define mmSMC_RESP_11 0xbc +#define mmSMC_MSG_ARG_8 0xbd +#define mmSMC_MSG_ARG_9 0xbe +#define mmSMC_MSG_ARG_10 0xbf +#define mmSMC_MSG_ARG_11 0x93 +#define ixSMC_SYSCON_RESET_CNTL 0x80000000 +#define ixSMC_SYSCON_CLOCK_CNTL_0 0x80000004 +#define ixSMC_SYSCON_CLOCK_CNTL_1 0x80000008 +#define ixSMC_SYSCON_CLOCK_CNTL_2 0x8000000c +#define ixSMC_SYSCON_MISC_CNTL 0x80000010 +#define ixSMC_SYSCON_MSG_ARG_0 0x80000068 +#define ixSMC_PC_C 0x80000370 +#define ixSMC_SCRATCH9 0x80000424 +#define mmGPIOPAD_SW_INT_STAT 0x180 +#define mmGPIOPAD_STRENGTH 0x181 +#define mmGPIOPAD_MASK 0x182 +#define mmGPIOPAD_A 0x183 +#define mmGPIOPAD_EN 0x184 +#define mmGPIOPAD_Y 0x185 +#define mmGPIOPAD_PINSTRAPS 0x186 +#define mmGPIOPAD_INT_STAT_EN 0x187 +#define mmGPIOPAD_INT_STAT 0x188 +#define mmGPIOPAD_INT_STAT_AK 0x189 +#define mmGPIOPAD_INT_EN 0x18a +#define mmGPIOPAD_INT_TYPE 0x18b +#define mmGPIOPAD_INT_POLARITY 0x18c +#define mmGPIOPAD_EXTERN_TRIG_CNTL 0x18d +#define mmGPIOPAD_RCVR_SEL 0x191 +#define mmGPIOPAD_PU_EN 0x192 +#define mmGPIOPAD_PD_EN 0x193 +#define mmCG_FPS_CNT 0x1b6 +#define mmSMU_IND_INDEX_0 0x1a6 +#define mmSMU_IND_DATA_0 0x1a7 +#define mmSMU_IND_INDEX_1 0x1a8 +#define mmSMU_IND_DATA_1 0x1a9 +#define mmSMU_IND_INDEX_2 0x1aa +#define mmSMU_IND_DATA_2 0x1ab +#define mmSMU_IND_INDEX_3 0x1ac +#define mmSMU_IND_DATA_3 0x1ad +#define mmSMU_IND_INDEX_4 0x1ae +#define mmSMU_IND_DATA_4 0x1af +#define mmSMU_IND_INDEX_5 0x1b0 +#define mmSMU_IND_DATA_5 0x1b1 +#define mmSMU_IND_INDEX_6 0x1b2 +#define mmSMU_IND_DATA_6 0x1b3 +#define mmSMU_IND_INDEX_7 0x1b4 +#define mmSMU_IND_DATA_7 0x1b5 +#define mmSMU_SMC_IND_INDEX 0x80 +#define mmSMU0_SMU_SMC_IND_INDEX 0x80 +#define mmSMU1_SMU_SMC_IND_INDEX 0x82 +#define mmSMU2_SMU_SMC_IND_INDEX 0x84 +#define mmSMU3_SMU_SMC_IND_INDEX 0x86 +#define mmSMU_SMC_IND_DATA 0x81 +#define mmSMU0_SMU_SMC_IND_DATA 0x81 +#define mmSMU1_SMU_SMC_IND_DATA 0x83 +#define mmSMU2_SMU_SMC_IND_DATA 0x85 +#define mmSMU3_SMU_SMC_IND_DATA 0x87 +#define ixRCU_UC_EVENTS 0xc0000004 +#define ixRCU_MISC_CTRL 0xc0000010 +#define ixRCU_VIRT_RESET_REQ 0xc0000024 +#define ixCC_RCU_FUSES 0xc00c0000 +#define ixCC_SMU_MISC_FUSES 0xc00c0004 +#define ixCC_SCLK_VID_FUSES 0xc00c0008 +#define ixCC_GIO_IOCCFG_FUSES 0xc00c000c +#define ixCC_GIO_IOC_FUSES 0xc00c0010 +#define ixCC_SMU_TST_EFUSE1_MISC 0xc00c001c +#define ixCC_TST_ID_STRAPS 0xc00c0020 +#define ixCC_FCTRL_FUSES 0xc00c0024 +#define ixCC_HARVEST_FUSES 0xc00c0028 +#define ixSMU_MAIN_PLL_OP_FREQ 0xe0003020 +#define ixSMU_STATUS 0xe0003088 +#define ixSMU_FIRMWARE 0xe00030a4 +#define ixSMU_INPUT_DATA 0xe00030b8 +#define ixSMU_EFUSE_0 0xc0100000 +#define ixFIRMWARE_FLAGS 0x3f000 +#define ixTDC_STATUS 0x3f004 +#define ixTDC_MV_AVERAGE 0x3f008 +#define ixTDC_VRM_LIMIT 0x3f00c +#define ixFEATURE_STATUS 0x3f010 +#define ixENTITY_TEMPERATURES_1 0x3f014 +#define ixMCARB_DRAM_TIMING_TABLE_1 0x3f018 +#define ixMCARB_DRAM_TIMING_TABLE_2 0x3f01c +#define ixMCARB_DRAM_TIMING_TABLE_3 0x3f020 +#define ixMCARB_DRAM_TIMING_TABLE_4 0x3f024 +#define ixMCARB_DRAM_TIMING_TABLE_5 0x3f028 +#define ixMCARB_DRAM_TIMING_TABLE_6 0x3f02c +#define ixMCARB_DRAM_TIMING_TABLE_7 0x3f030 +#define ixMCARB_DRAM_TIMING_TABLE_8 0x3f034 +#define ixMCARB_DRAM_TIMING_TABLE_9 0x3f038 +#define ixMCARB_DRAM_TIMING_TABLE_10 0x3f03c +#define ixMCARB_DRAM_TIMING_TABLE_11 0x3f040 +#define ixMCARB_DRAM_TIMING_TABLE_12 0x3f044 +#define ixMCARB_DRAM_TIMING_TABLE_13 0x3f048 +#define ixMCARB_DRAM_TIMING_TABLE_14 0x3f04c +#define ixMCARB_DRAM_TIMING_TABLE_15 0x3f050 +#define ixMCARB_DRAM_TIMING_TABLE_16 0x3f054 +#define ixMCARB_DRAM_TIMING_TABLE_17 0x3f058 +#define ixMCARB_DRAM_TIMING_TABLE_18 0x3f05c +#define ixMCARB_DRAM_TIMING_TABLE_19 0x3f060 +#define ixMCARB_DRAM_TIMING_TABLE_20 0x3f064 +#define ixMCARB_DRAM_TIMING_TABLE_21 0x3f068 +#define ixMCARB_DRAM_TIMING_TABLE_22 0x3f06c +#define ixMCARB_DRAM_TIMING_TABLE_23 0x3f070 +#define ixMCARB_DRAM_TIMING_TABLE_24 0x3f074 +#define ixMCARB_DRAM_TIMING_TABLE_25 0x3f078 +#define ixMCARB_DRAM_TIMING_TABLE_26 0x3f07c +#define ixMCARB_DRAM_TIMING_TABLE_27 0x3f080 +#define ixMCARB_DRAM_TIMING_TABLE_28 0x3f084 +#define ixMCARB_DRAM_TIMING_TABLE_29 0x3f088 +#define ixMCARB_DRAM_TIMING_TABLE_30 0x3f08c +#define ixMCARB_DRAM_TIMING_TABLE_31 0x3f090 +#define ixMCARB_DRAM_TIMING_TABLE_32 0x3f094 +#define ixMCARB_DRAM_TIMING_TABLE_33 0x3f098 +#define ixMCARB_DRAM_TIMING_TABLE_34 0x3f09c +#define ixMCARB_DRAM_TIMING_TABLE_35 0x3f0a0 +#define ixMCARB_DRAM_TIMING_TABLE_36 0x3f0a4 +#define ixMCARB_DRAM_TIMING_TABLE_37 0x3f0a8 +#define ixMCARB_DRAM_TIMING_TABLE_38 0x3f0ac +#define ixMCARB_DRAM_TIMING_TABLE_39 0x3f0b0 +#define ixMCARB_DRAM_TIMING_TABLE_40 0x3f0b4 +#define ixMCARB_DRAM_TIMING_TABLE_41 0x3f0b8 +#define ixMCARB_DRAM_TIMING_TABLE_42 0x3f0bc +#define ixMCARB_DRAM_TIMING_TABLE_43 0x3f0c0 +#define ixMCARB_DRAM_TIMING_TABLE_44 0x3f0c4 +#define ixMCARB_DRAM_TIMING_TABLE_45 0x3f0c8 +#define ixMCARB_DRAM_TIMING_TABLE_46 0x3f0cc +#define ixMCARB_DRAM_TIMING_TABLE_47 0x3f0d0 +#define ixMCARB_DRAM_TIMING_TABLE_48 0x3f0d4 +#define ixMCARB_DRAM_TIMING_TABLE_49 0x3f0d8 +#define ixMCARB_DRAM_TIMING_TABLE_50 0x3f0dc +#define ixMCARB_DRAM_TIMING_TABLE_51 0x3f0e0 +#define ixMCARB_DRAM_TIMING_TABLE_52 0x3f0e4 +#define ixMCARB_DRAM_TIMING_TABLE_53 0x3f0e8 +#define ixMCARB_DRAM_TIMING_TABLE_54 0x3f0ec +#define ixMCARB_DRAM_TIMING_TABLE_55 0x3f0f0 +#define ixMCARB_DRAM_TIMING_TABLE_56 0x3f0f4 +#define ixMCARB_DRAM_TIMING_TABLE_57 0x3f0f8 +#define ixMCARB_DRAM_TIMING_TABLE_58 0x3f0fc +#define ixMCARB_DRAM_TIMING_TABLE_59 0x3f100 +#define ixMCARB_DRAM_TIMING_TABLE_60 0x3f104 +#define ixMCARB_DRAM_TIMING_TABLE_61 0x3f108 +#define ixMCARB_DRAM_TIMING_TABLE_62 0x3f10c +#define ixMCARB_DRAM_TIMING_TABLE_63 0x3f110 +#define ixMCARB_DRAM_TIMING_TABLE_64 0x3f114 +#define ixMCARB_DRAM_TIMING_TABLE_65 0x3f118 +#define ixMCARB_DRAM_TIMING_TABLE_66 0x3f11c +#define ixMCARB_DRAM_TIMING_TABLE_67 0x3f120 +#define ixMCARB_DRAM_TIMING_TABLE_68 0x3f124 +#define ixMCARB_DRAM_TIMING_TABLE_69 0x3f128 +#define ixMCARB_DRAM_TIMING_TABLE_70 0x3f12c +#define ixMCARB_DRAM_TIMING_TABLE_71 0x3f130 +#define ixMCARB_DRAM_TIMING_TABLE_72 0x3f134 +#define ixMCARB_DRAM_TIMING_TABLE_73 0x3f138 +#define ixMCARB_DRAM_TIMING_TABLE_74 0x3f13c +#define ixMCARB_DRAM_TIMING_TABLE_75 0x3f140 +#define ixMCARB_DRAM_TIMING_TABLE_76 0x3f144 +#define ixMCARB_DRAM_TIMING_TABLE_77 0x3f148 +#define ixMCARB_DRAM_TIMING_TABLE_78 0x3f14c +#define ixMCARB_DRAM_TIMING_TABLE_79 0x3f150 +#define ixMCARB_DRAM_TIMING_TABLE_80 0x3f154 +#define ixMCARB_DRAM_TIMING_TABLE_81 0x3f158 +#define ixMCARB_DRAM_TIMING_TABLE_82 0x3f15c +#define ixMCARB_DRAM_TIMING_TABLE_83 0x3f160 +#define ixMCARB_DRAM_TIMING_TABLE_84 0x3f164 +#define ixMCARB_DRAM_TIMING_TABLE_85 0x3f168 +#define ixMCARB_DRAM_TIMING_TABLE_86 0x3f16c +#define ixMCARB_DRAM_TIMING_TABLE_87 0x3f170 +#define ixMCARB_DRAM_TIMING_TABLE_88 0x3f174 +#define ixMCARB_DRAM_TIMING_TABLE_89 0x3f178 +#define ixMCARB_DRAM_TIMING_TABLE_90 0x3f17c +#define ixMCARB_DRAM_TIMING_TABLE_91 0x3f180 +#define ixMCARB_DRAM_TIMING_TABLE_92 0x3f184 +#define ixMCARB_DRAM_TIMING_TABLE_93 0x3f188 +#define ixMCARB_DRAM_TIMING_TABLE_94 0x3f18c +#define ixMCARB_DRAM_TIMING_TABLE_95 0x3f190 +#define ixMCARB_DRAM_TIMING_TABLE_96 0x3f194 +#define ixDPM_TABLE_1 0x3f198 +#define ixDPM_TABLE_2 0x3f19c +#define ixDPM_TABLE_3 0x3f1a0 +#define ixDPM_TABLE_4 0x3f1a4 +#define ixDPM_TABLE_5 0x3f1a8 +#define ixDPM_TABLE_6 0x3f1ac +#define ixDPM_TABLE_7 0x3f1b0 +#define ixDPM_TABLE_8 0x3f1b4 +#define ixDPM_TABLE_9 0x3f1b8 +#define ixDPM_TABLE_10 0x3f1bc +#define ixDPM_TABLE_11 0x3f1c0 +#define ixDPM_TABLE_12 0x3f1c4 +#define ixDPM_TABLE_13 0x3f1c8 +#define ixDPM_TABLE_14 0x3f1cc +#define ixDPM_TABLE_15 0x3f1d0 +#define ixDPM_TABLE_16 0x3f1d4 +#define ixDPM_TABLE_17 0x3f1d8 +#define ixDPM_TABLE_18 0x3f1dc +#define ixDPM_TABLE_19 0x3f1e0 +#define ixDPM_TABLE_20 0x3f1e4 +#define ixDPM_TABLE_21 0x3f1e8 +#define ixDPM_TABLE_22 0x3f1ec +#define ixDPM_TABLE_23 0x3f1f0 +#define ixDPM_TABLE_24 0x3f1f4 +#define ixDPM_TABLE_25 0x3f1f8 +#define ixDPM_TABLE_26 0x3f1fc +#define ixDPM_TABLE_27 0x3f200 +#define ixDPM_TABLE_28 0x3f204 +#define ixDPM_TABLE_29 0x3f208 +#define ixDPM_TABLE_30 0x3f20c +#define ixDPM_TABLE_31 0x3f210 +#define ixDPM_TABLE_32 0x3f214 +#define ixDPM_TABLE_33 0x3f218 +#define ixDPM_TABLE_34 0x3f21c +#define ixDPM_TABLE_35 0x3f220 +#define ixDPM_TABLE_36 0x3f224 +#define ixDPM_TABLE_37 0x3f228 +#define ixDPM_TABLE_38 0x3f22c +#define ixDPM_TABLE_39 0x3f230 +#define ixDPM_TABLE_40 0x3f234 +#define ixDPM_TABLE_41 0x3f238 +#define ixDPM_TABLE_42 0x3f23c +#define ixDPM_TABLE_43 0x3f240 +#define ixDPM_TABLE_44 0x3f244 +#define ixDPM_TABLE_45 0x3f248 +#define ixDPM_TABLE_46 0x3f24c +#define ixDPM_TABLE_47 0x3f250 +#define ixDPM_TABLE_48 0x3f254 +#define ixDPM_TABLE_49 0x3f258 +#define ixDPM_TABLE_50 0x3f25c +#define ixDPM_TABLE_51 0x3f260 +#define ixDPM_TABLE_52 0x3f264 +#define ixDPM_TABLE_53 0x3f268 +#define ixDPM_TABLE_54 0x3f26c +#define ixDPM_TABLE_55 0x3f270 +#define ixDPM_TABLE_56 0x3f274 +#define ixDPM_TABLE_57 0x3f278 +#define ixDPM_TABLE_58 0x3f27c +#define ixDPM_TABLE_59 0x3f280 +#define ixDPM_TABLE_60 0x3f284 +#define ixDPM_TABLE_61 0x3f288 +#define ixDPM_TABLE_62 0x3f28c +#define ixDPM_TABLE_63 0x3f290 +#define ixDPM_TABLE_64 0x3f294 +#define ixDPM_TABLE_65 0x3f298 +#define ixDPM_TABLE_66 0x3f29c +#define ixDPM_TABLE_67 0x3f2a0 +#define ixDPM_TABLE_68 0x3f2a4 +#define ixDPM_TABLE_69 0x3f2a8 +#define ixDPM_TABLE_70 0x3f2ac +#define ixDPM_TABLE_71 0x3f2b0 +#define ixDPM_TABLE_72 0x3f2b4 +#define ixDPM_TABLE_73 0x3f2b8 +#define ixDPM_TABLE_74 0x3f2bc +#define ixDPM_TABLE_75 0x3f2c0 +#define ixDPM_TABLE_76 0x3f2c4 +#define ixDPM_TABLE_77 0x3f2c8 +#define ixDPM_TABLE_78 0x3f2cc +#define ixDPM_TABLE_79 0x3f2d0 +#define ixDPM_TABLE_80 0x3f2d4 +#define ixDPM_TABLE_81 0x3f2d8 +#define ixDPM_TABLE_82 0x3f2dc +#define ixDPM_TABLE_83 0x3f2e0 +#define ixDPM_TABLE_84 0x3f2e4 +#define ixDPM_TABLE_85 0x3f2e8 +#define ixDPM_TABLE_86 0x3f2ec +#define ixDPM_TABLE_87 0x3f2f0 +#define ixDPM_TABLE_88 0x3f2f4 +#define ixDPM_TABLE_89 0x3f2f8 +#define ixDPM_TABLE_90 0x3f2fc +#define ixDPM_TABLE_91 0x3f300 +#define ixDPM_TABLE_92 0x3f304 +#define ixDPM_TABLE_93 0x3f308 +#define ixDPM_TABLE_94 0x3f30c +#define ixDPM_TABLE_95 0x3f310 +#define ixDPM_TABLE_96 0x3f314 +#define ixDPM_TABLE_97 0x3f318 +#define ixDPM_TABLE_98 0x3f31c +#define ixDPM_TABLE_99 0x3f320 +#define ixDPM_TABLE_100 0x3f324 +#define ixDPM_TABLE_101 0x3f328 +#define ixDPM_TABLE_102 0x3f32c +#define ixDPM_TABLE_103 0x3f330 +#define ixDPM_TABLE_104 0x3f334 +#define ixDPM_TABLE_105 0x3f338 +#define ixDPM_TABLE_106 0x3f33c +#define ixDPM_TABLE_107 0x3f340 +#define ixDPM_TABLE_108 0x3f344 +#define ixDPM_TABLE_109 0x3f348 +#define ixDPM_TABLE_110 0x3f34c +#define ixDPM_TABLE_111 0x3f350 +#define ixDPM_TABLE_112 0x3f354 +#define ixDPM_TABLE_113 0x3f358 +#define ixDPM_TABLE_114 0x3f35c +#define ixDPM_TABLE_115 0x3f360 +#define ixDPM_TABLE_116 0x3f364 +#define ixDPM_TABLE_117 0x3f368 +#define ixDPM_TABLE_118 0x3f36c +#define ixDPM_TABLE_119 0x3f370 +#define ixDPM_TABLE_120 0x3f374 +#define ixDPM_TABLE_121 0x3f378 +#define ixDPM_TABLE_122 0x3f37c +#define ixDPM_TABLE_123 0x3f380 +#define ixDPM_TABLE_124 0x3f384 +#define ixDPM_TABLE_125 0x3f388 +#define ixDPM_TABLE_126 0x3f38c +#define ixDPM_TABLE_127 0x3f390 +#define ixDPM_TABLE_128 0x3f394 +#define ixDPM_TABLE_129 0x3f398 +#define ixDPM_TABLE_130 0x3f39c +#define ixDPM_TABLE_131 0x3f3a0 +#define ixDPM_TABLE_132 0x3f3a4 +#define ixDPM_TABLE_133 0x3f3a8 +#define ixDPM_TABLE_134 0x3f3ac +#define ixDPM_TABLE_135 0x3f3b0 +#define ixDPM_TABLE_136 0x3f3b4 +#define ixDPM_TABLE_137 0x3f3b8 +#define ixDPM_TABLE_138 0x3f3bc +#define ixDPM_TABLE_139 0x3f3c0 +#define ixDPM_TABLE_140 0x3f3c4 +#define ixDPM_TABLE_141 0x3f3c8 +#define ixDPM_TABLE_142 0x3f3cc +#define ixDPM_TABLE_143 0x3f3d0 +#define ixDPM_TABLE_144 0x3f3d4 +#define ixDPM_TABLE_145 0x3f3d8 +#define ixDPM_TABLE_146 0x3f3dc +#define ixDPM_TABLE_147 0x3f3e0 +#define ixDPM_TABLE_148 0x3f3e4 +#define ixDPM_TABLE_149 0x3f3e8 +#define ixDPM_TABLE_150 0x3f3ec +#define ixDPM_TABLE_151 0x3f3f0 +#define ixDPM_TABLE_152 0x3f3f4 +#define ixDPM_TABLE_153 0x3f3f8 +#define ixDPM_TABLE_154 0x3f3fc +#define ixDPM_TABLE_155 0x3f400 +#define ixDPM_TABLE_156 0x3f404 +#define ixDPM_TABLE_157 0x3f408 +#define ixDPM_TABLE_158 0x3f40c +#define ixDPM_TABLE_159 0x3f410 +#define ixDPM_TABLE_160 0x3f414 +#define ixDPM_TABLE_161 0x3f418 +#define ixDPM_TABLE_162 0x3f41c +#define ixDPM_TABLE_163 0x3f420 +#define ixDPM_TABLE_164 0x3f424 +#define ixDPM_TABLE_165 0x3f428 +#define ixDPM_TABLE_166 0x3f42c +#define ixDPM_TABLE_167 0x3f430 +#define ixDPM_TABLE_168 0x3f434 +#define ixDPM_TABLE_169 0x3f438 +#define ixDPM_TABLE_170 0x3f43c +#define ixDPM_TABLE_171 0x3f440 +#define ixDPM_TABLE_172 0x3f444 +#define ixDPM_TABLE_173 0x3f448 +#define ixDPM_TABLE_174 0x3f44c +#define ixDPM_TABLE_175 0x3f450 +#define ixDPM_TABLE_176 0x3f454 +#define ixDPM_TABLE_177 0x3f458 +#define ixDPM_TABLE_178 0x3f45c +#define ixDPM_TABLE_179 0x3f460 +#define ixDPM_TABLE_180 0x3f464 +#define ixDPM_TABLE_181 0x3f468 +#define ixDPM_TABLE_182 0x3f46c +#define ixDPM_TABLE_183 0x3f470 +#define ixDPM_TABLE_184 0x3f474 +#define ixDPM_TABLE_185 0x3f478 +#define ixDPM_TABLE_186 0x3f47c +#define ixDPM_TABLE_187 0x3f480 +#define ixDPM_TABLE_188 0x3f484 +#define ixDPM_TABLE_189 0x3f488 +#define ixDPM_TABLE_190 0x3f48c +#define ixDPM_TABLE_191 0x3f490 +#define ixDPM_TABLE_192 0x3f494 +#define ixDPM_TABLE_193 0x3f498 +#define ixDPM_TABLE_194 0x3f49c +#define ixDPM_TABLE_195 0x3f4a0 +#define ixDPM_TABLE_196 0x3f4a4 +#define ixDPM_TABLE_197 0x3f4a8 +#define ixDPM_TABLE_198 0x3f4ac +#define ixDPM_TABLE_199 0x3f4b0 +#define ixDPM_TABLE_200 0x3f4b4 +#define ixDPM_TABLE_201 0x3f4b8 +#define ixDPM_TABLE_202 0x3f4bc +#define ixDPM_TABLE_203 0x3f4c0 +#define ixDPM_TABLE_204 0x3f4c4 +#define ixDPM_TABLE_205 0x3f4c8 +#define ixDPM_TABLE_206 0x3f4cc +#define ixDPM_TABLE_207 0x3f4d0 +#define ixDPM_TABLE_208 0x3f4d4 +#define ixDPM_TABLE_209 0x3f4d8 +#define ixDPM_TABLE_210 0x3f4dc +#define ixDPM_TABLE_211 0x3f4e0 +#define ixDPM_TABLE_212 0x3f4e4 +#define ixDPM_TABLE_213 0x3f4e8 +#define ixDPM_TABLE_214 0x3f4ec +#define ixDPM_TABLE_215 0x3f4f0 +#define ixDPM_TABLE_216 0x3f4f4 +#define ixDPM_TABLE_217 0x3f4f8 +#define ixDPM_TABLE_218 0x3f4fc +#define ixDPM_TABLE_219 0x3f500 +#define ixDPM_TABLE_220 0x3f504 +#define ixDPM_TABLE_221 0x3f508 +#define ixDPM_TABLE_222 0x3f50c +#define ixDPM_TABLE_223 0x3f510 +#define ixDPM_TABLE_224 0x3f514 +#define ixDPM_TABLE_225 0x3f518 +#define ixDPM_TABLE_226 0x3f51c +#define ixDPM_TABLE_227 0x3f520 +#define ixDPM_TABLE_228 0x3f524 +#define ixDPM_TABLE_229 0x3f528 +#define ixDPM_TABLE_230 0x3f52c +#define ixDPM_TABLE_231 0x3f530 +#define ixDPM_TABLE_232 0x3f534 +#define ixDPM_TABLE_233 0x3f538 +#define ixDPM_TABLE_234 0x3f53c +#define ixDPM_TABLE_235 0x3f540 +#define ixDPM_TABLE_236 0x3f544 +#define ixDPM_TABLE_237 0x3f548 +#define ixDPM_TABLE_238 0x3f54c +#define ixDPM_TABLE_239 0x3f550 +#define ixDPM_TABLE_240 0x3f554 +#define ixDPM_TABLE_241 0x3f558 +#define ixDPM_TABLE_242 0x3f55c +#define ixDPM_TABLE_243 0x3f560 +#define ixDPM_TABLE_244 0x3f564 +#define ixDPM_TABLE_245 0x3f568 +#define ixDPM_TABLE_246 0x3f56c +#define ixDPM_TABLE_247 0x3f570 +#define ixDPM_TABLE_248 0x3f574 +#define ixDPM_TABLE_249 0x3f578 +#define ixDPM_TABLE_250 0x3f57c +#define ixDPM_TABLE_251 0x3f580 +#define ixDPM_TABLE_252 0x3f584 +#define ixDPM_TABLE_253 0x3f588 +#define ixDPM_TABLE_254 0x3f58c +#define ixDPM_TABLE_255 0x3f590 +#define ixDPM_TABLE_256 0x3f594 +#define ixDPM_TABLE_257 0x3f598 +#define ixDPM_TABLE_258 0x3f59c +#define ixDPM_TABLE_259 0x3f5a0 +#define ixDPM_TABLE_260 0x3f5a4 +#define ixDPM_TABLE_261 0x3f5a8 +#define ixDPM_TABLE_262 0x3f5ac +#define ixDPM_TABLE_263 0x3f5b0 +#define ixDPM_TABLE_264 0x3f5b4 +#define ixDPM_TABLE_265 0x3f5b8 +#define ixDPM_TABLE_266 0x3f5bc +#define ixDPM_TABLE_267 0x3f5c0 +#define ixDPM_TABLE_268 0x3f5c4 +#define ixDPM_TABLE_269 0x3f5c8 +#define ixDPM_TABLE_270 0x3f5cc +#define ixDPM_TABLE_271 0x3f5d0 +#define ixDPM_TABLE_272 0x3f5d4 +#define ixDPM_TABLE_273 0x3f5d8 +#define ixDPM_TABLE_274 0x3f5dc +#define ixDPM_TABLE_275 0x3f5e0 +#define ixDPM_TABLE_276 0x3f5e4 +#define ixDPM_TABLE_277 0x3f5e8 +#define ixDPM_TABLE_278 0x3f5ec +#define ixDPM_TABLE_279 0x3f5f0 +#define ixDPM_TABLE_280 0x3f5f4 +#define ixDPM_TABLE_281 0x3f5f8 +#define ixDPM_TABLE_282 0x3f5fc +#define ixDPM_TABLE_283 0x3f600 +#define ixDPM_TABLE_284 0x3f604 +#define ixDPM_TABLE_285 0x3f608 +#define ixDPM_TABLE_286 0x3f60c +#define ixDPM_TABLE_287 0x3f610 +#define ixDPM_TABLE_288 0x3f614 +#define ixDPM_TABLE_289 0x3f618 +#define ixDPM_TABLE_290 0x3f61c +#define ixDPM_TABLE_291 0x3f620 +#define ixDPM_TABLE_292 0x3f624 +#define ixDPM_TABLE_293 0x3f628 +#define ixDPM_TABLE_294 0x3f62c +#define ixDPM_TABLE_295 0x3f630 +#define ixDPM_TABLE_296 0x3f634 +#define ixDPM_TABLE_297 0x3f638 +#define ixDPM_TABLE_298 0x3f63c +#define ixDPM_TABLE_299 0x3f640 +#define ixDPM_TABLE_300 0x3f644 +#define ixDPM_TABLE_301 0x3f648 +#define ixDPM_TABLE_302 0x3f64c +#define ixDPM_TABLE_303 0x3f650 +#define ixDPM_TABLE_304 0x3f654 +#define ixDPM_TABLE_305 0x3f658 +#define ixDPM_TABLE_306 0x3f65c +#define ixDPM_TABLE_307 0x3f660 +#define ixDPM_TABLE_308 0x3f664 +#define ixDPM_TABLE_309 0x3f668 +#define ixDPM_TABLE_310 0x3f66c +#define ixDPM_TABLE_311 0x3f670 +#define ixDPM_TABLE_312 0x3f674 +#define ixDPM_TABLE_313 0x3f678 +#define ixDPM_TABLE_314 0x3f67c +#define ixDPM_TABLE_315 0x3f680 +#define ixDPM_TABLE_316 0x3f684 +#define ixDPM_TABLE_317 0x3f688 +#define ixDPM_TABLE_318 0x3f68c +#define ixDPM_TABLE_319 0x3f690 +#define ixDPM_TABLE_320 0x3f694 +#define ixDPM_TABLE_321 0x3f698 +#define ixDPM_TABLE_322 0x3f69c +#define ixDPM_TABLE_323 0x3f6a0 +#define ixDPM_TABLE_324 0x3f6a4 +#define ixDPM_TABLE_325 0x3f6a8 +#define ixDPM_TABLE_326 0x3f6ac +#define ixDPM_TABLE_327 0x3f6b0 +#define ixDPM_TABLE_328 0x3f6b4 +#define ixDPM_TABLE_329 0x3f6b8 +#define ixDPM_TABLE_330 0x3f6bc +#define ixDPM_TABLE_331 0x3f6c0 +#define ixDPM_TABLE_332 0x3f6c4 +#define ixDPM_TABLE_333 0x3f6c8 +#define ixDPM_TABLE_334 0x3f6cc +#define ixDPM_TABLE_335 0x3f6d0 +#define ixDPM_TABLE_336 0x3f6d4 +#define ixDPM_TABLE_337 0x3f6d8 +#define ixDPM_TABLE_338 0x3f6dc +#define ixDPM_TABLE_339 0x3f6e0 +#define ixDPM_TABLE_340 0x3f6e4 +#define ixDPM_TABLE_341 0x3f6e8 +#define ixDPM_TABLE_342 0x3f6ec +#define ixDPM_TABLE_343 0x3f6f0 +#define ixDPM_TABLE_344 0x3f6f4 +#define ixDPM_TABLE_345 0x3f6f8 +#define ixDPM_TABLE_346 0x3f6fc +#define ixDPM_TABLE_347 0x3f700 +#define ixDPM_TABLE_348 0x3f704 +#define ixDPM_TABLE_349 0x3f708 +#define ixDPM_TABLE_350 0x3f70c +#define ixDPM_TABLE_351 0x3f710 +#define ixDPM_TABLE_352 0x3f714 +#define ixDPM_TABLE_353 0x3f718 +#define ixDPM_TABLE_354 0x3f71c +#define ixDPM_TABLE_355 0x3f720 +#define ixDPM_TABLE_356 0x3f724 +#define ixDPM_TABLE_357 0x3f728 +#define ixDPM_TABLE_358 0x3f72c +#define ixDPM_TABLE_359 0x3f730 +#define ixDPM_TABLE_360 0x3f734 +#define ixDPM_TABLE_361 0x3f738 +#define ixDPM_TABLE_362 0x3f73c +#define ixDPM_TABLE_363 0x3f740 +#define ixDPM_TABLE_364 0x3f744 +#define ixDPM_TABLE_365 0x3f748 +#define ixDPM_TABLE_366 0x3f74c +#define ixDPM_TABLE_367 0x3f750 +#define ixDPM_TABLE_368 0x3f754 +#define ixDPM_TABLE_369 0x3f758 +#define ixDPM_TABLE_370 0x3f75c +#define ixDPM_TABLE_371 0x3f760 +#define ixDPM_TABLE_372 0x3f764 +#define ixDPM_TABLE_373 0x3f768 +#define ixDPM_TABLE_374 0x3f76c +#define ixDPM_TABLE_375 0x3f770 +#define ixDPM_TABLE_376 0x3f774 +#define ixDPM_TABLE_377 0x3f778 +#define ixDPM_TABLE_378 0x3f77c +#define ixDPM_TABLE_379 0x3f780 +#define ixDPM_TABLE_380 0x3f784 +#define ixDPM_TABLE_381 0x3f788 +#define ixDPM_TABLE_382 0x3f78c +#define ixDPM_TABLE_383 0x3f790 +#define ixDPM_TABLE_384 0x3f794 +#define ixDPM_TABLE_385 0x3f798 +#define ixDPM_TABLE_386 0x3f79c +#define ixDPM_TABLE_387 0x3f7a0 +#define ixDPM_TABLE_388 0x3f7a4 +#define ixDPM_TABLE_389 0x3f7a8 +#define ixDPM_TABLE_390 0x3f7ac +#define ixDPM_TABLE_391 0x3f7b0 +#define ixDPM_TABLE_392 0x3f7b4 +#define ixDPM_TABLE_393 0x3f7b8 +#define ixDPM_TABLE_394 0x3f7bc +#define ixDPM_TABLE_395 0x3f7c0 +#define ixDPM_TABLE_396 0x3f7c4 +#define ixDPM_TABLE_397 0x3f7c8 +#define ixDPM_TABLE_398 0x3f7cc +#define ixDPM_TABLE_399 0x3f7d0 +#define ixDPM_TABLE_400 0x3f7d4 +#define ixDPM_TABLE_401 0x3f7d8 +#define ixDPM_TABLE_402 0x3f7dc +#define ixDPM_TABLE_403 0x3f7e0 +#define ixDPM_TABLE_404 0x3f7e4 +#define ixDPM_TABLE_405 0x3f7e8 +#define ixDPM_TABLE_406 0x3f7ec +#define ixDPM_TABLE_407 0x3f7f0 +#define ixDPM_TABLE_408 0x3f7f4 +#define ixDPM_TABLE_409 0x3f7f8 +#define ixDPM_TABLE_410 0x3f7fc +#define ixDPM_TABLE_411 0x3f800 +#define ixDPM_TABLE_412 0x3f804 +#define ixDPM_TABLE_413 0x3f808 +#define ixDPM_TABLE_414 0x3f80c +#define ixDPM_TABLE_415 0x3f810 +#define ixDPM_TABLE_416 0x3f814 +#define ixDPM_TABLE_417 0x3f818 +#define ixDPM_TABLE_418 0x3f81c +#define ixDPM_TABLE_419 0x3f820 +#define ixDPM_TABLE_420 0x3f824 +#define ixDPM_TABLE_421 0x3f828 +#define ixDPM_TABLE_422 0x3f82c +#define ixDPM_TABLE_423 0x3f830 +#define ixDPM_TABLE_424 0x3f834 +#define ixDPM_TABLE_425 0x3f838 +#define ixDPM_TABLE_426 0x3f83c +#define ixDPM_TABLE_427 0x3f840 +#define ixDPM_TABLE_428 0x3f844 +#define ixDPM_TABLE_429 0x3f848 +#define ixDPM_TABLE_430 0x3f84c +#define ixDPM_TABLE_431 0x3f850 +#define ixDPM_TABLE_432 0x3f854 +#define ixDPM_TABLE_433 0x3f858 +#define ixDPM_TABLE_434 0x3f85c +#define ixDPM_TABLE_435 0x3f860 +#define ixDPM_TABLE_436 0x3f864 +#define ixDPM_TABLE_437 0x3f868 +#define ixDPM_TABLE_438 0x3f86c +#define ixDPM_TABLE_439 0x3f870 +#define ixDPM_TABLE_440 0x3f874 +#define ixSOFT_REGISTERS_TABLE_1 0x3f89c +#define ixSOFT_REGISTERS_TABLE_2 0x3f8a0 +#define ixSOFT_REGISTERS_TABLE_3 0x3f8a4 +#define ixSOFT_REGISTERS_TABLE_4 0x3f8a8 +#define ixSOFT_REGISTERS_TABLE_5 0x3f8ac +#define ixSOFT_REGISTERS_TABLE_6 0x3f8b0 +#define ixSOFT_REGISTERS_TABLE_7 0x3f8b4 +#define ixSOFT_REGISTERS_TABLE_8 0x3f8b8 +#define ixSOFT_REGISTERS_TABLE_9 0x3f8bc +#define ixSOFT_REGISTERS_TABLE_10 0x3f8c0 +#define ixSOFT_REGISTERS_TABLE_11 0x3f8c4 +#define ixSOFT_REGISTERS_TABLE_12 0x3f8c8 +#define ixSOFT_REGISTERS_TABLE_13 0x3f8cc +#define ixSOFT_REGISTERS_TABLE_14 0x3f8d0 +#define ixSOFT_REGISTERS_TABLE_15 0x3f8d4 +#define ixSOFT_REGISTERS_TABLE_16 0x3f8d8 +#define ixSOFT_REGISTERS_TABLE_17 0x3f8dc +#define ixSOFT_REGISTERS_TABLE_18 0x3f8e0 +#define ixSOFT_REGISTERS_TABLE_19 0x3f8e4 +#define ixSOFT_REGISTERS_TABLE_20 0x3f8e8 +#define ixSOFT_REGISTERS_TABLE_21 0x3f8ec +#define ixSOFT_REGISTERS_TABLE_22 0x3f8f0 +#define ixSOFT_REGISTERS_TABLE_23 0x3f8f4 +#define ixSOFT_REGISTERS_TABLE_24 0x3f8f8 +#define ixSOFT_REGISTERS_TABLE_25 0x3f8fc +#define ixSOFT_REGISTERS_TABLE_26 0x3f900 +#define ixSOFT_REGISTERS_TABLE_27 0x3f904 +#define ixSOFT_REGISTERS_TABLE_28 0x3f888 +#define ixSOFT_REGISTERS_TABLE_29 0x3f90c +#define ixSOFT_REGISTERS_TABLE_30 0x3f910 +#define ixPM_FUSES_1 0x3f914 +#define ixPM_FUSES_2 0x3f918 +#define ixPM_FUSES_3 0x3f91c +#define ixPM_FUSES_4 0x3f920 +#define ixPM_FUSES_5 0x3f924 +#define ixPM_FUSES_6 0x3f928 +#define ixPM_FUSES_7 0x3f92c +#define ixPM_FUSES_8 0x3f930 +#define ixPM_FUSES_9 0x3f934 +#define ixPM_FUSES_10 0x3f938 +#define ixPM_FUSES_11 0x3f93c +#define ixPM_FUSES_12 0x3f940 +#define ixPM_FUSES_13 0x3f944 +#define ixPM_FUSES_14 0x3f948 +#define ixPM_FUSES_15 0x3f94c +#define ixSMU_PM_STATUS_0 0x3fe00 +#define ixSMU_PM_STATUS_1 0x3fe04 +#define ixSMU_PM_STATUS_2 0x3fe08 +#define ixSMU_PM_STATUS_3 0x3fe0c +#define ixSMU_PM_STATUS_4 0x3fe10 +#define ixSMU_PM_STATUS_5 0x3fe14 +#define ixSMU_PM_STATUS_6 0x3fe18 +#define ixSMU_PM_STATUS_7 0x3fe1c +#define ixSMU_PM_STATUS_8 0x3fe20 +#define ixSMU_PM_STATUS_9 0x3fe24 +#define ixSMU_PM_STATUS_10 0x3fe28 +#define ixSMU_PM_STATUS_11 0x3fe2c +#define ixSMU_PM_STATUS_12 0x3fe30 +#define ixSMU_PM_STATUS_13 0x3fe34 +#define ixSMU_PM_STATUS_14 0x3fe38 +#define ixSMU_PM_STATUS_15 0x3fe3c +#define ixSMU_PM_STATUS_16 0x3fe40 +#define ixSMU_PM_STATUS_17 0x3fe44 +#define ixSMU_PM_STATUS_18 0x3fe48 +#define ixSMU_PM_STATUS_19 0x3fe4c +#define ixSMU_PM_STATUS_20 0x3fe50 +#define ixSMU_PM_STATUS_21 0x3fe54 +#define ixSMU_PM_STATUS_22 0x3fe58 +#define ixSMU_PM_STATUS_23 0x3fe5c +#define ixSMU_PM_STATUS_24 0x3fe60 +#define ixSMU_PM_STATUS_25 0x3fe64 +#define ixSMU_PM_STATUS_26 0x3fe68 +#define ixSMU_PM_STATUS_27 0x3fe6c +#define ixSMU_PM_STATUS_28 0x3fe70 +#define ixSMU_PM_STATUS_29 0x3fe74 +#define ixSMU_PM_STATUS_30 0x3fe78 +#define ixSMU_PM_STATUS_31 0x3fe7c +#define ixSMU_PM_STATUS_32 0x3fe80 +#define ixSMU_PM_STATUS_33 0x3fe84 +#define ixSMU_PM_STATUS_34 0x3fe88 +#define ixSMU_PM_STATUS_35 0x3fe8c +#define ixSMU_PM_STATUS_36 0x3fe90 +#define ixSMU_PM_STATUS_37 0x3fe94 +#define ixSMU_PM_STATUS_38 0x3fe98 +#define ixSMU_PM_STATUS_39 0x3fe9c +#define ixSMU_PM_STATUS_40 0x3fea0 +#define ixSMU_PM_STATUS_41 0x3fea4 +#define ixSMU_PM_STATUS_42 0x3fea8 +#define ixSMU_PM_STATUS_43 0x3feac +#define ixSMU_PM_STATUS_44 0x3feb0 +#define ixSMU_PM_STATUS_45 0x3feb4 +#define ixSMU_PM_STATUS_46 0x3feb8 +#define ixSMU_PM_STATUS_47 0x3febc +#define ixSMU_PM_STATUS_48 0x3fec0 +#define ixSMU_PM_STATUS_49 0x3fec4 +#define ixSMU_PM_STATUS_50 0x3fec8 +#define ixSMU_PM_STATUS_51 0x3fecc +#define ixSMU_PM_STATUS_52 0x3fed0 +#define ixSMU_PM_STATUS_53 0x3fed4 +#define ixSMU_PM_STATUS_54 0x3fed8 +#define ixSMU_PM_STATUS_55 0x3fedc +#define ixSMU_PM_STATUS_56 0x3fee0 +#define ixSMU_PM_STATUS_57 0x3fee4 +#define ixSMU_PM_STATUS_58 0x3fee8 +#define ixSMU_PM_STATUS_59 0x3feec +#define ixSMU_PM_STATUS_60 0x3fef0 +#define ixSMU_PM_STATUS_61 0x3fef4 +#define ixSMU_PM_STATUS_62 0x3fef8 +#define ixSMU_PM_STATUS_63 0x3fefc +#define ixSMU_PM_STATUS_64 0x3ff00 +#define ixSMU_PM_STATUS_65 0x3ff04 +#define ixSMU_PM_STATUS_66 0x3ff08 +#define ixSMU_PM_STATUS_67 0x3ff0c +#define ixSMU_PM_STATUS_68 0x3ff10 +#define ixSMU_PM_STATUS_69 0x3ff14 +#define ixSMU_PM_STATUS_70 0x3ff18 +#define ixSMU_PM_STATUS_71 0x3ff1c +#define ixSMU_PM_STATUS_72 0x3ff20 +#define ixSMU_PM_STATUS_73 0x3ff24 +#define ixSMU_PM_STATUS_74 0x3ff28 +#define ixSMU_PM_STATUS_75 0x3ff2c +#define ixSMU_PM_STATUS_76 0x3ff30 +#define ixSMU_PM_STATUS_77 0x3ff34 +#define ixSMU_PM_STATUS_78 0x3ff38 +#define ixSMU_PM_STATUS_79 0x3ff3c +#define ixSMU_PM_STATUS_80 0x3ff40 +#define ixSMU_PM_STATUS_81 0x3ff44 +#define ixSMU_PM_STATUS_82 0x3ff48 +#define ixSMU_PM_STATUS_83 0x3ff4c +#define ixSMU_PM_STATUS_84 0x3ff50 +#define ixSMU_PM_STATUS_85 0x3ff54 +#define ixSMU_PM_STATUS_86 0x3ff58 +#define ixSMU_PM_STATUS_87 0x3ff5c +#define ixSMU_PM_STATUS_88 0x3ff60 +#define ixSMU_PM_STATUS_89 0x3ff64 +#define ixSMU_PM_STATUS_90 0x3ff68 +#define ixSMU_PM_STATUS_91 0x3ff6c +#define ixSMU_PM_STATUS_92 0x3ff70 +#define ixSMU_PM_STATUS_93 0x3ff74 +#define ixSMU_PM_STATUS_94 0x3ff78 +#define ixSMU_PM_STATUS_95 0x3ff7c +#define ixSMU_PM_STATUS_96 0x3ff80 +#define ixSMU_PM_STATUS_97 0x3ff84 +#define ixSMU_PM_STATUS_98 0x3ff88 +#define ixSMU_PM_STATUS_99 0x3ff8c +#define ixSMU_PM_STATUS_100 0x3ff90 +#define ixSMU_PM_STATUS_101 0x3ff94 +#define ixSMU_PM_STATUS_102 0x3ff98 +#define ixSMU_PM_STATUS_103 0x3ff9c +#define ixSMU_PM_STATUS_104 0x3ffa0 +#define ixSMU_PM_STATUS_105 0x3ffa4 +#define ixSMU_PM_STATUS_106 0x3ffa8 +#define ixSMU_PM_STATUS_107 0x3ffac +#define ixSMU_PM_STATUS_108 0x3ffb0 +#define ixSMU_PM_STATUS_109 0x3ffb4 +#define ixSMU_PM_STATUS_110 0x3ffb8 +#define ixSMU_PM_STATUS_111 0x3ffbc +#define ixSMU_PM_STATUS_112 0x3ffc0 +#define ixSMU_PM_STATUS_113 0x3ffc4 +#define ixSMU_PM_STATUS_114 0x3ffc8 +#define ixSMU_PM_STATUS_115 0x3ffcc +#define ixSMU_PM_STATUS_116 0x3ffd0 +#define ixSMU_PM_STATUS_117 0x3ffd4 +#define ixSMU_PM_STATUS_118 0x3ffd8 +#define ixSMU_PM_STATUS_119 0x3ffdc +#define ixSMU_PM_STATUS_120 0x3ffe0 +#define ixSMU_PM_STATUS_121 0x3ffe4 +#define ixSMU_PM_STATUS_122 0x3ffe8 +#define ixSMU_PM_STATUS_123 0x3ffec +#define ixSMU_PM_STATUS_124 0x3fff0 +#define ixSMU_PM_STATUS_125 0x3fff4 +#define ixSMU_PM_STATUS_126 0x3fff8 +#define ixSMU_PM_STATUS_127 0x3fffc +#define ixCG_THERMAL_INT_ENA 0xc2100024 +#define ixCG_THERMAL_INT_CTRL 0xc2100028 +#define ixCG_THERMAL_INT_STATUS 0xc210002c +#define ixCG_THERMAL_CTRL 0xc0300004 +#define ixCG_THERMAL_STATUS 0xc0300008 +#define ixCG_THERMAL_INT 0xc030000c +#define ixCG_MULT_THERMAL_CTRL 0xc0300010 +#define ixCG_MULT_THERMAL_STATUS 0xc0300014 +#define ixTHM_TMON2_CTRL 0xc0300034 +#define ixTHM_TMON2_CTRL2 0xc0300038 +#define ixTHM_TMON2_CSR_WR 0xc0300054 +#define ixTHM_TMON2_CSR_RD 0xc0300058 +#define ixCG_FDO_CTRL0 0xc0300064 +#define ixCG_FDO_CTRL1 0xc0300068 +#define ixCG_FDO_CTRL2 0xc030006c +#define ixCG_TACH_CTRL 0xc0300070 +#define ixCG_TACH_STATUS 0xc0300074 +#define ixCC_THM_STRAPS0 0xc0300080 +#define ixTHM_TMON0_RDIL0_DATA 0xc0300100 +#define ixTHM_TMON0_RDIL1_DATA 0xc0300104 +#define ixTHM_TMON0_RDIL2_DATA 0xc0300108 +#define ixTHM_TMON0_RDIL3_DATA 0xc030010c +#define ixTHM_TMON0_RDIL4_DATA 0xc0300110 +#define ixTHM_TMON0_RDIL5_DATA 0xc0300114 +#define ixTHM_TMON0_RDIL6_DATA 0xc0300118 +#define ixTHM_TMON0_RDIL7_DATA 0xc030011c +#define ixTHM_TMON0_RDIL8_DATA 0xc0300120 +#define ixTHM_TMON0_RDIL9_DATA 0xc0300124 +#define ixTHM_TMON0_RDIL10_DATA 0xc0300128 +#define ixTHM_TMON0_RDIL11_DATA 0xc030012c +#define ixTHM_TMON0_RDIL12_DATA 0xc0300130 +#define ixTHM_TMON0_RDIL13_DATA 0xc0300134 +#define ixTHM_TMON0_RDIL14_DATA 0xc0300138 +#define ixTHM_TMON0_RDIL15_DATA 0xc030013c +#define ixTHM_TMON0_RDIR0_DATA 0xc0300140 +#define ixTHM_TMON0_RDIR1_DATA 0xc0300144 +#define ixTHM_TMON0_RDIR2_DATA 0xc0300148 +#define ixTHM_TMON0_RDIR3_DATA 0xc030014c +#define ixTHM_TMON0_RDIR4_DATA 0xc0300150 +#define ixTHM_TMON0_RDIR5_DATA 0xc0300154 +#define ixTHM_TMON0_RDIR6_DATA 0xc0300158 +#define ixTHM_TMON0_RDIR7_DATA 0xc030015c +#define ixTHM_TMON0_RDIR8_DATA 0xc0300160 +#define ixTHM_TMON0_RDIR9_DATA 0xc0300164 +#define ixTHM_TMON0_RDIR10_DATA 0xc0300168 +#define ixTHM_TMON0_RDIR11_DATA 0xc030016c +#define ixTHM_TMON0_RDIR12_DATA 0xc0300170 +#define ixTHM_TMON0_RDIR13_DATA 0xc0300174 +#define ixTHM_TMON0_RDIR14_DATA 0xc0300178 +#define ixTHM_TMON0_RDIR15_DATA 0xc030017c +#define ixTHM_TMON1_RDIL0_DATA 0xc0300180 +#define ixTHM_TMON1_RDIL1_DATA 0xc0300184 +#define ixTHM_TMON1_RDIL2_DATA 0xc0300188 +#define ixTHM_TMON1_RDIL3_DATA 0xc030018c +#define ixTHM_TMON1_RDIL4_DATA 0xc0300190 +#define ixTHM_TMON1_RDIL5_DATA 0xc0300194 +#define ixTHM_TMON1_RDIL6_DATA 0xc0300198 +#define ixTHM_TMON1_RDIL7_DATA 0xc030019c +#define ixTHM_TMON1_RDIL8_DATA 0xc03001a0 +#define ixTHM_TMON1_RDIL9_DATA 0xc03001a4 +#define ixTHM_TMON1_RDIL10_DATA 0xc03001a8 +#define ixTHM_TMON1_RDIL11_DATA 0xc03001ac +#define ixTHM_TMON1_RDIL12_DATA 0xc03001b0 +#define ixTHM_TMON1_RDIL13_DATA 0xc03001b4 +#define ixTHM_TMON1_RDIL14_DATA 0xc03001b8 +#define ixTHM_TMON1_RDIL15_DATA 0xc03001bc +#define ixTHM_TMON1_RDIR0_DATA 0xc03001c0 +#define ixTHM_TMON1_RDIR1_DATA 0xc03001c4 +#define ixTHM_TMON1_RDIR2_DATA 0xc03001c8 +#define ixTHM_TMON1_RDIR3_DATA 0xc03001cc +#define ixTHM_TMON1_RDIR4_DATA 0xc03001d0 +#define ixTHM_TMON1_RDIR5_DATA 0xc03001d4 +#define ixTHM_TMON1_RDIR6_DATA 0xc03001d8 +#define ixTHM_TMON1_RDIR7_DATA 0xc03001dc +#define ixTHM_TMON1_RDIR8_DATA 0xc03001e0 +#define ixTHM_TMON1_RDIR9_DATA 0xc03001e4 +#define ixTHM_TMON1_RDIR10_DATA 0xc03001e8 +#define ixTHM_TMON1_RDIR11_DATA 0xc03001ec +#define ixTHM_TMON1_RDIR12_DATA 0xc03001f0 +#define ixTHM_TMON1_RDIR13_DATA 0xc03001f4 +#define ixTHM_TMON1_RDIR14_DATA 0xc03001f8 +#define ixTHM_TMON1_RDIR15_DATA 0xc03001fc +#define ixTHM_TMON2_RDIL0_DATA 0xc0300200 +#define ixTHM_TMON2_RDIL1_DATA 0xc0300204 +#define ixTHM_TMON2_RDIL2_DATA 0xc0300208 +#define ixTHM_TMON2_RDIL3_DATA 0xc030020c +#define ixTHM_TMON2_RDIL4_DATA 0xc0300210 +#define ixTHM_TMON2_RDIL5_DATA 0xc0300214 +#define ixTHM_TMON2_RDIL6_DATA 0xc0300218 +#define ixTHM_TMON2_RDIL7_DATA 0xc030021c +#define ixTHM_TMON2_RDIL8_DATA 0xc0300220 +#define ixTHM_TMON2_RDIL9_DATA 0xc0300224 +#define ixTHM_TMON2_RDIL10_DATA 0xc0300228 +#define ixTHM_TMON2_RDIL11_DATA 0xc030022c +#define ixTHM_TMON2_RDIL12_DATA 0xc0300230 +#define ixTHM_TMON2_RDIL13_DATA 0xc0300234 +#define ixTHM_TMON2_RDIL14_DATA 0xc0300238 +#define ixTHM_TMON2_RDIL15_DATA 0xc030023c +#define ixTHM_TMON2_RDIR0_DATA 0xc0300240 +#define ixTHM_TMON2_RDIR1_DATA 0xc0300244 +#define ixTHM_TMON2_RDIR2_DATA 0xc0300248 +#define ixTHM_TMON2_RDIR3_DATA 0xc030024c +#define ixTHM_TMON2_RDIR4_DATA 0xc0300250 +#define ixTHM_TMON2_RDIR5_DATA 0xc0300254 +#define ixTHM_TMON2_RDIR6_DATA 0xc0300258 +#define ixTHM_TMON2_RDIR7_DATA 0xc030025c +#define ixTHM_TMON2_RDIR8_DATA 0xc0300260 +#define ixTHM_TMON2_RDIR9_DATA 0xc0300264 +#define ixTHM_TMON2_RDIR10_DATA 0xc0300268 +#define ixTHM_TMON2_RDIR11_DATA 0xc030026c +#define ixTHM_TMON2_RDIR12_DATA 0xc0300270 +#define ixTHM_TMON2_RDIR13_DATA 0xc0300274 +#define ixTHM_TMON2_RDIR14_DATA 0xc0300278 +#define ixTHM_TMON2_RDIR15_DATA 0xc030027c +#define ixTHM_TMON0_INT_DATA 0xc0300300 +#define ixTHM_TMON1_INT_DATA 0xc0300304 +#define ixTHM_TMON2_INT_DATA 0xc0300308 +#define ixTHM_TMON0_DEBUG 0xc0300310 +#define ixTHM_TMON1_DEBUG 0xc0300314 +#define ixTHM_TMON2_DEBUG 0xc0300318 +#define ixTHM_TMON0_STATUS 0xc0300320 +#define ixTHM_TMON1_STATUS 0xc0300324 +#define ixTHM_TMON2_STATUS 0xc0300328 +#define ixGENERAL_PWRMGT 0xc0200000 +#define ixCNB_PWRMGT_CNTL 0xc0200004 +#define ixSCLK_PWRMGT_CNTL 0xc0200008 +#define ixTARGET_AND_CURRENT_PROFILE_INDEX 0xc0200014 +#define ixPWR_PCC_CONTROL 0xc0200018 +#define ixPWR_PCC_GPIO_SELECT 0xc020001c +#define ixCG_FREQ_TRAN_VOTING_0 0xc02001a8 +#define ixCG_FREQ_TRAN_VOTING_1 0xc02001ac +#define ixCG_FREQ_TRAN_VOTING_2 0xc02001b0 +#define ixCG_FREQ_TRAN_VOTING_3 0xc02001b4 +#define ixCG_FREQ_TRAN_VOTING_4 0xc02001b8 +#define ixCG_FREQ_TRAN_VOTING_5 0xc02001bc +#define ixCG_FREQ_TRAN_VOTING_6 0xc02001c0 +#define ixCG_FREQ_TRAN_VOTING_7 0xc02001c4 +#define ixPLL_TEST_CNTL 0xc020003c +#define ixCG_STATIC_SCREEN_PARAMETER 0xc0200044 +#define ixCG_DISPLAY_GAP_CNTL 0xc0200060 +#define ixCG_DISPLAY_GAP_CNTL2 0xc0200230 +#define ixCG_ACPI_CNTL 0xc0200064 +#define ixSCLK_DEEP_SLEEP_CNTL 0xc0200080 +#define ixSCLK_DEEP_SLEEP_CNTL2 0xc0200084 +#define ixSCLK_DEEP_SLEEP_CNTL3 0xc020009c +#define ixSCLK_DEEP_SLEEP_MISC_CNTL 0xc0200088 +#define ixLCLK_DEEP_SLEEP_CNTL 0xc020008c +#define ixLCLK_DEEP_SLEEP_CNTL2 0xc0200310 +#define ixTARGET_AND_CURRENT_PROFILE_INDEX_1 0xc02000f0 +#define ixCG_ULV_PARAMETER 0xc020015c +#define ixSCLK_MIN_DIV 0xc02003ac +#define ixPWR_AVFS_SEL 0xc0200384 +#define ixPWR_AVFS_CNTL 0xc0200388 +#define ixPWR_AVFS0_CNTL_STATUS 0xc0200400 +#define ixPWR_AVFS1_CNTL_STATUS 0xc0200404 +#define ixPWR_AVFS2_CNTL_STATUS 0xc0200408 +#define ixPWR_AVFS3_CNTL_STATUS 0xc020040c +#define ixPWR_AVFS4_CNTL_STATUS 0xc0200410 +#define ixPWR_AVFS5_CNTL_STATUS 0xc0200414 +#define ixPWR_AVFS6_CNTL_STATUS 0xc0200418 +#define ixPWR_AVFS7_CNTL_STATUS 0xc020041c +#define ixPWR_AVFS8_CNTL_STATUS 0xc0200420 +#define ixPWR_AVFS9_CNTL_STATUS 0xc0200424 +#define ixPWR_AVFS10_CNTL_STATUS 0xc0200428 +#define ixPWR_AVFS11_CNTL_STATUS 0xc020042c +#define ixPWR_AVFS12_CNTL_STATUS 0xc0200430 +#define ixPWR_AVFS13_CNTL_STATUS 0xc0200434 +#define ixPWR_AVFS14_CNTL_STATUS 0xc0200438 +#define ixPWR_AVFS15_CNTL_STATUS 0xc020043c +#define ixPWR_AVFS16_CNTL_STATUS 0xc0200440 +#define ixPWR_AVFS17_CNTL_STATUS 0xc0200444 +#define ixPWR_AVFS18_CNTL_STATUS 0xc0200448 +#define ixPWR_AVFS19_CNTL_STATUS 0xc020044c +#define ixPWR_AVFS20_CNTL_STATUS 0xc0200450 +#define ixPWR_AVFS21_CNTL_STATUS 0xc0200454 +#define ixPWR_AVFS22_CNTL_STATUS 0xc0200458 +#define ixPWR_AVFS23_CNTL_STATUS 0xc020045c +#define ixPWR_AVFS24_CNTL_STATUS 0xc0200460 +#define ixPWR_AVFS25_CNTL_STATUS 0xc0200464 +#define ixPWR_AVFS26_CNTL_STATUS 0xc0200468 +#define ixPWR_AVFS27_CNTL_STATUS 0xc020046c +#define ixPWR_CKS_ENABLE 0xc020034c +#define ixPWR_CKS_CNTL 0xc0200350 +#define ixPWR_DISP_TIMER_CONTROL 0xc02003c0 +#define ixPWR_DISP_TIMER_DEBUG 0xc02003c4 +#define ixPWR_DISP_TIMER2_CONTROL 0xc02003c8 +#define ixPWR_DISP_TIMER2_DEBUG 0xc02003cc +#define ixPWR_DISP_TIMER_CONTROL2 0xc0200378 +#define ixVDDGFX_IDLE_PARAMETER 0xc020036c +#define ixVDDGFX_IDLE_CONTROL 0xc0200370 +#define ixVDDGFX_IDLE_EXIT 0xc0200374 +#define ixLCAC_MC0_CNTL 0xc0400130 +#define ixLCAC_MC0_OVR_SEL 0xc0400134 +#define ixLCAC_MC0_OVR_VAL 0xc0400138 +#define ixLCAC_MC1_CNTL 0xc040013c +#define ixLCAC_MC1_OVR_SEL 0xc0400140 +#define ixLCAC_MC1_OVR_VAL 0xc0400144 +#define ixLCAC_MC2_CNTL 0xc0400148 +#define ixLCAC_MC2_OVR_SEL 0xc040014c +#define ixLCAC_MC2_OVR_VAL 0xc0400150 +#define ixLCAC_MC3_CNTL 0xc0400154 +#define ixLCAC_MC3_OVR_SEL 0xc0400158 +#define ixLCAC_MC3_OVR_VAL 0xc040015c +#define ixLCAC_MC4_CNTL 0xc0400d60 +#define ixLCAC_MC4_OVR_SEL 0xc0400d64 +#define ixLCAC_MC4_OVR_VAL 0xc0400d68 +#define ixLCAC_MC5_CNTL 0xc0400d6c +#define ixLCAC_MC5_OVR_SEL 0xc0400d70 +#define ixLCAC_MC5_OVR_VAL 0xc0400d74 +#define ixLCAC_MC6_CNTL 0xc0400d78 +#define ixLCAC_MC6_OVR_SEL 0xc0400d7c +#define ixLCAC_MC6_OVR_VAL 0xc0400d80 +#define ixLCAC_MC7_CNTL 0xc0400d84 +#define ixLCAC_MC7_OVR_SEL 0xc0400d88 +#define ixLCAC_MC7_OVR_VAL 0xc0400d8c +#define ixLCAC_CPL_CNTL 0xc0400160 +#define ixLCAC_CPL_OVR_SEL 0xc0400164 +#define ixLCAC_CPL_OVR_VAL 0xc0400168 +#define mmROM_SMC_IND_INDEX 0x80 +#define mmROM0_ROM_SMC_IND_INDEX 0x80 +#define mmROM1_ROM_SMC_IND_INDEX 0x82 +#define mmROM2_ROM_SMC_IND_INDEX 0x84 +#define mmROM3_ROM_SMC_IND_INDEX 0x86 +#define mmROM_SMC_IND_DATA 0x81 +#define mmROM0_ROM_SMC_IND_DATA 0x81 +#define mmROM1_ROM_SMC_IND_DATA 0x83 +#define mmROM2_ROM_SMC_IND_DATA 0x85 +#define mmROM3_ROM_SMC_IND_DATA 0x87 +#define ixROM_CNTL 0xc0600000 +#define ixPAGE_MIRROR_CNTL 0xc0600004 +#define ixROM_STATUS 0xc0600008 +#define ixCGTT_ROM_CLK_CTRL0 0xc060000c +#define ixROM_INDEX 0xc0600010 +#define ixROM_DATA 0xc0600014 +#define ixROM_START 0xc0600018 +#define ixROM_SW_CNTL 0xc060001c +#define ixROM_SW_STATUS 0xc0600020 +#define ixROM_SW_COMMAND 0xc0600024 +#define ixROM_SW_DATA_1 0xc0600028 +#define ixROM_SW_DATA_2 0xc060002c +#define ixROM_SW_DATA_3 0xc0600030 +#define ixROM_SW_DATA_4 0xc0600034 +#define ixROM_SW_DATA_5 0xc0600038 +#define ixROM_SW_DATA_6 0xc060003c +#define ixROM_SW_DATA_7 0xc0600040 +#define ixROM_SW_DATA_8 0xc0600044 +#define ixROM_SW_DATA_9 0xc0600048 +#define ixROM_SW_DATA_10 0xc060004c +#define ixROM_SW_DATA_11 0xc0600050 +#define ixROM_SW_DATA_12 0xc0600054 +#define ixROM_SW_DATA_13 0xc0600058 +#define ixROM_SW_DATA_14 0xc060005c +#define ixROM_SW_DATA_15 0xc0600060 +#define ixROM_SW_DATA_16 0xc0600064 +#define ixROM_SW_DATA_17 0xc0600068 +#define ixROM_SW_DATA_18 0xc060006c +#define ixROM_SW_DATA_19 0xc0600070 +#define ixROM_SW_DATA_20 0xc0600074 +#define ixROM_SW_DATA_21 0xc0600078 +#define ixROM_SW_DATA_22 0xc060007c +#define ixROM_SW_DATA_23 0xc0600080 +#define ixROM_SW_DATA_24 0xc0600084 +#define ixROM_SW_DATA_25 0xc0600088 +#define ixROM_SW_DATA_26 0xc060008c +#define ixROM_SW_DATA_27 0xc0600090 +#define ixROM_SW_DATA_28 0xc0600094 +#define ixROM_SW_DATA_29 0xc0600098 +#define ixROM_SW_DATA_30 0xc060009c +#define ixROM_SW_DATA_31 0xc06000a0 +#define ixROM_SW_DATA_32 0xc06000a4 +#define ixROM_SW_DATA_33 0xc06000a8 +#define ixROM_SW_DATA_34 0xc06000ac +#define ixROM_SW_DATA_35 0xc06000b0 +#define ixROM_SW_DATA_36 0xc06000b4 +#define ixROM_SW_DATA_37 0xc06000b8 +#define ixROM_SW_DATA_38 0xc06000bc +#define ixROM_SW_DATA_39 0xc06000c0 +#define ixROM_SW_DATA_40 0xc06000c4 +#define ixROM_SW_DATA_41 0xc06000c8 +#define ixROM_SW_DATA_42 0xc06000cc +#define ixROM_SW_DATA_43 0xc06000d0 +#define ixROM_SW_DATA_44 0xc06000d4 +#define ixROM_SW_DATA_45 0xc06000d8 +#define ixROM_SW_DATA_46 0xc06000dc +#define ixROM_SW_DATA_47 0xc06000e0 +#define ixROM_SW_DATA_48 0xc06000e4 +#define ixROM_SW_DATA_49 0xc06000e8 +#define ixROM_SW_DATA_50 0xc06000ec +#define ixROM_SW_DATA_51 0xc06000f0 +#define ixROM_SW_DATA_52 0xc06000f4 +#define ixROM_SW_DATA_53 0xc06000f8 +#define ixROM_SW_DATA_54 0xc06000fc +#define ixROM_SW_DATA_55 0xc0600100 +#define ixROM_SW_DATA_56 0xc0600104 +#define ixROM_SW_DATA_57 0xc0600108 +#define ixROM_SW_DATA_58 0xc060010c +#define ixROM_SW_DATA_59 0xc0600110 +#define ixROM_SW_DATA_60 0xc0600114 +#define ixROM_SW_DATA_61 0xc0600118 +#define ixROM_SW_DATA_62 0xc060011c +#define ixROM_SW_DATA_63 0xc0600120 +#define ixROM_SW_DATA_64 0xc0600124 +#define mmGC_CAC_CGTT_CLK_CTRL 0x3292 +#define mmSE_CAC_CGTT_CLK_CTRL 0x3293 +#define mmGC_CAC_LKG_AGGR_LOWER 0x3296 +#define mmGC_CAC_LKG_AGGR_UPPER 0x3297 +#define ixGC_CAC_WEIGHT_CU_0 0x32 +#define ixGC_CAC_WEIGHT_CU_1 0x33 +#define ixGC_CAC_WEIGHT_CU_2 0x34 +#define ixGC_CAC_WEIGHT_CU_3 0x35 +#define ixGC_CAC_WEIGHT_CU_4 0x36 +#define ixGC_CAC_WEIGHT_CU_5 0x37 +#define ixGC_CAC_WEIGHT_CU_6 0x38 +#define ixGC_CAC_WEIGHT_CU_7 0x39 +#define ixGC_CAC_ACC_CU0 0xba +#define ixGC_CAC_ACC_CU1 0xbb +#define ixGC_CAC_ACC_CU2 0xbc +#define ixGC_CAC_ACC_CU3 0xbd +#define ixGC_CAC_ACC_CU4 0xbe +#define ixGC_CAC_ACC_CU5 0xbf +#define ixGC_CAC_ACC_CU6 0xc0 +#define ixGC_CAC_ACC_CU7 0xc1 +#define ixGC_CAC_ACC_CU8 0xc2 +#define ixGC_CAC_ACC_CU9 0xc3 +#define ixGC_CAC_ACC_CU10 0xc4 +#define ixGC_CAC_ACC_CU11 0xc5 +#define ixGC_CAC_ACC_CU12 0xc6 +#define ixGC_CAC_ACC_CU13 0xc7 +#define ixGC_CAC_ACC_CU14 0xc8 +#define ixGC_CAC_ACC_CU15 0xc9 +#define ixGC_CAC_OVRD_CU 0xe7 + +#endif /* SMU_7_1_3_D_H */ diff --git a/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_enum.h b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_enum.h new file mode 100644 index 000000000000..f19c4208d963 --- /dev/null +++ b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_enum.h @@ -0,0 +1,1282 @@ +/* + * SMU_7_1_3 Register documentation + * + * Copyright (C) 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef SMU_7_1_3_ENUM_H +#define SMU_7_1_3_ENUM_H + +#define CG_SRBM_START_ADDR 0x600 +#define CG_SRBM_END_ADDR 0x8ff +#define RCU_CCF_DWORDS0 0xa0 +#define RCU_CCF_BITS0 0x1400 +#define RCU_SAM_BYTES 0x2c +#define RCU_SAM_RTL_BYTES 0x2c +#define RCU_SMU_BYTES 0x14 +#define RCU_SMU_RTL_BYTES 0x14 +#define SFP_CHAIN_ADDR 0x1 +#define SFP_SADR 0x0 +#define SFP_EADR 0x37f +#define SAMU_KEY_CHAIN_ADR 0x0 +#define SAMU_KEY_SADR 0x280 +#define SAMU_KEY_EADR 0x2ab +#define SMU_KEY_CHAIN_ADR 0x0 +#define SMU_KEY_SADR 0x2ac +#define SMU_KEY_EADR 0x2bf +#define SMC_MSG_TEST 0x1 +#define SMC_MSG_PHY_LN_OFF 0x2 +#define SMC_MSG_PHY_LN_ON 0x3 +#define SMC_MSG_DDI_PHY_OFF 0x4 +#define SMC_MSG_DDI_PHY_ON 0x5 +#define SMC_MSG_CASCADE_PLL_OFF 0x6 +#define SMC_MSG_CASCADE_PLL_ON 0x7 +#define SMC_MSG_PWR_OFF_x16 0x8 +#define SMC_MSG_CONFIG_LCLK_DPM 0x9 +#define SMC_MSG_FLUSH_DATA_CACHE 0xa +#define SMC_MSG_FLUSH_INSTRUCTION_CACHE 0xb +#define SMC_MSG_CONFIG_VPC_ACCUMULATOR 0xc +#define SMC_MSG_CONFIG_BAPM 0xd +#define SMC_MSG_CONFIG_TDC_LIMIT 0xe +#define SMC_MSG_CONFIG_LPMx 0xf +#define SMC_MSG_CONFIG_HTC_LIMIT 0x10 +#define SMC_MSG_CONFIG_THERMAL_CNTL 0x11 +#define SMC_MSG_CONFIG_VOLTAGE_CNTL 0x12 +#define SMC_MSG_CONFIG_TDP_CNTL 0x13 +#define SMC_MSG_EN_PM_CNTL 0x14 +#define SMC_MSG_DIS_PM_CNTL 0x15 +#define SMC_MSG_CONFIG_NBDPM 0x16 +#define SMC_MSG_CONFIG_LOADLINE 0x17 +#define SMC_MSG_ADJUST_LOADLINE 0x18 +#define SMC_MSG_RESET 0x20 +#define SMC_MSG_VOLTAGE 0x25 +#define SMC_VERSION_MAJOR 0x7 +#define SMC_VERSION_MINOR 0x0 +#define SMC_HEADER_SIZE 0x40 +#define ROM_SIGNATURE 0xaa55 +typedef enum SurfaceEndian { + ENDIAN_NONE = 0x0, + ENDIAN_8IN16 = 0x1, + ENDIAN_8IN32 = 0x2, + ENDIAN_8IN64 = 0x3, +} SurfaceEndian; +typedef enum ArrayMode { + ARRAY_LINEAR_GENERAL = 0x0, + ARRAY_LINEAR_ALIGNED = 0x1, + ARRAY_1D_TILED_THIN1 = 0x2, + ARRAY_1D_TILED_THICK = 0x3, + ARRAY_2D_TILED_THIN1 = 0x4, + ARRAY_PRT_TILED_THIN1 = 0x5, + ARRAY_PRT_2D_TILED_THIN1 = 0x6, + ARRAY_2D_TILED_THICK = 0x7, + ARRAY_2D_TILED_XTHICK = 0x8, + ARRAY_PRT_TILED_THICK = 0x9, + ARRAY_PRT_2D_TILED_THICK = 0xa, + ARRAY_PRT_3D_TILED_THIN1 = 0xb, + ARRAY_3D_TILED_THIN1 = 0xc, + ARRAY_3D_TILED_THICK = 0xd, + ARRAY_3D_TILED_XTHICK = 0xe, + ARRAY_PRT_3D_TILED_THICK = 0xf, +} ArrayMode; +typedef enum PipeTiling { + CONFIG_1_PIPE = 0x0, + CONFIG_2_PIPE = 0x1, + CONFIG_4_PIPE = 0x2, + CONFIG_8_PIPE = 0x3, +} PipeTiling; +typedef enum BankTiling { + CONFIG_4_BANK = 0x0, + CONFIG_8_BANK = 0x1, +} BankTiling; +typedef enum GroupInterleave { + CONFIG_256B_GROUP = 0x0, + CONFIG_512B_GROUP = 0x1, +} GroupInterleave; +typedef enum RowTiling { + CONFIG_1KB_ROW = 0x0, + CONFIG_2KB_ROW = 0x1, + CONFIG_4KB_ROW = 0x2, + CONFIG_8KB_ROW = 0x3, + CONFIG_1KB_ROW_OPT = 0x4, + CONFIG_2KB_ROW_OPT = 0x5, + CONFIG_4KB_ROW_OPT = 0x6, + CONFIG_8KB_ROW_OPT = 0x7, +} RowTiling; +typedef enum BankSwapBytes { + CONFIG_128B_SWAPS = 0x0, + CONFIG_256B_SWAPS = 0x1, + CONFIG_512B_SWAPS = 0x2, + CONFIG_1KB_SWAPS = 0x3, +} BankSwapBytes; +typedef enum SampleSplitBytes { + CONFIG_1KB_SPLIT = 0x0, + CONFIG_2KB_SPLIT = 0x1, + CONFIG_4KB_SPLIT = 0x2, + CONFIG_8KB_SPLIT = 0x3, +} SampleSplitBytes; +typedef enum NumPipes { + ADDR_CONFIG_1_PIPE = 0x0, + ADDR_CONFIG_2_PIPE = 0x1, + ADDR_CONFIG_4_PIPE = 0x2, + ADDR_CONFIG_8_PIPE = 0x3, +} NumPipes; +typedef enum PipeInterleaveSize { + ADDR_CONFIG_PIPE_INTERLEAVE_256B = 0x0, + ADDR_CONFIG_PIPE_INTERLEAVE_512B = 0x1, +} PipeInterleaveSize; +typedef enum BankInterleaveSize { + ADDR_CONFIG_BANK_INTERLEAVE_1 = 0x0, + ADDR_CONFIG_BANK_INTERLEAVE_2 = 0x1, + ADDR_CONFIG_BANK_INTERLEAVE_4 = 0x2, + ADDR_CONFIG_BANK_INTERLEAVE_8 = 0x3, +} BankInterleaveSize; +typedef enum NumShaderEngines { + ADDR_CONFIG_1_SHADER_ENGINE = 0x0, + ADDR_CONFIG_2_SHADER_ENGINE = 0x1, +} NumShaderEngines; +typedef enum ShaderEngineTileSize { + ADDR_CONFIG_SE_TILE_16 = 0x0, + ADDR_CONFIG_SE_TILE_32 = 0x1, +} ShaderEngineTileSize; +typedef enum NumGPUs { + ADDR_CONFIG_1_GPU = 0x0, + ADDR_CONFIG_2_GPU = 0x1, + ADDR_CONFIG_4_GPU = 0x2, +} NumGPUs; +typedef enum MultiGPUTileSize { + ADDR_CONFIG_GPU_TILE_16 = 0x0, + ADDR_CONFIG_GPU_TILE_32 = 0x1, + ADDR_CONFIG_GPU_TILE_64 = 0x2, + ADDR_CONFIG_GPU_TILE_128 = 0x3, +} MultiGPUTileSize; +typedef enum RowSize { + ADDR_CONFIG_1KB_ROW = 0x0, + ADDR_CONFIG_2KB_ROW = 0x1, + ADDR_CONFIG_4KB_ROW = 0x2, +} RowSize; +typedef enum NumLowerPipes { + ADDR_CONFIG_1_LOWER_PIPES = 0x0, + ADDR_CONFIG_2_LOWER_PIPES = 0x1, +} NumLowerPipes; +typedef enum DebugBlockId { + DBG_CLIENT_BLKID_RESERVED = 0x0, + DBG_CLIENT_BLKID_dbg = 0x1, + DBG_CLIENT_BLKID_scf2 = 0x2, + DBG_CLIENT_BLKID_mcd5_0 = 0x3, + DBG_CLIENT_BLKID_mcd5_1 = 0x4, + DBG_CLIENT_BLKID_mcd6_0 = 0x5, + DBG_CLIENT_BLKID_mcd6_1 = 0x6, + DBG_CLIENT_BLKID_mcd7_0 = 0x7, + DBG_CLIENT_BLKID_mcd7_1 = 0x8, + DBG_CLIENT_BLKID_vmc = 0x9, + DBG_CLIENT_BLKID_sx30 = 0xa, + DBG_CLIENT_BLKID_mcd2_0 = 0xb, + DBG_CLIENT_BLKID_mcd2_1 = 0xc, + DBG_CLIENT_BLKID_bci1 = 0xd, + DBG_CLIENT_BLKID_xdma_dbg_client_wrapper = 0xe, + DBG_CLIENT_BLKID_mcc0 = 0xf, + DBG_CLIENT_BLKID_uvdf_0 = 0x10, + DBG_CLIENT_BLKID_uvdf_1 = 0x11, + DBG_CLIENT_BLKID_uvdf_2 = 0x12, + DBG_CLIENT_BLKID_bci0 = 0x13, + DBG_CLIENT_BLKID_vcec0_0 = 0x14, + DBG_CLIENT_BLKID_cb100 = 0x15, + DBG_CLIENT_BLKID_cb001 = 0x16, + DBG_CLIENT_BLKID_cb002 = 0x17, + DBG_CLIENT_BLKID_cb003 = 0x18, + DBG_CLIENT_BLKID_mcd4_0 = 0x19, + DBG_CLIENT_BLKID_mcd4_1 = 0x1a, + DBG_CLIENT_BLKID_tmonw00 = 0x1b, + DBG_CLIENT_BLKID_cb101 = 0x1c, + DBG_CLIENT_BLKID_cb102 = 0x1d, + DBG_CLIENT_BLKID_cb103 = 0x1e, + DBG_CLIENT_BLKID_sx10 = 0x1f, + DBG_CLIENT_BLKID_cb301 = 0x20, + DBG_CLIENT_BLKID_cb302 = 0x21, + DBG_CLIENT_BLKID_cb303 = 0x22, + DBG_CLIENT_BLKID_tmonw01 = 0x23, + DBG_CLIENT_BLKID_tmonw02 = 0x24, + DBG_CLIENT_BLKID_vcea0_0 = 0x25, + DBG_CLIENT_BLKID_vcea0_1 = 0x26, + DBG_CLIENT_BLKID_vcea0_2 = 0x27, + DBG_CLIENT_BLKID_vcea0_3 = 0x28, + DBG_CLIENT_BLKID_scf1 = 0x29, + DBG_CLIENT_BLKID_sx20 = 0x2a, + DBG_CLIENT_BLKID_spim1 = 0x2b, + DBG_CLIENT_BLKID_scb1 = 0x2c, + DBG_CLIENT_BLKID_pa10 = 0x2d, + DBG_CLIENT_BLKID_pa00 = 0x2e, + DBG_CLIENT_BLKID_gmcon = 0x2f, + DBG_CLIENT_BLKID_mcb = 0x30, + DBG_CLIENT_BLKID_vgt0 = 0x31, + DBG_CLIENT_BLKID_pc0 = 0x32, + DBG_CLIENT_BLKID_bci2 = 0x33, + DBG_CLIENT_BLKID_uvdb_0 = 0x34, + DBG_CLIENT_BLKID_spim3 = 0x35, + DBG_CLIENT_BLKID_scb3 = 0x36, + DBG_CLIENT_BLKID_cpc_0 = 0x37, + DBG_CLIENT_BLKID_cpc_1 = 0x38, + DBG_CLIENT_BLKID_uvdm_0 = 0x39, + DBG_CLIENT_BLKID_uvdm_1 = 0x3a, + DBG_CLIENT_BLKID_uvdm_2 = 0x3b, + DBG_CLIENT_BLKID_uvdm_3 = 0x3c, + DBG_CLIENT_BLKID_cb000 = 0x3d, + DBG_CLIENT_BLKID_spim0 = 0x3e, + DBG_CLIENT_BLKID_scb0 = 0x3f, + DBG_CLIENT_BLKID_mcc2 = 0x40, + DBG_CLIENT_BLKID_ds0 = 0x41, + DBG_CLIENT_BLKID_srbm = 0x42, + DBG_CLIENT_BLKID_ih = 0x43, + DBG_CLIENT_BLKID_sem = 0x44, + DBG_CLIENT_BLKID_sdma_0 = 0x45, + DBG_CLIENT_BLKID_sdma_1 = 0x46, + DBG_CLIENT_BLKID_hdp = 0x47, + DBG_CLIENT_BLKID_acp_0 = 0x48, + DBG_CLIENT_BLKID_acp_1 = 0x49, + DBG_CLIENT_BLKID_cb200 = 0x4a, + DBG_CLIENT_BLKID_scf3 = 0x4b, + DBG_CLIENT_BLKID_bci3 = 0x4c, + DBG_CLIENT_BLKID_mcd0_0 = 0x4d, + DBG_CLIENT_BLKID_mcd0_1 = 0x4e, + DBG_CLIENT_BLKID_pa11 = 0x4f, + DBG_CLIENT_BLKID_pa01 = 0x50, + DBG_CLIENT_BLKID_cb201 = 0x51, + DBG_CLIENT_BLKID_cb202 = 0x52, + DBG_CLIENT_BLKID_cb203 = 0x53, + DBG_CLIENT_BLKID_spim2 = 0x54, + DBG_CLIENT_BLKID_scb2 = 0x55, + DBG_CLIENT_BLKID_vgt2 = 0x56, + DBG_CLIENT_BLKID_pc2 = 0x57, + DBG_CLIENT_BLKID_smu_0 = 0x58, + DBG_CLIENT_BLKID_smu_1 = 0x59, + DBG_CLIENT_BLKID_smu_2 = 0x5a, + DBG_CLIENT_BLKID_cb1 = 0x5b, + DBG_CLIENT_BLKID_ia0 = 0x5c, + DBG_CLIENT_BLKID_wd = 0x5d, + DBG_CLIENT_BLKID_ia1 = 0x5e, + DBG_CLIENT_BLKID_scf0 = 0x5f, + DBG_CLIENT_BLKID_vgt1 = 0x60, + DBG_CLIENT_BLKID_pc1 = 0x61, + DBG_CLIENT_BLKID_cb0 = 0x62, + DBG_CLIENT_BLKID_gdc_one_0 = 0x63, + DBG_CLIENT_BLKID_gdc_one_1 = 0x64, + DBG_CLIENT_BLKID_gdc_one_2 = 0x65, + DBG_CLIENT_BLKID_gdc_one_3 = 0x66, + DBG_CLIENT_BLKID_gdc_one_4 = 0x67, + DBG_CLIENT_BLKID_gdc_one_5 = 0x68, + DBG_CLIENT_BLKID_gdc_one_6 = 0x69, + DBG_CLIENT_BLKID_gdc_one_7 = 0x6a, + DBG_CLIENT_BLKID_gdc_one_8 = 0x6b, + DBG_CLIENT_BLKID_gdc_one_9 = 0x6c, + DBG_CLIENT_BLKID_gdc_one_10 = 0x6d, + DBG_CLIENT_BLKID_gdc_one_11 = 0x6e, + DBG_CLIENT_BLKID_gdc_one_12 = 0x6f, + DBG_CLIENT_BLKID_gdc_one_13 = 0x70, + DBG_CLIENT_BLKID_gdc_one_14 = 0x71, + DBG_CLIENT_BLKID_gdc_one_15 = 0x72, + DBG_CLIENT_BLKID_gdc_one_16 = 0x73, + DBG_CLIENT_BLKID_gdc_one_17 = 0x74, + DBG_CLIENT_BLKID_gdc_one_18 = 0x75, + DBG_CLIENT_BLKID_gdc_one_19 = 0x76, + DBG_CLIENT_BLKID_gdc_one_20 = 0x77, + DBG_CLIENT_BLKID_gdc_one_21 = 0x78, + DBG_CLIENT_BLKID_gdc_one_22 = 0x79, + DBG_CLIENT_BLKID_gdc_one_23 = 0x7a, + DBG_CLIENT_BLKID_gdc_one_24 = 0x7b, + DBG_CLIENT_BLKID_gdc_one_25 = 0x7c, + DBG_CLIENT_BLKID_gdc_one_26 = 0x7d, + DBG_CLIENT_BLKID_gdc_one_27 = 0x7e, + DBG_CLIENT_BLKID_gdc_one_28 = 0x7f, + DBG_CLIENT_BLKID_gdc_one_29 = 0x80, + DBG_CLIENT_BLKID_gdc_one_30 = 0x81, + DBG_CLIENT_BLKID_gdc_one_31 = 0x82, + DBG_CLIENT_BLKID_gdc_one_32 = 0x83, + DBG_CLIENT_BLKID_gdc_one_33 = 0x84, + DBG_CLIENT_BLKID_gdc_one_34 = 0x85, + DBG_CLIENT_BLKID_gdc_one_35 = 0x86, + DBG_CLIENT_BLKID_vceb0_0 = 0x87, + DBG_CLIENT_BLKID_vgt3 = 0x88, + DBG_CLIENT_BLKID_pc3 = 0x89, + DBG_CLIENT_BLKID_mcd3_0 = 0x8a, + DBG_CLIENT_BLKID_mcd3_1 = 0x8b, + DBG_CLIENT_BLKID_uvdu_0 = 0x8c, + DBG_CLIENT_BLKID_uvdu_1 = 0x8d, + DBG_CLIENT_BLKID_uvdu_2 = 0x8e, + DBG_CLIENT_BLKID_uvdu_3 = 0x8f, + DBG_CLIENT_BLKID_uvdu_4 = 0x90, + DBG_CLIENT_BLKID_uvdu_5 = 0x91, + DBG_CLIENT_BLKID_uvdu_6 = 0x92, + DBG_CLIENT_BLKID_cb300 = 0x93, + DBG_CLIENT_BLKID_mcd1_0 = 0x94, + DBG_CLIENT_BLKID_mcd1_1 = 0x95, + DBG_CLIENT_BLKID_sx00 = 0x96, + DBG_CLIENT_BLKID_uvdc_0 = 0x97, + DBG_CLIENT_BLKID_uvdc_1 = 0x98, + DBG_CLIENT_BLKID_mcc3 = 0x99, + DBG_CLIENT_BLKID_mcc4 = 0x9a, + DBG_CLIENT_BLKID_mcc5 = 0x9b, + DBG_CLIENT_BLKID_mcc6 = 0x9c, + DBG_CLIENT_BLKID_mcc7 = 0x9d, + DBG_CLIENT_BLKID_cpg_0 = 0x9e, + DBG_CLIENT_BLKID_cpg_1 = 0x9f, + DBG_CLIENT_BLKID_gck = 0xa0, + DBG_CLIENT_BLKID_mcc1 = 0xa1, + DBG_CLIENT_BLKID_cpf_0 = 0xa2, + DBG_CLIENT_BLKID_cpf_1 = 0xa3, + DBG_CLIENT_BLKID_rlc = 0xa4, + DBG_CLIENT_BLKID_grbm = 0xa5, + DBG_CLIENT_BLKID_sammsp = 0xa6, + DBG_CLIENT_BLKID_dci_pg = 0xa7, + DBG_CLIENT_BLKID_dci_0 = 0xa8, + DBG_CLIENT_BLKID_dccg0_0 = 0xa9, + DBG_CLIENT_BLKID_dccg0_1 = 0xaa, + DBG_CLIENT_BLKID_dcfe01_0 = 0xab, + DBG_CLIENT_BLKID_dcfe02_0 = 0xac, + DBG_CLIENT_BLKID_dcfe03_0 = 0xad, + DBG_CLIENT_BLKID_dcfe04_0 = 0xae, + DBG_CLIENT_BLKID_dcfe05_0 = 0xaf, + DBG_CLIENT_BLKID_dcfe06_0 = 0xb0, + DBG_CLIENT_BLKID_mcq0_0 = 0xb1, + DBG_CLIENT_BLKID_mcq0_1 = 0xb2, + DBG_CLIENT_BLKID_mcq1_0 = 0xb3, + DBG_CLIENT_BLKID_mcq1_1 = 0xb4, + DBG_CLIENT_BLKID_mcq2_0 = 0xb5, + DBG_CLIENT_BLKID_mcq2_1 = 0xb6, + DBG_CLIENT_BLKID_mcq3_0 = 0xb7, + DBG_CLIENT_BLKID_mcq3_1 = 0xb8, + DBG_CLIENT_BLKID_mcq4_0 = 0xb9, + DBG_CLIENT_BLKID_mcq4_1 = 0xba, + DBG_CLIENT_BLKID_mcq5_0 = 0xbb, + DBG_CLIENT_BLKID_mcq5_1 = 0xbc, + DBG_CLIENT_BLKID_mcq6_0 = 0xbd, + DBG_CLIENT_BLKID_mcq6_1 = 0xbe, + DBG_CLIENT_BLKID_mcq7_0 = 0xbf, + DBG_CLIENT_BLKID_mcq7_1 = 0xc0, + DBG_CLIENT_BLKID_uvdi_0 = 0xc1, + DBG_CLIENT_BLKID_RESERVED_LAST = 0xc2, +} DebugBlockId; +typedef enum DebugBlockId_OLD { + DBG_BLOCK_ID_RESERVED = 0x0, + DBG_BLOCK_ID_DBG = 0x1, + DBG_BLOCK_ID_VMC = 0x2, + DBG_BLOCK_ID_PDMA = 0x3, + DBG_BLOCK_ID_CG = 0x4, + DBG_BLOCK_ID_SRBM = 0x5, + DBG_BLOCK_ID_GRBM = 0x6, + DBG_BLOCK_ID_RLC = 0x7, + DBG_BLOCK_ID_CSC = 0x8, + DBG_BLOCK_ID_SEM = 0x9, + DBG_BLOCK_ID_IH = 0xa, + DBG_BLOCK_ID_SC = 0xb, + DBG_BLOCK_ID_SQ = 0xc, + DBG_BLOCK_ID_AVP = 0xd, + DBG_BLOCK_ID_GMCON = 0xe, + DBG_BLOCK_ID_SMU = 0xf, + DBG_BLOCK_ID_DMA0 = 0x10, + DBG_BLOCK_ID_DMA1 = 0x11, + DBG_BLOCK_ID_SPIM = 0x12, + DBG_BLOCK_ID_GDS = 0x13, + DBG_BLOCK_ID_SPIS = 0x14, + DBG_BLOCK_ID_UNUSED0 = 0x15, + DBG_BLOCK_ID_PA0 = 0x16, + DBG_BLOCK_ID_PA1 = 0x17, + DBG_BLOCK_ID_CP0 = 0x18, + DBG_BLOCK_ID_CP1 = 0x19, + DBG_BLOCK_ID_CP2 = 0x1a, + DBG_BLOCK_ID_UNUSED1 = 0x1b, + DBG_BLOCK_ID_UVDU = 0x1c, + DBG_BLOCK_ID_UVDM = 0x1d, + DBG_BLOCK_ID_VCE = 0x1e, + DBG_BLOCK_ID_UNUSED2 = 0x1f, + DBG_BLOCK_ID_VGT0 = 0x20, + DBG_BLOCK_ID_VGT1 = 0x21, + DBG_BLOCK_ID_IA = 0x22, + DBG_BLOCK_ID_UNUSED3 = 0x23, + DBG_BLOCK_ID_SCT0 = 0x24, + DBG_BLOCK_ID_SCT1 = 0x25, + DBG_BLOCK_ID_SPM0 = 0x26, + DBG_BLOCK_ID_SPM1 = 0x27, + DBG_BLOCK_ID_TCAA = 0x28, + DBG_BLOCK_ID_TCAB = 0x29, + DBG_BLOCK_ID_TCCA = 0x2a, + DBG_BLOCK_ID_TCCB = 0x2b, + DBG_BLOCK_ID_MCC0 = 0x2c, + DBG_BLOCK_ID_MCC1 = 0x2d, + DBG_BLOCK_ID_MCC2 = 0x2e, + DBG_BLOCK_ID_MCC3 = 0x2f, + DBG_BLOCK_ID_SX0 = 0x30, + DBG_BLOCK_ID_SX1 = 0x31, + DBG_BLOCK_ID_SX2 = 0x32, + DBG_BLOCK_ID_SX3 = 0x33, + DBG_BLOCK_ID_UNUSED4 = 0x34, + DBG_BLOCK_ID_UNUSED5 = 0x35, + DBG_BLOCK_ID_UNUSED6 = 0x36, + DBG_BLOCK_ID_UNUSED7 = 0x37, + DBG_BLOCK_ID_PC0 = 0x38, + DBG_BLOCK_ID_PC1 = 0x39, + DBG_BLOCK_ID_UNUSED8 = 0x3a, + DBG_BLOCK_ID_UNUSED9 = 0x3b, + DBG_BLOCK_ID_UNUSED10 = 0x3c, + DBG_BLOCK_ID_UNUSED11 = 0x3d, + DBG_BLOCK_ID_MCB = 0x3e, + DBG_BLOCK_ID_UNUSED12 = 0x3f, + DBG_BLOCK_ID_SCB0 = 0x40, + DBG_BLOCK_ID_SCB1 = 0x41, + DBG_BLOCK_ID_UNUSED13 = 0x42, + DBG_BLOCK_ID_UNUSED14 = 0x43, + DBG_BLOCK_ID_SCF0 = 0x44, + DBG_BLOCK_ID_SCF1 = 0x45, + DBG_BLOCK_ID_UNUSED15 = 0x46, + DBG_BLOCK_ID_UNUSED16 = 0x47, + DBG_BLOCK_ID_BCI0 = 0x48, + DBG_BLOCK_ID_BCI1 = 0x49, + DBG_BLOCK_ID_BCI2 = 0x4a, + DBG_BLOCK_ID_BCI3 = 0x4b, + DBG_BLOCK_ID_UNUSED17 = 0x4c, + DBG_BLOCK_ID_UNUSED18 = 0x4d, + DBG_BLOCK_ID_UNUSED19 = 0x4e, + DBG_BLOCK_ID_UNUSED20 = 0x4f, + DBG_BLOCK_ID_CB00 = 0x50, + DBG_BLOCK_ID_CB01 = 0x51, + DBG_BLOCK_ID_CB02 = 0x52, + DBG_BLOCK_ID_CB03 = 0x53, + DBG_BLOCK_ID_CB04 = 0x54, + DBG_BLOCK_ID_UNUSED21 = 0x55, + DBG_BLOCK_ID_UNUSED22 = 0x56, + DBG_BLOCK_ID_UNUSED23 = 0x57, + DBG_BLOCK_ID_CB10 = 0x58, + DBG_BLOCK_ID_CB11 = 0x59, + DBG_BLOCK_ID_CB12 = 0x5a, + DBG_BLOCK_ID_CB13 = 0x5b, + DBG_BLOCK_ID_CB14 = 0x5c, + DBG_BLOCK_ID_UNUSED24 = 0x5d, + DBG_BLOCK_ID_UNUSED25 = 0x5e, + DBG_BLOCK_ID_UNUSED26 = 0x5f, + DBG_BLOCK_ID_TCP0 = 0x60, + DBG_BLOCK_ID_TCP1 = 0x61, + DBG_BLOCK_ID_TCP2 = 0x62, + DBG_BLOCK_ID_TCP3 = 0x63, + DBG_BLOCK_ID_TCP4 = 0x64, + DBG_BLOCK_ID_TCP5 = 0x65, + DBG_BLOCK_ID_TCP6 = 0x66, + DBG_BLOCK_ID_TCP7 = 0x67, + DBG_BLOCK_ID_TCP8 = 0x68, + DBG_BLOCK_ID_TCP9 = 0x69, + DBG_BLOCK_ID_TCP10 = 0x6a, + DBG_BLOCK_ID_TCP11 = 0x6b, + DBG_BLOCK_ID_TCP12 = 0x6c, + DBG_BLOCK_ID_TCP13 = 0x6d, + DBG_BLOCK_ID_TCP14 = 0x6e, + DBG_BLOCK_ID_TCP15 = 0x6f, + DBG_BLOCK_ID_TCP16 = 0x70, + DBG_BLOCK_ID_TCP17 = 0x71, + DBG_BLOCK_ID_TCP18 = 0x72, + DBG_BLOCK_ID_TCP19 = 0x73, + DBG_BLOCK_ID_TCP20 = 0x74, + DBG_BLOCK_ID_TCP21 = 0x75, + DBG_BLOCK_ID_TCP22 = 0x76, + DBG_BLOCK_ID_TCP23 = 0x77, + DBG_BLOCK_ID_TCP_RESERVED0 = 0x78, + DBG_BLOCK_ID_TCP_RESERVED1 = 0x79, + DBG_BLOCK_ID_TCP_RESERVED2 = 0x7a, + DBG_BLOCK_ID_TCP_RESERVED3 = 0x7b, + DBG_BLOCK_ID_TCP_RESERVED4 = 0x7c, + DBG_BLOCK_ID_TCP_RESERVED5 = 0x7d, + DBG_BLOCK_ID_TCP_RESERVED6 = 0x7e, + DBG_BLOCK_ID_TCP_RESERVED7 = 0x7f, + DBG_BLOCK_ID_DB00 = 0x80, + DBG_BLOCK_ID_DB01 = 0x81, + DBG_BLOCK_ID_DB02 = 0x82, + DBG_BLOCK_ID_DB03 = 0x83, + DBG_BLOCK_ID_DB04 = 0x84, + DBG_BLOCK_ID_UNUSED27 = 0x85, + DBG_BLOCK_ID_UNUSED28 = 0x86, + DBG_BLOCK_ID_UNUSED29 = 0x87, + DBG_BLOCK_ID_DB10 = 0x88, + DBG_BLOCK_ID_DB11 = 0x89, + DBG_BLOCK_ID_DB12 = 0x8a, + DBG_BLOCK_ID_DB13 = 0x8b, + DBG_BLOCK_ID_DB14 = 0x8c, + DBG_BLOCK_ID_UNUSED30 = 0x8d, + DBG_BLOCK_ID_UNUSED31 = 0x8e, + DBG_BLOCK_ID_UNUSED32 = 0x8f, + DBG_BLOCK_ID_TCC0 = 0x90, + DBG_BLOCK_ID_TCC1 = 0x91, + DBG_BLOCK_ID_TCC2 = 0x92, + DBG_BLOCK_ID_TCC3 = 0x93, + DBG_BLOCK_ID_TCC4 = 0x94, + DBG_BLOCK_ID_TCC5 = 0x95, + DBG_BLOCK_ID_TCC6 = 0x96, + DBG_BLOCK_ID_TCC7 = 0x97, + DBG_BLOCK_ID_SPS00 = 0x98, + DBG_BLOCK_ID_SPS01 = 0x99, + DBG_BLOCK_ID_SPS02 = 0x9a, + DBG_BLOCK_ID_SPS10 = 0x9b, + DBG_BLOCK_ID_SPS11 = 0x9c, + DBG_BLOCK_ID_SPS12 = 0x9d, + DBG_BLOCK_ID_UNUSED33 = 0x9e, + DBG_BLOCK_ID_UNUSED34 = 0x9f, + DBG_BLOCK_ID_TA00 = 0xa0, + DBG_BLOCK_ID_TA01 = 0xa1, + DBG_BLOCK_ID_TA02 = 0xa2, + DBG_BLOCK_ID_TA03 = 0xa3, + DBG_BLOCK_ID_TA04 = 0xa4, + DBG_BLOCK_ID_TA05 = 0xa5, + DBG_BLOCK_ID_TA06 = 0xa6, + DBG_BLOCK_ID_TA07 = 0xa7, + DBG_BLOCK_ID_TA08 = 0xa8, + DBG_BLOCK_ID_TA09 = 0xa9, + DBG_BLOCK_ID_TA0A = 0xaa, + DBG_BLOCK_ID_TA0B = 0xab, + DBG_BLOCK_ID_UNUSED35 = 0xac, + DBG_BLOCK_ID_UNUSED36 = 0xad, + DBG_BLOCK_ID_UNUSED37 = 0xae, + DBG_BLOCK_ID_UNUSED38 = 0xaf, + DBG_BLOCK_ID_TA10 = 0xb0, + DBG_BLOCK_ID_TA11 = 0xb1, + DBG_BLOCK_ID_TA12 = 0xb2, + DBG_BLOCK_ID_TA13 = 0xb3, + DBG_BLOCK_ID_TA14 = 0xb4, + DBG_BLOCK_ID_TA15 = 0xb5, + DBG_BLOCK_ID_TA16 = 0xb6, + DBG_BLOCK_ID_TA17 = 0xb7, + DBG_BLOCK_ID_TA18 = 0xb8, + DBG_BLOCK_ID_TA19 = 0xb9, + DBG_BLOCK_ID_TA1A = 0xba, + DBG_BLOCK_ID_TA1B = 0xbb, + DBG_BLOCK_ID_UNUSED39 = 0xbc, + DBG_BLOCK_ID_UNUSED40 = 0xbd, + DBG_BLOCK_ID_UNUSED41 = 0xbe, + DBG_BLOCK_ID_UNUSED42 = 0xbf, + DBG_BLOCK_ID_TD00 = 0xc0, + DBG_BLOCK_ID_TD01 = 0xc1, + DBG_BLOCK_ID_TD02 = 0xc2, + DBG_BLOCK_ID_TD03 = 0xc3, + DBG_BLOCK_ID_TD04 = 0xc4, + DBG_BLOCK_ID_TD05 = 0xc5, + DBG_BLOCK_ID_TD06 = 0xc6, + DBG_BLOCK_ID_TD07 = 0xc7, + DBG_BLOCK_ID_TD08 = 0xc8, + DBG_BLOCK_ID_TD09 = 0xc9, + DBG_BLOCK_ID_TD0A = 0xca, + DBG_BLOCK_ID_TD0B = 0xcb, + DBG_BLOCK_ID_UNUSED43 = 0xcc, + DBG_BLOCK_ID_UNUSED44 = 0xcd, + DBG_BLOCK_ID_UNUSED45 = 0xce, + DBG_BLOCK_ID_UNUSED46 = 0xcf, + DBG_BLOCK_ID_TD10 = 0xd0, + DBG_BLOCK_ID_TD11 = 0xd1, + DBG_BLOCK_ID_TD12 = 0xd2, + DBG_BLOCK_ID_TD13 = 0xd3, + DBG_BLOCK_ID_TD14 = 0xd4, + DBG_BLOCK_ID_TD15 = 0xd5, + DBG_BLOCK_ID_TD16 = 0xd6, + DBG_BLOCK_ID_TD17 = 0xd7, + DBG_BLOCK_ID_TD18 = 0xd8, + DBG_BLOCK_ID_TD19 = 0xd9, + DBG_BLOCK_ID_TD1A = 0xda, + DBG_BLOCK_ID_TD1B = 0xdb, + DBG_BLOCK_ID_UNUSED47 = 0xdc, + DBG_BLOCK_ID_UNUSED48 = 0xdd, + DBG_BLOCK_ID_UNUSED49 = 0xde, + DBG_BLOCK_ID_UNUSED50 = 0xdf, + DBG_BLOCK_ID_MCD0 = 0xe0, + DBG_BLOCK_ID_MCD1 = 0xe1, + DBG_BLOCK_ID_MCD2 = 0xe2, + DBG_BLOCK_ID_MCD3 = 0xe3, + DBG_BLOCK_ID_MCD4 = 0xe4, + DBG_BLOCK_ID_MCD5 = 0xe5, + DBG_BLOCK_ID_UNUSED51 = 0xe6, + DBG_BLOCK_ID_UNUSED52 = 0xe7, +} DebugBlockId_OLD; +typedef enum DebugBlockId_BY2 { + DBG_BLOCK_ID_RESERVED_BY2 = 0x0, + DBG_BLOCK_ID_VMC_BY2 = 0x1, + DBG_BLOCK_ID_CG_BY2 = 0x2, + DBG_BLOCK_ID_GRBM_BY2 = 0x3, + DBG_BLOCK_ID_CSC_BY2 = 0x4, + DBG_BLOCK_ID_IH_BY2 = 0x5, + DBG_BLOCK_ID_SQ_BY2 = 0x6, + DBG_BLOCK_ID_GMCON_BY2 = 0x7, + DBG_BLOCK_ID_DMA0_BY2 = 0x8, + DBG_BLOCK_ID_SPIM_BY2 = 0x9, + DBG_BLOCK_ID_SPIS_BY2 = 0xa, + DBG_BLOCK_ID_PA0_BY2 = 0xb, + DBG_BLOCK_ID_CP0_BY2 = 0xc, + DBG_BLOCK_ID_CP2_BY2 = 0xd, + DBG_BLOCK_ID_UVDU_BY2 = 0xe, + DBG_BLOCK_ID_VCE_BY2 = 0xf, + DBG_BLOCK_ID_VGT0_BY2 = 0x10, + DBG_BLOCK_ID_IA_BY2 = 0x11, + DBG_BLOCK_ID_SCT0_BY2 = 0x12, + DBG_BLOCK_ID_SPM0_BY2 = 0x13, + DBG_BLOCK_ID_TCAA_BY2 = 0x14, + DBG_BLOCK_ID_TCCA_BY2 = 0x15, + DBG_BLOCK_ID_MCC0_BY2 = 0x16, + DBG_BLOCK_ID_MCC2_BY2 = 0x17, + DBG_BLOCK_ID_SX0_BY2 = 0x18, + DBG_BLOCK_ID_SX2_BY2 = 0x19, + DBG_BLOCK_ID_UNUSED4_BY2 = 0x1a, + DBG_BLOCK_ID_UNUSED6_BY2 = 0x1b, + DBG_BLOCK_ID_PC0_BY2 = 0x1c, + DBG_BLOCK_ID_UNUSED8_BY2 = 0x1d, + DBG_BLOCK_ID_UNUSED10_BY2 = 0x1e, + DBG_BLOCK_ID_MCB_BY2 = 0x1f, + DBG_BLOCK_ID_SCB0_BY2 = 0x20, + DBG_BLOCK_ID_UNUSED13_BY2 = 0x21, + DBG_BLOCK_ID_SCF0_BY2 = 0x22, + DBG_BLOCK_ID_UNUSED15_BY2 = 0x23, + DBG_BLOCK_ID_BCI0_BY2 = 0x24, + DBG_BLOCK_ID_BCI2_BY2 = 0x25, + DBG_BLOCK_ID_UNUSED17_BY2 = 0x26, + DBG_BLOCK_ID_UNUSED19_BY2 = 0x27, + DBG_BLOCK_ID_CB00_BY2 = 0x28, + DBG_BLOCK_ID_CB02_BY2 = 0x29, + DBG_BLOCK_ID_CB04_BY2 = 0x2a, + DBG_BLOCK_ID_UNUSED22_BY2 = 0x2b, + DBG_BLOCK_ID_CB10_BY2 = 0x2c, + DBG_BLOCK_ID_CB12_BY2 = 0x2d, + DBG_BLOCK_ID_CB14_BY2 = 0x2e, + DBG_BLOCK_ID_UNUSED25_BY2 = 0x2f, + DBG_BLOCK_ID_TCP0_BY2 = 0x30, + DBG_BLOCK_ID_TCP2_BY2 = 0x31, + DBG_BLOCK_ID_TCP4_BY2 = 0x32, + DBG_BLOCK_ID_TCP6_BY2 = 0x33, + DBG_BLOCK_ID_TCP8_BY2 = 0x34, + DBG_BLOCK_ID_TCP10_BY2 = 0x35, + DBG_BLOCK_ID_TCP12_BY2 = 0x36, + DBG_BLOCK_ID_TCP14_BY2 = 0x37, + DBG_BLOCK_ID_TCP16_BY2 = 0x38, + DBG_BLOCK_ID_TCP18_BY2 = 0x39, + DBG_BLOCK_ID_TCP20_BY2 = 0x3a, + DBG_BLOCK_ID_TCP22_BY2 = 0x3b, + DBG_BLOCK_ID_TCP_RESERVED0_BY2 = 0x3c, + DBG_BLOCK_ID_TCP_RESERVED2_BY2 = 0x3d, + DBG_BLOCK_ID_TCP_RESERVED4_BY2 = 0x3e, + DBG_BLOCK_ID_TCP_RESERVED6_BY2 = 0x3f, + DBG_BLOCK_ID_DB00_BY2 = 0x40, + DBG_BLOCK_ID_DB02_BY2 = 0x41, + DBG_BLOCK_ID_DB04_BY2 = 0x42, + DBG_BLOCK_ID_UNUSED28_BY2 = 0x43, + DBG_BLOCK_ID_DB10_BY2 = 0x44, + DBG_BLOCK_ID_DB12_BY2 = 0x45, + DBG_BLOCK_ID_DB14_BY2 = 0x46, + DBG_BLOCK_ID_UNUSED31_BY2 = 0x47, + DBG_BLOCK_ID_TCC0_BY2 = 0x48, + DBG_BLOCK_ID_TCC2_BY2 = 0x49, + DBG_BLOCK_ID_TCC4_BY2 = 0x4a, + DBG_BLOCK_ID_TCC6_BY2 = 0x4b, + DBG_BLOCK_ID_SPS00_BY2 = 0x4c, + DBG_BLOCK_ID_SPS02_BY2 = 0x4d, + DBG_BLOCK_ID_SPS11_BY2 = 0x4e, + DBG_BLOCK_ID_UNUSED33_BY2 = 0x4f, + DBG_BLOCK_ID_TA00_BY2 = 0x50, + DBG_BLOCK_ID_TA02_BY2 = 0x51, + DBG_BLOCK_ID_TA04_BY2 = 0x52, + DBG_BLOCK_ID_TA06_BY2 = 0x53, + DBG_BLOCK_ID_TA08_BY2 = 0x54, + DBG_BLOCK_ID_TA0A_BY2 = 0x55, + DBG_BLOCK_ID_UNUSED35_BY2 = 0x56, + DBG_BLOCK_ID_UNUSED37_BY2 = 0x57, + DBG_BLOCK_ID_TA10_BY2 = 0x58, + DBG_BLOCK_ID_TA12_BY2 = 0x59, + DBG_BLOCK_ID_TA14_BY2 = 0x5a, + DBG_BLOCK_ID_TA16_BY2 = 0x5b, + DBG_BLOCK_ID_TA18_BY2 = 0x5c, + DBG_BLOCK_ID_TA1A_BY2 = 0x5d, + DBG_BLOCK_ID_UNUSED39_BY2 = 0x5e, + DBG_BLOCK_ID_UNUSED41_BY2 = 0x5f, + DBG_BLOCK_ID_TD00_BY2 = 0x60, + DBG_BLOCK_ID_TD02_BY2 = 0x61, + DBG_BLOCK_ID_TD04_BY2 = 0x62, + DBG_BLOCK_ID_TD06_BY2 = 0x63, + DBG_BLOCK_ID_TD08_BY2 = 0x64, + DBG_BLOCK_ID_TD0A_BY2 = 0x65, + DBG_BLOCK_ID_UNUSED43_BY2 = 0x66, + DBG_BLOCK_ID_UNUSED45_BY2 = 0x67, + DBG_BLOCK_ID_TD10_BY2 = 0x68, + DBG_BLOCK_ID_TD12_BY2 = 0x69, + DBG_BLOCK_ID_TD14_BY2 = 0x6a, + DBG_BLOCK_ID_TD16_BY2 = 0x6b, + DBG_BLOCK_ID_TD18_BY2 = 0x6c, + DBG_BLOCK_ID_TD1A_BY2 = 0x6d, + DBG_BLOCK_ID_UNUSED47_BY2 = 0x6e, + DBG_BLOCK_ID_UNUSED49_BY2 = 0x6f, + DBG_BLOCK_ID_MCD0_BY2 = 0x70, + DBG_BLOCK_ID_MCD2_BY2 = 0x71, + DBG_BLOCK_ID_MCD4_BY2 = 0x72, + DBG_BLOCK_ID_UNUSED51_BY2 = 0x73, +} DebugBlockId_BY2; +typedef enum DebugBlockId_BY4 { + DBG_BLOCK_ID_RESERVED_BY4 = 0x0, + DBG_BLOCK_ID_CG_BY4 = 0x1, + DBG_BLOCK_ID_CSC_BY4 = 0x2, + DBG_BLOCK_ID_SQ_BY4 = 0x3, + DBG_BLOCK_ID_DMA0_BY4 = 0x4, + DBG_BLOCK_ID_SPIS_BY4 = 0x5, + DBG_BLOCK_ID_CP0_BY4 = 0x6, + DBG_BLOCK_ID_UVDU_BY4 = 0x7, + DBG_BLOCK_ID_VGT0_BY4 = 0x8, + DBG_BLOCK_ID_SCT0_BY4 = 0x9, + DBG_BLOCK_ID_TCAA_BY4 = 0xa, + DBG_BLOCK_ID_MCC0_BY4 = 0xb, + DBG_BLOCK_ID_SX0_BY4 = 0xc, + DBG_BLOCK_ID_UNUSED4_BY4 = 0xd, + DBG_BLOCK_ID_PC0_BY4 = 0xe, + DBG_BLOCK_ID_UNUSED10_BY4 = 0xf, + DBG_BLOCK_ID_SCB0_BY4 = 0x10, + DBG_BLOCK_ID_SCF0_BY4 = 0x11, + DBG_BLOCK_ID_BCI0_BY4 = 0x12, + DBG_BLOCK_ID_UNUSED17_BY4 = 0x13, + DBG_BLOCK_ID_CB00_BY4 = 0x14, + DBG_BLOCK_ID_CB04_BY4 = 0x15, + DBG_BLOCK_ID_CB10_BY4 = 0x16, + DBG_BLOCK_ID_CB14_BY4 = 0x17, + DBG_BLOCK_ID_TCP0_BY4 = 0x18, + DBG_BLOCK_ID_TCP4_BY4 = 0x19, + DBG_BLOCK_ID_TCP8_BY4 = 0x1a, + DBG_BLOCK_ID_TCP12_BY4 = 0x1b, + DBG_BLOCK_ID_TCP16_BY4 = 0x1c, + DBG_BLOCK_ID_TCP20_BY4 = 0x1d, + DBG_BLOCK_ID_TCP_RESERVED0_BY4 = 0x1e, + DBG_BLOCK_ID_TCP_RESERVED4_BY4 = 0x1f, + DBG_BLOCK_ID_DB_BY4 = 0x20, + DBG_BLOCK_ID_DB04_BY4 = 0x21, + DBG_BLOCK_ID_DB10_BY4 = 0x22, + DBG_BLOCK_ID_DB14_BY4 = 0x23, + DBG_BLOCK_ID_TCC0_BY4 = 0x24, + DBG_BLOCK_ID_TCC4_BY4 = 0x25, + DBG_BLOCK_ID_SPS00_BY4 = 0x26, + DBG_BLOCK_ID_SPS11_BY4 = 0x27, + DBG_BLOCK_ID_TA00_BY4 = 0x28, + DBG_BLOCK_ID_TA04_BY4 = 0x29, + DBG_BLOCK_ID_TA08_BY4 = 0x2a, + DBG_BLOCK_ID_UNUSED35_BY4 = 0x2b, + DBG_BLOCK_ID_TA10_BY4 = 0x2c, + DBG_BLOCK_ID_TA14_BY4 = 0x2d, + DBG_BLOCK_ID_TA18_BY4 = 0x2e, + DBG_BLOCK_ID_UNUSED39_BY4 = 0x2f, + DBG_BLOCK_ID_TD00_BY4 = 0x30, + DBG_BLOCK_ID_TD04_BY4 = 0x31, + DBG_BLOCK_ID_TD08_BY4 = 0x32, + DBG_BLOCK_ID_UNUSED43_BY4 = 0x33, + DBG_BLOCK_ID_TD10_BY4 = 0x34, + DBG_BLOCK_ID_TD14_BY4 = 0x35, + DBG_BLOCK_ID_TD18_BY4 = 0x36, + DBG_BLOCK_ID_UNUSED47_BY4 = 0x37, + DBG_BLOCK_ID_MCD0_BY4 = 0x38, + DBG_BLOCK_ID_MCD4_BY4 = 0x39, +} DebugBlockId_BY4; +typedef enum DebugBlockId_BY8 { + DBG_BLOCK_ID_RESERVED_BY8 = 0x0, + DBG_BLOCK_ID_CSC_BY8 = 0x1, + DBG_BLOCK_ID_DMA0_BY8 = 0x2, + DBG_BLOCK_ID_CP0_BY8 = 0x3, + DBG_BLOCK_ID_VGT0_BY8 = 0x4, + DBG_BLOCK_ID_TCAA_BY8 = 0x5, + DBG_BLOCK_ID_SX0_BY8 = 0x6, + DBG_BLOCK_ID_PC0_BY8 = 0x7, + DBG_BLOCK_ID_SCB0_BY8 = 0x8, + DBG_BLOCK_ID_BCI0_BY8 = 0x9, + DBG_BLOCK_ID_CB00_BY8 = 0xa, + DBG_BLOCK_ID_CB10_BY8 = 0xb, + DBG_BLOCK_ID_TCP0_BY8 = 0xc, + DBG_BLOCK_ID_TCP8_BY8 = 0xd, + DBG_BLOCK_ID_TCP16_BY8 = 0xe, + DBG_BLOCK_ID_TCP_RESERVED0_BY8 = 0xf, + DBG_BLOCK_ID_DB00_BY8 = 0x10, + DBG_BLOCK_ID_DB10_BY8 = 0x11, + DBG_BLOCK_ID_TCC0_BY8 = 0x12, + DBG_BLOCK_ID_SPS00_BY8 = 0x13, + DBG_BLOCK_ID_TA00_BY8 = 0x14, + DBG_BLOCK_ID_TA08_BY8 = 0x15, + DBG_BLOCK_ID_TA10_BY8 = 0x16, + DBG_BLOCK_ID_TA18_BY8 = 0x17, + DBG_BLOCK_ID_TD00_BY8 = 0x18, + DBG_BLOCK_ID_TD08_BY8 = 0x19, + DBG_BLOCK_ID_TD10_BY8 = 0x1a, + DBG_BLOCK_ID_TD18_BY8 = 0x1b, + DBG_BLOCK_ID_MCD0_BY8 = 0x1c, +} DebugBlockId_BY8; +typedef enum DebugBlockId_BY16 { + DBG_BLOCK_ID_RESERVED_BY16 = 0x0, + DBG_BLOCK_ID_DMA0_BY16 = 0x1, + DBG_BLOCK_ID_VGT0_BY16 = 0x2, + DBG_BLOCK_ID_SX0_BY16 = 0x3, + DBG_BLOCK_ID_SCB0_BY16 = 0x4, + DBG_BLOCK_ID_CB00_BY16 = 0x5, + DBG_BLOCK_ID_TCP0_BY16 = 0x6, + DBG_BLOCK_ID_TCP16_BY16 = 0x7, + DBG_BLOCK_ID_DB00_BY16 = 0x8, + DBG_BLOCK_ID_TCC0_BY16 = 0x9, + DBG_BLOCK_ID_TA00_BY16 = 0xa, + DBG_BLOCK_ID_TA10_BY16 = 0xb, + DBG_BLOCK_ID_TD00_BY16 = 0xc, + DBG_BLOCK_ID_TD10_BY16 = 0xd, + DBG_BLOCK_ID_MCD0_BY16 = 0xe, +} DebugBlockId_BY16; +typedef enum ColorTransform { + DCC_CT_AUTO = 0x0, + DCC_CT_NONE = 0x1, + ABGR_TO_A_BG_G_RB = 0x2, + BGRA_TO_BG_G_RB_A = 0x3, +} ColorTransform; +typedef enum CompareRef { + REF_NEVER = 0x0, + REF_LESS = 0x1, + REF_EQUAL = 0x2, + REF_LEQUAL = 0x3, + REF_GREATER = 0x4, + REF_NOTEQUAL = 0x5, + REF_GEQUAL = 0x6, + REF_ALWAYS = 0x7, +} CompareRef; +typedef enum ReadSize { + READ_256_BITS = 0x0, + READ_512_BITS = 0x1, +} ReadSize; +typedef enum DepthFormat { + DEPTH_INVALID = 0x0, + DEPTH_16 = 0x1, + DEPTH_X8_24 = 0x2, + DEPTH_8_24 = 0x3, + DEPTH_X8_24_FLOAT = 0x4, + DEPTH_8_24_FLOAT = 0x5, + DEPTH_32_FLOAT = 0x6, + DEPTH_X24_8_32_FLOAT = 0x7, +} DepthFormat; +typedef enum ZFormat { + Z_INVALID = 0x0, + Z_16 = 0x1, + Z_24 = 0x2, + Z_32_FLOAT = 0x3, +} ZFormat; +typedef enum StencilFormat { + STENCIL_INVALID = 0x0, + STENCIL_8 = 0x1, +} StencilFormat; +typedef enum CmaskMode { + CMASK_CLEAR_NONE = 0x0, + CMASK_CLEAR_ONE = 0x1, + CMASK_CLEAR_ALL = 0x2, + CMASK_ANY_EXPANDED = 0x3, + CMASK_ALPHA0_FRAG1 = 0x4, + CMASK_ALPHA0_FRAG2 = 0x5, + CMASK_ALPHA0_FRAG4 = 0x6, + CMASK_ALPHA0_FRAGS = 0x7, + CMASK_ALPHA1_FRAG1 = 0x8, + CMASK_ALPHA1_FRAG2 = 0x9, + CMASK_ALPHA1_FRAG4 = 0xa, + CMASK_ALPHA1_FRAGS = 0xb, + CMASK_ALPHAX_FRAG1 = 0xc, + CMASK_ALPHAX_FRAG2 = 0xd, + CMASK_ALPHAX_FRAG4 = 0xe, + CMASK_ALPHAX_FRAGS = 0xf, +} CmaskMode; +typedef enum QuadExportFormat { + EXPORT_UNUSED = 0x0, + EXPORT_32_R = 0x1, + EXPORT_32_GR = 0x2, + EXPORT_32_AR = 0x3, + EXPORT_FP16_ABGR = 0x4, + EXPORT_UNSIGNED16_ABGR = 0x5, + EXPORT_SIGNED16_ABGR = 0x6, + EXPORT_32_ABGR = 0x7, +} QuadExportFormat; +typedef enum QuadExportFormatOld { + EXPORT_4P_32BPC_ABGR = 0x0, + EXPORT_4P_16BPC_ABGR = 0x1, + EXPORT_4P_32BPC_GR = 0x2, + EXPORT_4P_32BPC_AR = 0x3, + EXPORT_2P_32BPC_ABGR = 0x4, + EXPORT_8P_32BPC_R = 0x5, +} QuadExportFormatOld; +typedef enum ColorFormat { + COLOR_INVALID = 0x0, + COLOR_8 = 0x1, + COLOR_16 = 0x2, + COLOR_8_8 = 0x3, + COLOR_32 = 0x4, + COLOR_16_16 = 0x5, + COLOR_10_11_11 = 0x6, + COLOR_11_11_10 = 0x7, + COLOR_10_10_10_2 = 0x8, + COLOR_2_10_10_10 = 0x9, + COLOR_8_8_8_8 = 0xa, + COLOR_32_32 = 0xb, + COLOR_16_16_16_16 = 0xc, + COLOR_RESERVED_13 = 0xd, + COLOR_32_32_32_32 = 0xe, + COLOR_RESERVED_15 = 0xf, + COLOR_5_6_5 = 0x10, + COLOR_1_5_5_5 = 0x11, + COLOR_5_5_5_1 = 0x12, + COLOR_4_4_4_4 = 0x13, + COLOR_8_24 = 0x14, + COLOR_24_8 = 0x15, + COLOR_X24_8_32_FLOAT = 0x16, + COLOR_RESERVED_23 = 0x17, +} ColorFormat; +typedef enum SurfaceFormat { + FMT_INVALID = 0x0, + FMT_8 = 0x1, + FMT_16 = 0x2, + FMT_8_8 = 0x3, + FMT_32 = 0x4, + FMT_16_16 = 0x5, + FMT_10_11_11 = 0x6, + FMT_11_11_10 = 0x7, + FMT_10_10_10_2 = 0x8, + FMT_2_10_10_10 = 0x9, + FMT_8_8_8_8 = 0xa, + FMT_32_32 = 0xb, + FMT_16_16_16_16 = 0xc, + FMT_32_32_32 = 0xd, + FMT_32_32_32_32 = 0xe, + FMT_RESERVED_4 = 0xf, + FMT_5_6_5 = 0x10, + FMT_1_5_5_5 = 0x11, + FMT_5_5_5_1 = 0x12, + FMT_4_4_4_4 = 0x13, + FMT_8_24 = 0x14, + FMT_24_8 = 0x15, + FMT_X24_8_32_FLOAT = 0x16, + FMT_RESERVED_33 = 0x17, + FMT_11_11_10_FLOAT = 0x18, + FMT_16_FLOAT = 0x19, + FMT_32_FLOAT = 0x1a, + FMT_16_16_FLOAT = 0x1b, + FMT_8_24_FLOAT = 0x1c, + FMT_24_8_FLOAT = 0x1d, + FMT_32_32_FLOAT = 0x1e, + FMT_10_11_11_FLOAT = 0x1f, + FMT_16_16_16_16_FLOAT = 0x20, + FMT_3_3_2 = 0x21, + FMT_6_5_5 = 0x22, + FMT_32_32_32_32_FLOAT = 0x23, + FMT_RESERVED_36 = 0x24, + FMT_1 = 0x25, + FMT_1_REVERSED = 0x26, + FMT_GB_GR = 0x27, + FMT_BG_RG = 0x28, + FMT_32_AS_8 = 0x29, + FMT_32_AS_8_8 = 0x2a, + FMT_5_9_9_9_SHAREDEXP = 0x2b, + FMT_8_8_8 = 0x2c, + FMT_16_16_16 = 0x2d, + FMT_16_16_16_FLOAT = 0x2e, + FMT_4_4 = 0x2f, + FMT_32_32_32_FLOAT = 0x30, + FMT_BC1 = 0x31, + FMT_BC2 = 0x32, + FMT_BC3 = 0x33, + FMT_BC4 = 0x34, + FMT_BC5 = 0x35, + FMT_BC6 = 0x36, + FMT_BC7 = 0x37, + FMT_32_AS_32_32_32_32 = 0x38, + FMT_APC3 = 0x39, + FMT_APC4 = 0x3a, + FMT_APC5 = 0x3b, + FMT_APC6 = 0x3c, + FMT_APC7 = 0x3d, + FMT_CTX1 = 0x3e, + FMT_RESERVED_63 = 0x3f, +} SurfaceFormat; +typedef enum BUF_DATA_FORMAT { + BUF_DATA_FORMAT_INVALID = 0x0, + BUF_DATA_FORMAT_8 = 0x1, + BUF_DATA_FORMAT_16 = 0x2, + BUF_DATA_FORMAT_8_8 = 0x3, + BUF_DATA_FORMAT_32 = 0x4, + BUF_DATA_FORMAT_16_16 = 0x5, + BUF_DATA_FORMAT_10_11_11 = 0x6, + BUF_DATA_FORMAT_11_11_10 = 0x7, + BUF_DATA_FORMAT_10_10_10_2 = 0x8, + BUF_DATA_FORMAT_2_10_10_10 = 0x9, + BUF_DATA_FORMAT_8_8_8_8 = 0xa, + BUF_DATA_FORMAT_32_32 = 0xb, + BUF_DATA_FORMAT_16_16_16_16 = 0xc, + BUF_DATA_FORMAT_32_32_32 = 0xd, + BUF_DATA_FORMAT_32_32_32_32 = 0xe, + BUF_DATA_FORMAT_RESERVED_15 = 0xf, +} BUF_DATA_FORMAT; +typedef enum IMG_DATA_FORMAT { + IMG_DATA_FORMAT_INVALID = 0x0, + IMG_DATA_FORMAT_8 = 0x1, + IMG_DATA_FORMAT_16 = 0x2, + IMG_DATA_FORMAT_8_8 = 0x3, + IMG_DATA_FORMAT_32 = 0x4, + IMG_DATA_FORMAT_16_16 = 0x5, + IMG_DATA_FORMAT_10_11_11 = 0x6, + IMG_DATA_FORMAT_11_11_10 = 0x7, + IMG_DATA_FORMAT_10_10_10_2 = 0x8, + IMG_DATA_FORMAT_2_10_10_10 = 0x9, + IMG_DATA_FORMAT_8_8_8_8 = 0xa, + IMG_DATA_FORMAT_32_32 = 0xb, + IMG_DATA_FORMAT_16_16_16_16 = 0xc, + IMG_DATA_FORMAT_32_32_32 = 0xd, + IMG_DATA_FORMAT_32_32_32_32 = 0xe, + IMG_DATA_FORMAT_RESERVED_15 = 0xf, + IMG_DATA_FORMAT_5_6_5 = 0x10, + IMG_DATA_FORMAT_1_5_5_5 = 0x11, + IMG_DATA_FORMAT_5_5_5_1 = 0x12, + IMG_DATA_FORMAT_4_4_4_4 = 0x13, + IMG_DATA_FORMAT_8_24 = 0x14, + IMG_DATA_FORMAT_24_8 = 0x15, + IMG_DATA_FORMAT_X24_8_32 = 0x16, + IMG_DATA_FORMAT_RESERVED_23 = 0x17, + IMG_DATA_FORMAT_RESERVED_24 = 0x18, + IMG_DATA_FORMAT_RESERVED_25 = 0x19, + IMG_DATA_FORMAT_RESERVED_26 = 0x1a, + IMG_DATA_FORMAT_RESERVED_27 = 0x1b, + IMG_DATA_FORMAT_RESERVED_28 = 0x1c, + IMG_DATA_FORMAT_RESERVED_29 = 0x1d, + IMG_DATA_FORMAT_RESERVED_30 = 0x1e, + IMG_DATA_FORMAT_RESERVED_31 = 0x1f, + IMG_DATA_FORMAT_GB_GR = 0x20, + IMG_DATA_FORMAT_BG_RG = 0x21, + IMG_DATA_FORMAT_5_9_9_9 = 0x22, + IMG_DATA_FORMAT_BC1 = 0x23, + IMG_DATA_FORMAT_BC2 = 0x24, + IMG_DATA_FORMAT_BC3 = 0x25, + IMG_DATA_FORMAT_BC4 = 0x26, + IMG_DATA_FORMAT_BC5 = 0x27, + IMG_DATA_FORMAT_BC6 = 0x28, + IMG_DATA_FORMAT_BC7 = 0x29, + IMG_DATA_FORMAT_RESERVED_42 = 0x2a, + IMG_DATA_FORMAT_RESERVED_43 = 0x2b, + IMG_DATA_FORMAT_FMASK8_S2_F1 = 0x2c, + IMG_DATA_FORMAT_FMASK8_S4_F1 = 0x2d, + IMG_DATA_FORMAT_FMASK8_S8_F1 = 0x2e, + IMG_DATA_FORMAT_FMASK8_S2_F2 = 0x2f, + IMG_DATA_FORMAT_FMASK8_S4_F2 = 0x30, + IMG_DATA_FORMAT_FMASK8_S4_F4 = 0x31, + IMG_DATA_FORMAT_FMASK16_S16_F1 = 0x32, + IMG_DATA_FORMAT_FMASK16_S8_F2 = 0x33, + IMG_DATA_FORMAT_FMASK32_S16_F2 = 0x34, + IMG_DATA_FORMAT_FMASK32_S8_F4 = 0x35, + IMG_DATA_FORMAT_FMASK32_S8_F8 = 0x36, + IMG_DATA_FORMAT_FMASK64_S16_F4 = 0x37, + IMG_DATA_FORMAT_FMASK64_S16_F8 = 0x38, + IMG_DATA_FORMAT_4_4 = 0x39, + IMG_DATA_FORMAT_6_5_5 = 0x3a, + IMG_DATA_FORMAT_1 = 0x3b, + IMG_DATA_FORMAT_1_REVERSED = 0x3c, + IMG_DATA_FORMAT_32_AS_8 = 0x3d, + IMG_DATA_FORMAT_32_AS_8_8 = 0x3e, + IMG_DATA_FORMAT_32_AS_32_32_32_32 = 0x3f, +} IMG_DATA_FORMAT; +typedef enum BUF_NUM_FORMAT { + BUF_NUM_FORMAT_UNORM = 0x0, + BUF_NUM_FORMAT_SNORM = 0x1, + BUF_NUM_FORMAT_USCALED = 0x2, + BUF_NUM_FORMAT_SSCALED = 0x3, + BUF_NUM_FORMAT_UINT = 0x4, + BUF_NUM_FORMAT_SINT = 0x5, + BUF_NUM_FORMAT_RESERVED_6 = 0x6, + BUF_NUM_FORMAT_FLOAT = 0x7, +} BUF_NUM_FORMAT; +typedef enum IMG_NUM_FORMAT { + IMG_NUM_FORMAT_UNORM = 0x0, + IMG_NUM_FORMAT_SNORM = 0x1, + IMG_NUM_FORMAT_USCALED = 0x2, + IMG_NUM_FORMAT_SSCALED = 0x3, + IMG_NUM_FORMAT_UINT = 0x4, + IMG_NUM_FORMAT_SINT = 0x5, + IMG_NUM_FORMAT_RESERVED_6 = 0x6, + IMG_NUM_FORMAT_FLOAT = 0x7, + IMG_NUM_FORMAT_RESERVED_8 = 0x8, + IMG_NUM_FORMAT_SRGB = 0x9, + IMG_NUM_FORMAT_RESERVED_10 = 0xa, + IMG_NUM_FORMAT_RESERVED_11 = 0xb, + IMG_NUM_FORMAT_RESERVED_12 = 0xc, + IMG_NUM_FORMAT_RESERVED_13 = 0xd, + IMG_NUM_FORMAT_RESERVED_14 = 0xe, + IMG_NUM_FORMAT_RESERVED_15 = 0xf, +} IMG_NUM_FORMAT; +typedef enum TileType { + ARRAY_COLOR_TILE = 0x0, + ARRAY_DEPTH_TILE = 0x1, +} TileType; +typedef enum NonDispTilingOrder { + ADDR_SURF_MICRO_TILING_DISPLAY = 0x0, + ADDR_SURF_MICRO_TILING_NON_DISPLAY = 0x1, +} NonDispTilingOrder; +typedef enum MicroTileMode { + ADDR_SURF_DISPLAY_MICRO_TILING = 0x0, + ADDR_SURF_THIN_MICRO_TILING = 0x1, + ADDR_SURF_DEPTH_MICRO_TILING = 0x2, + ADDR_SURF_ROTATED_MICRO_TILING = 0x3, + ADDR_SURF_THICK_MICRO_TILING = 0x4, +} MicroTileMode; +typedef enum TileSplit { + ADDR_SURF_TILE_SPLIT_64B = 0x0, + ADDR_SURF_TILE_SPLIT_128B = 0x1, + ADDR_SURF_TILE_SPLIT_256B = 0x2, + ADDR_SURF_TILE_SPLIT_512B = 0x3, + ADDR_SURF_TILE_SPLIT_1KB = 0x4, + ADDR_SURF_TILE_SPLIT_2KB = 0x5, + ADDR_SURF_TILE_SPLIT_4KB = 0x6, +} TileSplit; +typedef enum SampleSplit { + ADDR_SURF_SAMPLE_SPLIT_1 = 0x0, + ADDR_SURF_SAMPLE_SPLIT_2 = 0x1, + ADDR_SURF_SAMPLE_SPLIT_4 = 0x2, + ADDR_SURF_SAMPLE_SPLIT_8 = 0x3, +} SampleSplit; +typedef enum PipeConfig { + ADDR_SURF_P2 = 0x0, + ADDR_SURF_P2_RESERVED0 = 0x1, + ADDR_SURF_P2_RESERVED1 = 0x2, + ADDR_SURF_P2_RESERVED2 = 0x3, + ADDR_SURF_P4_8x16 = 0x4, + ADDR_SURF_P4_16x16 = 0x5, + ADDR_SURF_P4_16x32 = 0x6, + ADDR_SURF_P4_32x32 = 0x7, + ADDR_SURF_P8_16x16_8x16 = 0x8, + ADDR_SURF_P8_16x32_8x16 = 0x9, + ADDR_SURF_P8_32x32_8x16 = 0xa, + ADDR_SURF_P8_16x32_16x16 = 0xb, + ADDR_SURF_P8_32x32_16x16 = 0xc, + ADDR_SURF_P8_32x32_16x32 = 0xd, + ADDR_SURF_P8_32x64_32x32 = 0xe, + ADDR_SURF_P8_RESERVED0 = 0xf, + ADDR_SURF_P16_32x32_8x16 = 0x10, + ADDR_SURF_P16_32x32_16x16 = 0x11, +} PipeConfig; +typedef enum NumBanks { + ADDR_SURF_2_BANK = 0x0, + ADDR_SURF_4_BANK = 0x1, + ADDR_SURF_8_BANK = 0x2, + ADDR_SURF_16_BANK = 0x3, +} NumBanks; +typedef enum BankWidth { + ADDR_SURF_BANK_WIDTH_1 = 0x0, + ADDR_SURF_BANK_WIDTH_2 = 0x1, + ADDR_SURF_BANK_WIDTH_4 = 0x2, + ADDR_SURF_BANK_WIDTH_8 = 0x3, +} BankWidth; +typedef enum BankHeight { + ADDR_SURF_BANK_HEIGHT_1 = 0x0, + ADDR_SURF_BANK_HEIGHT_2 = 0x1, + ADDR_SURF_BANK_HEIGHT_4 = 0x2, + ADDR_SURF_BANK_HEIGHT_8 = 0x3, +} BankHeight; +typedef enum BankWidthHeight { + ADDR_SURF_BANK_WH_1 = 0x0, + ADDR_SURF_BANK_WH_2 = 0x1, + ADDR_SURF_BANK_WH_4 = 0x2, + ADDR_SURF_BANK_WH_8 = 0x3, +} BankWidthHeight; +typedef enum MacroTileAspect { + ADDR_SURF_MACRO_ASPECT_1 = 0x0, + ADDR_SURF_MACRO_ASPECT_2 = 0x1, + ADDR_SURF_MACRO_ASPECT_4 = 0x2, + ADDR_SURF_MACRO_ASPECT_8 = 0x3, +} MacroTileAspect; +typedef enum GATCL1RequestType { + GATCL1_TYPE_NORMAL = 0x0, + GATCL1_TYPE_SHOOTDOWN = 0x1, + GATCL1_TYPE_BYPASS = 0x2, +} GATCL1RequestType; +typedef enum TCC_CACHE_POLICIES { + TCC_CACHE_POLICY_LRU = 0x0, + TCC_CACHE_POLICY_STREAM = 0x1, +} TCC_CACHE_POLICIES; +typedef enum MTYPE { + MTYPE_NC_NV = 0x0, + MTYPE_NC = 0x1, + MTYPE_CC = 0x2, + MTYPE_UC = 0x3, +} MTYPE; +typedef enum PERFMON_COUNTER_MODE { + PERFMON_COUNTER_MODE_ACCUM = 0x0, + PERFMON_COUNTER_MODE_ACTIVE_CYCLES = 0x1, + PERFMON_COUNTER_MODE_MAX = 0x2, + PERFMON_COUNTER_MODE_DIRTY = 0x3, + PERFMON_COUNTER_MODE_SAMPLE = 0x4, + PERFMON_COUNTER_MODE_CYCLES_SINCE_FIRST_EVENT = 0x5, + PERFMON_COUNTER_MODE_CYCLES_SINCE_LAST_EVENT = 0x6, + PERFMON_COUNTER_MODE_CYCLES_GE_HI = 0x7, + PERFMON_COUNTER_MODE_CYCLES_EQ_HI = 0x8, + PERFMON_COUNTER_MODE_INACTIVE_CYCLES = 0x9, + PERFMON_COUNTER_MODE_RESERVED = 0xf, +} PERFMON_COUNTER_MODE; +typedef enum PERFMON_SPM_MODE { + PERFMON_SPM_MODE_OFF = 0x0, + PERFMON_SPM_MODE_16BIT_CLAMP = 0x1, + PERFMON_SPM_MODE_16BIT_NO_CLAMP = 0x2, + PERFMON_SPM_MODE_32BIT_CLAMP = 0x3, + PERFMON_SPM_MODE_32BIT_NO_CLAMP = 0x4, + PERFMON_SPM_MODE_RESERVED_5 = 0x5, + PERFMON_SPM_MODE_RESERVED_6 = 0x6, + PERFMON_SPM_MODE_RESERVED_7 = 0x7, + PERFMON_SPM_MODE_TEST_MODE_0 = 0x8, + PERFMON_SPM_MODE_TEST_MODE_1 = 0x9, + PERFMON_SPM_MODE_TEST_MODE_2 = 0xa, +} PERFMON_SPM_MODE; +typedef enum SurfaceTiling { + ARRAY_LINEAR = 0x0, + ARRAY_TILED = 0x1, +} SurfaceTiling; +typedef enum SurfaceArray { + ARRAY_1D = 0x0, + ARRAY_2D = 0x1, + ARRAY_3D = 0x2, + ARRAY_3D_SLICE = 0x3, +} SurfaceArray; +typedef enum ColorArray { + ARRAY_2D_ALT_COLOR = 0x0, + ARRAY_2D_COLOR = 0x1, + ARRAY_3D_SLICE_COLOR = 0x3, +} ColorArray; +typedef enum DepthArray { + ARRAY_2D_ALT_DEPTH = 0x0, + ARRAY_2D_DEPTH = 0x1, +} DepthArray; +typedef enum ENUM_NUM_SIMD_PER_CU { + NUM_SIMD_PER_CU = 0x4, +} ENUM_NUM_SIMD_PER_CU; +typedef enum MEM_PWR_FORCE_CTRL { + NO_FORCE_REQUEST = 0x0, + FORCE_LIGHT_SLEEP_REQUEST = 0x1, + FORCE_DEEP_SLEEP_REQUEST = 0x2, + FORCE_SHUT_DOWN_REQUEST = 0x3, +} MEM_PWR_FORCE_CTRL; +typedef enum MEM_PWR_FORCE_CTRL2 { + NO_FORCE_REQ = 0x0, + FORCE_LIGHT_SLEEP_REQ = 0x1, +} MEM_PWR_FORCE_CTRL2; +typedef enum MEM_PWR_DIS_CTRL { + ENABLE_MEM_PWR_CTRL = 0x0, + DISABLE_MEM_PWR_CTRL = 0x1, +} MEM_PWR_DIS_CTRL; +typedef enum MEM_PWR_SEL_CTRL { + DYNAMIC_SHUT_DOWN_ENABLE = 0x0, + DYNAMIC_DEEP_SLEEP_ENABLE = 0x1, + DYNAMIC_LIGHT_SLEEP_ENABLE = 0x2, +} MEM_PWR_SEL_CTRL; +typedef enum MEM_PWR_SEL_CTRL2 { + DYNAMIC_DEEP_SLEEP_EN = 0x0, + DYNAMIC_LIGHT_SLEEP_EN = 0x1, +} MEM_PWR_SEL_CTRL2; + +#endif /* SMU_7_1_3_ENUM_H */ diff --git a/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_sh_mask.h new file mode 100644 index 000000000000..1ede9e274714 --- /dev/null +++ b/drivers/gpu/drm/amd/include/asic_reg/smu/smu_7_1_3_sh_mask.h @@ -0,0 +1,6080 @@ +/* + * SMU_7_1_3 Register documentation + * + * Copyright (C) 2014 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef SMU_7_1_3_SH_MASK_H +#define SMU_7_1_3_SH_MASK_H + +#define GCK_SMC_IND_INDEX__SMC_IND_ADDR_MASK 0xffffffff +#define GCK_SMC_IND_INDEX__SMC_IND_ADDR__SHIFT 0x0 +#define GCK_SMC_IND_DATA__SMC_IND_DATA_MASK 0xffffffff +#define GCK_SMC_IND_DATA__SMC_IND_DATA__SHIFT 0x0 +#define GCK_MCLK_FUSES__StartupMClkDid_MASK 0x7f +#define GCK_MCLK_FUSES__StartupMClkDid__SHIFT 0x0 +#define GCK_MCLK_FUSES__MClkADCA_MASK 0x780 +#define GCK_MCLK_FUSES__MClkADCA__SHIFT 0x7 +#define GCK_MCLK_FUSES__MClkDDCA_MASK 0x1800 +#define GCK_MCLK_FUSES__MClkDDCA__SHIFT 0xb +#define GCK_MCLK_FUSES__MClkDiDtWait_MASK 0xe000 +#define GCK_MCLK_FUSES__MClkDiDtWait__SHIFT 0xd +#define GCK_MCLK_FUSES__MClkDiDtFloor_MASK 0x30000 +#define GCK_MCLK_FUSES__MClkDiDtFloor__SHIFT 0x10 +#define CG_DCLK_CNTL__DCLK_DIVIDER_MASK 0x7f +#define CG_DCLK_CNTL__DCLK_DIVIDER__SHIFT 0x0 +#define CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK 0x100 +#define CG_DCLK_CNTL__DCLK_DIR_CNTL_EN__SHIFT 0x8 +#define CG_DCLK_CNTL__DCLK_DIR_CNTL_TOG_MASK 0x200 +#define CG_DCLK_CNTL__DCLK_DIR_CNTL_TOG__SHIFT 0x9 +#define CG_DCLK_CNTL__DCLK_DIR_CNTL_DIVIDER_MASK 0x1fc00 +#define CG_DCLK_CNTL__DCLK_DIR_CNTL_DIVIDER__SHIFT 0xa +#define CG_DCLK_STATUS__DCLK_STATUS_MASK 0x1 +#define CG_DCLK_STATUS__DCLK_STATUS__SHIFT 0x0 +#define CG_DCLK_STATUS__DCLK_DIR_CNTL_DONETOG_MASK 0x2 +#define CG_DCLK_STATUS__DCLK_DIR_CNTL_DONETOG__SHIFT 0x1 +#define CG_VCLK_CNTL__VCLK_DIVIDER_MASK 0x7f +#define CG_VCLK_CNTL__VCLK_DIVIDER__SHIFT 0x0 +#define CG_VCLK_CNTL__VCLK_DIR_CNTL_EN_MASK 0x100 +#define CG_VCLK_CNTL__VCLK_DIR_CNTL_EN__SHIFT 0x8 +#define CG_VCLK_CNTL__VCLK_DIR_CNTL_TOG_MASK 0x200 +#define CG_VCLK_CNTL__VCLK_DIR_CNTL_TOG__SHIFT 0x9 +#define CG_VCLK_CNTL__VCLK_DIR_CNTL_DIVIDER_MASK 0x1fc00 +#define CG_VCLK_CNTL__VCLK_DIR_CNTL_DIVIDER__SHIFT 0xa +#define CG_VCLK_STATUS__VCLK_STATUS_MASK 0x1 +#define CG_VCLK_STATUS__VCLK_STATUS__SHIFT 0x0 +#define CG_VCLK_STATUS__VCLK_DIR_CNTL_DONETOG_MASK 0x2 +#define CG_VCLK_STATUS__VCLK_DIR_CNTL_DONETOG__SHIFT 0x1 +#define CG_ECLK_CNTL__ECLK_DIVIDER_MASK 0x7f +#define CG_ECLK_CNTL__ECLK_DIVIDER__SHIFT 0x0 +#define CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK 0x100 +#define CG_ECLK_CNTL__ECLK_DIR_CNTL_EN__SHIFT 0x8 +#define CG_ECLK_CNTL__ECLK_DIR_CNTL_TOG_MASK 0x200 +#define CG_ECLK_CNTL__ECLK_DIR_CNTL_TOG__SHIFT 0x9 +#define CG_ECLK_CNTL__ECLK_DIR_CNTL_DIVIDER_MASK 0x1fc00 +#define CG_ECLK_CNTL__ECLK_DIR_CNTL_DIVIDER__SHIFT 0xa +#define CG_ECLK_STATUS__ECLK_STATUS_MASK 0x1 +#define CG_ECLK_STATUS__ECLK_STATUS__SHIFT 0x0 +#define CG_ECLK_STATUS__ECLK_DIR_CNTL_DONETOG_MASK 0x2 +#define CG_ECLK_STATUS__ECLK_DIR_CNTL_DONETOG__SHIFT 0x1 +#define CG_ACLK_CNTL__ACLK_DIVIDER_MASK 0x7f +#define CG_ACLK_CNTL__ACLK_DIVIDER__SHIFT 0x0 +#define CG_ACLK_CNTL__ACLK_DIR_CNTL_EN_MASK 0x100 +#define CG_ACLK_CNTL__ACLK_DIR_CNTL_EN__SHIFT 0x8 +#define CG_ACLK_CNTL__ACLK_DIR_CNTL_TOG_MASK 0x200 +#define CG_ACLK_CNTL__ACLK_DIR_CNTL_TOG__SHIFT 0x9 +#define CG_ACLK_CNTL__ACLK_DIR_CNTL_DIVIDER_MASK 0x1fc00 +#define CG_ACLK_CNTL__ACLK_DIR_CNTL_DIVIDER__SHIFT 0xa +#define CG_MCLK_CNTL__MCLK_DIVIDER_MASK 0x7f +#define CG_MCLK_CNTL__MCLK_DIVIDER__SHIFT 0x0 +#define CG_MCLK_CNTL__MCLK_DIR_CNTL_EN_MASK 0x100 +#define CG_MCLK_CNTL__MCLK_DIR_CNTL_EN__SHIFT 0x8 +#define CG_MCLK_CNTL__MCLK_DIR_CNTL_TOG_MASK 0x200 +#define CG_MCLK_CNTL__MCLK_DIR_CNTL_TOG__SHIFT 0x9 +#define CG_MCLK_CNTL__MCLK_DIR_CNTL_DIVIDER_MASK 0x1fc00 +#define CG_MCLK_CNTL__MCLK_DIR_CNTL_DIVIDER__SHIFT 0xa +#define CG_MCLK_STATUS__MCLK_STATUS_MASK 0x1 +#define CG_MCLK_STATUS__MCLK_STATUS__SHIFT 0x0 +#define CG_MCLK_STATUS__MCLK_DIR_CNTL_DONETOG_MASK 0x2 +#define CG_MCLK_STATUS__MCLK_DIR_CNTL_DONETOG__SHIFT 0x1 +#define GCK_DFS_BYPASS_CNTL__BYPASSECLK_MASK 0x1 +#define GCK_DFS_BYPASS_CNTL__BYPASSECLK__SHIFT 0x0 +#define GCK_DFS_BYPASS_CNTL__BYPASSLCLK_MASK 0x2 +#define GCK_DFS_BYPASS_CNTL__BYPASSLCLK__SHIFT 0x1 +#define GCK_DFS_BYPASS_CNTL__BYPASSEVCLK_MASK 0x4 +#define GCK_DFS_BYPASS_CNTL__BYPASSEVCLK__SHIFT 0x2 +#define GCK_DFS_BYPASS_CNTL__BYPASSDCLK_MASK 0x8 +#define GCK_DFS_BYPASS_CNTL__BYPASSDCLK__SHIFT 0x3 +#define GCK_DFS_BYPASS_CNTL__BYPASSVCLK_MASK 0x10 +#define GCK_DFS_BYPASS_CNTL__BYPASSVCLK__SHIFT 0x4 +#define GCK_DFS_BYPASS_CNTL__BYPASSDISPCLK_MASK 0x20 +#define GCK_DFS_BYPASS_CNTL__BYPASSDISPCLK__SHIFT 0x5 +#define GCK_DFS_BYPASS_CNTL__BYPASSDPREFCLK_MASK 0x40 +#define GCK_DFS_BYPASS_CNTL__BYPASSDPREFCLK__SHIFT 0x6 +#define GCK_DFS_BYPASS_CNTL__BYPASSACLK_MASK 0x80 +#define GCK_DFS_BYPASS_CNTL__BYPASSACLK__SHIFT 0x7 +#define GCK_DFS_BYPASS_CNTL__BYPASSADIVCLK_MASK 0x100 +#define GCK_DFS_BYPASS_CNTL__BYPASSADIVCLK__SHIFT 0x8 +#define GCK_DFS_BYPASS_CNTL__BYPASSPSPCLK_MASK 0x200 +#define GCK_DFS_BYPASS_CNTL__BYPASSPSPCLK__SHIFT 0x9 +#define GCK_DFS_BYPASS_CNTL__BYPASSSAMCLK_MASK 0x400 +#define GCK_DFS_BYPASS_CNTL__BYPASSSAMCLK__SHIFT 0xa +#define GCK_DFS_BYPASS_CNTL__BYPASSSCLK_MASK 0x800 +#define GCK_DFS_BYPASS_CNTL__BYPASSSCLK__SHIFT 0xb +#define GCK_DFS_BYPASS_CNTL__USE_SPLL_BYPASS_EN_MASK 0x1000 +#define GCK_DFS_BYPASS_CNTL__USE_SPLL_BYPASS_EN__SHIFT 0xc +#define GCK_DFS_BYPASS_CNTL__BYPASSMCLK_MASK 0x2000 +#define GCK_DFS_BYPASS_CNTL__BYPASSMCLK__SHIFT 0xd +#define CG_SPLL_FUNC_CNTL__SPLL_RESET_MASK 0x1 +#define CG_SPLL_FUNC_CNTL__SPLL_RESET__SHIFT 0x0 +#define CG_SPLL_FUNC_CNTL__SPLL_PWRON_MASK 0x2 +#define CG_SPLL_FUNC_CNTL__SPLL_PWRON__SHIFT 0x1 +#define CG_SPLL_FUNC_CNTL__SPLL_DIVEN_MASK 0x4 +#define CG_SPLL_FUNC_CNTL__SPLL_DIVEN__SHIFT 0x2 +#define CG_SPLL_FUNC_CNTL__SPLL_BYPASS_EN_MASK 0x8 +#define CG_SPLL_FUNC_CNTL__SPLL_BYPASS_EN__SHIFT 0x3 +#define CG_SPLL_FUNC_CNTL__SPLL_BYPASS_THRU_DFS_MASK 0x10 +#define CG_SPLL_FUNC_CNTL__SPLL_BYPASS_THRU_DFS__SHIFT 0x4 +#define CG_SPLL_FUNC_CNTL__SPLL_REF_DIV_MASK 0x7e0 +#define CG_SPLL_FUNC_CNTL__SPLL_REF_DIV__SHIFT 0x5 +#define CG_SPLL_FUNC_CNTL__SPLL_PDIV_A_UPDATE_MASK 0x800 +#define CG_SPLL_FUNC_CNTL__SPLL_PDIV_A_UPDATE__SHIFT 0xb +#define CG_SPLL_FUNC_CNTL__SPLL_PDIV_A_EN_MASK 0x1000 +#define CG_SPLL_FUNC_CNTL__SPLL_PDIV_A_EN__SHIFT 0xc +#define CG_SPLL_FUNC_CNTL__SPLL_PDIV_A_MASK 0x7f00000 +#define CG_SPLL_FUNC_CNTL__SPLL_PDIV_A__SHIFT 0x14 +#define CG_SPLL_FUNC_CNTL__SPLL_DIVA_ACK_MASK 0x8000000 +#define CG_SPLL_FUNC_CNTL__SPLL_DIVA_ACK__SHIFT 0x1b +#define CG_SPLL_FUNC_CNTL__SPLL_OTEST_LOCK_EN_MASK 0x10000000 +#define CG_SPLL_FUNC_CNTL__SPLL_OTEST_LOCK_EN__SHIFT 0x1c +#define CG_SPLL_FUNC_CNTL_2__SCLK_MUX_SEL_MASK 0x1ff +#define CG_SPLL_FUNC_CNTL_2__SCLK_MUX_SEL__SHIFT 0x0 +#define CG_SPLL_FUNC_CNTL_2__SPLL_CTLREQ_MASK 0x800 +#define CG_SPLL_FUNC_CNTL_2__SPLL_CTLREQ__SHIFT 0xb +#define CG_SPLL_FUNC_CNTL_2__SPLL_BYPASS_CHG_MASK 0x400000 +#define CG_SPLL_FUNC_CNTL_2__SPLL_BYPASS_CHG__SHIFT 0x16 +#define CG_SPLL_FUNC_CNTL_2__SPLL_CTLREQ_CHG_MASK 0x800000 +#define CG_SPLL_FUNC_CNTL_2__SPLL_CTLREQ_CHG__SHIFT 0x17 +#define CG_SPLL_FUNC_CNTL_2__SPLL_RESET_CHG_MASK 0x1000000 +#define CG_SPLL_FUNC_CNTL_2__SPLL_RESET_CHG__SHIFT 0x18 +#define CG_SPLL_FUNC_CNTL_2__SPLL_BABY_STEP_CHG_MASK 0x2000000 +#define CG_SPLL_FUNC_CNTL_2__SPLL_BABY_STEP_CHG__SHIFT 0x19 +#define CG_SPLL_FUNC_CNTL_2__SCLK_MUX_UPDATE_MASK 0x4000000 +#define CG_SPLL_FUNC_CNTL_2__SCLK_MUX_UPDATE__SHIFT 0x1a +#define CG_SPLL_FUNC_CNTL_2__SPLL_UNLOCK_CLEAR_MASK 0x8000000 +#define CG_SPLL_FUNC_CNTL_2__SPLL_UNLOCK_CLEAR__SHIFT 0x1b +#define CG_SPLL_FUNC_CNTL_2__SPLL_CLKF_UPDATE_MASK 0x10000000 +#define CG_SPLL_FUNC_CNTL_2__SPLL_CLKF_UPDATE__SHIFT 0x1c +#define CG_SPLL_FUNC_CNTL_2__SPLL_TEST_UNLOCK_CLR_MASK 0x40000000 +#define CG_SPLL_FUNC_CNTL_2__SPLL_TEST_UNLOCK_CLR__SHIFT 0x1e +#define CG_SPLL_FUNC_CNTL_3__SPLL_FB_DIV_MASK 0x3ffffff +#define CG_SPLL_FUNC_CNTL_3__SPLL_FB_DIV__SHIFT 0x0 +#define CG_SPLL_FUNC_CNTL_3__SPLL_DITHEN_MASK 0x10000000 +#define CG_SPLL_FUNC_CNTL_3__SPLL_DITHEN__SHIFT 0x1c +#define CG_SPLL_FUNC_CNTL_4__SPLL_SCLK_TEST_SEL_MASK 0xf +#define CG_SPLL_FUNC_CNTL_4__SPLL_SCLK_TEST_SEL__SHIFT 0x0 +#define CG_SPLL_FUNC_CNTL_4__SPLL_SCLK_EXT_SEL_MASK 0x60 +#define CG_SPLL_FUNC_CNTL_4__SPLL_SCLK_EXT_SEL__SHIFT 0x5 +#define CG_SPLL_FUNC_CNTL_4__SPLL_SCLK_EN_MASK 0x180 +#define CG_SPLL_FUNC_CNTL_4__SPLL_SCLK_EN__SHIFT 0x7 +#define CG_SPLL_FUNC_CNTL_4__SPLL_SPARE_MASK 0xe00 +#define CG_SPLL_FUNC_CNTL_4__SPLL_SPARE__SHIFT 0x9 +#define CG_SPLL_FUNC_CNTL_4__PCC_INC_DIV_MASK 0x7f000 +#define CG_SPLL_FUNC_CNTL_4__PCC_INC_DIV__SHIFT 0xc +#define CG_SPLL_FUNC_CNTL_4__TEST_FRAC_BYPASS_MASK 0x200000 +#define CG_SPLL_FUNC_CNTL_4__TEST_FRAC_BYPASS__SHIFT 0x15 +#define CG_SPLL_FUNC_CNTL_4__SPLL_ILOCK_MASK 0x800000 +#define CG_SPLL_FUNC_CNTL_4__SPLL_ILOCK__SHIFT 0x17 +#define CG_SPLL_FUNC_CNTL_4__SPLL_FBCLK_SEL_MASK 0x1000000 +#define CG_SPLL_FUNC_CNTL_4__SPLL_FBCLK_SEL__SHIFT 0x18 +#define CG_SPLL_FUNC_CNTL_4__SPLL_VCTRLADC_EN_MASK 0x2000000 +#define CG_SPLL_FUNC_CNTL_4__SPLL_VCTRLADC_EN__SHIFT 0x19 +#define CG_SPLL_FUNC_CNTL_4__SPLL_SCLK_EXT_MASK 0xc000000 +#define CG_SPLL_FUNC_CNTL_4__SPLL_SCLK_EXT__SHIFT 0x1a +#define CG_SPLL_FUNC_CNTL_4__SPLL_SPARE_EXT_MASK 0x70000000 +#define CG_SPLL_FUNC_CNTL_4__SPLL_SPARE_EXT__SHIFT 0x1c +#define CG_SPLL_FUNC_CNTL_4__SPLL_VTOI_BIAS_CNTL_MASK 0x80000000 +#define CG_SPLL_FUNC_CNTL_4__SPLL_VTOI_BIAS_CNTL__SHIFT 0x1f +#define CG_SPLL_FUNC_CNTL_5__FBDIV_SSC_BYPASS_MASK 0x1 +#define CG_SPLL_FUNC_CNTL_5__FBDIV_SSC_BYPASS__SHIFT 0x0 +#define CG_SPLL_FUNC_CNTL_5__RISEFBVCO_EN_MASK 0x2 +#define CG_SPLL_FUNC_CNTL_5__RISEFBVCO_EN__SHIFT 0x1 +#define CG_SPLL_FUNC_CNTL_5__PFD_RESET_CNTRL_MASK 0xc +#define CG_SPLL_FUNC_CNTL_5__PFD_RESET_CNTRL__SHIFT 0x2 +#define CG_SPLL_FUNC_CNTL_5__RESET_TIMER_MASK 0x30 +#define CG_SPLL_FUNC_CNTL_5__RESET_TIMER__SHIFT 0x4 +#define CG_SPLL_FUNC_CNTL_5__FAST_LOCK_CNTRL_MASK 0xc0 +#define CG_SPLL_FUNC_CNTL_5__FAST_LOCK_CNTRL__SHIFT 0x6 +#define CG_SPLL_FUNC_CNTL_5__FAST_LOCK_EN_MASK 0x100 +#define CG_SPLL_FUNC_CNTL_5__FAST_LOCK_EN__SHIFT 0x8 +#define CG_SPLL_FUNC_CNTL_5__RESET_ANTI_MUX_MASK 0x200 +#define CG_SPLL_FUNC_CNTL_5__RESET_ANTI_MUX__SHIFT 0x9 +#define CG_SPLL_FUNC_CNTL_6__SCLKMUX0_CLKOFF_CNT_MASK 0xff +#define CG_SPLL_FUNC_CNTL_6__SCLKMUX0_CLKOFF_CNT__SHIFT 0x0 +#define CG_SPLL_FUNC_CNTL_6__SCLKMUX1_CLKOFF_CNT_MASK 0xff00 +#define CG_SPLL_FUNC_CNTL_6__SCLKMUX1_CLKOFF_CNT__SHIFT 0x8 +#define CG_SPLL_FUNC_CNTL_6__SPLL_VCTL_EN_MASK 0x10000 +#define CG_SPLL_FUNC_CNTL_6__SPLL_VCTL_EN__SHIFT 0x10 +#define CG_SPLL_FUNC_CNTL_6__SPLL_VCTL_CNTRL_IN_MASK 0x1e0000 +#define CG_SPLL_FUNC_CNTL_6__SPLL_VCTL_CNTRL_IN__SHIFT 0x11 +#define CG_SPLL_FUNC_CNTL_6__SPLL_VCTL_CNTRL_OUT_MASK 0x1e00000 +#define CG_SPLL_FUNC_CNTL_6__SPLL_VCTL_CNTRL_OUT__SHIFT 0x15 +#define CG_SPLL_FUNC_CNTL_6__SPLL_LF_CNTR_MASK 0xfe000000 +#define CG_SPLL_FUNC_CNTL_6__SPLL_LF_CNTR__SHIFT 0x19 +#define CG_SPLL_FUNC_CNTL_7__SPLL_BW_CNTRL_MASK 0xfff +#define CG_SPLL_FUNC_CNTL_7__SPLL_BW_CNTRL__SHIFT 0x0 +#define SPLL_CNTL_MODE__SPLL_SW_DIR_CONTROL_MASK 0x1 +#define SPLL_CNTL_MODE__SPLL_SW_DIR_CONTROL__SHIFT 0x0 +#define SPLL_CNTL_MODE__SPLL_LEGACY_PDIV_MASK 0x2 +#define SPLL_CNTL_MODE__SPLL_LEGACY_PDIV__SHIFT 0x1 +#define SPLL_CNTL_MODE__SPLL_TEST_MASK 0x4 +#define SPLL_CNTL_MODE__SPLL_TEST__SHIFT 0x2 +#define SPLL_CNTL_MODE__SPLL_FASTEN_MASK 0x8 +#define SPLL_CNTL_MODE__SPLL_FASTEN__SHIFT 0x3 +#define SPLL_CNTL_MODE__SPLL_ENSAT_MASK 0x10 +#define SPLL_CNTL_MODE__SPLL_ENSAT__SHIFT 0x4 +#define SPLL_CNTL_MODE__SPLL_TEST_CLK_EXT_DIV_MASK 0xc00 +#define SPLL_CNTL_MODE__SPLL_TEST_CLK_EXT_DIV__SHIFT 0xa +#define SPLL_CNTL_MODE__SPLL_CTLREQ_DLY_CNT_MASK 0xff000 +#define SPLL_CNTL_MODE__SPLL_CTLREQ_DLY_CNT__SHIFT 0xc +#define SPLL_CNTL_MODE__SPLL_RESET_EN_MASK 0x10000000 +#define SPLL_CNTL_MODE__SPLL_RESET_EN__SHIFT 0x1c +#define SPLL_CNTL_MODE__SPLL_VCO_MODE_MASK 0x60000000 +#define SPLL_CNTL_MODE__SPLL_VCO_MODE__SHIFT 0x1d +#define CG_SPLL_SPREAD_SPECTRUM__SSEN_MASK 0x1 +#define CG_SPLL_SPREAD_SPECTRUM__SSEN__SHIFT 0x0 +#define CG_SPLL_SPREAD_SPECTRUM__CLKS_MASK 0xfff0 +#define CG_SPLL_SPREAD_SPECTRUM__CLKS__SHIFT 0x4 +#define CG_SPLL_SPREAD_SPECTRUM_2__CLKV_MASK 0x3ffffff +#define CG_SPLL_SPREAD_SPECTRUM_2__CLKV__SHIFT 0x0 +#define MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK 0xff00 +#define MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT 0x8 +#define CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK 0x2 +#define CG_CLKPIN_CNTL__XTALIN_DIVIDE__SHIFT 0x1 +#define CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK 0x4 +#define CG_CLKPIN_CNTL__BCLK_AS_XCLK__SHIFT 0x2 +#define CG_CLKPIN_CNTL_2__ENABLE_XCLK_MASK 0x1 +#define CG_CLKPIN_CNTL_2__ENABLE_XCLK__SHIFT 0x0 +#define CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK 0x8 +#define CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN__SHIFT 0x3 +#define CG_CLKPIN_CNTL_2__MUX_TCLK_TO_XCLK_MASK 0x100 +#define CG_CLKPIN_CNTL_2__MUX_TCLK_TO_XCLK__SHIFT 0x8 +#define CG_CLKPIN_CNTL_2__XO_IN_OSCIN_EN_MASK 0x4000 +#define CG_CLKPIN_CNTL_2__XO_IN_OSCIN_EN__SHIFT 0xe +#define CG_CLKPIN_CNTL_2__XO_IN_ICORE_CLK_OE_MASK 0x8000 +#define CG_CLKPIN_CNTL_2__XO_IN_ICORE_CLK_OE__SHIFT 0xf +#define CG_CLKPIN_CNTL_2__XO_IN_CML_RXEN_MASK 0x10000 +#define CG_CLKPIN_CNTL_2__XO_IN_CML_RXEN__SHIFT 0x10 +#define CG_CLKPIN_CNTL_2__XO_IN_BIDIR_CML_OE_MASK 0x20000 +#define CG_CLKPIN_CNTL_2__XO_IN_BIDIR_CML_OE__SHIFT 0x11 +#define CG_CLKPIN_CNTL_2__XO_IN2_OSCIN_EN_MASK 0x40000 +#define CG_CLKPIN_CNTL_2__XO_IN2_OSCIN_EN__SHIFT 0x12 +#define CG_CLKPIN_CNTL_2__XO_IN2_ICORE_CLK_OE_MASK 0x80000 +#define CG_CLKPIN_CNTL_2__XO_IN2_ICORE_CLK_OE__SHIFT 0x13 +#define CG_CLKPIN_CNTL_2__XO_IN2_CML_RXEN_MASK 0x100000 +#define CG_CLKPIN_CNTL_2__XO_IN2_CML_RXEN__SHIFT 0x14 +#define CG_CLKPIN_CNTL_2__XO_IN2_BIDIR_CML_OE_MASK 0x200000 +#define CG_CLKPIN_CNTL_2__XO_IN2_BIDIR_CML_OE__SHIFT 0x15 +#define CG_CLKPIN_CNTL_2__CML_CTRL_MASK 0xc00000 +#define CG_CLKPIN_CNTL_2__CML_CTRL__SHIFT 0x16 +#define CG_CLKPIN_CNTL_2__CLK_SPARE_MASK 0xff000000 +#define CG_CLKPIN_CNTL_2__CLK_SPARE__SHIFT 0x18 +#define CG_CLKPIN_CNTL_DC__OSC_EN_MASK 0x1 +#define CG_CLKPIN_CNTL_DC__OSC_EN__SHIFT 0x0 +#define CG_CLKPIN_CNTL_DC__XTL_LOW_GAIN_MASK 0x6 +#define CG_CLKPIN_CNTL_DC__XTL_LOW_GAIN__SHIFT 0x1 +#define CG_CLKPIN_CNTL_DC__XTL_XOCLK_DRV_R_EN_MASK 0x200 +#define CG_CLKPIN_CNTL_DC__XTL_XOCLK_DRV_R_EN__SHIFT 0x9 +#define CG_CLKPIN_CNTL_DC__XTALIN_SEL_MASK 0x1c00 +#define CG_CLKPIN_CNTL_DC__XTALIN_SEL__SHIFT 0xa +#define THM_CLK_CNTL__CMON_CLK_SEL_MASK 0xff +#define THM_CLK_CNTL__CMON_CLK_SEL__SHIFT 0x0 +#define THM_CLK_CNTL__TMON_CLK_SEL_MASK 0xff00 +#define THM_CLK_CNTL__TMON_CLK_SEL__SHIFT 0x8 +#define THM_CLK_CNTL__CTF_CLK_SHUTOFF_EN_MASK 0x10000 +#define THM_CLK_CNTL__CTF_CLK_SHUTOFF_EN__SHIFT 0x10 +#define MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK 0xff +#define MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT 0x0 +#define MISC_CLK_CTRL__ZCLK_SEL_MASK 0xff00 +#define MISC_CLK_CTRL__ZCLK_SEL__SHIFT 0x8 +#define MISC_CLK_CTRL__DFT_SMS_PG_CLK_SEL_MASK 0xff0000 +#define MISC_CLK_CTRL__DFT_SMS_PG_CLK_SEL__SHIFT 0x10 +#define GCK_PLL_TEST_CNTL__TST_SRC_SEL_MASK 0x1f +#define GCK_PLL_TEST_CNTL__TST_SRC_SEL__SHIFT 0x0 +#define GCK_PLL_TEST_CNTL__TST_REF_SEL_MASK 0x3e0 +#define GCK_PLL_TEST_CNTL__TST_REF_SEL__SHIFT 0x5 +#define GCK_PLL_TEST_CNTL__REF_TEST_COUNT_MASK 0x1fc00 +#define GCK_PLL_TEST_CNTL__REF_TEST_COUNT__SHIFT 0xa +#define GCK_PLL_TEST_CNTL__TST_RESET_MASK 0x20000 +#define GCK_PLL_TEST_CNTL__TST_RESET__SHIFT 0x11 +#define GCK_PLL_TEST_CNTL__TST_CLK_SEL_MODE_MASK 0x40000 +#define GCK_PLL_TEST_CNTL__TST_CLK_SEL_MODE__SHIFT 0x12 +#define GCK_PLL_TEST_CNTL_2__TEST_COUNT_MASK 0xfffe0000 +#define GCK_PLL_TEST_CNTL_2__TEST_COUNT__SHIFT 0x11 +#define GCK_ADFS_CLK_BYPASS_CNTL1__ECLK_BYPASS_CNTL_MASK 0x7 +#define GCK_ADFS_CLK_BYPASS_CNTL1__ECLK_BYPASS_CNTL__SHIFT 0x0 +#define GCK_ADFS_CLK_BYPASS_CNTL1__SCLK_BYPASS_CNTL_MASK 0x38 +#define GCK_ADFS_CLK_BYPASS_CNTL1__SCLK_BYPASS_CNTL__SHIFT 0x3 +#define GCK_ADFS_CLK_BYPASS_CNTL1__LCLK_BYPASS_CNTL_MASK 0x1c0 +#define GCK_ADFS_CLK_BYPASS_CNTL1__LCLK_BYPASS_CNTL__SHIFT 0x6 +#define GCK_ADFS_CLK_BYPASS_CNTL1__DCLK_BYPASS_CNTL_MASK 0xe00 +#define GCK_ADFS_CLK_BYPASS_CNTL1__DCLK_BYPASS_CNTL__SHIFT 0x9 +#define GCK_ADFS_CLK_BYPASS_CNTL1__VCLK_BYPASS_CNTL_MASK 0x7000 +#define GCK_ADFS_CLK_BYPASS_CNTL1__VCLK_BYPASS_CNTL__SHIFT 0xc +#define GCK_ADFS_CLK_BYPASS_CNTL1__DISPCLK_BYPASS_CNTL_MASK 0x38000 +#define GCK_ADFS_CLK_BYPASS_CNTL1__DISPCLK_BYPASS_CNTL__SHIFT 0xf +#define GCK_ADFS_CLK_BYPASS_CNTL1__DRREFCLK_BYPASS_CNTL_MASK 0x1c0000 +#define GCK_ADFS_CLK_BYPASS_CNTL1__DRREFCLK_BYPASS_CNTL__SHIFT 0x12 +#define GCK_ADFS_CLK_BYPASS_CNTL1__ACLK_BYPASS_CNTL_MASK 0xe00000 +#define GCK_ADFS_CLK_BYPASS_CNTL1__ACLK_BYPASS_CNTL__SHIFT 0x15 +#define GCK_ADFS_CLK_BYPASS_CNTL1__SAMCLK_BYPASS_CNTL_MASK 0x7000000 +#define GCK_ADFS_CLK_BYPASS_CNTL1__SAMCLK_BYPASS_CNTL__SHIFT 0x18 +#define GCK_ADFS_CLK_BYPASS_CNTL1__ACLK_DIV_BYPASS_CNTL_MASK 0x38000000 +#define GCK_ADFS_CLK_BYPASS_CNTL1__ACLK_DIV_BYPASS_CNTL__SHIFT 0x1b +#define SMC_IND_INDEX__SMC_IND_ADDR_MASK 0xffffffff +#define SMC_IND_INDEX__SMC_IND_ADDR__SHIFT 0x0 +#define SMC_IND_DATA__SMC_IND_DATA_MASK 0xffffffff +#define SMC_IND_DATA__SMC_IND_DATA__SHIFT 0x0 +#define SMC_IND_INDEX_0__SMC_IND_ADDR_MASK 0xffffffff +#define SMC_IND_INDEX_0__SMC_IND_ADDR__SHIFT 0x0 +#define SMC_IND_DATA_0__SMC_IND_DATA_MASK 0xffffffff +#define SMC_IND_DATA_0__SMC_IND_DATA__SHIFT 0x0 +#define SMC_IND_INDEX_1__SMC_IND_ADDR_MASK 0xffffffff +#define SMC_IND_INDEX_1__SMC_IND_ADDR__SHIFT 0x0 +#define SMC_IND_DATA_1__SMC_IND_DATA_MASK 0xffffffff +#define SMC_IND_DATA_1__SMC_IND_DATA__SHIFT 0x0 +#define SMC_IND_INDEX_2__SMC_IND_ADDR_MASK 0xffffffff +#define SMC_IND_INDEX_2__SMC_IND_ADDR__SHIFT 0x0 +#define SMC_IND_DATA_2__SMC_IND_DATA_MASK 0xffffffff +#define SMC_IND_DATA_2__SMC_IND_DATA__SHIFT 0x0 +#define SMC_IND_INDEX_3__SMC_IND_ADDR_MASK 0xffffffff +#define SMC_IND_INDEX_3__SMC_IND_ADDR__SHIFT 0x0 +#define SMC_IND_DATA_3__SMC_IND_DATA_MASK 0xffffffff +#define SMC_IND_DATA_3__SMC_IND_DATA__SHIFT 0x0 +#define SMC_IND_INDEX_4__SMC_IND_ADDR_MASK 0xffffffff +#define SMC_IND_INDEX_4__SMC_IND_ADDR__SHIFT 0x0 +#define SMC_IND_DATA_4__SMC_IND_DATA_MASK 0xffffffff +#define SMC_IND_DATA_4__SMC_IND_DATA__SHIFT 0x0 +#define SMC_IND_INDEX_5__SMC_IND_ADDR_MASK 0xffffffff +#define SMC_IND_INDEX_5__SMC_IND_ADDR__SHIFT 0x0 +#define SMC_IND_DATA_5__SMC_IND_DATA_MASK 0xffffffff +#define SMC_IND_DATA_5__SMC_IND_DATA__SHIFT 0x0 +#define SMC_IND_INDEX_6__SMC_IND_ADDR_MASK 0xffffffff +#define SMC_IND_INDEX_6__SMC_IND_ADDR__SHIFT 0x0 +#define SMC_IND_DATA_6__SMC_IND_DATA_MASK 0xffffffff +#define SMC_IND_DATA_6__SMC_IND_DATA__SHIFT 0x0 +#define SMC_IND_INDEX_7__SMC_IND_ADDR_MASK 0xffffffff +#define SMC_IND_INDEX_7__SMC_IND_ADDR__SHIFT 0x0 +#define SMC_IND_DATA_7__SMC_IND_DATA_MASK 0xffffffff +#define SMC_IND_DATA_7__SMC_IND_DATA__SHIFT 0x0 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_0_MASK 0x1 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_0__SHIFT 0x0 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_1_MASK 0x2 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_1__SHIFT 0x1 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_2_MASK 0x4 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_2__SHIFT 0x2 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_3_MASK 0x8 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_3__SHIFT 0x3 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_4_MASK 0x10 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_4__SHIFT 0x4 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_5_MASK 0x20 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_5__SHIFT 0x5 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_6_MASK 0x40 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_6__SHIFT 0x6 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_7_MASK 0x80 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_7__SHIFT 0x7 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_8_MASK 0x100 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_8__SHIFT 0x8 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_9_MASK 0x200 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_9__SHIFT 0x9 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_10_MASK 0x400 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_10__SHIFT 0xa +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_11_MASK 0x800 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_11__SHIFT 0xb +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_12_MASK 0x1000 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_12__SHIFT 0xc +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_13_MASK 0x2000 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_13__SHIFT 0xd +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_14_MASK 0x4000 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_14__SHIFT 0xe +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_15_MASK 0x8000 +#define SMC_IND_ACCESS_CNTL__AUTO_INCREMENT_IND_15__SHIFT 0xf +#define SMC_MESSAGE_0__SMC_MSG_MASK 0xffff +#define SMC_MESSAGE_0__SMC_MSG__SHIFT 0x0 +#define SMC_RESP_0__SMC_RESP_MASK 0xffff +#define SMC_RESP_0__SMC_RESP__SHIFT 0x0 +#define SMC_MESSAGE_1__SMC_MSG_MASK 0xffff +#define SMC_MESSAGE_1__SMC_MSG__SHIFT 0x0 +#define SMC_RESP_1__SMC_RESP_MASK 0xffff +#define SMC_RESP_1__SMC_RESP__SHIFT 0x0 +#define SMC_MESSAGE_2__SMC_MSG_MASK 0xffff +#define SMC_MESSAGE_2__SMC_MSG__SHIFT 0x0 +#define SMC_RESP_2__SMC_RESP_MASK 0xffff +#define SMC_RESP_2__SMC_RESP__SHIFT 0x0 +#define SMC_MESSAGE_3__SMC_MSG_MASK 0xffff +#define SMC_MESSAGE_3__SMC_MSG__SHIFT 0x0 +#define SMC_RESP_3__SMC_RESP_MASK 0xffff +#define SMC_RESP_3__SMC_RESP__SHIFT 0x0 +#define SMC_MESSAGE_4__SMC_MSG_MASK 0xffff +#define SMC_MESSAGE_4__SMC_MSG__SHIFT 0x0 +#define SMC_RESP_4__SMC_RESP_MASK 0xffff +#define SMC_RESP_4__SMC_RESP__SHIFT 0x0 +#define SMC_MESSAGE_5__SMC_MSG_MASK 0xffff +#define SMC_MESSAGE_5__SMC_MSG__SHIFT 0x0 +#define SMC_RESP_5__SMC_RESP_MASK 0xffff +#define SMC_RESP_5__SMC_RESP__SHIFT 0x0 +#define SMC_MESSAGE_6__SMC_MSG_MASK 0xffff +#define SMC_MESSAGE_6__SMC_MSG__SHIFT 0x0 +#define SMC_RESP_6__SMC_RESP_MASK 0xffff +#define SMC_RESP_6__SMC_RESP__SHIFT 0x0 +#define SMC_MESSAGE_7__SMC_MSG_MASK 0xffff +#define SMC_MESSAGE_7__SMC_MSG__SHIFT 0x0 +#define SMC_RESP_7__SMC_RESP_MASK 0xffff +#define SMC_RESP_7__SMC_RESP__SHIFT 0x0 +#define SMC_MSG_ARG_0__SMC_MSG_ARG_MASK 0xffffffff +#define SMC_MSG_ARG_0__SMC_MSG_ARG__SHIFT 0x0 +#define SMC_MSG_ARG_1__SMC_MSG_ARG_MASK 0xffffffff +#define SMC_MSG_ARG_1__SMC_MSG_ARG__SHIFT 0x0 +#define SMC_MSG_ARG_2__SMC_MSG_ARG_MASK 0xffffffff +#define SMC_MSG_ARG_2__SMC_MSG_ARG__SHIFT 0x0 +#define SMC_MSG_ARG_3__SMC_MSG_ARG_MASK 0xffffffff +#define SMC_MSG_ARG_3__SMC_MSG_ARG__SHIFT 0x0 +#define SMC_MSG_ARG_4__SMC_MSG_ARG_MASK 0xffffffff +#define SMC_MSG_ARG_4__SMC_MSG_ARG__SHIFT 0x0 +#define SMC_MSG_ARG_5__SMC_MSG_ARG_MASK 0xffffffff +#define SMC_MSG_ARG_5__SMC_MSG_ARG__SHIFT 0x0 +#define SMC_MSG_ARG_6__SMC_MSG_ARG_MASK 0xffffffff +#define SMC_MSG_ARG_6__SMC_MSG_ARG__SHIFT 0x0 +#define SMC_MSG_ARG_7__SMC_MSG_ARG_MASK 0xffffffff +#define SMC_MSG_ARG_7__SMC_MSG_ARG__SHIFT 0x0 +#define SMC_MESSAGE_8__SMC_MSG_MASK 0xffff +#define SMC_MESSAGE_8__SMC_MSG__SHIFT 0x0 +#define SMC_RESP_8__SMC_RESP_MASK 0xffff +#define SMC_RESP_8__SMC_RESP__SHIFT 0x0 +#define SMC_MESSAGE_9__SMC_MSG_MASK 0xffff +#define SMC_MESSAGE_9__SMC_MSG__SHIFT 0x0 +#define SMC_RESP_9__SMC_RESP_MASK 0xffff +#define SMC_RESP_9__SMC_RESP__SHIFT 0x0 +#define SMC_MESSAGE_10__SMC_MSG_MASK 0xffff +#define SMC_MESSAGE_10__SMC_MSG__SHIFT 0x0 +#define SMC_RESP_10__SMC_RESP_MASK 0xffff +#define SMC_RESP_10__SMC_RESP__SHIFT 0x0 +#define SMC_MESSAGE_11__SMC_MSG_MASK 0xffff +#define SMC_MESSAGE_11__SMC_MSG__SHIFT 0x0 +#define SMC_RESP_11__SMC_RESP_MASK 0xffff +#define SMC_RESP_11__SMC_RESP__SHIFT 0x0 +#define SMC_MSG_ARG_8__SMC_MSG_ARG_MASK 0xffffffff +#define SMC_MSG_ARG_8__SMC_MSG_ARG__SHIFT 0x0 +#define SMC_MSG_ARG_9__SMC_MSG_ARG_MASK 0xffffffff +#define SMC_MSG_ARG_9__SMC_MSG_ARG__SHIFT 0x0 +#define SMC_MSG_ARG_10__SMC_MSG_ARG_MASK 0xffffffff +#define SMC_MSG_ARG_10__SMC_MSG_ARG__SHIFT 0x0 +#define SMC_MSG_ARG_11__SMC_MSG_ARG_MASK 0xffffffff +#define SMC_MSG_ARG_11__SMC_MSG_ARG__SHIFT 0x0 +#define SMC_SYSCON_RESET_CNTL__rst_reg_MASK 0x1 +#define SMC_SYSCON_RESET_CNTL__rst_reg__SHIFT 0x0 +#define SMC_SYSCON_RESET_CNTL__srbm_soft_rst_override_MASK 0x2 +#define SMC_SYSCON_RESET_CNTL__srbm_soft_rst_override__SHIFT 0x1 +#define SMC_SYSCON_RESET_CNTL__RegReset_MASK 0x40000000 +#define SMC_SYSCON_RESET_CNTL__RegReset__SHIFT 0x1e +#define SMC_SYSCON_CLOCK_CNTL_0__ck_disable_MASK 0x1 +#define SMC_SYSCON_CLOCK_CNTL_0__ck_disable__SHIFT 0x0 +#define SMC_SYSCON_CLOCK_CNTL_0__auto_cg_en_MASK 0x2 +#define SMC_SYSCON_CLOCK_CNTL_0__auto_cg_en__SHIFT 0x1 +#define SMC_SYSCON_CLOCK_CNTL_0__auto_cg_timeout_MASK 0xffff00 +#define SMC_SYSCON_CLOCK_CNTL_0__auto_cg_timeout__SHIFT 0x8 +#define SMC_SYSCON_CLOCK_CNTL_0__cken_MASK 0x1000000 +#define SMC_SYSCON_CLOCK_CNTL_0__cken__SHIFT 0x18 +#define SMC_SYSCON_CLOCK_CNTL_1__auto_ck_disable_MASK 0x1 +#define SMC_SYSCON_CLOCK_CNTL_1__auto_ck_disable__SHIFT 0x0 +#define SMC_SYSCON_CLOCK_CNTL_2__wake_on_irq_MASK 0xffffffff +#define SMC_SYSCON_CLOCK_CNTL_2__wake_on_irq__SHIFT 0x0 +#define SMC_SYSCON_MISC_CNTL__dma_no_outstanding_MASK 0x2 +#define SMC_SYSCON_MISC_CNTL__dma_no_outstanding__SHIFT 0x1 +#define SMC_SYSCON_MSG_ARG_0__smc_msg_arg_MASK 0xffffffff +#define SMC_SYSCON_MSG_ARG_0__smc_msg_arg__SHIFT 0x0 +#define SMC_PC_C__smc_pc_c_MASK 0xffffffff +#define SMC_PC_C__smc_pc_c__SHIFT 0x0 +#define SMC_SCRATCH9__SCRATCH_VALUE_MASK 0xffffffff +#define SMC_SCRATCH9__SCRATCH_VALUE__SHIFT 0x0 +#define GPIOPAD_SW_INT_STAT__SW_INT_STAT_MASK 0x1 +#define GPIOPAD_SW_INT_STAT__SW_INT_STAT__SHIFT 0x0 +#define GPIOPAD_STRENGTH__GPIO_STRENGTH_SN_MASK 0xf +#define GPIOPAD_STRENGTH__GPIO_STRENGTH_SN__SHIFT 0x0 +#define GPIOPAD_STRENGTH__GPIO_STRENGTH_SP_MASK 0xf0 +#define GPIOPAD_STRENGTH__GPIO_STRENGTH_SP__SHIFT 0x4 +#define GPIOPAD_MASK__GPIO_MASK_MASK 0x7fffffff +#define GPIOPAD_MASK__GPIO_MASK__SHIFT 0x0 +#define GPIOPAD_A__GPIO_A_MASK 0x7fffffff +#define GPIOPAD_A__GPIO_A__SHIFT 0x0 +#define GPIOPAD_EN__GPIO_EN_MASK 0x7fffffff +#define GPIOPAD_EN__GPIO_EN__SHIFT 0x0 +#define GPIOPAD_Y__GPIO_Y_MASK 0x7fffffff +#define GPIOPAD_Y__GPIO_Y__SHIFT 0x0 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_0_MASK 0x1 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_0__SHIFT 0x0 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_1_MASK 0x2 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_1__SHIFT 0x1 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_2_MASK 0x4 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_2__SHIFT 0x2 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_3_MASK 0x8 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_3__SHIFT 0x3 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_4_MASK 0x10 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_4__SHIFT 0x4 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_5_MASK 0x20 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_5__SHIFT 0x5 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_6_MASK 0x40 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_6__SHIFT 0x6 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_7_MASK 0x80 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_7__SHIFT 0x7 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_8_MASK 0x100 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_8__SHIFT 0x8 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_9_MASK 0x200 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_9__SHIFT 0x9 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_10_MASK 0x400 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_10__SHIFT 0xa +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_11_MASK 0x800 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_11__SHIFT 0xb +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_12_MASK 0x1000 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_12__SHIFT 0xc +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_13_MASK 0x2000 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_13__SHIFT 0xd +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_14_MASK 0x4000 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_14__SHIFT 0xe +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_15_MASK 0x8000 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_15__SHIFT 0xf +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_16_MASK 0x10000 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_16__SHIFT 0x10 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_17_MASK 0x20000 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_17__SHIFT 0x11 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_18_MASK 0x40000 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_18__SHIFT 0x12 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_19_MASK 0x80000 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_19__SHIFT 0x13 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_20_MASK 0x100000 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_20__SHIFT 0x14 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_21_MASK 0x200000 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_21__SHIFT 0x15 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_22_MASK 0x400000 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_22__SHIFT 0x16 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_23_MASK 0x800000 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_23__SHIFT 0x17 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_24_MASK 0x1000000 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_24__SHIFT 0x18 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_25_MASK 0x2000000 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_25__SHIFT 0x19 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_26_MASK 0x4000000 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_26__SHIFT 0x1a +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_27_MASK 0x8000000 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_27__SHIFT 0x1b +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_28_MASK 0x10000000 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_28__SHIFT 0x1c +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_29_MASK 0x20000000 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_29__SHIFT 0x1d +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_30_MASK 0x40000000 +#define GPIOPAD_PINSTRAPS__GPIO_PINSTRAP_30__SHIFT 0x1e +#define GPIOPAD_INT_STAT_EN__GPIO_INT_STAT_EN_MASK 0x1fffffff +#define GPIOPAD_INT_STAT_EN__GPIO_INT_STAT_EN__SHIFT 0x0 +#define GPIOPAD_INT_STAT_EN__SW_INITIATED_INT_STAT_EN_MASK 0x80000000 +#define GPIOPAD_INT_STAT_EN__SW_INITIATED_INT_STAT_EN__SHIFT 0x1f +#define GPIOPAD_INT_STAT__GPIO_INT_STAT_MASK 0x1fffffff +#define GPIOPAD_INT_STAT__GPIO_INT_STAT__SHIFT 0x0 +#define GPIOPAD_INT_STAT__SW_INITIATED_INT_STAT_MASK 0x80000000 +#define GPIOPAD_INT_STAT__SW_INITIATED_INT_STAT__SHIFT 0x1f +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_0_MASK 0x1 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_0__SHIFT 0x0 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_1_MASK 0x2 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_1__SHIFT 0x1 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_2_MASK 0x4 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_2__SHIFT 0x2 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_3_MASK 0x8 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_3__SHIFT 0x3 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_4_MASK 0x10 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_4__SHIFT 0x4 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_5_MASK 0x20 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_5__SHIFT 0x5 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_6_MASK 0x40 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_6__SHIFT 0x6 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_7_MASK 0x80 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_7__SHIFT 0x7 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_8_MASK 0x100 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_8__SHIFT 0x8 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_9_MASK 0x200 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_9__SHIFT 0x9 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_10_MASK 0x400 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_10__SHIFT 0xa +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_11_MASK 0x800 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_11__SHIFT 0xb +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_12_MASK 0x1000 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_12__SHIFT 0xc +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_13_MASK 0x2000 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_13__SHIFT 0xd +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_14_MASK 0x4000 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_14__SHIFT 0xe +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_15_MASK 0x8000 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_15__SHIFT 0xf +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_16_MASK 0x10000 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_16__SHIFT 0x10 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_17_MASK 0x20000 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_17__SHIFT 0x11 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_18_MASK 0x40000 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_18__SHIFT 0x12 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_19_MASK 0x80000 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_19__SHIFT 0x13 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_20_MASK 0x100000 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_20__SHIFT 0x14 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_21_MASK 0x200000 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_21__SHIFT 0x15 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_22_MASK 0x400000 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_22__SHIFT 0x16 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_23_MASK 0x800000 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_23__SHIFT 0x17 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_24_MASK 0x1000000 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_24__SHIFT 0x18 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_25_MASK 0x2000000 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_25__SHIFT 0x19 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_26_MASK 0x4000000 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_26__SHIFT 0x1a +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_27_MASK 0x8000000 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_27__SHIFT 0x1b +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_28_MASK 0x10000000 +#define GPIOPAD_INT_STAT_AK__GPIO_INT_STAT_AK_28__SHIFT 0x1c +#define GPIOPAD_INT_STAT_AK__SW_INITIATED_INT_STAT_AK_MASK 0x80000000 +#define GPIOPAD_INT_STAT_AK__SW_INITIATED_INT_STAT_AK__SHIFT 0x1f +#define GPIOPAD_INT_EN__GPIO_INT_EN_MASK 0x1fffffff +#define GPIOPAD_INT_EN__GPIO_INT_EN__SHIFT 0x0 +#define GPIOPAD_INT_EN__SW_INITIATED_INT_EN_MASK 0x80000000 +#define GPIOPAD_INT_EN__SW_INITIATED_INT_EN__SHIFT 0x1f +#define GPIOPAD_INT_TYPE__GPIO_INT_TYPE_MASK 0x1fffffff +#define GPIOPAD_INT_TYPE__GPIO_INT_TYPE__SHIFT 0x0 +#define GPIOPAD_INT_TYPE__SW_INITIATED_INT_TYPE_MASK 0x80000000 +#define GPIOPAD_INT_TYPE__SW_INITIATED_INT_TYPE__SHIFT 0x1f +#define GPIOPAD_INT_POLARITY__GPIO_INT_POLARITY_MASK 0x1fffffff +#define GPIOPAD_INT_POLARITY__GPIO_INT_POLARITY__SHIFT 0x0 +#define GPIOPAD_INT_POLARITY__SW_INITIATED_INT_POLARITY_MASK 0x80000000 +#define GPIOPAD_INT_POLARITY__SW_INITIATED_INT_POLARITY__SHIFT 0x1f +#define GPIOPAD_EXTERN_TRIG_CNTL__EXTERN_TRIG_SEL_MASK 0x1f +#define GPIOPAD_EXTERN_TRIG_CNTL__EXTERN_TRIG_SEL__SHIFT 0x0 +#define GPIOPAD_EXTERN_TRIG_CNTL__EXTERN_TRIG_CLR_MASK 0x20 +#define GPIOPAD_EXTERN_TRIG_CNTL__EXTERN_TRIG_CLR__SHIFT 0x5 +#define GPIOPAD_EXTERN_TRIG_CNTL__EXTERN_TRIG_READ_MASK 0x40 +#define GPIOPAD_EXTERN_TRIG_CNTL__EXTERN_TRIG_READ__SHIFT 0x6 +#define GPIOPAD_RCVR_SEL__GPIO_RCVR_SEL_MASK 0x7fffffff +#define GPIOPAD_RCVR_SEL__GPIO_RCVR_SEL__SHIFT 0x0 +#define GPIOPAD_PU_EN__GPIO_PU_EN_MASK 0x7fffffff +#define GPIOPAD_PU_EN__GPIO_PU_EN__SHIFT 0x0 +#define GPIOPAD_PD_EN__GPIO_PD_EN_MASK 0x7fffffff +#define GPIOPAD_PD_EN__GPIO_PD_EN__SHIFT 0x0 +#define CG_FPS_CNT__FPS_CNT_MASK 0xffffffff +#define CG_FPS_CNT__FPS_CNT__SHIFT 0x0 +#define SMU_IND_INDEX_0__SMC_IND_ADDR_MASK 0xffffffff +#define SMU_IND_INDEX_0__SMC_IND_ADDR__SHIFT 0x0 +#define SMU_IND_DATA_0__SMC_IND_DATA_MASK 0xffffffff +#define SMU_IND_DATA_0__SMC_IND_DATA__SHIFT 0x0 +#define SMU_IND_INDEX_1__SMC_IND_ADDR_MASK 0xffffffff +#define SMU_IND_INDEX_1__SMC_IND_ADDR__SHIFT 0x0 +#define SMU_IND_DATA_1__SMC_IND_DATA_MASK 0xffffffff +#define SMU_IND_DATA_1__SMC_IND_DATA__SHIFT 0x0 +#define SMU_IND_INDEX_2__SMC_IND_ADDR_MASK 0xffffffff +#define SMU_IND_INDEX_2__SMC_IND_ADDR__SHIFT 0x0 +#define SMU_IND_DATA_2__SMC_IND_DATA_MASK 0xffffffff +#define SMU_IND_DATA_2__SMC_IND_DATA__SHIFT 0x0 +#define SMU_IND_INDEX_3__SMC_IND_ADDR_MASK 0xffffffff +#define SMU_IND_INDEX_3__SMC_IND_ADDR__SHIFT 0x0 +#define SMU_IND_DATA_3__SMC_IND_DATA_MASK 0xffffffff +#define SMU_IND_DATA_3__SMC_IND_DATA__SHIFT 0x0 +#define SMU_IND_INDEX_4__SMC_IND_ADDR_MASK 0xffffffff +#define SMU_IND_INDEX_4__SMC_IND_ADDR__SHIFT 0x0 +#define SMU_IND_DATA_4__SMC_IND_DATA_MASK 0xffffffff +#define SMU_IND_DATA_4__SMC_IND_DATA__SHIFT 0x0 +#define SMU_IND_INDEX_5__SMC_IND_ADDR_MASK 0xffffffff +#define SMU_IND_INDEX_5__SMC_IND_ADDR__SHIFT 0x0 +#define SMU_IND_DATA_5__SMC_IND_DATA_MASK 0xffffffff +#define SMU_IND_DATA_5__SMC_IND_DATA__SHIFT 0x0 +#define SMU_IND_INDEX_6__SMC_IND_ADDR_MASK 0xffffffff +#define SMU_IND_INDEX_6__SMC_IND_ADDR__SHIFT 0x0 +#define SMU_IND_DATA_6__SMC_IND_DATA_MASK 0xffffffff +#define SMU_IND_DATA_6__SMC_IND_DATA__SHIFT 0x0 +#define SMU_IND_INDEX_7__SMC_IND_ADDR_MASK 0xffffffff +#define SMU_IND_INDEX_7__SMC_IND_ADDR__SHIFT 0x0 +#define SMU_IND_DATA_7__SMC_IND_DATA_MASK 0xffffffff +#define SMU_IND_DATA_7__SMC_IND_DATA__SHIFT 0x0 +#define SMU_SMC_IND_INDEX__SMC_IND_ADDR_MASK 0xffffffff +#define SMU_SMC_IND_INDEX__SMC_IND_ADDR__SHIFT 0x0 +#define SMU_SMC_IND_DATA__SMC_IND_DATA_MASK 0xffffffff +#define SMU_SMC_IND_DATA__SMC_IND_DATA__SHIFT 0x0 +#define RCU_UC_EVENTS__RCU_TST_jpc_rep_req_MASK 0x1 +#define RCU_UC_EVENTS__RCU_TST_jpc_rep_req__SHIFT 0x0 +#define RCU_UC_EVENTS__TST_RCU_jpc_rep_done_MASK 0x2 +#define RCU_UC_EVENTS__TST_RCU_jpc_rep_done__SHIFT 0x1 +#define RCU_UC_EVENTS__drv_rst_mode_MASK 0x4 +#define RCU_UC_EVENTS__drv_rst_mode__SHIFT 0x2 +#define RCU_UC_EVENTS__SMU_DC_efuse_status_invalid_MASK 0x8 +#define RCU_UC_EVENTS__SMU_DC_efuse_status_invalid__SHIFT 0x3 +#define RCU_UC_EVENTS__TP_Tester_MASK 0x40 +#define RCU_UC_EVENTS__TP_Tester__SHIFT 0x6 +#define RCU_UC_EVENTS__boot_seq_done_MASK 0x80 +#define RCU_UC_EVENTS__boot_seq_done__SHIFT 0x7 +#define RCU_UC_EVENTS__sclk_deep_sleep_exit_MASK 0x100 +#define RCU_UC_EVENTS__sclk_deep_sleep_exit__SHIFT 0x8 +#define RCU_UC_EVENTS__BREAK_PT1_ACTIVE_MASK 0x200 +#define RCU_UC_EVENTS__BREAK_PT1_ACTIVE__SHIFT 0x9 +#define RCU_UC_EVENTS__BREAK_PT2_ACTIVE_MASK 0x400 +#define RCU_UC_EVENTS__BREAK_PT2_ACTIVE__SHIFT 0xa +#define RCU_UC_EVENTS__FCH_HALT_MASK 0x800 +#define RCU_UC_EVENTS__FCH_HALT__SHIFT 0xb +#define RCU_UC_EVENTS__RCU_GIO_fch_lockdown_MASK 0x2000 +#define RCU_UC_EVENTS__RCU_GIO_fch_lockdown__SHIFT 0xd +#define RCU_UC_EVENTS__INTERRUPTS_ENABLED_MASK 0x10000 +#define RCU_UC_EVENTS__INTERRUPTS_ENABLED__SHIFT 0x10 +#define RCU_UC_EVENTS__RCU_DtmCnt0_Done_MASK 0x20000 +#define RCU_UC_EVENTS__RCU_DtmCnt0_Done__SHIFT 0x11 +#define RCU_UC_EVENTS__RCU_DtmCnt1_Done_MASK 0x40000 +#define RCU_UC_EVENTS__RCU_DtmCnt1_Done__SHIFT 0x12 +#define RCU_UC_EVENTS__RCU_DtmCnt2_Done_MASK 0x80000 +#define RCU_UC_EVENTS__RCU_DtmCnt2_Done__SHIFT 0x13 +#define RCU_UC_EVENTS__irq31_sel_MASK 0x3000000 +#define RCU_UC_EVENTS__irq31_sel__SHIFT 0x18 +#define RCU_MISC_CTRL__REG_DRV_RST_MODE_MASK 0x2 +#define RCU_MISC_CTRL__REG_DRV_RST_MODE__SHIFT 0x1 +#define RCU_MISC_CTRL__REG_RCU_MEMREP_DIS_MASK 0x8 +#define RCU_MISC_CTRL__REG_RCU_MEMREP_DIS__SHIFT 0x3 +#define RCU_MISC_CTRL__REG_CC_FUSE_DISABLE_MASK 0x10 +#define RCU_MISC_CTRL__REG_CC_FUSE_DISABLE__SHIFT 0x4 +#define RCU_MISC_CTRL__REG_SAMU_FUSE_DISABLE_MASK 0x20 +#define RCU_MISC_CTRL__REG_SAMU_FUSE_DISABLE__SHIFT 0x5 +#define RCU_MISC_CTRL__REG_CC_SRBM_RD_DISABLE_MASK 0x100 +#define RCU_MISC_CTRL__REG_CC_SRBM_RD_DISABLE__SHIFT 0x8 +#define RCU_MISC_CTRL__BREAK_PT1_DONE_MASK 0x10000 +#define RCU_MISC_CTRL__BREAK_PT1_DONE__SHIFT 0x10 +#define RCU_MISC_CTRL__BREAK_PT2_DONE_MASK 0x20000 +#define RCU_MISC_CTRL__BREAK_PT2_DONE__SHIFT 0x11 +#define RCU_MISC_CTRL__SAMU_START_MASK 0x400000 +#define RCU_MISC_CTRL__SAMU_START__SHIFT 0x16 +#define RCU_MISC_CTRL__RST_PULSE_WIDTH_MASK 0xff800000 +#define RCU_MISC_CTRL__RST_PULSE_WIDTH__SHIFT 0x17 +#define RCU_VIRT_RESET_REQ__VF_MASK 0xffff +#define RCU_VIRT_RESET_REQ__VF__SHIFT 0x0 +#define RCU_VIRT_RESET_REQ__PF_MASK 0x80000000 +#define RCU_VIRT_RESET_REQ__PF__SHIFT 0x1f +#define CC_RCU_FUSES__GPU_DIS_MASK 0x2 +#define CC_RCU_FUSES__GPU_DIS__SHIFT 0x1 +#define CC_RCU_FUSES__DEBUG_DISABLE_MASK 0x4 +#define CC_RCU_FUSES__DEBUG_DISABLE__SHIFT 0x2 +#define CC_RCU_FUSES__EFUSE_RD_DISABLE_MASK 0x10 +#define CC_RCU_FUSES__EFUSE_RD_DISABLE__SHIFT 0x4 +#define CC_RCU_FUSES__CG_RST_GLB_REQ_DIS_MASK 0x20 +#define CC_RCU_FUSES__CG_RST_GLB_REQ_DIS__SHIFT 0x5 +#define CC_RCU_FUSES__DRV_RST_MODE_MASK 0x40 +#define CC_RCU_FUSES__DRV_RST_MODE__SHIFT 0x6 +#define CC_RCU_FUSES__ROM_DIS_MASK 0x80 +#define CC_RCU_FUSES__ROM_DIS__SHIFT 0x7 +#define CC_RCU_FUSES__JPC_REP_DISABLE_MASK 0x100 +#define CC_RCU_FUSES__JPC_REP_DISABLE__SHIFT 0x8 +#define CC_RCU_FUSES__RCU_BREAK_POINT1_MASK 0x200 +#define CC_RCU_FUSES__RCU_BREAK_POINT1__SHIFT 0x9 +#define CC_RCU_FUSES__RCU_BREAK_POINT2_MASK 0x400 +#define CC_RCU_FUSES__RCU_BREAK_POINT2__SHIFT 0xa +#define CC_RCU_FUSES__SMU_IOC_MST_DISABLE_MASK 0x4000 +#define CC_RCU_FUSES__SMU_IOC_MST_DISABLE__SHIFT 0xe +#define CC_RCU_FUSES__FCH_LOCKOUT_ENABLE_MASK 0x8000 +#define CC_RCU_FUSES__FCH_LOCKOUT_ENABLE__SHIFT 0xf +#define CC_RCU_FUSES__FCH_XFIRE_FILTER_ENABLE_MASK 0x10000 +#define CC_RCU_FUSES__FCH_XFIRE_FILTER_ENABLE__SHIFT 0x10 +#define CC_RCU_FUSES__XFIRE_DISABLE_MASK 0x20000 +#define CC_RCU_FUSES__XFIRE_DISABLE__SHIFT 0x11 +#define CC_RCU_FUSES__SAMU_FUSE_DISABLE_MASK 0x40000 +#define CC_RCU_FUSES__SAMU_FUSE_DISABLE__SHIFT 0x12 +#define CC_RCU_FUSES__BIF_RST_POLLING_DISABLE_MASK 0x80000 +#define CC_RCU_FUSES__BIF_RST_POLLING_DISABLE__SHIFT 0x13 +#define CC_RCU_FUSES__MEM_HARDREP_EN_MASK 0x200000 +#define CC_RCU_FUSES__MEM_HARDREP_EN__SHIFT 0x15 +#define CC_RCU_FUSES__PCIE_INIT_DISABLE_MASK 0x400000 +#define CC_RCU_FUSES__PCIE_INIT_DISABLE__SHIFT 0x16 +#define CC_RCU_FUSES__DSMU_DISABLE_MASK 0x800000 +#define CC_RCU_FUSES__DSMU_DISABLE__SHIFT 0x17 +#define CC_RCU_FUSES__WRP_FUSE_VALID_MASK 0x1000000 +#define CC_RCU_FUSES__WRP_FUSE_VALID__SHIFT 0x18 +#define CC_RCU_FUSES__PHY_FUSE_VALID_MASK 0x2000000 +#define CC_RCU_FUSES__PHY_FUSE_VALID__SHIFT 0x19 +#define CC_RCU_FUSES__RCU_SPARE_MASK 0xfc000000 +#define CC_RCU_FUSES__RCU_SPARE__SHIFT 0x1a +#define CC_SMU_MISC_FUSES__IOMMU_V2_DISABLE_MASK 0x2 +#define CC_SMU_MISC_FUSES__IOMMU_V2_DISABLE__SHIFT 0x1 +#define CC_SMU_MISC_FUSES__MinSClkDid_MASK 0x1fc +#define CC_SMU_MISC_FUSES__MinSClkDid__SHIFT 0x2 +#define CC_SMU_MISC_FUSES__MISC_SPARE_MASK 0x600 +#define CC_SMU_MISC_FUSES__MISC_SPARE__SHIFT 0x9 +#define CC_SMU_MISC_FUSES__PostResetGnbClkDid_MASK 0x3f800 +#define CC_SMU_MISC_FUSES__PostResetGnbClkDid__SHIFT 0xb +#define CC_SMU_MISC_FUSES__L2IMU_tn2_dtc_half_MASK 0x40000 +#define CC_SMU_MISC_FUSES__L2IMU_tn2_dtc_half__SHIFT 0x12 +#define CC_SMU_MISC_FUSES__L2IMU_tn2_ptc_half_MASK 0x80000 +#define CC_SMU_MISC_FUSES__L2IMU_tn2_ptc_half__SHIFT 0x13 +#define CC_SMU_MISC_FUSES__L2IMU_tn2_itc_half_MASK 0x100000 +#define CC_SMU_MISC_FUSES__L2IMU_tn2_itc_half__SHIFT 0x14 +#define CC_SMU_MISC_FUSES__L2IMU_tn2_pdc_half_MASK 0x200000 +#define CC_SMU_MISC_FUSES__L2IMU_tn2_pdc_half__SHIFT 0x15 +#define CC_SMU_MISC_FUSES__L2IMU_tn2_ptc_dis_MASK 0x400000 +#define CC_SMU_MISC_FUSES__L2IMU_tn2_ptc_dis__SHIFT 0x16 +#define CC_SMU_MISC_FUSES__L2IMU_tn2_itc_dis_MASK 0x800000 +#define CC_SMU_MISC_FUSES__L2IMU_tn2_itc_dis__SHIFT 0x17 +#define CC_SMU_MISC_FUSES__VCE_DISABLE_MASK 0x8000000 +#define CC_SMU_MISC_FUSES__VCE_DISABLE__SHIFT 0x1b +#define CC_SMU_MISC_FUSES__IOC_IOMMU_DISABLE_MASK 0x10000000 +#define CC_SMU_MISC_FUSES__IOC_IOMMU_DISABLE__SHIFT 0x1c +#define CC_SMU_MISC_FUSES__GNB_SPARE_MASK 0x60000000 +#define CC_SMU_MISC_FUSES__GNB_SPARE__SHIFT 0x1d +#define CC_SCLK_VID_FUSES__SClkVid0_MASK 0xff +#define CC_SCLK_VID_FUSES__SClkVid0__SHIFT 0x0 +#define CC_SCLK_VID_FUSES__SClkVid1_MASK 0xff00 +#define CC_SCLK_VID_FUSES__SClkVid1__SHIFT 0x8 +#define CC_SCLK_VID_FUSES__SClkVid2_MASK 0xff0000 +#define CC_SCLK_VID_FUSES__SClkVid2__SHIFT 0x10 +#define CC_SCLK_VID_FUSES__SClkVid3_MASK 0xff000000 +#define CC_SCLK_VID_FUSES__SClkVid3__SHIFT 0x18 +#define CC_GIO_IOCCFG_FUSES__NB_REV_ID_MASK 0x7fe +#define CC_GIO_IOCCFG_FUSES__NB_REV_ID__SHIFT 0x1 +#define CC_GIO_IOC_FUSES__IOC_FUSES_MASK 0x3e +#define CC_GIO_IOC_FUSES__IOC_FUSES__SHIFT 0x1 +#define CC_SMU_TST_EFUSE1_MISC__RF_RM_6_2_MASK 0x3e +#define CC_SMU_TST_EFUSE1_MISC__RF_RM_6_2__SHIFT 0x1 +#define CC_SMU_TST_EFUSE1_MISC__RME_MASK 0x40 +#define CC_SMU_TST_EFUSE1_MISC__RME__SHIFT 0x6 +#define CC_SMU_TST_EFUSE1_MISC__MBIST_DISABLE_MASK 0x80 +#define CC_SMU_TST_EFUSE1_MISC__MBIST_DISABLE__SHIFT 0x7 +#define CC_SMU_TST_EFUSE1_MISC__HARD_REPAIR_DISABLE_MASK 0x100 +#define CC_SMU_TST_EFUSE1_MISC__HARD_REPAIR_DISABLE__SHIFT 0x8 +#define CC_SMU_TST_EFUSE1_MISC__SOFT_REPAIR_DISABLE_MASK 0x200 +#define CC_SMU_TST_EFUSE1_MISC__SOFT_REPAIR_DISABLE__SHIFT 0x9 +#define CC_SMU_TST_EFUSE1_MISC__GPU_DIS_MASK 0x400 +#define CC_SMU_TST_EFUSE1_MISC__GPU_DIS__SHIFT 0xa +#define CC_SMU_TST_EFUSE1_MISC__SMS_PWRDWN_DISABLE_MASK 0x800 +#define CC_SMU_TST_EFUSE1_MISC__SMS_PWRDWN_DISABLE__SHIFT 0xb +#define CC_SMU_TST_EFUSE1_MISC__CRBBMP1500_DISA_MASK 0x1000 +#define CC_SMU_TST_EFUSE1_MISC__CRBBMP1500_DISA__SHIFT 0xc +#define CC_SMU_TST_EFUSE1_MISC__CRBBMP1500_DISB_MASK 0x2000 +#define CC_SMU_TST_EFUSE1_MISC__CRBBMP1500_DISB__SHIFT 0xd +#define CC_SMU_TST_EFUSE1_MISC__RM_RF8_MASK 0x4000 +#define CC_SMU_TST_EFUSE1_MISC__RM_RF8__SHIFT 0xe +#define CC_SMU_TST_EFUSE1_MISC__DFT_SPARE1_MASK 0x400000 +#define CC_SMU_TST_EFUSE1_MISC__DFT_SPARE1__SHIFT 0x16 +#define CC_SMU_TST_EFUSE1_MISC__DFT_SPARE2_MASK 0x800000 +#define CC_SMU_TST_EFUSE1_MISC__DFT_SPARE2__SHIFT 0x17 +#define CC_SMU_TST_EFUSE1_MISC__DFT_SPARE3_MASK 0x1000000 +#define CC_SMU_TST_EFUSE1_MISC__DFT_SPARE3__SHIFT 0x18 +#define CC_SMU_TST_EFUSE1_MISC__VCE_DISABLE_MASK 0x2000000 +#define CC_SMU_TST_EFUSE1_MISC__VCE_DISABLE__SHIFT 0x19 +#define CC_SMU_TST_EFUSE1_MISC__DCE_SCAN_DISABLE_MASK 0x4000000 +#define CC_SMU_TST_EFUSE1_MISC__DCE_SCAN_DISABLE__SHIFT 0x1a +#define CC_TST_ID_STRAPS__DEVICE_ID_MASK 0xffff0 +#define CC_TST_ID_STRAPS__DEVICE_ID__SHIFT 0x4 +#define CC_TST_ID_STRAPS__MAJOR_REV_ID_MASK 0xf00000 +#define CC_TST_ID_STRAPS__MAJOR_REV_ID__SHIFT 0x14 +#define CC_TST_ID_STRAPS__MINOR_REV_ID_MASK 0xf000000 +#define CC_TST_ID_STRAPS__MINOR_REV_ID__SHIFT 0x18 +#define CC_TST_ID_STRAPS__ATI_REV_ID_MASK 0xf0000000 +#define CC_TST_ID_STRAPS__ATI_REV_ID__SHIFT 0x1c +#define CC_FCTRL_FUSES__EXT_EFUSE_MACRO_PRESENT_MASK 0x2 +#define CC_FCTRL_FUSES__EXT_EFUSE_MACRO_PRESENT__SHIFT 0x1 +#define CC_HARVEST_FUSES__VCE_DISABLE_MASK 0x6 +#define CC_HARVEST_FUSES__VCE_DISABLE__SHIFT 0x1 +#define CC_HARVEST_FUSES__UVD_DISABLE_MASK 0x10 +#define CC_HARVEST_FUSES__UVD_DISABLE__SHIFT 0x4 +#define CC_HARVEST_FUSES__ACP_DISABLE_MASK 0x40 +#define CC_HARVEST_FUSES__ACP_DISABLE__SHIFT 0x6 +#define CC_HARVEST_FUSES__DC_DISABLE_MASK 0x3f00 +#define CC_HARVEST_FUSES__DC_DISABLE__SHIFT 0x8 +#define SMU_MAIN_PLL_OP_FREQ__PLL_OP_FREQ_MASK 0xffffffff +#define SMU_MAIN_PLL_OP_FREQ__PLL_OP_FREQ__SHIFT 0x0 +#define SMU_STATUS__SMU_DONE_MASK 0x1 +#define SMU_STATUS__SMU_DONE__SHIFT 0x0 +#define SMU_STATUS__SMU_PASS_MASK 0x2 +#define SMU_STATUS__SMU_PASS__SHIFT 0x1 +#define SMU_FIRMWARE__SMU_IN_PROG_MASK 0x1 +#define SMU_FIRMWARE__SMU_IN_PROG__SHIFT 0x0 +#define SMU_FIRMWARE__SMU_RD_DONE_MASK 0x6 +#define SMU_FIRMWARE__SMU_RD_DONE__SHIFT 0x1 +#define SMU_FIRMWARE__SMU_SRAM_RD_BLOCK_EN_MASK 0x8 +#define SMU_FIRMWARE__SMU_SRAM_RD_BLOCK_EN__SHIFT 0x3 +#define SMU_FIRMWARE__SMU_SRAM_WR_BLOCK_EN_MASK 0x10 +#define SMU_FIRMWARE__SMU_SRAM_WR_BLOCK_EN__SHIFT 0x4 +#define SMU_FIRMWARE__SMU_counter_MASK 0xf00 +#define SMU_FIRMWARE__SMU_counter__SHIFT 0x8 +#define SMU_FIRMWARE__SMU_MODE_MASK 0x10000 +#define SMU_FIRMWARE__SMU_MODE__SHIFT 0x10 +#define SMU_FIRMWARE__SMU_SEL_MASK 0x20000 +#define SMU_FIRMWARE__SMU_SEL__SHIFT 0x11 +#define SMU_INPUT_DATA__START_ADDR_MASK 0x7fffffff +#define SMU_INPUT_DATA__START_ADDR__SHIFT 0x0 +#define SMU_INPUT_DATA__AUTO_START_MASK 0x80000000 +#define SMU_INPUT_DATA__AUTO_START__SHIFT 0x1f +#define SMU_EFUSE_0__EFUSE_DATA_MASK 0xffffffff +#define SMU_EFUSE_0__EFUSE_DATA__SHIFT 0x0 +#define FIRMWARE_FLAGS__INTERRUPTS_ENABLED_MASK 0x1 +#define FIRMWARE_FLAGS__INTERRUPTS_ENABLED__SHIFT 0x0 +#define FIRMWARE_FLAGS__RESERVED_MASK 0xfffffe +#define FIRMWARE_FLAGS__RESERVED__SHIFT 0x1 +#define FIRMWARE_FLAGS__TEST_COUNT_MASK 0xff000000 +#define FIRMWARE_FLAGS__TEST_COUNT__SHIFT 0x18 +#define TDC_STATUS__VDD_Boost_MASK 0xff +#define TDC_STATUS__VDD_Boost__SHIFT 0x0 +#define TDC_STATUS__VDD_Throttle_MASK 0xff00 +#define TDC_STATUS__VDD_Throttle__SHIFT 0x8 +#define TDC_STATUS__VDDC_Boost_MASK 0xff0000 +#define TDC_STATUS__VDDC_Boost__SHIFT 0x10 +#define TDC_STATUS__VDDC_Throttle_MASK 0xff000000 +#define TDC_STATUS__VDDC_Throttle__SHIFT 0x18 +#define TDC_MV_AVERAGE__IDD_MASK 0xffff +#define TDC_MV_AVERAGE__IDD__SHIFT 0x0 +#define TDC_MV_AVERAGE__IDDC_MASK 0xffff0000 +#define TDC_MV_AVERAGE__IDDC__SHIFT 0x10 +#define TDC_VRM_LIMIT__IDD_MASK 0xffff +#define TDC_VRM_LIMIT__IDD__SHIFT 0x0 +#define TDC_VRM_LIMIT__IDDC_MASK 0xffff0000 +#define TDC_VRM_LIMIT__IDDC__SHIFT 0x10 +#define FEATURE_STATUS__SCLK_DPM_ON_MASK 0x1 +#define FEATURE_STATUS__SCLK_DPM_ON__SHIFT 0x0 +#define FEATURE_STATUS__MCLK_DPM_ON_MASK 0x2 +#define FEATURE_STATUS__MCLK_DPM_ON__SHIFT 0x1 +#define FEATURE_STATUS__LCLK_DPM_ON_MASK 0x4 +#define FEATURE_STATUS__LCLK_DPM_ON__SHIFT 0x2 +#define FEATURE_STATUS__UVD_DPM_ON_MASK 0x8 +#define FEATURE_STATUS__UVD_DPM_ON__SHIFT 0x3 +#define FEATURE_STATUS__VCE_DPM_ON_MASK 0x10 +#define FEATURE_STATUS__VCE_DPM_ON__SHIFT 0x4 +#define FEATURE_STATUS__SAMU_DPM_ON_MASK 0x20 +#define FEATURE_STATUS__SAMU_DPM_ON__SHIFT 0x5 +#define FEATURE_STATUS__ACP_DPM_ON_MASK 0x40 +#define FEATURE_STATUS__ACP_DPM_ON__SHIFT 0x6 +#define FEATURE_STATUS__PCIE_DPM_ON_MASK 0x80 +#define FEATURE_STATUS__PCIE_DPM_ON__SHIFT 0x7 +#define FEATURE_STATUS__BAPM_ON_MASK 0x100 +#define FEATURE_STATUS__BAPM_ON__SHIFT 0x8 +#define FEATURE_STATUS__LPMX_ON_MASK 0x200 +#define FEATURE_STATUS__LPMX_ON__SHIFT 0x9 +#define FEATURE_STATUS__NBDPM_ON_MASK 0x400 +#define FEATURE_STATUS__NBDPM_ON__SHIFT 0xa +#define FEATURE_STATUS__LHTC_ON_MASK 0x800 +#define FEATURE_STATUS__LHTC_ON__SHIFT 0xb +#define FEATURE_STATUS__VPC_ON_MASK 0x1000 +#define FEATURE_STATUS__VPC_ON__SHIFT 0xc +#define FEATURE_STATUS__VOLTAGE_CONTROLLER_ON_MASK 0x2000 +#define FEATURE_STATUS__VOLTAGE_CONTROLLER_ON__SHIFT 0xd +#define FEATURE_STATUS__TDC_LIMIT_ON_MASK 0x4000 +#define FEATURE_STATUS__TDC_LIMIT_ON__SHIFT 0xe +#define FEATURE_STATUS__GPU_CAC_ON_MASK 0x8000 +#define FEATURE_STATUS__GPU_CAC_ON__SHIFT 0xf +#define FEATURE_STATUS__AVS_ON_MASK 0x10000 +#define FEATURE_STATUS__AVS_ON__SHIFT 0x10 +#define FEATURE_STATUS__SPMI_ON_MASK 0x20000 +#define FEATURE_STATUS__SPMI_ON__SHIFT 0x11 +#define FEATURE_STATUS__SCLK_DPM_FORCED_MASK 0x40000 +#define FEATURE_STATUS__SCLK_DPM_FORCED__SHIFT 0x12 +#define FEATURE_STATUS__MCLK_DPM_FORCED_MASK 0x80000 +#define FEATURE_STATUS__MCLK_DPM_FORCED__SHIFT 0x13 +#define FEATURE_STATUS__LCLK_DPM_FORCED_MASK 0x100000 +#define FEATURE_STATUS__LCLK_DPM_FORCED__SHIFT 0x14 +#define FEATURE_STATUS__PCIE_DPM_FORCED_MASK 0x200000 +#define FEATURE_STATUS__PCIE_DPM_FORCED__SHIFT 0x15 +#define FEATURE_STATUS__RESERVED_MASK 0xffc00000 +#define FEATURE_STATUS__RESERVED__SHIFT 0x16 +#define ENTITY_TEMPERATURES_1__GPU_MASK 0xffffffff +#define ENTITY_TEMPERATURES_1__GPU__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_1__entries_0_0_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_1__entries_0_0_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_2__entries_0_0_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_2__entries_0_0_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_3__entries_0_0_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_3__entries_0_0_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_3__entries_0_0_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_3__entries_0_0_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_3__entries_0_0_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_3__entries_0_0_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_3__entries_0_0_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_3__entries_0_0_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_4__entries_0_1_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_4__entries_0_1_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_5__entries_0_1_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_5__entries_0_1_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_6__entries_0_1_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_6__entries_0_1_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_6__entries_0_1_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_6__entries_0_1_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_6__entries_0_1_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_6__entries_0_1_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_6__entries_0_1_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_6__entries_0_1_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_7__entries_0_2_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_7__entries_0_2_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_8__entries_0_2_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_8__entries_0_2_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_9__entries_0_2_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_9__entries_0_2_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_9__entries_0_2_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_9__entries_0_2_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_9__entries_0_2_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_9__entries_0_2_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_9__entries_0_2_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_9__entries_0_2_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_10__entries_0_3_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_10__entries_0_3_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_11__entries_0_3_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_11__entries_0_3_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_12__entries_0_3_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_12__entries_0_3_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_12__entries_0_3_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_12__entries_0_3_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_12__entries_0_3_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_12__entries_0_3_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_12__entries_0_3_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_12__entries_0_3_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_13__entries_1_0_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_13__entries_1_0_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_14__entries_1_0_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_14__entries_1_0_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_15__entries_1_0_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_15__entries_1_0_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_15__entries_1_0_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_15__entries_1_0_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_15__entries_1_0_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_15__entries_1_0_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_15__entries_1_0_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_15__entries_1_0_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_16__entries_1_1_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_16__entries_1_1_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_17__entries_1_1_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_17__entries_1_1_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_18__entries_1_1_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_18__entries_1_1_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_18__entries_1_1_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_18__entries_1_1_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_18__entries_1_1_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_18__entries_1_1_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_18__entries_1_1_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_18__entries_1_1_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_19__entries_1_2_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_19__entries_1_2_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_20__entries_1_2_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_20__entries_1_2_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_21__entries_1_2_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_21__entries_1_2_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_21__entries_1_2_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_21__entries_1_2_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_21__entries_1_2_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_21__entries_1_2_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_21__entries_1_2_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_21__entries_1_2_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_22__entries_1_3_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_22__entries_1_3_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_23__entries_1_3_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_23__entries_1_3_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_24__entries_1_3_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_24__entries_1_3_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_24__entries_1_3_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_24__entries_1_3_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_24__entries_1_3_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_24__entries_1_3_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_24__entries_1_3_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_24__entries_1_3_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_25__entries_2_0_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_25__entries_2_0_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_26__entries_2_0_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_26__entries_2_0_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_27__entries_2_0_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_27__entries_2_0_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_27__entries_2_0_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_27__entries_2_0_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_27__entries_2_0_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_27__entries_2_0_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_27__entries_2_0_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_27__entries_2_0_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_28__entries_2_1_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_28__entries_2_1_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_29__entries_2_1_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_29__entries_2_1_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_30__entries_2_1_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_30__entries_2_1_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_30__entries_2_1_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_30__entries_2_1_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_30__entries_2_1_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_30__entries_2_1_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_30__entries_2_1_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_30__entries_2_1_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_31__entries_2_2_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_31__entries_2_2_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_32__entries_2_2_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_32__entries_2_2_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_33__entries_2_2_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_33__entries_2_2_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_33__entries_2_2_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_33__entries_2_2_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_33__entries_2_2_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_33__entries_2_2_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_33__entries_2_2_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_33__entries_2_2_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_34__entries_2_3_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_34__entries_2_3_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_35__entries_2_3_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_35__entries_2_3_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_36__entries_2_3_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_36__entries_2_3_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_36__entries_2_3_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_36__entries_2_3_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_36__entries_2_3_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_36__entries_2_3_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_36__entries_2_3_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_36__entries_2_3_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_37__entries_3_0_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_37__entries_3_0_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_38__entries_3_0_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_38__entries_3_0_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_39__entries_3_0_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_39__entries_3_0_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_39__entries_3_0_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_39__entries_3_0_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_39__entries_3_0_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_39__entries_3_0_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_39__entries_3_0_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_39__entries_3_0_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_40__entries_3_1_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_40__entries_3_1_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_41__entries_3_1_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_41__entries_3_1_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_42__entries_3_1_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_42__entries_3_1_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_42__entries_3_1_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_42__entries_3_1_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_42__entries_3_1_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_42__entries_3_1_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_42__entries_3_1_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_42__entries_3_1_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_43__entries_3_2_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_43__entries_3_2_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_44__entries_3_2_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_44__entries_3_2_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_45__entries_3_2_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_45__entries_3_2_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_45__entries_3_2_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_45__entries_3_2_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_45__entries_3_2_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_45__entries_3_2_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_45__entries_3_2_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_45__entries_3_2_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_46__entries_3_3_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_46__entries_3_3_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_47__entries_3_3_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_47__entries_3_3_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_48__entries_3_3_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_48__entries_3_3_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_48__entries_3_3_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_48__entries_3_3_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_48__entries_3_3_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_48__entries_3_3_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_48__entries_3_3_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_48__entries_3_3_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_49__entries_4_0_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_49__entries_4_0_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_50__entries_4_0_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_50__entries_4_0_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_51__entries_4_0_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_51__entries_4_0_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_51__entries_4_0_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_51__entries_4_0_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_51__entries_4_0_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_51__entries_4_0_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_51__entries_4_0_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_51__entries_4_0_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_52__entries_4_1_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_52__entries_4_1_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_53__entries_4_1_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_53__entries_4_1_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_54__entries_4_1_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_54__entries_4_1_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_54__entries_4_1_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_54__entries_4_1_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_54__entries_4_1_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_54__entries_4_1_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_54__entries_4_1_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_54__entries_4_1_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_55__entries_4_2_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_55__entries_4_2_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_56__entries_4_2_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_56__entries_4_2_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_57__entries_4_2_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_57__entries_4_2_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_57__entries_4_2_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_57__entries_4_2_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_57__entries_4_2_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_57__entries_4_2_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_57__entries_4_2_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_57__entries_4_2_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_58__entries_4_3_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_58__entries_4_3_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_59__entries_4_3_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_59__entries_4_3_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_60__entries_4_3_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_60__entries_4_3_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_60__entries_4_3_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_60__entries_4_3_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_60__entries_4_3_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_60__entries_4_3_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_60__entries_4_3_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_60__entries_4_3_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_61__entries_5_0_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_61__entries_5_0_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_62__entries_5_0_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_62__entries_5_0_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_63__entries_5_0_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_63__entries_5_0_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_63__entries_5_0_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_63__entries_5_0_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_63__entries_5_0_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_63__entries_5_0_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_63__entries_5_0_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_63__entries_5_0_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_64__entries_5_1_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_64__entries_5_1_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_65__entries_5_1_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_65__entries_5_1_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_66__entries_5_1_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_66__entries_5_1_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_66__entries_5_1_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_66__entries_5_1_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_66__entries_5_1_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_66__entries_5_1_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_66__entries_5_1_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_66__entries_5_1_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_67__entries_5_2_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_67__entries_5_2_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_68__entries_5_2_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_68__entries_5_2_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_69__entries_5_2_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_69__entries_5_2_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_69__entries_5_2_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_69__entries_5_2_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_69__entries_5_2_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_69__entries_5_2_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_69__entries_5_2_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_69__entries_5_2_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_70__entries_5_3_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_70__entries_5_3_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_71__entries_5_3_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_71__entries_5_3_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_72__entries_5_3_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_72__entries_5_3_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_72__entries_5_3_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_72__entries_5_3_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_72__entries_5_3_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_72__entries_5_3_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_72__entries_5_3_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_72__entries_5_3_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_73__entries_6_0_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_73__entries_6_0_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_74__entries_6_0_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_74__entries_6_0_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_75__entries_6_0_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_75__entries_6_0_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_75__entries_6_0_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_75__entries_6_0_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_75__entries_6_0_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_75__entries_6_0_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_75__entries_6_0_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_75__entries_6_0_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_76__entries_6_1_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_76__entries_6_1_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_77__entries_6_1_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_77__entries_6_1_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_78__entries_6_1_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_78__entries_6_1_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_78__entries_6_1_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_78__entries_6_1_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_78__entries_6_1_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_78__entries_6_1_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_78__entries_6_1_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_78__entries_6_1_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_79__entries_6_2_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_79__entries_6_2_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_80__entries_6_2_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_80__entries_6_2_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_81__entries_6_2_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_81__entries_6_2_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_81__entries_6_2_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_81__entries_6_2_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_81__entries_6_2_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_81__entries_6_2_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_81__entries_6_2_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_81__entries_6_2_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_82__entries_6_3_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_82__entries_6_3_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_83__entries_6_3_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_83__entries_6_3_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_84__entries_6_3_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_84__entries_6_3_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_84__entries_6_3_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_84__entries_6_3_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_84__entries_6_3_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_84__entries_6_3_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_84__entries_6_3_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_84__entries_6_3_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_85__entries_7_0_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_85__entries_7_0_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_86__entries_7_0_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_86__entries_7_0_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_87__entries_7_0_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_87__entries_7_0_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_87__entries_7_0_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_87__entries_7_0_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_87__entries_7_0_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_87__entries_7_0_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_87__entries_7_0_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_87__entries_7_0_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_88__entries_7_1_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_88__entries_7_1_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_89__entries_7_1_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_89__entries_7_1_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_90__entries_7_1_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_90__entries_7_1_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_90__entries_7_1_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_90__entries_7_1_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_90__entries_7_1_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_90__entries_7_1_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_90__entries_7_1_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_90__entries_7_1_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_91__entries_7_2_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_91__entries_7_2_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_92__entries_7_2_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_92__entries_7_2_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_93__entries_7_2_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_93__entries_7_2_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_93__entries_7_2_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_93__entries_7_2_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_93__entries_7_2_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_93__entries_7_2_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_93__entries_7_2_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_93__entries_7_2_McArbBurstTime__SHIFT 0x18 +#define MCARB_DRAM_TIMING_TABLE_94__entries_7_3_McArbDramTiming_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_94__entries_7_3_McArbDramTiming__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_95__entries_7_3_McArbDramTiming2_MASK 0xffffffff +#define MCARB_DRAM_TIMING_TABLE_95__entries_7_3_McArbDramTiming2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_96__entries_7_3_padding_2_MASK 0xff +#define MCARB_DRAM_TIMING_TABLE_96__entries_7_3_padding_2__SHIFT 0x0 +#define MCARB_DRAM_TIMING_TABLE_96__entries_7_3_padding_1_MASK 0xff00 +#define MCARB_DRAM_TIMING_TABLE_96__entries_7_3_padding_1__SHIFT 0x8 +#define MCARB_DRAM_TIMING_TABLE_96__entries_7_3_padding_0_MASK 0xff0000 +#define MCARB_DRAM_TIMING_TABLE_96__entries_7_3_padding_0__SHIFT 0x10 +#define MCARB_DRAM_TIMING_TABLE_96__entries_7_3_McArbBurstTime_MASK 0xff000000 +#define MCARB_DRAM_TIMING_TABLE_96__entries_7_3_McArbBurstTime__SHIFT 0x18 +#define DPM_TABLE_1__GraphicsPIDController_Ki_MASK 0xffffffff +#define DPM_TABLE_1__GraphicsPIDController_Ki__SHIFT 0x0 +#define DPM_TABLE_2__GraphicsPIDController_LFWindupUpperLim_MASK 0xffffffff +#define DPM_TABLE_2__GraphicsPIDController_LFWindupUpperLim__SHIFT 0x0 +#define DPM_TABLE_3__GraphicsPIDController_LFWindupLowerLim_MASK 0xffffffff +#define DPM_TABLE_3__GraphicsPIDController_LFWindupLowerLim__SHIFT 0x0 +#define DPM_TABLE_4__GraphicsPIDController_StatePrecision_MASK 0xffffffff +#define DPM_TABLE_4__GraphicsPIDController_StatePrecision__SHIFT 0x0 +#define DPM_TABLE_5__GraphicsPIDController_LfPrecision_MASK 0xffffffff +#define DPM_TABLE_5__GraphicsPIDController_LfPrecision__SHIFT 0x0 +#define DPM_TABLE_6__GraphicsPIDController_LfOffset_MASK 0xffffffff +#define DPM_TABLE_6__GraphicsPIDController_LfOffset__SHIFT 0x0 +#define DPM_TABLE_7__GraphicsPIDController_MaxState_MASK 0xffffffff +#define DPM_TABLE_7__GraphicsPIDController_MaxState__SHIFT 0x0 +#define DPM_TABLE_8__GraphicsPIDController_MaxLfFraction_MASK 0xffffffff +#define DPM_TABLE_8__GraphicsPIDController_MaxLfFraction__SHIFT 0x0 +#define DPM_TABLE_9__GraphicsPIDController_StateShift_MASK 0xffffffff +#define DPM_TABLE_9__GraphicsPIDController_StateShift__SHIFT 0x0 +#define DPM_TABLE_10__MemoryPIDController_Ki_MASK 0xffffffff +#define DPM_TABLE_10__MemoryPIDController_Ki__SHIFT 0x0 +#define DPM_TABLE_11__MemoryPIDController_LFWindupUpperLim_MASK 0xffffffff +#define DPM_TABLE_11__MemoryPIDController_LFWindupUpperLim__SHIFT 0x0 +#define DPM_TABLE_12__MemoryPIDController_LFWindupLowerLim_MASK 0xffffffff +#define DPM_TABLE_12__MemoryPIDController_LFWindupLowerLim__SHIFT 0x0 +#define DPM_TABLE_13__MemoryPIDController_StatePrecision_MASK 0xffffffff +#define DPM_TABLE_13__MemoryPIDController_StatePrecision__SHIFT 0x0 +#define DPM_TABLE_14__MemoryPIDController_LfPrecision_MASK 0xffffffff +#define DPM_TABLE_14__MemoryPIDController_LfPrecision__SHIFT 0x0 +#define DPM_TABLE_15__MemoryPIDController_LfOffset_MASK 0xffffffff +#define DPM_TABLE_15__MemoryPIDController_LfOffset__SHIFT 0x0 +#define DPM_TABLE_16__MemoryPIDController_MaxState_MASK 0xffffffff +#define DPM_TABLE_16__MemoryPIDController_MaxState__SHIFT 0x0 +#define DPM_TABLE_17__MemoryPIDController_MaxLfFraction_MASK 0xffffffff +#define DPM_TABLE_17__MemoryPIDController_MaxLfFraction__SHIFT 0x0 +#define DPM_TABLE_18__MemoryPIDController_StateShift_MASK 0xffffffff +#define DPM_TABLE_18__MemoryPIDController_StateShift__SHIFT 0x0 +#define DPM_TABLE_19__LinkPIDController_Ki_MASK 0xffffffff +#define DPM_TABLE_19__LinkPIDController_Ki__SHIFT 0x0 +#define DPM_TABLE_20__LinkPIDController_LFWindupUpperLim_MASK 0xffffffff +#define DPM_TABLE_20__LinkPIDController_LFWindupUpperLim__SHIFT 0x0 +#define DPM_TABLE_21__LinkPIDController_LFWindupLowerLim_MASK 0xffffffff +#define DPM_TABLE_21__LinkPIDController_LFWindupLowerLim__SHIFT 0x0 +#define DPM_TABLE_22__LinkPIDController_StatePrecision_MASK 0xffffffff +#define DPM_TABLE_22__LinkPIDController_StatePrecision__SHIFT 0x0 +#define DPM_TABLE_23__LinkPIDController_LfPrecision_MASK 0xffffffff +#define DPM_TABLE_23__LinkPIDController_LfPrecision__SHIFT 0x0 +#define DPM_TABLE_24__LinkPIDController_LfOffset_MASK 0xffffffff +#define DPM_TABLE_24__LinkPIDController_LfOffset__SHIFT 0x0 +#define DPM_TABLE_25__LinkPIDController_MaxState_MASK 0xffffffff +#define DPM_TABLE_25__LinkPIDController_MaxState__SHIFT 0x0 +#define DPM_TABLE_26__LinkPIDController_MaxLfFraction_MASK 0xffffffff +#define DPM_TABLE_26__LinkPIDController_MaxLfFraction__SHIFT 0x0 +#define DPM_TABLE_27__LinkPIDController_StateShift_MASK 0xffffffff +#define DPM_TABLE_27__LinkPIDController_StateShift__SHIFT 0x0 +#define DPM_TABLE_28__SystemFlags_MASK 0xffffffff +#define DPM_TABLE_28__SystemFlags__SHIFT 0x0 +#define DPM_TABLE_29__VRConfig_MASK 0xffffffff +#define DPM_TABLE_29__VRConfig__SHIFT 0x0 +#define DPM_TABLE_30__SmioMask1_MASK 0xffffffff +#define DPM_TABLE_30__SmioMask1__SHIFT 0x0 +#define DPM_TABLE_31__SmioMask2_MASK 0xffffffff +#define DPM_TABLE_31__SmioMask2__SHIFT 0x0 +#define DPM_TABLE_32__SmioTable1_Pattern_0_padding_MASK 0xff +#define DPM_TABLE_32__SmioTable1_Pattern_0_padding__SHIFT 0x0 +#define DPM_TABLE_32__SmioTable1_Pattern_0_Smio_MASK 0xff00 +#define DPM_TABLE_32__SmioTable1_Pattern_0_Smio__SHIFT 0x8 +#define DPM_TABLE_32__SmioTable1_Pattern_0_Voltage_MASK 0xffff0000 +#define DPM_TABLE_32__SmioTable1_Pattern_0_Voltage__SHIFT 0x10 +#define DPM_TABLE_33__SmioTable1_Pattern_1_padding_MASK 0xff +#define DPM_TABLE_33__SmioTable1_Pattern_1_padding__SHIFT 0x0 +#define DPM_TABLE_33__SmioTable1_Pattern_1_Smio_MASK 0xff00 +#define DPM_TABLE_33__SmioTable1_Pattern_1_Smio__SHIFT 0x8 +#define DPM_TABLE_33__SmioTable1_Pattern_1_Voltage_MASK 0xffff0000 +#define DPM_TABLE_33__SmioTable1_Pattern_1_Voltage__SHIFT 0x10 +#define DPM_TABLE_34__SmioTable1_Pattern_2_padding_MASK 0xff +#define DPM_TABLE_34__SmioTable1_Pattern_2_padding__SHIFT 0x0 +#define DPM_TABLE_34__SmioTable1_Pattern_2_Smio_MASK 0xff00 +#define DPM_TABLE_34__SmioTable1_Pattern_2_Smio__SHIFT 0x8 +#define DPM_TABLE_34__SmioTable1_Pattern_2_Voltage_MASK 0xffff0000 +#define DPM_TABLE_34__SmioTable1_Pattern_2_Voltage__SHIFT 0x10 +#define DPM_TABLE_35__SmioTable1_Pattern_3_padding_MASK 0xff +#define DPM_TABLE_35__SmioTable1_Pattern_3_padding__SHIFT 0x0 +#define DPM_TABLE_35__SmioTable1_Pattern_3_Smio_MASK 0xff00 +#define DPM_TABLE_35__SmioTable1_Pattern_3_Smio__SHIFT 0x8 +#define DPM_TABLE_35__SmioTable1_Pattern_3_Voltage_MASK 0xffff0000 +#define DPM_TABLE_35__SmioTable1_Pattern_3_Voltage__SHIFT 0x10 +#define DPM_TABLE_36__SmioTable2_Pattern_0_padding_MASK 0xff +#define DPM_TABLE_36__SmioTable2_Pattern_0_padding__SHIFT 0x0 +#define DPM_TABLE_36__SmioTable2_Pattern_0_Smio_MASK 0xff00 +#define DPM_TABLE_36__SmioTable2_Pattern_0_Smio__SHIFT 0x8 +#define DPM_TABLE_36__SmioTable2_Pattern_0_Voltage_MASK 0xffff0000 +#define DPM_TABLE_36__SmioTable2_Pattern_0_Voltage__SHIFT 0x10 +#define DPM_TABLE_37__SmioTable2_Pattern_1_padding_MASK 0xff +#define DPM_TABLE_37__SmioTable2_Pattern_1_padding__SHIFT 0x0 +#define DPM_TABLE_37__SmioTable2_Pattern_1_Smio_MASK 0xff00 +#define DPM_TABLE_37__SmioTable2_Pattern_1_Smio__SHIFT 0x8 +#define DPM_TABLE_37__SmioTable2_Pattern_1_Voltage_MASK 0xffff0000 +#define DPM_TABLE_37__SmioTable2_Pattern_1_Voltage__SHIFT 0x10 +#define DPM_TABLE_38__SmioTable2_Pattern_2_padding_MASK 0xff +#define DPM_TABLE_38__SmioTable2_Pattern_2_padding__SHIFT 0x0 +#define DPM_TABLE_38__SmioTable2_Pattern_2_Smio_MASK 0xff00 +#define DPM_TABLE_38__SmioTable2_Pattern_2_Smio__SHIFT 0x8 +#define DPM_TABLE_38__SmioTable2_Pattern_2_Voltage_MASK 0xffff0000 +#define DPM_TABLE_38__SmioTable2_Pattern_2_Voltage__SHIFT 0x10 +#define DPM_TABLE_39__SmioTable2_Pattern_3_padding_MASK 0xff +#define DPM_TABLE_39__SmioTable2_Pattern_3_padding__SHIFT 0x0 +#define DPM_TABLE_39__SmioTable2_Pattern_3_Smio_MASK 0xff00 +#define DPM_TABLE_39__SmioTable2_Pattern_3_Smio__SHIFT 0x8 +#define DPM_TABLE_39__SmioTable2_Pattern_3_Voltage_MASK 0xffff0000 +#define DPM_TABLE_39__SmioTable2_Pattern_3_Voltage__SHIFT 0x10 +#define DPM_TABLE_40__VddcLevelCount_MASK 0xffffffff +#define DPM_TABLE_40__VddcLevelCount__SHIFT 0x0 +#define DPM_TABLE_41__VddciLevelCount_MASK 0xffffffff +#define DPM_TABLE_41__VddciLevelCount__SHIFT 0x0 +#define DPM_TABLE_42__VddGfxLevelCount_MASK 0xffffffff +#define DPM_TABLE_42__VddGfxLevelCount__SHIFT 0x0 +#define DPM_TABLE_43__MvddLevelCount_MASK 0xffffffff +#define DPM_TABLE_43__MvddLevelCount__SHIFT 0x0 +#define DPM_TABLE_44__VddcTable_1_MASK 0xffff +#define DPM_TABLE_44__VddcTable_1__SHIFT 0x0 +#define DPM_TABLE_44__VddcTable_0_MASK 0xffff0000 +#define DPM_TABLE_44__VddcTable_0__SHIFT 0x10 +#define DPM_TABLE_45__VddcTable_3_MASK 0xffff +#define DPM_TABLE_45__VddcTable_3__SHIFT 0x0 +#define DPM_TABLE_45__VddcTable_2_MASK 0xffff0000 +#define DPM_TABLE_45__VddcTable_2__SHIFT 0x10 +#define DPM_TABLE_46__VddcTable_5_MASK 0xffff +#define DPM_TABLE_46__VddcTable_5__SHIFT 0x0 +#define DPM_TABLE_46__VddcTable_4_MASK 0xffff0000 +#define DPM_TABLE_46__VddcTable_4__SHIFT 0x10 +#define DPM_TABLE_47__VddcTable_7_MASK 0xffff +#define DPM_TABLE_47__VddcTable_7__SHIFT 0x0 +#define DPM_TABLE_47__VddcTable_6_MASK 0xffff0000 +#define DPM_TABLE_47__VddcTable_6__SHIFT 0x10 +#define DPM_TABLE_48__VddcTable_9_MASK 0xffff +#define DPM_TABLE_48__VddcTable_9__SHIFT 0x0 +#define DPM_TABLE_48__VddcTable_8_MASK 0xffff0000 +#define DPM_TABLE_48__VddcTable_8__SHIFT 0x10 +#define DPM_TABLE_49__VddcTable_11_MASK 0xffff +#define DPM_TABLE_49__VddcTable_11__SHIFT 0x0 +#define DPM_TABLE_49__VddcTable_10_MASK 0xffff0000 +#define DPM_TABLE_49__VddcTable_10__SHIFT 0x10 +#define DPM_TABLE_50__VddcTable_13_MASK 0xffff +#define DPM_TABLE_50__VddcTable_13__SHIFT 0x0 +#define DPM_TABLE_50__VddcTable_12_MASK 0xffff0000 +#define DPM_TABLE_50__VddcTable_12__SHIFT 0x10 +#define DPM_TABLE_51__VddcTable_15_MASK 0xffff +#define DPM_TABLE_51__VddcTable_15__SHIFT 0x0 +#define DPM_TABLE_51__VddcTable_14_MASK 0xffff0000 +#define DPM_TABLE_51__VddcTable_14__SHIFT 0x10 +#define DPM_TABLE_52__VddGfxTable_1_MASK 0xffff +#define DPM_TABLE_52__VddGfxTable_1__SHIFT 0x0 +#define DPM_TABLE_52__VddGfxTable_0_MASK 0xffff0000 +#define DPM_TABLE_52__VddGfxTable_0__SHIFT 0x10 +#define DPM_TABLE_53__VddGfxTable_3_MASK 0xffff +#define DPM_TABLE_53__VddGfxTable_3__SHIFT 0x0 +#define DPM_TABLE_53__VddGfxTable_2_MASK 0xffff0000 +#define DPM_TABLE_53__VddGfxTable_2__SHIFT 0x10 +#define DPM_TABLE_54__VddGfxTable_5_MASK 0xffff +#define DPM_TABLE_54__VddGfxTable_5__SHIFT 0x0 +#define DPM_TABLE_54__VddGfxTable_4_MASK 0xffff0000 +#define DPM_TABLE_54__VddGfxTable_4__SHIFT 0x10 +#define DPM_TABLE_55__VddGfxTable_7_MASK 0xffff +#define DPM_TABLE_55__VddGfxTable_7__SHIFT 0x0 +#define DPM_TABLE_55__VddGfxTable_6_MASK 0xffff0000 +#define DPM_TABLE_55__VddGfxTable_6__SHIFT 0x10 +#define DPM_TABLE_56__VddGfxTable_9_MASK 0xffff +#define DPM_TABLE_56__VddGfxTable_9__SHIFT 0x0 +#define DPM_TABLE_56__VddGfxTable_8_MASK 0xffff0000 +#define DPM_TABLE_56__VddGfxTable_8__SHIFT 0x10 +#define DPM_TABLE_57__VddGfxTable_11_MASK 0xffff +#define DPM_TABLE_57__VddGfxTable_11__SHIFT 0x0 +#define DPM_TABLE_57__VddGfxTable_10_MASK 0xffff0000 +#define DPM_TABLE_57__VddGfxTable_10__SHIFT 0x10 +#define DPM_TABLE_58__VddGfxTable_13_MASK 0xffff +#define DPM_TABLE_58__VddGfxTable_13__SHIFT 0x0 +#define DPM_TABLE_58__VddGfxTable_12_MASK 0xffff0000 +#define DPM_TABLE_58__VddGfxTable_12__SHIFT 0x10 +#define DPM_TABLE_59__VddGfxTable_15_MASK 0xffff +#define DPM_TABLE_59__VddGfxTable_15__SHIFT 0x0 +#define DPM_TABLE_59__VddGfxTable_14_MASK 0xffff0000 +#define DPM_TABLE_59__VddGfxTable_14__SHIFT 0x10 +#define DPM_TABLE_60__VddciTable_1_MASK 0xffff +#define DPM_TABLE_60__VddciTable_1__SHIFT 0x0 +#define DPM_TABLE_60__VddciTable_0_MASK 0xffff0000 +#define DPM_TABLE_60__VddciTable_0__SHIFT 0x10 +#define DPM_TABLE_61__VddciTable_3_MASK 0xffff +#define DPM_TABLE_61__VddciTable_3__SHIFT 0x0 +#define DPM_TABLE_61__VddciTable_2_MASK 0xffff0000 +#define DPM_TABLE_61__VddciTable_2__SHIFT 0x10 +#define DPM_TABLE_62__VddciTable_5_MASK 0xffff +#define DPM_TABLE_62__VddciTable_5__SHIFT 0x0 +#define DPM_TABLE_62__VddciTable_4_MASK 0xffff0000 +#define DPM_TABLE_62__VddciTable_4__SHIFT 0x10 +#define DPM_TABLE_63__VddciTable_7_MASK 0xffff +#define DPM_TABLE_63__VddciTable_7__SHIFT 0x0 +#define DPM_TABLE_63__VddciTable_6_MASK 0xffff0000 +#define DPM_TABLE_63__VddciTable_6__SHIFT 0x10 +#define DPM_TABLE_64__BapmVddGfxVidHiSidd_3_MASK 0xff +#define DPM_TABLE_64__BapmVddGfxVidHiSidd_3__SHIFT 0x0 +#define DPM_TABLE_64__BapmVddGfxVidHiSidd_2_MASK 0xff00 +#define DPM_TABLE_64__BapmVddGfxVidHiSidd_2__SHIFT 0x8 +#define DPM_TABLE_64__BapmVddGfxVidHiSidd_1_MASK 0xff0000 +#define DPM_TABLE_64__BapmVddGfxVidHiSidd_1__SHIFT 0x10 +#define DPM_TABLE_64__BapmVddGfxVidHiSidd_0_MASK 0xff000000 +#define DPM_TABLE_64__BapmVddGfxVidHiSidd_0__SHIFT 0x18 +#define DPM_TABLE_65__BapmVddGfxVidHiSidd_7_MASK 0xff +#define DPM_TABLE_65__BapmVddGfxVidHiSidd_7__SHIFT 0x0 +#define DPM_TABLE_65__BapmVddGfxVidHiSidd_6_MASK 0xff00 +#define DPM_TABLE_65__BapmVddGfxVidHiSidd_6__SHIFT 0x8 +#define DPM_TABLE_65__BapmVddGfxVidHiSidd_5_MASK 0xff0000 +#define DPM_TABLE_65__BapmVddGfxVidHiSidd_5__SHIFT 0x10 +#define DPM_TABLE_65__BapmVddGfxVidHiSidd_4_MASK 0xff000000 +#define DPM_TABLE_65__BapmVddGfxVidHiSidd_4__SHIFT 0x18 +#define DPM_TABLE_66__BapmVddGfxVidHiSidd_11_MASK 0xff +#define DPM_TABLE_66__BapmVddGfxVidHiSidd_11__SHIFT 0x0 +#define DPM_TABLE_66__BapmVddGfxVidHiSidd_10_MASK 0xff00 +#define DPM_TABLE_66__BapmVddGfxVidHiSidd_10__SHIFT 0x8 +#define DPM_TABLE_66__BapmVddGfxVidHiSidd_9_MASK 0xff0000 +#define DPM_TABLE_66__BapmVddGfxVidHiSidd_9__SHIFT 0x10 +#define DPM_TABLE_66__BapmVddGfxVidHiSidd_8_MASK 0xff000000 +#define DPM_TABLE_66__BapmVddGfxVidHiSidd_8__SHIFT 0x18 +#define DPM_TABLE_67__BapmVddGfxVidHiSidd_15_MASK 0xff +#define DPM_TABLE_67__BapmVddGfxVidHiSidd_15__SHIFT 0x0 +#define DPM_TABLE_67__BapmVddGfxVidHiSidd_14_MASK 0xff00 +#define DPM_TABLE_67__BapmVddGfxVidHiSidd_14__SHIFT 0x8 +#define DPM_TABLE_67__BapmVddGfxVidHiSidd_13_MASK 0xff0000 +#define DPM_TABLE_67__BapmVddGfxVidHiSidd_13__SHIFT 0x10 +#define DPM_TABLE_67__BapmVddGfxVidHiSidd_12_MASK 0xff000000 +#define DPM_TABLE_67__BapmVddGfxVidHiSidd_12__SHIFT 0x18 +#define DPM_TABLE_68__BapmVddGfxVidLoSidd_3_MASK 0xff +#define DPM_TABLE_68__BapmVddGfxVidLoSidd_3__SHIFT 0x0 +#define DPM_TABLE_68__BapmVddGfxVidLoSidd_2_MASK 0xff00 +#define DPM_TABLE_68__BapmVddGfxVidLoSidd_2__SHIFT 0x8 +#define DPM_TABLE_68__BapmVddGfxVidLoSidd_1_MASK 0xff0000 +#define DPM_TABLE_68__BapmVddGfxVidLoSidd_1__SHIFT 0x10 +#define DPM_TABLE_68__BapmVddGfxVidLoSidd_0_MASK 0xff000000 +#define DPM_TABLE_68__BapmVddGfxVidLoSidd_0__SHIFT 0x18 +#define DPM_TABLE_69__BapmVddGfxVidLoSidd_7_MASK 0xff +#define DPM_TABLE_69__BapmVddGfxVidLoSidd_7__SHIFT 0x0 +#define DPM_TABLE_69__BapmVddGfxVidLoSidd_6_MASK 0xff00 +#define DPM_TABLE_69__BapmVddGfxVidLoSidd_6__SHIFT 0x8 +#define DPM_TABLE_69__BapmVddGfxVidLoSidd_5_MASK 0xff0000 +#define DPM_TABLE_69__BapmVddGfxVidLoSidd_5__SHIFT 0x10 +#define DPM_TABLE_69__BapmVddGfxVidLoSidd_4_MASK 0xff000000 +#define DPM_TABLE_69__BapmVddGfxVidLoSidd_4__SHIFT 0x18 +#define DPM_TABLE_70__BapmVddGfxVidLoSidd_11_MASK 0xff +#define DPM_TABLE_70__BapmVddGfxVidLoSidd_11__SHIFT 0x0 +#define DPM_TABLE_70__BapmVddGfxVidLoSidd_10_MASK 0xff00 +#define DPM_TABLE_70__BapmVddGfxVidLoSidd_10__SHIFT 0x8 +#define DPM_TABLE_70__BapmVddGfxVidLoSidd_9_MASK 0xff0000 +#define DPM_TABLE_70__BapmVddGfxVidLoSidd_9__SHIFT 0x10 +#define DPM_TABLE_70__BapmVddGfxVidLoSidd_8_MASK 0xff000000 +#define DPM_TABLE_70__BapmVddGfxVidLoSidd_8__SHIFT 0x18 +#define DPM_TABLE_71__BapmVddGfxVidLoSidd_15_MASK 0xff +#define DPM_TABLE_71__BapmVddGfxVidLoSidd_15__SHIFT 0x0 +#define DPM_TABLE_71__BapmVddGfxVidLoSidd_14_MASK 0xff00 +#define DPM_TABLE_71__BapmVddGfxVidLoSidd_14__SHIFT 0x8 +#define DPM_TABLE_71__BapmVddGfxVidLoSidd_13_MASK 0xff0000 +#define DPM_TABLE_71__BapmVddGfxVidLoSidd_13__SHIFT 0x10 +#define DPM_TABLE_71__BapmVddGfxVidLoSidd_12_MASK 0xff000000 +#define DPM_TABLE_71__BapmVddGfxVidLoSidd_12__SHIFT 0x18 +#define DPM_TABLE_72__BapmVddGfxVidHiSidd2_3_MASK 0xff +#define DPM_TABLE_72__BapmVddGfxVidHiSidd2_3__SHIFT 0x0 +#define DPM_TABLE_72__BapmVddGfxVidHiSidd2_2_MASK 0xff00 +#define DPM_TABLE_72__BapmVddGfxVidHiSidd2_2__SHIFT 0x8 +#define DPM_TABLE_72__BapmVddGfxVidHiSidd2_1_MASK 0xff0000 +#define DPM_TABLE_72__BapmVddGfxVidHiSidd2_1__SHIFT 0x10 +#define DPM_TABLE_72__BapmVddGfxVidHiSidd2_0_MASK 0xff000000 +#define DPM_TABLE_72__BapmVddGfxVidHiSidd2_0__SHIFT 0x18 +#define DPM_TABLE_73__BapmVddGfxVidHiSidd2_7_MASK 0xff +#define DPM_TABLE_73__BapmVddGfxVidHiSidd2_7__SHIFT 0x0 +#define DPM_TABLE_73__BapmVddGfxVidHiSidd2_6_MASK 0xff00 +#define DPM_TABLE_73__BapmVddGfxVidHiSidd2_6__SHIFT 0x8 +#define DPM_TABLE_73__BapmVddGfxVidHiSidd2_5_MASK 0xff0000 +#define DPM_TABLE_73__BapmVddGfxVidHiSidd2_5__SHIFT 0x10 +#define DPM_TABLE_73__BapmVddGfxVidHiSidd2_4_MASK 0xff000000 +#define DPM_TABLE_73__BapmVddGfxVidHiSidd2_4__SHIFT 0x18 +#define DPM_TABLE_74__BapmVddGfxVidHiSidd2_11_MASK 0xff +#define DPM_TABLE_74__BapmVddGfxVidHiSidd2_11__SHIFT 0x0 +#define DPM_TABLE_74__BapmVddGfxVidHiSidd2_10_MASK 0xff00 +#define DPM_TABLE_74__BapmVddGfxVidHiSidd2_10__SHIFT 0x8 +#define DPM_TABLE_74__BapmVddGfxVidHiSidd2_9_MASK 0xff0000 +#define DPM_TABLE_74__BapmVddGfxVidHiSidd2_9__SHIFT 0x10 +#define DPM_TABLE_74__BapmVddGfxVidHiSidd2_8_MASK 0xff000000 +#define DPM_TABLE_74__BapmVddGfxVidHiSidd2_8__SHIFT 0x18 +#define DPM_TABLE_75__BapmVddGfxVidHiSidd2_15_MASK 0xff +#define DPM_TABLE_75__BapmVddGfxVidHiSidd2_15__SHIFT 0x0 +#define DPM_TABLE_75__BapmVddGfxVidHiSidd2_14_MASK 0xff00 +#define DPM_TABLE_75__BapmVddGfxVidHiSidd2_14__SHIFT 0x8 +#define DPM_TABLE_75__BapmVddGfxVidHiSidd2_13_MASK 0xff0000 +#define DPM_TABLE_75__BapmVddGfxVidHiSidd2_13__SHIFT 0x10 +#define DPM_TABLE_75__BapmVddGfxVidHiSidd2_12_MASK 0xff000000 +#define DPM_TABLE_75__BapmVddGfxVidHiSidd2_12__SHIFT 0x18 +#define DPM_TABLE_76__BapmVddcVidHiSidd_3_MASK 0xff +#define DPM_TABLE_76__BapmVddcVidHiSidd_3__SHIFT 0x0 +#define DPM_TABLE_76__BapmVddcVidHiSidd_2_MASK 0xff00 +#define DPM_TABLE_76__BapmVddcVidHiSidd_2__SHIFT 0x8 +#define DPM_TABLE_76__BapmVddcVidHiSidd_1_MASK 0xff0000 +#define DPM_TABLE_76__BapmVddcVidHiSidd_1__SHIFT 0x10 +#define DPM_TABLE_76__BapmVddcVidHiSidd_0_MASK 0xff000000 +#define DPM_TABLE_76__BapmVddcVidHiSidd_0__SHIFT 0x18 +#define DPM_TABLE_77__BapmVddcVidHiSidd_7_MASK 0xff +#define DPM_TABLE_77__BapmVddcVidHiSidd_7__SHIFT 0x0 +#define DPM_TABLE_77__BapmVddcVidHiSidd_6_MASK 0xff00 +#define DPM_TABLE_77__BapmVddcVidHiSidd_6__SHIFT 0x8 +#define DPM_TABLE_77__BapmVddcVidHiSidd_5_MASK 0xff0000 +#define DPM_TABLE_77__BapmVddcVidHiSidd_5__SHIFT 0x10 +#define DPM_TABLE_77__BapmVddcVidHiSidd_4_MASK 0xff000000 +#define DPM_TABLE_77__BapmVddcVidHiSidd_4__SHIFT 0x18 +#define DPM_TABLE_78__BapmVddcVidHiSidd_11_MASK 0xff +#define DPM_TABLE_78__BapmVddcVidHiSidd_11__SHIFT 0x0 +#define DPM_TABLE_78__BapmVddcVidHiSidd_10_MASK 0xff00 +#define DPM_TABLE_78__BapmVddcVidHiSidd_10__SHIFT 0x8 +#define DPM_TABLE_78__BapmVddcVidHiSidd_9_MASK 0xff0000 +#define DPM_TABLE_78__BapmVddcVidHiSidd_9__SHIFT 0x10 +#define DPM_TABLE_78__BapmVddcVidHiSidd_8_MASK 0xff000000 +#define DPM_TABLE_78__BapmVddcVidHiSidd_8__SHIFT 0x18 +#define DPM_TABLE_79__BapmVddcVidHiSidd_15_MASK 0xff +#define DPM_TABLE_79__BapmVddcVidHiSidd_15__SHIFT 0x0 +#define DPM_TABLE_79__BapmVddcVidHiSidd_14_MASK 0xff00 +#define DPM_TABLE_79__BapmVddcVidHiSidd_14__SHIFT 0x8 +#define DPM_TABLE_79__BapmVddcVidHiSidd_13_MASK 0xff0000 +#define DPM_TABLE_79__BapmVddcVidHiSidd_13__SHIFT 0x10 +#define DPM_TABLE_79__BapmVddcVidHiSidd_12_MASK 0xff000000 +#define DPM_TABLE_79__BapmVddcVidHiSidd_12__SHIFT 0x18 +#define DPM_TABLE_80__BapmVddcVidLoSidd_3_MASK 0xff +#define DPM_TABLE_80__BapmVddcVidLoSidd_3__SHIFT 0x0 +#define DPM_TABLE_80__BapmVddcVidLoSidd_2_MASK 0xff00 +#define DPM_TABLE_80__BapmVddcVidLoSidd_2__SHIFT 0x8 +#define DPM_TABLE_80__BapmVddcVidLoSidd_1_MASK 0xff0000 +#define DPM_TABLE_80__BapmVddcVidLoSidd_1__SHIFT 0x10 +#define DPM_TABLE_80__BapmVddcVidLoSidd_0_MASK 0xff000000 +#define DPM_TABLE_80__BapmVddcVidLoSidd_0__SHIFT 0x18 +#define DPM_TABLE_81__BapmVddcVidLoSidd_7_MASK 0xff +#define DPM_TABLE_81__BapmVddcVidLoSidd_7__SHIFT 0x0 +#define DPM_TABLE_81__BapmVddcVidLoSidd_6_MASK 0xff00 +#define DPM_TABLE_81__BapmVddcVidLoSidd_6__SHIFT 0x8 +#define DPM_TABLE_81__BapmVddcVidLoSidd_5_MASK 0xff0000 +#define DPM_TABLE_81__BapmVddcVidLoSidd_5__SHIFT 0x10 +#define DPM_TABLE_81__BapmVddcVidLoSidd_4_MASK 0xff000000 +#define DPM_TABLE_81__BapmVddcVidLoSidd_4__SHIFT 0x18 +#define DPM_TABLE_82__BapmVddcVidLoSidd_11_MASK 0xff +#define DPM_TABLE_82__BapmVddcVidLoSidd_11__SHIFT 0x0 +#define DPM_TABLE_82__BapmVddcVidLoSidd_10_MASK 0xff00 +#define DPM_TABLE_82__BapmVddcVidLoSidd_10__SHIFT 0x8 +#define DPM_TABLE_82__BapmVddcVidLoSidd_9_MASK 0xff0000 +#define DPM_TABLE_82__BapmVddcVidLoSidd_9__SHIFT 0x10 +#define DPM_TABLE_82__BapmVddcVidLoSidd_8_MASK 0xff000000 +#define DPM_TABLE_82__BapmVddcVidLoSidd_8__SHIFT 0x18 +#define DPM_TABLE_83__BapmVddcVidLoSidd_15_MASK 0xff +#define DPM_TABLE_83__BapmVddcVidLoSidd_15__SHIFT 0x0 +#define DPM_TABLE_83__BapmVddcVidLoSidd_14_MASK 0xff00 +#define DPM_TABLE_83__BapmVddcVidLoSidd_14__SHIFT 0x8 +#define DPM_TABLE_83__BapmVddcVidLoSidd_13_MASK 0xff0000 +#define DPM_TABLE_83__BapmVddcVidLoSidd_13__SHIFT 0x10 +#define DPM_TABLE_83__BapmVddcVidLoSidd_12_MASK 0xff000000 +#define DPM_TABLE_83__BapmVddcVidLoSidd_12__SHIFT 0x18 +#define DPM_TABLE_84__BapmVddcVidHiSidd2_3_MASK 0xff +#define DPM_TABLE_84__BapmVddcVidHiSidd2_3__SHIFT 0x0 +#define DPM_TABLE_84__BapmVddcVidHiSidd2_2_MASK 0xff00 +#define DPM_TABLE_84__BapmVddcVidHiSidd2_2__SHIFT 0x8 +#define DPM_TABLE_84__BapmVddcVidHiSidd2_1_MASK 0xff0000 +#define DPM_TABLE_84__BapmVddcVidHiSidd2_1__SHIFT 0x10 +#define DPM_TABLE_84__BapmVddcVidHiSidd2_0_MASK 0xff000000 +#define DPM_TABLE_84__BapmVddcVidHiSidd2_0__SHIFT 0x18 +#define DPM_TABLE_85__BapmVddcVidHiSidd2_7_MASK 0xff +#define DPM_TABLE_85__BapmVddcVidHiSidd2_7__SHIFT 0x0 +#define DPM_TABLE_85__BapmVddcVidHiSidd2_6_MASK 0xff00 +#define DPM_TABLE_85__BapmVddcVidHiSidd2_6__SHIFT 0x8 +#define DPM_TABLE_85__BapmVddcVidHiSidd2_5_MASK 0xff0000 +#define DPM_TABLE_85__BapmVddcVidHiSidd2_5__SHIFT 0x10 +#define DPM_TABLE_85__BapmVddcVidHiSidd2_4_MASK 0xff000000 +#define DPM_TABLE_85__BapmVddcVidHiSidd2_4__SHIFT 0x18 +#define DPM_TABLE_86__BapmVddcVidHiSidd2_11_MASK 0xff +#define DPM_TABLE_86__BapmVddcVidHiSidd2_11__SHIFT 0x0 +#define DPM_TABLE_86__BapmVddcVidHiSidd2_10_MASK 0xff00 +#define DPM_TABLE_86__BapmVddcVidHiSidd2_10__SHIFT 0x8 +#define DPM_TABLE_86__BapmVddcVidHiSidd2_9_MASK 0xff0000 +#define DPM_TABLE_86__BapmVddcVidHiSidd2_9__SHIFT 0x10 +#define DPM_TABLE_86__BapmVddcVidHiSidd2_8_MASK 0xff000000 +#define DPM_TABLE_86__BapmVddcVidHiSidd2_8__SHIFT 0x18 +#define DPM_TABLE_87__BapmVddcVidHiSidd2_15_MASK 0xff +#define DPM_TABLE_87__BapmVddcVidHiSidd2_15__SHIFT 0x0 +#define DPM_TABLE_87__BapmVddcVidHiSidd2_14_MASK 0xff00 +#define DPM_TABLE_87__BapmVddcVidHiSidd2_14__SHIFT 0x8 +#define DPM_TABLE_87__BapmVddcVidHiSidd2_13_MASK 0xff0000 +#define DPM_TABLE_87__BapmVddcVidHiSidd2_13__SHIFT 0x10 +#define DPM_TABLE_87__BapmVddcVidHiSidd2_12_MASK 0xff000000 +#define DPM_TABLE_87__BapmVddcVidHiSidd2_12__SHIFT 0x18 +#define DPM_TABLE_88__MasterDeepSleepControl_MASK 0xff +#define DPM_TABLE_88__MasterDeepSleepControl__SHIFT 0x0 +#define DPM_TABLE_88__LinkLevelCount_MASK 0xff00 +#define DPM_TABLE_88__LinkLevelCount__SHIFT 0x8 +#define DPM_TABLE_88__MemoryDpmLevelCount_MASK 0xff0000 +#define DPM_TABLE_88__MemoryDpmLevelCount__SHIFT 0x10 +#define DPM_TABLE_88__GraphicsDpmLevelCount_MASK 0xff000000 +#define DPM_TABLE_88__GraphicsDpmLevelCount__SHIFT 0x18 +#define DPM_TABLE_89__SamuLevelCount_MASK 0xff +#define DPM_TABLE_89__SamuLevelCount__SHIFT 0x0 +#define DPM_TABLE_89__AcpLevelCount_MASK 0xff00 +#define DPM_TABLE_89__AcpLevelCount__SHIFT 0x8 +#define DPM_TABLE_89__VceLevelCount_MASK 0xff0000 +#define DPM_TABLE_89__VceLevelCount__SHIFT 0x10 +#define DPM_TABLE_89__UvdLevelCount_MASK 0xff000000 +#define DPM_TABLE_89__UvdLevelCount__SHIFT 0x18 +#define DPM_TABLE_90__Reserved_0_MASK 0xff +#define DPM_TABLE_90__Reserved_0__SHIFT 0x0 +#define DPM_TABLE_90__ThermOutMode_MASK 0xff00 +#define DPM_TABLE_90__ThermOutMode__SHIFT 0x8 +#define DPM_TABLE_90__ThermOutPolarity_MASK 0xff0000 +#define DPM_TABLE_90__ThermOutPolarity__SHIFT 0x10 +#define DPM_TABLE_90__ThermOutGpio_MASK 0xff000000 +#define DPM_TABLE_90__ThermOutGpio__SHIFT 0x18 +#define DPM_TABLE_91__Reserved_0_MASK 0xffffffff +#define DPM_TABLE_91__Reserved_0__SHIFT 0x0 +#define DPM_TABLE_92__Reserved_1_MASK 0xffffffff +#define DPM_TABLE_92__Reserved_1__SHIFT 0x0 +#define DPM_TABLE_93__Reserved_2_MASK 0xffffffff +#define DPM_TABLE_93__Reserved_2__SHIFT 0x0 +#define DPM_TABLE_94__Reserved_3_MASK 0xffffffff +#define DPM_TABLE_94__Reserved_3__SHIFT 0x0 +#define DPM_TABLE_95__GraphicsLevel_0_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_95__GraphicsLevel_0_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_95__GraphicsLevel_0_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_95__GraphicsLevel_0_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_95__GraphicsLevel_0_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_95__GraphicsLevel_0_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_95__GraphicsLevel_0_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_95__GraphicsLevel_0_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_96__GraphicsLevel_0_SclkFrequency_MASK 0xffffffff +#define DPM_TABLE_96__GraphicsLevel_0_SclkFrequency__SHIFT 0x0 +#define DPM_TABLE_97__GraphicsLevel_0_ActivityLevel_MASK 0xffff +#define DPM_TABLE_97__GraphicsLevel_0_ActivityLevel__SHIFT 0x0 +#define DPM_TABLE_97__GraphicsLevel_0_DeepSleepDivId_MASK 0xff0000 +#define DPM_TABLE_97__GraphicsLevel_0_DeepSleepDivId__SHIFT 0x10 +#define DPM_TABLE_97__GraphicsLevel_0_pcieDpmLevel_MASK 0xff000000 +#define DPM_TABLE_97__GraphicsLevel_0_pcieDpmLevel__SHIFT 0x18 +#define DPM_TABLE_98__GraphicsLevel_0_CgSpllFuncCntl3_MASK 0xffffffff +#define DPM_TABLE_98__GraphicsLevel_0_CgSpllFuncCntl3__SHIFT 0x0 +#define DPM_TABLE_99__GraphicsLevel_0_CgSpllFuncCntl4_MASK 0xffffffff +#define DPM_TABLE_99__GraphicsLevel_0_CgSpllFuncCntl4__SHIFT 0x0 +#define DPM_TABLE_100__GraphicsLevel_0_SpllSpreadSpectrum_MASK 0xffffffff +#define DPM_TABLE_100__GraphicsLevel_0_SpllSpreadSpectrum__SHIFT 0x0 +#define DPM_TABLE_101__GraphicsLevel_0_SpllSpreadSpectrum2_MASK 0xffffffff +#define DPM_TABLE_101__GraphicsLevel_0_SpllSpreadSpectrum2__SHIFT 0x0 +#define DPM_TABLE_102__GraphicsLevel_0_CcPwrDynRm_MASK 0xffffffff +#define DPM_TABLE_102__GraphicsLevel_0_CcPwrDynRm__SHIFT 0x0 +#define DPM_TABLE_103__GraphicsLevel_0_CcPwrDynRm1_MASK 0xffffffff +#define DPM_TABLE_103__GraphicsLevel_0_CcPwrDynRm1__SHIFT 0x0 +#define DPM_TABLE_104__GraphicsLevel_0_EnabledForThrottle_MASK 0xff +#define DPM_TABLE_104__GraphicsLevel_0_EnabledForThrottle__SHIFT 0x0 +#define DPM_TABLE_104__GraphicsLevel_0_EnabledForActivity_MASK 0xff00 +#define DPM_TABLE_104__GraphicsLevel_0_EnabledForActivity__SHIFT 0x8 +#define DPM_TABLE_104__GraphicsLevel_0_DisplayWatermark_MASK 0xff0000 +#define DPM_TABLE_104__GraphicsLevel_0_DisplayWatermark__SHIFT 0x10 +#define DPM_TABLE_104__GraphicsLevel_0_SclkDid_MASK 0xff000000 +#define DPM_TABLE_104__GraphicsLevel_0_SclkDid__SHIFT 0x18 +#define DPM_TABLE_105__GraphicsLevel_0_PowerThrottle_MASK 0xff +#define DPM_TABLE_105__GraphicsLevel_0_PowerThrottle__SHIFT 0x0 +#define DPM_TABLE_105__GraphicsLevel_0_VoltageDownHyst_MASK 0xff00 +#define DPM_TABLE_105__GraphicsLevel_0_VoltageDownHyst__SHIFT 0x8 +#define DPM_TABLE_105__GraphicsLevel_0_DownHyst_MASK 0xff0000 +#define DPM_TABLE_105__GraphicsLevel_0_DownHyst__SHIFT 0x10 +#define DPM_TABLE_105__GraphicsLevel_0_UpHyst_MASK 0xff000000 +#define DPM_TABLE_105__GraphicsLevel_0_UpHyst__SHIFT 0x18 +#define DPM_TABLE_106__GraphicsLevel_1_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_106__GraphicsLevel_1_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_106__GraphicsLevel_1_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_106__GraphicsLevel_1_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_106__GraphicsLevel_1_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_106__GraphicsLevel_1_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_106__GraphicsLevel_1_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_106__GraphicsLevel_1_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_107__GraphicsLevel_1_SclkFrequency_MASK 0xffffffff +#define DPM_TABLE_107__GraphicsLevel_1_SclkFrequency__SHIFT 0x0 +#define DPM_TABLE_108__GraphicsLevel_1_ActivityLevel_MASK 0xffff +#define DPM_TABLE_108__GraphicsLevel_1_ActivityLevel__SHIFT 0x0 +#define DPM_TABLE_108__GraphicsLevel_1_DeepSleepDivId_MASK 0xff0000 +#define DPM_TABLE_108__GraphicsLevel_1_DeepSleepDivId__SHIFT 0x10 +#define DPM_TABLE_108__GraphicsLevel_1_pcieDpmLevel_MASK 0xff000000 +#define DPM_TABLE_108__GraphicsLevel_1_pcieDpmLevel__SHIFT 0x18 +#define DPM_TABLE_109__GraphicsLevel_1_CgSpllFuncCntl3_MASK 0xffffffff +#define DPM_TABLE_109__GraphicsLevel_1_CgSpllFuncCntl3__SHIFT 0x0 +#define DPM_TABLE_110__GraphicsLevel_1_CgSpllFuncCntl4_MASK 0xffffffff +#define DPM_TABLE_110__GraphicsLevel_1_CgSpllFuncCntl4__SHIFT 0x0 +#define DPM_TABLE_111__GraphicsLevel_1_SpllSpreadSpectrum_MASK 0xffffffff +#define DPM_TABLE_111__GraphicsLevel_1_SpllSpreadSpectrum__SHIFT 0x0 +#define DPM_TABLE_112__GraphicsLevel_1_SpllSpreadSpectrum2_MASK 0xffffffff +#define DPM_TABLE_112__GraphicsLevel_1_SpllSpreadSpectrum2__SHIFT 0x0 +#define DPM_TABLE_113__GraphicsLevel_1_CcPwrDynRm_MASK 0xffffffff +#define DPM_TABLE_113__GraphicsLevel_1_CcPwrDynRm__SHIFT 0x0 +#define DPM_TABLE_114__GraphicsLevel_1_CcPwrDynRm1_MASK 0xffffffff +#define DPM_TABLE_114__GraphicsLevel_1_CcPwrDynRm1__SHIFT 0x0 +#define DPM_TABLE_115__GraphicsLevel_1_EnabledForThrottle_MASK 0xff +#define DPM_TABLE_115__GraphicsLevel_1_EnabledForThrottle__SHIFT 0x0 +#define DPM_TABLE_115__GraphicsLevel_1_EnabledForActivity_MASK 0xff00 +#define DPM_TABLE_115__GraphicsLevel_1_EnabledForActivity__SHIFT 0x8 +#define DPM_TABLE_115__GraphicsLevel_1_DisplayWatermark_MASK 0xff0000 +#define DPM_TABLE_115__GraphicsLevel_1_DisplayWatermark__SHIFT 0x10 +#define DPM_TABLE_115__GraphicsLevel_1_SclkDid_MASK 0xff000000 +#define DPM_TABLE_115__GraphicsLevel_1_SclkDid__SHIFT 0x18 +#define DPM_TABLE_116__GraphicsLevel_1_PowerThrottle_MASK 0xff +#define DPM_TABLE_116__GraphicsLevel_1_PowerThrottle__SHIFT 0x0 +#define DPM_TABLE_116__GraphicsLevel_1_VoltageDownHyst_MASK 0xff00 +#define DPM_TABLE_116__GraphicsLevel_1_VoltageDownHyst__SHIFT 0x8 +#define DPM_TABLE_116__GraphicsLevel_1_DownHyst_MASK 0xff0000 +#define DPM_TABLE_116__GraphicsLevel_1_DownHyst__SHIFT 0x10 +#define DPM_TABLE_116__GraphicsLevel_1_UpHyst_MASK 0xff000000 +#define DPM_TABLE_116__GraphicsLevel_1_UpHyst__SHIFT 0x18 +#define DPM_TABLE_117__GraphicsLevel_2_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_117__GraphicsLevel_2_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_117__GraphicsLevel_2_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_117__GraphicsLevel_2_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_117__GraphicsLevel_2_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_117__GraphicsLevel_2_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_117__GraphicsLevel_2_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_117__GraphicsLevel_2_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_118__GraphicsLevel_2_SclkFrequency_MASK 0xffffffff +#define DPM_TABLE_118__GraphicsLevel_2_SclkFrequency__SHIFT 0x0 +#define DPM_TABLE_119__GraphicsLevel_2_ActivityLevel_MASK 0xffff +#define DPM_TABLE_119__GraphicsLevel_2_ActivityLevel__SHIFT 0x0 +#define DPM_TABLE_119__GraphicsLevel_2_DeepSleepDivId_MASK 0xff0000 +#define DPM_TABLE_119__GraphicsLevel_2_DeepSleepDivId__SHIFT 0x10 +#define DPM_TABLE_119__GraphicsLevel_2_pcieDpmLevel_MASK 0xff000000 +#define DPM_TABLE_119__GraphicsLevel_2_pcieDpmLevel__SHIFT 0x18 +#define DPM_TABLE_120__GraphicsLevel_2_CgSpllFuncCntl3_MASK 0xffffffff +#define DPM_TABLE_120__GraphicsLevel_2_CgSpllFuncCntl3__SHIFT 0x0 +#define DPM_TABLE_121__GraphicsLevel_2_CgSpllFuncCntl4_MASK 0xffffffff +#define DPM_TABLE_121__GraphicsLevel_2_CgSpllFuncCntl4__SHIFT 0x0 +#define DPM_TABLE_122__GraphicsLevel_2_SpllSpreadSpectrum_MASK 0xffffffff +#define DPM_TABLE_122__GraphicsLevel_2_SpllSpreadSpectrum__SHIFT 0x0 +#define DPM_TABLE_123__GraphicsLevel_2_SpllSpreadSpectrum2_MASK 0xffffffff +#define DPM_TABLE_123__GraphicsLevel_2_SpllSpreadSpectrum2__SHIFT 0x0 +#define DPM_TABLE_124__GraphicsLevel_2_CcPwrDynRm_MASK 0xffffffff +#define DPM_TABLE_124__GraphicsLevel_2_CcPwrDynRm__SHIFT 0x0 +#define DPM_TABLE_125__GraphicsLevel_2_CcPwrDynRm1_MASK 0xffffffff +#define DPM_TABLE_125__GraphicsLevel_2_CcPwrDynRm1__SHIFT 0x0 +#define DPM_TABLE_126__GraphicsLevel_2_EnabledForThrottle_MASK 0xff +#define DPM_TABLE_126__GraphicsLevel_2_EnabledForThrottle__SHIFT 0x0 +#define DPM_TABLE_126__GraphicsLevel_2_EnabledForActivity_MASK 0xff00 +#define DPM_TABLE_126__GraphicsLevel_2_EnabledForActivity__SHIFT 0x8 +#define DPM_TABLE_126__GraphicsLevel_2_DisplayWatermark_MASK 0xff0000 +#define DPM_TABLE_126__GraphicsLevel_2_DisplayWatermark__SHIFT 0x10 +#define DPM_TABLE_126__GraphicsLevel_2_SclkDid_MASK 0xff000000 +#define DPM_TABLE_126__GraphicsLevel_2_SclkDid__SHIFT 0x18 +#define DPM_TABLE_127__GraphicsLevel_2_PowerThrottle_MASK 0xff +#define DPM_TABLE_127__GraphicsLevel_2_PowerThrottle__SHIFT 0x0 +#define DPM_TABLE_127__GraphicsLevel_2_VoltageDownHyst_MASK 0xff00 +#define DPM_TABLE_127__GraphicsLevel_2_VoltageDownHyst__SHIFT 0x8 +#define DPM_TABLE_127__GraphicsLevel_2_DownHyst_MASK 0xff0000 +#define DPM_TABLE_127__GraphicsLevel_2_DownHyst__SHIFT 0x10 +#define DPM_TABLE_127__GraphicsLevel_2_UpHyst_MASK 0xff000000 +#define DPM_TABLE_127__GraphicsLevel_2_UpHyst__SHIFT 0x18 +#define DPM_TABLE_128__GraphicsLevel_3_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_128__GraphicsLevel_3_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_128__GraphicsLevel_3_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_128__GraphicsLevel_3_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_128__GraphicsLevel_3_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_128__GraphicsLevel_3_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_128__GraphicsLevel_3_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_128__GraphicsLevel_3_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_129__GraphicsLevel_3_SclkFrequency_MASK 0xffffffff +#define DPM_TABLE_129__GraphicsLevel_3_SclkFrequency__SHIFT 0x0 +#define DPM_TABLE_130__GraphicsLevel_3_ActivityLevel_MASK 0xffff +#define DPM_TABLE_130__GraphicsLevel_3_ActivityLevel__SHIFT 0x0 +#define DPM_TABLE_130__GraphicsLevel_3_DeepSleepDivId_MASK 0xff0000 +#define DPM_TABLE_130__GraphicsLevel_3_DeepSleepDivId__SHIFT 0x10 +#define DPM_TABLE_130__GraphicsLevel_3_pcieDpmLevel_MASK 0xff000000 +#define DPM_TABLE_130__GraphicsLevel_3_pcieDpmLevel__SHIFT 0x18 +#define DPM_TABLE_131__GraphicsLevel_3_CgSpllFuncCntl3_MASK 0xffffffff +#define DPM_TABLE_131__GraphicsLevel_3_CgSpllFuncCntl3__SHIFT 0x0 +#define DPM_TABLE_132__GraphicsLevel_3_CgSpllFuncCntl4_MASK 0xffffffff +#define DPM_TABLE_132__GraphicsLevel_3_CgSpllFuncCntl4__SHIFT 0x0 +#define DPM_TABLE_133__GraphicsLevel_3_SpllSpreadSpectrum_MASK 0xffffffff +#define DPM_TABLE_133__GraphicsLevel_3_SpllSpreadSpectrum__SHIFT 0x0 +#define DPM_TABLE_134__GraphicsLevel_3_SpllSpreadSpectrum2_MASK 0xffffffff +#define DPM_TABLE_134__GraphicsLevel_3_SpllSpreadSpectrum2__SHIFT 0x0 +#define DPM_TABLE_135__GraphicsLevel_3_CcPwrDynRm_MASK 0xffffffff +#define DPM_TABLE_135__GraphicsLevel_3_CcPwrDynRm__SHIFT 0x0 +#define DPM_TABLE_136__GraphicsLevel_3_CcPwrDynRm1_MASK 0xffffffff +#define DPM_TABLE_136__GraphicsLevel_3_CcPwrDynRm1__SHIFT 0x0 +#define DPM_TABLE_137__GraphicsLevel_3_EnabledForThrottle_MASK 0xff +#define DPM_TABLE_137__GraphicsLevel_3_EnabledForThrottle__SHIFT 0x0 +#define DPM_TABLE_137__GraphicsLevel_3_EnabledForActivity_MASK 0xff00 +#define DPM_TABLE_137__GraphicsLevel_3_EnabledForActivity__SHIFT 0x8 +#define DPM_TABLE_137__GraphicsLevel_3_DisplayWatermark_MASK 0xff0000 +#define DPM_TABLE_137__GraphicsLevel_3_DisplayWatermark__SHIFT 0x10 +#define DPM_TABLE_137__GraphicsLevel_3_SclkDid_MASK 0xff000000 +#define DPM_TABLE_137__GraphicsLevel_3_SclkDid__SHIFT 0x18 +#define DPM_TABLE_138__GraphicsLevel_3_PowerThrottle_MASK 0xff +#define DPM_TABLE_138__GraphicsLevel_3_PowerThrottle__SHIFT 0x0 +#define DPM_TABLE_138__GraphicsLevel_3_VoltageDownHyst_MASK 0xff00 +#define DPM_TABLE_138__GraphicsLevel_3_VoltageDownHyst__SHIFT 0x8 +#define DPM_TABLE_138__GraphicsLevel_3_DownHyst_MASK 0xff0000 +#define DPM_TABLE_138__GraphicsLevel_3_DownHyst__SHIFT 0x10 +#define DPM_TABLE_138__GraphicsLevel_3_UpHyst_MASK 0xff000000 +#define DPM_TABLE_138__GraphicsLevel_3_UpHyst__SHIFT 0x18 +#define DPM_TABLE_139__GraphicsLevel_4_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_139__GraphicsLevel_4_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_139__GraphicsLevel_4_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_139__GraphicsLevel_4_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_139__GraphicsLevel_4_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_139__GraphicsLevel_4_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_139__GraphicsLevel_4_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_139__GraphicsLevel_4_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_140__GraphicsLevel_4_SclkFrequency_MASK 0xffffffff +#define DPM_TABLE_140__GraphicsLevel_4_SclkFrequency__SHIFT 0x0 +#define DPM_TABLE_141__GraphicsLevel_4_ActivityLevel_MASK 0xffff +#define DPM_TABLE_141__GraphicsLevel_4_ActivityLevel__SHIFT 0x0 +#define DPM_TABLE_141__GraphicsLevel_4_DeepSleepDivId_MASK 0xff0000 +#define DPM_TABLE_141__GraphicsLevel_4_DeepSleepDivId__SHIFT 0x10 +#define DPM_TABLE_141__GraphicsLevel_4_pcieDpmLevel_MASK 0xff000000 +#define DPM_TABLE_141__GraphicsLevel_4_pcieDpmLevel__SHIFT 0x18 +#define DPM_TABLE_142__GraphicsLevel_4_CgSpllFuncCntl3_MASK 0xffffffff +#define DPM_TABLE_142__GraphicsLevel_4_CgSpllFuncCntl3__SHIFT 0x0 +#define DPM_TABLE_143__GraphicsLevel_4_CgSpllFuncCntl4_MASK 0xffffffff +#define DPM_TABLE_143__GraphicsLevel_4_CgSpllFuncCntl4__SHIFT 0x0 +#define DPM_TABLE_144__GraphicsLevel_4_SpllSpreadSpectrum_MASK 0xffffffff +#define DPM_TABLE_144__GraphicsLevel_4_SpllSpreadSpectrum__SHIFT 0x0 +#define DPM_TABLE_145__GraphicsLevel_4_SpllSpreadSpectrum2_MASK 0xffffffff +#define DPM_TABLE_145__GraphicsLevel_4_SpllSpreadSpectrum2__SHIFT 0x0 +#define DPM_TABLE_146__GraphicsLevel_4_CcPwrDynRm_MASK 0xffffffff +#define DPM_TABLE_146__GraphicsLevel_4_CcPwrDynRm__SHIFT 0x0 +#define DPM_TABLE_147__GraphicsLevel_4_CcPwrDynRm1_MASK 0xffffffff +#define DPM_TABLE_147__GraphicsLevel_4_CcPwrDynRm1__SHIFT 0x0 +#define DPM_TABLE_148__GraphicsLevel_4_EnabledForThrottle_MASK 0xff +#define DPM_TABLE_148__GraphicsLevel_4_EnabledForThrottle__SHIFT 0x0 +#define DPM_TABLE_148__GraphicsLevel_4_EnabledForActivity_MASK 0xff00 +#define DPM_TABLE_148__GraphicsLevel_4_EnabledForActivity__SHIFT 0x8 +#define DPM_TABLE_148__GraphicsLevel_4_DisplayWatermark_MASK 0xff0000 +#define DPM_TABLE_148__GraphicsLevel_4_DisplayWatermark__SHIFT 0x10 +#define DPM_TABLE_148__GraphicsLevel_4_SclkDid_MASK 0xff000000 +#define DPM_TABLE_148__GraphicsLevel_4_SclkDid__SHIFT 0x18 +#define DPM_TABLE_149__GraphicsLevel_4_PowerThrottle_MASK 0xff +#define DPM_TABLE_149__GraphicsLevel_4_PowerThrottle__SHIFT 0x0 +#define DPM_TABLE_149__GraphicsLevel_4_VoltageDownHyst_MASK 0xff00 +#define DPM_TABLE_149__GraphicsLevel_4_VoltageDownHyst__SHIFT 0x8 +#define DPM_TABLE_149__GraphicsLevel_4_DownHyst_MASK 0xff0000 +#define DPM_TABLE_149__GraphicsLevel_4_DownHyst__SHIFT 0x10 +#define DPM_TABLE_149__GraphicsLevel_4_UpHyst_MASK 0xff000000 +#define DPM_TABLE_149__GraphicsLevel_4_UpHyst__SHIFT 0x18 +#define DPM_TABLE_150__GraphicsLevel_5_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_150__GraphicsLevel_5_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_150__GraphicsLevel_5_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_150__GraphicsLevel_5_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_150__GraphicsLevel_5_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_150__GraphicsLevel_5_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_150__GraphicsLevel_5_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_150__GraphicsLevel_5_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_151__GraphicsLevel_5_SclkFrequency_MASK 0xffffffff +#define DPM_TABLE_151__GraphicsLevel_5_SclkFrequency__SHIFT 0x0 +#define DPM_TABLE_152__GraphicsLevel_5_ActivityLevel_MASK 0xffff +#define DPM_TABLE_152__GraphicsLevel_5_ActivityLevel__SHIFT 0x0 +#define DPM_TABLE_152__GraphicsLevel_5_DeepSleepDivId_MASK 0xff0000 +#define DPM_TABLE_152__GraphicsLevel_5_DeepSleepDivId__SHIFT 0x10 +#define DPM_TABLE_152__GraphicsLevel_5_pcieDpmLevel_MASK 0xff000000 +#define DPM_TABLE_152__GraphicsLevel_5_pcieDpmLevel__SHIFT 0x18 +#define DPM_TABLE_153__GraphicsLevel_5_CgSpllFuncCntl3_MASK 0xffffffff +#define DPM_TABLE_153__GraphicsLevel_5_CgSpllFuncCntl3__SHIFT 0x0 +#define DPM_TABLE_154__GraphicsLevel_5_CgSpllFuncCntl4_MASK 0xffffffff +#define DPM_TABLE_154__GraphicsLevel_5_CgSpllFuncCntl4__SHIFT 0x0 +#define DPM_TABLE_155__GraphicsLevel_5_SpllSpreadSpectrum_MASK 0xffffffff +#define DPM_TABLE_155__GraphicsLevel_5_SpllSpreadSpectrum__SHIFT 0x0 +#define DPM_TABLE_156__GraphicsLevel_5_SpllSpreadSpectrum2_MASK 0xffffffff +#define DPM_TABLE_156__GraphicsLevel_5_SpllSpreadSpectrum2__SHIFT 0x0 +#define DPM_TABLE_157__GraphicsLevel_5_CcPwrDynRm_MASK 0xffffffff +#define DPM_TABLE_157__GraphicsLevel_5_CcPwrDynRm__SHIFT 0x0 +#define DPM_TABLE_158__GraphicsLevel_5_CcPwrDynRm1_MASK 0xffffffff +#define DPM_TABLE_158__GraphicsLevel_5_CcPwrDynRm1__SHIFT 0x0 +#define DPM_TABLE_159__GraphicsLevel_5_EnabledForThrottle_MASK 0xff +#define DPM_TABLE_159__GraphicsLevel_5_EnabledForThrottle__SHIFT 0x0 +#define DPM_TABLE_159__GraphicsLevel_5_EnabledForActivity_MASK 0xff00 +#define DPM_TABLE_159__GraphicsLevel_5_EnabledForActivity__SHIFT 0x8 +#define DPM_TABLE_159__GraphicsLevel_5_DisplayWatermark_MASK 0xff0000 +#define DPM_TABLE_159__GraphicsLevel_5_DisplayWatermark__SHIFT 0x10 +#define DPM_TABLE_159__GraphicsLevel_5_SclkDid_MASK 0xff000000 +#define DPM_TABLE_159__GraphicsLevel_5_SclkDid__SHIFT 0x18 +#define DPM_TABLE_160__GraphicsLevel_5_PowerThrottle_MASK 0xff +#define DPM_TABLE_160__GraphicsLevel_5_PowerThrottle__SHIFT 0x0 +#define DPM_TABLE_160__GraphicsLevel_5_VoltageDownHyst_MASK 0xff00 +#define DPM_TABLE_160__GraphicsLevel_5_VoltageDownHyst__SHIFT 0x8 +#define DPM_TABLE_160__GraphicsLevel_5_DownHyst_MASK 0xff0000 +#define DPM_TABLE_160__GraphicsLevel_5_DownHyst__SHIFT 0x10 +#define DPM_TABLE_160__GraphicsLevel_5_UpHyst_MASK 0xff000000 +#define DPM_TABLE_160__GraphicsLevel_5_UpHyst__SHIFT 0x18 +#define DPM_TABLE_161__GraphicsLevel_6_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_161__GraphicsLevel_6_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_161__GraphicsLevel_6_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_161__GraphicsLevel_6_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_161__GraphicsLevel_6_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_161__GraphicsLevel_6_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_161__GraphicsLevel_6_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_161__GraphicsLevel_6_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_162__GraphicsLevel_6_SclkFrequency_MASK 0xffffffff +#define DPM_TABLE_162__GraphicsLevel_6_SclkFrequency__SHIFT 0x0 +#define DPM_TABLE_163__GraphicsLevel_6_ActivityLevel_MASK 0xffff +#define DPM_TABLE_163__GraphicsLevel_6_ActivityLevel__SHIFT 0x0 +#define DPM_TABLE_163__GraphicsLevel_6_DeepSleepDivId_MASK 0xff0000 +#define DPM_TABLE_163__GraphicsLevel_6_DeepSleepDivId__SHIFT 0x10 +#define DPM_TABLE_163__GraphicsLevel_6_pcieDpmLevel_MASK 0xff000000 +#define DPM_TABLE_163__GraphicsLevel_6_pcieDpmLevel__SHIFT 0x18 +#define DPM_TABLE_164__GraphicsLevel_6_CgSpllFuncCntl3_MASK 0xffffffff +#define DPM_TABLE_164__GraphicsLevel_6_CgSpllFuncCntl3__SHIFT 0x0 +#define DPM_TABLE_165__GraphicsLevel_6_CgSpllFuncCntl4_MASK 0xffffffff +#define DPM_TABLE_165__GraphicsLevel_6_CgSpllFuncCntl4__SHIFT 0x0 +#define DPM_TABLE_166__GraphicsLevel_6_SpllSpreadSpectrum_MASK 0xffffffff +#define DPM_TABLE_166__GraphicsLevel_6_SpllSpreadSpectrum__SHIFT 0x0 +#define DPM_TABLE_167__GraphicsLevel_6_SpllSpreadSpectrum2_MASK 0xffffffff +#define DPM_TABLE_167__GraphicsLevel_6_SpllSpreadSpectrum2__SHIFT 0x0 +#define DPM_TABLE_168__GraphicsLevel_6_CcPwrDynRm_MASK 0xffffffff +#define DPM_TABLE_168__GraphicsLevel_6_CcPwrDynRm__SHIFT 0x0 +#define DPM_TABLE_169__GraphicsLevel_6_CcPwrDynRm1_MASK 0xffffffff +#define DPM_TABLE_169__GraphicsLevel_6_CcPwrDynRm1__SHIFT 0x0 +#define DPM_TABLE_170__GraphicsLevel_6_EnabledForThrottle_MASK 0xff +#define DPM_TABLE_170__GraphicsLevel_6_EnabledForThrottle__SHIFT 0x0 +#define DPM_TABLE_170__GraphicsLevel_6_EnabledForActivity_MASK 0xff00 +#define DPM_TABLE_170__GraphicsLevel_6_EnabledForActivity__SHIFT 0x8 +#define DPM_TABLE_170__GraphicsLevel_6_DisplayWatermark_MASK 0xff0000 +#define DPM_TABLE_170__GraphicsLevel_6_DisplayWatermark__SHIFT 0x10 +#define DPM_TABLE_170__GraphicsLevel_6_SclkDid_MASK 0xff000000 +#define DPM_TABLE_170__GraphicsLevel_6_SclkDid__SHIFT 0x18 +#define DPM_TABLE_171__GraphicsLevel_6_PowerThrottle_MASK 0xff +#define DPM_TABLE_171__GraphicsLevel_6_PowerThrottle__SHIFT 0x0 +#define DPM_TABLE_171__GraphicsLevel_6_VoltageDownHyst_MASK 0xff00 +#define DPM_TABLE_171__GraphicsLevel_6_VoltageDownHyst__SHIFT 0x8 +#define DPM_TABLE_171__GraphicsLevel_6_DownHyst_MASK 0xff0000 +#define DPM_TABLE_171__GraphicsLevel_6_DownHyst__SHIFT 0x10 +#define DPM_TABLE_171__GraphicsLevel_6_UpHyst_MASK 0xff000000 +#define DPM_TABLE_171__GraphicsLevel_6_UpHyst__SHIFT 0x18 +#define DPM_TABLE_172__GraphicsLevel_7_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_172__GraphicsLevel_7_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_172__GraphicsLevel_7_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_172__GraphicsLevel_7_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_172__GraphicsLevel_7_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_172__GraphicsLevel_7_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_172__GraphicsLevel_7_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_172__GraphicsLevel_7_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_173__GraphicsLevel_7_SclkFrequency_MASK 0xffffffff +#define DPM_TABLE_173__GraphicsLevel_7_SclkFrequency__SHIFT 0x0 +#define DPM_TABLE_174__GraphicsLevel_7_ActivityLevel_MASK 0xffff +#define DPM_TABLE_174__GraphicsLevel_7_ActivityLevel__SHIFT 0x0 +#define DPM_TABLE_174__GraphicsLevel_7_DeepSleepDivId_MASK 0xff0000 +#define DPM_TABLE_174__GraphicsLevel_7_DeepSleepDivId__SHIFT 0x10 +#define DPM_TABLE_174__GraphicsLevel_7_pcieDpmLevel_MASK 0xff000000 +#define DPM_TABLE_174__GraphicsLevel_7_pcieDpmLevel__SHIFT 0x18 +#define DPM_TABLE_175__GraphicsLevel_7_CgSpllFuncCntl3_MASK 0xffffffff +#define DPM_TABLE_175__GraphicsLevel_7_CgSpllFuncCntl3__SHIFT 0x0 +#define DPM_TABLE_176__GraphicsLevel_7_CgSpllFuncCntl4_MASK 0xffffffff +#define DPM_TABLE_176__GraphicsLevel_7_CgSpllFuncCntl4__SHIFT 0x0 +#define DPM_TABLE_177__GraphicsLevel_7_SpllSpreadSpectrum_MASK 0xffffffff +#define DPM_TABLE_177__GraphicsLevel_7_SpllSpreadSpectrum__SHIFT 0x0 +#define DPM_TABLE_178__GraphicsLevel_7_SpllSpreadSpectrum2_MASK 0xffffffff +#define DPM_TABLE_178__GraphicsLevel_7_SpllSpreadSpectrum2__SHIFT 0x0 +#define DPM_TABLE_179__GraphicsLevel_7_CcPwrDynRm_MASK 0xffffffff +#define DPM_TABLE_179__GraphicsLevel_7_CcPwrDynRm__SHIFT 0x0 +#define DPM_TABLE_180__GraphicsLevel_7_CcPwrDynRm1_MASK 0xffffffff +#define DPM_TABLE_180__GraphicsLevel_7_CcPwrDynRm1__SHIFT 0x0 +#define DPM_TABLE_181__GraphicsLevel_7_EnabledForThrottle_MASK 0xff +#define DPM_TABLE_181__GraphicsLevel_7_EnabledForThrottle__SHIFT 0x0 +#define DPM_TABLE_181__GraphicsLevel_7_EnabledForActivity_MASK 0xff00 +#define DPM_TABLE_181__GraphicsLevel_7_EnabledForActivity__SHIFT 0x8 +#define DPM_TABLE_181__GraphicsLevel_7_DisplayWatermark_MASK 0xff0000 +#define DPM_TABLE_181__GraphicsLevel_7_DisplayWatermark__SHIFT 0x10 +#define DPM_TABLE_181__GraphicsLevel_7_SclkDid_MASK 0xff000000 +#define DPM_TABLE_181__GraphicsLevel_7_SclkDid__SHIFT 0x18 +#define DPM_TABLE_182__GraphicsLevel_7_PowerThrottle_MASK 0xff +#define DPM_TABLE_182__GraphicsLevel_7_PowerThrottle__SHIFT 0x0 +#define DPM_TABLE_182__GraphicsLevel_7_VoltageDownHyst_MASK 0xff00 +#define DPM_TABLE_182__GraphicsLevel_7_VoltageDownHyst__SHIFT 0x8 +#define DPM_TABLE_182__GraphicsLevel_7_DownHyst_MASK 0xff0000 +#define DPM_TABLE_182__GraphicsLevel_7_DownHyst__SHIFT 0x10 +#define DPM_TABLE_182__GraphicsLevel_7_UpHyst_MASK 0xff000000 +#define DPM_TABLE_182__GraphicsLevel_7_UpHyst__SHIFT 0x18 +#define DPM_TABLE_183__MemoryACPILevel_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_183__MemoryACPILevel_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_183__MemoryACPILevel_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_183__MemoryACPILevel_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_183__MemoryACPILevel_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_183__MemoryACPILevel_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_183__MemoryACPILevel_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_183__MemoryACPILevel_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_184__MemoryACPILevel_MinMvdd_MASK 0xffffffff +#define DPM_TABLE_184__MemoryACPILevel_MinMvdd__SHIFT 0x0 +#define DPM_TABLE_185__MemoryACPILevel_MclkFrequency_MASK 0xffffffff +#define DPM_TABLE_185__MemoryACPILevel_MclkFrequency__SHIFT 0x0 +#define DPM_TABLE_186__MemoryACPILevel_EnabledForActivity_MASK 0xff +#define DPM_TABLE_186__MemoryACPILevel_EnabledForActivity__SHIFT 0x0 +#define DPM_TABLE_186__MemoryACPILevel_EnabledForThrottle_MASK 0xff00 +#define DPM_TABLE_186__MemoryACPILevel_EnabledForThrottle__SHIFT 0x8 +#define DPM_TABLE_186__MemoryACPILevel_FreqRange_MASK 0xff0000 +#define DPM_TABLE_186__MemoryACPILevel_FreqRange__SHIFT 0x10 +#define DPM_TABLE_186__MemoryACPILevel_StutterEnable_MASK 0xff000000 +#define DPM_TABLE_186__MemoryACPILevel_StutterEnable__SHIFT 0x18 +#define DPM_TABLE_187__MemoryACPILevel_padding_MASK 0xff +#define DPM_TABLE_187__MemoryACPILevel_padding__SHIFT 0x0 +#define DPM_TABLE_187__MemoryACPILevel_VoltageDownHyst_MASK 0xff00 +#define DPM_TABLE_187__MemoryACPILevel_VoltageDownHyst__SHIFT 0x8 +#define DPM_TABLE_187__MemoryACPILevel_DownHyst_MASK 0xff0000 +#define DPM_TABLE_187__MemoryACPILevel_DownHyst__SHIFT 0x10 +#define DPM_TABLE_187__MemoryACPILevel_UpHyst_MASK 0xff000000 +#define DPM_TABLE_187__MemoryACPILevel_UpHyst__SHIFT 0x18 +#define DPM_TABLE_188__MemoryACPILevel_MclkDivider_MASK 0xff +#define DPM_TABLE_188__MemoryACPILevel_MclkDivider__SHIFT 0x0 +#define DPM_TABLE_188__MemoryACPILevel_DisplayWatermark_MASK 0xff00 +#define DPM_TABLE_188__MemoryACPILevel_DisplayWatermark__SHIFT 0x8 +#define DPM_TABLE_188__MemoryACPILevel_ActivityLevel_MASK 0xffff0000 +#define DPM_TABLE_188__MemoryACPILevel_ActivityLevel__SHIFT 0x10 +#define DPM_TABLE_189__MemoryLevel_0_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_189__MemoryLevel_0_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_189__MemoryLevel_0_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_189__MemoryLevel_0_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_189__MemoryLevel_0_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_189__MemoryLevel_0_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_189__MemoryLevel_0_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_189__MemoryLevel_0_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_190__MemoryLevel_0_MinMvdd_MASK 0xffffffff +#define DPM_TABLE_190__MemoryLevel_0_MinMvdd__SHIFT 0x0 +#define DPM_TABLE_191__MemoryLevel_0_MclkFrequency_MASK 0xffffffff +#define DPM_TABLE_191__MemoryLevel_0_MclkFrequency__SHIFT 0x0 +#define DPM_TABLE_192__MemoryLevel_0_EnabledForActivity_MASK 0xff +#define DPM_TABLE_192__MemoryLevel_0_EnabledForActivity__SHIFT 0x0 +#define DPM_TABLE_192__MemoryLevel_0_EnabledForThrottle_MASK 0xff00 +#define DPM_TABLE_192__MemoryLevel_0_EnabledForThrottle__SHIFT 0x8 +#define DPM_TABLE_192__MemoryLevel_0_FreqRange_MASK 0xff0000 +#define DPM_TABLE_192__MemoryLevel_0_FreqRange__SHIFT 0x10 +#define DPM_TABLE_192__MemoryLevel_0_StutterEnable_MASK 0xff000000 +#define DPM_TABLE_192__MemoryLevel_0_StutterEnable__SHIFT 0x18 +#define DPM_TABLE_193__MemoryLevel_0_padding_MASK 0xff +#define DPM_TABLE_193__MemoryLevel_0_padding__SHIFT 0x0 +#define DPM_TABLE_193__MemoryLevel_0_VoltageDownHyst_MASK 0xff00 +#define DPM_TABLE_193__MemoryLevel_0_VoltageDownHyst__SHIFT 0x8 +#define DPM_TABLE_193__MemoryLevel_0_DownHyst_MASK 0xff0000 +#define DPM_TABLE_193__MemoryLevel_0_DownHyst__SHIFT 0x10 +#define DPM_TABLE_193__MemoryLevel_0_UpHyst_MASK 0xff000000 +#define DPM_TABLE_193__MemoryLevel_0_UpHyst__SHIFT 0x18 +#define DPM_TABLE_194__MemoryLevel_0_MclkDivider_MASK 0xff +#define DPM_TABLE_194__MemoryLevel_0_MclkDivider__SHIFT 0x0 +#define DPM_TABLE_194__MemoryLevel_0_DisplayWatermark_MASK 0xff00 +#define DPM_TABLE_194__MemoryLevel_0_DisplayWatermark__SHIFT 0x8 +#define DPM_TABLE_194__MemoryLevel_0_ActivityLevel_MASK 0xffff0000 +#define DPM_TABLE_194__MemoryLevel_0_ActivityLevel__SHIFT 0x10 +#define DPM_TABLE_195__MemoryLevel_1_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_195__MemoryLevel_1_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_195__MemoryLevel_1_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_195__MemoryLevel_1_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_195__MemoryLevel_1_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_195__MemoryLevel_1_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_195__MemoryLevel_1_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_195__MemoryLevel_1_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_196__MemoryLevel_1_MinMvdd_MASK 0xffffffff +#define DPM_TABLE_196__MemoryLevel_1_MinMvdd__SHIFT 0x0 +#define DPM_TABLE_197__MemoryLevel_1_MclkFrequency_MASK 0xffffffff +#define DPM_TABLE_197__MemoryLevel_1_MclkFrequency__SHIFT 0x0 +#define DPM_TABLE_198__MemoryLevel_1_EnabledForActivity_MASK 0xff +#define DPM_TABLE_198__MemoryLevel_1_EnabledForActivity__SHIFT 0x0 +#define DPM_TABLE_198__MemoryLevel_1_EnabledForThrottle_MASK 0xff00 +#define DPM_TABLE_198__MemoryLevel_1_EnabledForThrottle__SHIFT 0x8 +#define DPM_TABLE_198__MemoryLevel_1_FreqRange_MASK 0xff0000 +#define DPM_TABLE_198__MemoryLevel_1_FreqRange__SHIFT 0x10 +#define DPM_TABLE_198__MemoryLevel_1_StutterEnable_MASK 0xff000000 +#define DPM_TABLE_198__MemoryLevel_1_StutterEnable__SHIFT 0x18 +#define DPM_TABLE_199__MemoryLevel_1_padding_MASK 0xff +#define DPM_TABLE_199__MemoryLevel_1_padding__SHIFT 0x0 +#define DPM_TABLE_199__MemoryLevel_1_VoltageDownHyst_MASK 0xff00 +#define DPM_TABLE_199__MemoryLevel_1_VoltageDownHyst__SHIFT 0x8 +#define DPM_TABLE_199__MemoryLevel_1_DownHyst_MASK 0xff0000 +#define DPM_TABLE_199__MemoryLevel_1_DownHyst__SHIFT 0x10 +#define DPM_TABLE_199__MemoryLevel_1_UpHyst_MASK 0xff000000 +#define DPM_TABLE_199__MemoryLevel_1_UpHyst__SHIFT 0x18 +#define DPM_TABLE_200__MemoryLevel_1_MclkDivider_MASK 0xff +#define DPM_TABLE_200__MemoryLevel_1_MclkDivider__SHIFT 0x0 +#define DPM_TABLE_200__MemoryLevel_1_DisplayWatermark_MASK 0xff00 +#define DPM_TABLE_200__MemoryLevel_1_DisplayWatermark__SHIFT 0x8 +#define DPM_TABLE_200__MemoryLevel_1_ActivityLevel_MASK 0xffff0000 +#define DPM_TABLE_200__MemoryLevel_1_ActivityLevel__SHIFT 0x10 +#define DPM_TABLE_201__MemoryLevel_2_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_201__MemoryLevel_2_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_201__MemoryLevel_2_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_201__MemoryLevel_2_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_201__MemoryLevel_2_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_201__MemoryLevel_2_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_201__MemoryLevel_2_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_201__MemoryLevel_2_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_202__MemoryLevel_2_MinMvdd_MASK 0xffffffff +#define DPM_TABLE_202__MemoryLevel_2_MinMvdd__SHIFT 0x0 +#define DPM_TABLE_203__MemoryLevel_2_MclkFrequency_MASK 0xffffffff +#define DPM_TABLE_203__MemoryLevel_2_MclkFrequency__SHIFT 0x0 +#define DPM_TABLE_204__MemoryLevel_2_EnabledForActivity_MASK 0xff +#define DPM_TABLE_204__MemoryLevel_2_EnabledForActivity__SHIFT 0x0 +#define DPM_TABLE_204__MemoryLevel_2_EnabledForThrottle_MASK 0xff00 +#define DPM_TABLE_204__MemoryLevel_2_EnabledForThrottle__SHIFT 0x8 +#define DPM_TABLE_204__MemoryLevel_2_FreqRange_MASK 0xff0000 +#define DPM_TABLE_204__MemoryLevel_2_FreqRange__SHIFT 0x10 +#define DPM_TABLE_204__MemoryLevel_2_StutterEnable_MASK 0xff000000 +#define DPM_TABLE_204__MemoryLevel_2_StutterEnable__SHIFT 0x18 +#define DPM_TABLE_205__MemoryLevel_2_padding_MASK 0xff +#define DPM_TABLE_205__MemoryLevel_2_padding__SHIFT 0x0 +#define DPM_TABLE_205__MemoryLevel_2_VoltageDownHyst_MASK 0xff00 +#define DPM_TABLE_205__MemoryLevel_2_VoltageDownHyst__SHIFT 0x8 +#define DPM_TABLE_205__MemoryLevel_2_DownHyst_MASK 0xff0000 +#define DPM_TABLE_205__MemoryLevel_2_DownHyst__SHIFT 0x10 +#define DPM_TABLE_205__MemoryLevel_2_UpHyst_MASK 0xff000000 +#define DPM_TABLE_205__MemoryLevel_2_UpHyst__SHIFT 0x18 +#define DPM_TABLE_206__MemoryLevel_2_MclkDivider_MASK 0xff +#define DPM_TABLE_206__MemoryLevel_2_MclkDivider__SHIFT 0x0 +#define DPM_TABLE_206__MemoryLevel_2_DisplayWatermark_MASK 0xff00 +#define DPM_TABLE_206__MemoryLevel_2_DisplayWatermark__SHIFT 0x8 +#define DPM_TABLE_206__MemoryLevel_2_ActivityLevel_MASK 0xffff0000 +#define DPM_TABLE_206__MemoryLevel_2_ActivityLevel__SHIFT 0x10 +#define DPM_TABLE_207__MemoryLevel_3_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_207__MemoryLevel_3_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_207__MemoryLevel_3_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_207__MemoryLevel_3_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_207__MemoryLevel_3_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_207__MemoryLevel_3_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_207__MemoryLevel_3_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_207__MemoryLevel_3_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_208__MemoryLevel_3_MinMvdd_MASK 0xffffffff +#define DPM_TABLE_208__MemoryLevel_3_MinMvdd__SHIFT 0x0 +#define DPM_TABLE_209__MemoryLevel_3_MclkFrequency_MASK 0xffffffff +#define DPM_TABLE_209__MemoryLevel_3_MclkFrequency__SHIFT 0x0 +#define DPM_TABLE_210__MemoryLevel_3_EnabledForActivity_MASK 0xff +#define DPM_TABLE_210__MemoryLevel_3_EnabledForActivity__SHIFT 0x0 +#define DPM_TABLE_210__MemoryLevel_3_EnabledForThrottle_MASK 0xff00 +#define DPM_TABLE_210__MemoryLevel_3_EnabledForThrottle__SHIFT 0x8 +#define DPM_TABLE_210__MemoryLevel_3_FreqRange_MASK 0xff0000 +#define DPM_TABLE_210__MemoryLevel_3_FreqRange__SHIFT 0x10 +#define DPM_TABLE_210__MemoryLevel_3_StutterEnable_MASK 0xff000000 +#define DPM_TABLE_210__MemoryLevel_3_StutterEnable__SHIFT 0x18 +#define DPM_TABLE_211__MemoryLevel_3_padding_MASK 0xff +#define DPM_TABLE_211__MemoryLevel_3_padding__SHIFT 0x0 +#define DPM_TABLE_211__MemoryLevel_3_VoltageDownHyst_MASK 0xff00 +#define DPM_TABLE_211__MemoryLevel_3_VoltageDownHyst__SHIFT 0x8 +#define DPM_TABLE_211__MemoryLevel_3_DownHyst_MASK 0xff0000 +#define DPM_TABLE_211__MemoryLevel_3_DownHyst__SHIFT 0x10 +#define DPM_TABLE_211__MemoryLevel_3_UpHyst_MASK 0xff000000 +#define DPM_TABLE_211__MemoryLevel_3_UpHyst__SHIFT 0x18 +#define DPM_TABLE_212__MemoryLevel_3_MclkDivider_MASK 0xff +#define DPM_TABLE_212__MemoryLevel_3_MclkDivider__SHIFT 0x0 +#define DPM_TABLE_212__MemoryLevel_3_DisplayWatermark_MASK 0xff00 +#define DPM_TABLE_212__MemoryLevel_3_DisplayWatermark__SHIFT 0x8 +#define DPM_TABLE_212__MemoryLevel_3_ActivityLevel_MASK 0xffff0000 +#define DPM_TABLE_212__MemoryLevel_3_ActivityLevel__SHIFT 0x10 +#define DPM_TABLE_213__LinkLevel_0_SPC_MASK 0xff +#define DPM_TABLE_213__LinkLevel_0_SPC__SHIFT 0x0 +#define DPM_TABLE_213__LinkLevel_0_EnabledForActivity_MASK 0xff00 +#define DPM_TABLE_213__LinkLevel_0_EnabledForActivity__SHIFT 0x8 +#define DPM_TABLE_213__LinkLevel_0_PcieLaneCount_MASK 0xff0000 +#define DPM_TABLE_213__LinkLevel_0_PcieLaneCount__SHIFT 0x10 +#define DPM_TABLE_213__LinkLevel_0_PcieGenSpeed_MASK 0xff000000 +#define DPM_TABLE_213__LinkLevel_0_PcieGenSpeed__SHIFT 0x18 +#define DPM_TABLE_214__LinkLevel_0_DownThreshold_MASK 0xffffffff +#define DPM_TABLE_214__LinkLevel_0_DownThreshold__SHIFT 0x0 +#define DPM_TABLE_215__LinkLevel_0_UpThreshold_MASK 0xffffffff +#define DPM_TABLE_215__LinkLevel_0_UpThreshold__SHIFT 0x0 +#define DPM_TABLE_216__LinkLevel_0_Reserved_MASK 0xffffffff +#define DPM_TABLE_216__LinkLevel_0_Reserved__SHIFT 0x0 +#define DPM_TABLE_217__LinkLevel_1_SPC_MASK 0xff +#define DPM_TABLE_217__LinkLevel_1_SPC__SHIFT 0x0 +#define DPM_TABLE_217__LinkLevel_1_EnabledForActivity_MASK 0xff00 +#define DPM_TABLE_217__LinkLevel_1_EnabledForActivity__SHIFT 0x8 +#define DPM_TABLE_217__LinkLevel_1_PcieLaneCount_MASK 0xff0000 +#define DPM_TABLE_217__LinkLevel_1_PcieLaneCount__SHIFT 0x10 +#define DPM_TABLE_217__LinkLevel_1_PcieGenSpeed_MASK 0xff000000 +#define DPM_TABLE_217__LinkLevel_1_PcieGenSpeed__SHIFT 0x18 +#define DPM_TABLE_218__LinkLevel_1_DownThreshold_MASK 0xffffffff +#define DPM_TABLE_218__LinkLevel_1_DownThreshold__SHIFT 0x0 +#define DPM_TABLE_219__LinkLevel_1_UpThreshold_MASK 0xffffffff +#define DPM_TABLE_219__LinkLevel_1_UpThreshold__SHIFT 0x0 +#define DPM_TABLE_220__LinkLevel_1_Reserved_MASK 0xffffffff +#define DPM_TABLE_220__LinkLevel_1_Reserved__SHIFT 0x0 +#define DPM_TABLE_221__LinkLevel_2_SPC_MASK 0xff +#define DPM_TABLE_221__LinkLevel_2_SPC__SHIFT 0x0 +#define DPM_TABLE_221__LinkLevel_2_EnabledForActivity_MASK 0xff00 +#define DPM_TABLE_221__LinkLevel_2_EnabledForActivity__SHIFT 0x8 +#define DPM_TABLE_221__LinkLevel_2_PcieLaneCount_MASK 0xff0000 +#define DPM_TABLE_221__LinkLevel_2_PcieLaneCount__SHIFT 0x10 +#define DPM_TABLE_221__LinkLevel_2_PcieGenSpeed_MASK 0xff000000 +#define DPM_TABLE_221__LinkLevel_2_PcieGenSpeed__SHIFT 0x18 +#define DPM_TABLE_222__LinkLevel_2_DownThreshold_MASK 0xffffffff +#define DPM_TABLE_222__LinkLevel_2_DownThreshold__SHIFT 0x0 +#define DPM_TABLE_223__LinkLevel_2_UpThreshold_MASK 0xffffffff +#define DPM_TABLE_223__LinkLevel_2_UpThreshold__SHIFT 0x0 +#define DPM_TABLE_224__LinkLevel_2_Reserved_MASK 0xffffffff +#define DPM_TABLE_224__LinkLevel_2_Reserved__SHIFT 0x0 +#define DPM_TABLE_225__LinkLevel_3_SPC_MASK 0xff +#define DPM_TABLE_225__LinkLevel_3_SPC__SHIFT 0x0 +#define DPM_TABLE_225__LinkLevel_3_EnabledForActivity_MASK 0xff00 +#define DPM_TABLE_225__LinkLevel_3_EnabledForActivity__SHIFT 0x8 +#define DPM_TABLE_225__LinkLevel_3_PcieLaneCount_MASK 0xff0000 +#define DPM_TABLE_225__LinkLevel_3_PcieLaneCount__SHIFT 0x10 +#define DPM_TABLE_225__LinkLevel_3_PcieGenSpeed_MASK 0xff000000 +#define DPM_TABLE_225__LinkLevel_3_PcieGenSpeed__SHIFT 0x18 +#define DPM_TABLE_226__LinkLevel_3_DownThreshold_MASK 0xffffffff +#define DPM_TABLE_226__LinkLevel_3_DownThreshold__SHIFT 0x0 +#define DPM_TABLE_227__LinkLevel_3_UpThreshold_MASK 0xffffffff +#define DPM_TABLE_227__LinkLevel_3_UpThreshold__SHIFT 0x0 +#define DPM_TABLE_228__LinkLevel_3_Reserved_MASK 0xffffffff +#define DPM_TABLE_228__LinkLevel_3_Reserved__SHIFT 0x0 +#define DPM_TABLE_229__LinkLevel_4_SPC_MASK 0xff +#define DPM_TABLE_229__LinkLevel_4_SPC__SHIFT 0x0 +#define DPM_TABLE_229__LinkLevel_4_EnabledForActivity_MASK 0xff00 +#define DPM_TABLE_229__LinkLevel_4_EnabledForActivity__SHIFT 0x8 +#define DPM_TABLE_229__LinkLevel_4_PcieLaneCount_MASK 0xff0000 +#define DPM_TABLE_229__LinkLevel_4_PcieLaneCount__SHIFT 0x10 +#define DPM_TABLE_229__LinkLevel_4_PcieGenSpeed_MASK 0xff000000 +#define DPM_TABLE_229__LinkLevel_4_PcieGenSpeed__SHIFT 0x18 +#define DPM_TABLE_230__LinkLevel_4_DownThreshold_MASK 0xffffffff +#define DPM_TABLE_230__LinkLevel_4_DownThreshold__SHIFT 0x0 +#define DPM_TABLE_231__LinkLevel_4_UpThreshold_MASK 0xffffffff +#define DPM_TABLE_231__LinkLevel_4_UpThreshold__SHIFT 0x0 +#define DPM_TABLE_232__LinkLevel_4_Reserved_MASK 0xffffffff +#define DPM_TABLE_232__LinkLevel_4_Reserved__SHIFT 0x0 +#define DPM_TABLE_233__LinkLevel_5_SPC_MASK 0xff +#define DPM_TABLE_233__LinkLevel_5_SPC__SHIFT 0x0 +#define DPM_TABLE_233__LinkLevel_5_EnabledForActivity_MASK 0xff00 +#define DPM_TABLE_233__LinkLevel_5_EnabledForActivity__SHIFT 0x8 +#define DPM_TABLE_233__LinkLevel_5_PcieLaneCount_MASK 0xff0000 +#define DPM_TABLE_233__LinkLevel_5_PcieLaneCount__SHIFT 0x10 +#define DPM_TABLE_233__LinkLevel_5_PcieGenSpeed_MASK 0xff000000 +#define DPM_TABLE_233__LinkLevel_5_PcieGenSpeed__SHIFT 0x18 +#define DPM_TABLE_234__LinkLevel_5_DownThreshold_MASK 0xffffffff +#define DPM_TABLE_234__LinkLevel_5_DownThreshold__SHIFT 0x0 +#define DPM_TABLE_235__LinkLevel_5_UpThreshold_MASK 0xffffffff +#define DPM_TABLE_235__LinkLevel_5_UpThreshold__SHIFT 0x0 +#define DPM_TABLE_236__LinkLevel_5_Reserved_MASK 0xffffffff +#define DPM_TABLE_236__LinkLevel_5_Reserved__SHIFT 0x0 +#define DPM_TABLE_237__LinkLevel_6_SPC_MASK 0xff +#define DPM_TABLE_237__LinkLevel_6_SPC__SHIFT 0x0 +#define DPM_TABLE_237__LinkLevel_6_EnabledForActivity_MASK 0xff00 +#define DPM_TABLE_237__LinkLevel_6_EnabledForActivity__SHIFT 0x8 +#define DPM_TABLE_237__LinkLevel_6_PcieLaneCount_MASK 0xff0000 +#define DPM_TABLE_237__LinkLevel_6_PcieLaneCount__SHIFT 0x10 +#define DPM_TABLE_237__LinkLevel_6_PcieGenSpeed_MASK 0xff000000 +#define DPM_TABLE_237__LinkLevel_6_PcieGenSpeed__SHIFT 0x18 +#define DPM_TABLE_238__LinkLevel_6_DownThreshold_MASK 0xffffffff +#define DPM_TABLE_238__LinkLevel_6_DownThreshold__SHIFT 0x0 +#define DPM_TABLE_239__LinkLevel_6_UpThreshold_MASK 0xffffffff +#define DPM_TABLE_239__LinkLevel_6_UpThreshold__SHIFT 0x0 +#define DPM_TABLE_240__LinkLevel_6_Reserved_MASK 0xffffffff +#define DPM_TABLE_240__LinkLevel_6_Reserved__SHIFT 0x0 +#define DPM_TABLE_241__LinkLevel_7_SPC_MASK 0xff +#define DPM_TABLE_241__LinkLevel_7_SPC__SHIFT 0x0 +#define DPM_TABLE_241__LinkLevel_7_EnabledForActivity_MASK 0xff00 +#define DPM_TABLE_241__LinkLevel_7_EnabledForActivity__SHIFT 0x8 +#define DPM_TABLE_241__LinkLevel_7_PcieLaneCount_MASK 0xff0000 +#define DPM_TABLE_241__LinkLevel_7_PcieLaneCount__SHIFT 0x10 +#define DPM_TABLE_241__LinkLevel_7_PcieGenSpeed_MASK 0xff000000 +#define DPM_TABLE_241__LinkLevel_7_PcieGenSpeed__SHIFT 0x18 +#define DPM_TABLE_242__LinkLevel_7_DownThreshold_MASK 0xffffffff +#define DPM_TABLE_242__LinkLevel_7_DownThreshold__SHIFT 0x0 +#define DPM_TABLE_243__LinkLevel_7_UpThreshold_MASK 0xffffffff +#define DPM_TABLE_243__LinkLevel_7_UpThreshold__SHIFT 0x0 +#define DPM_TABLE_244__LinkLevel_7_Reserved_MASK 0xffffffff +#define DPM_TABLE_244__LinkLevel_7_Reserved__SHIFT 0x0 +#define DPM_TABLE_245__ACPILevel_Flags_MASK 0xffffffff +#define DPM_TABLE_245__ACPILevel_Flags__SHIFT 0x0 +#define DPM_TABLE_246__ACPILevel_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_246__ACPILevel_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_246__ACPILevel_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_246__ACPILevel_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_246__ACPILevel_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_246__ACPILevel_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_246__ACPILevel_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_246__ACPILevel_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_247__ACPILevel_SclkFrequency_MASK 0xffffffff +#define DPM_TABLE_247__ACPILevel_SclkFrequency__SHIFT 0x0 +#define DPM_TABLE_248__ACPILevel_padding_MASK 0xff +#define DPM_TABLE_248__ACPILevel_padding__SHIFT 0x0 +#define DPM_TABLE_248__ACPILevel_DeepSleepDivId_MASK 0xff00 +#define DPM_TABLE_248__ACPILevel_DeepSleepDivId__SHIFT 0x8 +#define DPM_TABLE_248__ACPILevel_DisplayWatermark_MASK 0xff0000 +#define DPM_TABLE_248__ACPILevel_DisplayWatermark__SHIFT 0x10 +#define DPM_TABLE_248__ACPILevel_SclkDid_MASK 0xff000000 +#define DPM_TABLE_248__ACPILevel_SclkDid__SHIFT 0x18 +#define DPM_TABLE_249__ACPILevel_CgSpllFuncCntl_MASK 0xffffffff +#define DPM_TABLE_249__ACPILevel_CgSpllFuncCntl__SHIFT 0x0 +#define DPM_TABLE_250__ACPILevel_CgSpllFuncCntl2_MASK 0xffffffff +#define DPM_TABLE_250__ACPILevel_CgSpllFuncCntl2__SHIFT 0x0 +#define DPM_TABLE_251__ACPILevel_CgSpllFuncCntl3_MASK 0xffffffff +#define DPM_TABLE_251__ACPILevel_CgSpllFuncCntl3__SHIFT 0x0 +#define DPM_TABLE_252__ACPILevel_CgSpllFuncCntl4_MASK 0xffffffff +#define DPM_TABLE_252__ACPILevel_CgSpllFuncCntl4__SHIFT 0x0 +#define DPM_TABLE_253__ACPILevel_SpllSpreadSpectrum_MASK 0xffffffff +#define DPM_TABLE_253__ACPILevel_SpllSpreadSpectrum__SHIFT 0x0 +#define DPM_TABLE_254__ACPILevel_SpllSpreadSpectrum2_MASK 0xffffffff +#define DPM_TABLE_254__ACPILevel_SpllSpreadSpectrum2__SHIFT 0x0 +#define DPM_TABLE_255__ACPILevel_CcPwrDynRm_MASK 0xffffffff +#define DPM_TABLE_255__ACPILevel_CcPwrDynRm__SHIFT 0x0 +#define DPM_TABLE_256__ACPILevel_CcPwrDynRm1_MASK 0xffffffff +#define DPM_TABLE_256__ACPILevel_CcPwrDynRm1__SHIFT 0x0 +#define DPM_TABLE_257__UvdLevel_0_VclkFrequency_MASK 0xffffffff +#define DPM_TABLE_257__UvdLevel_0_VclkFrequency__SHIFT 0x0 +#define DPM_TABLE_258__UvdLevel_0_DclkFrequency_MASK 0xffffffff +#define DPM_TABLE_258__UvdLevel_0_DclkFrequency__SHIFT 0x0 +#define DPM_TABLE_259__UvdLevel_0_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_259__UvdLevel_0_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_259__UvdLevel_0_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_259__UvdLevel_0_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_259__UvdLevel_0_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_259__UvdLevel_0_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_259__UvdLevel_0_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_259__UvdLevel_0_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_260__UvdLevel_0_padding_1_MASK 0xff +#define DPM_TABLE_260__UvdLevel_0_padding_1__SHIFT 0x0 +#define DPM_TABLE_260__UvdLevel_0_padding_0_MASK 0xff00 +#define DPM_TABLE_260__UvdLevel_0_padding_0__SHIFT 0x8 +#define DPM_TABLE_260__UvdLevel_0_DclkDivider_MASK 0xff0000 +#define DPM_TABLE_260__UvdLevel_0_DclkDivider__SHIFT 0x10 +#define DPM_TABLE_260__UvdLevel_0_VclkDivider_MASK 0xff000000 +#define DPM_TABLE_260__UvdLevel_0_VclkDivider__SHIFT 0x18 +#define DPM_TABLE_261__UvdLevel_1_VclkFrequency_MASK 0xffffffff +#define DPM_TABLE_261__UvdLevel_1_VclkFrequency__SHIFT 0x0 +#define DPM_TABLE_262__UvdLevel_1_DclkFrequency_MASK 0xffffffff +#define DPM_TABLE_262__UvdLevel_1_DclkFrequency__SHIFT 0x0 +#define DPM_TABLE_263__UvdLevel_1_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_263__UvdLevel_1_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_263__UvdLevel_1_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_263__UvdLevel_1_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_263__UvdLevel_1_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_263__UvdLevel_1_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_263__UvdLevel_1_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_263__UvdLevel_1_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_264__UvdLevel_1_padding_1_MASK 0xff +#define DPM_TABLE_264__UvdLevel_1_padding_1__SHIFT 0x0 +#define DPM_TABLE_264__UvdLevel_1_padding_0_MASK 0xff00 +#define DPM_TABLE_264__UvdLevel_1_padding_0__SHIFT 0x8 +#define DPM_TABLE_264__UvdLevel_1_DclkDivider_MASK 0xff0000 +#define DPM_TABLE_264__UvdLevel_1_DclkDivider__SHIFT 0x10 +#define DPM_TABLE_264__UvdLevel_1_VclkDivider_MASK 0xff000000 +#define DPM_TABLE_264__UvdLevel_1_VclkDivider__SHIFT 0x18 +#define DPM_TABLE_265__UvdLevel_2_VclkFrequency_MASK 0xffffffff +#define DPM_TABLE_265__UvdLevel_2_VclkFrequency__SHIFT 0x0 +#define DPM_TABLE_266__UvdLevel_2_DclkFrequency_MASK 0xffffffff +#define DPM_TABLE_266__UvdLevel_2_DclkFrequency__SHIFT 0x0 +#define DPM_TABLE_267__UvdLevel_2_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_267__UvdLevel_2_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_267__UvdLevel_2_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_267__UvdLevel_2_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_267__UvdLevel_2_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_267__UvdLevel_2_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_267__UvdLevel_2_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_267__UvdLevel_2_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_268__UvdLevel_2_padding_1_MASK 0xff +#define DPM_TABLE_268__UvdLevel_2_padding_1__SHIFT 0x0 +#define DPM_TABLE_268__UvdLevel_2_padding_0_MASK 0xff00 +#define DPM_TABLE_268__UvdLevel_2_padding_0__SHIFT 0x8 +#define DPM_TABLE_268__UvdLevel_2_DclkDivider_MASK 0xff0000 +#define DPM_TABLE_268__UvdLevel_2_DclkDivider__SHIFT 0x10 +#define DPM_TABLE_268__UvdLevel_2_VclkDivider_MASK 0xff000000 +#define DPM_TABLE_268__UvdLevel_2_VclkDivider__SHIFT 0x18 +#define DPM_TABLE_269__UvdLevel_3_VclkFrequency_MASK 0xffffffff +#define DPM_TABLE_269__UvdLevel_3_VclkFrequency__SHIFT 0x0 +#define DPM_TABLE_270__UvdLevel_3_DclkFrequency_MASK 0xffffffff +#define DPM_TABLE_270__UvdLevel_3_DclkFrequency__SHIFT 0x0 +#define DPM_TABLE_271__UvdLevel_3_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_271__UvdLevel_3_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_271__UvdLevel_3_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_271__UvdLevel_3_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_271__UvdLevel_3_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_271__UvdLevel_3_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_271__UvdLevel_3_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_271__UvdLevel_3_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_272__UvdLevel_3_padding_1_MASK 0xff +#define DPM_TABLE_272__UvdLevel_3_padding_1__SHIFT 0x0 +#define DPM_TABLE_272__UvdLevel_3_padding_0_MASK 0xff00 +#define DPM_TABLE_272__UvdLevel_3_padding_0__SHIFT 0x8 +#define DPM_TABLE_272__UvdLevel_3_DclkDivider_MASK 0xff0000 +#define DPM_TABLE_272__UvdLevel_3_DclkDivider__SHIFT 0x10 +#define DPM_TABLE_272__UvdLevel_3_VclkDivider_MASK 0xff000000 +#define DPM_TABLE_272__UvdLevel_3_VclkDivider__SHIFT 0x18 +#define DPM_TABLE_273__UvdLevel_4_VclkFrequency_MASK 0xffffffff +#define DPM_TABLE_273__UvdLevel_4_VclkFrequency__SHIFT 0x0 +#define DPM_TABLE_274__UvdLevel_4_DclkFrequency_MASK 0xffffffff +#define DPM_TABLE_274__UvdLevel_4_DclkFrequency__SHIFT 0x0 +#define DPM_TABLE_275__UvdLevel_4_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_275__UvdLevel_4_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_275__UvdLevel_4_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_275__UvdLevel_4_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_275__UvdLevel_4_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_275__UvdLevel_4_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_275__UvdLevel_4_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_275__UvdLevel_4_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_276__UvdLevel_4_padding_1_MASK 0xff +#define DPM_TABLE_276__UvdLevel_4_padding_1__SHIFT 0x0 +#define DPM_TABLE_276__UvdLevel_4_padding_0_MASK 0xff00 +#define DPM_TABLE_276__UvdLevel_4_padding_0__SHIFT 0x8 +#define DPM_TABLE_276__UvdLevel_4_DclkDivider_MASK 0xff0000 +#define DPM_TABLE_276__UvdLevel_4_DclkDivider__SHIFT 0x10 +#define DPM_TABLE_276__UvdLevel_4_VclkDivider_MASK 0xff000000 +#define DPM_TABLE_276__UvdLevel_4_VclkDivider__SHIFT 0x18 +#define DPM_TABLE_277__UvdLevel_5_VclkFrequency_MASK 0xffffffff +#define DPM_TABLE_277__UvdLevel_5_VclkFrequency__SHIFT 0x0 +#define DPM_TABLE_278__UvdLevel_5_DclkFrequency_MASK 0xffffffff +#define DPM_TABLE_278__UvdLevel_5_DclkFrequency__SHIFT 0x0 +#define DPM_TABLE_279__UvdLevel_5_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_279__UvdLevel_5_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_279__UvdLevel_5_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_279__UvdLevel_5_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_279__UvdLevel_5_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_279__UvdLevel_5_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_279__UvdLevel_5_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_279__UvdLevel_5_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_280__UvdLevel_5_padding_1_MASK 0xff +#define DPM_TABLE_280__UvdLevel_5_padding_1__SHIFT 0x0 +#define DPM_TABLE_280__UvdLevel_5_padding_0_MASK 0xff00 +#define DPM_TABLE_280__UvdLevel_5_padding_0__SHIFT 0x8 +#define DPM_TABLE_280__UvdLevel_5_DclkDivider_MASK 0xff0000 +#define DPM_TABLE_280__UvdLevel_5_DclkDivider__SHIFT 0x10 +#define DPM_TABLE_280__UvdLevel_5_VclkDivider_MASK 0xff000000 +#define DPM_TABLE_280__UvdLevel_5_VclkDivider__SHIFT 0x18 +#define DPM_TABLE_281__UvdLevel_6_VclkFrequency_MASK 0xffffffff +#define DPM_TABLE_281__UvdLevel_6_VclkFrequency__SHIFT 0x0 +#define DPM_TABLE_282__UvdLevel_6_DclkFrequency_MASK 0xffffffff +#define DPM_TABLE_282__UvdLevel_6_DclkFrequency__SHIFT 0x0 +#define DPM_TABLE_283__UvdLevel_6_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_283__UvdLevel_6_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_283__UvdLevel_6_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_283__UvdLevel_6_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_283__UvdLevel_6_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_283__UvdLevel_6_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_283__UvdLevel_6_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_283__UvdLevel_6_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_284__UvdLevel_6_padding_1_MASK 0xff +#define DPM_TABLE_284__UvdLevel_6_padding_1__SHIFT 0x0 +#define DPM_TABLE_284__UvdLevel_6_padding_0_MASK 0xff00 +#define DPM_TABLE_284__UvdLevel_6_padding_0__SHIFT 0x8 +#define DPM_TABLE_284__UvdLevel_6_DclkDivider_MASK 0xff0000 +#define DPM_TABLE_284__UvdLevel_6_DclkDivider__SHIFT 0x10 +#define DPM_TABLE_284__UvdLevel_6_VclkDivider_MASK 0xff000000 +#define DPM_TABLE_284__UvdLevel_6_VclkDivider__SHIFT 0x18 +#define DPM_TABLE_285__UvdLevel_7_VclkFrequency_MASK 0xffffffff +#define DPM_TABLE_285__UvdLevel_7_VclkFrequency__SHIFT 0x0 +#define DPM_TABLE_286__UvdLevel_7_DclkFrequency_MASK 0xffffffff +#define DPM_TABLE_286__UvdLevel_7_DclkFrequency__SHIFT 0x0 +#define DPM_TABLE_287__UvdLevel_7_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_287__UvdLevel_7_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_287__UvdLevel_7_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_287__UvdLevel_7_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_287__UvdLevel_7_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_287__UvdLevel_7_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_287__UvdLevel_7_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_287__UvdLevel_7_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_288__UvdLevel_7_padding_1_MASK 0xff +#define DPM_TABLE_288__UvdLevel_7_padding_1__SHIFT 0x0 +#define DPM_TABLE_288__UvdLevel_7_padding_0_MASK 0xff00 +#define DPM_TABLE_288__UvdLevel_7_padding_0__SHIFT 0x8 +#define DPM_TABLE_288__UvdLevel_7_DclkDivider_MASK 0xff0000 +#define DPM_TABLE_288__UvdLevel_7_DclkDivider__SHIFT 0x10 +#define DPM_TABLE_288__UvdLevel_7_VclkDivider_MASK 0xff000000 +#define DPM_TABLE_288__UvdLevel_7_VclkDivider__SHIFT 0x18 +#define DPM_TABLE_289__VceLevel_0_Frequency_MASK 0xffffffff +#define DPM_TABLE_289__VceLevel_0_Frequency__SHIFT 0x0 +#define DPM_TABLE_290__VceLevel_0_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_290__VceLevel_0_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_290__VceLevel_0_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_290__VceLevel_0_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_290__VceLevel_0_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_290__VceLevel_0_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_290__VceLevel_0_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_290__VceLevel_0_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_291__VceLevel_0_padding_2_MASK 0xff +#define DPM_TABLE_291__VceLevel_0_padding_2__SHIFT 0x0 +#define DPM_TABLE_291__VceLevel_0_padding_1_MASK 0xff00 +#define DPM_TABLE_291__VceLevel_0_padding_1__SHIFT 0x8 +#define DPM_TABLE_291__VceLevel_0_padding_0_MASK 0xff0000 +#define DPM_TABLE_291__VceLevel_0_padding_0__SHIFT 0x10 +#define DPM_TABLE_291__VceLevel_0_Divider_MASK 0xff000000 +#define DPM_TABLE_291__VceLevel_0_Divider__SHIFT 0x18 +#define DPM_TABLE_292__VceLevel_1_Frequency_MASK 0xffffffff +#define DPM_TABLE_292__VceLevel_1_Frequency__SHIFT 0x0 +#define DPM_TABLE_293__VceLevel_1_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_293__VceLevel_1_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_293__VceLevel_1_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_293__VceLevel_1_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_293__VceLevel_1_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_293__VceLevel_1_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_293__VceLevel_1_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_293__VceLevel_1_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_294__VceLevel_1_padding_2_MASK 0xff +#define DPM_TABLE_294__VceLevel_1_padding_2__SHIFT 0x0 +#define DPM_TABLE_294__VceLevel_1_padding_1_MASK 0xff00 +#define DPM_TABLE_294__VceLevel_1_padding_1__SHIFT 0x8 +#define DPM_TABLE_294__VceLevel_1_padding_0_MASK 0xff0000 +#define DPM_TABLE_294__VceLevel_1_padding_0__SHIFT 0x10 +#define DPM_TABLE_294__VceLevel_1_Divider_MASK 0xff000000 +#define DPM_TABLE_294__VceLevel_1_Divider__SHIFT 0x18 +#define DPM_TABLE_295__VceLevel_2_Frequency_MASK 0xffffffff +#define DPM_TABLE_295__VceLevel_2_Frequency__SHIFT 0x0 +#define DPM_TABLE_296__VceLevel_2_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_296__VceLevel_2_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_296__VceLevel_2_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_296__VceLevel_2_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_296__VceLevel_2_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_296__VceLevel_2_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_296__VceLevel_2_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_296__VceLevel_2_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_297__VceLevel_2_padding_2_MASK 0xff +#define DPM_TABLE_297__VceLevel_2_padding_2__SHIFT 0x0 +#define DPM_TABLE_297__VceLevel_2_padding_1_MASK 0xff00 +#define DPM_TABLE_297__VceLevel_2_padding_1__SHIFT 0x8 +#define DPM_TABLE_297__VceLevel_2_padding_0_MASK 0xff0000 +#define DPM_TABLE_297__VceLevel_2_padding_0__SHIFT 0x10 +#define DPM_TABLE_297__VceLevel_2_Divider_MASK 0xff000000 +#define DPM_TABLE_297__VceLevel_2_Divider__SHIFT 0x18 +#define DPM_TABLE_298__VceLevel_3_Frequency_MASK 0xffffffff +#define DPM_TABLE_298__VceLevel_3_Frequency__SHIFT 0x0 +#define DPM_TABLE_299__VceLevel_3_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_299__VceLevel_3_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_299__VceLevel_3_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_299__VceLevel_3_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_299__VceLevel_3_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_299__VceLevel_3_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_299__VceLevel_3_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_299__VceLevel_3_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_300__VceLevel_3_padding_2_MASK 0xff +#define DPM_TABLE_300__VceLevel_3_padding_2__SHIFT 0x0 +#define DPM_TABLE_300__VceLevel_3_padding_1_MASK 0xff00 +#define DPM_TABLE_300__VceLevel_3_padding_1__SHIFT 0x8 +#define DPM_TABLE_300__VceLevel_3_padding_0_MASK 0xff0000 +#define DPM_TABLE_300__VceLevel_3_padding_0__SHIFT 0x10 +#define DPM_TABLE_300__VceLevel_3_Divider_MASK 0xff000000 +#define DPM_TABLE_300__VceLevel_3_Divider__SHIFT 0x18 +#define DPM_TABLE_301__VceLevel_4_Frequency_MASK 0xffffffff +#define DPM_TABLE_301__VceLevel_4_Frequency__SHIFT 0x0 +#define DPM_TABLE_302__VceLevel_4_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_302__VceLevel_4_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_302__VceLevel_4_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_302__VceLevel_4_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_302__VceLevel_4_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_302__VceLevel_4_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_302__VceLevel_4_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_302__VceLevel_4_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_303__VceLevel_4_padding_2_MASK 0xff +#define DPM_TABLE_303__VceLevel_4_padding_2__SHIFT 0x0 +#define DPM_TABLE_303__VceLevel_4_padding_1_MASK 0xff00 +#define DPM_TABLE_303__VceLevel_4_padding_1__SHIFT 0x8 +#define DPM_TABLE_303__VceLevel_4_padding_0_MASK 0xff0000 +#define DPM_TABLE_303__VceLevel_4_padding_0__SHIFT 0x10 +#define DPM_TABLE_303__VceLevel_4_Divider_MASK 0xff000000 +#define DPM_TABLE_303__VceLevel_4_Divider__SHIFT 0x18 +#define DPM_TABLE_304__VceLevel_5_Frequency_MASK 0xffffffff +#define DPM_TABLE_304__VceLevel_5_Frequency__SHIFT 0x0 +#define DPM_TABLE_305__VceLevel_5_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_305__VceLevel_5_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_305__VceLevel_5_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_305__VceLevel_5_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_305__VceLevel_5_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_305__VceLevel_5_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_305__VceLevel_5_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_305__VceLevel_5_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_306__VceLevel_5_padding_2_MASK 0xff +#define DPM_TABLE_306__VceLevel_5_padding_2__SHIFT 0x0 +#define DPM_TABLE_306__VceLevel_5_padding_1_MASK 0xff00 +#define DPM_TABLE_306__VceLevel_5_padding_1__SHIFT 0x8 +#define DPM_TABLE_306__VceLevel_5_padding_0_MASK 0xff0000 +#define DPM_TABLE_306__VceLevel_5_padding_0__SHIFT 0x10 +#define DPM_TABLE_306__VceLevel_5_Divider_MASK 0xff000000 +#define DPM_TABLE_306__VceLevel_5_Divider__SHIFT 0x18 +#define DPM_TABLE_307__VceLevel_6_Frequency_MASK 0xffffffff +#define DPM_TABLE_307__VceLevel_6_Frequency__SHIFT 0x0 +#define DPM_TABLE_308__VceLevel_6_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_308__VceLevel_6_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_308__VceLevel_6_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_308__VceLevel_6_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_308__VceLevel_6_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_308__VceLevel_6_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_308__VceLevel_6_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_308__VceLevel_6_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_309__VceLevel_6_padding_2_MASK 0xff +#define DPM_TABLE_309__VceLevel_6_padding_2__SHIFT 0x0 +#define DPM_TABLE_309__VceLevel_6_padding_1_MASK 0xff00 +#define DPM_TABLE_309__VceLevel_6_padding_1__SHIFT 0x8 +#define DPM_TABLE_309__VceLevel_6_padding_0_MASK 0xff0000 +#define DPM_TABLE_309__VceLevel_6_padding_0__SHIFT 0x10 +#define DPM_TABLE_309__VceLevel_6_Divider_MASK 0xff000000 +#define DPM_TABLE_309__VceLevel_6_Divider__SHIFT 0x18 +#define DPM_TABLE_310__VceLevel_7_Frequency_MASK 0xffffffff +#define DPM_TABLE_310__VceLevel_7_Frequency__SHIFT 0x0 +#define DPM_TABLE_311__VceLevel_7_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_311__VceLevel_7_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_311__VceLevel_7_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_311__VceLevel_7_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_311__VceLevel_7_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_311__VceLevel_7_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_311__VceLevel_7_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_311__VceLevel_7_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_312__VceLevel_7_padding_2_MASK 0xff +#define DPM_TABLE_312__VceLevel_7_padding_2__SHIFT 0x0 +#define DPM_TABLE_312__VceLevel_7_padding_1_MASK 0xff00 +#define DPM_TABLE_312__VceLevel_7_padding_1__SHIFT 0x8 +#define DPM_TABLE_312__VceLevel_7_padding_0_MASK 0xff0000 +#define DPM_TABLE_312__VceLevel_7_padding_0__SHIFT 0x10 +#define DPM_TABLE_312__VceLevel_7_Divider_MASK 0xff000000 +#define DPM_TABLE_312__VceLevel_7_Divider__SHIFT 0x18 +#define DPM_TABLE_313__AcpLevel_0_Frequency_MASK 0xffffffff +#define DPM_TABLE_313__AcpLevel_0_Frequency__SHIFT 0x0 +#define DPM_TABLE_314__AcpLevel_0_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_314__AcpLevel_0_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_314__AcpLevel_0_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_314__AcpLevel_0_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_314__AcpLevel_0_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_314__AcpLevel_0_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_314__AcpLevel_0_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_314__AcpLevel_0_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_315__AcpLevel_0_padding_2_MASK 0xff +#define DPM_TABLE_315__AcpLevel_0_padding_2__SHIFT 0x0 +#define DPM_TABLE_315__AcpLevel_0_padding_1_MASK 0xff00 +#define DPM_TABLE_315__AcpLevel_0_padding_1__SHIFT 0x8 +#define DPM_TABLE_315__AcpLevel_0_padding_0_MASK 0xff0000 +#define DPM_TABLE_315__AcpLevel_0_padding_0__SHIFT 0x10 +#define DPM_TABLE_315__AcpLevel_0_Divider_MASK 0xff000000 +#define DPM_TABLE_315__AcpLevel_0_Divider__SHIFT 0x18 +#define DPM_TABLE_316__AcpLevel_1_Frequency_MASK 0xffffffff +#define DPM_TABLE_316__AcpLevel_1_Frequency__SHIFT 0x0 +#define DPM_TABLE_317__AcpLevel_1_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_317__AcpLevel_1_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_317__AcpLevel_1_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_317__AcpLevel_1_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_317__AcpLevel_1_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_317__AcpLevel_1_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_317__AcpLevel_1_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_317__AcpLevel_1_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_318__AcpLevel_1_padding_2_MASK 0xff +#define DPM_TABLE_318__AcpLevel_1_padding_2__SHIFT 0x0 +#define DPM_TABLE_318__AcpLevel_1_padding_1_MASK 0xff00 +#define DPM_TABLE_318__AcpLevel_1_padding_1__SHIFT 0x8 +#define DPM_TABLE_318__AcpLevel_1_padding_0_MASK 0xff0000 +#define DPM_TABLE_318__AcpLevel_1_padding_0__SHIFT 0x10 +#define DPM_TABLE_318__AcpLevel_1_Divider_MASK 0xff000000 +#define DPM_TABLE_318__AcpLevel_1_Divider__SHIFT 0x18 +#define DPM_TABLE_319__AcpLevel_2_Frequency_MASK 0xffffffff +#define DPM_TABLE_319__AcpLevel_2_Frequency__SHIFT 0x0 +#define DPM_TABLE_320__AcpLevel_2_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_320__AcpLevel_2_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_320__AcpLevel_2_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_320__AcpLevel_2_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_320__AcpLevel_2_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_320__AcpLevel_2_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_320__AcpLevel_2_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_320__AcpLevel_2_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_321__AcpLevel_2_padding_2_MASK 0xff +#define DPM_TABLE_321__AcpLevel_2_padding_2__SHIFT 0x0 +#define DPM_TABLE_321__AcpLevel_2_padding_1_MASK 0xff00 +#define DPM_TABLE_321__AcpLevel_2_padding_1__SHIFT 0x8 +#define DPM_TABLE_321__AcpLevel_2_padding_0_MASK 0xff0000 +#define DPM_TABLE_321__AcpLevel_2_padding_0__SHIFT 0x10 +#define DPM_TABLE_321__AcpLevel_2_Divider_MASK 0xff000000 +#define DPM_TABLE_321__AcpLevel_2_Divider__SHIFT 0x18 +#define DPM_TABLE_322__AcpLevel_3_Frequency_MASK 0xffffffff +#define DPM_TABLE_322__AcpLevel_3_Frequency__SHIFT 0x0 +#define DPM_TABLE_323__AcpLevel_3_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_323__AcpLevel_3_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_323__AcpLevel_3_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_323__AcpLevel_3_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_323__AcpLevel_3_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_323__AcpLevel_3_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_323__AcpLevel_3_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_323__AcpLevel_3_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_324__AcpLevel_3_padding_2_MASK 0xff +#define DPM_TABLE_324__AcpLevel_3_padding_2__SHIFT 0x0 +#define DPM_TABLE_324__AcpLevel_3_padding_1_MASK 0xff00 +#define DPM_TABLE_324__AcpLevel_3_padding_1__SHIFT 0x8 +#define DPM_TABLE_324__AcpLevel_3_padding_0_MASK 0xff0000 +#define DPM_TABLE_324__AcpLevel_3_padding_0__SHIFT 0x10 +#define DPM_TABLE_324__AcpLevel_3_Divider_MASK 0xff000000 +#define DPM_TABLE_324__AcpLevel_3_Divider__SHIFT 0x18 +#define DPM_TABLE_325__AcpLevel_4_Frequency_MASK 0xffffffff +#define DPM_TABLE_325__AcpLevel_4_Frequency__SHIFT 0x0 +#define DPM_TABLE_326__AcpLevel_4_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_326__AcpLevel_4_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_326__AcpLevel_4_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_326__AcpLevel_4_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_326__AcpLevel_4_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_326__AcpLevel_4_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_326__AcpLevel_4_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_326__AcpLevel_4_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_327__AcpLevel_4_padding_2_MASK 0xff +#define DPM_TABLE_327__AcpLevel_4_padding_2__SHIFT 0x0 +#define DPM_TABLE_327__AcpLevel_4_padding_1_MASK 0xff00 +#define DPM_TABLE_327__AcpLevel_4_padding_1__SHIFT 0x8 +#define DPM_TABLE_327__AcpLevel_4_padding_0_MASK 0xff0000 +#define DPM_TABLE_327__AcpLevel_4_padding_0__SHIFT 0x10 +#define DPM_TABLE_327__AcpLevel_4_Divider_MASK 0xff000000 +#define DPM_TABLE_327__AcpLevel_4_Divider__SHIFT 0x18 +#define DPM_TABLE_328__AcpLevel_5_Frequency_MASK 0xffffffff +#define DPM_TABLE_328__AcpLevel_5_Frequency__SHIFT 0x0 +#define DPM_TABLE_329__AcpLevel_5_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_329__AcpLevel_5_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_329__AcpLevel_5_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_329__AcpLevel_5_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_329__AcpLevel_5_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_329__AcpLevel_5_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_329__AcpLevel_5_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_329__AcpLevel_5_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_330__AcpLevel_5_padding_2_MASK 0xff +#define DPM_TABLE_330__AcpLevel_5_padding_2__SHIFT 0x0 +#define DPM_TABLE_330__AcpLevel_5_padding_1_MASK 0xff00 +#define DPM_TABLE_330__AcpLevel_5_padding_1__SHIFT 0x8 +#define DPM_TABLE_330__AcpLevel_5_padding_0_MASK 0xff0000 +#define DPM_TABLE_330__AcpLevel_5_padding_0__SHIFT 0x10 +#define DPM_TABLE_330__AcpLevel_5_Divider_MASK 0xff000000 +#define DPM_TABLE_330__AcpLevel_5_Divider__SHIFT 0x18 +#define DPM_TABLE_331__AcpLevel_6_Frequency_MASK 0xffffffff +#define DPM_TABLE_331__AcpLevel_6_Frequency__SHIFT 0x0 +#define DPM_TABLE_332__AcpLevel_6_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_332__AcpLevel_6_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_332__AcpLevel_6_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_332__AcpLevel_6_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_332__AcpLevel_6_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_332__AcpLevel_6_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_332__AcpLevel_6_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_332__AcpLevel_6_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_333__AcpLevel_6_padding_2_MASK 0xff +#define DPM_TABLE_333__AcpLevel_6_padding_2__SHIFT 0x0 +#define DPM_TABLE_333__AcpLevel_6_padding_1_MASK 0xff00 +#define DPM_TABLE_333__AcpLevel_6_padding_1__SHIFT 0x8 +#define DPM_TABLE_333__AcpLevel_6_padding_0_MASK 0xff0000 +#define DPM_TABLE_333__AcpLevel_6_padding_0__SHIFT 0x10 +#define DPM_TABLE_333__AcpLevel_6_Divider_MASK 0xff000000 +#define DPM_TABLE_333__AcpLevel_6_Divider__SHIFT 0x18 +#define DPM_TABLE_334__AcpLevel_7_Frequency_MASK 0xffffffff +#define DPM_TABLE_334__AcpLevel_7_Frequency__SHIFT 0x0 +#define DPM_TABLE_335__AcpLevel_7_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_335__AcpLevel_7_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_335__AcpLevel_7_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_335__AcpLevel_7_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_335__AcpLevel_7_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_335__AcpLevel_7_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_335__AcpLevel_7_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_335__AcpLevel_7_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_336__AcpLevel_7_padding_2_MASK 0xff +#define DPM_TABLE_336__AcpLevel_7_padding_2__SHIFT 0x0 +#define DPM_TABLE_336__AcpLevel_7_padding_1_MASK 0xff00 +#define DPM_TABLE_336__AcpLevel_7_padding_1__SHIFT 0x8 +#define DPM_TABLE_336__AcpLevel_7_padding_0_MASK 0xff0000 +#define DPM_TABLE_336__AcpLevel_7_padding_0__SHIFT 0x10 +#define DPM_TABLE_336__AcpLevel_7_Divider_MASK 0xff000000 +#define DPM_TABLE_336__AcpLevel_7_Divider__SHIFT 0x18 +#define DPM_TABLE_337__SamuLevel_0_Frequency_MASK 0xffffffff +#define DPM_TABLE_337__SamuLevel_0_Frequency__SHIFT 0x0 +#define DPM_TABLE_338__SamuLevel_0_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_338__SamuLevel_0_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_338__SamuLevel_0_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_338__SamuLevel_0_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_338__SamuLevel_0_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_338__SamuLevel_0_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_338__SamuLevel_0_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_338__SamuLevel_0_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_339__SamuLevel_0_padding_2_MASK 0xff +#define DPM_TABLE_339__SamuLevel_0_padding_2__SHIFT 0x0 +#define DPM_TABLE_339__SamuLevel_0_padding_1_MASK 0xff00 +#define DPM_TABLE_339__SamuLevel_0_padding_1__SHIFT 0x8 +#define DPM_TABLE_339__SamuLevel_0_padding_0_MASK 0xff0000 +#define DPM_TABLE_339__SamuLevel_0_padding_0__SHIFT 0x10 +#define DPM_TABLE_339__SamuLevel_0_Divider_MASK 0xff000000 +#define DPM_TABLE_339__SamuLevel_0_Divider__SHIFT 0x18 +#define DPM_TABLE_340__SamuLevel_1_Frequency_MASK 0xffffffff +#define DPM_TABLE_340__SamuLevel_1_Frequency__SHIFT 0x0 +#define DPM_TABLE_341__SamuLevel_1_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_341__SamuLevel_1_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_341__SamuLevel_1_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_341__SamuLevel_1_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_341__SamuLevel_1_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_341__SamuLevel_1_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_341__SamuLevel_1_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_341__SamuLevel_1_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_342__SamuLevel_1_padding_2_MASK 0xff +#define DPM_TABLE_342__SamuLevel_1_padding_2__SHIFT 0x0 +#define DPM_TABLE_342__SamuLevel_1_padding_1_MASK 0xff00 +#define DPM_TABLE_342__SamuLevel_1_padding_1__SHIFT 0x8 +#define DPM_TABLE_342__SamuLevel_1_padding_0_MASK 0xff0000 +#define DPM_TABLE_342__SamuLevel_1_padding_0__SHIFT 0x10 +#define DPM_TABLE_342__SamuLevel_1_Divider_MASK 0xff000000 +#define DPM_TABLE_342__SamuLevel_1_Divider__SHIFT 0x18 +#define DPM_TABLE_343__SamuLevel_2_Frequency_MASK 0xffffffff +#define DPM_TABLE_343__SamuLevel_2_Frequency__SHIFT 0x0 +#define DPM_TABLE_344__SamuLevel_2_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_344__SamuLevel_2_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_344__SamuLevel_2_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_344__SamuLevel_2_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_344__SamuLevel_2_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_344__SamuLevel_2_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_344__SamuLevel_2_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_344__SamuLevel_2_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_345__SamuLevel_2_padding_2_MASK 0xff +#define DPM_TABLE_345__SamuLevel_2_padding_2__SHIFT 0x0 +#define DPM_TABLE_345__SamuLevel_2_padding_1_MASK 0xff00 +#define DPM_TABLE_345__SamuLevel_2_padding_1__SHIFT 0x8 +#define DPM_TABLE_345__SamuLevel_2_padding_0_MASK 0xff0000 +#define DPM_TABLE_345__SamuLevel_2_padding_0__SHIFT 0x10 +#define DPM_TABLE_345__SamuLevel_2_Divider_MASK 0xff000000 +#define DPM_TABLE_345__SamuLevel_2_Divider__SHIFT 0x18 +#define DPM_TABLE_346__SamuLevel_3_Frequency_MASK 0xffffffff +#define DPM_TABLE_346__SamuLevel_3_Frequency__SHIFT 0x0 +#define DPM_TABLE_347__SamuLevel_3_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_347__SamuLevel_3_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_347__SamuLevel_3_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_347__SamuLevel_3_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_347__SamuLevel_3_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_347__SamuLevel_3_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_347__SamuLevel_3_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_347__SamuLevel_3_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_348__SamuLevel_3_padding_2_MASK 0xff +#define DPM_TABLE_348__SamuLevel_3_padding_2__SHIFT 0x0 +#define DPM_TABLE_348__SamuLevel_3_padding_1_MASK 0xff00 +#define DPM_TABLE_348__SamuLevel_3_padding_1__SHIFT 0x8 +#define DPM_TABLE_348__SamuLevel_3_padding_0_MASK 0xff0000 +#define DPM_TABLE_348__SamuLevel_3_padding_0__SHIFT 0x10 +#define DPM_TABLE_348__SamuLevel_3_Divider_MASK 0xff000000 +#define DPM_TABLE_348__SamuLevel_3_Divider__SHIFT 0x18 +#define DPM_TABLE_349__SamuLevel_4_Frequency_MASK 0xffffffff +#define DPM_TABLE_349__SamuLevel_4_Frequency__SHIFT 0x0 +#define DPM_TABLE_350__SamuLevel_4_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_350__SamuLevel_4_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_350__SamuLevel_4_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_350__SamuLevel_4_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_350__SamuLevel_4_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_350__SamuLevel_4_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_350__SamuLevel_4_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_350__SamuLevel_4_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_351__SamuLevel_4_padding_2_MASK 0xff +#define DPM_TABLE_351__SamuLevel_4_padding_2__SHIFT 0x0 +#define DPM_TABLE_351__SamuLevel_4_padding_1_MASK 0xff00 +#define DPM_TABLE_351__SamuLevel_4_padding_1__SHIFT 0x8 +#define DPM_TABLE_351__SamuLevel_4_padding_0_MASK 0xff0000 +#define DPM_TABLE_351__SamuLevel_4_padding_0__SHIFT 0x10 +#define DPM_TABLE_351__SamuLevel_4_Divider_MASK 0xff000000 +#define DPM_TABLE_351__SamuLevel_4_Divider__SHIFT 0x18 +#define DPM_TABLE_352__SamuLevel_5_Frequency_MASK 0xffffffff +#define DPM_TABLE_352__SamuLevel_5_Frequency__SHIFT 0x0 +#define DPM_TABLE_353__SamuLevel_5_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_353__SamuLevel_5_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_353__SamuLevel_5_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_353__SamuLevel_5_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_353__SamuLevel_5_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_353__SamuLevel_5_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_353__SamuLevel_5_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_353__SamuLevel_5_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_354__SamuLevel_5_padding_2_MASK 0xff +#define DPM_TABLE_354__SamuLevel_5_padding_2__SHIFT 0x0 +#define DPM_TABLE_354__SamuLevel_5_padding_1_MASK 0xff00 +#define DPM_TABLE_354__SamuLevel_5_padding_1__SHIFT 0x8 +#define DPM_TABLE_354__SamuLevel_5_padding_0_MASK 0xff0000 +#define DPM_TABLE_354__SamuLevel_5_padding_0__SHIFT 0x10 +#define DPM_TABLE_354__SamuLevel_5_Divider_MASK 0xff000000 +#define DPM_TABLE_354__SamuLevel_5_Divider__SHIFT 0x18 +#define DPM_TABLE_355__SamuLevel_6_Frequency_MASK 0xffffffff +#define DPM_TABLE_355__SamuLevel_6_Frequency__SHIFT 0x0 +#define DPM_TABLE_356__SamuLevel_6_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_356__SamuLevel_6_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_356__SamuLevel_6_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_356__SamuLevel_6_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_356__SamuLevel_6_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_356__SamuLevel_6_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_356__SamuLevel_6_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_356__SamuLevel_6_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_357__SamuLevel_6_padding_2_MASK 0xff +#define DPM_TABLE_357__SamuLevel_6_padding_2__SHIFT 0x0 +#define DPM_TABLE_357__SamuLevel_6_padding_1_MASK 0xff00 +#define DPM_TABLE_357__SamuLevel_6_padding_1__SHIFT 0x8 +#define DPM_TABLE_357__SamuLevel_6_padding_0_MASK 0xff0000 +#define DPM_TABLE_357__SamuLevel_6_padding_0__SHIFT 0x10 +#define DPM_TABLE_357__SamuLevel_6_Divider_MASK 0xff000000 +#define DPM_TABLE_357__SamuLevel_6_Divider__SHIFT 0x18 +#define DPM_TABLE_358__SamuLevel_7_Frequency_MASK 0xffffffff +#define DPM_TABLE_358__SamuLevel_7_Frequency__SHIFT 0x0 +#define DPM_TABLE_359__SamuLevel_7_MinVoltage_Phases_MASK 0xff +#define DPM_TABLE_359__SamuLevel_7_MinVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_359__SamuLevel_7_MinVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_359__SamuLevel_7_MinVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_359__SamuLevel_7_MinVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_359__SamuLevel_7_MinVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_359__SamuLevel_7_MinVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_359__SamuLevel_7_MinVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_360__SamuLevel_7_padding_2_MASK 0xff +#define DPM_TABLE_360__SamuLevel_7_padding_2__SHIFT 0x0 +#define DPM_TABLE_360__SamuLevel_7_padding_1_MASK 0xff00 +#define DPM_TABLE_360__SamuLevel_7_padding_1__SHIFT 0x8 +#define DPM_TABLE_360__SamuLevel_7_padding_0_MASK 0xff0000 +#define DPM_TABLE_360__SamuLevel_7_padding_0__SHIFT 0x10 +#define DPM_TABLE_360__SamuLevel_7_Divider_MASK 0xff000000 +#define DPM_TABLE_360__SamuLevel_7_Divider__SHIFT 0x18 +#define DPM_TABLE_361__Ulv_CcPwrDynRm_MASK 0xffffffff +#define DPM_TABLE_361__Ulv_CcPwrDynRm__SHIFT 0x0 +#define DPM_TABLE_362__Ulv_CcPwrDynRm1_MASK 0xffffffff +#define DPM_TABLE_362__Ulv_CcPwrDynRm1__SHIFT 0x0 +#define DPM_TABLE_363__Ulv_VddcPhase_MASK 0xff +#define DPM_TABLE_363__Ulv_VddcPhase__SHIFT 0x0 +#define DPM_TABLE_363__Ulv_VddcOffsetVid_MASK 0xff00 +#define DPM_TABLE_363__Ulv_VddcOffsetVid__SHIFT 0x8 +#define DPM_TABLE_363__Ulv_VddcOffset_MASK 0xffff0000 +#define DPM_TABLE_363__Ulv_VddcOffset__SHIFT 0x10 +#define DPM_TABLE_364__Ulv_Reserved_MASK 0xffffffff +#define DPM_TABLE_364__Ulv_Reserved__SHIFT 0x0 +#define DPM_TABLE_365__SclkStepSize_MASK 0xffffffff +#define DPM_TABLE_365__SclkStepSize__SHIFT 0x0 +#define DPM_TABLE_366__Smio_0_MASK 0xffffffff +#define DPM_TABLE_366__Smio_0__SHIFT 0x0 +#define DPM_TABLE_367__Smio_1_MASK 0xffffffff +#define DPM_TABLE_367__Smio_1__SHIFT 0x0 +#define DPM_TABLE_368__Smio_2_MASK 0xffffffff +#define DPM_TABLE_368__Smio_2__SHIFT 0x0 +#define DPM_TABLE_369__Smio_3_MASK 0xffffffff +#define DPM_TABLE_369__Smio_3__SHIFT 0x0 +#define DPM_TABLE_370__Smio_4_MASK 0xffffffff +#define DPM_TABLE_370__Smio_4__SHIFT 0x0 +#define DPM_TABLE_371__Smio_5_MASK 0xffffffff +#define DPM_TABLE_371__Smio_5__SHIFT 0x0 +#define DPM_TABLE_372__Smio_6_MASK 0xffffffff +#define DPM_TABLE_372__Smio_6__SHIFT 0x0 +#define DPM_TABLE_373__Smio_7_MASK 0xffffffff +#define DPM_TABLE_373__Smio_7__SHIFT 0x0 +#define DPM_TABLE_374__Smio_8_MASK 0xffffffff +#define DPM_TABLE_374__Smio_8__SHIFT 0x0 +#define DPM_TABLE_375__Smio_9_MASK 0xffffffff +#define DPM_TABLE_375__Smio_9__SHIFT 0x0 +#define DPM_TABLE_376__Smio_10_MASK 0xffffffff +#define DPM_TABLE_376__Smio_10__SHIFT 0x0 +#define DPM_TABLE_377__Smio_11_MASK 0xffffffff +#define DPM_TABLE_377__Smio_11__SHIFT 0x0 +#define DPM_TABLE_378__Smio_12_MASK 0xffffffff +#define DPM_TABLE_378__Smio_12__SHIFT 0x0 +#define DPM_TABLE_379__Smio_13_MASK 0xffffffff +#define DPM_TABLE_379__Smio_13__SHIFT 0x0 +#define DPM_TABLE_380__Smio_14_MASK 0xffffffff +#define DPM_TABLE_380__Smio_14__SHIFT 0x0 +#define DPM_TABLE_381__Smio_15_MASK 0xffffffff +#define DPM_TABLE_381__Smio_15__SHIFT 0x0 +#define DPM_TABLE_382__Smio_16_MASK 0xffffffff +#define DPM_TABLE_382__Smio_16__SHIFT 0x0 +#define DPM_TABLE_383__Smio_17_MASK 0xffffffff +#define DPM_TABLE_383__Smio_17__SHIFT 0x0 +#define DPM_TABLE_384__Smio_18_MASK 0xffffffff +#define DPM_TABLE_384__Smio_18__SHIFT 0x0 +#define DPM_TABLE_385__Smio_19_MASK 0xffffffff +#define DPM_TABLE_385__Smio_19__SHIFT 0x0 +#define DPM_TABLE_386__Smio_20_MASK 0xffffffff +#define DPM_TABLE_386__Smio_20__SHIFT 0x0 +#define DPM_TABLE_387__Smio_21_MASK 0xffffffff +#define DPM_TABLE_387__Smio_21__SHIFT 0x0 +#define DPM_TABLE_388__Smio_22_MASK 0xffffffff +#define DPM_TABLE_388__Smio_22__SHIFT 0x0 +#define DPM_TABLE_389__Smio_23_MASK 0xffffffff +#define DPM_TABLE_389__Smio_23__SHIFT 0x0 +#define DPM_TABLE_390__Smio_24_MASK 0xffffffff +#define DPM_TABLE_390__Smio_24__SHIFT 0x0 +#define DPM_TABLE_391__Smio_25_MASK 0xffffffff +#define DPM_TABLE_391__Smio_25__SHIFT 0x0 +#define DPM_TABLE_392__Smio_26_MASK 0xffffffff +#define DPM_TABLE_392__Smio_26__SHIFT 0x0 +#define DPM_TABLE_393__Smio_27_MASK 0xffffffff +#define DPM_TABLE_393__Smio_27__SHIFT 0x0 +#define DPM_TABLE_394__Smio_28_MASK 0xffffffff +#define DPM_TABLE_394__Smio_28__SHIFT 0x0 +#define DPM_TABLE_395__Smio_29_MASK 0xffffffff +#define DPM_TABLE_395__Smio_29__SHIFT 0x0 +#define DPM_TABLE_396__Smio_30_MASK 0xffffffff +#define DPM_TABLE_396__Smio_30__SHIFT 0x0 +#define DPM_TABLE_397__Smio_31_MASK 0xffffffff +#define DPM_TABLE_397__Smio_31__SHIFT 0x0 +#define DPM_TABLE_398__SamuBootLevel_MASK 0xff +#define DPM_TABLE_398__SamuBootLevel__SHIFT 0x0 +#define DPM_TABLE_398__AcpBootLevel_MASK 0xff00 +#define DPM_TABLE_398__AcpBootLevel__SHIFT 0x8 +#define DPM_TABLE_398__VceBootLevel_MASK 0xff0000 +#define DPM_TABLE_398__VceBootLevel__SHIFT 0x10 +#define DPM_TABLE_398__UvdBootLevel_MASK 0xff000000 +#define DPM_TABLE_398__UvdBootLevel__SHIFT 0x18 +#define DPM_TABLE_399__GraphicsInterval_MASK 0xff +#define DPM_TABLE_399__GraphicsInterval__SHIFT 0x0 +#define DPM_TABLE_399__GraphicsThermThrottleEnable_MASK 0xff00 +#define DPM_TABLE_399__GraphicsThermThrottleEnable__SHIFT 0x8 +#define DPM_TABLE_399__GraphicsVoltageChangeEnable_MASK 0xff0000 +#define DPM_TABLE_399__GraphicsVoltageChangeEnable__SHIFT 0x10 +#define DPM_TABLE_399__GraphicsBootLevel_MASK 0xff000000 +#define DPM_TABLE_399__GraphicsBootLevel__SHIFT 0x18 +#define DPM_TABLE_400__TemperatureLimitHigh_MASK 0xffff +#define DPM_TABLE_400__TemperatureLimitHigh__SHIFT 0x0 +#define DPM_TABLE_400__ThermalInterval_MASK 0xff0000 +#define DPM_TABLE_400__ThermalInterval__SHIFT 0x10 +#define DPM_TABLE_400__VoltageInterval_MASK 0xff000000 +#define DPM_TABLE_400__VoltageInterval__SHIFT 0x18 +#define DPM_TABLE_401__MemoryVoltageChangeEnable_MASK 0xff +#define DPM_TABLE_401__MemoryVoltageChangeEnable__SHIFT 0x0 +#define DPM_TABLE_401__MemoryBootLevel_MASK 0xff00 +#define DPM_TABLE_401__MemoryBootLevel__SHIFT 0x8 +#define DPM_TABLE_401__TemperatureLimitLow_MASK 0xffff0000 +#define DPM_TABLE_401__TemperatureLimitLow__SHIFT 0x10 +#define DPM_TABLE_402__MemoryThermThrottleEnable_MASK 0xff +#define DPM_TABLE_402__MemoryThermThrottleEnable__SHIFT 0x0 +#define DPM_TABLE_402__MemoryInterval_MASK 0xff00 +#define DPM_TABLE_402__MemoryInterval__SHIFT 0x8 +#define DPM_TABLE_402__BootMVdd_MASK 0xffff0000 +#define DPM_TABLE_402__BootMVdd__SHIFT 0x10 +#define DPM_TABLE_403__PhaseResponseTime_MASK 0xffff +#define DPM_TABLE_403__PhaseResponseTime__SHIFT 0x0 +#define DPM_TABLE_403__VoltageResponseTime_MASK 0xffff0000 +#define DPM_TABLE_403__VoltageResponseTime__SHIFT 0x10 +#define DPM_TABLE_404__DTEMode_MASK 0xff +#define DPM_TABLE_404__DTEMode__SHIFT 0x0 +#define DPM_TABLE_404__DTEInterval_MASK 0xff00 +#define DPM_TABLE_404__DTEInterval__SHIFT 0x8 +#define DPM_TABLE_404__PCIeGenInterval_MASK 0xff0000 +#define DPM_TABLE_404__PCIeGenInterval__SHIFT 0x10 +#define DPM_TABLE_404__PCIeBootLinkLevel_MASK 0xff000000 +#define DPM_TABLE_404__PCIeBootLinkLevel__SHIFT 0x18 +#define DPM_TABLE_405__ThermGpio_MASK 0xff +#define DPM_TABLE_405__ThermGpio__SHIFT 0x0 +#define DPM_TABLE_405__AcDcGpio_MASK 0xff00 +#define DPM_TABLE_405__AcDcGpio__SHIFT 0x8 +#define DPM_TABLE_405__VRHotGpio_MASK 0xff0000 +#define DPM_TABLE_405__VRHotGpio__SHIFT 0x10 +#define DPM_TABLE_405__SVI2Enable_MASK 0xff000000 +#define DPM_TABLE_405__SVI2Enable__SHIFT 0x18 +#define DPM_TABLE_406__PPM_TemperatureLimit_MASK 0xffff +#define DPM_TABLE_406__PPM_TemperatureLimit__SHIFT 0x0 +#define DPM_TABLE_406__PPM_PkgPwrLimit_MASK 0xffff0000 +#define DPM_TABLE_406__PPM_PkgPwrLimit__SHIFT 0x10 +#define DPM_TABLE_407__TargetTdp_MASK 0xffff +#define DPM_TABLE_407__TargetTdp__SHIFT 0x0 +#define DPM_TABLE_407__DefaultTdp_MASK 0xffff0000 +#define DPM_TABLE_407__DefaultTdp__SHIFT 0x10 +#define DPM_TABLE_408__FpsLowThreshold_MASK 0xffff +#define DPM_TABLE_408__FpsLowThreshold__SHIFT 0x0 +#define DPM_TABLE_408__FpsHighThreshold_MASK 0xffff0000 +#define DPM_TABLE_408__FpsHighThreshold__SHIFT 0x10 +#define DPM_TABLE_409__BAPMTI_R_0_1_0_MASK 0xffff +#define DPM_TABLE_409__BAPMTI_R_0_1_0__SHIFT 0x0 +#define DPM_TABLE_409__BAPMTI_R_0_0_0_MASK 0xffff0000 +#define DPM_TABLE_409__BAPMTI_R_0_0_0__SHIFT 0x10 +#define DPM_TABLE_410__BAPMTI_R_1_0_0_MASK 0xffff +#define DPM_TABLE_410__BAPMTI_R_1_0_0__SHIFT 0x0 +#define DPM_TABLE_410__BAPMTI_R_0_2_0_MASK 0xffff0000 +#define DPM_TABLE_410__BAPMTI_R_0_2_0__SHIFT 0x10 +#define DPM_TABLE_411__BAPMTI_R_1_2_0_MASK 0xffff +#define DPM_TABLE_411__BAPMTI_R_1_2_0__SHIFT 0x0 +#define DPM_TABLE_411__BAPMTI_R_1_1_0_MASK 0xffff0000 +#define DPM_TABLE_411__BAPMTI_R_1_1_0__SHIFT 0x10 +#define DPM_TABLE_412__BAPMTI_R_2_1_0_MASK 0xffff +#define DPM_TABLE_412__BAPMTI_R_2_1_0__SHIFT 0x0 +#define DPM_TABLE_412__BAPMTI_R_2_0_0_MASK 0xffff0000 +#define DPM_TABLE_412__BAPMTI_R_2_0_0__SHIFT 0x10 +#define DPM_TABLE_413__BAPMTI_R_3_0_0_MASK 0xffff +#define DPM_TABLE_413__BAPMTI_R_3_0_0__SHIFT 0x0 +#define DPM_TABLE_413__BAPMTI_R_2_2_0_MASK 0xffff0000 +#define DPM_TABLE_413__BAPMTI_R_2_2_0__SHIFT 0x10 +#define DPM_TABLE_414__BAPMTI_R_3_2_0_MASK 0xffff +#define DPM_TABLE_414__BAPMTI_R_3_2_0__SHIFT 0x0 +#define DPM_TABLE_414__BAPMTI_R_3_1_0_MASK 0xffff0000 +#define DPM_TABLE_414__BAPMTI_R_3_1_0__SHIFT 0x10 +#define DPM_TABLE_415__BAPMTI_R_4_1_0_MASK 0xffff +#define DPM_TABLE_415__BAPMTI_R_4_1_0__SHIFT 0x0 +#define DPM_TABLE_415__BAPMTI_R_4_0_0_MASK 0xffff0000 +#define DPM_TABLE_415__BAPMTI_R_4_0_0__SHIFT 0x10 +#define DPM_TABLE_416__BAPMTI_RC_0_0_0_MASK 0xffff +#define DPM_TABLE_416__BAPMTI_RC_0_0_0__SHIFT 0x0 +#define DPM_TABLE_416__BAPMTI_R_4_2_0_MASK 0xffff0000 +#define DPM_TABLE_416__BAPMTI_R_4_2_0__SHIFT 0x10 +#define DPM_TABLE_417__BAPMTI_RC_0_2_0_MASK 0xffff +#define DPM_TABLE_417__BAPMTI_RC_0_2_0__SHIFT 0x0 +#define DPM_TABLE_417__BAPMTI_RC_0_1_0_MASK 0xffff0000 +#define DPM_TABLE_417__BAPMTI_RC_0_1_0__SHIFT 0x10 +#define DPM_TABLE_418__BAPMTI_RC_1_1_0_MASK 0xffff +#define DPM_TABLE_418__BAPMTI_RC_1_1_0__SHIFT 0x0 +#define DPM_TABLE_418__BAPMTI_RC_1_0_0_MASK 0xffff0000 +#define DPM_TABLE_418__BAPMTI_RC_1_0_0__SHIFT 0x10 +#define DPM_TABLE_419__BAPMTI_RC_2_0_0_MASK 0xffff +#define DPM_TABLE_419__BAPMTI_RC_2_0_0__SHIFT 0x0 +#define DPM_TABLE_419__BAPMTI_RC_1_2_0_MASK 0xffff0000 +#define DPM_TABLE_419__BAPMTI_RC_1_2_0__SHIFT 0x10 +#define DPM_TABLE_420__BAPMTI_RC_2_2_0_MASK 0xffff +#define DPM_TABLE_420__BAPMTI_RC_2_2_0__SHIFT 0x0 +#define DPM_TABLE_420__BAPMTI_RC_2_1_0_MASK 0xffff0000 +#define DPM_TABLE_420__BAPMTI_RC_2_1_0__SHIFT 0x10 +#define DPM_TABLE_421__BAPMTI_RC_3_1_0_MASK 0xffff +#define DPM_TABLE_421__BAPMTI_RC_3_1_0__SHIFT 0x0 +#define DPM_TABLE_421__BAPMTI_RC_3_0_0_MASK 0xffff0000 +#define DPM_TABLE_421__BAPMTI_RC_3_0_0__SHIFT 0x10 +#define DPM_TABLE_422__BAPMTI_RC_4_0_0_MASK 0xffff +#define DPM_TABLE_422__BAPMTI_RC_4_0_0__SHIFT 0x0 +#define DPM_TABLE_422__BAPMTI_RC_3_2_0_MASK 0xffff0000 +#define DPM_TABLE_422__BAPMTI_RC_3_2_0__SHIFT 0x10 +#define DPM_TABLE_423__BAPMTI_RC_4_2_0_MASK 0xffff +#define DPM_TABLE_423__BAPMTI_RC_4_2_0__SHIFT 0x0 +#define DPM_TABLE_423__BAPMTI_RC_4_1_0_MASK 0xffff0000 +#define DPM_TABLE_423__BAPMTI_RC_4_1_0__SHIFT 0x10 +#define DPM_TABLE_424__GpuTjHyst_MASK 0xff +#define DPM_TABLE_424__GpuTjHyst__SHIFT 0x0 +#define DPM_TABLE_424__GpuTjMax_MASK 0xff00 +#define DPM_TABLE_424__GpuTjMax__SHIFT 0x8 +#define DPM_TABLE_424__DTETjOffset_MASK 0xff0000 +#define DPM_TABLE_424__DTETjOffset__SHIFT 0x10 +#define DPM_TABLE_424__DTEAmbientTempBase_MASK 0xff000000 +#define DPM_TABLE_424__DTEAmbientTempBase__SHIFT 0x18 +#define DPM_TABLE_425__BootVoltage_Phases_MASK 0xff +#define DPM_TABLE_425__BootVoltage_Phases__SHIFT 0x0 +#define DPM_TABLE_425__BootVoltage_VddGfx_MASK 0xff00 +#define DPM_TABLE_425__BootVoltage_VddGfx__SHIFT 0x8 +#define DPM_TABLE_425__BootVoltage_Vddci_MASK 0xff0000 +#define DPM_TABLE_425__BootVoltage_Vddci__SHIFT 0x10 +#define DPM_TABLE_425__BootVoltage_Vddc_MASK 0xff000000 +#define DPM_TABLE_425__BootVoltage_Vddc__SHIFT 0x18 +#define DPM_TABLE_426__BAPM_TEMP_GRADIENT_MASK 0xffffffff +#define DPM_TABLE_426__BAPM_TEMP_GRADIENT__SHIFT 0x0 +#define DPM_TABLE_427__LowSclkInterruptThreshold_MASK 0xffffffff +#define DPM_TABLE_427__LowSclkInterruptThreshold__SHIFT 0x0 +#define DPM_TABLE_428__VddGfxReChkWait_MASK 0xffffffff +#define DPM_TABLE_428__VddGfxReChkWait__SHIFT 0x0 +#define DPM_TABLE_429__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_padding_1_MASK 0xff +#define DPM_TABLE_429__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_padding_1__SHIFT 0x0 +#define DPM_TABLE_429__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_padding_0_MASK 0xff00 +#define DPM_TABLE_429__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_padding_0__SHIFT 0x8 +#define DPM_TABLE_429__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_maxVID_MASK 0xff0000 +#define DPM_TABLE_429__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_maxVID__SHIFT 0x10 +#define DPM_TABLE_429__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_minVID_MASK 0xff000000 +#define DPM_TABLE_429__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_minVID__SHIFT 0x18 +#define DPM_TABLE_430__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_setting_3_MASK 0xff +#define DPM_TABLE_430__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_setting_3__SHIFT 0x0 +#define DPM_TABLE_430__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_setting_2_MASK 0xff00 +#define DPM_TABLE_430__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_setting_2__SHIFT 0x8 +#define DPM_TABLE_430__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_setting_1_MASK 0xff0000 +#define DPM_TABLE_430__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_setting_1__SHIFT 0x10 +#define DPM_TABLE_430__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_setting_0_MASK 0xff000000 +#define DPM_TABLE_430__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_setting_0__SHIFT 0x18 +#define DPM_TABLE_431__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_setting_7_MASK 0xff +#define DPM_TABLE_431__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_setting_7__SHIFT 0x0 +#define DPM_TABLE_431__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_setting_6_MASK 0xff00 +#define DPM_TABLE_431__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_setting_6__SHIFT 0x8 +#define DPM_TABLE_431__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_setting_5_MASK 0xff0000 +#define DPM_TABLE_431__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_setting_5__SHIFT 0x10 +#define DPM_TABLE_431__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_setting_4_MASK 0xff000000 +#define DPM_TABLE_431__ClockStretcherDataTable_ClockStretcherDataTableEntry_0_setting_4__SHIFT 0x18 +#define DPM_TABLE_432__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_padding_1_MASK 0xff +#define DPM_TABLE_432__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_padding_1__SHIFT 0x0 +#define DPM_TABLE_432__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_padding_0_MASK 0xff00 +#define DPM_TABLE_432__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_padding_0__SHIFT 0x8 +#define DPM_TABLE_432__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_maxVID_MASK 0xff0000 +#define DPM_TABLE_432__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_maxVID__SHIFT 0x10 +#define DPM_TABLE_432__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_minVID_MASK 0xff000000 +#define DPM_TABLE_432__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_minVID__SHIFT 0x18 +#define DPM_TABLE_433__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_setting_3_MASK 0xff +#define DPM_TABLE_433__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_setting_3__SHIFT 0x0 +#define DPM_TABLE_433__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_setting_2_MASK 0xff00 +#define DPM_TABLE_433__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_setting_2__SHIFT 0x8 +#define DPM_TABLE_433__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_setting_1_MASK 0xff0000 +#define DPM_TABLE_433__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_setting_1__SHIFT 0x10 +#define DPM_TABLE_433__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_setting_0_MASK 0xff000000 +#define DPM_TABLE_433__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_setting_0__SHIFT 0x18 +#define DPM_TABLE_434__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_setting_7_MASK 0xff +#define DPM_TABLE_434__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_setting_7__SHIFT 0x0 +#define DPM_TABLE_434__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_setting_6_MASK 0xff00 +#define DPM_TABLE_434__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_setting_6__SHIFT 0x8 +#define DPM_TABLE_434__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_setting_5_MASK 0xff0000 +#define DPM_TABLE_434__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_setting_5__SHIFT 0x10 +#define DPM_TABLE_434__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_setting_4_MASK 0xff000000 +#define DPM_TABLE_434__ClockStretcherDataTable_ClockStretcherDataTableEntry_1_setting_4__SHIFT 0x18 +#define DPM_TABLE_435__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_padding_1_MASK 0xff +#define DPM_TABLE_435__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_padding_1__SHIFT 0x0 +#define DPM_TABLE_435__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_padding_0_MASK 0xff00 +#define DPM_TABLE_435__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_padding_0__SHIFT 0x8 +#define DPM_TABLE_435__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_maxVID_MASK 0xff0000 +#define DPM_TABLE_435__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_maxVID__SHIFT 0x10 +#define DPM_TABLE_435__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_minVID_MASK 0xff000000 +#define DPM_TABLE_435__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_minVID__SHIFT 0x18 +#define DPM_TABLE_436__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_setting_3_MASK 0xff +#define DPM_TABLE_436__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_setting_3__SHIFT 0x0 +#define DPM_TABLE_436__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_setting_2_MASK 0xff00 +#define DPM_TABLE_436__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_setting_2__SHIFT 0x8 +#define DPM_TABLE_436__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_setting_1_MASK 0xff0000 +#define DPM_TABLE_436__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_setting_1__SHIFT 0x10 +#define DPM_TABLE_436__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_setting_0_MASK 0xff000000 +#define DPM_TABLE_436__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_setting_0__SHIFT 0x18 +#define DPM_TABLE_437__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_setting_7_MASK 0xff +#define DPM_TABLE_437__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_setting_7__SHIFT 0x0 +#define DPM_TABLE_437__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_setting_6_MASK 0xff00 +#define DPM_TABLE_437__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_setting_6__SHIFT 0x8 +#define DPM_TABLE_437__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_setting_5_MASK 0xff0000 +#define DPM_TABLE_437__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_setting_5__SHIFT 0x10 +#define DPM_TABLE_437__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_setting_4_MASK 0xff000000 +#define DPM_TABLE_437__ClockStretcherDataTable_ClockStretcherDataTableEntry_2_setting_4__SHIFT 0x18 +#define DPM_TABLE_438__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_padding_1_MASK 0xff +#define DPM_TABLE_438__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_padding_1__SHIFT 0x0 +#define DPM_TABLE_438__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_padding_0_MASK 0xff00 +#define DPM_TABLE_438__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_padding_0__SHIFT 0x8 +#define DPM_TABLE_438__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_maxVID_MASK 0xff0000 +#define DPM_TABLE_438__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_maxVID__SHIFT 0x10 +#define DPM_TABLE_438__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_minVID_MASK 0xff000000 +#define DPM_TABLE_438__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_minVID__SHIFT 0x18 +#define DPM_TABLE_439__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_setting_3_MASK 0xff +#define DPM_TABLE_439__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_setting_3__SHIFT 0x0 +#define DPM_TABLE_439__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_setting_2_MASK 0xff00 +#define DPM_TABLE_439__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_setting_2__SHIFT 0x8 +#define DPM_TABLE_439__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_setting_1_MASK 0xff0000 +#define DPM_TABLE_439__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_setting_1__SHIFT 0x10 +#define DPM_TABLE_439__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_setting_0_MASK 0xff000000 +#define DPM_TABLE_439__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_setting_0__SHIFT 0x18 +#define DPM_TABLE_440__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_setting_7_MASK 0xff +#define DPM_TABLE_440__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_setting_7__SHIFT 0x0 +#define DPM_TABLE_440__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_setting_6_MASK 0xff00 +#define DPM_TABLE_440__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_setting_6__SHIFT 0x8 +#define DPM_TABLE_440__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_setting_5_MASK 0xff0000 +#define DPM_TABLE_440__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_setting_5__SHIFT 0x10 +#define DPM_TABLE_440__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_setting_4_MASK 0xff000000 +#define DPM_TABLE_440__ClockStretcherDataTable_ClockStretcherDataTableEntry_3_setting_4__SHIFT 0x18 +#define SOFT_REGISTERS_TABLE_1__RefClockFrequency_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_1__RefClockFrequency__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_2__PmTimerPeriod_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_2__PmTimerPeriod__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_3__FeatureEnables_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_3__FeatureEnables__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_4__PreVBlankGap_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_4__PreVBlankGap__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_5__VBlankTimeout_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_5__VBlankTimeout__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_6__TrainTimeGap_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_6__TrainTimeGap__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_7__MvddSwitchTime_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_7__MvddSwitchTime__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_8__LongestAcpiTrainTime_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_8__LongestAcpiTrainTime__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_9__AcpiDelay_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_9__AcpiDelay__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_10__G5TrainTime_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_10__G5TrainTime__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_11__DelayMpllPwron_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_11__DelayMpllPwron__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_12__VoltageChangeTimeout_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_12__VoltageChangeTimeout__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_13__HandshakeDisables_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_13__HandshakeDisables__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_14__DisplayPhy4Config_MASK 0xff +#define SOFT_REGISTERS_TABLE_14__DisplayPhy4Config__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_14__DisplayPhy3Config_MASK 0xff00 +#define SOFT_REGISTERS_TABLE_14__DisplayPhy3Config__SHIFT 0x8 +#define SOFT_REGISTERS_TABLE_14__DisplayPhy2Config_MASK 0xff0000 +#define SOFT_REGISTERS_TABLE_14__DisplayPhy2Config__SHIFT 0x10 +#define SOFT_REGISTERS_TABLE_14__DisplayPhy1Config_MASK 0xff000000 +#define SOFT_REGISTERS_TABLE_14__DisplayPhy1Config__SHIFT 0x18 +#define SOFT_REGISTERS_TABLE_15__DisplayPhy8Config_MASK 0xff +#define SOFT_REGISTERS_TABLE_15__DisplayPhy8Config__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_15__DisplayPhy7Config_MASK 0xff00 +#define SOFT_REGISTERS_TABLE_15__DisplayPhy7Config__SHIFT 0x8 +#define SOFT_REGISTERS_TABLE_15__DisplayPhy6Config_MASK 0xff0000 +#define SOFT_REGISTERS_TABLE_15__DisplayPhy6Config__SHIFT 0x10 +#define SOFT_REGISTERS_TABLE_15__DisplayPhy5Config_MASK 0xff000000 +#define SOFT_REGISTERS_TABLE_15__DisplayPhy5Config__SHIFT 0x18 +#define SOFT_REGISTERS_TABLE_16__AverageGraphicsActivity_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_16__AverageGraphicsActivity__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_17__AverageMemoryActivity_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_17__AverageMemoryActivity__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_18__AverageGioActivity_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_18__AverageGioActivity__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_19__PCIeDpmEnabledLevels_MASK 0xff +#define SOFT_REGISTERS_TABLE_19__PCIeDpmEnabledLevels__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_19__LClkDpmEnabledLevels_MASK 0xff00 +#define SOFT_REGISTERS_TABLE_19__LClkDpmEnabledLevels__SHIFT 0x8 +#define SOFT_REGISTERS_TABLE_19__MClkDpmEnabledLevels_MASK 0xff0000 +#define SOFT_REGISTERS_TABLE_19__MClkDpmEnabledLevels__SHIFT 0x10 +#define SOFT_REGISTERS_TABLE_19__SClkDpmEnabledLevels_MASK 0xff000000 +#define SOFT_REGISTERS_TABLE_19__SClkDpmEnabledLevels__SHIFT 0x18 +#define SOFT_REGISTERS_TABLE_20__VCEDpmEnabledLevels_MASK 0xff +#define SOFT_REGISTERS_TABLE_20__VCEDpmEnabledLevels__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_20__ACPDpmEnabledLevels_MASK 0xff00 +#define SOFT_REGISTERS_TABLE_20__ACPDpmEnabledLevels__SHIFT 0x8 +#define SOFT_REGISTERS_TABLE_20__SAMUDpmEnabledLevels_MASK 0xff0000 +#define SOFT_REGISTERS_TABLE_20__SAMUDpmEnabledLevels__SHIFT 0x10 +#define SOFT_REGISTERS_TABLE_20__UVDDpmEnabledLevels_MASK 0xff000000 +#define SOFT_REGISTERS_TABLE_20__UVDDpmEnabledLevels__SHIFT 0x18 +#define SOFT_REGISTERS_TABLE_21__DRAM_LOG_ADDR_H_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_21__DRAM_LOG_ADDR_H__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_22__DRAM_LOG_ADDR_L_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_22__DRAM_LOG_ADDR_L__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_23__DRAM_LOG_PHY_ADDR_H_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_23__DRAM_LOG_PHY_ADDR_H__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_24__DRAM_LOG_PHY_ADDR_L_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_24__DRAM_LOG_PHY_ADDR_L__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_25__DRAM_LOG_BUFF_SIZE_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_25__DRAM_LOG_BUFF_SIZE__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_26__UlvEnterCount_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_26__UlvEnterCount__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_27__UlvTime_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_27__UlvTime__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_28__UcodeLoadStatus_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_28__UcodeLoadStatus__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_29__Reserved_0_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_29__Reserved_0__SHIFT 0x0 +#define SOFT_REGISTERS_TABLE_30__Reserved_1_MASK 0xffffffff +#define SOFT_REGISTERS_TABLE_30__Reserved_1__SHIFT 0x0 +#define PM_FUSES_1__SviLoadLineOffsetVddC_MASK 0xff +#define PM_FUSES_1__SviLoadLineOffsetVddC__SHIFT 0x0 +#define PM_FUSES_1__SviLoadLineTrimVddC_MASK 0xff00 +#define PM_FUSES_1__SviLoadLineTrimVddC__SHIFT 0x8 +#define PM_FUSES_1__SviLoadLineVddC_MASK 0xff0000 +#define PM_FUSES_1__SviLoadLineVddC__SHIFT 0x10 +#define PM_FUSES_1__SviLoadLineEn_MASK 0xff000000 +#define PM_FUSES_1__SviLoadLineEn__SHIFT 0x18 +#define PM_FUSES_2__TDC_MAWt_MASK 0xff +#define PM_FUSES_2__TDC_MAWt__SHIFT 0x0 +#define PM_FUSES_2__TDC_VDDC_ThrottleReleaseLimitPerc_MASK 0xff00 +#define PM_FUSES_2__TDC_VDDC_ThrottleReleaseLimitPerc__SHIFT 0x8 +#define PM_FUSES_2__TDC_VDDC_PkgLimit_MASK 0xffff0000 +#define PM_FUSES_2__TDC_VDDC_PkgLimit__SHIFT 0x10 +#define PM_FUSES_3__Reserved_MASK 0xff +#define PM_FUSES_3__Reserved__SHIFT 0x0 +#define PM_FUSES_3__LPMLTemperatureMax_MASK 0xff00 +#define PM_FUSES_3__LPMLTemperatureMax__SHIFT 0x8 +#define PM_FUSES_3__LPMLTemperatureMin_MASK 0xff0000 +#define PM_FUSES_3__LPMLTemperatureMin__SHIFT 0x10 +#define PM_FUSES_3__TdcWaterfallCtl_MASK 0xff000000 +#define PM_FUSES_3__TdcWaterfallCtl__SHIFT 0x18 +#define PM_FUSES_4__LPMLTemperatureScaler_3_MASK 0xff +#define PM_FUSES_4__LPMLTemperatureScaler_3__SHIFT 0x0 +#define PM_FUSES_4__LPMLTemperatureScaler_2_MASK 0xff00 +#define PM_FUSES_4__LPMLTemperatureScaler_2__SHIFT 0x8 +#define PM_FUSES_4__LPMLTemperatureScaler_1_MASK 0xff0000 +#define PM_FUSES_4__LPMLTemperatureScaler_1__SHIFT 0x10 +#define PM_FUSES_4__LPMLTemperatureScaler_0_MASK 0xff000000 +#define PM_FUSES_4__LPMLTemperatureScaler_0__SHIFT 0x18 +#define PM_FUSES_5__LPMLTemperatureScaler_7_MASK 0xff +#define PM_FUSES_5__LPMLTemperatureScaler_7__SHIFT 0x0 +#define PM_FUSES_5__LPMLTemperatureScaler_6_MASK 0xff00 +#define PM_FUSES_5__LPMLTemperatureScaler_6__SHIFT 0x8 +#define PM_FUSES_5__LPMLTemperatureScaler_5_MASK 0xff0000 +#define PM_FUSES_5__LPMLTemperatureScaler_5__SHIFT 0x10 +#define PM_FUSES_5__LPMLTemperatureScaler_4_MASK 0xff000000 +#define PM_FUSES_5__LPMLTemperatureScaler_4__SHIFT 0x18 +#define PM_FUSES_6__LPMLTemperatureScaler_11_MASK 0xff +#define PM_FUSES_6__LPMLTemperatureScaler_11__SHIFT 0x0 +#define PM_FUSES_6__LPMLTemperatureScaler_10_MASK 0xff00 +#define PM_FUSES_6__LPMLTemperatureScaler_10__SHIFT 0x8 +#define PM_FUSES_6__LPMLTemperatureScaler_9_MASK 0xff0000 +#define PM_FUSES_6__LPMLTemperatureScaler_9__SHIFT 0x10 +#define PM_FUSES_6__LPMLTemperatureScaler_8_MASK 0xff000000 +#define PM_FUSES_6__LPMLTemperatureScaler_8__SHIFT 0x18 +#define PM_FUSES_7__LPMLTemperatureScaler_15_MASK 0xff +#define PM_FUSES_7__LPMLTemperatureScaler_15__SHIFT 0x0 +#define PM_FUSES_7__LPMLTemperatureScaler_14_MASK 0xff00 +#define PM_FUSES_7__LPMLTemperatureScaler_14__SHIFT 0x8 +#define PM_FUSES_7__LPMLTemperatureScaler_13_MASK 0xff0000 +#define PM_FUSES_7__LPMLTemperatureScaler_13__SHIFT 0x10 +#define PM_FUSES_7__LPMLTemperatureScaler_12_MASK 0xff000000 +#define PM_FUSES_7__LPMLTemperatureScaler_12__SHIFT 0x18 +#define PM_FUSES_8__FuzzyFan_ErrorRateSetDelta_MASK 0xffff +#define PM_FUSES_8__FuzzyFan_ErrorRateSetDelta__SHIFT 0x0 +#define PM_FUSES_8__FuzzyFan_ErrorSetDelta_MASK 0xffff0000 +#define PM_FUSES_8__FuzzyFan_ErrorSetDelta__SHIFT 0x10 +#define PM_FUSES_9__Reserved6_MASK 0xffff +#define PM_FUSES_9__Reserved6__SHIFT 0x0 +#define PM_FUSES_9__FuzzyFan_PwmSetDelta_MASK 0xffff0000 +#define PM_FUSES_9__FuzzyFan_PwmSetDelta__SHIFT 0x10 +#define PM_FUSES_10__GnbLPML_3_MASK 0xff +#define PM_FUSES_10__GnbLPML_3__SHIFT 0x0 +#define PM_FUSES_10__GnbLPML_2_MASK 0xff00 +#define PM_FUSES_10__GnbLPML_2__SHIFT 0x8 +#define PM_FUSES_10__GnbLPML_1_MASK 0xff0000 +#define PM_FUSES_10__GnbLPML_1__SHIFT 0x10 +#define PM_FUSES_10__GnbLPML_0_MASK 0xff000000 +#define PM_FUSES_10__GnbLPML_0__SHIFT 0x18 +#define PM_FUSES_11__GnbLPML_7_MASK 0xff +#define PM_FUSES_11__GnbLPML_7__SHIFT 0x0 +#define PM_FUSES_11__GnbLPML_6_MASK 0xff00 +#define PM_FUSES_11__GnbLPML_6__SHIFT 0x8 +#define PM_FUSES_11__GnbLPML_5_MASK 0xff0000 +#define PM_FUSES_11__GnbLPML_5__SHIFT 0x10 +#define PM_FUSES_11__GnbLPML_4_MASK 0xff000000 +#define PM_FUSES_11__GnbLPML_4__SHIFT 0x18 +#define PM_FUSES_12__GnbLPML_11_MASK 0xff +#define PM_FUSES_12__GnbLPML_11__SHIFT 0x0 +#define PM_FUSES_12__GnbLPML_10_MASK 0xff00 +#define PM_FUSES_12__GnbLPML_10__SHIFT 0x8 +#define PM_FUSES_12__GnbLPML_9_MASK 0xff0000 +#define PM_FUSES_12__GnbLPML_9__SHIFT 0x10 +#define PM_FUSES_12__GnbLPML_8_MASK 0xff000000 +#define PM_FUSES_12__GnbLPML_8__SHIFT 0x18 +#define PM_FUSES_13__GnbLPML_15_MASK 0xff +#define PM_FUSES_13__GnbLPML_15__SHIFT 0x0 +#define PM_FUSES_13__GnbLPML_14_MASK 0xff00 +#define PM_FUSES_13__GnbLPML_14__SHIFT 0x8 +#define PM_FUSES_13__GnbLPML_13_MASK 0xff0000 +#define PM_FUSES_13__GnbLPML_13__SHIFT 0x10 +#define PM_FUSES_13__GnbLPML_12_MASK 0xff000000 +#define PM_FUSES_13__GnbLPML_12__SHIFT 0x18 +#define PM_FUSES_14__Reserved1_1_MASK 0xff +#define PM_FUSES_14__Reserved1_1__SHIFT 0x0 +#define PM_FUSES_14__Reserved1_0_MASK 0xff00 +#define PM_FUSES_14__Reserved1_0__SHIFT 0x8 +#define PM_FUSES_14__GnbLPMLMinVid_MASK 0xff0000 +#define PM_FUSES_14__GnbLPMLMinVid__SHIFT 0x10 +#define PM_FUSES_14__GnbLPMLMaxVid_MASK 0xff000000 +#define PM_FUSES_14__GnbLPMLMaxVid__SHIFT 0x18 +#define PM_FUSES_15__BapmVddCBaseLeakageLoSidd_MASK 0xffff +#define PM_FUSES_15__BapmVddCBaseLeakageLoSidd__SHIFT 0x0 +#define PM_FUSES_15__BapmVddCBaseLeakageHiSidd_MASK 0xffff0000 +#define PM_FUSES_15__BapmVddCBaseLeakageHiSidd__SHIFT 0x10 +#define SMU_PM_STATUS_0__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_0__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_1__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_1__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_2__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_2__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_3__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_3__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_4__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_4__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_5__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_5__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_6__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_6__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_7__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_7__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_8__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_8__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_9__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_9__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_10__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_10__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_11__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_11__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_12__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_12__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_13__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_13__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_14__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_14__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_15__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_15__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_16__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_16__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_17__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_17__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_18__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_18__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_19__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_19__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_20__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_20__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_21__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_21__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_22__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_22__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_23__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_23__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_24__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_24__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_25__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_25__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_26__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_26__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_27__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_27__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_28__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_28__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_29__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_29__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_30__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_30__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_31__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_31__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_32__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_32__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_33__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_33__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_34__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_34__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_35__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_35__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_36__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_36__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_37__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_37__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_38__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_38__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_39__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_39__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_40__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_40__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_41__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_41__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_42__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_42__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_43__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_43__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_44__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_44__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_45__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_45__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_46__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_46__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_47__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_47__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_48__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_48__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_49__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_49__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_50__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_50__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_51__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_51__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_52__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_52__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_53__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_53__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_54__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_54__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_55__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_55__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_56__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_56__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_57__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_57__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_58__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_58__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_59__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_59__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_60__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_60__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_61__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_61__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_62__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_62__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_63__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_63__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_64__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_64__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_65__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_65__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_66__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_66__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_67__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_67__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_68__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_68__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_69__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_69__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_70__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_70__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_71__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_71__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_72__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_72__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_73__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_73__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_74__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_74__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_75__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_75__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_76__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_76__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_77__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_77__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_78__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_78__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_79__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_79__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_80__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_80__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_81__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_81__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_82__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_82__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_83__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_83__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_84__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_84__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_85__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_85__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_86__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_86__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_87__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_87__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_88__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_88__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_89__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_89__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_90__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_90__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_91__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_91__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_92__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_92__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_93__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_93__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_94__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_94__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_95__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_95__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_96__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_96__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_97__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_97__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_98__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_98__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_99__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_99__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_100__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_100__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_101__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_101__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_102__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_102__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_103__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_103__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_104__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_104__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_105__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_105__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_106__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_106__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_107__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_107__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_108__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_108__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_109__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_109__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_110__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_110__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_111__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_111__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_112__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_112__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_113__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_113__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_114__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_114__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_115__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_115__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_116__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_116__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_117__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_117__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_118__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_118__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_119__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_119__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_120__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_120__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_121__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_121__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_122__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_122__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_123__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_123__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_124__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_124__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_125__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_125__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_126__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_126__DATA__SHIFT 0x0 +#define SMU_PM_STATUS_127__DATA_MASK 0xffffffff +#define SMU_PM_STATUS_127__DATA__SHIFT 0x0 +#define CG_THERMAL_INT_ENA__THERM_INTH_SET_MASK 0x1 +#define CG_THERMAL_INT_ENA__THERM_INTH_SET__SHIFT 0x0 +#define CG_THERMAL_INT_ENA__THERM_INTL_SET_MASK 0x2 +#define CG_THERMAL_INT_ENA__THERM_INTL_SET__SHIFT 0x1 +#define CG_THERMAL_INT_ENA__THERM_TRIGGER_SET_MASK 0x4 +#define CG_THERMAL_INT_ENA__THERM_TRIGGER_SET__SHIFT 0x2 +#define CG_THERMAL_INT_ENA__THERM_INTH_CLR_MASK 0x8 +#define CG_THERMAL_INT_ENA__THERM_INTH_CLR__SHIFT 0x3 +#define CG_THERMAL_INT_ENA__THERM_INTL_CLR_MASK 0x10 +#define CG_THERMAL_INT_ENA__THERM_INTL_CLR__SHIFT 0x4 +#define CG_THERMAL_INT_ENA__THERM_TRIGGER_CLR_MASK 0x20 +#define CG_THERMAL_INT_ENA__THERM_TRIGGER_CLR__SHIFT 0x5 +#define CG_THERMAL_INT_CTRL__DIG_THERM_INTH_MASK 0xff +#define CG_THERMAL_INT_CTRL__DIG_THERM_INTH__SHIFT 0x0 +#define CG_THERMAL_INT_CTRL__DIG_THERM_INTL_MASK 0xff00 +#define CG_THERMAL_INT_CTRL__DIG_THERM_INTL__SHIFT 0x8 +#define CG_THERMAL_INT_CTRL__GNB_TEMP_THRESHOLD_MASK 0xff0000 +#define CG_THERMAL_INT_CTRL__GNB_TEMP_THRESHOLD__SHIFT 0x10 +#define CG_THERMAL_INT_CTRL__THERM_INTH_MASK_MASK 0x1000000 +#define CG_THERMAL_INT_CTRL__THERM_INTH_MASK__SHIFT 0x18 +#define CG_THERMAL_INT_CTRL__THERM_INTL_MASK_MASK 0x2000000 +#define CG_THERMAL_INT_CTRL__THERM_INTL_MASK__SHIFT 0x19 +#define CG_THERMAL_INT_CTRL__THERM_TRIGGER_MASK_MASK 0x4000000 +#define CG_THERMAL_INT_CTRL__THERM_TRIGGER_MASK__SHIFT 0x1a +#define CG_THERMAL_INT_CTRL__THERM_TRIGGER_CNB_MASK_MASK 0x8000000 +#define CG_THERMAL_INT_CTRL__THERM_TRIGGER_CNB_MASK__SHIFT 0x1b +#define CG_THERMAL_INT_CTRL__THERM_GNB_HW_ENA_MASK 0x10000000 +#define CG_THERMAL_INT_CTRL__THERM_GNB_HW_ENA__SHIFT 0x1c +#define CG_THERMAL_INT_STATUS__THERM_INTH_DETECT_MASK 0x1 +#define CG_THERMAL_INT_STATUS__THERM_INTH_DETECT__SHIFT 0x0 +#define CG_THERMAL_INT_STATUS__THERM_INTL_DETECT_MASK 0x2 +#define CG_THERMAL_INT_STATUS__THERM_INTL_DETECT__SHIFT 0x1 +#define CG_THERMAL_INT_STATUS__THERM_TRIGGER_DETECT_MASK 0x4 +#define CG_THERMAL_INT_STATUS__THERM_TRIGGER_DETECT__SHIFT 0x2 +#define CG_THERMAL_INT_STATUS__THERM_TRIGGER_CNB_DETECT_MASK 0x8 +#define CG_THERMAL_INT_STATUS__THERM_TRIGGER_CNB_DETECT__SHIFT 0x3 +#define CG_THERMAL_CTRL__DPM_EVENT_SRC_MASK 0x7 +#define CG_THERMAL_CTRL__DPM_EVENT_SRC__SHIFT 0x0 +#define CG_THERMAL_CTRL__THERM_INC_CLK_MASK 0x8 +#define CG_THERMAL_CTRL__THERM_INC_CLK__SHIFT 0x3 +#define CG_THERMAL_CTRL__SPARE_MASK 0x3ff0 +#define CG_THERMAL_CTRL__SPARE__SHIFT 0x4 +#define CG_THERMAL_CTRL__DIG_THERM_DPM_MASK 0x3fc000 +#define CG_THERMAL_CTRL__DIG_THERM_DPM__SHIFT 0xe +#define CG_THERMAL_CTRL__RESERVED_MASK 0x1c00000 +#define CG_THERMAL_CTRL__RESERVED__SHIFT 0x16 +#define CG_THERMAL_CTRL__CTF_PAD_POLARITY_MASK 0x2000000 +#define CG_THERMAL_CTRL__CTF_PAD_POLARITY__SHIFT 0x19 +#define CG_THERMAL_CTRL__CTF_PAD_EN_MASK 0x4000000 +#define CG_THERMAL_CTRL__CTF_PAD_EN__SHIFT 0x1a +#define CG_THERMAL_STATUS__SPARE_MASK 0x1ff +#define CG_THERMAL_STATUS__SPARE__SHIFT 0x0 +#define CG_THERMAL_STATUS__FDO_PWM_DUTY_MASK 0x1fe00 +#define CG_THERMAL_STATUS__FDO_PWM_DUTY__SHIFT 0x9 +#define CG_THERMAL_STATUS__THERM_ALERT_MASK 0x20000 +#define CG_THERMAL_STATUS__THERM_ALERT__SHIFT 0x11 +#define CG_THERMAL_STATUS__GEN_STATUS_MASK 0x3c0000 +#define CG_THERMAL_STATUS__GEN_STATUS__SHIFT 0x12 +#define CG_THERMAL_INT__DIG_THERM_CTF_MASK 0xff +#define CG_THERMAL_INT__DIG_THERM_CTF__SHIFT 0x0 +#define CG_THERMAL_INT__DIG_THERM_INTH_MASK 0xff00 +#define CG_THERMAL_INT__DIG_THERM_INTH__SHIFT 0x8 +#define CG_THERMAL_INT__DIG_THERM_INTL_MASK 0xff0000 +#define CG_THERMAL_INT__DIG_THERM_INTL__SHIFT 0x10 +#define CG_THERMAL_INT__THERM_INT_MASK_MASK 0xf000000 +#define CG_THERMAL_INT__THERM_INT_MASK__SHIFT 0x18 +#define CG_MULT_THERMAL_CTRL__TS_FILTER_MASK 0xf +#define CG_MULT_THERMAL_CTRL__TS_FILTER__SHIFT 0x0 +#define CG_MULT_THERMAL_CTRL__UNUSED_MASK 0x1f0 +#define CG_MULT_THERMAL_CTRL__UNUSED__SHIFT 0x4 +#define CG_MULT_THERMAL_CTRL__THERMAL_RANGE_RST_MASK 0x200 +#define CG_MULT_THERMAL_CTRL__THERMAL_RANGE_RST__SHIFT 0x9 +#define CG_MULT_THERMAL_CTRL__TEMP_SEL_MASK 0xff00000 +#define CG_MULT_THERMAL_CTRL__TEMP_SEL__SHIFT 0x14 +#define CG_MULT_THERMAL_CTRL__THM_READY_CLEAR_MASK 0x10000000 +#define CG_MULT_THERMAL_CTRL__THM_READY_CLEAR__SHIFT 0x1c +#define CG_MULT_THERMAL_STATUS__ASIC_MAX_TEMP_MASK 0x1ff +#define CG_MULT_THERMAL_STATUS__ASIC_MAX_TEMP__SHIFT 0x0 +#define CG_MULT_THERMAL_STATUS__CTF_TEMP_MASK 0x3fe00 +#define CG_MULT_THERMAL_STATUS__CTF_TEMP__SHIFT 0x9 +#define THM_TMON2_CTRL__POWER_DOWN_MASK 0x1 +#define THM_TMON2_CTRL__POWER_DOWN__SHIFT 0x0 +#define THM_TMON2_CTRL__BGADJ_MASK 0x1fe +#define THM_TMON2_CTRL__BGADJ__SHIFT 0x1 +#define THM_TMON2_CTRL__BGADJ_MODE_MASK 0x200 +#define THM_TMON2_CTRL__BGADJ_MODE__SHIFT 0x9 +#define THM_TMON2_CTRL__TMON_PAUSE_MASK 0x400 +#define THM_TMON2_CTRL__TMON_PAUSE__SHIFT 0xa +#define THM_TMON2_CTRL__INT_MEAS_EN_MASK 0x800 +#define THM_TMON2_CTRL__INT_MEAS_EN__SHIFT 0xb +#define THM_TMON2_CTRL__DEBUG_MODE_MASK 0x1000 +#define THM_TMON2_CTRL__DEBUG_MODE__SHIFT 0xc +#define THM_TMON2_CTRL__EN_CFG_SERDES_MASK 0x2000 +#define THM_TMON2_CTRL__EN_CFG_SERDES__SHIFT 0xd +#define THM_TMON2_CTRL2__RDIL_PRESENT_MASK 0xffff +#define THM_TMON2_CTRL2__RDIL_PRESENT__SHIFT 0x0 +#define THM_TMON2_CTRL2__RDIR_PRESENT_MASK 0xffff0000 +#define THM_TMON2_CTRL2__RDIR_PRESENT__SHIFT 0x10 +#define THM_TMON2_CSR_WR__CSR_WRITE_MASK 0x1 +#define THM_TMON2_CSR_WR__CSR_WRITE__SHIFT 0x0 +#define THM_TMON2_CSR_WR__CSR_READ_MASK 0x2 +#define THM_TMON2_CSR_WR__CSR_READ__SHIFT 0x1 +#define THM_TMON2_CSR_WR__CSR_ADDR_MASK 0xffc +#define THM_TMON2_CSR_WR__CSR_ADDR__SHIFT 0x2 +#define THM_TMON2_CSR_WR__WRITE_DATA_MASK 0xfff000 +#define THM_TMON2_CSR_WR__WRITE_DATA__SHIFT 0xc +#define THM_TMON2_CSR_WR__SPARE_MASK 0x1000000 +#define THM_TMON2_CSR_WR__SPARE__SHIFT 0x18 +#define THM_TMON2_CSR_RD__READ_DATA_MASK 0xfff +#define THM_TMON2_CSR_RD__READ_DATA__SHIFT 0x0 +#define CG_FDO_CTRL0__FDO_STATIC_DUTY_MASK 0xff +#define CG_FDO_CTRL0__FDO_STATIC_DUTY__SHIFT 0x0 +#define CG_FDO_CTRL0__FAN_SPINUP_DUTY_MASK 0xff00 +#define CG_FDO_CTRL0__FAN_SPINUP_DUTY__SHIFT 0x8 +#define CG_FDO_CTRL0__FDO_PWM_MANUAL_MASK 0x10000 +#define CG_FDO_CTRL0__FDO_PWM_MANUAL__SHIFT 0x10 +#define CG_FDO_CTRL0__FDO_PWM_HYSTER_MASK 0x7e0000 +#define CG_FDO_CTRL0__FDO_PWM_HYSTER__SHIFT 0x11 +#define CG_FDO_CTRL0__FDO_PWM_RAMP_EN_MASK 0x800000 +#define CG_FDO_CTRL0__FDO_PWM_RAMP_EN__SHIFT 0x17 +#define CG_FDO_CTRL0__FDO_PWM_RAMP_MASK 0xff000000 +#define CG_FDO_CTRL0__FDO_PWM_RAMP__SHIFT 0x18 +#define CG_FDO_CTRL1__FMAX_DUTY100_MASK 0xff +#define CG_FDO_CTRL1__FMAX_DUTY100__SHIFT 0x0 +#define CG_FDO_CTRL1__FMIN_DUTY_MASK 0xff00 +#define CG_FDO_CTRL1__FMIN_DUTY__SHIFT 0x8 +#define CG_FDO_CTRL1__M_MASK 0xff0000 +#define CG_FDO_CTRL1__M__SHIFT 0x10 +#define CG_FDO_CTRL1__RESERVED_MASK 0x3f000000 +#define CG_FDO_CTRL1__RESERVED__SHIFT 0x18 +#define CG_FDO_CTRL1__FDO_PWRDNB_MASK 0x40000000 +#define CG_FDO_CTRL1__FDO_PWRDNB__SHIFT 0x1e +#define CG_FDO_CTRL2__TMIN_MASK 0xff +#define CG_FDO_CTRL2__TMIN__SHIFT 0x0 +#define CG_FDO_CTRL2__FAN_SPINUP_TIME_MASK 0x700 +#define CG_FDO_CTRL2__FAN_SPINUP_TIME__SHIFT 0x8 +#define CG_FDO_CTRL2__FDO_PWM_MODE_MASK 0x3800 +#define CG_FDO_CTRL2__FDO_PWM_MODE__SHIFT 0xb +#define CG_FDO_CTRL2__TMIN_HYSTER_MASK 0x1c000 +#define CG_FDO_CTRL2__TMIN_HYSTER__SHIFT 0xe +#define CG_FDO_CTRL2__TMAX_MASK 0x1fe0000 +#define CG_FDO_CTRL2__TMAX__SHIFT 0x11 +#define CG_FDO_CTRL2__TACH_PWM_RESP_RATE_MASK 0xfe000000 +#define CG_FDO_CTRL2__TACH_PWM_RESP_RATE__SHIFT 0x19 +#define CG_TACH_CTRL__EDGE_PER_REV_MASK 0x7 +#define CG_TACH_CTRL__EDGE_PER_REV__SHIFT 0x0 +#define CG_TACH_CTRL__TARGET_PERIOD_MASK 0xfffffff8 +#define CG_TACH_CTRL__TARGET_PERIOD__SHIFT 0x3 +#define CG_TACH_STATUS__TACH_PERIOD_MASK 0xffffffff +#define CG_TACH_STATUS__TACH_PERIOD__SHIFT 0x0 +#define CC_THM_STRAPS0__TMON0_BGADJ_MASK 0x1fe +#define CC_THM_STRAPS0__TMON0_BGADJ__SHIFT 0x1 +#define CC_THM_STRAPS0__TMON1_BGADJ_MASK 0x1fe00 +#define CC_THM_STRAPS0__TMON1_BGADJ__SHIFT 0x9 +#define CC_THM_STRAPS0__TMON_CMON_FUSE_SEL_MASK 0x20000 +#define CC_THM_STRAPS0__TMON_CMON_FUSE_SEL__SHIFT 0x11 +#define CC_THM_STRAPS0__NUM_ACQ_MASK 0x1c0000 +#define CC_THM_STRAPS0__NUM_ACQ__SHIFT 0x12 +#define CC_THM_STRAPS0__TMON_CLK_SEL_MASK 0xe00000 +#define CC_THM_STRAPS0__TMON_CLK_SEL__SHIFT 0x15 +#define CC_THM_STRAPS0__TMON_CONFIG_SOURCE_MASK 0x1000000 +#define CC_THM_STRAPS0__TMON_CONFIG_SOURCE__SHIFT 0x18 +#define CC_THM_STRAPS0__CTF_DISABLE_MASK 0x2000000 +#define CC_THM_STRAPS0__CTF_DISABLE__SHIFT 0x19 +#define CC_THM_STRAPS0__TMON0_DISABLE_MASK 0x4000000 +#define CC_THM_STRAPS0__TMON0_DISABLE__SHIFT 0x1a +#define CC_THM_STRAPS0__TMON1_DISABLE_MASK 0x8000000 +#define CC_THM_STRAPS0__TMON1_DISABLE__SHIFT 0x1b +#define CC_THM_STRAPS0__TMON2_DISABLE_MASK 0x10000000 +#define CC_THM_STRAPS0__TMON2_DISABLE__SHIFT 0x1c +#define CC_THM_STRAPS0__TMON3_DISABLE_MASK 0x20000000 +#define CC_THM_STRAPS0__TMON3_DISABLE__SHIFT 0x1d +#define CC_THM_STRAPS0__UNUSED_MASK 0x80000000 +#define CC_THM_STRAPS0__UNUSED__SHIFT 0x1f +#define THM_TMON0_RDIL0_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIL0_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIL0_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIL0_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIL0_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIL0_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIL1_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIL1_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIL1_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIL1_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIL1_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIL1_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIL2_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIL2_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIL2_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIL2_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIL2_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIL2_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIL3_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIL3_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIL3_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIL3_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIL3_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIL3_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIL4_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIL4_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIL4_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIL4_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIL4_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIL4_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIL5_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIL5_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIL5_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIL5_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIL5_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIL5_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIL6_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIL6_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIL6_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIL6_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIL6_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIL6_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIL7_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIL7_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIL7_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIL7_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIL7_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIL7_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIL8_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIL8_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIL8_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIL8_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIL8_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIL8_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIL9_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIL9_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIL9_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIL9_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIL9_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIL9_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIL10_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIL10_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIL10_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIL10_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIL10_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIL10_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIL11_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIL11_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIL11_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIL11_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIL11_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIL11_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIL12_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIL12_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIL12_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIL12_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIL12_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIL12_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIL13_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIL13_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIL13_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIL13_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIL13_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIL13_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIL14_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIL14_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIL14_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIL14_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIL14_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIL14_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIL15_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIL15_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIL15_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIL15_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIL15_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIL15_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIR0_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIR0_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIR0_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIR0_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIR0_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIR0_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIR1_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIR1_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIR1_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIR1_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIR1_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIR1_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIR2_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIR2_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIR2_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIR2_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIR2_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIR2_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIR3_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIR3_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIR3_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIR3_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIR3_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIR3_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIR4_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIR4_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIR4_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIR4_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIR4_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIR4_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIR5_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIR5_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIR5_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIR5_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIR5_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIR5_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIR6_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIR6_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIR6_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIR6_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIR6_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIR6_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIR7_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIR7_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIR7_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIR7_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIR7_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIR7_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIR8_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIR8_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIR8_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIR8_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIR8_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIR8_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIR9_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIR9_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIR9_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIR9_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIR9_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIR9_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIR10_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIR10_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIR10_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIR10_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIR10_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIR10_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIR11_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIR11_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIR11_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIR11_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIR11_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIR11_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIR12_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIR12_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIR12_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIR12_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIR12_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIR12_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIR13_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIR13_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIR13_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIR13_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIR13_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIR13_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIR14_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIR14_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIR14_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIR14_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIR14_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIR14_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_RDIR15_DATA__Z_MASK 0x7ff +#define THM_TMON0_RDIR15_DATA__Z__SHIFT 0x0 +#define THM_TMON0_RDIR15_DATA__VALID_MASK 0x800 +#define THM_TMON0_RDIR15_DATA__VALID__SHIFT 0xb +#define THM_TMON0_RDIR15_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_RDIR15_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIL0_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIL0_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIL0_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIL0_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIL0_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIL0_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIL1_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIL1_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIL1_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIL1_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIL1_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIL1_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIL2_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIL2_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIL2_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIL2_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIL2_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIL2_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIL3_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIL3_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIL3_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIL3_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIL3_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIL3_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIL4_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIL4_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIL4_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIL4_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIL4_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIL4_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIL5_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIL5_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIL5_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIL5_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIL5_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIL5_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIL6_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIL6_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIL6_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIL6_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIL6_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIL6_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIL7_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIL7_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIL7_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIL7_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIL7_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIL7_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIL8_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIL8_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIL8_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIL8_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIL8_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIL8_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIL9_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIL9_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIL9_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIL9_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIL9_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIL9_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIL10_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIL10_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIL10_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIL10_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIL10_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIL10_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIL11_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIL11_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIL11_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIL11_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIL11_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIL11_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIL12_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIL12_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIL12_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIL12_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIL12_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIL12_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIL13_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIL13_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIL13_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIL13_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIL13_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIL13_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIL14_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIL14_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIL14_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIL14_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIL14_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIL14_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIL15_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIL15_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIL15_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIL15_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIL15_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIL15_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIR0_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIR0_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIR0_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIR0_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIR0_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIR0_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIR1_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIR1_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIR1_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIR1_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIR1_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIR1_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIR2_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIR2_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIR2_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIR2_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIR2_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIR2_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIR3_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIR3_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIR3_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIR3_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIR3_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIR3_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIR4_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIR4_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIR4_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIR4_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIR4_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIR4_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIR5_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIR5_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIR5_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIR5_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIR5_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIR5_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIR6_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIR6_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIR6_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIR6_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIR6_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIR6_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIR7_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIR7_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIR7_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIR7_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIR7_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIR7_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIR8_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIR8_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIR8_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIR8_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIR8_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIR8_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIR9_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIR9_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIR9_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIR9_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIR9_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIR9_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIR10_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIR10_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIR10_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIR10_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIR10_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIR10_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIR11_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIR11_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIR11_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIR11_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIR11_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIR11_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIR12_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIR12_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIR12_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIR12_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIR12_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIR12_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIR13_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIR13_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIR13_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIR13_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIR13_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIR13_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIR14_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIR14_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIR14_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIR14_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIR14_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIR14_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_RDIR15_DATA__Z_MASK 0x7ff +#define THM_TMON1_RDIR15_DATA__Z__SHIFT 0x0 +#define THM_TMON1_RDIR15_DATA__VALID_MASK 0x800 +#define THM_TMON1_RDIR15_DATA__VALID__SHIFT 0xb +#define THM_TMON1_RDIR15_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_RDIR15_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIL0_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIL0_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIL0_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIL0_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIL0_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIL0_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIL1_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIL1_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIL1_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIL1_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIL1_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIL1_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIL2_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIL2_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIL2_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIL2_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIL2_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIL2_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIL3_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIL3_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIL3_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIL3_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIL3_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIL3_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIL4_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIL4_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIL4_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIL4_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIL4_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIL4_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIL5_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIL5_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIL5_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIL5_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIL5_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIL5_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIL6_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIL6_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIL6_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIL6_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIL6_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIL6_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIL7_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIL7_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIL7_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIL7_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIL7_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIL7_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIL8_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIL8_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIL8_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIL8_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIL8_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIL8_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIL9_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIL9_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIL9_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIL9_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIL9_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIL9_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIL10_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIL10_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIL10_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIL10_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIL10_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIL10_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIL11_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIL11_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIL11_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIL11_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIL11_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIL11_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIL12_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIL12_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIL12_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIL12_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIL12_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIL12_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIL13_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIL13_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIL13_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIL13_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIL13_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIL13_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIL14_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIL14_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIL14_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIL14_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIL14_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIL14_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIL15_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIL15_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIL15_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIL15_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIL15_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIL15_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIR0_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIR0_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIR0_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIR0_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIR0_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIR0_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIR1_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIR1_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIR1_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIR1_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIR1_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIR1_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIR2_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIR2_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIR2_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIR2_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIR2_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIR2_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIR3_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIR3_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIR3_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIR3_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIR3_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIR3_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIR4_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIR4_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIR4_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIR4_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIR4_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIR4_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIR5_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIR5_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIR5_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIR5_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIR5_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIR5_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIR6_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIR6_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIR6_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIR6_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIR6_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIR6_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIR7_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIR7_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIR7_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIR7_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIR7_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIR7_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIR8_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIR8_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIR8_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIR8_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIR8_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIR8_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIR9_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIR9_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIR9_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIR9_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIR9_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIR9_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIR10_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIR10_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIR10_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIR10_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIR10_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIR10_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIR11_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIR11_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIR11_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIR11_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIR11_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIR11_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIR12_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIR12_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIR12_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIR12_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIR12_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIR12_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIR13_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIR13_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIR13_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIR13_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIR13_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIR13_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIR14_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIR14_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIR14_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIR14_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIR14_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIR14_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_RDIR15_DATA__Z_MASK 0x7ff +#define THM_TMON2_RDIR15_DATA__Z__SHIFT 0x0 +#define THM_TMON2_RDIR15_DATA__VALID_MASK 0x800 +#define THM_TMON2_RDIR15_DATA__VALID__SHIFT 0xb +#define THM_TMON2_RDIR15_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_RDIR15_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_INT_DATA__Z_MASK 0x7ff +#define THM_TMON0_INT_DATA__Z__SHIFT 0x0 +#define THM_TMON0_INT_DATA__VALID_MASK 0x800 +#define THM_TMON0_INT_DATA__VALID__SHIFT 0xb +#define THM_TMON0_INT_DATA__TEMP_MASK 0xfff000 +#define THM_TMON0_INT_DATA__TEMP__SHIFT 0xc +#define THM_TMON1_INT_DATA__Z_MASK 0x7ff +#define THM_TMON1_INT_DATA__Z__SHIFT 0x0 +#define THM_TMON1_INT_DATA__VALID_MASK 0x800 +#define THM_TMON1_INT_DATA__VALID__SHIFT 0xb +#define THM_TMON1_INT_DATA__TEMP_MASK 0xfff000 +#define THM_TMON1_INT_DATA__TEMP__SHIFT 0xc +#define THM_TMON2_INT_DATA__Z_MASK 0x7ff +#define THM_TMON2_INT_DATA__Z__SHIFT 0x0 +#define THM_TMON2_INT_DATA__VALID_MASK 0x800 +#define THM_TMON2_INT_DATA__VALID__SHIFT 0xb +#define THM_TMON2_INT_DATA__TEMP_MASK 0xfff000 +#define THM_TMON2_INT_DATA__TEMP__SHIFT 0xc +#define THM_TMON0_DEBUG__DEBUG_RDI_MASK 0x1f +#define THM_TMON0_DEBUG__DEBUG_RDI__SHIFT 0x0 +#define THM_TMON0_DEBUG__DEBUG_Z_MASK 0xffe0 +#define THM_TMON0_DEBUG__DEBUG_Z__SHIFT 0x5 +#define THM_TMON1_DEBUG__DEBUG_RDI_MASK 0x1f +#define THM_TMON1_DEBUG__DEBUG_RDI__SHIFT 0x0 +#define THM_TMON1_DEBUG__DEBUG_Z_MASK 0xffe0 +#define THM_TMON1_DEBUG__DEBUG_Z__SHIFT 0x5 +#define THM_TMON2_DEBUG__DEBUG_RDI_MASK 0x1f +#define THM_TMON2_DEBUG__DEBUG_RDI__SHIFT 0x0 +#define THM_TMON2_DEBUG__DEBUG_Z_MASK 0xffe0 +#define THM_TMON2_DEBUG__DEBUG_Z__SHIFT 0x5 +#define THM_TMON0_STATUS__CURRENT_RDI_MASK 0x1f +#define THM_TMON0_STATUS__CURRENT_RDI__SHIFT 0x0 +#define THM_TMON0_STATUS__MEAS_DONE_MASK 0x20 +#define THM_TMON0_STATUS__MEAS_DONE__SHIFT 0x5 +#define THM_TMON1_STATUS__CURRENT_RDI_MASK 0x1f +#define THM_TMON1_STATUS__CURRENT_RDI__SHIFT 0x0 +#define THM_TMON1_STATUS__MEAS_DONE_MASK 0x20 +#define THM_TMON1_STATUS__MEAS_DONE__SHIFT 0x5 +#define THM_TMON2_STATUS__CURRENT_RDI_MASK 0x1f +#define THM_TMON2_STATUS__CURRENT_RDI__SHIFT 0x0 +#define THM_TMON2_STATUS__MEAS_DONE_MASK 0x20 +#define THM_TMON2_STATUS__MEAS_DONE__SHIFT 0x5 +#define GENERAL_PWRMGT__GLOBAL_PWRMGT_EN_MASK 0x1 +#define GENERAL_PWRMGT__GLOBAL_PWRMGT_EN__SHIFT 0x0 +#define GENERAL_PWRMGT__STATIC_PM_EN_MASK 0x2 +#define GENERAL_PWRMGT__STATIC_PM_EN__SHIFT 0x1 +#define GENERAL_PWRMGT__THERMAL_PROTECTION_DIS_MASK 0x4 +#define GENERAL_PWRMGT__THERMAL_PROTECTION_DIS__SHIFT 0x2 +#define GENERAL_PWRMGT__THERMAL_PROTECTION_TYPE_MASK 0x8 +#define GENERAL_PWRMGT__THERMAL_PROTECTION_TYPE__SHIFT 0x3 +#define GENERAL_PWRMGT__SW_SMIO_INDEX_MASK 0x40 +#define GENERAL_PWRMGT__SW_SMIO_INDEX__SHIFT 0x6 +#define GENERAL_PWRMGT__LOW_VOLT_D2_ACPI_MASK 0x100 +#define GENERAL_PWRMGT__LOW_VOLT_D2_ACPI__SHIFT 0x8 +#define GENERAL_PWRMGT__LOW_VOLT_D3_ACPI_MASK 0x200 +#define GENERAL_PWRMGT__LOW_VOLT_D3_ACPI__SHIFT 0x9 +#define GENERAL_PWRMGT__VOLT_PWRMGT_EN_MASK 0x400 +#define GENERAL_PWRMGT__VOLT_PWRMGT_EN__SHIFT 0xa +#define GENERAL_PWRMGT__SPARE11_MASK 0x800 +#define GENERAL_PWRMGT__SPARE11__SHIFT 0xb +#define GENERAL_PWRMGT__GPU_COUNTER_ACPI_MASK 0x4000 +#define GENERAL_PWRMGT__GPU_COUNTER_ACPI__SHIFT 0xe +#define GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK 0x8000 +#define GENERAL_PWRMGT__GPU_COUNTER_CLK__SHIFT 0xf +#define GENERAL_PWRMGT__GPU_COUNTER_OFF_MASK 0x10000 +#define GENERAL_PWRMGT__GPU_COUNTER_OFF__SHIFT 0x10 +#define GENERAL_PWRMGT__GPU_COUNTER_INTF_OFF_MASK 0x20000 +#define GENERAL_PWRMGT__GPU_COUNTER_INTF_OFF__SHIFT 0x11 +#define GENERAL_PWRMGT__SPARE18_MASK 0x40000 +#define GENERAL_PWRMGT__SPARE18__SHIFT 0x12 +#define GENERAL_PWRMGT__ACPI_D3_VID_MASK 0x180000 +#define GENERAL_PWRMGT__ACPI_D3_VID__SHIFT 0x13 +#define GENERAL_PWRMGT__DYN_SPREAD_SPECTRUM_EN_MASK 0x800000 +#define GENERAL_PWRMGT__DYN_SPREAD_SPECTRUM_EN__SHIFT 0x17 +#define GENERAL_PWRMGT__SPARE27_MASK 0x8000000 +#define GENERAL_PWRMGT__SPARE27__SHIFT 0x1b +#define GENERAL_PWRMGT__SPARE_MASK 0xf0000000 +#define GENERAL_PWRMGT__SPARE__SHIFT 0x1c +#define CNB_PWRMGT_CNTL__GNB_SLOW_MODE_MASK 0x3 +#define CNB_PWRMGT_CNTL__GNB_SLOW_MODE__SHIFT 0x0 +#define CNB_PWRMGT_CNTL__GNB_SLOW_MASK 0x4 +#define CNB_PWRMGT_CNTL__GNB_SLOW__SHIFT 0x2 +#define CNB_PWRMGT_CNTL__FORCE_NB_PS1_MASK 0x8 +#define CNB_PWRMGT_CNTL__FORCE_NB_PS1__SHIFT 0x3 +#define CNB_PWRMGT_CNTL__DPM_ENABLED_MASK 0x10 +#define CNB_PWRMGT_CNTL__DPM_ENABLED__SHIFT 0x4 +#define CNB_PWRMGT_CNTL__SPARE_MASK 0xffffffe0 +#define CNB_PWRMGT_CNTL__SPARE__SHIFT 0x5 +#define SCLK_PWRMGT_CNTL__SCLK_PWRMGT_OFF_MASK 0x1 +#define SCLK_PWRMGT_CNTL__SCLK_PWRMGT_OFF__SHIFT 0x0 +#define SCLK_PWRMGT_CNTL__RESET_BUSY_CNT_MASK 0x10 +#define SCLK_PWRMGT_CNTL__RESET_BUSY_CNT__SHIFT 0x4 +#define SCLK_PWRMGT_CNTL__RESET_SCLK_CNT_MASK 0x20 +#define SCLK_PWRMGT_CNTL__RESET_SCLK_CNT__SHIFT 0x5 +#define SCLK_PWRMGT_CNTL__DYN_LIGHT_SLEEP_EN_MASK 0x4000 +#define SCLK_PWRMGT_CNTL__DYN_LIGHT_SLEEP_EN__SHIFT 0xe +#define SCLK_PWRMGT_CNTL__AUTO_SCLK_PULSE_SKIP_MASK 0x8000 +#define SCLK_PWRMGT_CNTL__AUTO_SCLK_PULSE_SKIP__SHIFT 0xf +#define SCLK_PWRMGT_CNTL__LIGHT_SLEEP_COUNTER_MASK 0x1f0000 +#define SCLK_PWRMGT_CNTL__LIGHT_SLEEP_COUNTER__SHIFT 0x10 +#define SCLK_PWRMGT_CNTL__DYNAMIC_PM_EN_MASK 0x200000 +#define SCLK_PWRMGT_CNTL__DYNAMIC_PM_EN__SHIFT 0x15 +#define TARGET_AND_CURRENT_PROFILE_INDEX__TARGET_STATE_MASK 0xf +#define TARGET_AND_CURRENT_PROFILE_INDEX__TARGET_STATE__SHIFT 0x0 +#define TARGET_AND_CURRENT_PROFILE_INDEX__CURRENT_STATE_MASK 0xf0 +#define TARGET_AND_CURRENT_PROFILE_INDEX__CURRENT_STATE__SHIFT 0x4 +#define TARGET_AND_CURRENT_PROFILE_INDEX__CURR_MCLK_INDEX_MASK 0xf00 +#define TARGET_AND_CURRENT_PROFILE_INDEX__CURR_MCLK_INDEX__SHIFT 0x8 +#define TARGET_AND_CURRENT_PROFILE_INDEX__TARG_MCLK_INDEX_MASK 0xf000 +#define TARGET_AND_CURRENT_PROFILE_INDEX__TARG_MCLK_INDEX__SHIFT 0xc +#define TARGET_AND_CURRENT_PROFILE_INDEX__CURR_SCLK_INDEX_MASK 0x1f0000 +#define TARGET_AND_CURRENT_PROFILE_INDEX__CURR_SCLK_INDEX__SHIFT 0x10 +#define TARGET_AND_CURRENT_PROFILE_INDEX__TARG_SCLK_INDEX_MASK 0x3e00000 +#define TARGET_AND_CURRENT_PROFILE_INDEX__TARG_SCLK_INDEX__SHIFT 0x15 +#define TARGET_AND_CURRENT_PROFILE_INDEX__CURR_LCLK_INDEX_MASK 0x1c000000 +#define TARGET_AND_CURRENT_PROFILE_INDEX__CURR_LCLK_INDEX__SHIFT 0x1a +#define TARGET_AND_CURRENT_PROFILE_INDEX__TARG_LCLK_INDEX_MASK 0xe0000000 +#define TARGET_AND_CURRENT_PROFILE_INDEX__TARG_LCLK_INDEX__SHIFT 0x1d +#define PWR_PCC_CONTROL__PCC_POLARITY_MASK 0x1 +#define PWR_PCC_CONTROL__PCC_POLARITY__SHIFT 0x0 +#define PWR_PCC_GPIO_SELECT__GPIO_MASK 0xffffffff +#define PWR_PCC_GPIO_SELECT__GPIO__SHIFT 0x0 +#define CG_FREQ_TRAN_VOTING_0__BIF_FREQ_THROTTLING_VOTE_EN_MASK 0x1 +#define CG_FREQ_TRAN_VOTING_0__BIF_FREQ_THROTTLING_VOTE_EN__SHIFT 0x0 +#define CG_FREQ_TRAN_VOTING_0__HDP_FREQ_THROTTLING_VOTE_EN_MASK 0x2 +#define CG_FREQ_TRAN_VOTING_0__HDP_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1 +#define CG_FREQ_TRAN_VOTING_0__ROM_FREQ_THROTTLING_VOTE_EN_MASK 0x4 +#define CG_FREQ_TRAN_VOTING_0__ROM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x2 +#define CG_FREQ_TRAN_VOTING_0__IH_SEM_FREQ_THROTTLING_VOTE_EN_MASK 0x8 +#define CG_FREQ_TRAN_VOTING_0__IH_SEM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x3 +#define CG_FREQ_TRAN_VOTING_0__PDMA_FREQ_THROTTLING_VOTE_EN_MASK 0x10 +#define CG_FREQ_TRAN_VOTING_0__PDMA_FREQ_THROTTLING_VOTE_EN__SHIFT 0x4 +#define CG_FREQ_TRAN_VOTING_0__DRM_FREQ_THROTTLING_VOTE_EN_MASK 0x20 +#define CG_FREQ_TRAN_VOTING_0__DRM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x5 +#define CG_FREQ_TRAN_VOTING_0__IDCT_FREQ_THROTTLING_VOTE_EN_MASK 0x40 +#define CG_FREQ_TRAN_VOTING_0__IDCT_FREQ_THROTTLING_VOTE_EN__SHIFT 0x6 +#define CG_FREQ_TRAN_VOTING_0__ACP_FREQ_THROTTLING_VOTE_EN_MASK 0x80 +#define CG_FREQ_TRAN_VOTING_0__ACP_FREQ_THROTTLING_VOTE_EN__SHIFT 0x7 +#define CG_FREQ_TRAN_VOTING_0__SDMA_FREQ_THROTTLING_VOTE_EN_MASK 0x100 +#define CG_FREQ_TRAN_VOTING_0__SDMA_FREQ_THROTTLING_VOTE_EN__SHIFT 0x8 +#define CG_FREQ_TRAN_VOTING_0__UVD_FREQ_THROTTLING_VOTE_EN_MASK 0x200 +#define CG_FREQ_TRAN_VOTING_0__UVD_FREQ_THROTTLING_VOTE_EN__SHIFT 0x9 +#define CG_FREQ_TRAN_VOTING_0__VCE_FREQ_THROTTLING_VOTE_EN_MASK 0x400 +#define CG_FREQ_TRAN_VOTING_0__VCE_FREQ_THROTTLING_VOTE_EN__SHIFT 0xa +#define CG_FREQ_TRAN_VOTING_0__DC_AZ_FREQ_THROTTLING_VOTE_EN_MASK 0x800 +#define CG_FREQ_TRAN_VOTING_0__DC_AZ_FREQ_THROTTLING_VOTE_EN__SHIFT 0xb +#define CG_FREQ_TRAN_VOTING_0__SAM_FREQ_THROTTLING_VOTE_EN_MASK 0x1000 +#define CG_FREQ_TRAN_VOTING_0__SAM_FREQ_THROTTLING_VOTE_EN__SHIFT 0xc +#define CG_FREQ_TRAN_VOTING_0__AVP_FREQ_THROTTLING_VOTE_EN_MASK 0x2000 +#define CG_FREQ_TRAN_VOTING_0__AVP_FREQ_THROTTLING_VOTE_EN__SHIFT 0xd +#define CG_FREQ_TRAN_VOTING_0__GRBM_0_FREQ_THROTTLING_VOTE_EN_MASK 0x4000 +#define CG_FREQ_TRAN_VOTING_0__GRBM_0_FREQ_THROTTLING_VOTE_EN__SHIFT 0xe +#define CG_FREQ_TRAN_VOTING_0__GRBM_1_FREQ_THROTTLING_VOTE_EN_MASK 0x8000 +#define CG_FREQ_TRAN_VOTING_0__GRBM_1_FREQ_THROTTLING_VOTE_EN__SHIFT 0xf +#define CG_FREQ_TRAN_VOTING_0__GRBM_2_FREQ_THROTTLING_VOTE_EN_MASK 0x10000 +#define CG_FREQ_TRAN_VOTING_0__GRBM_2_FREQ_THROTTLING_VOTE_EN__SHIFT 0x10 +#define CG_FREQ_TRAN_VOTING_0__GRBM_3_FREQ_THROTTLING_VOTE_EN_MASK 0x20000 +#define CG_FREQ_TRAN_VOTING_0__GRBM_3_FREQ_THROTTLING_VOTE_EN__SHIFT 0x11 +#define CG_FREQ_TRAN_VOTING_0__GRBM_4_FREQ_THROTTLING_VOTE_EN_MASK 0x40000 +#define CG_FREQ_TRAN_VOTING_0__GRBM_4_FREQ_THROTTLING_VOTE_EN__SHIFT 0x12 +#define CG_FREQ_TRAN_VOTING_0__GRBM_5_FREQ_THROTTLING_VOTE_EN_MASK 0x80000 +#define CG_FREQ_TRAN_VOTING_0__GRBM_5_FREQ_THROTTLING_VOTE_EN__SHIFT 0x13 +#define CG_FREQ_TRAN_VOTING_0__GRBM_6_FREQ_THROTTLING_VOTE_EN_MASK 0x100000 +#define CG_FREQ_TRAN_VOTING_0__GRBM_6_FREQ_THROTTLING_VOTE_EN__SHIFT 0x14 +#define CG_FREQ_TRAN_VOTING_0__GRBM_7_FREQ_THROTTLING_VOTE_EN_MASK 0x200000 +#define CG_FREQ_TRAN_VOTING_0__GRBM_7_FREQ_THROTTLING_VOTE_EN__SHIFT 0x15 +#define CG_FREQ_TRAN_VOTING_0__GRBM_8_FREQ_THROTTLING_VOTE_EN_MASK 0x400000 +#define CG_FREQ_TRAN_VOTING_0__GRBM_8_FREQ_THROTTLING_VOTE_EN__SHIFT 0x16 +#define CG_FREQ_TRAN_VOTING_0__GRBM_9_FREQ_THROTTLING_VOTE_EN_MASK 0x800000 +#define CG_FREQ_TRAN_VOTING_0__GRBM_9_FREQ_THROTTLING_VOTE_EN__SHIFT 0x17 +#define CG_FREQ_TRAN_VOTING_0__GRBM_10_FREQ_THROTTLING_VOTE_EN_MASK 0x1000000 +#define CG_FREQ_TRAN_VOTING_0__GRBM_10_FREQ_THROTTLING_VOTE_EN__SHIFT 0x18 +#define CG_FREQ_TRAN_VOTING_0__GRBM_11_FREQ_THROTTLING_VOTE_EN_MASK 0x2000000 +#define CG_FREQ_TRAN_VOTING_0__GRBM_11_FREQ_THROTTLING_VOTE_EN__SHIFT 0x19 +#define CG_FREQ_TRAN_VOTING_0__GRBM_12_FREQ_THROTTLING_VOTE_EN_MASK 0x4000000 +#define CG_FREQ_TRAN_VOTING_0__GRBM_12_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1a +#define CG_FREQ_TRAN_VOTING_0__GRBM_13_FREQ_THROTTLING_VOTE_EN_MASK 0x8000000 +#define CG_FREQ_TRAN_VOTING_0__GRBM_13_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1b +#define CG_FREQ_TRAN_VOTING_0__GRBM_14_FREQ_THROTTLING_VOTE_EN_MASK 0x10000000 +#define CG_FREQ_TRAN_VOTING_0__GRBM_14_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1c +#define CG_FREQ_TRAN_VOTING_0__GRBM_15_FREQ_THROTTLING_VOTE_EN_MASK 0x20000000 +#define CG_FREQ_TRAN_VOTING_0__GRBM_15_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1d +#define CG_FREQ_TRAN_VOTING_0__RLC_FREQ_THROTTLING_VOTE_EN_MASK 0x40000000 +#define CG_FREQ_TRAN_VOTING_0__RLC_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1e +#define CG_FREQ_TRAN_VOTING_1__BIF_FREQ_THROTTLING_VOTE_EN_MASK 0x1 +#define CG_FREQ_TRAN_VOTING_1__BIF_FREQ_THROTTLING_VOTE_EN__SHIFT 0x0 +#define CG_FREQ_TRAN_VOTING_1__HDP_FREQ_THROTTLING_VOTE_EN_MASK 0x2 +#define CG_FREQ_TRAN_VOTING_1__HDP_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1 +#define CG_FREQ_TRAN_VOTING_1__ROM_FREQ_THROTTLING_VOTE_EN_MASK 0x4 +#define CG_FREQ_TRAN_VOTING_1__ROM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x2 +#define CG_FREQ_TRAN_VOTING_1__IH_SEM_FREQ_THROTTLING_VOTE_EN_MASK 0x8 +#define CG_FREQ_TRAN_VOTING_1__IH_SEM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x3 +#define CG_FREQ_TRAN_VOTING_1__PDMA_FREQ_THROTTLING_VOTE_EN_MASK 0x10 +#define CG_FREQ_TRAN_VOTING_1__PDMA_FREQ_THROTTLING_VOTE_EN__SHIFT 0x4 +#define CG_FREQ_TRAN_VOTING_1__DRM_FREQ_THROTTLING_VOTE_EN_MASK 0x20 +#define CG_FREQ_TRAN_VOTING_1__DRM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x5 +#define CG_FREQ_TRAN_VOTING_1__IDCT_FREQ_THROTTLING_VOTE_EN_MASK 0x40 +#define CG_FREQ_TRAN_VOTING_1__IDCT_FREQ_THROTTLING_VOTE_EN__SHIFT 0x6 +#define CG_FREQ_TRAN_VOTING_1__ACP_FREQ_THROTTLING_VOTE_EN_MASK 0x80 +#define CG_FREQ_TRAN_VOTING_1__ACP_FREQ_THROTTLING_VOTE_EN__SHIFT 0x7 +#define CG_FREQ_TRAN_VOTING_1__SDMA_FREQ_THROTTLING_VOTE_EN_MASK 0x100 +#define CG_FREQ_TRAN_VOTING_1__SDMA_FREQ_THROTTLING_VOTE_EN__SHIFT 0x8 +#define CG_FREQ_TRAN_VOTING_1__UVD_FREQ_THROTTLING_VOTE_EN_MASK 0x200 +#define CG_FREQ_TRAN_VOTING_1__UVD_FREQ_THROTTLING_VOTE_EN__SHIFT 0x9 +#define CG_FREQ_TRAN_VOTING_1__VCE_FREQ_THROTTLING_VOTE_EN_MASK 0x400 +#define CG_FREQ_TRAN_VOTING_1__VCE_FREQ_THROTTLING_VOTE_EN__SHIFT 0xa +#define CG_FREQ_TRAN_VOTING_1__DC_AZ_FREQ_THROTTLING_VOTE_EN_MASK 0x800 +#define CG_FREQ_TRAN_VOTING_1__DC_AZ_FREQ_THROTTLING_VOTE_EN__SHIFT 0xb +#define CG_FREQ_TRAN_VOTING_1__SAM_FREQ_THROTTLING_VOTE_EN_MASK 0x1000 +#define CG_FREQ_TRAN_VOTING_1__SAM_FREQ_THROTTLING_VOTE_EN__SHIFT 0xc +#define CG_FREQ_TRAN_VOTING_1__AVP_FREQ_THROTTLING_VOTE_EN_MASK 0x2000 +#define CG_FREQ_TRAN_VOTING_1__AVP_FREQ_THROTTLING_VOTE_EN__SHIFT 0xd +#define CG_FREQ_TRAN_VOTING_1__GRBM_0_FREQ_THROTTLING_VOTE_EN_MASK 0x4000 +#define CG_FREQ_TRAN_VOTING_1__GRBM_0_FREQ_THROTTLING_VOTE_EN__SHIFT 0xe +#define CG_FREQ_TRAN_VOTING_1__GRBM_1_FREQ_THROTTLING_VOTE_EN_MASK 0x8000 +#define CG_FREQ_TRAN_VOTING_1__GRBM_1_FREQ_THROTTLING_VOTE_EN__SHIFT 0xf +#define CG_FREQ_TRAN_VOTING_1__GRBM_2_FREQ_THROTTLING_VOTE_EN_MASK 0x10000 +#define CG_FREQ_TRAN_VOTING_1__GRBM_2_FREQ_THROTTLING_VOTE_EN__SHIFT 0x10 +#define CG_FREQ_TRAN_VOTING_1__GRBM_3_FREQ_THROTTLING_VOTE_EN_MASK 0x20000 +#define CG_FREQ_TRAN_VOTING_1__GRBM_3_FREQ_THROTTLING_VOTE_EN__SHIFT 0x11 +#define CG_FREQ_TRAN_VOTING_1__GRBM_4_FREQ_THROTTLING_VOTE_EN_MASK 0x40000 +#define CG_FREQ_TRAN_VOTING_1__GRBM_4_FREQ_THROTTLING_VOTE_EN__SHIFT 0x12 +#define CG_FREQ_TRAN_VOTING_1__GRBM_5_FREQ_THROTTLING_VOTE_EN_MASK 0x80000 +#define CG_FREQ_TRAN_VOTING_1__GRBM_5_FREQ_THROTTLING_VOTE_EN__SHIFT 0x13 +#define CG_FREQ_TRAN_VOTING_1__GRBM_6_FREQ_THROTTLING_VOTE_EN_MASK 0x100000 +#define CG_FREQ_TRAN_VOTING_1__GRBM_6_FREQ_THROTTLING_VOTE_EN__SHIFT 0x14 +#define CG_FREQ_TRAN_VOTING_1__GRBM_7_FREQ_THROTTLING_VOTE_EN_MASK 0x200000 +#define CG_FREQ_TRAN_VOTING_1__GRBM_7_FREQ_THROTTLING_VOTE_EN__SHIFT 0x15 +#define CG_FREQ_TRAN_VOTING_1__GRBM_8_FREQ_THROTTLING_VOTE_EN_MASK 0x400000 +#define CG_FREQ_TRAN_VOTING_1__GRBM_8_FREQ_THROTTLING_VOTE_EN__SHIFT 0x16 +#define CG_FREQ_TRAN_VOTING_1__GRBM_9_FREQ_THROTTLING_VOTE_EN_MASK 0x800000 +#define CG_FREQ_TRAN_VOTING_1__GRBM_9_FREQ_THROTTLING_VOTE_EN__SHIFT 0x17 +#define CG_FREQ_TRAN_VOTING_1__GRBM_10_FREQ_THROTTLING_VOTE_EN_MASK 0x1000000 +#define CG_FREQ_TRAN_VOTING_1__GRBM_10_FREQ_THROTTLING_VOTE_EN__SHIFT 0x18 +#define CG_FREQ_TRAN_VOTING_1__GRBM_11_FREQ_THROTTLING_VOTE_EN_MASK 0x2000000 +#define CG_FREQ_TRAN_VOTING_1__GRBM_11_FREQ_THROTTLING_VOTE_EN__SHIFT 0x19 +#define CG_FREQ_TRAN_VOTING_1__GRBM_12_FREQ_THROTTLING_VOTE_EN_MASK 0x4000000 +#define CG_FREQ_TRAN_VOTING_1__GRBM_12_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1a +#define CG_FREQ_TRAN_VOTING_1__GRBM_13_FREQ_THROTTLING_VOTE_EN_MASK 0x8000000 +#define CG_FREQ_TRAN_VOTING_1__GRBM_13_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1b +#define CG_FREQ_TRAN_VOTING_1__GRBM_14_FREQ_THROTTLING_VOTE_EN_MASK 0x10000000 +#define CG_FREQ_TRAN_VOTING_1__GRBM_14_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1c +#define CG_FREQ_TRAN_VOTING_1__GRBM_15_FREQ_THROTTLING_VOTE_EN_MASK 0x20000000 +#define CG_FREQ_TRAN_VOTING_1__GRBM_15_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1d +#define CG_FREQ_TRAN_VOTING_1__RLC_FREQ_THROTTLING_VOTE_EN_MASK 0x40000000 +#define CG_FREQ_TRAN_VOTING_1__RLC_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1e +#define CG_FREQ_TRAN_VOTING_2__BIF_FREQ_THROTTLING_VOTE_EN_MASK 0x1 +#define CG_FREQ_TRAN_VOTING_2__BIF_FREQ_THROTTLING_VOTE_EN__SHIFT 0x0 +#define CG_FREQ_TRAN_VOTING_2__HDP_FREQ_THROTTLING_VOTE_EN_MASK 0x2 +#define CG_FREQ_TRAN_VOTING_2__HDP_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1 +#define CG_FREQ_TRAN_VOTING_2__ROM_FREQ_THROTTLING_VOTE_EN_MASK 0x4 +#define CG_FREQ_TRAN_VOTING_2__ROM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x2 +#define CG_FREQ_TRAN_VOTING_2__IH_SEM_FREQ_THROTTLING_VOTE_EN_MASK 0x8 +#define CG_FREQ_TRAN_VOTING_2__IH_SEM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x3 +#define CG_FREQ_TRAN_VOTING_2__PDMA_FREQ_THROTTLING_VOTE_EN_MASK 0x10 +#define CG_FREQ_TRAN_VOTING_2__PDMA_FREQ_THROTTLING_VOTE_EN__SHIFT 0x4 +#define CG_FREQ_TRAN_VOTING_2__DRM_FREQ_THROTTLING_VOTE_EN_MASK 0x20 +#define CG_FREQ_TRAN_VOTING_2__DRM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x5 +#define CG_FREQ_TRAN_VOTING_2__IDCT_FREQ_THROTTLING_VOTE_EN_MASK 0x40 +#define CG_FREQ_TRAN_VOTING_2__IDCT_FREQ_THROTTLING_VOTE_EN__SHIFT 0x6 +#define CG_FREQ_TRAN_VOTING_2__ACP_FREQ_THROTTLING_VOTE_EN_MASK 0x80 +#define CG_FREQ_TRAN_VOTING_2__ACP_FREQ_THROTTLING_VOTE_EN__SHIFT 0x7 +#define CG_FREQ_TRAN_VOTING_2__SDMA_FREQ_THROTTLING_VOTE_EN_MASK 0x100 +#define CG_FREQ_TRAN_VOTING_2__SDMA_FREQ_THROTTLING_VOTE_EN__SHIFT 0x8 +#define CG_FREQ_TRAN_VOTING_2__UVD_FREQ_THROTTLING_VOTE_EN_MASK 0x200 +#define CG_FREQ_TRAN_VOTING_2__UVD_FREQ_THROTTLING_VOTE_EN__SHIFT 0x9 +#define CG_FREQ_TRAN_VOTING_2__VCE_FREQ_THROTTLING_VOTE_EN_MASK 0x400 +#define CG_FREQ_TRAN_VOTING_2__VCE_FREQ_THROTTLING_VOTE_EN__SHIFT 0xa +#define CG_FREQ_TRAN_VOTING_2__DC_AZ_FREQ_THROTTLING_VOTE_EN_MASK 0x800 +#define CG_FREQ_TRAN_VOTING_2__DC_AZ_FREQ_THROTTLING_VOTE_EN__SHIFT 0xb +#define CG_FREQ_TRAN_VOTING_2__SAM_FREQ_THROTTLING_VOTE_EN_MASK 0x1000 +#define CG_FREQ_TRAN_VOTING_2__SAM_FREQ_THROTTLING_VOTE_EN__SHIFT 0xc +#define CG_FREQ_TRAN_VOTING_2__AVP_FREQ_THROTTLING_VOTE_EN_MASK 0x2000 +#define CG_FREQ_TRAN_VOTING_2__AVP_FREQ_THROTTLING_VOTE_EN__SHIFT 0xd +#define CG_FREQ_TRAN_VOTING_2__GRBM_0_FREQ_THROTTLING_VOTE_EN_MASK 0x4000 +#define CG_FREQ_TRAN_VOTING_2__GRBM_0_FREQ_THROTTLING_VOTE_EN__SHIFT 0xe +#define CG_FREQ_TRAN_VOTING_2__GRBM_1_FREQ_THROTTLING_VOTE_EN_MASK 0x8000 +#define CG_FREQ_TRAN_VOTING_2__GRBM_1_FREQ_THROTTLING_VOTE_EN__SHIFT 0xf +#define CG_FREQ_TRAN_VOTING_2__GRBM_2_FREQ_THROTTLING_VOTE_EN_MASK 0x10000 +#define CG_FREQ_TRAN_VOTING_2__GRBM_2_FREQ_THROTTLING_VOTE_EN__SHIFT 0x10 +#define CG_FREQ_TRAN_VOTING_2__GRBM_3_FREQ_THROTTLING_VOTE_EN_MASK 0x20000 +#define CG_FREQ_TRAN_VOTING_2__GRBM_3_FREQ_THROTTLING_VOTE_EN__SHIFT 0x11 +#define CG_FREQ_TRAN_VOTING_2__GRBM_4_FREQ_THROTTLING_VOTE_EN_MASK 0x40000 +#define CG_FREQ_TRAN_VOTING_2__GRBM_4_FREQ_THROTTLING_VOTE_EN__SHIFT 0x12 +#define CG_FREQ_TRAN_VOTING_2__GRBM_5_FREQ_THROTTLING_VOTE_EN_MASK 0x80000 +#define CG_FREQ_TRAN_VOTING_2__GRBM_5_FREQ_THROTTLING_VOTE_EN__SHIFT 0x13 +#define CG_FREQ_TRAN_VOTING_2__GRBM_6_FREQ_THROTTLING_VOTE_EN_MASK 0x100000 +#define CG_FREQ_TRAN_VOTING_2__GRBM_6_FREQ_THROTTLING_VOTE_EN__SHIFT 0x14 +#define CG_FREQ_TRAN_VOTING_2__GRBM_7_FREQ_THROTTLING_VOTE_EN_MASK 0x200000 +#define CG_FREQ_TRAN_VOTING_2__GRBM_7_FREQ_THROTTLING_VOTE_EN__SHIFT 0x15 +#define CG_FREQ_TRAN_VOTING_2__GRBM_8_FREQ_THROTTLING_VOTE_EN_MASK 0x400000 +#define CG_FREQ_TRAN_VOTING_2__GRBM_8_FREQ_THROTTLING_VOTE_EN__SHIFT 0x16 +#define CG_FREQ_TRAN_VOTING_2__GRBM_9_FREQ_THROTTLING_VOTE_EN_MASK 0x800000 +#define CG_FREQ_TRAN_VOTING_2__GRBM_9_FREQ_THROTTLING_VOTE_EN__SHIFT 0x17 +#define CG_FREQ_TRAN_VOTING_2__GRBM_10_FREQ_THROTTLING_VOTE_EN_MASK 0x1000000 +#define CG_FREQ_TRAN_VOTING_2__GRBM_10_FREQ_THROTTLING_VOTE_EN__SHIFT 0x18 +#define CG_FREQ_TRAN_VOTING_2__GRBM_11_FREQ_THROTTLING_VOTE_EN_MASK 0x2000000 +#define CG_FREQ_TRAN_VOTING_2__GRBM_11_FREQ_THROTTLING_VOTE_EN__SHIFT 0x19 +#define CG_FREQ_TRAN_VOTING_2__GRBM_12_FREQ_THROTTLING_VOTE_EN_MASK 0x4000000 +#define CG_FREQ_TRAN_VOTING_2__GRBM_12_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1a +#define CG_FREQ_TRAN_VOTING_2__GRBM_13_FREQ_THROTTLING_VOTE_EN_MASK 0x8000000 +#define CG_FREQ_TRAN_VOTING_2__GRBM_13_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1b +#define CG_FREQ_TRAN_VOTING_2__GRBM_14_FREQ_THROTTLING_VOTE_EN_MASK 0x10000000 +#define CG_FREQ_TRAN_VOTING_2__GRBM_14_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1c +#define CG_FREQ_TRAN_VOTING_2__GRBM_15_FREQ_THROTTLING_VOTE_EN_MASK 0x20000000 +#define CG_FREQ_TRAN_VOTING_2__GRBM_15_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1d +#define CG_FREQ_TRAN_VOTING_2__RLC_FREQ_THROTTLING_VOTE_EN_MASK 0x40000000 +#define CG_FREQ_TRAN_VOTING_2__RLC_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1e +#define CG_FREQ_TRAN_VOTING_3__BIF_FREQ_THROTTLING_VOTE_EN_MASK 0x1 +#define CG_FREQ_TRAN_VOTING_3__BIF_FREQ_THROTTLING_VOTE_EN__SHIFT 0x0 +#define CG_FREQ_TRAN_VOTING_3__HDP_FREQ_THROTTLING_VOTE_EN_MASK 0x2 +#define CG_FREQ_TRAN_VOTING_3__HDP_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1 +#define CG_FREQ_TRAN_VOTING_3__ROM_FREQ_THROTTLING_VOTE_EN_MASK 0x4 +#define CG_FREQ_TRAN_VOTING_3__ROM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x2 +#define CG_FREQ_TRAN_VOTING_3__IH_SEM_FREQ_THROTTLING_VOTE_EN_MASK 0x8 +#define CG_FREQ_TRAN_VOTING_3__IH_SEM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x3 +#define CG_FREQ_TRAN_VOTING_3__PDMA_FREQ_THROTTLING_VOTE_EN_MASK 0x10 +#define CG_FREQ_TRAN_VOTING_3__PDMA_FREQ_THROTTLING_VOTE_EN__SHIFT 0x4 +#define CG_FREQ_TRAN_VOTING_3__DRM_FREQ_THROTTLING_VOTE_EN_MASK 0x20 +#define CG_FREQ_TRAN_VOTING_3__DRM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x5 +#define CG_FREQ_TRAN_VOTING_3__IDCT_FREQ_THROTTLING_VOTE_EN_MASK 0x40 +#define CG_FREQ_TRAN_VOTING_3__IDCT_FREQ_THROTTLING_VOTE_EN__SHIFT 0x6 +#define CG_FREQ_TRAN_VOTING_3__ACP_FREQ_THROTTLING_VOTE_EN_MASK 0x80 +#define CG_FREQ_TRAN_VOTING_3__ACP_FREQ_THROTTLING_VOTE_EN__SHIFT 0x7 +#define CG_FREQ_TRAN_VOTING_3__SDMA_FREQ_THROTTLING_VOTE_EN_MASK 0x100 +#define CG_FREQ_TRAN_VOTING_3__SDMA_FREQ_THROTTLING_VOTE_EN__SHIFT 0x8 +#define CG_FREQ_TRAN_VOTING_3__UVD_FREQ_THROTTLING_VOTE_EN_MASK 0x200 +#define CG_FREQ_TRAN_VOTING_3__UVD_FREQ_THROTTLING_VOTE_EN__SHIFT 0x9 +#define CG_FREQ_TRAN_VOTING_3__VCE_FREQ_THROTTLING_VOTE_EN_MASK 0x400 +#define CG_FREQ_TRAN_VOTING_3__VCE_FREQ_THROTTLING_VOTE_EN__SHIFT 0xa +#define CG_FREQ_TRAN_VOTING_3__DC_AZ_FREQ_THROTTLING_VOTE_EN_MASK 0x800 +#define CG_FREQ_TRAN_VOTING_3__DC_AZ_FREQ_THROTTLING_VOTE_EN__SHIFT 0xb +#define CG_FREQ_TRAN_VOTING_3__SAM_FREQ_THROTTLING_VOTE_EN_MASK 0x1000 +#define CG_FREQ_TRAN_VOTING_3__SAM_FREQ_THROTTLING_VOTE_EN__SHIFT 0xc +#define CG_FREQ_TRAN_VOTING_3__AVP_FREQ_THROTTLING_VOTE_EN_MASK 0x2000 +#define CG_FREQ_TRAN_VOTING_3__AVP_FREQ_THROTTLING_VOTE_EN__SHIFT 0xd +#define CG_FREQ_TRAN_VOTING_3__GRBM_0_FREQ_THROTTLING_VOTE_EN_MASK 0x4000 +#define CG_FREQ_TRAN_VOTING_3__GRBM_0_FREQ_THROTTLING_VOTE_EN__SHIFT 0xe +#define CG_FREQ_TRAN_VOTING_3__GRBM_1_FREQ_THROTTLING_VOTE_EN_MASK 0x8000 +#define CG_FREQ_TRAN_VOTING_3__GRBM_1_FREQ_THROTTLING_VOTE_EN__SHIFT 0xf +#define CG_FREQ_TRAN_VOTING_3__GRBM_2_FREQ_THROTTLING_VOTE_EN_MASK 0x10000 +#define CG_FREQ_TRAN_VOTING_3__GRBM_2_FREQ_THROTTLING_VOTE_EN__SHIFT 0x10 +#define CG_FREQ_TRAN_VOTING_3__GRBM_3_FREQ_THROTTLING_VOTE_EN_MASK 0x20000 +#define CG_FREQ_TRAN_VOTING_3__GRBM_3_FREQ_THROTTLING_VOTE_EN__SHIFT 0x11 +#define CG_FREQ_TRAN_VOTING_3__GRBM_4_FREQ_THROTTLING_VOTE_EN_MASK 0x40000 +#define CG_FREQ_TRAN_VOTING_3__GRBM_4_FREQ_THROTTLING_VOTE_EN__SHIFT 0x12 +#define CG_FREQ_TRAN_VOTING_3__GRBM_5_FREQ_THROTTLING_VOTE_EN_MASK 0x80000 +#define CG_FREQ_TRAN_VOTING_3__GRBM_5_FREQ_THROTTLING_VOTE_EN__SHIFT 0x13 +#define CG_FREQ_TRAN_VOTING_3__GRBM_6_FREQ_THROTTLING_VOTE_EN_MASK 0x100000 +#define CG_FREQ_TRAN_VOTING_3__GRBM_6_FREQ_THROTTLING_VOTE_EN__SHIFT 0x14 +#define CG_FREQ_TRAN_VOTING_3__GRBM_7_FREQ_THROTTLING_VOTE_EN_MASK 0x200000 +#define CG_FREQ_TRAN_VOTING_3__GRBM_7_FREQ_THROTTLING_VOTE_EN__SHIFT 0x15 +#define CG_FREQ_TRAN_VOTING_3__GRBM_8_FREQ_THROTTLING_VOTE_EN_MASK 0x400000 +#define CG_FREQ_TRAN_VOTING_3__GRBM_8_FREQ_THROTTLING_VOTE_EN__SHIFT 0x16 +#define CG_FREQ_TRAN_VOTING_3__GRBM_9_FREQ_THROTTLING_VOTE_EN_MASK 0x800000 +#define CG_FREQ_TRAN_VOTING_3__GRBM_9_FREQ_THROTTLING_VOTE_EN__SHIFT 0x17 +#define CG_FREQ_TRAN_VOTING_3__GRBM_10_FREQ_THROTTLING_VOTE_EN_MASK 0x1000000 +#define CG_FREQ_TRAN_VOTING_3__GRBM_10_FREQ_THROTTLING_VOTE_EN__SHIFT 0x18 +#define CG_FREQ_TRAN_VOTING_3__GRBM_11_FREQ_THROTTLING_VOTE_EN_MASK 0x2000000 +#define CG_FREQ_TRAN_VOTING_3__GRBM_11_FREQ_THROTTLING_VOTE_EN__SHIFT 0x19 +#define CG_FREQ_TRAN_VOTING_3__GRBM_12_FREQ_THROTTLING_VOTE_EN_MASK 0x4000000 +#define CG_FREQ_TRAN_VOTING_3__GRBM_12_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1a +#define CG_FREQ_TRAN_VOTING_3__GRBM_13_FREQ_THROTTLING_VOTE_EN_MASK 0x8000000 +#define CG_FREQ_TRAN_VOTING_3__GRBM_13_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1b +#define CG_FREQ_TRAN_VOTING_3__GRBM_14_FREQ_THROTTLING_VOTE_EN_MASK 0x10000000 +#define CG_FREQ_TRAN_VOTING_3__GRBM_14_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1c +#define CG_FREQ_TRAN_VOTING_3__GRBM_15_FREQ_THROTTLING_VOTE_EN_MASK 0x20000000 +#define CG_FREQ_TRAN_VOTING_3__GRBM_15_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1d +#define CG_FREQ_TRAN_VOTING_3__RLC_FREQ_THROTTLING_VOTE_EN_MASK 0x40000000 +#define CG_FREQ_TRAN_VOTING_3__RLC_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1e +#define CG_FREQ_TRAN_VOTING_4__BIF_FREQ_THROTTLING_VOTE_EN_MASK 0x1 +#define CG_FREQ_TRAN_VOTING_4__BIF_FREQ_THROTTLING_VOTE_EN__SHIFT 0x0 +#define CG_FREQ_TRAN_VOTING_4__HDP_FREQ_THROTTLING_VOTE_EN_MASK 0x2 +#define CG_FREQ_TRAN_VOTING_4__HDP_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1 +#define CG_FREQ_TRAN_VOTING_4__ROM_FREQ_THROTTLING_VOTE_EN_MASK 0x4 +#define CG_FREQ_TRAN_VOTING_4__ROM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x2 +#define CG_FREQ_TRAN_VOTING_4__IH_SEM_FREQ_THROTTLING_VOTE_EN_MASK 0x8 +#define CG_FREQ_TRAN_VOTING_4__IH_SEM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x3 +#define CG_FREQ_TRAN_VOTING_4__PDMA_FREQ_THROTTLING_VOTE_EN_MASK 0x10 +#define CG_FREQ_TRAN_VOTING_4__PDMA_FREQ_THROTTLING_VOTE_EN__SHIFT 0x4 +#define CG_FREQ_TRAN_VOTING_4__DRM_FREQ_THROTTLING_VOTE_EN_MASK 0x20 +#define CG_FREQ_TRAN_VOTING_4__DRM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x5 +#define CG_FREQ_TRAN_VOTING_4__IDCT_FREQ_THROTTLING_VOTE_EN_MASK 0x40 +#define CG_FREQ_TRAN_VOTING_4__IDCT_FREQ_THROTTLING_VOTE_EN__SHIFT 0x6 +#define CG_FREQ_TRAN_VOTING_4__ACP_FREQ_THROTTLING_VOTE_EN_MASK 0x80 +#define CG_FREQ_TRAN_VOTING_4__ACP_FREQ_THROTTLING_VOTE_EN__SHIFT 0x7 +#define CG_FREQ_TRAN_VOTING_4__SDMA_FREQ_THROTTLING_VOTE_EN_MASK 0x100 +#define CG_FREQ_TRAN_VOTING_4__SDMA_FREQ_THROTTLING_VOTE_EN__SHIFT 0x8 +#define CG_FREQ_TRAN_VOTING_4__UVD_FREQ_THROTTLING_VOTE_EN_MASK 0x200 +#define CG_FREQ_TRAN_VOTING_4__UVD_FREQ_THROTTLING_VOTE_EN__SHIFT 0x9 +#define CG_FREQ_TRAN_VOTING_4__VCE_FREQ_THROTTLING_VOTE_EN_MASK 0x400 +#define CG_FREQ_TRAN_VOTING_4__VCE_FREQ_THROTTLING_VOTE_EN__SHIFT 0xa +#define CG_FREQ_TRAN_VOTING_4__DC_AZ_FREQ_THROTTLING_VOTE_EN_MASK 0x800 +#define CG_FREQ_TRAN_VOTING_4__DC_AZ_FREQ_THROTTLING_VOTE_EN__SHIFT 0xb +#define CG_FREQ_TRAN_VOTING_4__SAM_FREQ_THROTTLING_VOTE_EN_MASK 0x1000 +#define CG_FREQ_TRAN_VOTING_4__SAM_FREQ_THROTTLING_VOTE_EN__SHIFT 0xc +#define CG_FREQ_TRAN_VOTING_4__AVP_FREQ_THROTTLING_VOTE_EN_MASK 0x2000 +#define CG_FREQ_TRAN_VOTING_4__AVP_FREQ_THROTTLING_VOTE_EN__SHIFT 0xd +#define CG_FREQ_TRAN_VOTING_4__GRBM_0_FREQ_THROTTLING_VOTE_EN_MASK 0x4000 +#define CG_FREQ_TRAN_VOTING_4__GRBM_0_FREQ_THROTTLING_VOTE_EN__SHIFT 0xe +#define CG_FREQ_TRAN_VOTING_4__GRBM_1_FREQ_THROTTLING_VOTE_EN_MASK 0x8000 +#define CG_FREQ_TRAN_VOTING_4__GRBM_1_FREQ_THROTTLING_VOTE_EN__SHIFT 0xf +#define CG_FREQ_TRAN_VOTING_4__GRBM_2_FREQ_THROTTLING_VOTE_EN_MASK 0x10000 +#define CG_FREQ_TRAN_VOTING_4__GRBM_2_FREQ_THROTTLING_VOTE_EN__SHIFT 0x10 +#define CG_FREQ_TRAN_VOTING_4__GRBM_3_FREQ_THROTTLING_VOTE_EN_MASK 0x20000 +#define CG_FREQ_TRAN_VOTING_4__GRBM_3_FREQ_THROTTLING_VOTE_EN__SHIFT 0x11 +#define CG_FREQ_TRAN_VOTING_4__GRBM_4_FREQ_THROTTLING_VOTE_EN_MASK 0x40000 +#define CG_FREQ_TRAN_VOTING_4__GRBM_4_FREQ_THROTTLING_VOTE_EN__SHIFT 0x12 +#define CG_FREQ_TRAN_VOTING_4__GRBM_5_FREQ_THROTTLING_VOTE_EN_MASK 0x80000 +#define CG_FREQ_TRAN_VOTING_4__GRBM_5_FREQ_THROTTLING_VOTE_EN__SHIFT 0x13 +#define CG_FREQ_TRAN_VOTING_4__GRBM_6_FREQ_THROTTLING_VOTE_EN_MASK 0x100000 +#define CG_FREQ_TRAN_VOTING_4__GRBM_6_FREQ_THROTTLING_VOTE_EN__SHIFT 0x14 +#define CG_FREQ_TRAN_VOTING_4__GRBM_7_FREQ_THROTTLING_VOTE_EN_MASK 0x200000 +#define CG_FREQ_TRAN_VOTING_4__GRBM_7_FREQ_THROTTLING_VOTE_EN__SHIFT 0x15 +#define CG_FREQ_TRAN_VOTING_4__GRBM_8_FREQ_THROTTLING_VOTE_EN_MASK 0x400000 +#define CG_FREQ_TRAN_VOTING_4__GRBM_8_FREQ_THROTTLING_VOTE_EN__SHIFT 0x16 +#define CG_FREQ_TRAN_VOTING_4__GRBM_9_FREQ_THROTTLING_VOTE_EN_MASK 0x800000 +#define CG_FREQ_TRAN_VOTING_4__GRBM_9_FREQ_THROTTLING_VOTE_EN__SHIFT 0x17 +#define CG_FREQ_TRAN_VOTING_4__GRBM_10_FREQ_THROTTLING_VOTE_EN_MASK 0x1000000 +#define CG_FREQ_TRAN_VOTING_4__GRBM_10_FREQ_THROTTLING_VOTE_EN__SHIFT 0x18 +#define CG_FREQ_TRAN_VOTING_4__GRBM_11_FREQ_THROTTLING_VOTE_EN_MASK 0x2000000 +#define CG_FREQ_TRAN_VOTING_4__GRBM_11_FREQ_THROTTLING_VOTE_EN__SHIFT 0x19 +#define CG_FREQ_TRAN_VOTING_4__GRBM_12_FREQ_THROTTLING_VOTE_EN_MASK 0x4000000 +#define CG_FREQ_TRAN_VOTING_4__GRBM_12_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1a +#define CG_FREQ_TRAN_VOTING_4__GRBM_13_FREQ_THROTTLING_VOTE_EN_MASK 0x8000000 +#define CG_FREQ_TRAN_VOTING_4__GRBM_13_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1b +#define CG_FREQ_TRAN_VOTING_4__GRBM_14_FREQ_THROTTLING_VOTE_EN_MASK 0x10000000 +#define CG_FREQ_TRAN_VOTING_4__GRBM_14_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1c +#define CG_FREQ_TRAN_VOTING_4__GRBM_15_FREQ_THROTTLING_VOTE_EN_MASK 0x20000000 +#define CG_FREQ_TRAN_VOTING_4__GRBM_15_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1d +#define CG_FREQ_TRAN_VOTING_4__RLC_FREQ_THROTTLING_VOTE_EN_MASK 0x40000000 +#define CG_FREQ_TRAN_VOTING_4__RLC_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1e +#define CG_FREQ_TRAN_VOTING_5__BIF_FREQ_THROTTLING_VOTE_EN_MASK 0x1 +#define CG_FREQ_TRAN_VOTING_5__BIF_FREQ_THROTTLING_VOTE_EN__SHIFT 0x0 +#define CG_FREQ_TRAN_VOTING_5__HDP_FREQ_THROTTLING_VOTE_EN_MASK 0x2 +#define CG_FREQ_TRAN_VOTING_5__HDP_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1 +#define CG_FREQ_TRAN_VOTING_5__ROM_FREQ_THROTTLING_VOTE_EN_MASK 0x4 +#define CG_FREQ_TRAN_VOTING_5__ROM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x2 +#define CG_FREQ_TRAN_VOTING_5__IH_SEM_FREQ_THROTTLING_VOTE_EN_MASK 0x8 +#define CG_FREQ_TRAN_VOTING_5__IH_SEM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x3 +#define CG_FREQ_TRAN_VOTING_5__PDMA_FREQ_THROTTLING_VOTE_EN_MASK 0x10 +#define CG_FREQ_TRAN_VOTING_5__PDMA_FREQ_THROTTLING_VOTE_EN__SHIFT 0x4 +#define CG_FREQ_TRAN_VOTING_5__DRM_FREQ_THROTTLING_VOTE_EN_MASK 0x20 +#define CG_FREQ_TRAN_VOTING_5__DRM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x5 +#define CG_FREQ_TRAN_VOTING_5__IDCT_FREQ_THROTTLING_VOTE_EN_MASK 0x40 +#define CG_FREQ_TRAN_VOTING_5__IDCT_FREQ_THROTTLING_VOTE_EN__SHIFT 0x6 +#define CG_FREQ_TRAN_VOTING_5__ACP_FREQ_THROTTLING_VOTE_EN_MASK 0x80 +#define CG_FREQ_TRAN_VOTING_5__ACP_FREQ_THROTTLING_VOTE_EN__SHIFT 0x7 +#define CG_FREQ_TRAN_VOTING_5__SDMA_FREQ_THROTTLING_VOTE_EN_MASK 0x100 +#define CG_FREQ_TRAN_VOTING_5__SDMA_FREQ_THROTTLING_VOTE_EN__SHIFT 0x8 +#define CG_FREQ_TRAN_VOTING_5__UVD_FREQ_THROTTLING_VOTE_EN_MASK 0x200 +#define CG_FREQ_TRAN_VOTING_5__UVD_FREQ_THROTTLING_VOTE_EN__SHIFT 0x9 +#define CG_FREQ_TRAN_VOTING_5__VCE_FREQ_THROTTLING_VOTE_EN_MASK 0x400 +#define CG_FREQ_TRAN_VOTING_5__VCE_FREQ_THROTTLING_VOTE_EN__SHIFT 0xa +#define CG_FREQ_TRAN_VOTING_5__DC_AZ_FREQ_THROTTLING_VOTE_EN_MASK 0x800 +#define CG_FREQ_TRAN_VOTING_5__DC_AZ_FREQ_THROTTLING_VOTE_EN__SHIFT 0xb +#define CG_FREQ_TRAN_VOTING_5__SAM_FREQ_THROTTLING_VOTE_EN_MASK 0x1000 +#define CG_FREQ_TRAN_VOTING_5__SAM_FREQ_THROTTLING_VOTE_EN__SHIFT 0xc +#define CG_FREQ_TRAN_VOTING_5__AVP_FREQ_THROTTLING_VOTE_EN_MASK 0x2000 +#define CG_FREQ_TRAN_VOTING_5__AVP_FREQ_THROTTLING_VOTE_EN__SHIFT 0xd +#define CG_FREQ_TRAN_VOTING_5__GRBM_0_FREQ_THROTTLING_VOTE_EN_MASK 0x4000 +#define CG_FREQ_TRAN_VOTING_5__GRBM_0_FREQ_THROTTLING_VOTE_EN__SHIFT 0xe +#define CG_FREQ_TRAN_VOTING_5__GRBM_1_FREQ_THROTTLING_VOTE_EN_MASK 0x8000 +#define CG_FREQ_TRAN_VOTING_5__GRBM_1_FREQ_THROTTLING_VOTE_EN__SHIFT 0xf +#define CG_FREQ_TRAN_VOTING_5__GRBM_2_FREQ_THROTTLING_VOTE_EN_MASK 0x10000 +#define CG_FREQ_TRAN_VOTING_5__GRBM_2_FREQ_THROTTLING_VOTE_EN__SHIFT 0x10 +#define CG_FREQ_TRAN_VOTING_5__GRBM_3_FREQ_THROTTLING_VOTE_EN_MASK 0x20000 +#define CG_FREQ_TRAN_VOTING_5__GRBM_3_FREQ_THROTTLING_VOTE_EN__SHIFT 0x11 +#define CG_FREQ_TRAN_VOTING_5__GRBM_4_FREQ_THROTTLING_VOTE_EN_MASK 0x40000 +#define CG_FREQ_TRAN_VOTING_5__GRBM_4_FREQ_THROTTLING_VOTE_EN__SHIFT 0x12 +#define CG_FREQ_TRAN_VOTING_5__GRBM_5_FREQ_THROTTLING_VOTE_EN_MASK 0x80000 +#define CG_FREQ_TRAN_VOTING_5__GRBM_5_FREQ_THROTTLING_VOTE_EN__SHIFT 0x13 +#define CG_FREQ_TRAN_VOTING_5__GRBM_6_FREQ_THROTTLING_VOTE_EN_MASK 0x100000 +#define CG_FREQ_TRAN_VOTING_5__GRBM_6_FREQ_THROTTLING_VOTE_EN__SHIFT 0x14 +#define CG_FREQ_TRAN_VOTING_5__GRBM_7_FREQ_THROTTLING_VOTE_EN_MASK 0x200000 +#define CG_FREQ_TRAN_VOTING_5__GRBM_7_FREQ_THROTTLING_VOTE_EN__SHIFT 0x15 +#define CG_FREQ_TRAN_VOTING_5__GRBM_8_FREQ_THROTTLING_VOTE_EN_MASK 0x400000 +#define CG_FREQ_TRAN_VOTING_5__GRBM_8_FREQ_THROTTLING_VOTE_EN__SHIFT 0x16 +#define CG_FREQ_TRAN_VOTING_5__GRBM_9_FREQ_THROTTLING_VOTE_EN_MASK 0x800000 +#define CG_FREQ_TRAN_VOTING_5__GRBM_9_FREQ_THROTTLING_VOTE_EN__SHIFT 0x17 +#define CG_FREQ_TRAN_VOTING_5__GRBM_10_FREQ_THROTTLING_VOTE_EN_MASK 0x1000000 +#define CG_FREQ_TRAN_VOTING_5__GRBM_10_FREQ_THROTTLING_VOTE_EN__SHIFT 0x18 +#define CG_FREQ_TRAN_VOTING_5__GRBM_11_FREQ_THROTTLING_VOTE_EN_MASK 0x2000000 +#define CG_FREQ_TRAN_VOTING_5__GRBM_11_FREQ_THROTTLING_VOTE_EN__SHIFT 0x19 +#define CG_FREQ_TRAN_VOTING_5__GRBM_12_FREQ_THROTTLING_VOTE_EN_MASK 0x4000000 +#define CG_FREQ_TRAN_VOTING_5__GRBM_12_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1a +#define CG_FREQ_TRAN_VOTING_5__GRBM_13_FREQ_THROTTLING_VOTE_EN_MASK 0x8000000 +#define CG_FREQ_TRAN_VOTING_5__GRBM_13_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1b +#define CG_FREQ_TRAN_VOTING_5__GRBM_14_FREQ_THROTTLING_VOTE_EN_MASK 0x10000000 +#define CG_FREQ_TRAN_VOTING_5__GRBM_14_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1c +#define CG_FREQ_TRAN_VOTING_5__GRBM_15_FREQ_THROTTLING_VOTE_EN_MASK 0x20000000 +#define CG_FREQ_TRAN_VOTING_5__GRBM_15_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1d +#define CG_FREQ_TRAN_VOTING_5__RLC_FREQ_THROTTLING_VOTE_EN_MASK 0x40000000 +#define CG_FREQ_TRAN_VOTING_5__RLC_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1e +#define CG_FREQ_TRAN_VOTING_6__BIF_FREQ_THROTTLING_VOTE_EN_MASK 0x1 +#define CG_FREQ_TRAN_VOTING_6__BIF_FREQ_THROTTLING_VOTE_EN__SHIFT 0x0 +#define CG_FREQ_TRAN_VOTING_6__HDP_FREQ_THROTTLING_VOTE_EN_MASK 0x2 +#define CG_FREQ_TRAN_VOTING_6__HDP_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1 +#define CG_FREQ_TRAN_VOTING_6__ROM_FREQ_THROTTLING_VOTE_EN_MASK 0x4 +#define CG_FREQ_TRAN_VOTING_6__ROM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x2 +#define CG_FREQ_TRAN_VOTING_6__IH_SEM_FREQ_THROTTLING_VOTE_EN_MASK 0x8 +#define CG_FREQ_TRAN_VOTING_6__IH_SEM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x3 +#define CG_FREQ_TRAN_VOTING_6__PDMA_FREQ_THROTTLING_VOTE_EN_MASK 0x10 +#define CG_FREQ_TRAN_VOTING_6__PDMA_FREQ_THROTTLING_VOTE_EN__SHIFT 0x4 +#define CG_FREQ_TRAN_VOTING_6__DRM_FREQ_THROTTLING_VOTE_EN_MASK 0x20 +#define CG_FREQ_TRAN_VOTING_6__DRM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x5 +#define CG_FREQ_TRAN_VOTING_6__IDCT_FREQ_THROTTLING_VOTE_EN_MASK 0x40 +#define CG_FREQ_TRAN_VOTING_6__IDCT_FREQ_THROTTLING_VOTE_EN__SHIFT 0x6 +#define CG_FREQ_TRAN_VOTING_6__ACP_FREQ_THROTTLING_VOTE_EN_MASK 0x80 +#define CG_FREQ_TRAN_VOTING_6__ACP_FREQ_THROTTLING_VOTE_EN__SHIFT 0x7 +#define CG_FREQ_TRAN_VOTING_6__SDMA_FREQ_THROTTLING_VOTE_EN_MASK 0x100 +#define CG_FREQ_TRAN_VOTING_6__SDMA_FREQ_THROTTLING_VOTE_EN__SHIFT 0x8 +#define CG_FREQ_TRAN_VOTING_6__UVD_FREQ_THROTTLING_VOTE_EN_MASK 0x200 +#define CG_FREQ_TRAN_VOTING_6__UVD_FREQ_THROTTLING_VOTE_EN__SHIFT 0x9 +#define CG_FREQ_TRAN_VOTING_6__VCE_FREQ_THROTTLING_VOTE_EN_MASK 0x400 +#define CG_FREQ_TRAN_VOTING_6__VCE_FREQ_THROTTLING_VOTE_EN__SHIFT 0xa +#define CG_FREQ_TRAN_VOTING_6__DC_AZ_FREQ_THROTTLING_VOTE_EN_MASK 0x800 +#define CG_FREQ_TRAN_VOTING_6__DC_AZ_FREQ_THROTTLING_VOTE_EN__SHIFT 0xb +#define CG_FREQ_TRAN_VOTING_6__SAM_FREQ_THROTTLING_VOTE_EN_MASK 0x1000 +#define CG_FREQ_TRAN_VOTING_6__SAM_FREQ_THROTTLING_VOTE_EN__SHIFT 0xc +#define CG_FREQ_TRAN_VOTING_6__AVP_FREQ_THROTTLING_VOTE_EN_MASK 0x2000 +#define CG_FREQ_TRAN_VOTING_6__AVP_FREQ_THROTTLING_VOTE_EN__SHIFT 0xd +#define CG_FREQ_TRAN_VOTING_6__GRBM_0_FREQ_THROTTLING_VOTE_EN_MASK 0x4000 +#define CG_FREQ_TRAN_VOTING_6__GRBM_0_FREQ_THROTTLING_VOTE_EN__SHIFT 0xe +#define CG_FREQ_TRAN_VOTING_6__GRBM_1_FREQ_THROTTLING_VOTE_EN_MASK 0x8000 +#define CG_FREQ_TRAN_VOTING_6__GRBM_1_FREQ_THROTTLING_VOTE_EN__SHIFT 0xf +#define CG_FREQ_TRAN_VOTING_6__GRBM_2_FREQ_THROTTLING_VOTE_EN_MASK 0x10000 +#define CG_FREQ_TRAN_VOTING_6__GRBM_2_FREQ_THROTTLING_VOTE_EN__SHIFT 0x10 +#define CG_FREQ_TRAN_VOTING_6__GRBM_3_FREQ_THROTTLING_VOTE_EN_MASK 0x20000 +#define CG_FREQ_TRAN_VOTING_6__GRBM_3_FREQ_THROTTLING_VOTE_EN__SHIFT 0x11 +#define CG_FREQ_TRAN_VOTING_6__GRBM_4_FREQ_THROTTLING_VOTE_EN_MASK 0x40000 +#define CG_FREQ_TRAN_VOTING_6__GRBM_4_FREQ_THROTTLING_VOTE_EN__SHIFT 0x12 +#define CG_FREQ_TRAN_VOTING_6__GRBM_5_FREQ_THROTTLING_VOTE_EN_MASK 0x80000 +#define CG_FREQ_TRAN_VOTING_6__GRBM_5_FREQ_THROTTLING_VOTE_EN__SHIFT 0x13 +#define CG_FREQ_TRAN_VOTING_6__GRBM_6_FREQ_THROTTLING_VOTE_EN_MASK 0x100000 +#define CG_FREQ_TRAN_VOTING_6__GRBM_6_FREQ_THROTTLING_VOTE_EN__SHIFT 0x14 +#define CG_FREQ_TRAN_VOTING_6__GRBM_7_FREQ_THROTTLING_VOTE_EN_MASK 0x200000 +#define CG_FREQ_TRAN_VOTING_6__GRBM_7_FREQ_THROTTLING_VOTE_EN__SHIFT 0x15 +#define CG_FREQ_TRAN_VOTING_6__GRBM_8_FREQ_THROTTLING_VOTE_EN_MASK 0x400000 +#define CG_FREQ_TRAN_VOTING_6__GRBM_8_FREQ_THROTTLING_VOTE_EN__SHIFT 0x16 +#define CG_FREQ_TRAN_VOTING_6__GRBM_9_FREQ_THROTTLING_VOTE_EN_MASK 0x800000 +#define CG_FREQ_TRAN_VOTING_6__GRBM_9_FREQ_THROTTLING_VOTE_EN__SHIFT 0x17 +#define CG_FREQ_TRAN_VOTING_6__GRBM_10_FREQ_THROTTLING_VOTE_EN_MASK 0x1000000 +#define CG_FREQ_TRAN_VOTING_6__GRBM_10_FREQ_THROTTLING_VOTE_EN__SHIFT 0x18 +#define CG_FREQ_TRAN_VOTING_6__GRBM_11_FREQ_THROTTLING_VOTE_EN_MASK 0x2000000 +#define CG_FREQ_TRAN_VOTING_6__GRBM_11_FREQ_THROTTLING_VOTE_EN__SHIFT 0x19 +#define CG_FREQ_TRAN_VOTING_6__GRBM_12_FREQ_THROTTLING_VOTE_EN_MASK 0x4000000 +#define CG_FREQ_TRAN_VOTING_6__GRBM_12_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1a +#define CG_FREQ_TRAN_VOTING_6__GRBM_13_FREQ_THROTTLING_VOTE_EN_MASK 0x8000000 +#define CG_FREQ_TRAN_VOTING_6__GRBM_13_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1b +#define CG_FREQ_TRAN_VOTING_6__GRBM_14_FREQ_THROTTLING_VOTE_EN_MASK 0x10000000 +#define CG_FREQ_TRAN_VOTING_6__GRBM_14_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1c +#define CG_FREQ_TRAN_VOTING_6__GRBM_15_FREQ_THROTTLING_VOTE_EN_MASK 0x20000000 +#define CG_FREQ_TRAN_VOTING_6__GRBM_15_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1d +#define CG_FREQ_TRAN_VOTING_6__RLC_FREQ_THROTTLING_VOTE_EN_MASK 0x40000000 +#define CG_FREQ_TRAN_VOTING_6__RLC_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1e +#define CG_FREQ_TRAN_VOTING_7__BIF_FREQ_THROTTLING_VOTE_EN_MASK 0x1 +#define CG_FREQ_TRAN_VOTING_7__BIF_FREQ_THROTTLING_VOTE_EN__SHIFT 0x0 +#define CG_FREQ_TRAN_VOTING_7__HDP_FREQ_THROTTLING_VOTE_EN_MASK 0x2 +#define CG_FREQ_TRAN_VOTING_7__HDP_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1 +#define CG_FREQ_TRAN_VOTING_7__ROM_FREQ_THROTTLING_VOTE_EN_MASK 0x4 +#define CG_FREQ_TRAN_VOTING_7__ROM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x2 +#define CG_FREQ_TRAN_VOTING_7__IH_SEM_FREQ_THROTTLING_VOTE_EN_MASK 0x8 +#define CG_FREQ_TRAN_VOTING_7__IH_SEM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x3 +#define CG_FREQ_TRAN_VOTING_7__PDMA_FREQ_THROTTLING_VOTE_EN_MASK 0x10 +#define CG_FREQ_TRAN_VOTING_7__PDMA_FREQ_THROTTLING_VOTE_EN__SHIFT 0x4 +#define CG_FREQ_TRAN_VOTING_7__DRM_FREQ_THROTTLING_VOTE_EN_MASK 0x20 +#define CG_FREQ_TRAN_VOTING_7__DRM_FREQ_THROTTLING_VOTE_EN__SHIFT 0x5 +#define CG_FREQ_TRAN_VOTING_7__IDCT_FREQ_THROTTLING_VOTE_EN_MASK 0x40 +#define CG_FREQ_TRAN_VOTING_7__IDCT_FREQ_THROTTLING_VOTE_EN__SHIFT 0x6 +#define CG_FREQ_TRAN_VOTING_7__ACP_FREQ_THROTTLING_VOTE_EN_MASK 0x80 +#define CG_FREQ_TRAN_VOTING_7__ACP_FREQ_THROTTLING_VOTE_EN__SHIFT 0x7 +#define CG_FREQ_TRAN_VOTING_7__SDMA_FREQ_THROTTLING_VOTE_EN_MASK 0x100 +#define CG_FREQ_TRAN_VOTING_7__SDMA_FREQ_THROTTLING_VOTE_EN__SHIFT 0x8 +#define CG_FREQ_TRAN_VOTING_7__UVD_FREQ_THROTTLING_VOTE_EN_MASK 0x200 +#define CG_FREQ_TRAN_VOTING_7__UVD_FREQ_THROTTLING_VOTE_EN__SHIFT 0x9 +#define CG_FREQ_TRAN_VOTING_7__VCE_FREQ_THROTTLING_VOTE_EN_MASK 0x400 +#define CG_FREQ_TRAN_VOTING_7__VCE_FREQ_THROTTLING_VOTE_EN__SHIFT 0xa +#define CG_FREQ_TRAN_VOTING_7__DC_AZ_FREQ_THROTTLING_VOTE_EN_MASK 0x800 +#define CG_FREQ_TRAN_VOTING_7__DC_AZ_FREQ_THROTTLING_VOTE_EN__SHIFT 0xb +#define CG_FREQ_TRAN_VOTING_7__SAM_FREQ_THROTTLING_VOTE_EN_MASK 0x1000 +#define CG_FREQ_TRAN_VOTING_7__SAM_FREQ_THROTTLING_VOTE_EN__SHIFT 0xc +#define CG_FREQ_TRAN_VOTING_7__AVP_FREQ_THROTTLING_VOTE_EN_MASK 0x2000 +#define CG_FREQ_TRAN_VOTING_7__AVP_FREQ_THROTTLING_VOTE_EN__SHIFT 0xd +#define CG_FREQ_TRAN_VOTING_7__GRBM_0_FREQ_THROTTLING_VOTE_EN_MASK 0x4000 +#define CG_FREQ_TRAN_VOTING_7__GRBM_0_FREQ_THROTTLING_VOTE_EN__SHIFT 0xe +#define CG_FREQ_TRAN_VOTING_7__GRBM_1_FREQ_THROTTLING_VOTE_EN_MASK 0x8000 +#define CG_FREQ_TRAN_VOTING_7__GRBM_1_FREQ_THROTTLING_VOTE_EN__SHIFT 0xf +#define CG_FREQ_TRAN_VOTING_7__GRBM_2_FREQ_THROTTLING_VOTE_EN_MASK 0x10000 +#define CG_FREQ_TRAN_VOTING_7__GRBM_2_FREQ_THROTTLING_VOTE_EN__SHIFT 0x10 +#define CG_FREQ_TRAN_VOTING_7__GRBM_3_FREQ_THROTTLING_VOTE_EN_MASK 0x20000 +#define CG_FREQ_TRAN_VOTING_7__GRBM_3_FREQ_THROTTLING_VOTE_EN__SHIFT 0x11 +#define CG_FREQ_TRAN_VOTING_7__GRBM_4_FREQ_THROTTLING_VOTE_EN_MASK 0x40000 +#define CG_FREQ_TRAN_VOTING_7__GRBM_4_FREQ_THROTTLING_VOTE_EN__SHIFT 0x12 +#define CG_FREQ_TRAN_VOTING_7__GRBM_5_FREQ_THROTTLING_VOTE_EN_MASK 0x80000 +#define CG_FREQ_TRAN_VOTING_7__GRBM_5_FREQ_THROTTLING_VOTE_EN__SHIFT 0x13 +#define CG_FREQ_TRAN_VOTING_7__GRBM_6_FREQ_THROTTLING_VOTE_EN_MASK 0x100000 +#define CG_FREQ_TRAN_VOTING_7__GRBM_6_FREQ_THROTTLING_VOTE_EN__SHIFT 0x14 +#define CG_FREQ_TRAN_VOTING_7__GRBM_7_FREQ_THROTTLING_VOTE_EN_MASK 0x200000 +#define CG_FREQ_TRAN_VOTING_7__GRBM_7_FREQ_THROTTLING_VOTE_EN__SHIFT 0x15 +#define CG_FREQ_TRAN_VOTING_7__GRBM_8_FREQ_THROTTLING_VOTE_EN_MASK 0x400000 +#define CG_FREQ_TRAN_VOTING_7__GRBM_8_FREQ_THROTTLING_VOTE_EN__SHIFT 0x16 +#define CG_FREQ_TRAN_VOTING_7__GRBM_9_FREQ_THROTTLING_VOTE_EN_MASK 0x800000 +#define CG_FREQ_TRAN_VOTING_7__GRBM_9_FREQ_THROTTLING_VOTE_EN__SHIFT 0x17 +#define CG_FREQ_TRAN_VOTING_7__GRBM_10_FREQ_THROTTLING_VOTE_EN_MASK 0x1000000 +#define CG_FREQ_TRAN_VOTING_7__GRBM_10_FREQ_THROTTLING_VOTE_EN__SHIFT 0x18 +#define CG_FREQ_TRAN_VOTING_7__GRBM_11_FREQ_THROTTLING_VOTE_EN_MASK 0x2000000 +#define CG_FREQ_TRAN_VOTING_7__GRBM_11_FREQ_THROTTLING_VOTE_EN__SHIFT 0x19 +#define CG_FREQ_TRAN_VOTING_7__GRBM_12_FREQ_THROTTLING_VOTE_EN_MASK 0x4000000 +#define CG_FREQ_TRAN_VOTING_7__GRBM_12_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1a +#define CG_FREQ_TRAN_VOTING_7__GRBM_13_FREQ_THROTTLING_VOTE_EN_MASK 0x8000000 +#define CG_FREQ_TRAN_VOTING_7__GRBM_13_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1b +#define CG_FREQ_TRAN_VOTING_7__GRBM_14_FREQ_THROTTLING_VOTE_EN_MASK 0x10000000 +#define CG_FREQ_TRAN_VOTING_7__GRBM_14_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1c +#define CG_FREQ_TRAN_VOTING_7__GRBM_15_FREQ_THROTTLING_VOTE_EN_MASK 0x20000000 +#define CG_FREQ_TRAN_VOTING_7__GRBM_15_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1d +#define CG_FREQ_TRAN_VOTING_7__RLC_FREQ_THROTTLING_VOTE_EN_MASK 0x40000000 +#define CG_FREQ_TRAN_VOTING_7__RLC_FREQ_THROTTLING_VOTE_EN__SHIFT 0x1e +#define PLL_TEST_CNTL__TST_SRC_SEL_MASK 0xf +#define PLL_TEST_CNTL__TST_SRC_SEL__SHIFT 0x0 +#define PLL_TEST_CNTL__TST_REF_SEL_MASK 0xf0 +#define PLL_TEST_CNTL__TST_REF_SEL__SHIFT 0x4 +#define PLL_TEST_CNTL__REF_TEST_COUNT_MASK 0x7f00 +#define PLL_TEST_CNTL__REF_TEST_COUNT__SHIFT 0x8 +#define PLL_TEST_CNTL__TST_RESET_MASK 0x8000 +#define PLL_TEST_CNTL__TST_RESET__SHIFT 0xf +#define PLL_TEST_CNTL__TEST_COUNT_MASK 0xfffe0000 +#define PLL_TEST_CNTL__TEST_COUNT__SHIFT 0x11 +#define CG_STATIC_SCREEN_PARAMETER__STATIC_SCREEN_THRESHOLD_MASK 0xffff +#define CG_STATIC_SCREEN_PARAMETER__STATIC_SCREEN_THRESHOLD__SHIFT 0x0 +#define CG_STATIC_SCREEN_PARAMETER__STATIC_SCREEN_THRESHOLD_UNIT_MASK 0xf0000 +#define CG_STATIC_SCREEN_PARAMETER__STATIC_SCREEN_THRESHOLD_UNIT__SHIFT 0x10 +#define CG_DISPLAY_GAP_CNTL__DISP_GAP_MASK 0x3 +#define CG_DISPLAY_GAP_CNTL__DISP_GAP__SHIFT 0x0 +#define CG_DISPLAY_GAP_CNTL__VBI_TIMER_COUNT_MASK 0x3fff0 +#define CG_DISPLAY_GAP_CNTL__VBI_TIMER_COUNT__SHIFT 0x4 +#define CG_DISPLAY_GAP_CNTL__VBI_TIMER_UNIT_MASK 0x700000 +#define CG_DISPLAY_GAP_CNTL__VBI_TIMER_UNIT__SHIFT 0x14 +#define CG_DISPLAY_GAP_CNTL__DISP_GAP_MCHG_MASK 0x3000000 +#define CG_DISPLAY_GAP_CNTL__DISP_GAP_MCHG__SHIFT 0x18 +#define CG_DISPLAY_GAP_CNTL__VBI_TIMER_DISABLE_MASK 0x10000000 +#define CG_DISPLAY_GAP_CNTL__VBI_TIMER_DISABLE__SHIFT 0x1c +#define CG_DISPLAY_GAP_CNTL2__VBI_PREDICTION_MASK 0xffffffff +#define CG_DISPLAY_GAP_CNTL2__VBI_PREDICTION__SHIFT 0x0 +#define CG_ACPI_CNTL__SCLK_ACPI_DIV_MASK 0x7f +#define CG_ACPI_CNTL__SCLK_ACPI_DIV__SHIFT 0x0 +#define CG_ACPI_CNTL__SCLK_CHANGE_SKIP_MASK 0x80 +#define CG_ACPI_CNTL__SCLK_CHANGE_SKIP__SHIFT 0x7 +#define SCLK_DEEP_SLEEP_CNTL__DIV_ID_MASK 0x7 +#define SCLK_DEEP_SLEEP_CNTL__DIV_ID__SHIFT 0x0 +#define SCLK_DEEP_SLEEP_CNTL__RAMP_DIS_MASK 0x8 +#define SCLK_DEEP_SLEEP_CNTL__RAMP_DIS__SHIFT 0x3 +#define SCLK_DEEP_SLEEP_CNTL__HYSTERESIS_MASK 0xfff0 +#define SCLK_DEEP_SLEEP_CNTL__HYSTERESIS__SHIFT 0x4 +#define SCLK_DEEP_SLEEP_CNTL__SCLK_RUNNING_MASK_MASK 0x10000 +#define SCLK_DEEP_SLEEP_CNTL__SCLK_RUNNING_MASK__SHIFT 0x10 +#define SCLK_DEEP_SLEEP_CNTL__SELF_REFRESH_MASK_MASK 0x20000 +#define SCLK_DEEP_SLEEP_CNTL__SELF_REFRESH_MASK__SHIFT 0x11 +#define SCLK_DEEP_SLEEP_CNTL__ALLOW_NBPSTATE_MASK_MASK 0x40000 +#define SCLK_DEEP_SLEEP_CNTL__ALLOW_NBPSTATE_MASK__SHIFT 0x12 +#define SCLK_DEEP_SLEEP_CNTL__BIF_BUSY_MASK_MASK 0x80000 +#define SCLK_DEEP_SLEEP_CNTL__BIF_BUSY_MASK__SHIFT 0x13 +#define SCLK_DEEP_SLEEP_CNTL__UVD_BUSY_MASK_MASK 0x100000 +#define SCLK_DEEP_SLEEP_CNTL__UVD_BUSY_MASK__SHIFT 0x14 +#define SCLK_DEEP_SLEEP_CNTL__MC0SRBM_BUSY_MASK_MASK 0x200000 +#define SCLK_DEEP_SLEEP_CNTL__MC0SRBM_BUSY_MASK__SHIFT 0x15 +#define SCLK_DEEP_SLEEP_CNTL__MC1SRBM_BUSY_MASK_MASK 0x400000 +#define SCLK_DEEP_SLEEP_CNTL__MC1SRBM_BUSY_MASK__SHIFT 0x16 +#define SCLK_DEEP_SLEEP_CNTL__MC_ALLOW_MASK_MASK 0x800000 +#define SCLK_DEEP_SLEEP_CNTL__MC_ALLOW_MASK__SHIFT 0x17 +#define SCLK_DEEP_SLEEP_CNTL__SMU_BUSY_MASK_MASK 0x1000000 +#define SCLK_DEEP_SLEEP_CNTL__SMU_BUSY_MASK__SHIFT 0x18 +#define SCLK_DEEP_SLEEP_CNTL__SELF_REFRESH_NLC_MASK_MASK 0x2000000 +#define SCLK_DEEP_SLEEP_CNTL__SELF_REFRESH_NLC_MASK__SHIFT 0x19 +#define SCLK_DEEP_SLEEP_CNTL__FAST_EXIT_REQ_NBPSTATE_MASK 0x4000000 +#define SCLK_DEEP_SLEEP_CNTL__FAST_EXIT_REQ_NBPSTATE__SHIFT 0x1a +#define SCLK_DEEP_SLEEP_CNTL__DEEP_SLEEP_ENTRY_MODE_MASK 0x8000000 +#define SCLK_DEEP_SLEEP_CNTL__DEEP_SLEEP_ENTRY_MODE__SHIFT 0x1b +#define SCLK_DEEP_SLEEP_CNTL__MBUS2_ACTIVE_MASK_MASK 0x10000000 +#define SCLK_DEEP_SLEEP_CNTL__MBUS2_ACTIVE_MASK__SHIFT 0x1c +#define SCLK_DEEP_SLEEP_CNTL__VCE_BUSY_MASK_MASK 0x20000000 +#define SCLK_DEEP_SLEEP_CNTL__VCE_BUSY_MASK__SHIFT 0x1d +#define SCLK_DEEP_SLEEP_CNTL__AZ_BUSY_MASK_MASK 0x40000000 +#define SCLK_DEEP_SLEEP_CNTL__AZ_BUSY_MASK__SHIFT 0x1e +#define SCLK_DEEP_SLEEP_CNTL__ENABLE_DS_MASK 0x80000000 +#define SCLK_DEEP_SLEEP_CNTL__ENABLE_DS__SHIFT 0x1f +#define SCLK_DEEP_SLEEP_CNTL2__RLC_BUSY_MASK_MASK 0x1 +#define SCLK_DEEP_SLEEP_CNTL2__RLC_BUSY_MASK__SHIFT 0x0 +#define SCLK_DEEP_SLEEP_CNTL2__HDP_BUSY_MASK_MASK 0x2 +#define SCLK_DEEP_SLEEP_CNTL2__HDP_BUSY_MASK__SHIFT 0x1 +#define SCLK_DEEP_SLEEP_CNTL2__ROM_BUSY_MASK_MASK 0x4 +#define SCLK_DEEP_SLEEP_CNTL2__ROM_BUSY_MASK__SHIFT 0x2 +#define SCLK_DEEP_SLEEP_CNTL2__IH_SEM_BUSY_MASK_MASK 0x8 +#define SCLK_DEEP_SLEEP_CNTL2__IH_SEM_BUSY_MASK__SHIFT 0x3 +#define SCLK_DEEP_SLEEP_CNTL2__PDMA_BUSY_MASK_MASK 0x10 +#define SCLK_DEEP_SLEEP_CNTL2__PDMA_BUSY_MASK__SHIFT 0x4 +#define SCLK_DEEP_SLEEP_CNTL2__IDCT_BUSY_MASK_MASK 0x40 +#define SCLK_DEEP_SLEEP_CNTL2__IDCT_BUSY_MASK__SHIFT 0x6 +#define SCLK_DEEP_SLEEP_CNTL2__SDMA_BUSY_MASK_MASK 0x80 +#define SCLK_DEEP_SLEEP_CNTL2__SDMA_BUSY_MASK__SHIFT 0x7 +#define SCLK_DEEP_SLEEP_CNTL2__DC_AZ_BUSY_MASK_MASK 0x100 +#define SCLK_DEEP_SLEEP_CNTL2__DC_AZ_BUSY_MASK__SHIFT 0x8 +#define SCLK_DEEP_SLEEP_CNTL2__ACP_SMU_ALLOW_DSLEEP_STUTTER_MASK_MASK 0x200 +#define SCLK_DEEP_SLEEP_CNTL2__ACP_SMU_ALLOW_DSLEEP_STUTTER_MASK__SHIFT 0x9 +#define SCLK_DEEP_SLEEP_CNTL2__UVD_CG_MC_STAT_BUSY_MASK_MASK 0x400 +#define SCLK_DEEP_SLEEP_CNTL2__UVD_CG_MC_STAT_BUSY_MASK__SHIFT 0xa +#define SCLK_DEEP_SLEEP_CNTL2__VCE_CG_MC_STAT_BUSY_MASK_MASK 0x800 +#define SCLK_DEEP_SLEEP_CNTL2__VCE_CG_MC_STAT_BUSY_MASK__SHIFT 0xb +#define SCLK_DEEP_SLEEP_CNTL2__SAM_CG_MC_STAT_BUSY_MASK_MASK 0x1000 +#define SCLK_DEEP_SLEEP_CNTL2__SAM_CG_MC_STAT_BUSY_MASK__SHIFT 0xc +#define SCLK_DEEP_SLEEP_CNTL2__SAM_CG_STATUS_BUSY_MASK_MASK 0x2000 +#define SCLK_DEEP_SLEEP_CNTL2__SAM_CG_STATUS_BUSY_MASK__SHIFT 0xd +#define SCLK_DEEP_SLEEP_CNTL2__RLC_SMU_GFXCLK_OFF_MASK_MASK 0x4000 +#define SCLK_DEEP_SLEEP_CNTL2__RLC_SMU_GFXCLK_OFF_MASK__SHIFT 0xe +#define SCLK_DEEP_SLEEP_CNTL2__SHALLOW_DIV_ID_MASK 0xe00000 +#define SCLK_DEEP_SLEEP_CNTL2__SHALLOW_DIV_ID__SHIFT 0x15 +#define SCLK_DEEP_SLEEP_CNTL2__INOUT_CUSHION_MASK 0xff000000 +#define SCLK_DEEP_SLEEP_CNTL2__INOUT_CUSHION__SHIFT 0x18 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_0_SMU_BUSY_MASK_MASK 0x1 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_0_SMU_BUSY_MASK__SHIFT 0x0 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_1_SMU_BUSY_MASK_MASK 0x2 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_1_SMU_BUSY_MASK__SHIFT 0x1 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_2_SMU_BUSY_MASK_MASK 0x4 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_2_SMU_BUSY_MASK__SHIFT 0x2 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_3_SMU_BUSY_MASK_MASK 0x8 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_3_SMU_BUSY_MASK__SHIFT 0x3 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_4_SMU_BUSY_MASK_MASK 0x10 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_4_SMU_BUSY_MASK__SHIFT 0x4 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_5_SMU_BUSY_MASK_MASK 0x20 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_5_SMU_BUSY_MASK__SHIFT 0x5 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_6_SMU_BUSY_MASK_MASK 0x40 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_6_SMU_BUSY_MASK__SHIFT 0x6 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_7_SMU_BUSY_MASK_MASK 0x80 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_7_SMU_BUSY_MASK__SHIFT 0x7 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_8_SMU_BUSY_MASK_MASK 0x100 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_8_SMU_BUSY_MASK__SHIFT 0x8 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_9_SMU_BUSY_MASK_MASK 0x200 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_9_SMU_BUSY_MASK__SHIFT 0x9 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_10_SMU_BUSY_MASK_MASK 0x400 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_10_SMU_BUSY_MASK__SHIFT 0xa +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_11_SMU_BUSY_MASK_MASK 0x800 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_11_SMU_BUSY_MASK__SHIFT 0xb +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_12_SMU_BUSY_MASK_MASK 0x1000 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_12_SMU_BUSY_MASK__SHIFT 0xc +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_13_SMU_BUSY_MASK_MASK 0x2000 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_13_SMU_BUSY_MASK__SHIFT 0xd +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_14_SMU_BUSY_MASK_MASK 0x4000 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_14_SMU_BUSY_MASK__SHIFT 0xe +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_15_SMU_BUSY_MASK_MASK 0x8000 +#define SCLK_DEEP_SLEEP_CNTL3__GRBM_15_SMU_BUSY_MASK__SHIFT 0xf +#define SCLK_DEEP_SLEEP_MISC_CNTL__DPM_DS_DIV_ID_MASK 0x7 +#define SCLK_DEEP_SLEEP_MISC_CNTL__DPM_DS_DIV_ID__SHIFT 0x0 +#define SCLK_DEEP_SLEEP_MISC_CNTL__DPM_SS_DIV_ID_MASK 0x38 +#define SCLK_DEEP_SLEEP_MISC_CNTL__DPM_SS_DIV_ID__SHIFT 0x3 +#define SCLK_DEEP_SLEEP_MISC_CNTL__OCP_ENABLE_MASK 0x10000 +#define SCLK_DEEP_SLEEP_MISC_CNTL__OCP_ENABLE__SHIFT 0x10 +#define SCLK_DEEP_SLEEP_MISC_CNTL__OCP_DS_DIV_ID_MASK 0xe0000 +#define SCLK_DEEP_SLEEP_MISC_CNTL__OCP_DS_DIV_ID__SHIFT 0x11 +#define SCLK_DEEP_SLEEP_MISC_CNTL__OCP_SS_DIV_ID_MASK 0x700000 +#define SCLK_DEEP_SLEEP_MISC_CNTL__OCP_SS_DIV_ID__SHIFT 0x14 +#define LCLK_DEEP_SLEEP_CNTL__DIV_ID_MASK 0x7 +#define LCLK_DEEP_SLEEP_CNTL__DIV_ID__SHIFT 0x0 +#define LCLK_DEEP_SLEEP_CNTL__RAMP_DIS_MASK 0x8 +#define LCLK_DEEP_SLEEP_CNTL__RAMP_DIS__SHIFT 0x3 +#define LCLK_DEEP_SLEEP_CNTL__HYSTERESIS_MASK 0xfff0 +#define LCLK_DEEP_SLEEP_CNTL__HYSTERESIS__SHIFT 0x4 +#define LCLK_DEEP_SLEEP_CNTL__RESERVED_MASK 0x7fff0000 +#define LCLK_DEEP_SLEEP_CNTL__RESERVED__SHIFT 0x10 +#define LCLK_DEEP_SLEEP_CNTL__ENABLE_DS_MASK 0x80000000 +#define LCLK_DEEP_SLEEP_CNTL__ENABLE_DS__SHIFT 0x1f +#define LCLK_DEEP_SLEEP_CNTL2__RFE_BUSY_MASK_MASK 0x1 +#define LCLK_DEEP_SLEEP_CNTL2__RFE_BUSY_MASK__SHIFT 0x0 +#define LCLK_DEEP_SLEEP_CNTL2__BIF_CG_LCLK_BUSY_MASK_MASK 0x2 +#define LCLK_DEEP_SLEEP_CNTL2__BIF_CG_LCLK_BUSY_MASK__SHIFT 0x1 +#define LCLK_DEEP_SLEEP_CNTL2__L1IMU_SMU_IDLE_MASK_MASK 0x4 +#define LCLK_DEEP_SLEEP_CNTL2__L1IMU_SMU_IDLE_MASK__SHIFT 0x2 +#define LCLK_DEEP_SLEEP_CNTL2__RESERVED_BIT3_MASK 0x8 +#define LCLK_DEEP_SLEEP_CNTL2__RESERVED_BIT3__SHIFT 0x3 +#define LCLK_DEEP_SLEEP_CNTL2__SCLK_RUNNING_MASK_MASK 0x10 +#define LCLK_DEEP_SLEEP_CNTL2__SCLK_RUNNING_MASK__SHIFT 0x4 +#define LCLK_DEEP_SLEEP_CNTL2__SMU_BUSY_MASK_MASK 0x20 +#define LCLK_DEEP_SLEEP_CNTL2__SMU_BUSY_MASK__SHIFT 0x5 +#define LCLK_DEEP_SLEEP_CNTL2__PCIE_LCLK_IDLE1_MASK_MASK 0x40 +#define LCLK_DEEP_SLEEP_CNTL2__PCIE_LCLK_IDLE1_MASK__SHIFT 0x6 +#define LCLK_DEEP_SLEEP_CNTL2__PCIE_LCLK_IDLE2_MASK_MASK 0x80 +#define LCLK_DEEP_SLEEP_CNTL2__PCIE_LCLK_IDLE2_MASK__SHIFT 0x7 +#define LCLK_DEEP_SLEEP_CNTL2__PCIE_LCLK_IDLE3_MASK_MASK 0x100 +#define LCLK_DEEP_SLEEP_CNTL2__PCIE_LCLK_IDLE3_MASK__SHIFT 0x8 +#define LCLK_DEEP_SLEEP_CNTL2__PCIE_LCLK_IDLE4_MASK_MASK 0x200 +#define LCLK_DEEP_SLEEP_CNTL2__PCIE_LCLK_IDLE4_MASK__SHIFT 0x9 +#define LCLK_DEEP_SLEEP_CNTL2__L1IMUGPP_IDLE_MASK_MASK 0x400 +#define LCLK_DEEP_SLEEP_CNTL2__L1IMUGPP_IDLE_MASK__SHIFT 0xa +#define LCLK_DEEP_SLEEP_CNTL2__L1IMUGPPSB_IDLE_MASK_MASK 0x800 +#define LCLK_DEEP_SLEEP_CNTL2__L1IMUGPPSB_IDLE_MASK__SHIFT 0xb +#define LCLK_DEEP_SLEEP_CNTL2__L1IMUBIF_IDLE_MASK_MASK 0x1000 +#define LCLK_DEEP_SLEEP_CNTL2__L1IMUBIF_IDLE_MASK__SHIFT 0xc +#define LCLK_DEEP_SLEEP_CNTL2__L1IMUINTGEN_IDLE_MASK_MASK 0x2000 +#define LCLK_DEEP_SLEEP_CNTL2__L1IMUINTGEN_IDLE_MASK__SHIFT 0xd +#define LCLK_DEEP_SLEEP_CNTL2__L2IMU_IDLE_MASK_MASK 0x4000 +#define LCLK_DEEP_SLEEP_CNTL2__L2IMU_IDLE_MASK__SHIFT 0xe +#define LCLK_DEEP_SLEEP_CNTL2__ORB_IDLE_MASK_MASK 0x8000 +#define LCLK_DEEP_SLEEP_CNTL2__ORB_IDLE_MASK__SHIFT 0xf +#define LCLK_DEEP_SLEEP_CNTL2__ON_INB_WAKE_MASK_MASK 0x10000 +#define LCLK_DEEP_SLEEP_CNTL2__ON_INB_WAKE_MASK__SHIFT 0x10 +#define LCLK_DEEP_SLEEP_CNTL2__ON_INB_WAKE_ACK_MASK_MASK 0x20000 +#define LCLK_DEEP_SLEEP_CNTL2__ON_INB_WAKE_ACK_MASK__SHIFT 0x11 +#define LCLK_DEEP_SLEEP_CNTL2__ON_OUTB_WAKE_MASK_MASK 0x40000 +#define LCLK_DEEP_SLEEP_CNTL2__ON_OUTB_WAKE_MASK__SHIFT 0x12 +#define LCLK_DEEP_SLEEP_CNTL2__ON_OUTB_WAKE_ACK_MASK_MASK 0x80000 +#define LCLK_DEEP_SLEEP_CNTL2__ON_OUTB_WAKE_ACK_MASK__SHIFT 0x13 +#define LCLK_DEEP_SLEEP_CNTL2__DMAACTIVE_MASK_MASK 0x100000 +#define LCLK_DEEP_SLEEP_CNTL2__DMAACTIVE_MASK__SHIFT 0x14 +#define LCLK_DEEP_SLEEP_CNTL2__RLC_SMU_GFXCLK_OFF_MASK_MASK 0x200000 +#define LCLK_DEEP_SLEEP_CNTL2__RLC_SMU_GFXCLK_OFF_MASK__SHIFT 0x15 +#define LCLK_DEEP_SLEEP_CNTL2__RESERVED_MASK 0xffc00000 +#define LCLK_DEEP_SLEEP_CNTL2__RESERVED__SHIFT 0x16 +#define TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_VDDCI_INDEX_MASK 0xf +#define TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_VDDCI_INDEX__SHIFT 0x0 +#define TARGET_AND_CURRENT_PROFILE_INDEX_1__TARG_VDDCI_INDEX_MASK 0xf0 +#define TARGET_AND_CURRENT_PROFILE_INDEX_1__TARG_VDDCI_INDEX__SHIFT 0x4 +#define TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_MVDD_INDEX_MASK 0xf00 +#define TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_MVDD_INDEX__SHIFT 0x8 +#define TARGET_AND_CURRENT_PROFILE_INDEX_1__TARG_MVDD_INDEX_MASK 0xf000 +#define TARGET_AND_CURRENT_PROFILE_INDEX_1__TARG_MVDD_INDEX__SHIFT 0xc +#define TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_VDDC_INDEX_MASK 0xf0000 +#define TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_VDDC_INDEX__SHIFT 0x10 +#define TARGET_AND_CURRENT_PROFILE_INDEX_1__TARG_VDDC_INDEX_MASK 0xf00000 +#define TARGET_AND_CURRENT_PROFILE_INDEX_1__TARG_VDDC_INDEX__SHIFT 0x14 +#define TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_PCIE_INDEX_MASK 0xf000000 +#define TARGET_AND_CURRENT_PROFILE_INDEX_1__CURR_PCIE_INDEX__SHIFT 0x18 +#define TARGET_AND_CURRENT_PROFILE_INDEX_1__TARG_PCIE_INDEX_MASK 0xf0000000 +#define TARGET_AND_CURRENT_PROFILE_INDEX_1__TARG_PCIE_INDEX__SHIFT 0x1c +#define CG_ULV_PARAMETER__ULV_THRESHOLD_MASK 0xffff +#define CG_ULV_PARAMETER__ULV_THRESHOLD__SHIFT 0x0 +#define CG_ULV_PARAMETER__ULV_THRESHOLD_UNIT_MASK 0xf0000 +#define CG_ULV_PARAMETER__ULV_THRESHOLD_UNIT__SHIFT 0x10 +#define SCLK_MIN_DIV__FRACV_MASK 0xfff +#define SCLK_MIN_DIV__FRACV__SHIFT 0x0 +#define SCLK_MIN_DIV__INTV_MASK 0x7f000 +#define SCLK_MIN_DIV__INTV__SHIFT 0xc +#define PWR_AVFS_SEL__AvfsSel_MASK 0xfffffff +#define PWR_AVFS_SEL__AvfsSel__SHIFT 0x0 +#define PWR_AVFS_CNTL__MmBusIn_MASK 0xff +#define PWR_AVFS_CNTL__MmBusIn__SHIFT 0x0 +#define PWR_AVFS_CNTL__MmLclRdEn_MASK 0x100 +#define PWR_AVFS_CNTL__MmLclRdEn__SHIFT 0x8 +#define PWR_AVFS_CNTL__MmLclWrEn_MASK 0x200 +#define PWR_AVFS_CNTL__MmLclWrEn__SHIFT 0x9 +#define PWR_AVFS_CNTL__MmLclSz_MASK 0xc00 +#define PWR_AVFS_CNTL__MmLclSz__SHIFT 0xa +#define PWR_AVFS_CNTL__MmState_MASK 0x3f000 +#define PWR_AVFS_CNTL__MmState__SHIFT 0xc +#define PWR_AVFS_CNTL__PsmScanMode_MASK 0x40000 +#define PWR_AVFS_CNTL__PsmScanMode__SHIFT 0x12 +#define PWR_AVFS_CNTL__PsmGater_MASK 0x80000 +#define PWR_AVFS_CNTL__PsmGater__SHIFT 0x13 +#define PWR_AVFS_CNTL__PsmTrst_MASK 0x100000 +#define PWR_AVFS_CNTL__PsmTrst__SHIFT 0x14 +#define PWR_AVFS_CNTL__PsmEn_MASK 0x200000 +#define PWR_AVFS_CNTL__PsmEn__SHIFT 0x15 +#define PWR_AVFS_CNTL__SkipPhaseEn_MASK 0x400000 +#define PWR_AVFS_CNTL__SkipPhaseEn__SHIFT 0x16 +#define PWR_AVFS_CNTL__Isolate_MASK 0x800000 +#define PWR_AVFS_CNTL__Isolate__SHIFT 0x17 +#define PWR_AVFS_CNTL__AvfsRst_MASK 0x1000000 +#define PWR_AVFS_CNTL__AvfsRst__SHIFT 0x18 +#define PWR_AVFS_CNTL__PccIsolateEn_MASK 0x2000000 +#define PWR_AVFS_CNTL__PccIsolateEn__SHIFT 0x19 +#define PWR_AVFS_CNTL__DeepSleepIsolateEn_MASK 0x4000000 +#define PWR_AVFS_CNTL__DeepSleepIsolateEn__SHIFT 0x1a +#define PWR_AVFS0_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS0_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS0_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS0_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS0_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS0_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS1_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS1_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS1_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS1_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS1_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS1_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS2_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS2_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS2_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS2_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS2_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS2_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS3_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS3_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS3_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS3_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS3_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS3_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS4_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS4_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS4_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS4_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS4_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS4_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS5_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS5_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS5_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS5_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS5_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS5_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS6_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS6_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS6_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS6_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS6_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS6_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS7_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS7_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS7_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS7_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS7_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS7_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS8_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS8_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS8_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS8_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS8_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS8_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS9_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS9_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS9_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS9_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS9_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS9_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS10_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS10_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS10_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS10_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS10_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS10_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS11_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS11_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS11_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS11_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS11_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS11_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS12_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS12_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS12_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS12_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS12_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS12_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS13_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS13_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS13_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS13_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS13_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS13_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS14_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS14_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS14_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS14_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS14_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS14_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS15_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS15_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS15_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS15_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS15_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS15_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS16_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS16_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS16_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS16_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS16_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS16_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS17_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS17_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS17_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS17_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS17_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS17_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS18_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS18_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS18_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS18_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS18_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS18_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS19_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS19_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS19_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS19_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS19_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS19_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS20_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS20_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS20_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS20_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS20_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS20_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS21_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS21_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS21_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS21_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS21_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS21_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS22_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS22_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS22_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS22_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS22_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS22_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS23_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS23_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS23_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS23_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS23_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS23_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS24_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS24_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS24_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS24_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS24_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS24_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS25_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS25_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS25_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS25_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS25_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS25_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS26_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS26_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS26_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS26_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS26_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS26_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_AVFS27_CNTL_STATUS__MmDatOut_MASK 0xff +#define PWR_AVFS27_CNTL_STATUS__MmDatOut__SHIFT 0x0 +#define PWR_AVFS27_CNTL_STATUS__PsmTdo_MASK 0x100 +#define PWR_AVFS27_CNTL_STATUS__PsmTdo__SHIFT 0x8 +#define PWR_AVFS27_CNTL_STATUS__AlarmFlag_MASK 0x200 +#define PWR_AVFS27_CNTL_STATUS__AlarmFlag__SHIFT 0x9 +#define PWR_CKS_ENABLE__STRETCH_ENABLE_MASK 0x1 +#define PWR_CKS_ENABLE__STRETCH_ENABLE__SHIFT 0x0 +#define PWR_CKS_ENABLE__masterReset_MASK 0x2 +#define PWR_CKS_ENABLE__masterReset__SHIFT 0x1 +#define PWR_CKS_ENABLE__staticEnable_MASK 0x4 +#define PWR_CKS_ENABLE__staticEnable__SHIFT 0x2 +#define PWR_CKS_ENABLE__IGNORE_DROOP_DETECT_MASK 0x8 +#define PWR_CKS_ENABLE__IGNORE_DROOP_DETECT__SHIFT 0x3 +#define PWR_CKS_ENABLE__PCC_HAND_SHAKE_EN_MASK 0x10 +#define PWR_CKS_ENABLE__PCC_HAND_SHAKE_EN__SHIFT 0x4 +#define PWR_CKS_ENABLE__MET_CTRL_SEL_MASK 0x60 +#define PWR_CKS_ENABLE__MET_CTRL_SEL__SHIFT 0x5 +#define PWR_CKS_ENABLE__DS_HAND_SHAKE_EN_MASK 0x80 +#define PWR_CKS_ENABLE__DS_HAND_SHAKE_EN__SHIFT 0x7 +#define PWR_CKS_CNTL__CKS_BYPASS_MASK 0x1 +#define PWR_CKS_CNTL__CKS_BYPASS__SHIFT 0x0 +#define PWR_CKS_CNTL__CKS_PCCEnable_MASK 0x2 +#define PWR_CKS_CNTL__CKS_PCCEnable__SHIFT 0x1 +#define PWR_CKS_CNTL__CKS_TEMP_COMP_MASK 0x4 +#define PWR_CKS_CNTL__CKS_TEMP_COMP__SHIFT 0x2 +#define PWR_CKS_CNTL__CKS_STRETCH_AMOUNT_MASK 0x78 +#define PWR_CKS_CNTL__CKS_STRETCH_AMOUNT__SHIFT 0x3 +#define PWR_CKS_CNTL__CKS_SKIP_PHASE_BYPASS_MASK 0x80 +#define PWR_CKS_CNTL__CKS_SKIP_PHASE_BYPASS__SHIFT 0x7 +#define PWR_CKS_CNTL__CKS_SAMPLE_SIZE_MASK 0xf00 +#define PWR_CKS_CNTL__CKS_SAMPLE_SIZE__SHIFT 0x8 +#define PWR_CKS_CNTL__CKS_FSM_WAIT_CYCLES_MASK 0xf000 +#define PWR_CKS_CNTL__CKS_FSM_WAIT_CYCLES__SHIFT 0xc +#define PWR_CKS_CNTL__CKS_USE_FOR_LOW_FREQ_MASK 0x10000 +#define PWR_CKS_CNTL__CKS_USE_FOR_LOW_FREQ__SHIFT 0x10 +#define PWR_CKS_CNTL__CKS_NO_EXTRA_COARSE_STEP_MASK 0x20000 +#define PWR_CKS_CNTL__CKS_NO_EXTRA_COARSE_STEP__SHIFT 0x11 +#define PWR_CKS_CNTL__CKS_LDO_REFSEL_MASK 0x3c0000 +#define PWR_CKS_CNTL__CKS_LDO_REFSEL__SHIFT 0x12 +#define PWR_CKS_CNTL__DDT_DEBUS_SEL_MASK 0x400000 +#define PWR_CKS_CNTL__DDT_DEBUS_SEL__SHIFT 0x16 +#define PWR_CKS_CNTL__CKS_LDO_READY_COUNT_VAL_MASK 0x7f800000 +#define PWR_CKS_CNTL__CKS_LDO_READY_COUNT_VAL__SHIFT 0x17 +#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_COUNT_MASK 0x1ffffff +#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_COUNT__SHIFT 0x0 +#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_ENABLE_MASK 0x2000000 +#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_ENABLE__SHIFT 0x19 +#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_DISABLE_MASK 0x4000000 +#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_DISABLE__SHIFT 0x1a +#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_MASK_MASK 0x8000000 +#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_MASK__SHIFT 0x1b +#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_STAT_AK_MASK 0x10000000 +#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_STAT_AK__SHIFT 0x1c +#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_TYPE_MASK 0x20000000 +#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_TYPE__SHIFT 0x1d +#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_MODE_MASK 0x40000000 +#define PWR_DISP_TIMER_CONTROL__DISP_TIMER_INT_MODE__SHIFT 0x1e +#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_INT_RUNNING_MASK 0x1 +#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_INT_RUNNING__SHIFT 0x0 +#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_INT_STAT_MASK 0x2 +#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_INT_STAT__SHIFT 0x1 +#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_INT_MASK 0x4 +#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_INT__SHIFT 0x2 +#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_RUN_VAL_MASK 0xffffff80 +#define PWR_DISP_TIMER_DEBUG__DISP_TIMER_RUN_VAL__SHIFT 0x7 +#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_COUNT_MASK 0x1ffffff +#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_COUNT__SHIFT 0x0 +#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_ENABLE_MASK 0x2000000 +#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_ENABLE__SHIFT 0x19 +#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_DISABLE_MASK 0x4000000 +#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_DISABLE__SHIFT 0x1a +#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_MASK_MASK 0x8000000 +#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_MASK__SHIFT 0x1b +#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_STAT_AK_MASK 0x10000000 +#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_STAT_AK__SHIFT 0x1c +#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_TYPE_MASK 0x20000000 +#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_TYPE__SHIFT 0x1d +#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_MODE_MASK 0x40000000 +#define PWR_DISP_TIMER2_CONTROL__DISP_TIMER_INT_MODE__SHIFT 0x1e +#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_INT_RUNNING_MASK 0x1 +#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_INT_RUNNING__SHIFT 0x0 +#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_INT_STAT_MASK 0x2 +#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_INT_STAT__SHIFT 0x1 +#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_INT_MASK 0x4 +#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_INT__SHIFT 0x2 +#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_RUN_VAL_MASK 0xffffff80 +#define PWR_DISP_TIMER2_DEBUG__DISP_TIMER_RUN_VAL__SHIFT 0x7 +#define PWR_DISP_TIMER_CONTROL2__DISP_TIMER_PULSE_WIDTH_MASK 0x3ff +#define PWR_DISP_TIMER_CONTROL2__DISP_TIMER_PULSE_WIDTH__SHIFT 0x0 +#define VDDGFX_IDLE_PARAMETER__VDDGFX_IDLE_THRESHOLD_MASK 0xffff +#define VDDGFX_IDLE_PARAMETER__VDDGFX_IDLE_THRESHOLD__SHIFT 0x0 +#define VDDGFX_IDLE_PARAMETER__VDDGFX_IDLE_THRESHOLD_UNIT_MASK 0xf0000 +#define VDDGFX_IDLE_PARAMETER__VDDGFX_IDLE_THRESHOLD_UNIT__SHIFT 0x10 +#define VDDGFX_IDLE_CONTROL__VDDGFX_IDLE_EN_MASK 0x1 +#define VDDGFX_IDLE_CONTROL__VDDGFX_IDLE_EN__SHIFT 0x0 +#define VDDGFX_IDLE_CONTROL__VDDGFX_IDLE_DETECT_MASK 0x2 +#define VDDGFX_IDLE_CONTROL__VDDGFX_IDLE_DETECT__SHIFT 0x1 +#define VDDGFX_IDLE_CONTROL__FORCE_VDDGFX_IDLE_EXIT_MASK 0x4 +#define VDDGFX_IDLE_CONTROL__FORCE_VDDGFX_IDLE_EXIT__SHIFT 0x2 +#define VDDGFX_IDLE_CONTROL__SMC_VDDGFX_IDLE_STATE_MASK 0x8 +#define VDDGFX_IDLE_CONTROL__SMC_VDDGFX_IDLE_STATE__SHIFT 0x3 +#define VDDGFX_IDLE_EXIT__BIF_EXIT_REQ_MASK 0x1 +#define VDDGFX_IDLE_EXIT__BIF_EXIT_REQ__SHIFT 0x0 +#define LCAC_MC0_CNTL__MC0_ENABLE_MASK 0x1 +#define LCAC_MC0_CNTL__MC0_ENABLE__SHIFT 0x0 +#define LCAC_MC0_CNTL__MC0_THRESHOLD_MASK 0x1fffe +#define LCAC_MC0_CNTL__MC0_THRESHOLD__SHIFT 0x1 +#define LCAC_MC0_CNTL__MC0_BLOCK_ID_MASK 0x3e0000 +#define LCAC_MC0_CNTL__MC0_BLOCK_ID__SHIFT 0x11 +#define LCAC_MC0_CNTL__MC0_SIGNAL_ID_MASK 0x3fc00000 +#define LCAC_MC0_CNTL__MC0_SIGNAL_ID__SHIFT 0x16 +#define LCAC_MC0_OVR_SEL__MC0_OVR_SEL_MASK 0xffffffff +#define LCAC_MC0_OVR_SEL__MC0_OVR_SEL__SHIFT 0x0 +#define LCAC_MC0_OVR_VAL__MC0_OVR_VAL_MASK 0xffffffff +#define LCAC_MC0_OVR_VAL__MC0_OVR_VAL__SHIFT 0x0 +#define LCAC_MC1_CNTL__MC1_ENABLE_MASK 0x1 +#define LCAC_MC1_CNTL__MC1_ENABLE__SHIFT 0x0 +#define LCAC_MC1_CNTL__MC1_THRESHOLD_MASK 0x1fffe +#define LCAC_MC1_CNTL__MC1_THRESHOLD__SHIFT 0x1 +#define LCAC_MC1_CNTL__MC1_BLOCK_ID_MASK 0x3e0000 +#define LCAC_MC1_CNTL__MC1_BLOCK_ID__SHIFT 0x11 +#define LCAC_MC1_CNTL__MC1_SIGNAL_ID_MASK 0x3fc00000 +#define LCAC_MC1_CNTL__MC1_SIGNAL_ID__SHIFT 0x16 +#define LCAC_MC1_OVR_SEL__MC1_OVR_SEL_MASK 0xffffffff +#define LCAC_MC1_OVR_SEL__MC1_OVR_SEL__SHIFT 0x0 +#define LCAC_MC1_OVR_VAL__MC1_OVR_VAL_MASK 0xffffffff +#define LCAC_MC1_OVR_VAL__MC1_OVR_VAL__SHIFT 0x0 +#define LCAC_MC2_CNTL__MC2_ENABLE_MASK 0x1 +#define LCAC_MC2_CNTL__MC2_ENABLE__SHIFT 0x0 +#define LCAC_MC2_CNTL__MC2_THRESHOLD_MASK 0x1fffe +#define LCAC_MC2_CNTL__MC2_THRESHOLD__SHIFT 0x1 +#define LCAC_MC2_CNTL__MC2_BLOCK_ID_MASK 0x3e0000 +#define LCAC_MC2_CNTL__MC2_BLOCK_ID__SHIFT 0x11 +#define LCAC_MC2_CNTL__MC2_SIGNAL_ID_MASK 0x3fc00000 +#define LCAC_MC2_CNTL__MC2_SIGNAL_ID__SHIFT 0x16 +#define LCAC_MC2_OVR_SEL__MC2_OVR_SEL_MASK 0xffffffff +#define LCAC_MC2_OVR_SEL__MC2_OVR_SEL__SHIFT 0x0 +#define LCAC_MC2_OVR_VAL__MC2_OVR_VAL_MASK 0xffffffff +#define LCAC_MC2_OVR_VAL__MC2_OVR_VAL__SHIFT 0x0 +#define LCAC_MC3_CNTL__MC3_ENABLE_MASK 0x1 +#define LCAC_MC3_CNTL__MC3_ENABLE__SHIFT 0x0 +#define LCAC_MC3_CNTL__MC3_THRESHOLD_MASK 0x1fffe +#define LCAC_MC3_CNTL__MC3_THRESHOLD__SHIFT 0x1 +#define LCAC_MC3_CNTL__MC3_BLOCK_ID_MASK 0x3e0000 +#define LCAC_MC3_CNTL__MC3_BLOCK_ID__SHIFT 0x11 +#define LCAC_MC3_CNTL__MC3_SIGNAL_ID_MASK 0x3fc00000 +#define LCAC_MC3_CNTL__MC3_SIGNAL_ID__SHIFT 0x16 +#define LCAC_MC3_OVR_SEL__MC3_OVR_SEL_MASK 0xffffffff +#define LCAC_MC3_OVR_SEL__MC3_OVR_SEL__SHIFT 0x0 +#define LCAC_MC3_OVR_VAL__MC3_OVR_VAL_MASK 0xffffffff +#define LCAC_MC3_OVR_VAL__MC3_OVR_VAL__SHIFT 0x0 +#define LCAC_MC4_CNTL__MC4_ENABLE_MASK 0x1 +#define LCAC_MC4_CNTL__MC4_ENABLE__SHIFT 0x0 +#define LCAC_MC4_CNTL__MC4_THRESHOLD_MASK 0x1fffe +#define LCAC_MC4_CNTL__MC4_THRESHOLD__SHIFT 0x1 +#define LCAC_MC4_CNTL__MC4_BLOCK_ID_MASK 0x3e0000 +#define LCAC_MC4_CNTL__MC4_BLOCK_ID__SHIFT 0x11 +#define LCAC_MC4_CNTL__MC4_SIGNAL_ID_MASK 0x3fc00000 +#define LCAC_MC4_CNTL__MC4_SIGNAL_ID__SHIFT 0x16 +#define LCAC_MC4_OVR_SEL__MC4_OVR_SEL_MASK 0xffffffff +#define LCAC_MC4_OVR_SEL__MC4_OVR_SEL__SHIFT 0x0 +#define LCAC_MC4_OVR_VAL__MC4_OVR_VAL_MASK 0xffffffff +#define LCAC_MC4_OVR_VAL__MC4_OVR_VAL__SHIFT 0x0 +#define LCAC_MC5_CNTL__MC5_ENABLE_MASK 0x1 +#define LCAC_MC5_CNTL__MC5_ENABLE__SHIFT 0x0 +#define LCAC_MC5_CNTL__MC5_THRESHOLD_MASK 0x1fffe +#define LCAC_MC5_CNTL__MC5_THRESHOLD__SHIFT 0x1 +#define LCAC_MC5_CNTL__MC5_BLOCK_ID_MASK 0x3e0000 +#define LCAC_MC5_CNTL__MC5_BLOCK_ID__SHIFT 0x11 +#define LCAC_MC5_CNTL__MC5_SIGNAL_ID_MASK 0x3fc00000 +#define LCAC_MC5_CNTL__MC5_SIGNAL_ID__SHIFT 0x16 +#define LCAC_MC5_OVR_SEL__MC5_OVR_SEL_MASK 0xffffffff +#define LCAC_MC5_OVR_SEL__MC5_OVR_SEL__SHIFT 0x0 +#define LCAC_MC5_OVR_VAL__MC5_OVR_VAL_MASK 0xffffffff +#define LCAC_MC5_OVR_VAL__MC5_OVR_VAL__SHIFT 0x0 +#define LCAC_MC6_CNTL__MC6_ENABLE_MASK 0x1 +#define LCAC_MC6_CNTL__MC6_ENABLE__SHIFT 0x0 +#define LCAC_MC6_CNTL__MC6_THRESHOLD_MASK 0x1fffe +#define LCAC_MC6_CNTL__MC6_THRESHOLD__SHIFT 0x1 +#define LCAC_MC6_CNTL__MC6_BLOCK_ID_MASK 0x3e0000 +#define LCAC_MC6_CNTL__MC6_BLOCK_ID__SHIFT 0x11 +#define LCAC_MC6_CNTL__MC6_SIGNAL_ID_MASK 0x3fc00000 +#define LCAC_MC6_CNTL__MC6_SIGNAL_ID__SHIFT 0x16 +#define LCAC_MC6_OVR_SEL__MC6_OVR_SEL_MASK 0xffffffff +#define LCAC_MC6_OVR_SEL__MC6_OVR_SEL__SHIFT 0x0 +#define LCAC_MC6_OVR_VAL__MC6_OVR_VAL_MASK 0xffffffff +#define LCAC_MC6_OVR_VAL__MC6_OVR_VAL__SHIFT 0x0 +#define LCAC_MC7_CNTL__MC7_ENABLE_MASK 0x1 +#define LCAC_MC7_CNTL__MC7_ENABLE__SHIFT 0x0 +#define LCAC_MC7_CNTL__MC7_THRESHOLD_MASK 0x1fffe +#define LCAC_MC7_CNTL__MC7_THRESHOLD__SHIFT 0x1 +#define LCAC_MC7_CNTL__MC7_BLOCK_ID_MASK 0x3e0000 +#define LCAC_MC7_CNTL__MC7_BLOCK_ID__SHIFT 0x11 +#define LCAC_MC7_CNTL__MC7_SIGNAL_ID_MASK 0x3fc00000 +#define LCAC_MC7_CNTL__MC7_SIGNAL_ID__SHIFT 0x16 +#define LCAC_MC7_OVR_SEL__MC7_OVR_SEL_MASK 0xffffffff +#define LCAC_MC7_OVR_SEL__MC7_OVR_SEL__SHIFT 0x0 +#define LCAC_MC7_OVR_VAL__MC7_OVR_VAL_MASK 0xffffffff +#define LCAC_MC7_OVR_VAL__MC7_OVR_VAL__SHIFT 0x0 +#define LCAC_CPL_CNTL__CPL_ENABLE_MASK 0x1 +#define LCAC_CPL_CNTL__CPL_ENABLE__SHIFT 0x0 +#define LCAC_CPL_CNTL__CPL_THRESHOLD_MASK 0x1fffe +#define LCAC_CPL_CNTL__CPL_THRESHOLD__SHIFT 0x1 +#define LCAC_CPL_CNTL__CPL_BLOCK_ID_MASK 0x3e0000 +#define LCAC_CPL_CNTL__CPL_BLOCK_ID__SHIFT 0x11 +#define LCAC_CPL_CNTL__CPL_SIGNAL_ID_MASK 0x3fc00000 +#define LCAC_CPL_CNTL__CPL_SIGNAL_ID__SHIFT 0x16 +#define LCAC_CPL_OVR_SEL__CPL_OVR_SEL_MASK 0xffffffff +#define LCAC_CPL_OVR_SEL__CPL_OVR_SEL__SHIFT 0x0 +#define LCAC_CPL_OVR_VAL__CPL_OVR_VAL_MASK 0xffffffff +#define LCAC_CPL_OVR_VAL__CPL_OVR_VAL__SHIFT 0x0 +#define ROM_SMC_IND_INDEX__SMC_IND_ADDR_MASK 0xffffffff +#define ROM_SMC_IND_INDEX__SMC_IND_ADDR__SHIFT 0x0 +#define ROM_SMC_IND_DATA__SMC_IND_DATA_MASK 0xffffffff +#define ROM_SMC_IND_DATA__SMC_IND_DATA__SHIFT 0x0 +#define ROM_CNTL__SCK_OVERWRITE_MASK 0x2 +#define ROM_CNTL__SCK_OVERWRITE__SHIFT 0x1 +#define ROM_CNTL__CLOCK_GATING_EN_MASK 0x4 +#define ROM_CNTL__CLOCK_GATING_EN__SHIFT 0x2 +#define ROM_CNTL__CSB_ACTIVE_TO_SCK_SETUP_TIME_MASK 0xff00 +#define ROM_CNTL__CSB_ACTIVE_TO_SCK_SETUP_TIME__SHIFT 0x8 +#define ROM_CNTL__CSB_ACTIVE_TO_SCK_HOLD_TIME_MASK 0xff0000 +#define ROM_CNTL__CSB_ACTIVE_TO_SCK_HOLD_TIME__SHIFT 0x10 +#define ROM_CNTL__SCK_PRESCALE_REFCLK_MASK 0xf000000 +#define ROM_CNTL__SCK_PRESCALE_REFCLK__SHIFT 0x18 +#define ROM_CNTL__SCK_PRESCALE_CRYSTAL_CLK_MASK 0xf0000000 +#define ROM_CNTL__SCK_PRESCALE_CRYSTAL_CLK__SHIFT 0x1c +#define PAGE_MIRROR_CNTL__PAGE_MIRROR_BASE_ADDR_MASK 0xffffff +#define PAGE_MIRROR_CNTL__PAGE_MIRROR_BASE_ADDR__SHIFT 0x0 +#define PAGE_MIRROR_CNTL__PAGE_MIRROR_INVALIDATE_MASK 0x1000000 +#define PAGE_MIRROR_CNTL__PAGE_MIRROR_INVALIDATE__SHIFT 0x18 +#define PAGE_MIRROR_CNTL__PAGE_MIRROR_ENABLE_MASK 0x2000000 +#define PAGE_MIRROR_CNTL__PAGE_MIRROR_ENABLE__SHIFT 0x19 +#define PAGE_MIRROR_CNTL__PAGE_MIRROR_USAGE_MASK 0xc000000 +#define PAGE_MIRROR_CNTL__PAGE_MIRROR_USAGE__SHIFT 0x1a +#define ROM_STATUS__ROM_BUSY_MASK 0x1 +#define ROM_STATUS__ROM_BUSY__SHIFT 0x0 +#define CGTT_ROM_CLK_CTRL0__ON_DELAY_MASK 0xf +#define CGTT_ROM_CLK_CTRL0__ON_DELAY__SHIFT 0x0 +#define CGTT_ROM_CLK_CTRL0__OFF_HYSTERESIS_MASK 0xff0 +#define CGTT_ROM_CLK_CTRL0__OFF_HYSTERESIS__SHIFT 0x4 +#define CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE1_MASK 0x40000000 +#define CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE1__SHIFT 0x1e +#define CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK 0x80000000 +#define CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0__SHIFT 0x1f +#define ROM_INDEX__ROM_INDEX_MASK 0xffffff +#define ROM_INDEX__ROM_INDEX__SHIFT 0x0 +#define ROM_DATA__ROM_DATA_MASK 0xffffffff +#define ROM_DATA__ROM_DATA__SHIFT 0x0 +#define ROM_START__ROM_START_MASK 0xffffff +#define ROM_START__ROM_START__SHIFT 0x0 +#define ROM_SW_CNTL__DATA_SIZE_MASK 0xffff +#define ROM_SW_CNTL__DATA_SIZE__SHIFT 0x0 +#define ROM_SW_CNTL__COMMAND_SIZE_MASK 0x30000 +#define ROM_SW_CNTL__COMMAND_SIZE__SHIFT 0x10 +#define ROM_SW_CNTL__ROM_SW_RETURN_DATA_ENABLE_MASK 0x40000 +#define ROM_SW_CNTL__ROM_SW_RETURN_DATA_ENABLE__SHIFT 0x12 +#define ROM_SW_STATUS__ROM_SW_DONE_MASK 0x1 +#define ROM_SW_STATUS__ROM_SW_DONE__SHIFT 0x0 +#define ROM_SW_COMMAND__ROM_SW_INSTRUCTION_MASK 0xff +#define ROM_SW_COMMAND__ROM_SW_INSTRUCTION__SHIFT 0x0 +#define ROM_SW_COMMAND__ROM_SW_ADDRESS_MASK 0xffffff00 +#define ROM_SW_COMMAND__ROM_SW_ADDRESS__SHIFT 0x8 +#define ROM_SW_DATA_1__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_1__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_2__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_2__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_3__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_3__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_4__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_4__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_5__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_5__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_6__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_6__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_7__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_7__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_8__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_8__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_9__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_9__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_10__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_10__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_11__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_11__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_12__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_12__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_13__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_13__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_14__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_14__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_15__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_15__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_16__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_16__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_17__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_17__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_18__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_18__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_19__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_19__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_20__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_20__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_21__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_21__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_22__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_22__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_23__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_23__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_24__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_24__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_25__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_25__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_26__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_26__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_27__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_27__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_28__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_28__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_29__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_29__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_30__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_30__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_31__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_31__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_32__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_32__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_33__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_33__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_34__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_34__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_35__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_35__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_36__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_36__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_37__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_37__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_38__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_38__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_39__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_39__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_40__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_40__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_41__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_41__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_42__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_42__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_43__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_43__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_44__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_44__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_45__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_45__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_46__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_46__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_47__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_47__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_48__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_48__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_49__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_49__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_50__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_50__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_51__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_51__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_52__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_52__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_53__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_53__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_54__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_54__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_55__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_55__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_56__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_56__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_57__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_57__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_58__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_58__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_59__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_59__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_60__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_60__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_61__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_61__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_62__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_62__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_63__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_63__ROM_SW_DATA__SHIFT 0x0 +#define ROM_SW_DATA_64__ROM_SW_DATA_MASK 0xffffffff +#define ROM_SW_DATA_64__ROM_SW_DATA__SHIFT 0x0 +#define GC_CAC_CGTT_CLK_CTRL__ON_DELAY_MASK 0xf +#define GC_CAC_CGTT_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define GC_CAC_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define GC_CAC_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define GC_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK 0x40000000 +#define GC_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT 0x1e +#define GC_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_REG_MASK 0x80000000 +#define GC_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT 0x1f +#define SE_CAC_CGTT_CLK_CTRL__ON_DELAY_MASK 0xf +#define SE_CAC_CGTT_CLK_CTRL__ON_DELAY__SHIFT 0x0 +#define SE_CAC_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK 0xff0 +#define SE_CAC_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT 0x4 +#define SE_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK 0x40000000 +#define SE_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT 0x1e +#define SE_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_REG_MASK 0x80000000 +#define SE_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT 0x1f +#define GC_CAC_LKG_AGGR_LOWER__LKG_AGGR_31_0_MASK 0xffffffff +#define GC_CAC_LKG_AGGR_LOWER__LKG_AGGR_31_0__SHIFT 0x0 +#define GC_CAC_LKG_AGGR_UPPER__LKG_AGGR_63_32_MASK 0xffffffff +#define GC_CAC_LKG_AGGR_UPPER__LKG_AGGR_63_32__SHIFT 0x0 +#define GC_CAC_WEIGHT_CU_0__WEIGHT_CU_SIG0_MASK 0xffff +#define GC_CAC_WEIGHT_CU_0__WEIGHT_CU_SIG0__SHIFT 0x0 +#define GC_CAC_WEIGHT_CU_0__WEIGHT_CU_SIG1_MASK 0xffff0000 +#define GC_CAC_WEIGHT_CU_0__WEIGHT_CU_SIG1__SHIFT 0x10 +#define GC_CAC_WEIGHT_CU_1__WEIGHT_CU_SIG2_MASK 0xffff +#define GC_CAC_WEIGHT_CU_1__WEIGHT_CU_SIG2__SHIFT 0x0 +#define GC_CAC_WEIGHT_CU_1__WEIGHT_CU_SIG3_MASK 0xffff0000 +#define GC_CAC_WEIGHT_CU_1__WEIGHT_CU_SIG3__SHIFT 0x10 +#define GC_CAC_WEIGHT_CU_2__WEIGHT_CU_SIG4_MASK 0xffff +#define GC_CAC_WEIGHT_CU_2__WEIGHT_CU_SIG4__SHIFT 0x0 +#define GC_CAC_WEIGHT_CU_2__WEIGHT_CU_SIG5_MASK 0xffff0000 +#define GC_CAC_WEIGHT_CU_2__WEIGHT_CU_SIG5__SHIFT 0x10 +#define GC_CAC_WEIGHT_CU_3__WEIGHT_CU_SIG6_MASK 0xffff +#define GC_CAC_WEIGHT_CU_3__WEIGHT_CU_SIG6__SHIFT 0x0 +#define GC_CAC_WEIGHT_CU_3__WEIGHT_CU_SIG7_MASK 0xffff0000 +#define GC_CAC_WEIGHT_CU_3__WEIGHT_CU_SIG7__SHIFT 0x10 +#define GC_CAC_WEIGHT_CU_4__WEIGHT_CU_SIG8_MASK 0xffff +#define GC_CAC_WEIGHT_CU_4__WEIGHT_CU_SIG8__SHIFT 0x0 +#define GC_CAC_WEIGHT_CU_4__WEIGHT_CU_SIG9_MASK 0xffff0000 +#define GC_CAC_WEIGHT_CU_4__WEIGHT_CU_SIG9__SHIFT 0x10 +#define GC_CAC_WEIGHT_CU_5__WEIGHT_CU_SIG10_MASK 0xffff +#define GC_CAC_WEIGHT_CU_5__WEIGHT_CU_SIG10__SHIFT 0x0 +#define GC_CAC_WEIGHT_CU_5__WEIGHT_CU_SIG11_MASK 0xffff0000 +#define GC_CAC_WEIGHT_CU_5__WEIGHT_CU_SIG11__SHIFT 0x10 +#define GC_CAC_WEIGHT_CU_6__WEIGHT_CU_SIG12_MASK 0xffff +#define GC_CAC_WEIGHT_CU_6__WEIGHT_CU_SIG12__SHIFT 0x0 +#define GC_CAC_WEIGHT_CU_6__WEIGHT_CU_SIG13_MASK 0xffff0000 +#define GC_CAC_WEIGHT_CU_6__WEIGHT_CU_SIG13__SHIFT 0x10 +#define GC_CAC_WEIGHT_CU_7__WEIGHT_CU_SIG14_MASK 0xffff +#define GC_CAC_WEIGHT_CU_7__WEIGHT_CU_SIG14__SHIFT 0x0 +#define GC_CAC_WEIGHT_CU_7__WEIGHT_CU_SIG15_MASK 0xffff0000 +#define GC_CAC_WEIGHT_CU_7__WEIGHT_CU_SIG15__SHIFT 0x10 +#define GC_CAC_ACC_CU0__ACCUMULATOR_31_0_MASK 0xffffffff +#define GC_CAC_ACC_CU0__ACCUMULATOR_31_0__SHIFT 0x0 +#define GC_CAC_ACC_CU1__ACCUMULATOR_31_0_MASK 0xffffffff +#define GC_CAC_ACC_CU1__ACCUMULATOR_31_0__SHIFT 0x0 +#define GC_CAC_ACC_CU2__ACCUMULATOR_31_0_MASK 0xffffffff +#define GC_CAC_ACC_CU2__ACCUMULATOR_31_0__SHIFT 0x0 +#define GC_CAC_ACC_CU3__ACCUMULATOR_31_0_MASK 0xffffffff +#define GC_CAC_ACC_CU3__ACCUMULATOR_31_0__SHIFT 0x0 +#define GC_CAC_ACC_CU4__ACCUMULATOR_31_0_MASK 0xffffffff +#define GC_CAC_ACC_CU4__ACCUMULATOR_31_0__SHIFT 0x0 +#define GC_CAC_ACC_CU5__ACCUMULATOR_31_0_MASK 0xffffffff +#define GC_CAC_ACC_CU5__ACCUMULATOR_31_0__SHIFT 0x0 +#define GC_CAC_ACC_CU6__ACCUMULATOR_31_0_MASK 0xffffffff +#define GC_CAC_ACC_CU6__ACCUMULATOR_31_0__SHIFT 0x0 +#define GC_CAC_ACC_CU7__ACCUMULATOR_31_0_MASK 0xffffffff +#define GC_CAC_ACC_CU7__ACCUMULATOR_31_0__SHIFT 0x0 +#define GC_CAC_ACC_CU8__ACCUMULATOR_31_0_MASK 0xffffffff +#define GC_CAC_ACC_CU8__ACCUMULATOR_31_0__SHIFT 0x0 +#define GC_CAC_ACC_CU9__ACCUMULATOR_31_0_MASK 0xffffffff +#define GC_CAC_ACC_CU9__ACCUMULATOR_31_0__SHIFT 0x0 +#define GC_CAC_ACC_CU10__ACCUMULATOR_31_0_MASK 0xffffffff +#define GC_CAC_ACC_CU10__ACCUMULATOR_31_0__SHIFT 0x0 +#define GC_CAC_ACC_CU11__ACCUMULATOR_31_0_MASK 0xffffffff +#define GC_CAC_ACC_CU11__ACCUMULATOR_31_0__SHIFT 0x0 +#define GC_CAC_ACC_CU12__ACCUMULATOR_31_0_MASK 0xffffffff +#define GC_CAC_ACC_CU12__ACCUMULATOR_31_0__SHIFT 0x0 +#define GC_CAC_ACC_CU13__ACCUMULATOR_31_0_MASK 0xffffffff +#define GC_CAC_ACC_CU13__ACCUMULATOR_31_0__SHIFT 0x0 +#define GC_CAC_ACC_CU14__ACCUMULATOR_31_0_MASK 0xffffffff +#define GC_CAC_ACC_CU14__ACCUMULATOR_31_0__SHIFT 0x0 +#define GC_CAC_ACC_CU15__ACCUMULATOR_31_0_MASK 0xffffffff +#define GC_CAC_ACC_CU15__ACCUMULATOR_31_0__SHIFT 0x0 +#define GC_CAC_OVRD_CU__OVRRD_SELECT_MASK 0xffff +#define GC_CAC_OVRD_CU__OVRRD_SELECT__SHIFT 0x0 +#define GC_CAC_OVRD_CU__OVRRD_VALUE_MASK 0xffff0000 +#define GC_CAC_OVRD_CU__OVRRD_VALUE__SHIFT 0x10 + +#endif /* SMU_7_1_3_SH_MASK_H */ diff --git a/drivers/gpu/drm/amd/amdgpu/atom-bits.h b/drivers/gpu/drm/amd/include/atom-bits.h similarity index 100% rename from drivers/gpu/drm/amd/amdgpu/atom-bits.h rename to drivers/gpu/drm/amd/include/atom-bits.h diff --git a/drivers/gpu/drm/amd/amdgpu/atom-names.h b/drivers/gpu/drm/amd/include/atom-names.h similarity index 100% rename from drivers/gpu/drm/amd/amdgpu/atom-names.h rename to drivers/gpu/drm/amd/include/atom-names.h diff --git a/drivers/gpu/drm/amd/amdgpu/atom-types.h b/drivers/gpu/drm/amd/include/atom-types.h similarity index 100% rename from drivers/gpu/drm/amd/amdgpu/atom-types.h rename to drivers/gpu/drm/amd/include/atom-types.h diff --git a/drivers/gpu/drm/amd/amdgpu/atombios.h b/drivers/gpu/drm/amd/include/atombios.h similarity index 100% rename from drivers/gpu/drm/amd/amdgpu/atombios.h rename to drivers/gpu/drm/amd/include/atombios.h diff --git a/drivers/gpu/drm/amd/include/cgs_common.h b/drivers/gpu/drm/amd/include/cgs_common.h new file mode 100644 index 000000000000..992dcd8a5c6a --- /dev/null +++ b/drivers/gpu/drm/amd/include/cgs_common.h @@ -0,0 +1,624 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#ifndef _CGS_COMMON_H +#define _CGS_COMMON_H + +#include "amd_shared.h" + +/** + * enum cgs_gpu_mem_type - GPU memory types + */ +enum cgs_gpu_mem_type { + CGS_GPU_MEM_TYPE__VISIBLE_FB, + CGS_GPU_MEM_TYPE__INVISIBLE_FB, + CGS_GPU_MEM_TYPE__VISIBLE_CONTIG_FB, + CGS_GPU_MEM_TYPE__INVISIBLE_CONTIG_FB, + CGS_GPU_MEM_TYPE__GART_CACHEABLE, + CGS_GPU_MEM_TYPE__GART_WRITECOMBINE +}; + +/** + * enum cgs_ind_reg - Indirect register spaces + */ +enum cgs_ind_reg { + CGS_IND_REG__MMIO, + CGS_IND_REG__PCIE, + CGS_IND_REG__SMC, + CGS_IND_REG__UVD_CTX, + CGS_IND_REG__DIDT, + CGS_IND_REG__AUDIO_ENDPT +}; + +/** + * enum cgs_clock - Clocks controlled by the SMU + */ +enum cgs_clock { + CGS_CLOCK__SCLK, + CGS_CLOCK__MCLK, + CGS_CLOCK__VCLK, + CGS_CLOCK__DCLK, + CGS_CLOCK__ECLK, + CGS_CLOCK__ACLK, + CGS_CLOCK__ICLK, + /* ... */ +}; + +/** + * enum cgs_engine - Engines that can be statically power-gated + */ +enum cgs_engine { + CGS_ENGINE__UVD, + CGS_ENGINE__VCE, + CGS_ENGINE__VP8, + CGS_ENGINE__ACP_DMA, + CGS_ENGINE__ACP_DSP0, + CGS_ENGINE__ACP_DSP1, + CGS_ENGINE__ISP, + /* ... */ +}; + +/** + * enum cgs_voltage_planes - Voltage planes for external camera HW + */ +enum cgs_voltage_planes { + CGS_VOLTAGE_PLANE__SENSOR0, + CGS_VOLTAGE_PLANE__SENSOR1, + /* ... */ +}; + +/* + * enum cgs_ucode_id - Firmware types for different IPs + */ +enum cgs_ucode_id { + CGS_UCODE_ID_SMU = 0, + CGS_UCODE_ID_SDMA0, + CGS_UCODE_ID_SDMA1, + CGS_UCODE_ID_CP_CE, + CGS_UCODE_ID_CP_PFP, + CGS_UCODE_ID_CP_ME, + CGS_UCODE_ID_CP_MEC, + CGS_UCODE_ID_CP_MEC_JT1, + CGS_UCODE_ID_CP_MEC_JT2, + CGS_UCODE_ID_GMCON_RENG, + CGS_UCODE_ID_RLC_G, + CGS_UCODE_ID_MAXIMUM, +}; + +/** + * struct cgs_clock_limits - Clock limits + * + * Clocks are specified in 10KHz units. + */ +struct cgs_clock_limits { + unsigned min; /**< Minimum supported frequency */ + unsigned max; /**< Maxumim supported frequency */ + unsigned sustainable; /**< Thermally sustainable frequency */ +}; + +/** + * struct cgs_firmware_info - Firmware information + */ +struct cgs_firmware_info { + uint16_t version; + uint16_t feature_version; + uint32_t image_size; + uint64_t mc_addr; + void *kptr; +}; + +typedef unsigned long cgs_handle_t; + +/** + * cgs_gpu_mem_info() - Return information about memory heaps + * @cgs_device: opaque device handle + * @type: memory type + * @mc_start: Start MC address of the heap (output) + * @mc_size: MC address space size (output) + * @mem_size: maximum amount of memory available for allocation (output) + * + * This function returns information about memory heaps. The type + * parameter is used to select the memory heap. The mc_start and + * mc_size for GART heaps may be bigger than the memory available for + * allocation. + * + * mc_start and mc_size are undefined for non-contiguous FB memory + * types, since buffers allocated with these types may or may not be + * GART mapped. + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_gpu_mem_info_t)(void *cgs_device, enum cgs_gpu_mem_type type, + uint64_t *mc_start, uint64_t *mc_size, + uint64_t *mem_size); + +/** + * cgs_gmap_kmem() - map kernel memory to GART aperture + * @cgs_device: opaque device handle + * @kmem: pointer to kernel memory + * @size: size to map + * @min_offset: minimum offset from start of GART aperture + * @max_offset: maximum offset from start of GART aperture + * @kmem_handle: kernel memory handle (output) + * @mcaddr: MC address (output) + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_gmap_kmem_t)(void *cgs_device, void *kmem, uint64_t size, + uint64_t min_offset, uint64_t max_offset, + cgs_handle_t *kmem_handle, uint64_t *mcaddr); + +/** + * cgs_gunmap_kmem() - unmap kernel memory + * @cgs_device: opaque device handle + * @kmem_handle: kernel memory handle returned by gmap_kmem + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_gunmap_kmem_t)(void *cgs_device, cgs_handle_t kmem_handle); + +/** + * cgs_alloc_gpu_mem() - Allocate GPU memory + * @cgs_device: opaque device handle + * @type: memory type + * @size: size in bytes + * @align: alignment in bytes + * @min_offset: minimum offset from start of heap + * @max_offset: maximum offset from start of heap + * @handle: memory handle (output) + * + * The memory types CGS_GPU_MEM_TYPE_*_CONTIG_FB force contiguous + * memory allocation. This guarantees that the MC address returned by + * cgs_gmap_gpu_mem is not mapped through the GART. The non-contiguous + * FB memory types may be GART mapped depending on memory + * fragmentation and memory allocator policies. + * + * If min/max_offset are non-0, the allocation will be forced to + * reside between these offsets in its respective memory heap. The + * base address that the offset relates to, depends on the memory + * type. + * + * - CGS_GPU_MEM_TYPE__*_CONTIG_FB: FB MC base address + * - CGS_GPU_MEM_TYPE__GART_*: GART aperture base address + * - others: undefined, don't use with max_offset + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_alloc_gpu_mem_t)(void *cgs_device, enum cgs_gpu_mem_type type, + uint64_t size, uint64_t align, + uint64_t min_offset, uint64_t max_offset, + cgs_handle_t *handle); + +/** + * cgs_free_gpu_mem() - Free GPU memory + * @cgs_device: opaque device handle + * @handle: memory handle returned by alloc or import + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_free_gpu_mem_t)(void *cgs_device, cgs_handle_t handle); + +/** + * cgs_gmap_gpu_mem() - GPU-map GPU memory + * @cgs_device: opaque device handle + * @handle: memory handle returned by alloc or import + * @mcaddr: MC address (output) + * + * Ensures that a buffer is GPU accessible and returns its MC address. + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_gmap_gpu_mem_t)(void *cgs_device, cgs_handle_t handle, + uint64_t *mcaddr); + +/** + * cgs_gunmap_gpu_mem() - GPU-unmap GPU memory + * @cgs_device: opaque device handle + * @handle: memory handle returned by alloc or import + * + * Allows the buffer to be migrated while it's not used by the GPU. + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_gunmap_gpu_mem_t)(void *cgs_device, cgs_handle_t handle); + +/** + * cgs_kmap_gpu_mem() - Kernel-map GPU memory + * + * @cgs_device: opaque device handle + * @handle: memory handle returned by alloc or import + * @map: Kernel virtual address the memory was mapped to (output) + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_kmap_gpu_mem_t)(void *cgs_device, cgs_handle_t handle, + void **map); + +/** + * cgs_kunmap_gpu_mem() - Kernel-unmap GPU memory + * @cgs_device: opaque device handle + * @handle: memory handle returned by alloc or import + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_kunmap_gpu_mem_t)(void *cgs_device, cgs_handle_t handle); + +/** + * cgs_read_register() - Read an MMIO register + * @cgs_device: opaque device handle + * @offset: register offset + * + * Return: register value + */ +typedef uint32_t (*cgs_read_register_t)(void *cgs_device, unsigned offset); + +/** + * cgs_write_register() - Write an MMIO register + * @cgs_device: opaque device handle + * @offset: register offset + * @value: register value + */ +typedef void (*cgs_write_register_t)(void *cgs_device, unsigned offset, + uint32_t value); + +/** + * cgs_read_ind_register() - Read an indirect register + * @cgs_device: opaque device handle + * @offset: register offset + * + * Return: register value + */ +typedef uint32_t (*cgs_read_ind_register_t)(void *cgs_device, enum cgs_ind_reg space, + unsigned index); + +/** + * cgs_write_ind_register() - Write an indirect register + * @cgs_device: opaque device handle + * @offset: register offset + * @value: register value + */ +typedef void (*cgs_write_ind_register_t)(void *cgs_device, enum cgs_ind_reg space, + unsigned index, uint32_t value); + +/** + * cgs_read_pci_config_byte() - Read byte from PCI configuration space + * @cgs_device: opaque device handle + * @addr: address + * + * Return: Value read + */ +typedef uint8_t (*cgs_read_pci_config_byte_t)(void *cgs_device, unsigned addr); + +/** + * cgs_read_pci_config_word() - Read word from PCI configuration space + * @cgs_device: opaque device handle + * @addr: address, must be word-aligned + * + * Return: Value read + */ +typedef uint16_t (*cgs_read_pci_config_word_t)(void *cgs_device, unsigned addr); + +/** + * cgs_read_pci_config_dword() - Read dword from PCI configuration space + * @cgs_device: opaque device handle + * @addr: address, must be dword-aligned + * + * Return: Value read + */ +typedef uint32_t (*cgs_read_pci_config_dword_t)(void *cgs_device, + unsigned addr); + +/** + * cgs_write_pci_config_byte() - Write byte to PCI configuration space + * @cgs_device: opaque device handle + * @addr: address + * @value: value to write + */ +typedef void (*cgs_write_pci_config_byte_t)(void *cgs_device, unsigned addr, + uint8_t value); + +/** + * cgs_write_pci_config_word() - Write byte to PCI configuration space + * @cgs_device: opaque device handle + * @addr: address, must be word-aligned + * @value: value to write + */ +typedef void (*cgs_write_pci_config_word_t)(void *cgs_device, unsigned addr, + uint16_t value); + +/** + * cgs_write_pci_config_dword() - Write byte to PCI configuration space + * @cgs_device: opaque device handle + * @addr: address, must be dword-aligned + * @value: value to write + */ +typedef void (*cgs_write_pci_config_dword_t)(void *cgs_device, unsigned addr, + uint32_t value); + +/** + * cgs_atom_get_data_table() - Get a pointer to an ATOM BIOS data table + * @cgs_device: opaque device handle + * @table: data table index + * @size: size of the table (output, may be NULL) + * @frev: table format revision (output, may be NULL) + * @crev: table content revision (output, may be NULL) + * + * Return: Pointer to start of the table, or NULL on failure + */ +typedef const void *(*cgs_atom_get_data_table_t)( + void *cgs_device, unsigned table, + uint16_t *size, uint8_t *frev, uint8_t *crev); + +/** + * cgs_atom_get_cmd_table_revs() - Get ATOM BIOS command table revisions + * @cgs_device: opaque device handle + * @table: data table index + * @frev: table format revision (output, may be NULL) + * @crev: table content revision (output, may be NULL) + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_atom_get_cmd_table_revs_t)(void *cgs_device, unsigned table, + uint8_t *frev, uint8_t *crev); + +/** + * cgs_atom_exec_cmd_table() - Execute an ATOM BIOS command table + * @cgs_device: opaque device handle + * @table: command table index + * @args: arguments + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_atom_exec_cmd_table_t)(void *cgs_device, + unsigned table, void *args); + +/** + * cgs_create_pm_request() - Create a power management request + * @cgs_device: opaque device handle + * @request: handle of created PM request (output) + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_create_pm_request_t)(void *cgs_device, cgs_handle_t *request); + +/** + * cgs_destroy_pm_request() - Destroy a power management request + * @cgs_device: opaque device handle + * @request: handle of created PM request + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_destroy_pm_request_t)(void *cgs_device, cgs_handle_t request); + +/** + * cgs_set_pm_request() - Activate or deactiveate a PM request + * @cgs_device: opaque device handle + * @request: PM request handle + * @active: 0 = deactivate, non-0 = activate + * + * While a PM request is active, its minimum clock requests are taken + * into account as the requested engines are powered up. When the + * request is inactive, the engines may be powered down and clocks may + * be lower, depending on other PM requests by other driver + * components. + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_set_pm_request_t)(void *cgs_device, cgs_handle_t request, + int active); + +/** + * cgs_pm_request_clock() - Request a minimum frequency for a specific clock + * @cgs_device: opaque device handle + * @request: PM request handle + * @clock: which clock? + * @freq: requested min. frequency in 10KHz units (0 to clear request) + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_pm_request_clock_t)(void *cgs_device, cgs_handle_t request, + enum cgs_clock clock, unsigned freq); + +/** + * cgs_pm_request_engine() - Request an engine to be powered up + * @cgs_device: opaque device handle + * @request: PM request handle + * @engine: which engine? + * @powered: 0 = powered down, non-0 = powered up + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_pm_request_engine_t)(void *cgs_device, cgs_handle_t request, + enum cgs_engine engine, int powered); + +/** + * cgs_pm_query_clock_limits() - Query clock frequency limits + * @cgs_device: opaque device handle + * @clock: which clock? + * @limits: clock limits + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_pm_query_clock_limits_t)(void *cgs_device, + enum cgs_clock clock, + struct cgs_clock_limits *limits); + +/** + * cgs_set_camera_voltages() - Apply specific voltages to PMIC voltage planes + * @cgs_device: opaque device handle + * @mask: bitmask of voltages to change (1<ops->func(dev, ##__VA_ARGS__)) +#define CGS_OS_CALL(func,dev,...) \ + (((struct cgs_device *)dev)->os_ops->func(dev, ##__VA_ARGS__)) + +#define cgs_gpu_mem_info(dev,type,mc_start,mc_size,mem_size) \ + CGS_CALL(gpu_mem_info,dev,type,mc_start,mc_size,mem_size) +#define cgs_gmap_kmem(dev,kmem,size,min_off,max_off,kmem_handle,mcaddr) \ + CGS_CALL(gmap_kmem,dev,kmem,size,min_off,max_off,kmem_handle,mcaddr) +#define cgs_gunmap_kmem(dev,kmem_handle) \ + CGS_CALL(gunmap_kmem,dev,keme_handle) +#define cgs_alloc_gpu_mem(dev,type,size,align,min_off,max_off,handle) \ + CGS_CALL(alloc_gpu_mem,dev,type,size,align,min_off,max_off,handle) +#define cgs_free_gpu_mem(dev,handle) \ + CGS_CALL(free_gpu_mem,dev,handle) +#define cgs_gmap_gpu_mem(dev,handle,mcaddr) \ + CGS_CALL(gmap_gpu_mem,dev,handle,mcaddr) +#define cgs_gunmap_gpu_mem(dev,handle) \ + CGS_CALL(gunmap_gpu_mem,dev,handle) +#define cgs_kmap_gpu_mem(dev,handle,map) \ + CGS_CALL(kmap_gpu_mem,dev,handle,map) +#define cgs_kunmap_gpu_mem(dev,handle) \ + CGS_CALL(kunmap_gpu_mem,dev,handle) + +#define cgs_read_register(dev,offset) \ + CGS_CALL(read_register,dev,offset) +#define cgs_write_register(dev,offset,value) \ + CGS_CALL(write_register,dev,offset,value) +#define cgs_read_ind_register(dev,space,index) \ + CGS_CALL(read_ind_register,dev,space,index) +#define cgs_write_ind_register(dev,space,index,value) \ + CGS_CALL(write_ind_register,dev,space,index,value) + +#define cgs_read_pci_config_byte(dev,addr) \ + CGS_CALL(read_pci_config_byte,dev,addr) +#define cgs_read_pci_config_word(dev,addr) \ + CGS_CALL(read_pci_config_word,dev,addr) +#define cgs_read_pci_config_dword(dev,addr) \ + CGS_CALL(read_pci_config_dword,dev,addr) +#define cgs_write_pci_config_byte(dev,addr,value) \ + CGS_CALL(write_pci_config_byte,dev,addr,value) +#define cgs_write_pci_config_word(dev,addr,value) \ + CGS_CALL(write_pci_config_word,dev,addr,value) +#define cgs_write_pci_config_dword(dev,addr,value) \ + CGS_CALL(write_pci_config_dword,dev,addr,value) + +#define cgs_atom_get_data_table(dev,table,size,frev,crev) \ + CGS_CALL(atom_get_data_table,dev,table,size,frev,crev) +#define cgs_atom_get_cmd_table_revs(dev,table,frev,crev) \ + CGS_CALL(atom_get_cmd_table_revs,dev,table,frev,crev) +#define cgs_atom_exec_cmd_table(dev,table,args) \ + CGS_CALL(atom_exec_cmd_table,dev,table,args) + +#define cgs_create_pm_request(dev,request) \ + CGS_CALL(create_pm_request,dev,request) +#define cgs_destroy_pm_request(dev,request) \ + CGS_CALL(destroy_pm_request,dev,request) +#define cgs_set_pm_request(dev,request,active) \ + CGS_CALL(set_pm_request,dev,request,active) +#define cgs_pm_request_clock(dev,request,clock,freq) \ + CGS_CALL(pm_request_clock,dev,request,clock,freq) +#define cgs_pm_request_engine(dev,request,engine,powered) \ + CGS_CALL(pm_request_engine,dev,request,engine,powered) +#define cgs_pm_query_clock_limits(dev,clock,limits) \ + CGS_CALL(pm_query_clock_limits,dev,clock,limits) +#define cgs_set_camera_voltages(dev,mask,voltages) \ + CGS_CALL(set_camera_voltages,dev,mask,voltages) +#define cgs_get_firmware_info(dev, type, info) \ + CGS_CALL(get_firmware_info, dev, type, info) +#define cgs_set_powergating_state(dev, block_type, state) \ + CGS_CALL(set_powergating_state, dev, block_type, state) +#define cgs_set_clockgating_state(dev, block_type, state) \ + CGS_CALL(set_clockgating_state, dev, block_type, state) + +#endif /* _CGS_COMMON_H */ diff --git a/drivers/gpu/drm/amd/include/cgs_linux.h b/drivers/gpu/drm/amd/include/cgs_linux.h new file mode 100644 index 000000000000..488642f08267 --- /dev/null +++ b/drivers/gpu/drm/amd/include/cgs_linux.h @@ -0,0 +1,135 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#ifndef _CGS_LINUX_H +#define _CGS_LINUX_H + +#include "cgs_common.h" + +/** + * cgs_import_gpu_mem() - Import dmabuf handle + * @cgs_device: opaque device handle + * @dmabuf_fd: DMABuf file descriptor + * @handle: memory handle (output) + * + * Must be called in the process context that dmabuf_fd belongs to. + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_import_gpu_mem_t)(void *cgs_device, int dmabuf_fd, + cgs_handle_t *handle); + +/** + * cgs_irq_source_set_func() - Callback for enabling/disabling interrupt sources + * @private_data: private data provided to cgs_add_irq_source + * @src_id: interrupt source ID + * @type: interrupt type + * @enabled: 0 = disable source, non-0 = enable source + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_irq_source_set_func_t)(void *private_data, + unsigned src_id, unsigned type, + int enabled); + +/** + * cgs_irq_handler_func() - Interrupt handler callback + * @private_data: private data provided to cgs_add_irq_source + * @src_id: interrupt source ID + * @iv_entry: pointer to raw ih ring entry + * + * This callback runs in interrupt context. + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_irq_handler_func_t)(void *private_data, + unsigned src_id, const uint32_t *iv_entry); + +/** + * cgs_add_irq_source() - Add an IRQ source + * @cgs_device: opaque device handle + * @src_id: interrupt source ID + * @num_types: number of interrupt types that can be independently enabled + * @set: callback function to enable/disable an interrupt type + * @handler: interrupt handler callback + * @private_data: private data to pass to callback functions + * + * The same IRQ source can be added only once. Adding an IRQ source + * indicates ownership of that IRQ source and all its IRQ types. + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_add_irq_source_t)(void *cgs_device, unsigned src_id, + unsigned num_types, + cgs_irq_source_set_func_t set, + cgs_irq_handler_func_t handler, + void *private_data); + +/** + * cgs_irq_get() - Request enabling an IRQ source and type + * @cgs_device: opaque device handle + * @src_id: interrupt source ID + * @type: interrupt type + * + * cgs_irq_get and cgs_irq_put calls must be balanced. They count + * "references" to IRQ sources. + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_irq_get_t)(void *cgs_device, unsigned src_id, unsigned type); + +/** + * cgs_irq_put() - Indicate IRQ source is no longer needed + * @cgs_device: opaque device handle + * @src_id: interrupt source ID + * @type: interrupt type + * + * cgs_irq_get and cgs_irq_put calls must be balanced. They count + * "references" to IRQ sources. Even after cgs_irq_put is called, the + * IRQ handler may still be called if there are more refecences to + * the IRQ source. + * + * Return: 0 on success, -errno otherwise + */ +typedef int (*cgs_irq_put_t)(void *cgs_device, unsigned src_id, unsigned type); + +struct cgs_os_ops { + cgs_import_gpu_mem_t import_gpu_mem; + + /* IRQ handling */ + cgs_add_irq_source_t add_irq_source; + cgs_irq_get_t irq_get; + cgs_irq_put_t irq_put; +}; + +#define cgs_import_gpu_mem(dev,dmabuf_fd,handle) \ + CGS_OS_CALL(import_gpu_mem,dev,dmabuf_fd,handle) +#define cgs_add_irq_source(dev,src_id,num_types,set,handler,private_data) \ + CGS_OS_CALL(add_irq_source,dev,src_id,num_types,set,handler, \ + private_data) +#define cgs_irq_get(dev,src_id,type) \ + CGS_OS_CALL(irq_get,dev,src_id,type) +#define cgs_irq_put(dev,src_id,type) \ + CGS_OS_CALL(irq_put,dev,src_id,type) + +#endif /* _CGS_LINUX_H */ diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h index 9080daa116b6..888250b33ea8 100644 --- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h +++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h @@ -52,7 +52,8 @@ enum kgd_engine_type { KGD_ENGINE_MEC1, KGD_ENGINE_MEC2, KGD_ENGINE_RLC, - KGD_ENGINE_SDMA, + KGD_ENGINE_SDMA1, + KGD_ENGINE_SDMA2, KGD_ENGINE_MAX }; diff --git a/drivers/gpu/drm/amd/amdgpu/pptable.h b/drivers/gpu/drm/amd/include/pptable.h similarity index 98% rename from drivers/gpu/drm/amd/amdgpu/pptable.h rename to drivers/gpu/drm/amd/include/pptable.h index 0030f726e68c..ee6978b30b77 100644 --- a/drivers/gpu/drm/amd/amdgpu/pptable.h +++ b/drivers/gpu/drm/amd/include/pptable.h @@ -146,6 +146,9 @@ typedef struct _ATOM_PPLIB_EXTENDEDHEADER #define ATOM_PP_PLATFORM_CAP_VRHOT_GPIO_CONFIGURABLE 0x00200000 // Does the driver supports VR HOT GPIO Configurable. #define ATOM_PP_PLATFORM_CAP_TEMP_INVERSION 0x00400000 // Does the driver supports Temp Inversion feature. #define ATOM_PP_PLATFORM_CAP_EVV 0x00800000 +#define ATOM_PP_PLATFORM_COMBINE_PCC_WITH_THERMAL_SIGNAL 0x01000000 +#define ATOM_PP_PLATFORM_LOAD_POST_PRODUCTION_FIRMWARE 0x02000000 +#define ATOM_PP_PLATFORM_CAP_DISABLE_USING_ACTUAL_TEMPERATURE_FOR_POWER_CALC 0x04000000 typedef struct _ATOM_PPLIB_POWERPLAYTABLE { @@ -673,7 +676,8 @@ typedef struct _ATOM_PPLIB_POWERTUNE_Table_V1 UCHAR revid; ATOM_PowerTune_Table power_tune_table; USHORT usMaximumPowerDeliveryLimit; - USHORT usReserve[7]; + USHORT usTjMax; + USHORT usReserve[6]; } ATOM_PPLIB_POWERTUNE_Table_V1; #define ATOM_PPM_A_A 1 diff --git a/drivers/gpu/drm/amd/include/vi_structs.h b/drivers/gpu/drm/amd/include/vi_structs.h new file mode 100644 index 000000000000..65cfacd7a66c --- /dev/null +++ b/drivers/gpu/drm/amd/include/vi_structs.h @@ -0,0 +1,417 @@ +/* + * Copyright 2012 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef VI_STRUCTS_H_ +#define VI_STRUCTS_H_ + +struct vi_sdma_mqd { + uint32_t sdmax_rlcx_rb_cntl; + uint32_t sdmax_rlcx_rb_base; + uint32_t sdmax_rlcx_rb_base_hi; + uint32_t sdmax_rlcx_rb_rptr; + uint32_t sdmax_rlcx_rb_wptr; + uint32_t sdmax_rlcx_rb_wptr_poll_cntl; + uint32_t sdmax_rlcx_rb_wptr_poll_addr_hi; + uint32_t sdmax_rlcx_rb_wptr_poll_addr_lo; + uint32_t sdmax_rlcx_rb_rptr_addr_hi; + uint32_t sdmax_rlcx_rb_rptr_addr_lo; + uint32_t sdmax_rlcx_ib_cntl; + uint32_t sdmax_rlcx_ib_rptr; + uint32_t sdmax_rlcx_ib_offset; + uint32_t sdmax_rlcx_ib_base_lo; + uint32_t sdmax_rlcx_ib_base_hi; + uint32_t sdmax_rlcx_ib_size; + uint32_t sdmax_rlcx_skip_cntl; + uint32_t sdmax_rlcx_context_status; + uint32_t sdmax_rlcx_doorbell; + uint32_t sdmax_rlcx_virtual_addr; + uint32_t sdmax_rlcx_ape1_cntl; + uint32_t sdmax_rlcx_doorbell_log; + uint32_t reserved_22; + uint32_t reserved_23; + uint32_t reserved_24; + uint32_t reserved_25; + uint32_t reserved_26; + uint32_t reserved_27; + uint32_t reserved_28; + uint32_t reserved_29; + uint32_t reserved_30; + uint32_t reserved_31; + uint32_t reserved_32; + uint32_t reserved_33; + uint32_t reserved_34; + uint32_t reserved_35; + uint32_t reserved_36; + uint32_t reserved_37; + uint32_t reserved_38; + uint32_t reserved_39; + uint32_t reserved_40; + uint32_t reserved_41; + uint32_t reserved_42; + uint32_t reserved_43; + uint32_t reserved_44; + uint32_t reserved_45; + uint32_t reserved_46; + uint32_t reserved_47; + uint32_t reserved_48; + uint32_t reserved_49; + uint32_t reserved_50; + uint32_t reserved_51; + uint32_t reserved_52; + uint32_t reserved_53; + uint32_t reserved_54; + uint32_t reserved_55; + uint32_t reserved_56; + uint32_t reserved_57; + uint32_t reserved_58; + uint32_t reserved_59; + uint32_t reserved_60; + uint32_t reserved_61; + uint32_t reserved_62; + uint32_t reserved_63; + uint32_t reserved_64; + uint32_t reserved_65; + uint32_t reserved_66; + uint32_t reserved_67; + uint32_t reserved_68; + uint32_t reserved_69; + uint32_t reserved_70; + uint32_t reserved_71; + uint32_t reserved_72; + uint32_t reserved_73; + uint32_t reserved_74; + uint32_t reserved_75; + uint32_t reserved_76; + uint32_t reserved_77; + uint32_t reserved_78; + uint32_t reserved_79; + uint32_t reserved_80; + uint32_t reserved_81; + uint32_t reserved_82; + uint32_t reserved_83; + uint32_t reserved_84; + uint32_t reserved_85; + uint32_t reserved_86; + uint32_t reserved_87; + uint32_t reserved_88; + uint32_t reserved_89; + uint32_t reserved_90; + uint32_t reserved_91; + uint32_t reserved_92; + uint32_t reserved_93; + uint32_t reserved_94; + uint32_t reserved_95; + uint32_t reserved_96; + uint32_t reserved_97; + uint32_t reserved_98; + uint32_t reserved_99; + uint32_t reserved_100; + uint32_t reserved_101; + uint32_t reserved_102; + uint32_t reserved_103; + uint32_t reserved_104; + uint32_t reserved_105; + uint32_t reserved_106; + uint32_t reserved_107; + uint32_t reserved_108; + uint32_t reserved_109; + uint32_t reserved_110; + uint32_t reserved_111; + uint32_t reserved_112; + uint32_t reserved_113; + uint32_t reserved_114; + uint32_t reserved_115; + uint32_t reserved_116; + uint32_t reserved_117; + uint32_t reserved_118; + uint32_t reserved_119; + uint32_t reserved_120; + uint32_t reserved_121; + uint32_t reserved_122; + uint32_t reserved_123; + uint32_t reserved_124; + uint32_t reserved_125; + uint32_t reserved_126; + uint32_t reserved_127; +}; + +struct vi_mqd { + uint32_t header; + uint32_t compute_dispatch_initiator; + uint32_t compute_dim_x; + uint32_t compute_dim_y; + uint32_t compute_dim_z; + uint32_t compute_start_x; + uint32_t compute_start_y; + uint32_t compute_start_z; + uint32_t compute_num_thread_x; + uint32_t compute_num_thread_y; + uint32_t compute_num_thread_z; + uint32_t compute_pipelinestat_enable; + uint32_t compute_perfcount_enable; + uint32_t compute_pgm_lo; + uint32_t compute_pgm_hi; + uint32_t compute_tba_lo; + uint32_t compute_tba_hi; + uint32_t compute_tma_lo; + uint32_t compute_tma_hi; + uint32_t compute_pgm_rsrc1; + uint32_t compute_pgm_rsrc2; + uint32_t compute_vmid; + uint32_t compute_resource_limits; + uint32_t compute_static_thread_mgmt_se0; + uint32_t compute_static_thread_mgmt_se1; + uint32_t compute_tmpring_size; + uint32_t compute_static_thread_mgmt_se2; + uint32_t compute_static_thread_mgmt_se3; + uint32_t compute_restart_x; + uint32_t compute_restart_y; + uint32_t compute_restart_z; + uint32_t compute_thread_trace_enable; + uint32_t compute_misc_reserved; + uint32_t compute_dispatch_id; + uint32_t compute_threadgroup_id; + uint32_t compute_relaunch; + uint32_t compute_wave_restore_addr_lo; + uint32_t compute_wave_restore_addr_hi; + uint32_t compute_wave_restore_control; + uint32_t reserved_39; + uint32_t reserved_40; + uint32_t reserved_41; + uint32_t reserved_42; + uint32_t reserved_43; + uint32_t reserved_44; + uint32_t reserved_45; + uint32_t reserved_46; + uint32_t reserved_47; + uint32_t reserved_48; + uint32_t reserved_49; + uint32_t reserved_50; + uint32_t reserved_51; + uint32_t reserved_52; + uint32_t reserved_53; + uint32_t reserved_54; + uint32_t reserved_55; + uint32_t reserved_56; + uint32_t reserved_57; + uint32_t reserved_58; + uint32_t reserved_59; + uint32_t reserved_60; + uint32_t reserved_61; + uint32_t reserved_62; + uint32_t reserved_63; + uint32_t reserved_64; + uint32_t compute_user_data_0; + uint32_t compute_user_data_1; + uint32_t compute_user_data_2; + uint32_t compute_user_data_3; + uint32_t compute_user_data_4; + uint32_t compute_user_data_5; + uint32_t compute_user_data_6; + uint32_t compute_user_data_7; + uint32_t compute_user_data_8; + uint32_t compute_user_data_9; + uint32_t compute_user_data_10; + uint32_t compute_user_data_11; + uint32_t compute_user_data_12; + uint32_t compute_user_data_13; + uint32_t compute_user_data_14; + uint32_t compute_user_data_15; + uint32_t cp_compute_csinvoc_count_lo; + uint32_t cp_compute_csinvoc_count_hi; + uint32_t reserved_83; + uint32_t reserved_84; + uint32_t reserved_85; + uint32_t cp_mqd_query_time_lo; + uint32_t cp_mqd_query_time_hi; + uint32_t cp_mqd_connect_start_time_lo; + uint32_t cp_mqd_connect_start_time_hi; + uint32_t cp_mqd_connect_end_time_lo; + uint32_t cp_mqd_connect_end_time_hi; + uint32_t cp_mqd_connect_end_wf_count; + uint32_t cp_mqd_connect_end_pq_rptr; + uint32_t cp_mqd_connect_end_pq_wptr; + uint32_t cp_mqd_connect_end_ib_rptr; + uint32_t reserved_96; + uint32_t reserved_97; + uint32_t cp_mqd_save_start_time_lo; + uint32_t cp_mqd_save_start_time_hi; + uint32_t cp_mqd_save_end_time_lo; + uint32_t cp_mqd_save_end_time_hi; + uint32_t cp_mqd_restore_start_time_lo; + uint32_t cp_mqd_restore_start_time_hi; + uint32_t cp_mqd_restore_end_time_lo; + uint32_t cp_mqd_restore_end_time_hi; + uint32_t reserved_106; + uint32_t reserved_107; + uint32_t gds_cs_ctxsw_cnt0; + uint32_t gds_cs_ctxsw_cnt1; + uint32_t gds_cs_ctxsw_cnt2; + uint32_t gds_cs_ctxsw_cnt3; + uint32_t reserved_112; + uint32_t reserved_113; + uint32_t cp_pq_exe_status_lo; + uint32_t cp_pq_exe_status_hi; + uint32_t cp_packet_id_lo; + uint32_t cp_packet_id_hi; + uint32_t cp_packet_exe_status_lo; + uint32_t cp_packet_exe_status_hi; + uint32_t gds_save_base_addr_lo; + uint32_t gds_save_base_addr_hi; + uint32_t gds_save_mask_lo; + uint32_t gds_save_mask_hi; + uint32_t ctx_save_base_addr_lo; + uint32_t ctx_save_base_addr_hi; + uint32_t reserved_126; + uint32_t reserved_127; + uint32_t cp_mqd_base_addr_lo; + uint32_t cp_mqd_base_addr_hi; + uint32_t cp_hqd_active; + uint32_t cp_hqd_vmid; + uint32_t cp_hqd_persistent_state; + uint32_t cp_hqd_pipe_priority; + uint32_t cp_hqd_queue_priority; + uint32_t cp_hqd_quantum; + uint32_t cp_hqd_pq_base_lo; + uint32_t cp_hqd_pq_base_hi; + uint32_t cp_hqd_pq_rptr; + uint32_t cp_hqd_pq_rptr_report_addr_lo; + uint32_t cp_hqd_pq_rptr_report_addr_hi; + uint32_t cp_hqd_pq_wptr_poll_addr_lo; + uint32_t cp_hqd_pq_wptr_poll_addr_hi; + uint32_t cp_hqd_pq_doorbell_control; + uint32_t cp_hqd_pq_wptr; + uint32_t cp_hqd_pq_control; + uint32_t cp_hqd_ib_base_addr_lo; + uint32_t cp_hqd_ib_base_addr_hi; + uint32_t cp_hqd_ib_rptr; + uint32_t cp_hqd_ib_control; + uint32_t cp_hqd_iq_timer; + uint32_t cp_hqd_iq_rptr; + uint32_t cp_hqd_dequeue_request; + uint32_t cp_hqd_dma_offload; + uint32_t cp_hqd_sema_cmd; + uint32_t cp_hqd_msg_type; + uint32_t cp_hqd_atomic0_preop_lo; + uint32_t cp_hqd_atomic0_preop_hi; + uint32_t cp_hqd_atomic1_preop_lo; + uint32_t cp_hqd_atomic1_preop_hi; + uint32_t cp_hqd_hq_status0; + uint32_t cp_hqd_hq_control0; + uint32_t cp_mqd_control; + uint32_t cp_hqd_hq_status1; + uint32_t cp_hqd_hq_control1; + uint32_t cp_hqd_eop_base_addr_lo; + uint32_t cp_hqd_eop_base_addr_hi; + uint32_t cp_hqd_eop_control; + uint32_t cp_hqd_eop_rptr; + uint32_t cp_hqd_eop_wptr; + uint32_t cp_hqd_eop_done_events; + uint32_t cp_hqd_ctx_save_base_addr_lo; + uint32_t cp_hqd_ctx_save_base_addr_hi; + uint32_t cp_hqd_ctx_save_control; + uint32_t cp_hqd_cntl_stack_offset; + uint32_t cp_hqd_cntl_stack_size; + uint32_t cp_hqd_wg_state_offset; + uint32_t cp_hqd_ctx_save_size; + uint32_t cp_hqd_gds_resource_state; + uint32_t cp_hqd_error; + uint32_t cp_hqd_eop_wptr_mem; + uint32_t cp_hqd_eop_dones; + uint32_t reserved_182; + uint32_t reserved_183; + uint32_t reserved_184; + uint32_t reserved_185; + uint32_t reserved_186; + uint32_t reserved_187; + uint32_t reserved_188; + uint32_t reserved_189; + uint32_t reserved_190; + uint32_t reserved_191; + uint32_t iqtimer_pkt_header; + uint32_t iqtimer_pkt_dw0; + uint32_t iqtimer_pkt_dw1; + uint32_t iqtimer_pkt_dw2; + uint32_t iqtimer_pkt_dw3; + uint32_t iqtimer_pkt_dw4; + uint32_t iqtimer_pkt_dw5; + uint32_t iqtimer_pkt_dw6; + uint32_t iqtimer_pkt_dw7; + uint32_t iqtimer_pkt_dw8; + uint32_t iqtimer_pkt_dw9; + uint32_t iqtimer_pkt_dw10; + uint32_t iqtimer_pkt_dw11; + uint32_t iqtimer_pkt_dw12; + uint32_t iqtimer_pkt_dw13; + uint32_t iqtimer_pkt_dw14; + uint32_t iqtimer_pkt_dw15; + uint32_t iqtimer_pkt_dw16; + uint32_t iqtimer_pkt_dw17; + uint32_t iqtimer_pkt_dw18; + uint32_t iqtimer_pkt_dw19; + uint32_t iqtimer_pkt_dw20; + uint32_t iqtimer_pkt_dw21; + uint32_t iqtimer_pkt_dw22; + uint32_t iqtimer_pkt_dw23; + uint32_t iqtimer_pkt_dw24; + uint32_t iqtimer_pkt_dw25; + uint32_t iqtimer_pkt_dw26; + uint32_t iqtimer_pkt_dw27; + uint32_t iqtimer_pkt_dw28; + uint32_t iqtimer_pkt_dw29; + uint32_t iqtimer_pkt_dw30; + uint32_t iqtimer_pkt_dw31; + uint32_t reserved_225; + uint32_t reserved_226; + uint32_t reserved_227; + uint32_t set_resources_header; + uint32_t set_resources_dw1; + uint32_t set_resources_dw2; + uint32_t set_resources_dw3; + uint32_t set_resources_dw4; + uint32_t set_resources_dw5; + uint32_t set_resources_dw6; + uint32_t set_resources_dw7; + uint32_t reserved_236; + uint32_t reserved_237; + uint32_t reserved_238; + uint32_t reserved_239; + uint32_t queue_doorbell_id0; + uint32_t queue_doorbell_id1; + uint32_t queue_doorbell_id2; + uint32_t queue_doorbell_id3; + uint32_t queue_doorbell_id4; + uint32_t queue_doorbell_id5; + uint32_t queue_doorbell_id6; + uint32_t queue_doorbell_id7; + uint32_t queue_doorbell_id8; + uint32_t queue_doorbell_id9; + uint32_t queue_doorbell_id10; + uint32_t queue_doorbell_id11; + uint32_t queue_doorbell_id12; + uint32_t queue_doorbell_id13; + uint32_t queue_doorbell_id14; + uint32_t queue_doorbell_id15; +}; + +#endif /* VI_STRUCTS_H_ */ diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c new file mode 100644 index 000000000000..9259f1b6664c --- /dev/null +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c @@ -0,0 +1,424 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#include +#include +#include +#include +#include "gpu_scheduler.h" + +static struct amd_sched_job * +amd_sched_entity_pop_job(struct amd_sched_entity *entity); +static void amd_sched_wakeup(struct amd_gpu_scheduler *sched); + +/* Initialize a given run queue struct */ +static void amd_sched_rq_init(struct amd_sched_rq *rq) +{ + spin_lock_init(&rq->lock); + INIT_LIST_HEAD(&rq->entities); + rq->current_entity = NULL; +} + +static void amd_sched_rq_add_entity(struct amd_sched_rq *rq, + struct amd_sched_entity *entity) +{ + spin_lock(&rq->lock); + list_add_tail(&entity->list, &rq->entities); + spin_unlock(&rq->lock); +} + +static void amd_sched_rq_remove_entity(struct amd_sched_rq *rq, + struct amd_sched_entity *entity) +{ + spin_lock(&rq->lock); + list_del_init(&entity->list); + if (rq->current_entity == entity) + rq->current_entity = NULL; + spin_unlock(&rq->lock); +} + +/** + * Select next job from a specified run queue with round robin policy. + * Return NULL if nothing available. + */ +static struct amd_sched_job * +amd_sched_rq_select_job(struct amd_sched_rq *rq) +{ + struct amd_sched_entity *entity; + struct amd_sched_job *job; + + spin_lock(&rq->lock); + + entity = rq->current_entity; + if (entity) { + list_for_each_entry_continue(entity, &rq->entities, list) { + job = amd_sched_entity_pop_job(entity); + if (job) { + rq->current_entity = entity; + spin_unlock(&rq->lock); + return job; + } + } + } + + list_for_each_entry(entity, &rq->entities, list) { + + job = amd_sched_entity_pop_job(entity); + if (job) { + rq->current_entity = entity; + spin_unlock(&rq->lock); + return job; + } + + if (entity == rq->current_entity) + break; + } + + spin_unlock(&rq->lock); + + return NULL; +} + +/** + * Init a context entity used by scheduler when submit to HW ring. + * + * @sched The pointer to the scheduler + * @entity The pointer to a valid amd_sched_entity + * @rq The run queue this entity belongs + * @kernel If this is an entity for the kernel + * @jobs The max number of jobs in the job queue + * + * return 0 if succeed. negative error code on failure +*/ +int amd_sched_entity_init(struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity, + struct amd_sched_rq *rq, + uint32_t jobs) +{ + if (!(sched && entity && rq)) + return -EINVAL; + + memset(entity, 0, sizeof(struct amd_sched_entity)); + entity->belongto_rq = rq; + entity->scheduler = sched; + entity->fence_context = fence_context_alloc(1); + if(kfifo_alloc(&entity->job_queue, + jobs * sizeof(void *), + GFP_KERNEL)) + return -EINVAL; + + spin_lock_init(&entity->queue_lock); + atomic_set(&entity->fence_seq, 0); + + /* Add the entity to the run queue */ + amd_sched_rq_add_entity(rq, entity); + return 0; +} + +/** + * Query if entity is initialized + * + * @sched Pointer to scheduler instance + * @entity The pointer to a valid scheduler entity + * + * return true if entity is initialized, false otherwise +*/ +static bool amd_sched_entity_is_initialized(struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity) +{ + return entity->scheduler == sched && + entity->belongto_rq != NULL; +} + +/** + * Check if entity is idle + * + * @entity The pointer to a valid scheduler entity + * + * Return true if entity don't has any unscheduled jobs. + */ +static bool amd_sched_entity_is_idle(struct amd_sched_entity *entity) +{ + rmb(); + if (kfifo_is_empty(&entity->job_queue)) + return true; + + return false; +} + +/** + * Destroy a context entity + * + * @sched Pointer to scheduler instance + * @entity The pointer to a valid scheduler entity + * + * Cleanup and free the allocated resources. + */ +void amd_sched_entity_fini(struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity) +{ + struct amd_sched_rq *rq = entity->belongto_rq; + + if (!amd_sched_entity_is_initialized(sched, entity)) + return; + + /** + * The client will not queue more IBs during this fini, consume existing + * queued IBs + */ + wait_event(sched->job_scheduled, amd_sched_entity_is_idle(entity)); + + amd_sched_rq_remove_entity(rq, entity); + kfifo_free(&entity->job_queue); +} + +static void amd_sched_entity_wakeup(struct fence *f, struct fence_cb *cb) +{ + struct amd_sched_entity *entity = + container_of(cb, struct amd_sched_entity, cb); + entity->dependency = NULL; + fence_put(f); + amd_sched_wakeup(entity->scheduler); +} + +static struct amd_sched_job * +amd_sched_entity_pop_job(struct amd_sched_entity *entity) +{ + struct amd_gpu_scheduler *sched = entity->scheduler; + struct amd_sched_job *job; + + if (ACCESS_ONCE(entity->dependency)) + return NULL; + + if (!kfifo_out_peek(&entity->job_queue, &job, sizeof(job))) + return NULL; + + while ((entity->dependency = sched->ops->dependency(job))) { + + if (fence_add_callback(entity->dependency, &entity->cb, + amd_sched_entity_wakeup)) + fence_put(entity->dependency); + else + return NULL; + } + + return job; +} + +/** + * Helper to submit a job to the job queue + * + * @job The pointer to job required to submit + * + * Returns true if we could submit the job. + */ +static bool amd_sched_entity_in(struct amd_sched_job *job) +{ + struct amd_sched_entity *entity = job->s_entity; + bool added, first = false; + + spin_lock(&entity->queue_lock); + added = kfifo_in(&entity->job_queue, &job, sizeof(job)) == sizeof(job); + + if (added && kfifo_len(&entity->job_queue) == sizeof(job)) + first = true; + + spin_unlock(&entity->queue_lock); + + /* first job wakes up scheduler */ + if (first) + amd_sched_wakeup(job->sched); + + return added; +} + +/** + * Submit a job to the job queue + * + * @job The pointer to job required to submit + * + * Returns 0 for success, negative error code otherwise. + */ +int amd_sched_entity_push_job(struct amd_sched_job *sched_job) +{ + struct amd_sched_entity *entity = sched_job->s_entity; + struct amd_sched_fence *fence = amd_sched_fence_create( + entity, sched_job->owner); + + if (!fence) + return -ENOMEM; + + fence_get(&fence->base); + sched_job->s_fence = fence; + + wait_event(entity->scheduler->job_scheduled, + amd_sched_entity_in(sched_job)); + + return 0; +} + +/** + * Return ture if we can push more jobs to the hw. + */ +static bool amd_sched_ready(struct amd_gpu_scheduler *sched) +{ + return atomic_read(&sched->hw_rq_count) < + sched->hw_submission_limit; +} + +/** + * Wake up the scheduler when it is ready + */ +static void amd_sched_wakeup(struct amd_gpu_scheduler *sched) +{ + if (amd_sched_ready(sched)) + wake_up_interruptible(&sched->wake_up_worker); +} + +/** + * Select next to run +*/ +static struct amd_sched_job * +amd_sched_select_job(struct amd_gpu_scheduler *sched) +{ + struct amd_sched_job *job; + + if (!amd_sched_ready(sched)) + return NULL; + + /* Kernel run queue has higher priority than normal run queue*/ + job = amd_sched_rq_select_job(&sched->kernel_rq); + if (job == NULL) + job = amd_sched_rq_select_job(&sched->sched_rq); + + return job; +} + +static void amd_sched_process_job(struct fence *f, struct fence_cb *cb) +{ + struct amd_sched_job *sched_job = + container_of(cb, struct amd_sched_job, cb); + struct amd_gpu_scheduler *sched; + + sched = sched_job->sched; + amd_sched_fence_signal(sched_job->s_fence); + atomic_dec(&sched->hw_rq_count); + fence_put(&sched_job->s_fence->base); + sched->ops->process_job(sched_job); + wake_up_interruptible(&sched->wake_up_worker); +} + +static int amd_sched_main(void *param) +{ + struct sched_param sparam = {.sched_priority = 1}; + struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param; + int r, count; + + sched_setscheduler(current, SCHED_FIFO, &sparam); + + while (!kthread_should_stop()) { + struct amd_sched_entity *entity; + struct amd_sched_job *job; + struct fence *fence; + + wait_event_interruptible(sched->wake_up_worker, + kthread_should_stop() || + (job = amd_sched_select_job(sched))); + + if (!job) + continue; + + entity = job->s_entity; + atomic_inc(&sched->hw_rq_count); + fence = sched->ops->run_job(job); + if (fence) { + r = fence_add_callback(fence, &job->cb, + amd_sched_process_job); + if (r == -ENOENT) + amd_sched_process_job(fence, &job->cb); + else if (r) + DRM_ERROR("fence add callback failed (%d)\n", r); + fence_put(fence); + } + + count = kfifo_out(&entity->job_queue, &job, sizeof(job)); + WARN_ON(count != sizeof(job)); + wake_up(&sched->job_scheduled); + } + return 0; +} + +/** + * Create a gpu scheduler + * + * @ops The backend operations for this scheduler. + * @ring The the ring id for the scheduler. + * @hw_submissions Number of hw submissions to do. + * + * Return the pointer to scheduler for success, otherwise return NULL +*/ +struct amd_gpu_scheduler *amd_sched_create(struct amd_sched_backend_ops *ops, + unsigned ring, unsigned hw_submission, + void *priv) +{ + struct amd_gpu_scheduler *sched; + + sched = kzalloc(sizeof(struct amd_gpu_scheduler), GFP_KERNEL); + if (!sched) + return NULL; + + sched->ops = ops; + sched->ring_id = ring; + sched->hw_submission_limit = hw_submission; + sched->priv = priv; + snprintf(sched->name, sizeof(sched->name), "amdgpu[%d]", ring); + amd_sched_rq_init(&sched->sched_rq); + amd_sched_rq_init(&sched->kernel_rq); + + init_waitqueue_head(&sched->wake_up_worker); + init_waitqueue_head(&sched->job_scheduled); + atomic_set(&sched->hw_rq_count, 0); + /* Each scheduler will run on a seperate kernel thread */ + sched->thread = kthread_run(amd_sched_main, sched, sched->name); + if (IS_ERR(sched->thread)) { + DRM_ERROR("Failed to create scheduler for id %d.\n", ring); + kfree(sched); + return NULL; + } + + return sched; +} + +/** + * Destroy a gpu scheduler + * + * @sched The pointer to the scheduler + * + * return 0 if succeed. -1 if failed. + */ +int amd_sched_destroy(struct amd_gpu_scheduler *sched) +{ + kthread_stop(sched->thread); + kfree(sched); + return 0; +} diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h new file mode 100644 index 000000000000..2af0e4d4d817 --- /dev/null +++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h @@ -0,0 +1,134 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef _GPU_SCHEDULER_H_ +#define _GPU_SCHEDULER_H_ + +#include +#include + +struct amd_gpu_scheduler; +struct amd_sched_rq; + +/** + * A scheduler entity is a wrapper around a job queue or a group + * of other entities. Entities take turns emitting jobs from their + * job queues to corresponding hardware ring based on scheduling + * policy. +*/ +struct amd_sched_entity { + struct list_head list; + struct amd_sched_rq *belongto_rq; + atomic_t fence_seq; + /* the job_queue maintains the jobs submitted by clients */ + struct kfifo job_queue; + spinlock_t queue_lock; + struct amd_gpu_scheduler *scheduler; + uint64_t fence_context; + struct fence *dependency; + struct fence_cb cb; +}; + +/** + * Run queue is a set of entities scheduling command submissions for + * one specific ring. It implements the scheduling policy that selects + * the next entity to emit commands from. +*/ +struct amd_sched_rq { + spinlock_t lock; + struct list_head entities; + struct amd_sched_entity *current_entity; +}; + +struct amd_sched_fence { + struct fence base; + struct amd_gpu_scheduler *scheduler; + spinlock_t lock; + void *owner; +}; + +struct amd_sched_job { + struct fence_cb cb; + struct amd_gpu_scheduler *sched; + struct amd_sched_entity *s_entity; + struct amd_sched_fence *s_fence; + void *owner; +}; + +extern const struct fence_ops amd_sched_fence_ops; +static inline struct amd_sched_fence *to_amd_sched_fence(struct fence *f) +{ + struct amd_sched_fence *__f = container_of(f, struct amd_sched_fence, base); + + if (__f->base.ops == &amd_sched_fence_ops) + return __f; + + return NULL; +} + +/** + * Define the backend operations called by the scheduler, + * these functions should be implemented in driver side +*/ +struct amd_sched_backend_ops { + struct fence *(*dependency)(struct amd_sched_job *job); + struct fence *(*run_job)(struct amd_sched_job *job); + void (*process_job)(struct amd_sched_job *job); +}; + +/** + * One scheduler is implemented for each hardware ring +*/ +struct amd_gpu_scheduler { + struct task_struct *thread; + struct amd_sched_rq sched_rq; + struct amd_sched_rq kernel_rq; + atomic_t hw_rq_count; + struct amd_sched_backend_ops *ops; + uint32_t ring_id; + wait_queue_head_t wake_up_worker; + wait_queue_head_t job_scheduled; + uint32_t hw_submission_limit; + char name[20]; + void *priv; +}; + +struct amd_gpu_scheduler * +amd_sched_create(struct amd_sched_backend_ops *ops, + uint32_t ring, uint32_t hw_submission, void *priv); +int amd_sched_destroy(struct amd_gpu_scheduler *sched); + +int amd_sched_entity_init(struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity, + struct amd_sched_rq *rq, + uint32_t jobs); +void amd_sched_entity_fini(struct amd_gpu_scheduler *sched, + struct amd_sched_entity *entity); +int amd_sched_entity_push_job(struct amd_sched_job *sched_job); + +struct amd_sched_fence *amd_sched_fence_create( + struct amd_sched_entity *s_entity, void *owner); +void amd_sched_fence_signal(struct amd_sched_fence *fence); + + +#endif diff --git a/drivers/gpu/drm/amd/scheduler/sched_fence.c b/drivers/gpu/drm/amd/scheduler/sched_fence.c new file mode 100644 index 000000000000..e62c37920e11 --- /dev/null +++ b/drivers/gpu/drm/amd/scheduler/sched_fence.c @@ -0,0 +1,81 @@ +/* + * Copyright 2015 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ +#include +#include +#include +#include +#include "gpu_scheduler.h" + +struct amd_sched_fence *amd_sched_fence_create(struct amd_sched_entity *s_entity, void *owner) +{ + struct amd_sched_fence *fence = NULL; + unsigned seq; + + fence = kzalloc(sizeof(struct amd_sched_fence), GFP_KERNEL); + if (fence == NULL) + return NULL; + fence->owner = owner; + fence->scheduler = s_entity->scheduler; + spin_lock_init(&fence->lock); + + seq = atomic_inc_return(&s_entity->fence_seq); + fence_init(&fence->base, &amd_sched_fence_ops, &fence->lock, + s_entity->fence_context, seq); + + return fence; +} + +void amd_sched_fence_signal(struct amd_sched_fence *fence) +{ + int ret = fence_signal(&fence->base); + if (!ret) + FENCE_TRACE(&fence->base, "signaled from irq context\n"); + else + FENCE_TRACE(&fence->base, "was already signaled\n"); +} + +static const char *amd_sched_fence_get_driver_name(struct fence *fence) +{ + return "amd_sched"; +} + +static const char *amd_sched_fence_get_timeline_name(struct fence *f) +{ + struct amd_sched_fence *fence = to_amd_sched_fence(f); + return (const char *)fence->scheduler->name; +} + +static bool amd_sched_fence_enable_signaling(struct fence *f) +{ + return true; +} + +const struct fence_ops amd_sched_fence_ops = { + .get_driver_name = amd_sched_fence_get_driver_name, + .get_timeline_name = amd_sched_fence_get_timeline_name, + .enable_signaling = amd_sched_fence_enable_signaling, + .signaled = NULL, + .wait = fence_default_wait, + .release = NULL, +}; diff --git a/drivers/gpu/drm/armada/armada_fbdev.c b/drivers/gpu/drm/armada/armada_fbdev.c index 7838e731b0de..7d03c51abcb9 100644 --- a/drivers/gpu/drm/armada/armada_fbdev.c +++ b/drivers/gpu/drm/armada/armada_fbdev.c @@ -22,9 +22,9 @@ static /*const*/ struct fb_ops armada_fb_ops = { .owner = THIS_MODULE, .fb_check_var = drm_fb_helper_check_var, .fb_set_par = drm_fb_helper_set_par, - .fb_fillrect = cfb_fillrect, - .fb_copyarea = cfb_copyarea, - .fb_imageblit = cfb_imageblit, + .fb_fillrect = drm_fb_helper_cfb_fillrect, + .fb_copyarea = drm_fb_helper_cfb_copyarea, + .fb_imageblit = drm_fb_helper_cfb_imageblit, .fb_pan_display = drm_fb_helper_pan_display, .fb_blank = drm_fb_helper_blank, .fb_setcmap = drm_fb_helper_setcmap, @@ -80,18 +80,12 @@ static int armada_fb_create(struct drm_fb_helper *fbh, if (IS_ERR(dfb)) return PTR_ERR(dfb); - info = framebuffer_alloc(0, dev->dev); - if (!info) { - ret = -ENOMEM; + info = drm_fb_helper_alloc_fbi(fbh); + if (IS_ERR(info)) { + ret = PTR_ERR(info); goto err_fballoc; } - ret = fb_alloc_cmap(&info->cmap, 256, 0); - if (ret) { - ret = -ENOMEM; - goto err_fbcmap; - } - strlcpy(info->fix.id, "armada-drmfb", sizeof(info->fix.id)); info->par = fbh; info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT; @@ -101,7 +95,7 @@ static int armada_fb_create(struct drm_fb_helper *fbh, info->screen_size = obj->obj.size; info->screen_base = ptr; fbh->fb = &dfb->fb; - fbh->fbdev = info; + drm_fb_helper_fill_fix(info, dfb->fb.pitches[0], dfb->fb.depth); drm_fb_helper_fill_var(info, fbh, sizes->fb_width, sizes->fb_height); @@ -111,8 +105,6 @@ static int armada_fb_create(struct drm_fb_helper *fbh, return 0; - err_fbcmap: - framebuffer_release(info); err_fballoc: dfb->fb.funcs->destroy(&dfb->fb); return ret; @@ -171,6 +163,7 @@ int armada_fbdev_init(struct drm_device *dev) return 0; err_fb_setup: + drm_fb_helper_release_fbi(fbh); drm_fb_helper_fini(fbh); err_fb_helper: priv->fbdev = NULL; @@ -191,14 +184,8 @@ void armada_fbdev_fini(struct drm_device *dev) struct drm_fb_helper *fbh = priv->fbdev; if (fbh) { - struct fb_info *info = fbh->fbdev; - - if (info) { - unregister_framebuffer(info); - if (info->cmap.len) - fb_dealloc_cmap(&info->cmap); - framebuffer_release(info); - } + drm_fb_helper_unregister_fbi(fbh); + drm_fb_helper_release_fbi(fbh); drm_fb_helper_fini(fbh); diff --git a/drivers/gpu/drm/ast/ast_fb.c b/drivers/gpu/drm/ast/ast_fb.c index ff68eefae273..f31db28a684b 100644 --- a/drivers/gpu/drm/ast/ast_fb.c +++ b/drivers/gpu/drm/ast/ast_fb.c @@ -125,7 +125,7 @@ static void ast_fillrect(struct fb_info *info, const struct fb_fillrect *rect) { struct ast_fbdev *afbdev = info->par; - sys_fillrect(info, rect); + drm_fb_helper_sys_fillrect(info, rect); ast_dirty_update(afbdev, rect->dx, rect->dy, rect->width, rect->height); } @@ -134,7 +134,7 @@ static void ast_copyarea(struct fb_info *info, const struct fb_copyarea *area) { struct ast_fbdev *afbdev = info->par; - sys_copyarea(info, area); + drm_fb_helper_sys_copyarea(info, area); ast_dirty_update(afbdev, area->dx, area->dy, area->width, area->height); } @@ -143,7 +143,7 @@ static void ast_imageblit(struct fb_info *info, const struct fb_image *image) { struct ast_fbdev *afbdev = info->par; - sys_imageblit(info, image); + drm_fb_helper_sys_imageblit(info, image); ast_dirty_update(afbdev, image->dx, image->dy, image->width, image->height); } @@ -193,7 +193,6 @@ static int astfb_create(struct drm_fb_helper *helper, struct drm_framebuffer *fb; struct fb_info *info; int size, ret; - struct device *device = &dev->pdev->dev; void *sysram; struct drm_gem_object *gobj = NULL; struct ast_bo *bo = NULL; @@ -217,40 +216,28 @@ static int astfb_create(struct drm_fb_helper *helper, if (!sysram) return -ENOMEM; - info = framebuffer_alloc(0, device); - if (!info) { - ret = -ENOMEM; - goto out; + info = drm_fb_helper_alloc_fbi(helper); + if (IS_ERR(info)) { + ret = PTR_ERR(info); + goto err_free_vram; } info->par = afbdev; ret = ast_framebuffer_init(dev, &afbdev->afb, &mode_cmd, gobj); if (ret) - goto out; + goto err_release_fbi; afbdev->sysram = sysram; afbdev->size = size; fb = &afbdev->afb.base; afbdev->helper.fb = fb; - afbdev->helper.fbdev = info; strcpy(info->fix.id, "astdrmfb"); info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT; info->fbops = &astfb_ops; - ret = fb_alloc_cmap(&info->cmap, 256, 0); - if (ret) { - ret = -ENOMEM; - goto out; - } - - info->apertures = alloc_apertures(1); - if (!info->apertures) { - ret = -ENOMEM; - goto out; - } info->apertures->ranges[0].base = pci_resource_start(dev->pdev, 0); info->apertures->ranges[0].size = pci_resource_len(dev->pdev, 0); @@ -266,7 +253,11 @@ static int astfb_create(struct drm_fb_helper *helper, fb->width, fb->height); return 0; -out: + +err_release_fbi: + drm_fb_helper_release_fbi(helper); +err_free_vram: + vfree(afbdev->sysram); return ret; } @@ -297,15 +288,10 @@ static const struct drm_fb_helper_funcs ast_fb_helper_funcs = { static void ast_fbdev_destroy(struct drm_device *dev, struct ast_fbdev *afbdev) { - struct fb_info *info; struct ast_framebuffer *afb = &afbdev->afb; - if (afbdev->helper.fbdev) { - info = afbdev->helper.fbdev; - unregister_framebuffer(info); - if (info->cmap.len) - fb_dealloc_cmap(&info->cmap); - framebuffer_release(info); - } + + drm_fb_helper_unregister_fbi(&afbdev->helper); + drm_fb_helper_release_fbi(&afbdev->helper); if (afb->obj) { drm_gem_object_unreference_unlocked(afb->obj); @@ -377,5 +363,5 @@ void ast_fbdev_set_suspend(struct drm_device *dev, int state) if (!ast->fbdev) return; - fb_set_suspend(ast->fbdev->helper.fbdev, state); + drm_fb_helper_set_suspend(&ast->fbdev->helper, state); } diff --git a/drivers/gpu/drm/ast/ast_main.c b/drivers/gpu/drm/ast/ast_main.c index 035dacc93382..838217f8ce7d 100644 --- a/drivers/gpu/drm/ast/ast_main.c +++ b/drivers/gpu/drm/ast/ast_main.c @@ -571,24 +571,18 @@ ast_dumb_mmap_offset(struct drm_file *file, uint64_t *offset) { struct drm_gem_object *obj; - int ret; struct ast_bo *bo; - mutex_lock(&dev->struct_mutex); obj = drm_gem_object_lookup(dev, file, handle); - if (obj == NULL) { - ret = -ENOENT; - goto out_unlock; - } + if (obj == NULL) + return -ENOENT; bo = gem_to_ast_bo(obj); *offset = ast_bo_mmap_offset(bo); - drm_gem_object_unreference(obj); - ret = 0; -out_unlock: - mutex_unlock(&dev->struct_mutex); - return ret; + drm_gem_object_unreference_unlocked(obj); + + return 0; } diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c index 5ae5c6923128..9f6e234e7029 100644 --- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c +++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c @@ -239,7 +239,8 @@ static int atmel_hlcdc_crtc_atomic_check(struct drm_crtc *c, return atmel_hlcdc_plane_prepare_disc_area(s); } -static void atmel_hlcdc_crtc_atomic_begin(struct drm_crtc *c) +static void atmel_hlcdc_crtc_atomic_begin(struct drm_crtc *c, + struct drm_crtc_state *old_s) { struct atmel_hlcdc_crtc *crtc = drm_crtc_to_atmel_hlcdc_crtc(c); @@ -253,7 +254,8 @@ static void atmel_hlcdc_crtc_atomic_begin(struct drm_crtc *c) } } -static void atmel_hlcdc_crtc_atomic_flush(struct drm_crtc *crtc) +static void atmel_hlcdc_crtc_atomic_flush(struct drm_crtc *crtc, + struct drm_crtc_state *old_s) { /* TODO: write common plane control register if available */ } diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c index ef6182bc8e5e..8bc62ec407f9 100644 --- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c +++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c @@ -29,6 +29,115 @@ #define ATMEL_HLCDC_LAYER_IRQS_OFFSET 8 +static const struct atmel_hlcdc_layer_desc atmel_hlcdc_at91sam9n12_layers[] = { + { + .name = "base", + .formats = &atmel_hlcdc_plane_rgb_formats, + .regs_offset = 0x40, + .id = 0, + .type = ATMEL_HLCDC_BASE_LAYER, + .nconfigs = 5, + .layout = { + .xstride = { 2 }, + .default_color = 3, + .general_config = 4, + }, + }, +}; + +static const struct atmel_hlcdc_dc_desc atmel_hlcdc_dc_at91sam9n12 = { + .min_width = 0, + .min_height = 0, + .max_width = 1280, + .max_height = 860, + .nlayers = ARRAY_SIZE(atmel_hlcdc_at91sam9n12_layers), + .layers = atmel_hlcdc_at91sam9n12_layers, +}; + +static const struct atmel_hlcdc_layer_desc atmel_hlcdc_at91sam9x5_layers[] = { + { + .name = "base", + .formats = &atmel_hlcdc_plane_rgb_formats, + .regs_offset = 0x40, + .id = 0, + .type = ATMEL_HLCDC_BASE_LAYER, + .nconfigs = 5, + .layout = { + .xstride = { 2 }, + .default_color = 3, + .general_config = 4, + .disc_pos = 5, + .disc_size = 6, + }, + }, + { + .name = "overlay1", + .formats = &atmel_hlcdc_plane_rgb_formats, + .regs_offset = 0x100, + .id = 1, + .type = ATMEL_HLCDC_OVERLAY_LAYER, + .nconfigs = 10, + .layout = { + .pos = 2, + .size = 3, + .xstride = { 4 }, + .pstride = { 5 }, + .default_color = 6, + .chroma_key = 7, + .chroma_key_mask = 8, + .general_config = 9, + }, + }, + { + .name = "high-end-overlay", + .formats = &atmel_hlcdc_plane_rgb_and_yuv_formats, + .regs_offset = 0x280, + .id = 2, + .type = ATMEL_HLCDC_OVERLAY_LAYER, + .nconfigs = 17, + .layout = { + .pos = 2, + .size = 3, + .memsize = 4, + .xstride = { 5, 7 }, + .pstride = { 6, 8 }, + .default_color = 9, + .chroma_key = 10, + .chroma_key_mask = 11, + .general_config = 12, + .csc = 14, + }, + }, + { + .name = "cursor", + .formats = &atmel_hlcdc_plane_rgb_formats, + .regs_offset = 0x340, + .id = 3, + .type = ATMEL_HLCDC_CURSOR_LAYER, + .nconfigs = 10, + .max_width = 128, + .max_height = 128, + .layout = { + .pos = 2, + .size = 3, + .xstride = { 4 }, + .default_color = 6, + .chroma_key = 7, + .chroma_key_mask = 8, + .general_config = 9, + }, + }, +}; + +static const struct atmel_hlcdc_dc_desc atmel_hlcdc_dc_at91sam9x5 = { + .min_width = 0, + .min_height = 0, + .max_width = 800, + .max_height = 600, + .nlayers = ARRAY_SIZE(atmel_hlcdc_at91sam9x5_layers), + .layers = atmel_hlcdc_at91sam9x5_layers, +}; + static const struct atmel_hlcdc_layer_desc atmel_hlcdc_sama5d3_layers[] = { { .name = "base", @@ -132,11 +241,105 @@ static const struct atmel_hlcdc_dc_desc atmel_hlcdc_dc_sama5d3 = { .layers = atmel_hlcdc_sama5d3_layers, }; +static const struct atmel_hlcdc_layer_desc atmel_hlcdc_sama5d4_layers[] = { + { + .name = "base", + .formats = &atmel_hlcdc_plane_rgb_formats, + .regs_offset = 0x40, + .id = 0, + .type = ATMEL_HLCDC_BASE_LAYER, + .nconfigs = 7, + .layout = { + .xstride = { 2 }, + .default_color = 3, + .general_config = 4, + .disc_pos = 5, + .disc_size = 6, + }, + }, + { + .name = "overlay1", + .formats = &atmel_hlcdc_plane_rgb_formats, + .regs_offset = 0x140, + .id = 1, + .type = ATMEL_HLCDC_OVERLAY_LAYER, + .nconfigs = 10, + .layout = { + .pos = 2, + .size = 3, + .xstride = { 4 }, + .pstride = { 5 }, + .default_color = 6, + .chroma_key = 7, + .chroma_key_mask = 8, + .general_config = 9, + }, + }, + { + .name = "overlay2", + .formats = &atmel_hlcdc_plane_rgb_formats, + .regs_offset = 0x240, + .id = 2, + .type = ATMEL_HLCDC_OVERLAY_LAYER, + .nconfigs = 10, + .layout = { + .pos = 2, + .size = 3, + .xstride = { 4 }, + .pstride = { 5 }, + .default_color = 6, + .chroma_key = 7, + .chroma_key_mask = 8, + .general_config = 9, + }, + }, + { + .name = "high-end-overlay", + .formats = &atmel_hlcdc_plane_rgb_and_yuv_formats, + .regs_offset = 0x340, + .id = 3, + .type = ATMEL_HLCDC_OVERLAY_LAYER, + .nconfigs = 42, + .layout = { + .pos = 2, + .size = 3, + .memsize = 4, + .xstride = { 5, 7 }, + .pstride = { 6, 8 }, + .default_color = 9, + .chroma_key = 10, + .chroma_key_mask = 11, + .general_config = 12, + .csc = 14, + }, + }, +}; + +static const struct atmel_hlcdc_dc_desc atmel_hlcdc_dc_sama5d4 = { + .min_width = 0, + .min_height = 0, + .max_width = 2048, + .max_height = 2048, + .nlayers = ARRAY_SIZE(atmel_hlcdc_sama5d4_layers), + .layers = atmel_hlcdc_sama5d4_layers, +}; static const struct of_device_id atmel_hlcdc_of_match[] = { + { + .compatible = "atmel,at91sam9n12-hlcdc", + .data = &atmel_hlcdc_dc_at91sam9n12, + }, + { + .compatible = "atmel,at91sam9x5-hlcdc", + .data = &atmel_hlcdc_dc_at91sam9x5, + }, { .compatible = "atmel,sama5d3-hlcdc", .data = &atmel_hlcdc_dc_sama5d3, }, + { + .compatible = "atmel,sama5d4-hlcdc", + .data = &atmel_hlcdc_dc_sama5d4, + }, { /* sentinel */ }, }; @@ -485,7 +688,9 @@ static const struct file_operations fops = { }; static struct drm_driver atmel_hlcdc_dc_driver = { - .driver_features = DRIVER_HAVE_IRQ | DRIVER_GEM | DRIVER_MODESET, + .driver_features = DRIVER_HAVE_IRQ | DRIVER_GEM | + DRIVER_MODESET | DRIVER_PRIME | + DRIVER_ATOMIC, .preclose = atmel_hlcdc_dc_preclose, .lastclose = atmel_hlcdc_dc_lastclose, .irq_handler = atmel_hlcdc_dc_irq_handler, @@ -497,6 +702,15 @@ static struct drm_driver atmel_hlcdc_dc_driver = { .disable_vblank = atmel_hlcdc_dc_disable_vblank, .gem_free_object = drm_gem_cma_free_object, .gem_vm_ops = &drm_gem_cma_vm_ops, + .prime_handle_to_fd = drm_gem_prime_handle_to_fd, + .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_import = drm_gem_prime_import, + .gem_prime_export = drm_gem_prime_export, + .gem_prime_get_sg_table = drm_gem_cma_prime_get_sg_table, + .gem_prime_import_sg_table = drm_gem_cma_prime_import_sg_table, + .gem_prime_vmap = drm_gem_cma_prime_vmap, + .gem_prime_vunmap = drm_gem_cma_prime_vunmap, + .gem_prime_mmap = drm_gem_cma_prime_mmap, .dumb_create = drm_gem_cma_dumb_create, .dumb_map_offset = drm_gem_cma_dumb_map_offset, .dumb_destroy = drm_gem_dumb_destroy, diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_output.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_output.c index 9c4513005310..067e4c144bd6 100644 --- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_output.c +++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_output.c @@ -126,12 +126,16 @@ atmel_hlcdc_rgb_encoder_mode_set(struct drm_encoder *encoder, if (info->num_bus_formats) { switch (info->bus_formats[0]) { + case MEDIA_BUS_FMT_RGB565_1X16: + cfg |= ATMEL_HLCDC_CONNECTOR_RGB565 << 8; + break; case MEDIA_BUS_FMT_RGB666_1X18: cfg |= ATMEL_HLCDC_CONNECTOR_RGB666 << 8; break; case MEDIA_BUS_FMT_RGB888_1X24: cfg |= ATMEL_HLCDC_CONNECTOR_RGB888 << 8; break; + case MEDIA_BUS_FMT_RGB444_1X12: default: break; } diff --git a/drivers/gpu/drm/bochs/bochs_drv.c b/drivers/gpu/drm/bochs/bochs_drv.c index 98837bde2d25..7f1a3604b19f 100644 --- a/drivers/gpu/drm/bochs/bochs_drv.c +++ b/drivers/gpu/drm/bochs/bochs_drv.c @@ -109,7 +109,7 @@ static int bochs_pm_suspend(struct device *dev) if (bochs->fb.initialized) { console_lock(); - fb_set_suspend(bochs->fb.helper.fbdev, 1); + drm_fb_helper_set_suspend(&bochs->fb.helper, 1); console_unlock(); } @@ -126,7 +126,7 @@ static int bochs_pm_resume(struct device *dev) if (bochs->fb.initialized) { console_lock(); - fb_set_suspend(bochs->fb.helper.fbdev, 0); + drm_fb_helper_set_suspend(&bochs->fb.helper, 0); console_unlock(); } diff --git a/drivers/gpu/drm/bochs/bochs_fbdev.c b/drivers/gpu/drm/bochs/bochs_fbdev.c index 976d9798dc99..09a0637aab3e 100644 --- a/drivers/gpu/drm/bochs/bochs_fbdev.c +++ b/drivers/gpu/drm/bochs/bochs_fbdev.c @@ -24,9 +24,9 @@ static struct fb_ops bochsfb_ops = { .owner = THIS_MODULE, .fb_check_var = drm_fb_helper_check_var, .fb_set_par = drm_fb_helper_set_par, - .fb_fillrect = sys_fillrect, - .fb_copyarea = sys_copyarea, - .fb_imageblit = sys_imageblit, + .fb_fillrect = drm_fb_helper_sys_fillrect, + .fb_copyarea = drm_fb_helper_sys_copyarea, + .fb_imageblit = drm_fb_helper_sys_imageblit, .fb_pan_display = drm_fb_helper_pan_display, .fb_blank = drm_fb_helper_blank, .fb_setcmap = drm_fb_helper_setcmap, @@ -56,11 +56,9 @@ static int bochsfb_create(struct drm_fb_helper *helper, { struct bochs_device *bochs = container_of(helper, struct bochs_device, fb.helper); - struct drm_device *dev = bochs->dev; struct fb_info *info; struct drm_framebuffer *fb; struct drm_mode_fb_cmd2 mode_cmd; - struct device *device = &dev->pdev->dev; struct drm_gem_object *gobj = NULL; struct bochs_bo *bo = NULL; int size, ret; @@ -106,22 +104,23 @@ static int bochsfb_create(struct drm_fb_helper *helper, ttm_bo_unreserve(&bo->bo); /* init fb device */ - info = framebuffer_alloc(0, device); - if (info == NULL) - return -ENOMEM; + info = drm_fb_helper_alloc_fbi(helper); + if (IS_ERR(info)) + return PTR_ERR(info); info->par = &bochs->fb.helper; ret = bochs_framebuffer_init(bochs->dev, &bochs->fb.gfb, &mode_cmd, gobj); - if (ret) + if (ret) { + drm_fb_helper_release_fbi(helper); return ret; + } bochs->fb.size = size; /* setup helper */ fb = &bochs->fb.gfb.base; bochs->fb.helper.fb = fb; - bochs->fb.helper.fbdev = info; strcpy(info->fix.id, "bochsdrmfb"); @@ -139,30 +138,17 @@ static int bochsfb_create(struct drm_fb_helper *helper, info->fix.smem_start = 0; info->fix.smem_len = size; - ret = fb_alloc_cmap(&info->cmap, 256, 0); - if (ret) { - DRM_ERROR("%s: can't allocate color map\n", info->fix.id); - return -ENOMEM; - } - return 0; } static int bochs_fbdev_destroy(struct bochs_device *bochs) { struct bochs_framebuffer *gfb = &bochs->fb.gfb; - struct fb_info *info; DRM_DEBUG_DRIVER("\n"); - if (bochs->fb.helper.fbdev) { - info = bochs->fb.helper.fbdev; - - unregister_framebuffer(info); - if (info->cmap.len) - fb_dealloc_cmap(&info->cmap); - framebuffer_release(info); - } + drm_fb_helper_unregister_fbi(&bochs->fb.helper); + drm_fb_helper_release_fbi(&bochs->fb.helper); if (gfb->obj) { drm_gem_object_unreference_unlocked(gfb->obj); diff --git a/drivers/gpu/drm/bochs/bochs_mm.c b/drivers/gpu/drm/bochs/bochs_mm.c index 66286ff518d4..f69e6bf9bb0e 100644 --- a/drivers/gpu/drm/bochs/bochs_mm.c +++ b/drivers/gpu/drm/bochs/bochs_mm.c @@ -454,25 +454,17 @@ int bochs_dumb_mmap_offset(struct drm_file *file, struct drm_device *dev, uint32_t handle, uint64_t *offset) { struct drm_gem_object *obj; - int ret; struct bochs_bo *bo; - mutex_lock(&dev->struct_mutex); obj = drm_gem_object_lookup(dev, file, handle); - if (obj == NULL) { - ret = -ENOENT; - goto out_unlock; - } + if (obj == NULL) + return -ENOENT; bo = gem_to_bochs_bo(obj); *offset = bochs_bo_mmap_offset(bo); - drm_gem_object_unreference(obj); - ret = 0; -out_unlock: - mutex_unlock(&dev->struct_mutex); - return ret; - + drm_gem_object_unreference_unlocked(obj); + return 0; } /* ---------------------------------------------------------------------- */ diff --git a/drivers/gpu/drm/bridge/Kconfig b/drivers/gpu/drm/bridge/Kconfig index acef3223772c..2de52a53a803 100644 --- a/drivers/gpu/drm/bridge/Kconfig +++ b/drivers/gpu/drm/bridge/Kconfig @@ -1,24 +1,32 @@ +config DRM_BRIDGE + def_bool y + depends on DRM + help + Bridge registration and lookup framework. + +menu "Display Interface Bridges" + depends on DRM && DRM_BRIDGE + config DRM_DW_HDMI tristate - depends on DRM select DRM_KMS_HELPER -config DRM_PTN3460 - tristate "PTN3460 DP/LVDS bridge" - depends on DRM +config DRM_NXP_PTN3460 + tristate "NXP PTN3460 DP/LVDS bridge" depends on OF select DRM_KMS_HELPER select DRM_PANEL ---help--- - ptn3460 eDP-LVDS bridge chip driver. + NXP PTN3460 eDP-LVDS bridge chip driver. -config DRM_PS8622 +config DRM_PARADE_PS8622 tristate "Parade eDP/LVDS bridge" - depends on DRM depends on OF select DRM_PANEL select DRM_KMS_HELPER select BACKLIGHT_LCD_SUPPORT select BACKLIGHT_CLASS_DEVICE ---help--- - parade eDP-LVDS bridge chip driver. + Parade eDP-LVDS bridge chip driver. + +endmenu diff --git a/drivers/gpu/drm/bridge/Makefile b/drivers/gpu/drm/bridge/Makefile index 8dfebd984370..e2eef1c2f4c3 100644 --- a/drivers/gpu/drm/bridge/Makefile +++ b/drivers/gpu/drm/bridge/Makefile @@ -1,5 +1,5 @@ ccflags-y := -Iinclude/drm -obj-$(CONFIG_DRM_PS8622) += ps8622.o -obj-$(CONFIG_DRM_PTN3460) += ptn3460.o obj-$(CONFIG_DRM_DW_HDMI) += dw_hdmi.o +obj-$(CONFIG_DRM_NXP_PTN3460) += nxp-ptn3460.o +obj-$(CONFIG_DRM_PARADE_PS8622) += parade-ps8622.o diff --git a/drivers/gpu/drm/bridge/dw_hdmi.c b/drivers/gpu/drm/bridge/dw_hdmi.c index 816d104ca4da..0083d4e7e7e2 100644 --- a/drivers/gpu/drm/bridge/dw_hdmi.c +++ b/drivers/gpu/drm/bridge/dw_hdmi.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -81,10 +82,6 @@ static const u16 csc_coeff_rgb_in_eitu709[3][4] = { }; struct hdmi_vmode { - bool mdvi; - bool mhsyncpolarity; - bool mvsyncpolarity; - bool minterlaced; bool mdataenablepolarity; unsigned int mpixelclock; @@ -123,12 +120,20 @@ struct dw_hdmi { bool phy_enabled; struct drm_display_mode previous_mode; - struct regmap *regmap; struct i2c_adapter *ddc; void __iomem *regs; + bool sink_is_hdmi; + bool sink_has_audio; + struct mutex mutex; /* for state below and previous_mode */ + bool disabled; /* DRM has disabled our bridge */ + + spinlock_t audio_lock; struct mutex audio_mutex; unsigned int sample_rate; + unsigned int audio_cts; + unsigned int audio_n; + bool audio_enable; int ratio; void (*write)(struct dw_hdmi *hdmi, u8 val, int offset); @@ -335,42 +340,76 @@ static unsigned int hdmi_compute_cts(unsigned int freq, unsigned long pixel_clk, } static void hdmi_set_clk_regenerator(struct dw_hdmi *hdmi, - unsigned long pixel_clk) + unsigned long pixel_clk, unsigned int sample_rate, unsigned int ratio) { - unsigned int clk_n, clk_cts; + unsigned int n, cts; - clk_n = hdmi_compute_n(hdmi->sample_rate, pixel_clk, - hdmi->ratio); - clk_cts = hdmi_compute_cts(hdmi->sample_rate, pixel_clk, - hdmi->ratio); - - if (!clk_cts) { - dev_dbg(hdmi->dev, "%s: pixel clock not supported: %lu\n", - __func__, pixel_clk); - return; + n = hdmi_compute_n(sample_rate, pixel_clk, ratio); + cts = hdmi_compute_cts(sample_rate, pixel_clk, ratio); + if (!cts) { + dev_err(hdmi->dev, + "%s: pixel clock/sample rate not supported: %luMHz / %ukHz\n", + __func__, pixel_clk, sample_rate); } - dev_dbg(hdmi->dev, "%s: samplerate=%d ratio=%d pixelclk=%lu N=%d cts=%d\n", - __func__, hdmi->sample_rate, hdmi->ratio, - pixel_clk, clk_n, clk_cts); + dev_dbg(hdmi->dev, "%s: samplerate=%ukHz ratio=%d pixelclk=%luMHz N=%d cts=%d\n", + __func__, sample_rate, ratio, pixel_clk, n, cts); - hdmi_set_cts_n(hdmi, clk_cts, clk_n); + spin_lock_irq(&hdmi->audio_lock); + hdmi->audio_n = n; + hdmi->audio_cts = cts; + hdmi_set_cts_n(hdmi, cts, hdmi->audio_enable ? n : 0); + spin_unlock_irq(&hdmi->audio_lock); } static void hdmi_init_clk_regenerator(struct dw_hdmi *hdmi) { mutex_lock(&hdmi->audio_mutex); - hdmi_set_clk_regenerator(hdmi, 74250000); + hdmi_set_clk_regenerator(hdmi, 74250000, hdmi->sample_rate, + hdmi->ratio); mutex_unlock(&hdmi->audio_mutex); } static void hdmi_clk_regenerator_update_pixel_clock(struct dw_hdmi *hdmi) { mutex_lock(&hdmi->audio_mutex); - hdmi_set_clk_regenerator(hdmi, hdmi->hdmi_data.video_mode.mpixelclock); + hdmi_set_clk_regenerator(hdmi, hdmi->hdmi_data.video_mode.mpixelclock, + hdmi->sample_rate, hdmi->ratio); mutex_unlock(&hdmi->audio_mutex); } +void dw_hdmi_set_sample_rate(struct dw_hdmi *hdmi, unsigned int rate) +{ + mutex_lock(&hdmi->audio_mutex); + hdmi->sample_rate = rate; + hdmi_set_clk_regenerator(hdmi, hdmi->hdmi_data.video_mode.mpixelclock, + hdmi->sample_rate, hdmi->ratio); + mutex_unlock(&hdmi->audio_mutex); +} +EXPORT_SYMBOL_GPL(dw_hdmi_set_sample_rate); + +void dw_hdmi_audio_enable(struct dw_hdmi *hdmi) +{ + unsigned long flags; + + spin_lock_irqsave(&hdmi->audio_lock, flags); + hdmi->audio_enable = true; + hdmi_set_cts_n(hdmi, hdmi->audio_cts, hdmi->audio_n); + spin_unlock_irqrestore(&hdmi->audio_lock, flags); +} +EXPORT_SYMBOL_GPL(dw_hdmi_audio_enable); + +void dw_hdmi_audio_disable(struct dw_hdmi *hdmi) +{ + unsigned long flags; + + spin_lock_irqsave(&hdmi->audio_lock, flags); + hdmi->audio_enable = false; + hdmi_set_cts_n(hdmi, hdmi->audio_cts, 0); + spin_unlock_irqrestore(&hdmi->audio_lock, flags); +} +EXPORT_SYMBOL_GPL(dw_hdmi_audio_disable); + /* * this submodule is responsible for the video data synchronization. * for example, for RGB 4:4:4 input, the data map is defined as @@ -701,9 +740,9 @@ static int hdmi_phy_i2c_write(struct dw_hdmi *hdmi, unsigned short data, return 0; } -static void dw_hdmi_phy_enable_power(struct dw_hdmi *hdmi, u8 enable) +static void dw_hdmi_phy_enable_powerdown(struct dw_hdmi *hdmi, bool enable) { - hdmi_mask_writeb(hdmi, enable, HDMI_PHY_CONF0, + hdmi_mask_writeb(hdmi, !enable, HDMI_PHY_CONF0, HDMI_PHY_CONF0_PDZ_OFFSET, HDMI_PHY_CONF0_PDZ_MASK); } @@ -753,12 +792,12 @@ static void dw_hdmi_phy_sel_interface_control(struct dw_hdmi *hdmi, u8 enable) static int hdmi_phy_configure(struct dw_hdmi *hdmi, unsigned char prep, unsigned char res, int cscon) { - unsigned res_idx, i; + unsigned res_idx; u8 val, msec; - const struct dw_hdmi_plat_data *plat_data = hdmi->plat_data; - const struct dw_hdmi_mpll_config *mpll_config = plat_data->mpll_cfg; - const struct dw_hdmi_curr_ctrl *curr_ctrl = plat_data->cur_ctr; - const struct dw_hdmi_phy_config *phy_config = plat_data->phy_config; + const struct dw_hdmi_plat_data *pdata = hdmi->plat_data; + const struct dw_hdmi_mpll_config *mpll_config = pdata->mpll_cfg; + const struct dw_hdmi_curr_ctrl *curr_ctrl = pdata->cur_ctr; + const struct dw_hdmi_phy_config *phy_config = pdata->phy_config; if (prep) return -EINVAL; @@ -778,6 +817,30 @@ static int hdmi_phy_configure(struct dw_hdmi *hdmi, unsigned char prep, return -EINVAL; } + /* PLL/MPLL Cfg - always match on final entry */ + for (; mpll_config->mpixelclock != ~0UL; mpll_config++) + if (hdmi->hdmi_data.video_mode.mpixelclock <= + mpll_config->mpixelclock) + break; + + for (; curr_ctrl->mpixelclock != ~0UL; curr_ctrl++) + if (hdmi->hdmi_data.video_mode.mpixelclock <= + curr_ctrl->mpixelclock) + break; + + for (; phy_config->mpixelclock != ~0UL; phy_config++) + if (hdmi->hdmi_data.video_mode.mpixelclock <= + phy_config->mpixelclock) + break; + + if (mpll_config->mpixelclock == ~0UL || + curr_ctrl->mpixelclock == ~0UL || + phy_config->mpixelclock == ~0UL) { + dev_err(hdmi->dev, "Pixel clock %d - unsupported by HDMI\n", + hdmi->hdmi_data.video_mode.mpixelclock); + return -EINVAL; + } + /* Enable csc path */ if (cscon) val = HDMI_MC_FLOWCTRL_FEED_THROUGH_OFF_CSC_IN_PATH; @@ -803,48 +866,23 @@ static int hdmi_phy_configure(struct dw_hdmi *hdmi, unsigned char prep, HDMI_PHY_I2CM_SLAVE_ADDR); hdmi_phy_test_clear(hdmi, 0); - /* PLL/MPLL Cfg - always match on final entry */ - for (i = 0; mpll_config[i].mpixelclock != (~0UL); i++) - if (hdmi->hdmi_data.video_mode.mpixelclock <= - mpll_config[i].mpixelclock) - break; - - hdmi_phy_i2c_write(hdmi, mpll_config[i].res[res_idx].cpce, 0x06); - hdmi_phy_i2c_write(hdmi, mpll_config[i].res[res_idx].gmp, 0x15); - - for (i = 0; curr_ctrl[i].mpixelclock != (~0UL); i++) - if (hdmi->hdmi_data.video_mode.mpixelclock <= - curr_ctrl[i].mpixelclock) - break; - - if (curr_ctrl[i].mpixelclock == (~0UL)) { - dev_err(hdmi->dev, "Pixel clock %d - unsupported by HDMI\n", - hdmi->hdmi_data.video_mode.mpixelclock); - return -EINVAL; - } + hdmi_phy_i2c_write(hdmi, mpll_config->res[res_idx].cpce, 0x06); + hdmi_phy_i2c_write(hdmi, mpll_config->res[res_idx].gmp, 0x15); /* CURRCTRL */ - hdmi_phy_i2c_write(hdmi, curr_ctrl[i].curr[res_idx], 0x10); + hdmi_phy_i2c_write(hdmi, curr_ctrl->curr[res_idx], 0x10); hdmi_phy_i2c_write(hdmi, 0x0000, 0x13); /* PLLPHBYCTRL */ hdmi_phy_i2c_write(hdmi, 0x0006, 0x17); - for (i = 0; phy_config[i].mpixelclock != (~0UL); i++) - if (hdmi->hdmi_data.video_mode.mpixelclock <= - phy_config[i].mpixelclock) - break; - - /* RESISTANCE TERM 133Ohm Cfg */ - hdmi_phy_i2c_write(hdmi, phy_config[i].term, 0x19); /* TXTERM */ - /* PREEMP Cgf 0.00 */ - hdmi_phy_i2c_write(hdmi, phy_config[i].sym_ctr, 0x09); /* CKSYMTXCTRL */ - /* TX/CK LVL 10 */ - hdmi_phy_i2c_write(hdmi, phy_config[i].vlev_ctr, 0x0E); /* VLEVCTRL */ + hdmi_phy_i2c_write(hdmi, phy_config->term, 0x19); /* TXTERM */ + hdmi_phy_i2c_write(hdmi, phy_config->sym_ctr, 0x09); /* CKSYMTXCTRL */ + hdmi_phy_i2c_write(hdmi, phy_config->vlev_ctr, 0x0E); /* VLEVCTRL */ /* REMOVE CLK TERM */ hdmi_phy_i2c_write(hdmi, 0x8000, 0x05); /* CKCALCTRL */ - dw_hdmi_phy_enable_power(hdmi, 1); + dw_hdmi_phy_enable_powerdown(hdmi, false); /* toggle TMDS enable */ dw_hdmi_phy_enable_tmds(hdmi, 0); @@ -879,18 +917,17 @@ static int hdmi_phy_configure(struct dw_hdmi *hdmi, unsigned char prep, static int dw_hdmi_phy_init(struct dw_hdmi *hdmi) { int i, ret; - bool cscon = false; + bool cscon; /*check csc whether needed activated in HDMI mode */ - cscon = (is_color_space_conversion(hdmi) && - !hdmi->hdmi_data.video_mode.mdvi); + cscon = hdmi->sink_is_hdmi && is_color_space_conversion(hdmi); /* HDMI Phy spec says to do the phy initialization sequence twice */ for (i = 0; i < 2; i++) { dw_hdmi_phy_sel_data_en_pol(hdmi, 1); dw_hdmi_phy_sel_interface_control(hdmi, 0); dw_hdmi_phy_enable_tmds(hdmi, 0); - dw_hdmi_phy_enable_power(hdmi, 0); + dw_hdmi_phy_enable_powerdown(hdmi, true); /* Enable CSC */ ret = hdmi_phy_configure(hdmi, 0, 8, cscon); @@ -921,74 +958,76 @@ static void hdmi_tx_hdcp_config(struct dw_hdmi *hdmi) HDMI_A_HDCPCFG1_ENCRYPTIONDISABLE_MASK, HDMI_A_HDCPCFG1); } -static void hdmi_config_AVI(struct dw_hdmi *hdmi) +static void hdmi_config_AVI(struct dw_hdmi *hdmi, struct drm_display_mode *mode) { - u8 val, pix_fmt, under_scan; - u8 act_ratio, coded_ratio, colorimetry, ext_colorimetry; - bool aspect_16_9; + struct hdmi_avi_infoframe frame; + u8 val; - aspect_16_9 = false; /* FIXME */ + /* Initialise info frame from DRM mode */ + drm_hdmi_avi_infoframe_from_display_mode(&frame, mode); - /* AVI Data Byte 1 */ if (hdmi->hdmi_data.enc_out_format == YCBCR444) - pix_fmt = HDMI_FC_AVICONF0_PIX_FMT_YCBCR444; + frame.colorspace = HDMI_COLORSPACE_YUV444; else if (hdmi->hdmi_data.enc_out_format == YCBCR422_8BITS) - pix_fmt = HDMI_FC_AVICONF0_PIX_FMT_YCBCR422; + frame.colorspace = HDMI_COLORSPACE_YUV422; else - pix_fmt = HDMI_FC_AVICONF0_PIX_FMT_RGB; - - under_scan = HDMI_FC_AVICONF0_SCAN_INFO_NODATA; - - /* - * Active format identification data is present in the AVI InfoFrame. - * Under scan info, no bar data - */ - val = pix_fmt | under_scan | - HDMI_FC_AVICONF0_ACTIVE_FMT_INFO_PRESENT | - HDMI_FC_AVICONF0_BAR_DATA_NO_DATA; - - hdmi_writeb(hdmi, val, HDMI_FC_AVICONF0); - - /* AVI Data Byte 2 -Set the Aspect Ratio */ - if (aspect_16_9) { - act_ratio = HDMI_FC_AVICONF1_ACTIVE_ASPECT_RATIO_16_9; - coded_ratio = HDMI_FC_AVICONF1_CODED_ASPECT_RATIO_16_9; - } else { - act_ratio = HDMI_FC_AVICONF1_ACTIVE_ASPECT_RATIO_4_3; - coded_ratio = HDMI_FC_AVICONF1_CODED_ASPECT_RATIO_4_3; - } + frame.colorspace = HDMI_COLORSPACE_RGB; /* Set up colorimetry */ if (hdmi->hdmi_data.enc_out_format == XVYCC444) { - colorimetry = HDMI_FC_AVICONF1_COLORIMETRY_EXTENDED_INFO; + frame.colorimetry = HDMI_COLORIMETRY_EXTENDED; if (hdmi->hdmi_data.colorimetry == HDMI_COLORIMETRY_ITU_601) - ext_colorimetry = - HDMI_FC_AVICONF2_EXT_COLORIMETRY_XVYCC601; + frame.extended_colorimetry = + HDMI_EXTENDED_COLORIMETRY_XV_YCC_601; else /*hdmi->hdmi_data.colorimetry == HDMI_COLORIMETRY_ITU_709*/ - ext_colorimetry = - HDMI_FC_AVICONF2_EXT_COLORIMETRY_XVYCC709; + frame.extended_colorimetry = + HDMI_EXTENDED_COLORIMETRY_XV_YCC_709; } else if (hdmi->hdmi_data.enc_out_format != RGB) { - if (hdmi->hdmi_data.colorimetry == HDMI_COLORIMETRY_ITU_601) - colorimetry = HDMI_FC_AVICONF1_COLORIMETRY_SMPTE; - else /*hdmi->hdmi_data.colorimetry == HDMI_COLORIMETRY_ITU_709*/ - colorimetry = HDMI_FC_AVICONF1_COLORIMETRY_ITUR; - ext_colorimetry = HDMI_FC_AVICONF2_EXT_COLORIMETRY_XVYCC601; + frame.colorimetry = hdmi->hdmi_data.colorimetry; + frame.extended_colorimetry = HDMI_EXTENDED_COLORIMETRY_XV_YCC_601; } else { /* Carries no data */ - colorimetry = HDMI_FC_AVICONF1_COLORIMETRY_NO_DATA; - ext_colorimetry = HDMI_FC_AVICONF2_EXT_COLORIMETRY_XVYCC601; + frame.colorimetry = HDMI_COLORIMETRY_NONE; + frame.extended_colorimetry = HDMI_EXTENDED_COLORIMETRY_XV_YCC_601; } - val = colorimetry | coded_ratio | act_ratio; + frame.scan_mode = HDMI_SCAN_MODE_NONE; + + /* + * The Designware IP uses a different byte format from standard + * AVI info frames, though generally the bits are in the correct + * bytes. + */ + + /* + * AVI data byte 1 differences: Colorspace in bits 4,5 rather than 5,6, + * active aspect present in bit 6 rather than 4. + */ + val = (frame.colorspace & 3) << 4 | (frame.scan_mode & 0x3); + if (frame.active_aspect & 15) + val |= HDMI_FC_AVICONF0_ACTIVE_FMT_INFO_PRESENT; + if (frame.top_bar || frame.bottom_bar) + val |= HDMI_FC_AVICONF0_BAR_DATA_HORIZ_BAR; + if (frame.left_bar || frame.right_bar) + val |= HDMI_FC_AVICONF0_BAR_DATA_VERT_BAR; + hdmi_writeb(hdmi, val, HDMI_FC_AVICONF0); + + /* AVI data byte 2 differences: none */ + val = ((frame.colorimetry & 0x3) << 6) | + ((frame.picture_aspect & 0x3) << 4) | + (frame.active_aspect & 0xf); hdmi_writeb(hdmi, val, HDMI_FC_AVICONF1); - /* AVI Data Byte 3 */ - val = HDMI_FC_AVICONF2_IT_CONTENT_NO_DATA | ext_colorimetry | - HDMI_FC_AVICONF2_RGB_QUANT_DEFAULT | - HDMI_FC_AVICONF2_SCALING_NONE; + /* AVI data byte 3 differences: none */ + val = ((frame.extended_colorimetry & 0x7) << 4) | + ((frame.quantization_range & 0x3) << 2) | + (frame.nups & 0x3); + if (frame.itc) + val |= HDMI_FC_AVICONF2_IT_CONTENT_VALID; hdmi_writeb(hdmi, val, HDMI_FC_AVICONF2); - /* AVI Data Byte 4 */ - hdmi_writeb(hdmi, hdmi->vic, HDMI_FC_AVIVID); + /* AVI data byte 4 differences: none */ + val = frame.video_code & 0x7f; + hdmi_writeb(hdmi, val, HDMI_FC_AVIVID); /* AVI Data Byte 5- set up input and output pixel repetition */ val = (((hdmi->hdmi_data.video_mode.mpixelrepetitioninput + 1) << @@ -999,20 +1038,23 @@ static void hdmi_config_AVI(struct dw_hdmi *hdmi) HDMI_FC_PRCONF_OUTPUT_PR_FACTOR_MASK); hdmi_writeb(hdmi, val, HDMI_FC_PRCONF); - /* IT Content and quantization range = don't care */ - val = HDMI_FC_AVICONF3_IT_CONTENT_TYPE_GRAPHICS | - HDMI_FC_AVICONF3_QUANT_RANGE_LIMITED; + /* + * AVI data byte 5 differences: content type in 0,1 rather than 4,5, + * ycc range in bits 2,3 rather than 6,7 + */ + val = ((frame.ycc_quantization_range & 0x3) << 2) | + (frame.content_type & 0x3); hdmi_writeb(hdmi, val, HDMI_FC_AVICONF3); /* AVI Data Bytes 6-13 */ - hdmi_writeb(hdmi, 0, HDMI_FC_AVIETB0); - hdmi_writeb(hdmi, 0, HDMI_FC_AVIETB1); - hdmi_writeb(hdmi, 0, HDMI_FC_AVISBB0); - hdmi_writeb(hdmi, 0, HDMI_FC_AVISBB1); - hdmi_writeb(hdmi, 0, HDMI_FC_AVIELB0); - hdmi_writeb(hdmi, 0, HDMI_FC_AVIELB1); - hdmi_writeb(hdmi, 0, HDMI_FC_AVISRB0); - hdmi_writeb(hdmi, 0, HDMI_FC_AVISRB1); + hdmi_writeb(hdmi, frame.top_bar & 0xff, HDMI_FC_AVIETB0); + hdmi_writeb(hdmi, (frame.top_bar >> 8) & 0xff, HDMI_FC_AVIETB1); + hdmi_writeb(hdmi, frame.bottom_bar & 0xff, HDMI_FC_AVISBB0); + hdmi_writeb(hdmi, (frame.bottom_bar >> 8) & 0xff, HDMI_FC_AVISBB1); + hdmi_writeb(hdmi, frame.left_bar & 0xff, HDMI_FC_AVIELB0); + hdmi_writeb(hdmi, (frame.left_bar >> 8) & 0xff, HDMI_FC_AVIELB1); + hdmi_writeb(hdmi, frame.right_bar & 0xff, HDMI_FC_AVISRB0); + hdmi_writeb(hdmi, (frame.right_bar >> 8) & 0xff, HDMI_FC_AVISRB1); } static void hdmi_av_composer(struct dw_hdmi *hdmi, @@ -1022,9 +1064,6 @@ static void hdmi_av_composer(struct dw_hdmi *hdmi, struct hdmi_vmode *vmode = &hdmi->hdmi_data.video_mode; int hblank, vblank, h_de_hs, v_de_vs, hsync_len, vsync_len; - vmode->mhsyncpolarity = !!(mode->flags & DRM_MODE_FLAG_PHSYNC); - vmode->mvsyncpolarity = !!(mode->flags & DRM_MODE_FLAG_PVSYNC); - vmode->minterlaced = !!(mode->flags & DRM_MODE_FLAG_INTERLACE); vmode->mpixelclock = mode->clock * 1000; dev_dbg(hdmi->dev, "final pixclk = %d\n", vmode->mpixelclock); @@ -1034,13 +1073,13 @@ static void hdmi_av_composer(struct dw_hdmi *hdmi, HDMI_FC_INVIDCONF_HDCP_KEEPOUT_ACTIVE : HDMI_FC_INVIDCONF_HDCP_KEEPOUT_INACTIVE); - inv_val |= (vmode->mvsyncpolarity ? + inv_val |= mode->flags & DRM_MODE_FLAG_PVSYNC ? HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_ACTIVE_HIGH : - HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_ACTIVE_LOW); + HDMI_FC_INVIDCONF_VSYNC_IN_POLARITY_ACTIVE_LOW; - inv_val |= (vmode->mhsyncpolarity ? + inv_val |= mode->flags & DRM_MODE_FLAG_PHSYNC ? HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_ACTIVE_HIGH : - HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_ACTIVE_LOW); + HDMI_FC_INVIDCONF_HSYNC_IN_POLARITY_ACTIVE_LOW; inv_val |= (vmode->mdataenablepolarity ? HDMI_FC_INVIDCONF_DE_IN_POLARITY_ACTIVE_HIGH : @@ -1049,17 +1088,17 @@ static void hdmi_av_composer(struct dw_hdmi *hdmi, if (hdmi->vic == 39) inv_val |= HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_ACTIVE_HIGH; else - inv_val |= (vmode->minterlaced ? + inv_val |= mode->flags & DRM_MODE_FLAG_INTERLACE ? HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_ACTIVE_HIGH : - HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_ACTIVE_LOW); + HDMI_FC_INVIDCONF_R_V_BLANK_IN_OSC_ACTIVE_LOW; - inv_val |= (vmode->minterlaced ? + inv_val |= mode->flags & DRM_MODE_FLAG_INTERLACE ? HDMI_FC_INVIDCONF_IN_I_P_INTERLACED : - HDMI_FC_INVIDCONF_IN_I_P_PROGRESSIVE); + HDMI_FC_INVIDCONF_IN_I_P_PROGRESSIVE; - inv_val |= (vmode->mdvi ? - HDMI_FC_INVIDCONF_DVI_MODEZ_DVI_MODE : - HDMI_FC_INVIDCONF_DVI_MODEZ_HDMI_MODE); + inv_val |= hdmi->sink_is_hdmi ? + HDMI_FC_INVIDCONF_DVI_MODEZ_HDMI_MODE : + HDMI_FC_INVIDCONF_DVI_MODEZ_DVI_MODE; hdmi_writeb(hdmi, inv_val, HDMI_FC_INVIDCONF); @@ -1105,7 +1144,7 @@ static void dw_hdmi_phy_disable(struct dw_hdmi *hdmi) return; dw_hdmi_phy_enable_tmds(hdmi, 0); - dw_hdmi_phy_enable_power(hdmi, 0); + dw_hdmi_phy_enable_powerdown(hdmi, true); hdmi->phy_enabled = false; } @@ -1186,10 +1225,8 @@ static int dw_hdmi_setup(struct dw_hdmi *hdmi, struct drm_display_mode *mode) if (!hdmi->vic) { dev_dbg(hdmi->dev, "Non-CEA mode used in HDMI\n"); - hdmi->hdmi_data.video_mode.mdvi = true; } else { dev_dbg(hdmi->dev, "CEA mode used vic=%d\n", hdmi->vic); - hdmi->hdmi_data.video_mode.mdvi = false; } if ((hdmi->vic == 6) || (hdmi->vic == 7) || @@ -1200,18 +1237,7 @@ static int dw_hdmi_setup(struct dw_hdmi *hdmi, struct drm_display_mode *mode) else hdmi->hdmi_data.colorimetry = HDMI_COLORIMETRY_ITU_709; - if ((hdmi->vic == 10) || (hdmi->vic == 11) || - (hdmi->vic == 12) || (hdmi->vic == 13) || - (hdmi->vic == 14) || (hdmi->vic == 15) || - (hdmi->vic == 25) || (hdmi->vic == 26) || - (hdmi->vic == 27) || (hdmi->vic == 28) || - (hdmi->vic == 29) || (hdmi->vic == 30) || - (hdmi->vic == 35) || (hdmi->vic == 36) || - (hdmi->vic == 37) || (hdmi->vic == 38)) - hdmi->hdmi_data.video_mode.mpixelrepetitionoutput = 1; - else - hdmi->hdmi_data.video_mode.mpixelrepetitionoutput = 0; - + hdmi->hdmi_data.video_mode.mpixelrepetitionoutput = 0; hdmi->hdmi_data.video_mode.mpixelrepetitioninput = 0; /* TODO: Get input format from IPU (via FB driver interface) */ @@ -1235,18 +1261,22 @@ static int dw_hdmi_setup(struct dw_hdmi *hdmi, struct drm_display_mode *mode) /* HDMI Initialization Step B.3 */ dw_hdmi_enable_video_path(hdmi); - /* not for DVI mode */ - if (hdmi->hdmi_data.video_mode.mdvi) { - dev_dbg(hdmi->dev, "%s DVI mode\n", __func__); - } else { - dev_dbg(hdmi->dev, "%s CEA mode\n", __func__); + if (hdmi->sink_has_audio) { + dev_dbg(hdmi->dev, "sink has audio support\n"); /* HDMI Initialization Step E - Configure audio */ hdmi_clk_regenerator_update_pixel_clock(hdmi); hdmi_enable_audio_clk(hdmi); + } + + /* not for DVI mode */ + if (hdmi->sink_is_hdmi) { + dev_dbg(hdmi->dev, "%s HDMI mode\n", __func__); /* HDMI Initialization Step F - Configure AVI InfoFrame */ - hdmi_config_AVI(hdmi); + hdmi_config_AVI(hdmi, mode); + } else { + dev_dbg(hdmi->dev, "%s DVI mode\n", __func__); } hdmi_video_packetize(hdmi); @@ -1255,7 +1285,7 @@ static int dw_hdmi_setup(struct dw_hdmi *hdmi, struct drm_display_mode *mode) hdmi_tx_hdcp_config(hdmi); dw_hdmi_clear_overflow(hdmi); - if (hdmi->cable_plugin && !hdmi->hdmi_data.video_mode.mdvi) + if (hdmi->cable_plugin && hdmi->sink_is_hdmi) hdmi_enable_overflow_interrupts(hdmi); return 0; @@ -1348,10 +1378,12 @@ static void dw_hdmi_bridge_mode_set(struct drm_bridge *bridge, { struct dw_hdmi *hdmi = bridge->driver_private; - dw_hdmi_setup(hdmi, mode); + mutex_lock(&hdmi->mutex); /* Store the display mode for plugin/DKMS poweron events */ memcpy(&hdmi->previous_mode, mode, sizeof(hdmi->previous_mode)); + + mutex_unlock(&hdmi->mutex); } static bool dw_hdmi_bridge_mode_fixup(struct drm_bridge *bridge, @@ -1365,14 +1397,20 @@ static void dw_hdmi_bridge_disable(struct drm_bridge *bridge) { struct dw_hdmi *hdmi = bridge->driver_private; + mutex_lock(&hdmi->mutex); + hdmi->disabled = true; dw_hdmi_poweroff(hdmi); + mutex_unlock(&hdmi->mutex); } static void dw_hdmi_bridge_enable(struct drm_bridge *bridge) { struct dw_hdmi *hdmi = bridge->driver_private; + mutex_lock(&hdmi->mutex); dw_hdmi_poweron(hdmi); + hdmi->disabled = false; + mutex_unlock(&hdmi->mutex); } static void dw_hdmi_bridge_nop(struct drm_bridge *bridge) @@ -1405,6 +1443,8 @@ static int dw_hdmi_connector_get_modes(struct drm_connector *connector) dev_dbg(hdmi->dev, "got edid: width[%d] x height[%d]\n", edid->width_cm, edid->height_cm); + hdmi->sink_is_hdmi = drm_detect_hdmi_monitor(edid); + hdmi->sink_has_audio = drm_detect_monitor_audio(edid); drm_mode_connector_update_edid_property(connector, edid); ret = drm_add_edid_modes(connector, edid); kfree(edid); @@ -1423,6 +1463,10 @@ dw_hdmi_connector_mode_valid(struct drm_connector *connector, struct dw_hdmi, connector); enum drm_mode_status mode_status = MODE_OK; + /* We don't support double-clocked modes */ + if (mode->flags & DRM_MODE_FLAG_DBLCLK) + return MODE_BAD; + if (hdmi->plat_data->mode_valid) mode_status = hdmi->plat_data->mode_valid(connector, mode); @@ -1489,21 +1533,21 @@ static irqreturn_t dw_hdmi_irq(int irq, void *dev_id) phy_int_pol = hdmi_readb(hdmi, HDMI_PHY_POL0); if (intr_stat & HDMI_IH_PHY_STAT0_HPD) { + hdmi_modb(hdmi, ~phy_int_pol, HDMI_PHY_HPD, HDMI_PHY_POL0); + mutex_lock(&hdmi->mutex); if (phy_int_pol & HDMI_PHY_HPD) { dev_dbg(hdmi->dev, "EVENT=plugin\n"); - hdmi_modb(hdmi, 0, HDMI_PHY_HPD, HDMI_PHY_POL0); - - dw_hdmi_poweron(hdmi); + if (!hdmi->disabled) + dw_hdmi_poweron(hdmi); } else { dev_dbg(hdmi->dev, "EVENT=plugout\n"); - hdmi_modb(hdmi, HDMI_PHY_HPD, HDMI_PHY_HPD, - HDMI_PHY_POL0); - - dw_hdmi_poweroff(hdmi); + if (!hdmi->disabled) + dw_hdmi_poweroff(hdmi); } - drm_helper_hpd_irq_event(hdmi->connector.dev); + mutex_unlock(&hdmi->mutex); + drm_helper_hpd_irq_event(hdmi->bridge->dev); } hdmi_writeb(hdmi, intr_stat, HDMI_IH_PHY_STAT0); @@ -1570,8 +1614,11 @@ int dw_hdmi_bind(struct device *dev, struct device *master, hdmi->sample_rate = 48000; hdmi->ratio = 100; hdmi->encoder = encoder; + hdmi->disabled = true; + mutex_init(&hdmi->mutex); mutex_init(&hdmi->audio_mutex); + spin_lock_init(&hdmi->audio_lock); of_property_read_u32(np, "reg-io-width", &val); diff --git a/drivers/gpu/drm/bridge/dw_hdmi.h b/drivers/gpu/drm/bridge/dw_hdmi.h index 175dbc89a824..ee7f7ed2ab12 100644 --- a/drivers/gpu/drm/bridge/dw_hdmi.h +++ b/drivers/gpu/drm/bridge/dw_hdmi.h @@ -7,8 +7,8 @@ * (at your option) any later version. */ -#ifndef __IMX_HDMI_H__ -#define __IMX_HDMI_H__ +#ifndef __DW_HDMI_H__ +#define __DW_HDMI_H__ /* Identification Registers */ #define HDMI_DESIGN_ID 0x0000 @@ -525,7 +525,7 @@ /* I2C Master Registers (E-DDC) */ #define HDMI_I2CM_SLAVE 0x7E00 -#define HDMI_I2CMESS 0x7E01 +#define HDMI_I2CM_ADDRESS 0x7E01 #define HDMI_I2CM_DATAO 0x7E02 #define HDMI_I2CM_DATAI 0x7E03 #define HDMI_I2CM_OPERATION 0x7E04 @@ -1031,4 +1031,4 @@ enum { HDMI_A_VIDPOLCFG_HSYNCPOL_ACTIVE_LOW = 0x0, }; -#endif /* __IMX_HDMI_H__ */ +#endif /* __DW_HDMI_H__ */ diff --git a/drivers/gpu/drm/bridge/ptn3460.c b/drivers/gpu/drm/bridge/nxp-ptn3460.c similarity index 100% rename from drivers/gpu/drm/bridge/ptn3460.c rename to drivers/gpu/drm/bridge/nxp-ptn3460.c diff --git a/drivers/gpu/drm/bridge/ps8622.c b/drivers/gpu/drm/bridge/parade-ps8622.c similarity index 100% rename from drivers/gpu/drm/bridge/ps8622.c rename to drivers/gpu/drm/bridge/parade-ps8622.c diff --git a/drivers/gpu/drm/cirrus/cirrus_drv.c b/drivers/gpu/drm/cirrus/cirrus_drv.c index b9140032962d..b1619e29a564 100644 --- a/drivers/gpu/drm/cirrus/cirrus_drv.c +++ b/drivers/gpu/drm/cirrus/cirrus_drv.c @@ -92,7 +92,7 @@ static int cirrus_pm_suspend(struct device *dev) if (cdev->mode_info.gfbdev) { console_lock(); - fb_set_suspend(cdev->mode_info.gfbdev->helper.fbdev, 1); + drm_fb_helper_set_suspend(&cdev->mode_info.gfbdev->helper, 1); console_unlock(); } @@ -109,7 +109,7 @@ static int cirrus_pm_resume(struct device *dev) if (cdev->mode_info.gfbdev) { console_lock(); - fb_set_suspend(cdev->mode_info.gfbdev->helper.fbdev, 0); + drm_fb_helper_set_suspend(&cdev->mode_info.gfbdev->helper, 0); console_unlock(); } diff --git a/drivers/gpu/drm/cirrus/cirrus_fbdev.c b/drivers/gpu/drm/cirrus/cirrus_fbdev.c index 13ddf1c4bb8e..589103bcc06c 100644 --- a/drivers/gpu/drm/cirrus/cirrus_fbdev.c +++ b/drivers/gpu/drm/cirrus/cirrus_fbdev.c @@ -98,7 +98,7 @@ static void cirrus_fillrect(struct fb_info *info, const struct fb_fillrect *rect) { struct cirrus_fbdev *afbdev = info->par; - sys_fillrect(info, rect); + drm_fb_helper_sys_fillrect(info, rect); cirrus_dirty_update(afbdev, rect->dx, rect->dy, rect->width, rect->height); } @@ -107,7 +107,7 @@ static void cirrus_copyarea(struct fb_info *info, const struct fb_copyarea *area) { struct cirrus_fbdev *afbdev = info->par; - sys_copyarea(info, area); + drm_fb_helper_sys_copyarea(info, area); cirrus_dirty_update(afbdev, area->dx, area->dy, area->width, area->height); } @@ -116,7 +116,7 @@ static void cirrus_imageblit(struct fb_info *info, const struct fb_image *image) { struct cirrus_fbdev *afbdev = info->par; - sys_imageblit(info, image); + drm_fb_helper_sys_imageblit(info, image); cirrus_dirty_update(afbdev, image->dx, image->dy, image->width, image->height); } @@ -165,12 +165,10 @@ static int cirrusfb_create(struct drm_fb_helper *helper, { struct cirrus_fbdev *gfbdev = container_of(helper, struct cirrus_fbdev, helper); - struct drm_device *dev = gfbdev->helper.dev; struct cirrus_device *cdev = gfbdev->helper.dev->dev_private; struct fb_info *info; struct drm_framebuffer *fb; struct drm_mode_fb_cmd2 mode_cmd; - struct device *device = &dev->pdev->dev; void *sysram; struct drm_gem_object *gobj = NULL; struct cirrus_bo *bo = NULL; @@ -195,9 +193,9 @@ static int cirrusfb_create(struct drm_fb_helper *helper, if (!sysram) return -ENOMEM; - info = framebuffer_alloc(0, device); - if (info == NULL) - return -ENOMEM; + info = drm_fb_helper_alloc_fbi(helper); + if (IS_ERR(info)) + return PTR_ERR(info); info->par = gfbdev; @@ -216,11 +214,9 @@ static int cirrusfb_create(struct drm_fb_helper *helper, /* setup helper */ gfbdev->helper.fb = fb; - gfbdev->helper.fbdev = info; strcpy(info->fix.id, "cirrusdrmfb"); - info->flags = FBINFO_DEFAULT; info->fbops = &cirrusfb_ops; @@ -229,11 +225,6 @@ static int cirrusfb_create(struct drm_fb_helper *helper, sizes->fb_height); /* setup aperture base/size for vesafb takeover */ - info->apertures = alloc_apertures(1); - if (!info->apertures) { - ret = -ENOMEM; - goto out_iounmap; - } info->apertures->ranges[0].base = cdev->dev->mode_config.fb_base; info->apertures->ranges[0].size = cdev->mc.vram_size; @@ -246,13 +237,6 @@ static int cirrusfb_create(struct drm_fb_helper *helper, info->fix.mmio_start = 0; info->fix.mmio_len = 0; - ret = fb_alloc_cmap(&info->cmap, 256, 0); - if (ret) { - DRM_ERROR("%s: can't allocate color map\n", info->fix.id); - ret = -ENOMEM; - goto out_iounmap; - } - DRM_INFO("fb mappable at 0x%lX\n", info->fix.smem_start); DRM_INFO("vram aper at 0x%lX\n", (unsigned long)info->fix.smem_start); DRM_INFO("size %lu\n", (unsigned long)info->fix.smem_len); @@ -260,24 +244,15 @@ static int cirrusfb_create(struct drm_fb_helper *helper, DRM_INFO(" pitch is %d\n", fb->pitches[0]); return 0; -out_iounmap: - return ret; } static int cirrus_fbdev_destroy(struct drm_device *dev, struct cirrus_fbdev *gfbdev) { - struct fb_info *info; struct cirrus_framebuffer *gfb = &gfbdev->gfb; - if (gfbdev->helper.fbdev) { - info = gfbdev->helper.fbdev; - - unregister_framebuffer(info); - if (info->cmap.len) - fb_dealloc_cmap(&info->cmap); - framebuffer_release(info); - } + drm_fb_helper_unregister_fbi(&gfbdev->helper); + drm_fb_helper_release_fbi(&gfbdev->helper); if (gfb->obj) { drm_gem_object_unreference_unlocked(gfb->obj); diff --git a/drivers/gpu/drm/cirrus/cirrus_main.c b/drivers/gpu/drm/cirrus/cirrus_main.c index e4b976658087..055fd86ba717 100644 --- a/drivers/gpu/drm/cirrus/cirrus_main.c +++ b/drivers/gpu/drm/cirrus/cirrus_main.c @@ -293,25 +293,18 @@ cirrus_dumb_mmap_offset(struct drm_file *file, uint64_t *offset) { struct drm_gem_object *obj; - int ret; struct cirrus_bo *bo; - mutex_lock(&dev->struct_mutex); obj = drm_gem_object_lookup(dev, file, handle); - if (obj == NULL) { - ret = -ENOENT; - goto out_unlock; - } + if (obj == NULL) + return -ENOENT; bo = gem_to_cirrus_bo(obj); *offset = cirrus_bo_mmap_offset(bo); - drm_gem_object_unreference(obj); - ret = 0; -out_unlock: - mutex_unlock(&dev->struct_mutex); - return ret; + drm_gem_object_unreference_unlocked(obj); + return 0; } bool cirrus_check_framebuffer(struct cirrus_device *cdev, int width, int height, diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c index f6f2fb58eb37..f7d5166f89b2 100644 --- a/drivers/gpu/drm/drm_atomic.c +++ b/drivers/gpu/drm/drm_atomic.c @@ -153,9 +153,15 @@ void drm_atomic_state_default_clear(struct drm_atomic_state *state) if (!connector) continue; - WARN_ON(!drm_modeset_is_locked(&config->connection_mutex)); - - connector->funcs->atomic_destroy_state(connector, + /* + * FIXME: Async commits can race with connector unplugging and + * there's currently nothing that prevents cleanup up state for + * deleted connectors. As long as the callback doesn't look at + * the connector we'll be fine though, so make sure that's the + * case by setting all connector pointers to NULL. + */ + state->connector_states[i]->connector = NULL; + connector->funcs->atomic_destroy_state(NULL, state->connector_states[i]); state->connectors[i] = NULL; state->connector_states[i] = NULL; @@ -1063,7 +1069,7 @@ drm_atomic_add_affected_connectors(struct drm_atomic_state *state, * Changed connectors are already in @state, so only need to look at the * current configuration. */ - list_for_each_entry(connector, &config->connector_list, head) { + drm_for_each_connector(connector, state->dev) { if (connector->state->crtc != crtc) continue; @@ -1463,24 +1469,18 @@ int drm_mode_atomic_ioctl(struct drm_device *dev, if (get_user(obj_id, objs_ptr + copied_objs)) { ret = -EFAULT; - goto fail; + goto out; } obj = drm_mode_object_find(dev, obj_id, DRM_MODE_OBJECT_ANY); if (!obj || !obj->properties) { ret = -ENOENT; - goto fail; - } - - if (obj->type == DRM_MODE_OBJECT_PLANE) { - plane = obj_to_plane(obj); - plane_mask |= (1 << drm_plane_index(plane)); - plane->old_fb = plane->fb; + goto out; } if (get_user(count_props, count_props_ptr + copied_objs)) { ret = -EFAULT; - goto fail; + goto out; } copied_objs++; @@ -1492,28 +1492,35 @@ int drm_mode_atomic_ioctl(struct drm_device *dev, if (get_user(prop_id, props_ptr + copied_props)) { ret = -EFAULT; - goto fail; + goto out; } prop = drm_property_find(dev, prop_id); if (!prop) { ret = -ENOENT; - goto fail; + goto out; } if (copy_from_user(&prop_value, prop_values_ptr + copied_props, sizeof(prop_value))) { ret = -EFAULT; - goto fail; + goto out; } ret = atomic_set_prop(state, obj, prop, prop_value); if (ret) - goto fail; + goto out; copied_props++; } + + if (obj->type == DRM_MODE_OBJECT_PLANE && count_props && + !(arg->flags & DRM_MODE_ATOMIC_TEST_ONLY)) { + plane = obj_to_plane(obj); + plane_mask |= (1 << drm_plane_index(plane)); + plane->old_fb = plane->fb; + } } if (arg->flags & DRM_MODE_PAGE_FLIP_EVENT) { @@ -1523,7 +1530,7 @@ int drm_mode_atomic_ioctl(struct drm_device *dev, e = create_vblank_event(dev, file_priv, arg->user_data); if (!e) { ret = -ENOMEM; - goto fail; + goto out; } crtc_state->event = e; @@ -1531,15 +1538,18 @@ int drm_mode_atomic_ioctl(struct drm_device *dev, } if (arg->flags & DRM_MODE_ATOMIC_TEST_ONLY) { + /* + * Unlike commit, check_only does not clean up state. + * Below we call drm_atomic_state_free for it. + */ ret = drm_atomic_check_only(state); - /* _check_only() does not free state, unlike _commit() */ - drm_atomic_state_free(state); } else if (arg->flags & DRM_MODE_ATOMIC_NONBLOCK) { ret = drm_atomic_async_commit(state); } else { ret = drm_atomic_commit(state); } +out: /* if succeeded, fixup legacy plane crtc/fb ptrs before dropping * locks (ie. while it is still safe to deref plane->state). We * need to do this here because the driver entry points cannot @@ -1552,41 +1562,40 @@ int drm_mode_atomic_ioctl(struct drm_device *dev, drm_framebuffer_reference(new_fb); plane->fb = new_fb; plane->crtc = plane->state->crtc; - } else { - plane->old_fb = NULL; - } - if (plane->old_fb) { - drm_framebuffer_unreference(plane->old_fb); - plane->old_fb = NULL; + + if (plane->old_fb) + drm_framebuffer_unreference(plane->old_fb); } + plane->old_fb = NULL; } - drm_modeset_drop_locks(&ctx); - drm_modeset_acquire_fini(&ctx); + if (ret && arg->flags & DRM_MODE_PAGE_FLIP_EVENT) { + /* + * TEST_ONLY and PAGE_FLIP_EVENT are mutually exclusive, + * if they weren't, this code should be called on success + * for TEST_ONLY too. + */ - return ret; - -fail: - if (ret == -EDEADLK) - goto backoff; - - if (arg->flags & DRM_MODE_PAGE_FLIP_EVENT) { for_each_crtc_in_state(state, crtc, crtc_state, i) { - destroy_vblank_event(dev, file_priv, crtc_state->event); - crtc_state->event = NULL; + if (!crtc_state->event) + continue; + + destroy_vblank_event(dev, file_priv, + crtc_state->event); } } - drm_atomic_state_free(state); + if (ret == -EDEADLK) { + drm_atomic_state_clear(state); + drm_modeset_backoff(&ctx); + goto retry; + } + + if (ret || arg->flags & DRM_MODE_ATOMIC_TEST_ONLY) + drm_atomic_state_free(state); drm_modeset_drop_locks(&ctx); drm_modeset_acquire_fini(&ctx); return ret; - -backoff: - drm_atomic_state_clear(state); - drm_modeset_backoff(&ctx); - - goto retry; } diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c index 9dcc7280e572..aecb5d69bc2d 100644 --- a/drivers/gpu/drm/drm_atomic_helper.c +++ b/drivers/gpu/drm/drm_atomic_helper.c @@ -89,7 +89,7 @@ get_current_crtc_for_encoder(struct drm_device *dev, WARN_ON(!drm_modeset_is_locked(&config->connection_mutex)); - list_for_each_entry(connector, &config->connector_list, head) { + drm_for_each_connector(connector, dev) { if (connector->state->best_encoder != encoder) continue; @@ -124,7 +124,7 @@ steal_encoder(struct drm_atomic_state *state, if (IS_ERR(crtc_state)) return PTR_ERR(crtc_state); - crtc_state->mode_changed = true; + crtc_state->connectors_changed = true; list_for_each_entry(connector, &config->connector_list, head) { if (connector->state->best_encoder != encoder) @@ -174,14 +174,14 @@ update_connector_routing(struct drm_atomic_state *state, int conn_idx) idx = drm_crtc_index(connector->state->crtc); crtc_state = state->crtc_states[idx]; - crtc_state->mode_changed = true; + crtc_state->connectors_changed = true; } if (connector_state->crtc) { idx = drm_crtc_index(connector_state->crtc); crtc_state = state->crtc_states[idx]; - crtc_state->mode_changed = true; + crtc_state->connectors_changed = true; } } @@ -241,7 +241,7 @@ update_connector_routing(struct drm_atomic_state *state, int conn_idx) idx = drm_crtc_index(connector_state->crtc); crtc_state = state->crtc_states[idx]; - crtc_state->mode_changed = true; + crtc_state->connectors_changed = true; DRM_DEBUG_ATOMIC("[CONNECTOR:%d:%s] using [ENCODER:%d:%s] on [CRTC:%d]\n", connector->base.id, @@ -264,7 +264,8 @@ mode_fixup(struct drm_atomic_state *state) bool ret; for_each_crtc_in_state(state, crtc, crtc_state, i) { - if (!crtc_state->mode_changed) + if (!crtc_state->mode_changed && + !crtc_state->connectors_changed) continue; drm_mode_copy(&crtc_state->adjusted_mode, &crtc_state->mode); @@ -306,7 +307,7 @@ mode_fixup(struct drm_atomic_state *state) encoder->base.id, encoder->name); return ret; } - } else { + } else if (funcs->mode_fixup) { ret = funcs->mode_fixup(encoder, &crtc_state->mode, &crtc_state->adjusted_mode); if (!ret) { @@ -320,7 +321,8 @@ mode_fixup(struct drm_atomic_state *state) for_each_crtc_in_state(state, crtc, crtc_state, i) { const struct drm_crtc_helper_funcs *funcs; - if (!crtc_state->mode_changed) + if (!crtc_state->mode_changed && + !crtc_state->connectors_changed) continue; funcs = crtc->helper_private; @@ -346,9 +348,14 @@ mode_fixup(struct drm_atomic_state *state) * * Check the state object to see if the requested state is physically possible. * This does all the crtc and connector related computations for an atomic - * update. It computes and updates crtc_state->mode_changed, adds any additional - * connectors needed for full modesets and calls down into ->mode_fixup - * functions of the driver backend. + * update and adds any additional connectors needed for full modesets and calls + * down into ->mode_fixup functions of the driver backend. + * + * crtc_state->mode_changed is set when the input mode is changed. + * crtc_state->connectors_changed is set when a connector is added or + * removed from the crtc. + * crtc_state->active_changed is set when crtc_state->active changes, + * which is used for dpms. * * IMPORTANT: * @@ -381,7 +388,17 @@ drm_atomic_helper_check_modeset(struct drm_device *dev, if (crtc->state->enable != crtc_state->enable) { DRM_DEBUG_ATOMIC("[CRTC:%d] enable changed\n", crtc->base.id); + + /* + * For clarity this assignment is done here, but + * enable == 0 is only true when there are no + * connectors and a NULL mode. + * + * The other way around is true as well. enable != 0 + * iff connectors are attached and a mode is set. + */ crtc_state->mode_changed = true; + crtc_state->connectors_changed = true; } } @@ -456,6 +473,9 @@ EXPORT_SYMBOL(drm_atomic_helper_check_modeset); * This does all the plane update related checks using by calling into the * ->atomic_check hooks provided by the driver. * + * It also sets crtc_state->planes_changed to indicate that a crtc has + * updated planes. + * * RETURNS * Zero for success or -errno */ @@ -648,15 +668,29 @@ drm_atomic_helper_update_legacy_modeset_state(struct drm_device *dev, struct drm_crtc_state *old_crtc_state; int i; - /* clear out existing links */ + /* clear out existing links and update dpms */ for_each_connector_in_state(old_state, connector, old_conn_state, i) { - if (!connector->encoder) - continue; + if (connector->encoder) { + WARN_ON(!connector->encoder->crtc); - WARN_ON(!connector->encoder->crtc); + connector->encoder->crtc = NULL; + connector->encoder = NULL; + } - connector->encoder->crtc = NULL; - connector->encoder = NULL; + crtc = connector->state->crtc; + if ((!crtc && old_conn_state->crtc) || + (crtc && drm_atomic_crtc_needs_modeset(crtc->state))) { + struct drm_property *dpms_prop = + dev->mode_config.dpms_property; + int mode = DRM_MODE_DPMS_OFF; + + if (crtc && crtc->state->active) + mode = DRM_MODE_DPMS_ON; + + connector->dpms = mode; + drm_object_property_set_value(&connector->base, + dpms_prop, mode); + } } /* set new links */ @@ -673,10 +707,16 @@ drm_atomic_helper_update_legacy_modeset_state(struct drm_device *dev, /* set legacy state in the crtc structure */ for_each_crtc_in_state(old_state, crtc, old_crtc_state, i) { + struct drm_plane *primary = crtc->primary; + crtc->mode = crtc->state->mode; crtc->enabled = crtc->state->enable; - crtc->x = crtc->primary->state->src_x >> 16; - crtc->y = crtc->primary->state->src_y >> 16; + + if (drm_atomic_get_existing_plane_state(old_state, primary) && + primary->state->crtc == crtc) { + crtc->x = primary->state->src_x >> 16; + crtc->y = primary->state->src_y >> 16; + } if (crtc->state->enable) drm_calc_timestamping_constants(crtc, @@ -750,7 +790,7 @@ crtc_set_mode(struct drm_device *dev, struct drm_atomic_state *old_state) * This function shuts down all the outputs that need to be shut down and * prepares them (if required) with the new mode. * - * For compatability with legacy crtc helpers this should be called before + * For compatibility with legacy crtc helpers this should be called before * drm_atomic_helper_commit_planes(), which is what the default commit function * does. But drivers with different needs can group the modeset commits together * and do the plane commits at the end. This is useful for drivers doing runtime @@ -775,7 +815,7 @@ EXPORT_SYMBOL(drm_atomic_helper_commit_modeset_disables); * This function enables all the outputs with the new configuration which had to * be turned off for the update. * - * For compatability with legacy crtc helpers this should be called after + * For compatibility with legacy crtc helpers this should be called after * drm_atomic_helper_commit_planes(), which is what the default commit function * does. But drivers with different needs can group the modeset commits together * and do the plane commits at the end. This is useful for drivers doing runtime @@ -926,7 +966,7 @@ drm_atomic_helper_wait_for_vblanks(struct drm_device *dev, continue; old_crtc_state->enable = true; - old_crtc_state->last_vblank_count = drm_vblank_count(dev, i); + old_crtc_state->last_vblank_count = drm_crtc_vblank_count(crtc); } for_each_crtc_in_state(old_state, crtc, old_crtc_state, i) { @@ -935,7 +975,7 @@ drm_atomic_helper_wait_for_vblanks(struct drm_device *dev, ret = wait_event_timeout(dev->vblank[i].queue, old_crtc_state->last_vblank_count != - drm_vblank_count(dev, i), + drm_crtc_vblank_count(crtc), msecs_to_jiffies(50)); drm_crtc_vblank_put(crtc); @@ -1146,7 +1186,7 @@ void drm_atomic_helper_commit_planes(struct drm_device *dev, if (!funcs || !funcs->atomic_begin) continue; - funcs->atomic_begin(crtc); + funcs->atomic_begin(crtc, old_crtc_state); } for_each_plane_in_state(old_state, plane, old_plane_state, i) { @@ -1176,7 +1216,7 @@ void drm_atomic_helper_commit_planes(struct drm_device *dev, if (!funcs || !funcs->atomic_flush) continue; - funcs->atomic_flush(crtc); + funcs->atomic_flush(crtc, old_crtc_state); } } EXPORT_SYMBOL(drm_atomic_helper_commit_planes); @@ -1212,7 +1252,7 @@ drm_atomic_helper_commit_planes_on_crtc(struct drm_crtc_state *old_crtc_state) crtc_funcs = crtc->helper_private; if (crtc_funcs && crtc_funcs->atomic_begin) - crtc_funcs->atomic_begin(crtc); + crtc_funcs->atomic_begin(crtc, old_crtc_state); drm_for_each_plane_mask(plane, crtc->dev, plane_mask) { struct drm_plane_state *old_plane_state = @@ -1235,7 +1275,7 @@ drm_atomic_helper_commit_planes_on_crtc(struct drm_crtc_state *old_crtc_state) } if (crtc_funcs && crtc_funcs->atomic_flush) - crtc_funcs->atomic_flush(crtc); + crtc_funcs->atomic_flush(crtc, old_crtc_state); } EXPORT_SYMBOL(drm_atomic_helper_commit_planes_on_crtc); @@ -1923,10 +1963,6 @@ int drm_atomic_helper_page_flip(struct drm_crtc *crtc, if (ret != 0) goto fail; - /* TODO: ->page_flip is the only driver callback where the core - * doesn't update plane->fb. For now patch it up here. */ - plane->fb = plane->state->fb; - /* Driver takes ownership of state on successful async commit. */ return 0; fail: @@ -1960,9 +1996,12 @@ EXPORT_SYMBOL(drm_atomic_helper_page_flip); * implementing the legacy DPMS connector interface. It computes the new desired * ->active state for the corresponding CRTC (if the connector is enabled) and * updates it. + * + * Returns: + * Returns 0 on success, negative errno numbers on failure. */ -void drm_atomic_helper_connector_dpms(struct drm_connector *connector, - int mode) +int drm_atomic_helper_connector_dpms(struct drm_connector *connector, + int mode) { struct drm_mode_config *config = &connector->dev->mode_config; struct drm_atomic_state *state; @@ -1971,6 +2010,7 @@ void drm_atomic_helper_connector_dpms(struct drm_connector *connector, struct drm_connector *tmp_connector; int ret; bool active = false; + int old_mode = connector->dpms; if (mode != DRM_MODE_DPMS_ON) mode = DRM_MODE_DPMS_OFF; @@ -1979,22 +2019,23 @@ void drm_atomic_helper_connector_dpms(struct drm_connector *connector, crtc = connector->state->crtc; if (!crtc) - return; + return 0; - /* FIXME: ->dpms has no return value so can't forward the -ENOMEM. */ state = drm_atomic_state_alloc(connector->dev); if (!state) - return; + return -ENOMEM; state->acquire_ctx = drm_modeset_legacy_acquire_ctx(crtc); retry: crtc_state = drm_atomic_get_crtc_state(state, crtc); - if (IS_ERR(crtc_state)) - return; + if (IS_ERR(crtc_state)) { + ret = PTR_ERR(crtc_state); + goto fail; + } WARN_ON(!drm_modeset_is_locked(&config->connection_mutex)); - list_for_each_entry(tmp_connector, &config->connector_list, head) { + drm_for_each_connector(tmp_connector, connector->dev) { if (tmp_connector->state->crtc != crtc) continue; @@ -2009,17 +2050,16 @@ void drm_atomic_helper_connector_dpms(struct drm_connector *connector, if (ret != 0) goto fail; - /* Driver takes ownership of state on successful async commit. */ - return; + /* Driver takes ownership of state on successful commit. */ + return 0; fail: if (ret == -EDEADLK) goto backoff; + connector->dpms = old_mode; drm_atomic_state_free(state); - WARN(1, "Driver bug: Changing ->active failed with ret=%i\n", ret); - - return; + return ret; backoff: drm_atomic_state_clear(state); drm_atomic_legacy_backoff(state); @@ -2080,6 +2120,7 @@ void __drm_atomic_helper_crtc_duplicate_state(struct drm_crtc *crtc, state->mode_changed = false; state->active_changed = false; state->planes_changed = false; + state->connectors_changed = false; state->event = NULL; } EXPORT_SYMBOL(__drm_atomic_helper_crtc_duplicate_state); diff --git a/drivers/gpu/drm/drm_context.c b/drivers/gpu/drm/drm_context.c index 9b23525c0ed0..192a5f9eeb74 100644 --- a/drivers/gpu/drm/drm_context.c +++ b/drivers/gpu/drm/drm_context.c @@ -53,6 +53,10 @@ struct drm_ctx_list { */ void drm_legacy_ctxbitmap_free(struct drm_device * dev, int ctx_handle) { + if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) && + drm_core_check_feature(dev, DRIVER_MODESET)) + return; + mutex_lock(&dev->struct_mutex); idr_remove(&dev->ctx_idr, ctx_handle); mutex_unlock(&dev->struct_mutex); @@ -85,10 +89,13 @@ static int drm_legacy_ctxbitmap_next(struct drm_device * dev) * * Initialise the drm_device::ctx_idr */ -int drm_legacy_ctxbitmap_init(struct drm_device * dev) +void drm_legacy_ctxbitmap_init(struct drm_device * dev) { + if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) && + drm_core_check_feature(dev, DRIVER_MODESET)) + return; + idr_init(&dev->ctx_idr); - return 0; } /** @@ -101,6 +108,10 @@ int drm_legacy_ctxbitmap_init(struct drm_device * dev) */ void drm_legacy_ctxbitmap_cleanup(struct drm_device * dev) { + if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) && + drm_core_check_feature(dev, DRIVER_MODESET)) + return; + mutex_lock(&dev->struct_mutex); idr_destroy(&dev->ctx_idr); mutex_unlock(&dev->struct_mutex); @@ -119,6 +130,10 @@ void drm_legacy_ctxbitmap_flush(struct drm_device *dev, struct drm_file *file) { struct drm_ctx_list *pos, *tmp; + if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) && + drm_core_check_feature(dev, DRIVER_MODESET)) + return; + mutex_lock(&dev->ctxlist_mutex); list_for_each_entry_safe(pos, tmp, &dev->ctxlist, head) { @@ -161,6 +176,10 @@ int drm_legacy_getsareactx(struct drm_device *dev, void *data, struct drm_local_map *map; struct drm_map_list *_entry; + if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) && + drm_core_check_feature(dev, DRIVER_MODESET)) + return -EINVAL; + mutex_lock(&dev->struct_mutex); map = idr_find(&dev->ctx_idr, request->ctx_id); @@ -205,6 +224,10 @@ int drm_legacy_setsareactx(struct drm_device *dev, void *data, struct drm_local_map *map = NULL; struct drm_map_list *r_list = NULL; + if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) && + drm_core_check_feature(dev, DRIVER_MODESET)) + return -EINVAL; + mutex_lock(&dev->struct_mutex); list_for_each_entry(r_list, &dev->maplist, head) { if (r_list->map @@ -305,6 +328,10 @@ int drm_legacy_resctx(struct drm_device *dev, void *data, struct drm_ctx ctx; int i; + if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) && + drm_core_check_feature(dev, DRIVER_MODESET)) + return -EINVAL; + if (res->count >= DRM_RESERVED_CONTEXTS) { memset(&ctx, 0, sizeof(ctx)); for (i = 0; i < DRM_RESERVED_CONTEXTS; i++) { @@ -335,6 +362,10 @@ int drm_legacy_addctx(struct drm_device *dev, void *data, struct drm_ctx_list *ctx_entry; struct drm_ctx *ctx = data; + if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) && + drm_core_check_feature(dev, DRIVER_MODESET)) + return -EINVAL; + ctx->handle = drm_legacy_ctxbitmap_next(dev); if (ctx->handle == DRM_KERNEL_CONTEXT) { /* Skip kernel's context and get a new one. */ @@ -378,6 +409,10 @@ int drm_legacy_getctx(struct drm_device *dev, void *data, { struct drm_ctx *ctx = data; + if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) && + drm_core_check_feature(dev, DRIVER_MODESET)) + return -EINVAL; + /* This is 0, because we don't handle any context flags */ ctx->flags = 0; @@ -400,6 +435,10 @@ int drm_legacy_switchctx(struct drm_device *dev, void *data, { struct drm_ctx *ctx = data; + if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) && + drm_core_check_feature(dev, DRIVER_MODESET)) + return -EINVAL; + DRM_DEBUG("%d\n", ctx->handle); return drm_context_switch(dev, dev->last_context, ctx->handle); } @@ -420,6 +459,10 @@ int drm_legacy_newctx(struct drm_device *dev, void *data, { struct drm_ctx *ctx = data; + if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) && + drm_core_check_feature(dev, DRIVER_MODESET)) + return -EINVAL; + DRM_DEBUG("%d\n", ctx->handle); drm_context_switch_complete(dev, file_priv, ctx->handle); @@ -442,6 +485,10 @@ int drm_legacy_rmctx(struct drm_device *dev, void *data, { struct drm_ctx *ctx = data; + if (!drm_core_check_feature(dev, DRIVER_KMS_LEGACY_CONTEXT) && + drm_core_check_feature(dev, DRIVER_MODESET)) + return -EINVAL; + DRM_DEBUG("%d\n", ctx->handle); if (ctx->handle != DRM_KERNEL_CONTEXT) { if (dev->driver->context_dtor) diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c index fed748311b92..33d877c65ced 100644 --- a/drivers/gpu/drm/drm_crtc.c +++ b/drivers/gpu/drm/drm_crtc.c @@ -615,7 +615,7 @@ void drm_framebuffer_remove(struct drm_framebuffer *fb) if (atomic_read(&fb->refcount.refcount) > 1) { drm_modeset_lock_all(dev); /* remove from any CRTC */ - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + drm_for_each_crtc(crtc, dev) { if (crtc->primary->fb == fb) { /* should turn off the crtc */ memset(&set, 0, sizeof(struct drm_mode_set)); @@ -627,7 +627,7 @@ void drm_framebuffer_remove(struct drm_framebuffer *fb) } } - list_for_each_entry(plane, &dev->mode_config.plane_list, head) { + drm_for_each_plane(plane, dev) { if (plane->fb == fb) drm_plane_force_disable(plane); } @@ -736,7 +736,7 @@ unsigned int drm_crtc_index(struct drm_crtc *crtc) unsigned int index = 0; struct drm_crtc *tmp; - list_for_each_entry(tmp, &crtc->dev->mode_config.crtc_list, head) { + drm_for_each_crtc(tmp, crtc->dev) { if (tmp == crtc) return index; @@ -988,7 +988,7 @@ unsigned int drm_connector_index(struct drm_connector *connector) WARN_ON(!drm_modeset_is_locked(&config->connection_mutex)); - list_for_each_entry(tmp, &connector->dev->mode_config.connector_list, head) { + drm_for_each_connector(tmp, connector->dev) { if (tmp == connector) return index; @@ -1054,7 +1054,7 @@ void drm_connector_unplug_all(struct drm_device *dev) { struct drm_connector *connector; - /* taking the mode config mutex ends up in a clash with sysfs */ + /* FIXME: taking the mode config mutex ends up in a clash with sysfs */ list_for_each_entry(connector, &dev->mode_config.connector_list, head) drm_connector_unregister(connector); @@ -1151,7 +1151,7 @@ EXPORT_SYMBOL(drm_encoder_cleanup); int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane, unsigned long possible_crtcs, const struct drm_plane_funcs *funcs, - const uint32_t *formats, uint32_t format_count, + const uint32_t *formats, unsigned int format_count, enum drm_plane_type type) { struct drm_mode_config *config = &dev->mode_config; @@ -1225,7 +1225,7 @@ EXPORT_SYMBOL(drm_universal_plane_init); int drm_plane_init(struct drm_device *dev, struct drm_plane *plane, unsigned long possible_crtcs, const struct drm_plane_funcs *funcs, - const uint32_t *formats, uint32_t format_count, + const uint32_t *formats, unsigned int format_count, bool is_primary) { enum drm_plane_type type; @@ -1280,7 +1280,7 @@ unsigned int drm_plane_index(struct drm_plane *plane) unsigned int index = 0; struct drm_plane *tmp; - list_for_each_entry(tmp, &plane->dev->mode_config.plane_list, head) { + drm_for_each_plane(tmp, plane->dev) { if (tmp == plane) return index; @@ -1305,7 +1305,7 @@ drm_plane_from_index(struct drm_device *dev, int idx) struct drm_plane *plane; unsigned int i = 0; - list_for_each_entry(plane, &dev->mode_config.plane_list, head) { + drm_for_each_plane(plane, dev) { if (i == idx) return plane; i++; @@ -1679,70 +1679,6 @@ int drm_mode_create_suggested_offset_properties(struct drm_device *dev) } EXPORT_SYMBOL(drm_mode_create_suggested_offset_properties); -static int drm_mode_group_init(struct drm_device *dev, struct drm_mode_group *group) -{ - uint32_t total_objects = 0; - - total_objects += dev->mode_config.num_crtc; - total_objects += dev->mode_config.num_connector; - total_objects += dev->mode_config.num_encoder; - - group->id_list = kcalloc(total_objects, sizeof(uint32_t), GFP_KERNEL); - if (!group->id_list) - return -ENOMEM; - - group->num_crtcs = 0; - group->num_connectors = 0; - group->num_encoders = 0; - return 0; -} - -void drm_mode_group_destroy(struct drm_mode_group *group) -{ - kfree(group->id_list); - group->id_list = NULL; -} - -/* - * NOTE: Driver's shouldn't ever call drm_mode_group_init_legacy_group - it is - * the drm core's responsibility to set up mode control groups. - */ -int drm_mode_group_init_legacy_group(struct drm_device *dev, - struct drm_mode_group *group) -{ - struct drm_crtc *crtc; - struct drm_encoder *encoder; - struct drm_connector *connector; - int ret; - - ret = drm_mode_group_init(dev, group); - if (ret) - return ret; - - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) - group->id_list[group->num_crtcs++] = crtc->base.id; - - list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) - group->id_list[group->num_crtcs + group->num_encoders++] = - encoder->base.id; - - list_for_each_entry(connector, &dev->mode_config.connector_list, head) - group->id_list[group->num_crtcs + group->num_encoders + - group->num_connectors++] = connector->base.id; - - return 0; -} -EXPORT_SYMBOL(drm_mode_group_init_legacy_group); - -void drm_reinit_primary_mode_group(struct drm_device *dev) -{ - drm_modeset_lock_all(dev); - drm_mode_group_destroy(&dev->primary->mode_group); - drm_mode_group_init_legacy_group(dev, &dev->primary->mode_group); - drm_modeset_unlock_all(dev); -} -EXPORT_SYMBOL(drm_reinit_primary_mode_group); - /** * drm_mode_getresources - get graphics configuration * @dev: drm device for the ioctl @@ -1771,12 +1707,11 @@ int drm_mode_getresources(struct drm_device *dev, void *data, int crtc_count = 0; int fb_count = 0; int encoder_count = 0; - int copied = 0, i; + int copied = 0; uint32_t __user *fb_id; uint32_t __user *crtc_id; uint32_t __user *connector_id; uint32_t __user *encoder_id; - struct drm_mode_group *mode_group; if (!drm_core_check_feature(dev, DRIVER_MODESET)) return -EINVAL; @@ -1809,24 +1744,14 @@ int drm_mode_getresources(struct drm_device *dev, void *data, /* mode_config.mutex protects the connector list against e.g. DP MST * connector hot-adding. CRTC/Plane lists are invariant. */ mutex_lock(&dev->mode_config.mutex); - if (!drm_is_primary_client(file_priv)) { + drm_for_each_crtc(crtc, dev) + crtc_count++; - mode_group = NULL; - list_for_each(lh, &dev->mode_config.crtc_list) - crtc_count++; + drm_for_each_connector(connector, dev) + connector_count++; - list_for_each(lh, &dev->mode_config.connector_list) - connector_count++; - - list_for_each(lh, &dev->mode_config.encoder_list) - encoder_count++; - } else { - - mode_group = &file_priv->master->minor->mode_group; - crtc_count = mode_group->num_crtcs; - connector_count = mode_group->num_connectors; - encoder_count = mode_group->num_encoders; - } + drm_for_each_encoder(encoder, dev) + encoder_count++; card_res->max_height = dev->mode_config.max_height; card_res->min_height = dev->mode_config.min_height; @@ -1837,25 +1762,13 @@ int drm_mode_getresources(struct drm_device *dev, void *data, if (card_res->count_crtcs >= crtc_count) { copied = 0; crtc_id = (uint32_t __user *)(unsigned long)card_res->crtc_id_ptr; - if (!mode_group) { - list_for_each_entry(crtc, &dev->mode_config.crtc_list, - head) { - DRM_DEBUG_KMS("[CRTC:%d]\n", crtc->base.id); - if (put_user(crtc->base.id, crtc_id + copied)) { - ret = -EFAULT; - goto out; - } - copied++; - } - } else { - for (i = 0; i < mode_group->num_crtcs; i++) { - if (put_user(mode_group->id_list[i], - crtc_id + copied)) { - ret = -EFAULT; - goto out; - } - copied++; + drm_for_each_crtc(crtc, dev) { + DRM_DEBUG_KMS("[CRTC:%d]\n", crtc->base.id); + if (put_user(crtc->base.id, crtc_id + copied)) { + ret = -EFAULT; + goto out; } + copied++; } } card_res->count_crtcs = crtc_count; @@ -1864,29 +1777,15 @@ int drm_mode_getresources(struct drm_device *dev, void *data, if (card_res->count_encoders >= encoder_count) { copied = 0; encoder_id = (uint32_t __user *)(unsigned long)card_res->encoder_id_ptr; - if (!mode_group) { - list_for_each_entry(encoder, - &dev->mode_config.encoder_list, - head) { - DRM_DEBUG_KMS("[ENCODER:%d:%s]\n", encoder->base.id, - encoder->name); - if (put_user(encoder->base.id, encoder_id + - copied)) { - ret = -EFAULT; - goto out; - } - copied++; + drm_for_each_encoder(encoder, dev) { + DRM_DEBUG_KMS("[ENCODER:%d:%s]\n", encoder->base.id, + encoder->name); + if (put_user(encoder->base.id, encoder_id + + copied)) { + ret = -EFAULT; + goto out; } - } else { - for (i = mode_group->num_crtcs; i < mode_group->num_crtcs + mode_group->num_encoders; i++) { - if (put_user(mode_group->id_list[i], - encoder_id + copied)) { - ret = -EFAULT; - goto out; - } - copied++; - } - + copied++; } } card_res->count_encoders = encoder_count; @@ -1895,31 +1794,16 @@ int drm_mode_getresources(struct drm_device *dev, void *data, if (card_res->count_connectors >= connector_count) { copied = 0; connector_id = (uint32_t __user *)(unsigned long)card_res->connector_id_ptr; - if (!mode_group) { - list_for_each_entry(connector, - &dev->mode_config.connector_list, - head) { - DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", - connector->base.id, - connector->name); - if (put_user(connector->base.id, - connector_id + copied)) { - ret = -EFAULT; - goto out; - } - copied++; - } - } else { - int start = mode_group->num_crtcs + - mode_group->num_encoders; - for (i = start; i < start + mode_group->num_connectors; i++) { - if (put_user(mode_group->id_list[i], - connector_id + copied)) { - ret = -EFAULT; - goto out; - } - copied++; + drm_for_each_connector(connector, dev) { + DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", + connector->base.id, + connector->name); + if (put_user(connector->base.id, + connector_id + copied)) { + ret = -EFAULT; + goto out; } + copied++; } } card_res->count_connectors = connector_count; @@ -2187,7 +2071,7 @@ static struct drm_crtc *drm_encoder_get_crtc(struct drm_encoder *encoder) /* For atomic drivers only state objects are synchronously updated and * protected by modeset locks, so check those first. */ - list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + drm_for_each_connector(connector, dev) { if (!connector->state) continue; @@ -2291,7 +2175,7 @@ int drm_mode_getplane_res(struct drm_device *dev, void *data, plane_ptr = (uint32_t __user *)(unsigned long)plane_resp->plane_id_ptr; /* Plane lists are invariant, no locking needed. */ - list_for_each_entry(plane, &config->plane_list, head) { + drm_for_each_plane(plane, dev) { /* * Unless userspace set the 'universal planes' * capability bit, only advertise overlays. @@ -2596,7 +2480,7 @@ int drm_mode_set_config_internal(struct drm_mode_set *set) * connectors from it), hence we need to refcount the fbs across all * crtcs. Atomic modeset will have saner semantics ... */ - list_for_each_entry(tmp, &crtc->dev->mode_config.crtc_list, head) + drm_for_each_crtc(tmp, crtc->dev) tmp->primary->old_fb = tmp->primary->fb; fb = set->fb; @@ -2607,7 +2491,7 @@ int drm_mode_set_config_internal(struct drm_mode_set *set) crtc->primary->fb = fb; } - list_for_each_entry(tmp, &crtc->dev->mode_config.crtc_list, head) { + drm_for_each_crtc(tmp, crtc->dev) { if (tmp->primary->fb) drm_framebuffer_reference(tmp->primary->fb); if (tmp->primary->old_fb) @@ -4301,7 +4185,6 @@ void drm_property_unreference_blob(struct drm_property_blob *blob) mutex_unlock(&dev->mode_config.blob_lock); else might_lock(&dev->mode_config.blob_lock); - } EXPORT_SYMBOL(drm_property_unreference_blob); @@ -4472,9 +4355,7 @@ static int drm_property_replace_global_blob(struct drm_device *dev, goto err_created; } - if (old_blob) - drm_property_unreference_blob(old_blob); - + drm_property_unreference_blob(old_blob); *replace = new_blob; return 0; @@ -4872,9 +4753,9 @@ static int drm_mode_connector_set_obj_prop(struct drm_mode_object *obj, /* Do DPMS ourselves */ if (property == connector->dev->mode_config.dpms_property) { - if (connector->funcs->dpms) - (*connector->funcs->dpms)(connector, (int)value); ret = 0; + if (connector->funcs->dpms) + ret = (*connector->funcs->dpms)(connector, (int)value); } else if (connector->funcs->set_property) ret = connector->funcs->set_property(connector, property, value); @@ -5349,13 +5230,7 @@ int drm_mode_page_flip_ioctl(struct drm_device *dev, /* Keep the old fb, don't unref it. */ crtc->primary->old_fb = NULL; } else { - /* - * Warn if the driver hasn't properly updated the crtc->fb - * field to reflect that the new framebuffer is now used. - * Failing to do so will screw with the reference counting - * on framebuffers. - */ - WARN_ON(crtc->primary->fb != fb); + crtc->primary->fb = fb; /* Unref only the old framebuffer. */ fb = NULL; } @@ -5386,21 +5261,23 @@ void drm_mode_config_reset(struct drm_device *dev) struct drm_encoder *encoder; struct drm_connector *connector; - list_for_each_entry(plane, &dev->mode_config.plane_list, head) + drm_for_each_plane(plane, dev) if (plane->funcs->reset) plane->funcs->reset(plane); - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) + drm_for_each_crtc(crtc, dev) if (crtc->funcs->reset) crtc->funcs->reset(crtc); - list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) + drm_for_each_encoder(encoder, dev) if (encoder->funcs->reset) encoder->funcs->reset(encoder); - list_for_each_entry(connector, &dev->mode_config.connector_list, head) + mutex_lock(&dev->mode_config.mutex); + drm_for_each_connector(connector, dev) if (connector->funcs->reset) connector->funcs->reset(connector); + mutex_unlock(&dev->mode_config.mutex); } EXPORT_SYMBOL(drm_mode_config_reset); diff --git a/drivers/gpu/drm/drm_crtc_helper.c b/drivers/gpu/drm/drm_crtc_helper.c index 393114df88a3..ef534758a02c 100644 --- a/drivers/gpu/drm/drm_crtc_helper.c +++ b/drivers/gpu/drm/drm_crtc_helper.c @@ -121,7 +121,7 @@ bool drm_helper_encoder_in_use(struct drm_encoder *encoder) WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex)); } - list_for_each_entry(connector, &dev->mode_config.connector_list, head) + drm_for_each_connector(connector, dev) if (connector->encoder == encoder) return true; return false; @@ -151,7 +151,7 @@ bool drm_helper_crtc_in_use(struct drm_crtc *crtc) if (!oops_in_progress) WARN_ON(!mutex_is_locked(&dev->mode_config.mutex)); - list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) + drm_for_each_encoder(encoder, dev) if (encoder->crtc == crtc && drm_helper_encoder_in_use(encoder)) return true; return false; @@ -180,7 +180,7 @@ static void __drm_helper_disable_unused_functions(struct drm_device *dev) drm_warn_on_modeset_not_all_locked(dev); - list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { + drm_for_each_encoder(encoder, dev) { if (!drm_helper_encoder_in_use(encoder)) { drm_encoder_disable(encoder); /* disconnect encoder from any connector */ @@ -188,7 +188,7 @@ static void __drm_helper_disable_unused_functions(struct drm_device *dev) } } - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + drm_for_each_crtc(crtc, dev) { const struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private; crtc->enabled = drm_helper_crtc_in_use(crtc); if (!crtc->enabled) { @@ -230,7 +230,7 @@ drm_crtc_prepare_encoders(struct drm_device *dev) const struct drm_encoder_helper_funcs *encoder_funcs; struct drm_encoder *encoder; - list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { + drm_for_each_encoder(encoder, dev) { encoder_funcs = encoder->helper_private; /* Disable unused encoders */ if (encoder->crtc == NULL) @@ -305,7 +305,7 @@ bool drm_crtc_helper_set_mode(struct drm_crtc *crtc, * adjust it according to limitations or connector properties, and also * a chance to reject the mode entirely. */ - list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { + drm_for_each_encoder(encoder, dev) { if (encoder->crtc != crtc) continue; @@ -334,7 +334,7 @@ bool drm_crtc_helper_set_mode(struct drm_crtc *crtc, crtc->hwmode = *adjusted_mode; /* Prepare the encoders and CRTCs before setting the mode. */ - list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { + drm_for_each_encoder(encoder, dev) { if (encoder->crtc != crtc) continue; @@ -359,7 +359,7 @@ bool drm_crtc_helper_set_mode(struct drm_crtc *crtc, if (!ret) goto done; - list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { + drm_for_each_encoder(encoder, dev) { if (encoder->crtc != crtc) continue; @@ -376,7 +376,7 @@ bool drm_crtc_helper_set_mode(struct drm_crtc *crtc, /* Now enable the clocks, plane, pipe, and connectors that we set up. */ crtc_funcs->commit(crtc); - list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { + drm_for_each_encoder(encoder, dev) { if (encoder->crtc != crtc) continue; @@ -418,11 +418,11 @@ drm_crtc_helper_disable(struct drm_crtc *crtc) struct drm_encoder *encoder; /* Decouple all encoders and their attached connectors from this crtc */ - list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { + drm_for_each_encoder(encoder, dev) { if (encoder->crtc != crtc) continue; - list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + drm_for_each_connector(connector, dev) { if (connector->encoder != encoder) continue; @@ -519,12 +519,12 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set) * restored, not the drivers personal bookkeeping. */ count = 0; - list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { + drm_for_each_encoder(encoder, dev) { save_encoders[count++] = *encoder; } count = 0; - list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + drm_for_each_connector(connector, dev) { save_connectors[count++] = *connector; } @@ -562,7 +562,7 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set) /* a) traverse passed in connector list and get encoders for them */ count = 0; - list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + drm_for_each_connector(connector, dev) { const struct drm_connector_helper_funcs *connector_funcs = connector->helper_private; new_encoder = connector->encoder; @@ -602,7 +602,7 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set) } count = 0; - list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + drm_for_each_connector(connector, dev) { if (!connector->encoder) continue; @@ -685,12 +685,12 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set) fail: /* Restore all previous data. */ count = 0; - list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { + drm_for_each_encoder(encoder, dev) { *encoder = save_encoders[count++]; } count = 0; - list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + drm_for_each_connector(connector, dev) { *connector = save_connectors[count++]; } @@ -712,7 +712,7 @@ static int drm_helper_choose_encoder_dpms(struct drm_encoder *encoder) struct drm_connector *connector; struct drm_device *dev = encoder->dev; - list_for_each_entry(connector, &dev->mode_config.connector_list, head) + drm_for_each_connector(connector, dev) if (connector->encoder == encoder) if (connector->dpms < dpms) dpms = connector->dpms; @@ -746,7 +746,7 @@ static int drm_helper_choose_crtc_dpms(struct drm_crtc *crtc) struct drm_connector *connector; struct drm_device *dev = crtc->dev; - list_for_each_entry(connector, &dev->mode_config.connector_list, head) + drm_for_each_connector(connector, dev) if (connector->encoder && connector->encoder->crtc == crtc) if (connector->dpms < dpms) dpms = connector->dpms; @@ -762,15 +762,18 @@ static int drm_helper_choose_crtc_dpms(struct drm_crtc *crtc) * implementing the DPMS connector attribute. It computes the new desired DPMS * state for all encoders and crtcs in the output mesh and calls the ->dpms() * callback provided by the driver appropriately. + * + * Returns: + * Always returns 0. */ -void drm_helper_connector_dpms(struct drm_connector *connector, int mode) +int drm_helper_connector_dpms(struct drm_connector *connector, int mode) { struct drm_encoder *encoder = connector->encoder; struct drm_crtc *crtc = encoder ? encoder->crtc : NULL; int old_dpms, encoder_dpms = DRM_MODE_DPMS_OFF; if (mode == connector->dpms) - return; + return 0; old_dpms = connector->dpms; connector->dpms = mode; @@ -802,7 +805,7 @@ void drm_helper_connector_dpms(struct drm_connector *connector, int mode) } } - return; + return 0; } EXPORT_SYMBOL(drm_helper_connector_dpms); @@ -862,7 +865,7 @@ void drm_helper_resume_force_mode(struct drm_device *dev) bool ret; drm_modeset_lock_all(dev); - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + drm_for_each_crtc(crtc, dev) { if (!crtc->enabled) continue; @@ -876,7 +879,7 @@ void drm_helper_resume_force_mode(struct drm_device *dev) /* Turn off outputs that were already powered off */ if (drm_helper_choose_crtc_dpms(crtc)) { - list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) { + drm_for_each_encoder(encoder, dev) { if(encoder->crtc != crtc) continue; @@ -928,15 +931,15 @@ int drm_helper_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *mod if (crtc->funcs->atomic_duplicate_state) crtc_state = crtc->funcs->atomic_duplicate_state(crtc); else { - crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL); - if (!crtc_state) - return -ENOMEM; - if (crtc->state) - __drm_atomic_helper_crtc_duplicate_state(crtc, crtc_state); - else - crtc_state->crtc = crtc; + if (!crtc->state) + drm_atomic_helper_crtc_reset(crtc); + + crtc_state = drm_atomic_helper_crtc_duplicate_state(crtc); } + if (!crtc_state) + return -ENOMEM; + crtc_state->planes_changed = true; crtc_state->mode_changed = true; ret = drm_atomic_set_mode_for_crtc(crtc_state, mode); @@ -957,11 +960,11 @@ int drm_helper_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *mod ret = drm_helper_crtc_mode_set_base(crtc, x, y, old_fb); out: - if (crtc->funcs->atomic_destroy_state) - crtc->funcs->atomic_destroy_state(crtc, crtc_state); - else { - __drm_atomic_helper_crtc_destroy_state(crtc, crtc_state); - kfree(crtc_state); + if (crtc_state) { + if (crtc->funcs->atomic_destroy_state) + crtc->funcs->atomic_destroy_state(crtc, crtc_state); + else + drm_atomic_helper_crtc_destroy_state(crtc, crtc_state); } return ret; diff --git a/drivers/gpu/drm/drm_dp_helper.c b/drivers/gpu/drm/drm_dp_helper.c index 80a02a412607..291734e87fca 100644 --- a/drivers/gpu/drm/drm_dp_helper.c +++ b/drivers/gpu/drm/drm_dp_helper.c @@ -159,6 +159,8 @@ int drm_dp_bw_code_to_link_rate(u8 link_bw) } EXPORT_SYMBOL(drm_dp_bw_code_to_link_rate); +#define AUX_RETRY_INTERVAL 500 /* us */ + /** * DOC: dp helpers * @@ -213,7 +215,7 @@ static int drm_dp_dpcd_access(struct drm_dp_aux *aux, u8 request, return -EIO; case DP_AUX_NATIVE_REPLY_DEFER: - usleep_range(400, 500); + usleep_range(AUX_RETRY_INTERVAL, AUX_RETRY_INTERVAL + 100); break; } } @@ -422,6 +424,90 @@ static u32 drm_dp_i2c_functionality(struct i2c_adapter *adapter) I2C_FUNC_10BIT_ADDR; } +#define AUX_PRECHARGE_LEN 10 /* 10 to 16 */ +#define AUX_SYNC_LEN (16 + 4) /* preamble + AUX_SYNC_END */ +#define AUX_STOP_LEN 4 +#define AUX_CMD_LEN 4 +#define AUX_ADDRESS_LEN 20 +#define AUX_REPLY_PAD_LEN 4 +#define AUX_LENGTH_LEN 8 + +/* + * Calculate the duration of the AUX request/reply in usec. Gives the + * "best" case estimate, ie. successful while as short as possible. + */ +static int drm_dp_aux_req_duration(const struct drm_dp_aux_msg *msg) +{ + int len = AUX_PRECHARGE_LEN + AUX_SYNC_LEN + AUX_STOP_LEN + + AUX_CMD_LEN + AUX_ADDRESS_LEN + AUX_LENGTH_LEN; + + if ((msg->request & DP_AUX_I2C_READ) == 0) + len += msg->size * 8; + + return len; +} + +static int drm_dp_aux_reply_duration(const struct drm_dp_aux_msg *msg) +{ + int len = AUX_PRECHARGE_LEN + AUX_SYNC_LEN + AUX_STOP_LEN + + AUX_CMD_LEN + AUX_REPLY_PAD_LEN; + + /* + * For read we expect what was asked. For writes there will + * be 0 or 1 data bytes. Assume 0 for the "best" case. + */ + if (msg->request & DP_AUX_I2C_READ) + len += msg->size * 8; + + return len; +} + +#define I2C_START_LEN 1 +#define I2C_STOP_LEN 1 +#define I2C_ADDR_LEN 9 /* ADDRESS + R/W + ACK/NACK */ +#define I2C_DATA_LEN 9 /* DATA + ACK/NACK */ + +/* + * Calculate the length of the i2c transfer in usec, assuming + * the i2c bus speed is as specified. Gives the the "worst" + * case estimate, ie. successful while as long as possible. + * Doesn't account the the "MOT" bit, and instead assumes each + * message includes a START, ADDRESS and STOP. Neither does it + * account for additional random variables such as clock stretching. + */ +static int drm_dp_i2c_msg_duration(const struct drm_dp_aux_msg *msg, + int i2c_speed_khz) +{ + /* AUX bitrate is 1MHz, i2c bitrate as specified */ + return DIV_ROUND_UP((I2C_START_LEN + I2C_ADDR_LEN + + msg->size * I2C_DATA_LEN + + I2C_STOP_LEN) * 1000, i2c_speed_khz); +} + +/* + * Deterine how many retries should be attempted to successfully transfer + * the specified message, based on the estimated durations of the + * i2c and AUX transfers. + */ +static int drm_dp_i2c_retry_count(const struct drm_dp_aux_msg *msg, + int i2c_speed_khz) +{ + int aux_time_us = drm_dp_aux_req_duration(msg) + + drm_dp_aux_reply_duration(msg); + int i2c_time_us = drm_dp_i2c_msg_duration(msg, i2c_speed_khz); + + return DIV_ROUND_UP(i2c_time_us, aux_time_us + AUX_RETRY_INTERVAL); +} + +/* + * FIXME currently assumes 10 kHz as some real world devices seem + * to require it. We should query/set the speed via DPCD if supported. + */ +static int dp_aux_i2c_speed_khz __read_mostly = 10; +module_param_unsafe(dp_aux_i2c_speed_khz, int, 0644); +MODULE_PARM_DESC(dp_aux_i2c_speed_khz, + "Assumed speed of the i2c bus in kHz, (1-400, default 10)"); + /* * Transfer a single I2C-over-AUX message and handle various error conditions, * retrying the transaction as appropriate. It is assumed that the @@ -434,13 +520,16 @@ static int drm_dp_i2c_do_msg(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg) { unsigned int retry, defer_i2c; int ret; - /* * DP1.2 sections 2.7.7.1.5.6.1 and 2.7.7.1.6.6.1: A DP Source device * is required to retry at least seven times upon receiving AUX_DEFER * before giving up the AUX transaction. + * + * We also try to account for the i2c bus speed. */ - for (retry = 0, defer_i2c = 0; retry < (7 + defer_i2c); retry++) { + int max_retries = max(7, drm_dp_i2c_retry_count(msg, dp_aux_i2c_speed_khz)); + + for (retry = 0, defer_i2c = 0; retry < (max_retries + defer_i2c); retry++) { mutex_lock(&aux->hw_mutex); ret = aux->transfer(aux, msg); mutex_unlock(&aux->hw_mutex); @@ -476,7 +565,7 @@ static int drm_dp_i2c_do_msg(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg) * For now just defer for long enough to hopefully be * safe for all use-cases. */ - usleep_range(500, 600); + usleep_range(AUX_RETRY_INTERVAL, AUX_RETRY_INTERVAL + 100); continue; default: @@ -506,7 +595,7 @@ static int drm_dp_i2c_do_msg(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg) aux->i2c_defer_count++; if (defer_i2c < 7) defer_i2c++; - usleep_range(400, 500); + usleep_range(AUX_RETRY_INTERVAL, AUX_RETRY_INTERVAL + 100); continue; default: diff --git a/drivers/gpu/drm/drm_dp_mst_topology.c b/drivers/gpu/drm/drm_dp_mst_topology.c index eb603f1defc2..e23df5fd3836 100644 --- a/drivers/gpu/drm/drm_dp_mst_topology.c +++ b/drivers/gpu/drm/drm_dp_mst_topology.c @@ -2632,6 +2632,16 @@ void drm_dp_mst_dump_topology(struct seq_file *m, seq_printf(m, "%02x ", buf[i]); seq_printf(m, "\n"); + /* dump the standard OUI branch header */ + ret = drm_dp_dpcd_read(mgr->aux, DP_BRANCH_OUI, buf, DP_BRANCH_OUI_HEADER_SIZE); + seq_printf(m, "branch oui: "); + for (i = 0; i < 0x3; i++) + seq_printf(m, "%02x", buf[i]); + seq_printf(m, " devid: "); + for (i = 0x3; i < 0x8; i++) + seq_printf(m, "%c", buf[i]); + seq_printf(m, " revision: hw: %x.%x sw: %x.%x", buf[0x9] >> 4, buf[0x9] & 0xf, buf[0xa], buf[0xb]); + seq_printf(m, "\n"); bret = dump_dp_payload_table(mgr, buf); if (bret == true) { seq_printf(m, "payload table: "); diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c index b7bf4ce8c012..53d09a19f7e1 100644 --- a/drivers/gpu/drm/drm_drv.c +++ b/drivers/gpu/drm/drm_drv.c @@ -285,7 +285,6 @@ static void drm_minor_free(struct drm_device *dev, unsigned int type) if (!minor) return; - drm_mode_group_destroy(&minor->mode_group); put_device(minor->kdev); spin_lock_irqsave(&drm_minor_lock, flags); @@ -582,11 +581,7 @@ struct drm_device *drm_dev_alloc(struct drm_driver *driver, if (drm_ht_create(&dev->map_hash, 12)) goto err_minors; - ret = drm_legacy_ctxbitmap_init(dev); - if (ret) { - DRM_ERROR("Cannot allocate memory for context bitmap.\n"); - goto err_ht; - } + drm_legacy_ctxbitmap_init(dev); if (drm_core_check_feature(dev, DRIVER_GEM)) { ret = drm_gem_init(dev); @@ -600,7 +595,6 @@ struct drm_device *drm_dev_alloc(struct drm_driver *driver, err_ctxbitmap: drm_legacy_ctxbitmap_cleanup(dev); -err_ht: drm_ht_remove(&dev->map_hash); err_minors: drm_minor_free(dev, DRM_MINOR_LEGACY); @@ -705,20 +699,9 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags) goto err_minors; } - /* setup grouping for legacy outputs */ - if (drm_core_check_feature(dev, DRIVER_MODESET)) { - ret = drm_mode_group_init_legacy_group(dev, - &dev->primary->mode_group); - if (ret) - goto err_unload; - } - ret = 0; goto out_unlock; -err_unload: - if (dev->driver->unload) - dev->driver->unload(dev); err_minors: drm_minor_unregister(dev, DRM_MINOR_LEGACY); drm_minor_unregister(dev, DRM_MINOR_RENDER); diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c index 7087da37dae0..05bb7311ac5d 100644 --- a/drivers/gpu/drm/drm_edid.c +++ b/drivers/gpu/drm/drm_edid.c @@ -3413,7 +3413,7 @@ struct drm_connector *drm_select_eld(struct drm_encoder *encoder, WARN_ON(!mutex_is_locked(&dev->mode_config.mutex)); WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex)); - list_for_each_entry(connector, &dev->mode_config.connector_list, head) + drm_for_each_connector(connector, dev) if (connector->encoder == encoder && connector->eld[0]) return connector; @@ -3802,7 +3802,7 @@ int drm_add_modes_noedid(struct drm_connector *connector, struct drm_display_mode *mode; struct drm_device *dev = connector->dev; - count = sizeof(drm_dmt_modes) / sizeof(struct drm_display_mode); + count = ARRAY_SIZE(drm_dmt_modes); if (hdisplay < 0) hdisplay = 0; if (vdisplay < 0) diff --git a/drivers/gpu/drm/drm_fb_cma_helper.c b/drivers/gpu/drm/drm_fb_cma_helper.c index 5c1aca443e54..c19a62561183 100644 --- a/drivers/gpu/drm/drm_fb_cma_helper.c +++ b/drivers/gpu/drm/drm_fb_cma_helper.c @@ -209,23 +209,11 @@ int drm_fb_cma_debugfs_show(struct seq_file *m, void *arg) struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; struct drm_framebuffer *fb; - int ret; - ret = mutex_lock_interruptible(&dev->mode_config.mutex); - if (ret) - return ret; - - ret = mutex_lock_interruptible(&dev->struct_mutex); - if (ret) { - mutex_unlock(&dev->mode_config.mutex); - return ret; - } - - list_for_each_entry(fb, &dev->mode_config.fb_list, head) + mutex_lock(&dev->mode_config.fb_lock); + drm_for_each_fb(fb, dev) drm_fb_cma_describe(fb, m); - - mutex_unlock(&dev->struct_mutex); - mutex_unlock(&dev->mode_config.mutex); + mutex_unlock(&dev->mode_config.fb_lock); return 0; } @@ -234,9 +222,9 @@ EXPORT_SYMBOL_GPL(drm_fb_cma_debugfs_show); static struct fb_ops drm_fbdev_cma_ops = { .owner = THIS_MODULE, - .fb_fillrect = sys_fillrect, - .fb_copyarea = sys_copyarea, - .fb_imageblit = sys_imageblit, + .fb_fillrect = drm_fb_helper_sys_fillrect, + .fb_copyarea = drm_fb_helper_sys_copyarea, + .fb_imageblit = drm_fb_helper_sys_imageblit, .fb_check_var = drm_fb_helper_check_var, .fb_set_par = drm_fb_helper_set_par, .fb_blank = drm_fb_helper_blank, @@ -275,10 +263,9 @@ static int drm_fbdev_cma_create(struct drm_fb_helper *helper, if (IS_ERR(obj)) return -ENOMEM; - fbi = framebuffer_alloc(0, dev->dev); - if (!fbi) { - dev_err(dev->dev, "Failed to allocate framebuffer info.\n"); - ret = -ENOMEM; + fbi = drm_fb_helper_alloc_fbi(helper); + if (IS_ERR(fbi)) { + ret = PTR_ERR(fbi); goto err_drm_gem_cma_free_object; } @@ -286,23 +273,16 @@ static int drm_fbdev_cma_create(struct drm_fb_helper *helper, if (IS_ERR(fbdev_cma->fb)) { dev_err(dev->dev, "Failed to allocate DRM framebuffer.\n"); ret = PTR_ERR(fbdev_cma->fb); - goto err_framebuffer_release; + goto err_fb_info_destroy; } fb = &fbdev_cma->fb->fb; helper->fb = fb; - helper->fbdev = fbi; fbi->par = helper; fbi->flags = FBINFO_FLAG_DEFAULT; fbi->fbops = &drm_fbdev_cma_ops; - ret = fb_alloc_cmap(&fbi->cmap, 256, 0); - if (ret) { - dev_err(dev->dev, "Failed to allocate color map.\n"); - goto err_drm_fb_cma_destroy; - } - drm_fb_helper_fill_fix(fbi, fb->pitches[0], fb->depth); drm_fb_helper_fill_var(fbi, helper, sizes->fb_width, sizes->fb_height); @@ -317,11 +297,8 @@ static int drm_fbdev_cma_create(struct drm_fb_helper *helper, return 0; -err_drm_fb_cma_destroy: - drm_framebuffer_unregister_private(fb); - drm_fb_cma_destroy(fb); -err_framebuffer_release: - framebuffer_release(fbi); +err_fb_info_destroy: + drm_fb_helper_release_fbi(helper); err_drm_gem_cma_free_object: drm_gem_cma_free_object(&obj->base); return ret; @@ -397,20 +374,8 @@ EXPORT_SYMBOL_GPL(drm_fbdev_cma_init); */ void drm_fbdev_cma_fini(struct drm_fbdev_cma *fbdev_cma) { - if (fbdev_cma->fb_helper.fbdev) { - struct fb_info *info; - int ret; - - info = fbdev_cma->fb_helper.fbdev; - ret = unregister_framebuffer(info); - if (ret < 0) - DRM_DEBUG_KMS("failed unregister_framebuffer()\n"); - - if (info->cmap.len) - fb_dealloc_cmap(&info->cmap); - - framebuffer_release(info); - } + drm_fb_helper_unregister_fbi(&fbdev_cma->fb_helper); + drm_fb_helper_release_fbi(&fbdev_cma->fb_helper); if (fbdev_cma->fb) { drm_framebuffer_unregister_private(&fbdev_cma->fb->fb); diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c index cac422916c7a..418d299f3b12 100644 --- a/drivers/gpu/drm/drm_fb_helper.c +++ b/drivers/gpu/drm/drm_fb_helper.c @@ -56,8 +56,8 @@ static LIST_HEAD(kernel_fb_helper_list); * Teardown is done with drm_fb_helper_fini(). * * At runtime drivers should restore the fbdev console by calling - * drm_fb_helper_restore_fbdev_mode() from their ->lastclose callback. They - * should also notify the fb helper code from updates to the output + * drm_fb_helper_restore_fbdev_mode_unlocked() from their ->lastclose callback. + * They should also notify the fb helper code from updates to the output * configuration by calling drm_fb_helper_hotplug_event(). For easier * integration with the output polling code in drm_crtc_helper.c the modeset * code provides a ->output_poll_changed callback. @@ -89,8 +89,9 @@ static LIST_HEAD(kernel_fb_helper_list); * connectors to the fbdev, e.g. if some are reserved for special purposes or * not adequate to be used for the fbcon. * - * Since this is part of the initial setup before the fbdev is published, no - * locking is required. + * This function is protected against concurrent connector hotadds/removals + * using drm_fb_helper_add_one_connector() and + * drm_fb_helper_remove_one_connector(). */ int drm_fb_helper_single_add_all_connectors(struct drm_fb_helper *fb_helper) { @@ -98,7 +99,8 @@ int drm_fb_helper_single_add_all_connectors(struct drm_fb_helper *fb_helper) struct drm_connector *connector; int i; - list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + mutex_lock(&dev->mode_config.mutex); + drm_for_each_connector(connector, dev) { struct drm_fb_helper_connector *fb_helper_connector; fb_helper_connector = kzalloc(sizeof(struct drm_fb_helper_connector), GFP_KERNEL); @@ -108,6 +110,7 @@ int drm_fb_helper_single_add_all_connectors(struct drm_fb_helper *fb_helper) fb_helper_connector->connector = connector; fb_helper->connector_info[fb_helper->connector_count++] = fb_helper_connector; } + mutex_unlock(&dev->mode_config.mutex); return 0; fail: for (i = 0; i < fb_helper->connector_count; i++) { @@ -115,6 +118,8 @@ int drm_fb_helper_single_add_all_connectors(struct drm_fb_helper *fb_helper) fb_helper->connector_info[i] = NULL; } fb_helper->connector_count = 0; + mutex_unlock(&dev->mode_config.mutex); + return -ENOMEM; } EXPORT_SYMBOL(drm_fb_helper_single_add_all_connectors); @@ -163,11 +168,14 @@ static void remove_from_modeset(struct drm_mode_set *set, } set->num_connectors--; - /* because i915 is pissy about this.. + /* * TODO maybe need to makes sure we set it back to !=NULL somewhere? */ - if (set->num_connectors == 0) + if (set->num_connectors == 0) { set->fb = NULL; + drm_mode_destroy(connector->dev, set->mode); + set->mode = NULL; + } } int drm_fb_helper_remove_one_connector(struct drm_fb_helper *fb_helper, @@ -269,7 +277,7 @@ static struct drm_framebuffer *drm_mode_config_fb(struct drm_crtc *crtc) struct drm_device *dev = crtc->dev; struct drm_crtc *c; - list_for_each_entry(c, &dev->mode_config.crtc_list, head) { + drm_for_each_crtc(c, dev) { if (crtc->base.id == c->base.id) return c->primary->fb; } @@ -321,7 +329,7 @@ static bool restore_fbdev_mode(struct drm_fb_helper *fb_helper) drm_warn_on_modeset_not_all_locked(dev); - list_for_each_entry(plane, &dev->mode_config.plane_list, head) { + drm_for_each_plane(plane, dev) { if (plane->type != DRM_PLANE_TYPE_PRIMARY) drm_plane_force_disable(plane); @@ -349,21 +357,6 @@ static bool restore_fbdev_mode(struct drm_fb_helper *fb_helper) } return error; } -/** - * drm_fb_helper_restore_fbdev_mode - restore fbdev configuration - * @fb_helper: fbcon to restore - * - * This should be called from driver's drm ->lastclose callback - * when implementing an fbcon on top of kms using this helper. This ensures that - * the user isn't greeted with a black screen when e.g. X dies. - * - * Use this variant if you need to bypass locking (panic), or already - * hold all modeset locks. Otherwise use drm_fb_helper_restore_fbdev_mode_unlocked() - */ -static bool drm_fb_helper_restore_fbdev_mode(struct drm_fb_helper *fb_helper) -{ - return restore_fbdev_mode(fb_helper); -} /** * drm_fb_helper_restore_fbdev_mode_unlocked - restore fbdev configuration @@ -393,6 +386,31 @@ bool drm_fb_helper_restore_fbdev_mode_unlocked(struct drm_fb_helper *fb_helper) } EXPORT_SYMBOL(drm_fb_helper_restore_fbdev_mode_unlocked); +static bool drm_fb_helper_is_bound(struct drm_fb_helper *fb_helper) +{ + struct drm_device *dev = fb_helper->dev; + struct drm_crtc *crtc; + int bound = 0, crtcs_bound = 0; + + /* Sometimes user space wants everything disabled, so don't steal the + * display if there's a master. */ + if (dev->primary->master) + return false; + + drm_for_each_crtc(crtc, dev) { + if (crtc->primary->fb) + crtcs_bound++; + if (crtc->primary->fb == fb_helper->fb) + bound++; + } + + if (bound < crtcs_bound) + return false; + + return true; +} + +#ifdef CONFIG_MAGIC_SYSRQ /* * restore fbcon display for all kms driver's using this helper, used for sysrq * and panic handling. @@ -411,67 +429,15 @@ static bool drm_fb_helper_force_kernel_mode(void) if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) continue; - /* - * NOTE: Use trylock mode to avoid deadlocks and sleeping in - * panic context. - */ - if (__drm_modeset_lock_all(dev, true) != 0) { - error = true; - continue; - } - - ret = drm_fb_helper_restore_fbdev_mode(helper); + drm_modeset_lock_all(dev); + ret = restore_fbdev_mode(helper); if (ret) error = true; - drm_modeset_unlock_all(dev); } return error; } -static int drm_fb_helper_panic(struct notifier_block *n, unsigned long ununsed, - void *panic_str) -{ - /* - * It's a waste of time and effort to switch back to text console - * if the kernel should reboot before panic messages can be seen. - */ - if (panic_timeout < 0) - return 0; - - pr_err("panic occurred, switching back to text console\n"); - return drm_fb_helper_force_kernel_mode(); -} - -static struct notifier_block paniced = { - .notifier_call = drm_fb_helper_panic, -}; - -static bool drm_fb_helper_is_bound(struct drm_fb_helper *fb_helper) -{ - struct drm_device *dev = fb_helper->dev; - struct drm_crtc *crtc; - int bound = 0, crtcs_bound = 0; - - /* Sometimes user space wants everything disabled, so don't steal the - * display if there's a master. */ - if (dev->primary->master) - return false; - - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { - if (crtc->primary->fb) - crtcs_bound++; - if (crtc->primary->fb == fb_helper->fb) - bound++; - } - - if (bound < crtcs_bound) - return false; - - return true; -} - -#ifdef CONFIG_MAGIC_SYSRQ static void drm_fb_helper_restore_work_fn(struct work_struct *ignored) { bool ret; @@ -503,14 +469,6 @@ static void drm_fb_helper_dpms(struct fb_info *info, int dpms_mode) struct drm_connector *connector; int i, j; - /* - * fbdev->blank can be called from irq context in case of a panic. - * Since we already have our own special panic handler which will - * restore the fbdev console mode completely, just bail out early. - */ - if (oops_in_progress) - return; - /* * For each CRTC in this fb, turn the connectors on/off. */ @@ -544,6 +502,9 @@ static void drm_fb_helper_dpms(struct fb_info *info, int dpms_mode) */ int drm_fb_helper_blank(int blank, struct fb_info *info) { + if (oops_in_progress) + return -EBUSY; + switch (blank) { /* Display: On; HSync: On, VSync: On */ case FB_BLANK_UNBLANK: @@ -655,7 +616,7 @@ int drm_fb_helper_init(struct drm_device *dev, } i = 0; - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { + drm_for_each_crtc(crtc, dev) { fb_helper->crtc_info[i].mode_set.crtc = crtc; i++; } @@ -667,14 +628,91 @@ int drm_fb_helper_init(struct drm_device *dev, } EXPORT_SYMBOL(drm_fb_helper_init); +/** + * drm_fb_helper_alloc_fbi - allocate fb_info and some of its members + * @fb_helper: driver-allocated fbdev helper + * + * A helper to alloc fb_info and the members cmap and apertures. Called + * by the driver within the fb_probe fb_helper callback function. + * + * RETURNS: + * fb_info pointer if things went okay, pointer containing error code + * otherwise + */ +struct fb_info *drm_fb_helper_alloc_fbi(struct drm_fb_helper *fb_helper) +{ + struct device *dev = fb_helper->dev->dev; + struct fb_info *info; + int ret; + + info = framebuffer_alloc(0, dev); + if (!info) + return ERR_PTR(-ENOMEM); + + ret = fb_alloc_cmap(&info->cmap, 256, 0); + if (ret) + goto err_release; + + info->apertures = alloc_apertures(1); + if (!info->apertures) { + ret = -ENOMEM; + goto err_free_cmap; + } + + fb_helper->fbdev = info; + + return info; + +err_free_cmap: + fb_dealloc_cmap(&info->cmap); +err_release: + framebuffer_release(info); + return ERR_PTR(ret); +} +EXPORT_SYMBOL(drm_fb_helper_alloc_fbi); + +/** + * drm_fb_helper_unregister_fbi - unregister fb_info framebuffer device + * @fb_helper: driver-allocated fbdev helper + * + * A wrapper around unregister_framebuffer, to release the fb_info + * framebuffer device + */ +void drm_fb_helper_unregister_fbi(struct drm_fb_helper *fb_helper) +{ + if (fb_helper && fb_helper->fbdev) + unregister_framebuffer(fb_helper->fbdev); +} +EXPORT_SYMBOL(drm_fb_helper_unregister_fbi); + +/** + * drm_fb_helper_release_fbi - dealloc fb_info and its members + * @fb_helper: driver-allocated fbdev helper + * + * A helper to free memory taken by fb_info and the members cmap and + * apertures + */ +void drm_fb_helper_release_fbi(struct drm_fb_helper *fb_helper) +{ + if (fb_helper) { + struct fb_info *info = fb_helper->fbdev; + + if (info) { + if (info->cmap.len) + fb_dealloc_cmap(&info->cmap); + framebuffer_release(info); + } + + fb_helper->fbdev = NULL; + } +} +EXPORT_SYMBOL(drm_fb_helper_release_fbi); + void drm_fb_helper_fini(struct drm_fb_helper *fb_helper) { if (!list_empty(&fb_helper->kernel_fb_list)) { list_del(&fb_helper->kernel_fb_list); if (list_empty(&kernel_fb_helper_list)) { - pr_info("drm: unregistered panic notifier\n"); - atomic_notifier_chain_unregister(&panic_notifier_list, - &paniced); unregister_sysrq_key('v', &sysrq_drm_fb_helper_restore_op); } } @@ -684,6 +722,149 @@ void drm_fb_helper_fini(struct drm_fb_helper *fb_helper) } EXPORT_SYMBOL(drm_fb_helper_fini); +/** + * drm_fb_helper_unlink_fbi - wrapper around unlink_framebuffer + * @fb_helper: driver-allocated fbdev helper + * + * A wrapper around unlink_framebuffer implemented by fbdev core + */ +void drm_fb_helper_unlink_fbi(struct drm_fb_helper *fb_helper) +{ + if (fb_helper && fb_helper->fbdev) + unlink_framebuffer(fb_helper->fbdev); +} +EXPORT_SYMBOL(drm_fb_helper_unlink_fbi); + +/** + * drm_fb_helper_sys_read - wrapper around fb_sys_read + * @info: fb_info struct pointer + * @buf: userspace buffer to read from framebuffer memory + * @count: number of bytes to read from framebuffer memory + * @ppos: read offset within framebuffer memory + * + * A wrapper around fb_sys_read implemented by fbdev core + */ +ssize_t drm_fb_helper_sys_read(struct fb_info *info, char __user *buf, + size_t count, loff_t *ppos) +{ + return fb_sys_read(info, buf, count, ppos); +} +EXPORT_SYMBOL(drm_fb_helper_sys_read); + +/** + * drm_fb_helper_sys_write - wrapper around fb_sys_write + * @info: fb_info struct pointer + * @buf: userspace buffer to write to framebuffer memory + * @count: number of bytes to write to framebuffer memory + * @ppos: write offset within framebuffer memory + * + * A wrapper around fb_sys_write implemented by fbdev core + */ +ssize_t drm_fb_helper_sys_write(struct fb_info *info, const char __user *buf, + size_t count, loff_t *ppos) +{ + return fb_sys_write(info, buf, count, ppos); +} +EXPORT_SYMBOL(drm_fb_helper_sys_write); + +/** + * drm_fb_helper_sys_fillrect - wrapper around sys_fillrect + * @info: fbdev registered by the helper + * @rect: info about rectangle to fill + * + * A wrapper around sys_fillrect implemented by fbdev core + */ +void drm_fb_helper_sys_fillrect(struct fb_info *info, + const struct fb_fillrect *rect) +{ + sys_fillrect(info, rect); +} +EXPORT_SYMBOL(drm_fb_helper_sys_fillrect); + +/** + * drm_fb_helper_sys_copyarea - wrapper around sys_copyarea + * @info: fbdev registered by the helper + * @area: info about area to copy + * + * A wrapper around sys_copyarea implemented by fbdev core + */ +void drm_fb_helper_sys_copyarea(struct fb_info *info, + const struct fb_copyarea *area) +{ + sys_copyarea(info, area); +} +EXPORT_SYMBOL(drm_fb_helper_sys_copyarea); + +/** + * drm_fb_helper_sys_imageblit - wrapper around sys_imageblit + * @info: fbdev registered by the helper + * @image: info about image to blit + * + * A wrapper around sys_imageblit implemented by fbdev core + */ +void drm_fb_helper_sys_imageblit(struct fb_info *info, + const struct fb_image *image) +{ + sys_imageblit(info, image); +} +EXPORT_SYMBOL(drm_fb_helper_sys_imageblit); + +/** + * drm_fb_helper_cfb_fillrect - wrapper around cfb_fillrect + * @info: fbdev registered by the helper + * @rect: info about rectangle to fill + * + * A wrapper around cfb_imageblit implemented by fbdev core + */ +void drm_fb_helper_cfb_fillrect(struct fb_info *info, + const struct fb_fillrect *rect) +{ + cfb_fillrect(info, rect); +} +EXPORT_SYMBOL(drm_fb_helper_cfb_fillrect); + +/** + * drm_fb_helper_cfb_copyarea - wrapper around cfb_copyarea + * @info: fbdev registered by the helper + * @area: info about area to copy + * + * A wrapper around cfb_copyarea implemented by fbdev core + */ +void drm_fb_helper_cfb_copyarea(struct fb_info *info, + const struct fb_copyarea *area) +{ + cfb_copyarea(info, area); +} +EXPORT_SYMBOL(drm_fb_helper_cfb_copyarea); + +/** + * drm_fb_helper_cfb_imageblit - wrapper around cfb_imageblit + * @info: fbdev registered by the helper + * @image: info about image to blit + * + * A wrapper around cfb_imageblit implemented by fbdev core + */ +void drm_fb_helper_cfb_imageblit(struct fb_info *info, + const struct fb_image *image) +{ + cfb_imageblit(info, image); +} +EXPORT_SYMBOL(drm_fb_helper_cfb_imageblit); + +/** + * drm_fb_helper_set_suspend - wrapper around fb_set_suspend + * @fb_helper: driver-allocated fbdev helper + * @state: desired state, zero to resume, non-zero to suspend + * + * A wrapper around fb_set_suspend implemented by fbdev core + */ +void drm_fb_helper_set_suspend(struct drm_fb_helper *fb_helper, int state) +{ + if (fb_helper && fb_helper->fbdev) + fb_set_suspend(fb_helper->fbdev, state); +} +EXPORT_SYMBOL(drm_fb_helper_set_suspend); + static int setcolreg(struct drm_crtc *crtc, u16 red, u16 green, u16 blue, u16 regno, struct fb_info *info) { @@ -771,9 +952,10 @@ int drm_fb_helper_setcmap(struct fb_cmap *cmap, struct fb_info *info) int i, j, rc = 0; int start; - if (__drm_modeset_lock_all(dev, !!oops_in_progress)) { + if (oops_in_progress) return -EBUSY; - } + + drm_modeset_lock_all(dev); if (!drm_fb_helper_is_bound(fb_helper)) { drm_modeset_unlock_all(dev); return -EBUSY; @@ -922,6 +1104,9 @@ int drm_fb_helper_set_par(struct fb_info *info) struct drm_fb_helper *fb_helper = info->par; struct fb_var_screeninfo *var = &info->var; + if (oops_in_progress) + return -EBUSY; + if (var->pixclock != 0) { DRM_ERROR("PIXEL CLOCK SET\n"); return -EINVAL; @@ -947,9 +1132,10 @@ int drm_fb_helper_pan_display(struct fb_var_screeninfo *var, int ret = 0; int i; - if (__drm_modeset_lock_all(dev, !!oops_in_progress)) { + if (oops_in_progress) return -EBUSY; - } + + drm_modeset_lock_all(dev); if (!drm_fb_helper_is_bound(fb_helper)) { drm_modeset_unlock_all(dev); return -EBUSY; @@ -1109,12 +1295,7 @@ static int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper, dev_info(fb_helper->dev->dev, "fb%d: %s frame buffer device\n", info->node, info->fix.id); - /* Switch back to kernel console on panic */ - /* multi card linked list maybe */ if (list_empty(&kernel_fb_helper_list)) { - dev_info(fb_helper->dev->dev, "registered panic notifier\n"); - atomic_notifier_chain_register(&panic_notifier_list, - &paniced); register_sysrq_key('v', &sysrq_drm_fb_helper_restore_op); } diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c index 16a164770713..3c2d4abd71c5 100644 --- a/drivers/gpu/drm/drm_gem.c +++ b/drivers/gpu/drm/drm_gem.c @@ -766,7 +766,7 @@ drm_gem_object_free(struct kref *kref) struct drm_gem_object *obj = (struct drm_gem_object *) kref; struct drm_device *dev = obj->dev; - BUG_ON(!mutex_is_locked(&dev->struct_mutex)); + WARN_ON(!mutex_is_locked(&dev->struct_mutex)); if (dev->driver->gem_free_object != NULL) dev->driver->gem_free_object(obj); @@ -778,22 +778,14 @@ void drm_gem_vm_open(struct vm_area_struct *vma) struct drm_gem_object *obj = vma->vm_private_data; drm_gem_object_reference(obj); - - mutex_lock(&obj->dev->struct_mutex); - drm_vm_open_locked(obj->dev, vma); - mutex_unlock(&obj->dev->struct_mutex); } EXPORT_SYMBOL(drm_gem_vm_open); void drm_gem_vm_close(struct vm_area_struct *vma) { struct drm_gem_object *obj = vma->vm_private_data; - struct drm_device *dev = obj->dev; - mutex_lock(&dev->struct_mutex); - drm_vm_close_locked(obj->dev, vma); - drm_gem_object_unreference(obj); - mutex_unlock(&dev->struct_mutex); + drm_gem_object_unreference_unlocked(obj); } EXPORT_SYMBOL(drm_gem_vm_close); @@ -850,7 +842,6 @@ int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size, */ drm_gem_object_reference(obj); - drm_vm_open_locked(dev, vma); return 0; } EXPORT_SYMBOL(drm_gem_mmap_obj); diff --git a/drivers/gpu/drm/drm_gem_cma_helper.c b/drivers/gpu/drm/drm_gem_cma_helper.c index bd75f303da63..86cc793cdf79 100644 --- a/drivers/gpu/drm/drm_gem_cma_helper.c +++ b/drivers/gpu/drm/drm_gem_cma_helper.c @@ -289,20 +289,15 @@ int drm_gem_cma_dumb_map_offset(struct drm_file *file_priv, { struct drm_gem_object *gem_obj; - mutex_lock(&drm->struct_mutex); - gem_obj = drm_gem_object_lookup(drm, file_priv, handle); if (!gem_obj) { dev_err(drm->dev, "failed to lookup GEM object\n"); - mutex_unlock(&drm->struct_mutex); return -EINVAL; } *offset = drm_vma_node_offset_addr(&gem_obj->vma_node); - drm_gem_object_unreference(gem_obj); - - mutex_unlock(&drm->struct_mutex); + drm_gem_object_unreference_unlocked(gem_obj); return 0; } @@ -381,11 +376,8 @@ void drm_gem_cma_describe(struct drm_gem_cma_object *cma_obj, struct seq_file *m) { struct drm_gem_object *obj = &cma_obj->base; - struct drm_device *dev = obj->dev; uint64_t off; - WARN_ON(!mutex_is_locked(&dev->struct_mutex)); - off = drm_vma_node_start(&obj->vma_node); seq_printf(m, "%2d (%2d) %08llx %pad %p %zu", diff --git a/drivers/gpu/drm/drm_ioc32.c b/drivers/gpu/drm/drm_ioc32.c index 9cfcd0aef0df..ddfa6014c2c2 100644 --- a/drivers/gpu/drm/drm_ioc32.c +++ b/drivers/gpu/drm/drm_ioc32.c @@ -95,7 +95,7 @@ static int compat_drm_version(struct file *file, unsigned int cmd, return -EFAULT; version = compat_alloc_user_space(sizeof(*version)); - if (!access_ok(VERIFY_WRITE, version, sizeof(*version))) + if (!version) return -EFAULT; if (__put_user(v32.name_len, &version->name_len) || __put_user((void __user *)(unsigned long)v32.name, @@ -142,7 +142,7 @@ static int compat_drm_getunique(struct file *file, unsigned int cmd, return -EFAULT; u = compat_alloc_user_space(sizeof(*u)); - if (!access_ok(VERIFY_WRITE, u, sizeof(*u))) + if (!u) return -EFAULT; if (__put_user(uq32.unique_len, &u->unique_len) || __put_user((void __user *)(unsigned long)uq32.unique, @@ -170,7 +170,7 @@ static int compat_drm_setunique(struct file *file, unsigned int cmd, return -EFAULT; u = compat_alloc_user_space(sizeof(*u)); - if (!access_ok(VERIFY_WRITE, u, sizeof(*u))) + if (!u) return -EFAULT; if (__put_user(uq32.unique_len, &u->unique_len) || __put_user((void __user *)(unsigned long)uq32.unique, @@ -202,7 +202,7 @@ static int compat_drm_getmap(struct file *file, unsigned int cmd, return -EFAULT; map = compat_alloc_user_space(sizeof(*map)); - if (!access_ok(VERIFY_WRITE, map, sizeof(*map))) + if (!map) return -EFAULT; if (__put_user(idx, &map->offset)) return -EFAULT; @@ -239,7 +239,7 @@ static int compat_drm_addmap(struct file *file, unsigned int cmd, return -EFAULT; map = compat_alloc_user_space(sizeof(*map)); - if (!access_ok(VERIFY_WRITE, map, sizeof(*map))) + if (!map) return -EFAULT; if (__put_user(m32.offset, &map->offset) || __put_user(m32.size, &map->size) @@ -279,7 +279,7 @@ static int compat_drm_rmmap(struct file *file, unsigned int cmd, return -EFAULT; map = compat_alloc_user_space(sizeof(*map)); - if (!access_ok(VERIFY_WRITE, map, sizeof(*map))) + if (!map) return -EFAULT; if (__put_user((void *)(unsigned long)handle, &map->handle)) return -EFAULT; @@ -308,7 +308,7 @@ static int compat_drm_getclient(struct file *file, unsigned int cmd, return -EFAULT; client = compat_alloc_user_space(sizeof(*client)); - if (!access_ok(VERIFY_WRITE, client, sizeof(*client))) + if (!client) return -EFAULT; if (__put_user(idx, &client->idx)) return -EFAULT; @@ -347,7 +347,7 @@ static int compat_drm_getstats(struct file *file, unsigned int cmd, int i, err; stats = compat_alloc_user_space(sizeof(*stats)); - if (!access_ok(VERIFY_WRITE, stats, sizeof(*stats))) + if (!stats) return -EFAULT; err = drm_ioctl(file, DRM_IOCTL_GET_STATS, (unsigned long)stats); @@ -384,8 +384,7 @@ static int compat_drm_addbufs(struct file *file, unsigned int cmd, unsigned long agp_start; buf = compat_alloc_user_space(sizeof(*buf)); - if (!access_ok(VERIFY_WRITE, buf, sizeof(*buf)) - || !access_ok(VERIFY_WRITE, argp, sizeof(*argp))) + if (!buf || !access_ok(VERIFY_WRITE, argp, sizeof(*argp))) return -EFAULT; if (__copy_in_user(buf, argp, offsetof(drm_buf_desc32_t, agp_start)) @@ -416,7 +415,7 @@ static int compat_drm_markbufs(struct file *file, unsigned int cmd, return -EFAULT; buf = compat_alloc_user_space(sizeof(*buf)); - if (!access_ok(VERIFY_WRITE, buf, sizeof(*buf))) + if (!buf) return -EFAULT; if (__put_user(b32.size, &buf->size) @@ -457,7 +456,7 @@ static int compat_drm_infobufs(struct file *file, unsigned int cmd, nbytes = sizeof(*request) + count * sizeof(struct drm_buf_desc); request = compat_alloc_user_space(nbytes); - if (!access_ok(VERIFY_WRITE, request, nbytes)) + if (!request) return -EFAULT; list = (struct drm_buf_desc *) (request + 1); @@ -518,7 +517,7 @@ static int compat_drm_mapbufs(struct file *file, unsigned int cmd, return -EINVAL; nbytes = sizeof(*request) + count * sizeof(struct drm_buf_pub); request = compat_alloc_user_space(nbytes); - if (!access_ok(VERIFY_WRITE, request, nbytes)) + if (!request) return -EFAULT; list = (struct drm_buf_pub *) (request + 1); @@ -565,7 +564,7 @@ static int compat_drm_freebufs(struct file *file, unsigned int cmd, return -EFAULT; request = compat_alloc_user_space(sizeof(*request)); - if (!access_ok(VERIFY_WRITE, request, sizeof(*request))) + if (!request) return -EFAULT; if (__put_user(req32.count, &request->count) || __put_user((int __user *)(unsigned long)req32.list, @@ -591,7 +590,7 @@ static int compat_drm_setsareactx(struct file *file, unsigned int cmd, return -EFAULT; request = compat_alloc_user_space(sizeof(*request)); - if (!access_ok(VERIFY_WRITE, request, sizeof(*request))) + if (!request) return -EFAULT; if (__put_user(req32.ctx_id, &request->ctx_id) || __put_user((void *)(unsigned long)req32.handle, @@ -615,7 +614,7 @@ static int compat_drm_getsareactx(struct file *file, unsigned int cmd, return -EFAULT; request = compat_alloc_user_space(sizeof(*request)); - if (!access_ok(VERIFY_WRITE, request, sizeof(*request))) + if (!request) return -EFAULT; if (__put_user(ctx_id, &request->ctx_id)) return -EFAULT; @@ -648,7 +647,7 @@ static int compat_drm_resctx(struct file *file, unsigned int cmd, return -EFAULT; res = compat_alloc_user_space(sizeof(*res)); - if (!access_ok(VERIFY_WRITE, res, sizeof(*res))) + if (!res) return -EFAULT; if (__put_user(res32.count, &res->count) || __put_user((struct drm_ctx __user *) (unsigned long)res32.contexts, @@ -691,7 +690,7 @@ static int compat_drm_dma(struct file *file, unsigned int cmd, return -EFAULT; d = compat_alloc_user_space(sizeof(*d)); - if (!access_ok(VERIFY_WRITE, d, sizeof(*d))) + if (!d) return -EFAULT; if (__put_user(d32.context, &d->context) @@ -766,7 +765,7 @@ static int compat_drm_agp_info(struct file *file, unsigned int cmd, int err; info = compat_alloc_user_space(sizeof(*info)); - if (!access_ok(VERIFY_WRITE, info, sizeof(*info))) + if (!info) return -EFAULT; err = drm_ioctl(file, DRM_IOCTL_AGP_INFO, (unsigned long)info); @@ -809,7 +808,7 @@ static int compat_drm_agp_alloc(struct file *file, unsigned int cmd, return -EFAULT; request = compat_alloc_user_space(sizeof(*request)); - if (!access_ok(VERIFY_WRITE, request, sizeof(*request)) + if (!request || __put_user(req32.size, &request->size) || __put_user(req32.type, &request->type)) return -EFAULT; @@ -836,7 +835,7 @@ static int compat_drm_agp_free(struct file *file, unsigned int cmd, u32 handle; request = compat_alloc_user_space(sizeof(*request)); - if (!access_ok(VERIFY_WRITE, request, sizeof(*request)) + if (!request || get_user(handle, &argp->handle) || __put_user(handle, &request->handle)) return -EFAULT; @@ -860,7 +859,7 @@ static int compat_drm_agp_bind(struct file *file, unsigned int cmd, return -EFAULT; request = compat_alloc_user_space(sizeof(*request)); - if (!access_ok(VERIFY_WRITE, request, sizeof(*request)) + if (!request || __put_user(req32.handle, &request->handle) || __put_user(req32.offset, &request->offset)) return -EFAULT; @@ -876,7 +875,7 @@ static int compat_drm_agp_unbind(struct file *file, unsigned int cmd, u32 handle; request = compat_alloc_user_space(sizeof(*request)); - if (!access_ok(VERIFY_WRITE, request, sizeof(*request)) + if (!request || get_user(handle, &argp->handle) || __put_user(handle, &request->handle)) return -EFAULT; @@ -899,8 +898,7 @@ static int compat_drm_sg_alloc(struct file *file, unsigned int cmd, unsigned long x; request = compat_alloc_user_space(sizeof(*request)); - if (!access_ok(VERIFY_WRITE, request, sizeof(*request)) - || !access_ok(VERIFY_WRITE, argp, sizeof(*argp)) + if (!request || !access_ok(VERIFY_WRITE, argp, sizeof(*argp)) || __get_user(x, &argp->size) || __put_user(x, &request->size)) return -EFAULT; @@ -925,8 +923,7 @@ static int compat_drm_sg_free(struct file *file, unsigned int cmd, unsigned long x; request = compat_alloc_user_space(sizeof(*request)); - if (!access_ok(VERIFY_WRITE, request, sizeof(*request)) - || !access_ok(VERIFY_WRITE, argp, sizeof(*argp)) + if (!request || !access_ok(VERIFY_WRITE, argp, sizeof(*argp)) || __get_user(x, &argp->handle) || __put_user(x << PAGE_SHIFT, &request->handle)) return -EFAULT; @@ -954,7 +951,7 @@ static int compat_drm_update_draw(struct file *file, unsigned int cmd, return -EFAULT; request = compat_alloc_user_space(sizeof(*request)); - if (!access_ok(VERIFY_WRITE, request, sizeof(*request)) || + if (!request || __put_user(update32.handle, &request->handle) || __put_user(update32.type, &request->type) || __put_user(update32.num, &request->num) || @@ -996,7 +993,7 @@ static int compat_drm_wait_vblank(struct file *file, unsigned int cmd, return -EFAULT; request = compat_alloc_user_space(sizeof(*request)); - if (!access_ok(VERIFY_WRITE, request, sizeof(*request)) + if (!request || __put_user(req32.request.type, &request->request.type) || __put_user(req32.request.sequence, &request->request.sequence) || __put_user(req32.request.signal, &request->request.signal)) diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c index b1d303fa2327..9a860ca1e9d7 100644 --- a/drivers/gpu/drm/drm_ioctl.c +++ b/drivers/gpu/drm/drm_ioctl.c @@ -480,7 +480,7 @@ static int drm_version(struct drm_device *dev, void *data, * indicated permissions. If so, returns zero. Otherwise returns an * error code suitable for ioctl return. */ -static int drm_ioctl_permit(u32 flags, struct drm_file *file_priv) +int drm_ioctl_permit(u32 flags, struct drm_file *file_priv) { /* ROOT_ONLY is only for CAP_SYS_ADMIN */ if (unlikely((flags & DRM_ROOT_ONLY) && !capable(CAP_SYS_ADMIN))) @@ -508,6 +508,7 @@ static int drm_ioctl_permit(u32 flags, struct drm_file *file_priv) return 0; } +EXPORT_SYMBOL(drm_ioctl_permit); #define DRM_IOCTL_DEF(ioctl, _func, _flags) \ [DRM_IOCTL_NR(ioctl)] = { \ diff --git a/drivers/gpu/drm/drm_irq.c b/drivers/gpu/drm/drm_irq.c index b50fa0afd907..22d207e211e7 100644 --- a/drivers/gpu/drm/drm_irq.c +++ b/drivers/gpu/drm/drm_irq.c @@ -43,8 +43,8 @@ #include /* Access macro for slots in vblank timestamp ringbuffer. */ -#define vblanktimestamp(dev, crtc, count) \ - ((dev)->vblank[crtc].time[(count) % DRM_VBLANKTIME_RBSIZE]) +#define vblanktimestamp(dev, pipe, count) \ + ((dev)->vblank[pipe].time[(count) % DRM_VBLANKTIME_RBSIZE]) /* Retry timestamp calculation up to 3 times to satisfy * drm_timestamp_precision before giving up. @@ -57,7 +57,7 @@ #define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000 static bool -drm_get_last_vbltimestamp(struct drm_device *dev, int crtc, +drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe, struct timeval *tvblank, unsigned flags); static unsigned int drm_timestamp_precision = 20; /* Default to 20 usecs. */ @@ -107,7 +107,7 @@ static void store_vblank(struct drm_device *dev, int crtc, /** * drm_update_vblank_count - update the master vblank counter * @dev: DRM device - * @crtc: counter to update + * @pipe: counter to update * * Call back into the driver to update the appropriate vblank counter * (specified by @crtc). Deal with wraparound, if it occurred, and @@ -120,9 +120,9 @@ static void store_vblank(struct drm_device *dev, int crtc, * Note: caller must hold dev->vbl_lock since this reads & writes * device vblank fields. */ -static void drm_update_vblank_count(struct drm_device *dev, int crtc) +static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe) { - struct drm_vblank_crtc *vblank = &dev->vblank[crtc]; + struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; u32 cur_vblank, diff; bool rc; struct timeval t_vblank; @@ -140,21 +140,21 @@ static void drm_update_vblank_count(struct drm_device *dev, int crtc) * corresponding vblank timestamp. */ do { - cur_vblank = dev->driver->get_vblank_counter(dev, crtc); - rc = drm_get_last_vbltimestamp(dev, crtc, &t_vblank, 0); - } while (cur_vblank != dev->driver->get_vblank_counter(dev, crtc)); + cur_vblank = dev->driver->get_vblank_counter(dev, pipe); + rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, 0); + } while (cur_vblank != dev->driver->get_vblank_counter(dev, pipe)); /* Deal with counter wrap */ diff = cur_vblank - vblank->last; if (cur_vblank < vblank->last) { diff += dev->max_vblank_count + 1; - DRM_DEBUG("last_vblank[%d]=0x%x, cur_vblank=0x%x => diff=0x%x\n", - crtc, vblank->last, cur_vblank, diff); + DRM_DEBUG("last_vblank[%u]=0x%x, cur_vblank=0x%x => diff=0x%x\n", + pipe, vblank->last, cur_vblank, diff); } - DRM_DEBUG("updating vblank count on crtc %d, missed %d\n", - crtc, diff); + DRM_DEBUG("updating vblank count on crtc %u, missed %d\n", + pipe, diff); if (diff == 0) return; @@ -167,7 +167,7 @@ static void drm_update_vblank_count(struct drm_device *dev, int crtc) if (!rc) t_vblank = (struct timeval) {0, 0}; - store_vblank(dev, crtc, diff, &t_vblank); + store_vblank(dev, pipe, diff, &t_vblank); } /* @@ -176,9 +176,9 @@ static void drm_update_vblank_count(struct drm_device *dev, int crtc) * are preserved, even if there are any spurious vblank irq's after * disable. */ -static void vblank_disable_and_save(struct drm_device *dev, int crtc) +static void vblank_disable_and_save(struct drm_device *dev, unsigned int pipe) { - struct drm_vblank_crtc *vblank = &dev->vblank[crtc]; + struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; unsigned long irqflags; u32 vblcount; s64 diff_ns; @@ -206,8 +206,8 @@ static void vblank_disable_and_save(struct drm_device *dev, int crtc) * vblank interrupt is disabled. */ if (!vblank->enabled && - drm_get_last_vbltimestamp(dev, crtc, &tvblank, 0)) { - drm_update_vblank_count(dev, crtc); + drm_get_last_vbltimestamp(dev, pipe, &tvblank, 0)) { + drm_update_vblank_count(dev, pipe); spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags); return; } @@ -218,7 +218,7 @@ static void vblank_disable_and_save(struct drm_device *dev, int crtc) * hardware potentially runtime suspended. */ if (vblank->enabled) { - dev->driver->disable_vblank(dev, crtc); + dev->driver->disable_vblank(dev, pipe); vblank->enabled = false; } @@ -235,9 +235,9 @@ static void vblank_disable_and_save(struct drm_device *dev, int crtc) * delayed gpu counter increment. */ do { - vblank->last = dev->driver->get_vblank_counter(dev, crtc); - vblrc = drm_get_last_vbltimestamp(dev, crtc, &tvblank, 0); - } while (vblank->last != dev->driver->get_vblank_counter(dev, crtc) && (--count) && vblrc); + vblank->last = dev->driver->get_vblank_counter(dev, pipe); + vblrc = drm_get_last_vbltimestamp(dev, pipe, &tvblank, 0); + } while (vblank->last != dev->driver->get_vblank_counter(dev, pipe) && (--count) && vblrc); if (!count) vblrc = 0; @@ -247,7 +247,7 @@ static void vblank_disable_and_save(struct drm_device *dev, int crtc) */ vblcount = vblank->count; diff_ns = timeval_to_ns(&tvblank) - - timeval_to_ns(&vblanktimestamp(dev, crtc, vblcount)); + timeval_to_ns(&vblanktimestamp(dev, pipe, vblcount)); /* If there is at least 1 msec difference between the last stored * timestamp and tvblank, then we are currently executing our @@ -262,7 +262,7 @@ static void vblank_disable_and_save(struct drm_device *dev, int crtc) * hope for the best. */ if (vblrc && (abs64(diff_ns) > 1000000)) - store_vblank(dev, crtc, 1, &tvblank); + store_vblank(dev, pipe, 1, &tvblank); spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags); } @@ -271,16 +271,16 @@ static void vblank_disable_fn(unsigned long arg) { struct drm_vblank_crtc *vblank = (void *)arg; struct drm_device *dev = vblank->dev; + unsigned int pipe = vblank->pipe; unsigned long irqflags; - int crtc = vblank->crtc; if (!dev->vblank_disable_allowed) return; spin_lock_irqsave(&dev->vbl_lock, irqflags); if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) { - DRM_DEBUG("disabling vblank on crtc %d\n", crtc); - vblank_disable_and_save(dev, crtc); + DRM_DEBUG("disabling vblank on crtc %u\n", pipe); + vblank_disable_and_save(dev, pipe); } spin_unlock_irqrestore(&dev->vbl_lock, irqflags); } @@ -293,14 +293,14 @@ static void vblank_disable_fn(unsigned long arg) */ void drm_vblank_cleanup(struct drm_device *dev) { - int crtc; + unsigned int pipe; /* Bail if the driver didn't call drm_vblank_init() */ if (dev->num_crtcs == 0) return; - for (crtc = 0; crtc < dev->num_crtcs; crtc++) { - struct drm_vblank_crtc *vblank = &dev->vblank[crtc]; + for (pipe = 0; pipe < dev->num_crtcs; pipe++) { + struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; WARN_ON(vblank->enabled && drm_core_check_feature(dev, DRIVER_MODESET)); @@ -316,17 +316,18 @@ EXPORT_SYMBOL(drm_vblank_cleanup); /** * drm_vblank_init - initialize vblank support - * @dev: drm_device - * @num_crtcs: number of crtcs supported by @dev + * @dev: DRM device + * @num_crtcs: number of CRTCs supported by @dev * * This function initializes vblank support for @num_crtcs display pipelines. * * Returns: * Zero on success or a negative error code on failure. */ -int drm_vblank_init(struct drm_device *dev, int num_crtcs) +int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs) { - int i, ret = -ENOMEM; + int ret = -ENOMEM; + unsigned int i; spin_lock_init(&dev->vbl_lock); spin_lock_init(&dev->vblank_time_lock); @@ -341,7 +342,7 @@ int drm_vblank_init(struct drm_device *dev, int num_crtcs) struct drm_vblank_crtc *vblank = &dev->vblank[i]; vblank->dev = dev; - vblank->crtc = i; + vblank->pipe = i; init_waitqueue_head(&vblank->queue); setup_timer(&vblank->disable_timer, vblank_disable_fn, (unsigned long)vblank); @@ -624,17 +625,17 @@ void drm_calc_timestamping_constants(struct drm_crtc *crtc, if (mode->flags & DRM_MODE_FLAG_INTERLACE) framedur_ns /= 2; } else - DRM_ERROR("crtc %d: Can't calculate constants, dotclock = 0!\n", + DRM_ERROR("crtc %u: Can't calculate constants, dotclock = 0!\n", crtc->base.id); crtc->pixeldur_ns = pixeldur_ns; crtc->linedur_ns = linedur_ns; crtc->framedur_ns = framedur_ns; - DRM_DEBUG("crtc %d: hwmode: htotal %d, vtotal %d, vdisplay %d\n", + DRM_DEBUG("crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n", crtc->base.id, mode->crtc_htotal, mode->crtc_vtotal, mode->crtc_vdisplay); - DRM_DEBUG("crtc %d: clock %d kHz framedur %d linedur %d, pixeldur %d\n", + DRM_DEBUG("crtc %u: clock %d kHz framedur %d linedur %d, pixeldur %d\n", crtc->base.id, dotclock, framedur_ns, linedur_ns, pixeldur_ns); } @@ -643,7 +644,7 @@ EXPORT_SYMBOL(drm_calc_timestamping_constants); /** * drm_calc_vbltimestamp_from_scanoutpos - precise vblank timestamp helper * @dev: DRM device - * @crtc: Which CRTC's vblank timestamp to retrieve + * @pipe: index of CRTC whose vblank timestamp to retrieve * @max_error: Desired maximum allowable error in timestamps (nanosecs) * On return contains true maximum error of timestamp * @vblank_time: Pointer to struct timeval which should receive the timestamp @@ -686,7 +687,8 @@ EXPORT_SYMBOL(drm_calc_timestamping_constants); * DRM_VBLANKTIME_INVBL - Timestamp taken while scanout was in vblank interval. * */ -int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc, +int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, + unsigned int pipe, int *max_error, struct timeval *vblank_time, unsigned flags, @@ -700,8 +702,8 @@ int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc, int framedur_ns, linedur_ns, pixeldur_ns, delta_ns, duration_ns; bool invbl; - if (crtc < 0 || crtc >= dev->num_crtcs) { - DRM_ERROR("Invalid crtc %d\n", crtc); + if (pipe >= dev->num_crtcs) { + DRM_ERROR("Invalid crtc %u\n", pipe); return -EINVAL; } @@ -720,7 +722,7 @@ int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc, * Happens during initial modesetting of a crtc. */ if (framedur_ns == 0) { - DRM_DEBUG("crtc %d: Noop due to uninitialized mode.\n", crtc); + DRM_DEBUG("crtc %u: Noop due to uninitialized mode.\n", pipe); return -EAGAIN; } @@ -736,13 +738,13 @@ int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc, * Get vertical and horizontal scanout position vpos, hpos, * and bounding timestamps stime, etime, pre/post query. */ - vbl_status = dev->driver->get_scanout_position(dev, crtc, flags, &vpos, + vbl_status = dev->driver->get_scanout_position(dev, pipe, flags, &vpos, &hpos, &stime, &etime); /* Return as no-op if scanout query unsupported or failed. */ if (!(vbl_status & DRM_SCANOUTPOS_VALID)) { - DRM_DEBUG("crtc %d : scanoutpos query failed [%d].\n", - crtc, vbl_status); + DRM_DEBUG("crtc %u : scanoutpos query failed [%d].\n", + pipe, vbl_status); return -EIO; } @@ -756,8 +758,8 @@ int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc, /* Noisy system timing? */ if (i == DRM_TIMESTAMP_MAXRETRIES) { - DRM_DEBUG("crtc %d: Noisy timestamp %d us > %d us [%d reps].\n", - crtc, duration_ns/1000, *max_error/1000, i); + DRM_DEBUG("crtc %u: Noisy timestamp %d us > %d us [%d reps].\n", + pipe, duration_ns/1000, *max_error/1000, i); } /* Return upper bound of timestamp precision error. */ @@ -790,8 +792,8 @@ int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc, etime = ktime_sub_ns(etime, delta_ns); *vblank_time = ktime_to_timeval(etime); - DRM_DEBUG("crtc %d : v %d p(%d,%d)@ %ld.%ld -> %ld.%ld [e %d us, %d rep]\n", - crtc, (int)vbl_status, hpos, vpos, + DRM_DEBUG("crtc %u : v %d p(%d,%d)@ %ld.%ld -> %ld.%ld [e %d us, %d rep]\n", + pipe, (int)vbl_status, hpos, vpos, (long)tv_etime.tv_sec, (long)tv_etime.tv_usec, (long)vblank_time->tv_sec, (long)vblank_time->tv_usec, duration_ns/1000, i); @@ -816,7 +818,7 @@ static struct timeval get_drm_timestamp(void) * drm_get_last_vbltimestamp - retrieve raw timestamp for the most recent * vblank interval * @dev: DRM device - * @crtc: which CRTC's vblank timestamp to retrieve + * @pipe: index of CRTC whose vblank timestamp to retrieve * @tvblank: Pointer to target struct timeval which should receive the timestamp * @flags: Flags to pass to driver: * 0 = Default, @@ -833,7 +835,7 @@ static struct timeval get_drm_timestamp(void) * True if timestamp is considered to be very precise, false otherwise. */ static bool -drm_get_last_vbltimestamp(struct drm_device *dev, int crtc, +drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe, struct timeval *tvblank, unsigned flags) { int ret; @@ -843,7 +845,7 @@ drm_get_last_vbltimestamp(struct drm_device *dev, int crtc, /* Query driver if possible and precision timestamping enabled. */ if (dev->driver->get_vblank_timestamp && (max_error > 0)) { - ret = dev->driver->get_vblank_timestamp(dev, crtc, &max_error, + ret = dev->driver->get_vblank_timestamp(dev, pipe, &max_error, tvblank, flags); if (ret > 0) return true; @@ -860,7 +862,7 @@ drm_get_last_vbltimestamp(struct drm_device *dev, int crtc, /** * drm_vblank_count - retrieve "cooked" vblank counter value * @dev: DRM device - * @crtc: which counter to retrieve + * @pipe: index of CRTC for which to retrieve the counter * * Fetches the "cooked" vblank count value that represents the number of * vblank events since the system was booted, including lost events due to @@ -871,12 +873,13 @@ drm_get_last_vbltimestamp(struct drm_device *dev, int crtc, * Returns: * The software vblank counter. */ -u32 drm_vblank_count(struct drm_device *dev, int crtc) +u32 drm_vblank_count(struct drm_device *dev, int pipe) { - struct drm_vblank_crtc *vblank = &dev->vblank[crtc]; + struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; - if (WARN_ON(crtc >= dev->num_crtcs)) + if (WARN_ON(pipe >= dev->num_crtcs)) return 0; + return vblank->count; } EXPORT_SYMBOL(drm_vblank_count); @@ -901,11 +904,10 @@ u32 drm_crtc_vblank_count(struct drm_crtc *crtc) EXPORT_SYMBOL(drm_crtc_vblank_count); /** - * drm_vblank_count_and_time - retrieve "cooked" vblank counter value - * and the system timestamp corresponding to that vblank counter value. - * + * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the + * system timestamp corresponding to that vblank counter value. * @dev: DRM device - * @crtc: which counter to retrieve + * @pipe: index of CRTC whose counter to retrieve * @vblanktime: Pointer to struct timeval to receive the vblank timestamp. * * Fetches the "cooked" vblank count value that represents the number of @@ -913,13 +915,13 @@ EXPORT_SYMBOL(drm_crtc_vblank_count); * modesetting activity. Returns corresponding system timestamp of the time * of the vblank interval that corresponds to the current vblank counter value. */ -u32 drm_vblank_count_and_time(struct drm_device *dev, int crtc, +u32 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe, struct timeval *vblanktime) { - struct drm_vblank_crtc *vblank = &dev->vblank[crtc]; + struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; u32 cur_vblank; - if (WARN_ON(crtc >= dev->num_crtcs)) + if (WARN_ON(pipe >= dev->num_crtcs)) return 0; /* @@ -930,7 +932,7 @@ u32 drm_vblank_count_and_time(struct drm_device *dev, int crtc, do { cur_vblank = vblank->count; smp_rmb(); - *vblanktime = vblanktimestamp(dev, crtc, cur_vblank); + *vblanktime = vblanktimestamp(dev, pipe, cur_vblank); smp_rmb(); } while (cur_vblank != vblank->count); @@ -957,7 +959,7 @@ static void send_vblank_event(struct drm_device *dev, /** * drm_send_vblank_event - helper to send vblank event after pageflip * @dev: DRM device - * @crtc: CRTC in question + * @pipe: CRTC index * @e: the event to send * * Updates sequence # and timestamp on event, and sends it to userspace. @@ -965,20 +967,20 @@ static void send_vblank_event(struct drm_device *dev, * * This is the legacy version of drm_crtc_send_vblank_event(). */ -void drm_send_vblank_event(struct drm_device *dev, int crtc, - struct drm_pending_vblank_event *e) +void drm_send_vblank_event(struct drm_device *dev, unsigned int pipe, + struct drm_pending_vblank_event *e) { struct timeval now; unsigned int seq; - if (crtc >= 0) { - seq = drm_vblank_count_and_time(dev, crtc, &now); + if (dev->num_crtcs > 0) { + seq = drm_vblank_count_and_time(dev, pipe, &now); } else { seq = 0; now = get_drm_timestamp(); } - e->pipe = crtc; + e->pipe = pipe; send_vblank_event(dev, e, seq, &now); } EXPORT_SYMBOL(drm_send_vblank_event); @@ -1003,11 +1005,14 @@ EXPORT_SYMBOL(drm_crtc_send_vblank_event); /** * drm_vblank_enable - enable the vblank interrupt on a CRTC * @dev: DRM device - * @crtc: CRTC in question + * @pipe: CRTC index + * + * Returns: + * Zero on success or a negative error code on failure. */ -static int drm_vblank_enable(struct drm_device *dev, int crtc) +static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe) { - struct drm_vblank_crtc *vblank = &dev->vblank[crtc]; + struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; int ret = 0; assert_spin_locked(&dev->vbl_lock); @@ -1022,13 +1027,13 @@ static int drm_vblank_enable(struct drm_device *dev, int crtc) * timestamps. Filtercode in drm_handle_vblank() will * prevent double-accounting of same vblank interval. */ - ret = dev->driver->enable_vblank(dev, crtc); - DRM_DEBUG("enabling vblank on crtc %d, ret: %d\n", crtc, ret); + ret = dev->driver->enable_vblank(dev, pipe); + DRM_DEBUG("enabling vblank on crtc %u, ret: %d\n", pipe, ret); if (ret) atomic_dec(&vblank->refcount); else { vblank->enabled = true; - drm_update_vblank_count(dev, crtc); + drm_update_vblank_count(dev, pipe); } } @@ -1040,7 +1045,7 @@ static int drm_vblank_enable(struct drm_device *dev, int crtc) /** * drm_vblank_get - get a reference count on vblank events * @dev: DRM device - * @crtc: which CRTC to own + * @pipe: index of CRTC to own * * Acquire a reference count on vblank events to avoid having them disabled * while in use. @@ -1048,24 +1053,24 @@ static int drm_vblank_enable(struct drm_device *dev, int crtc) * This is the legacy version of drm_crtc_vblank_get(). * * Returns: - * Zero on success, nonzero on failure. + * Zero on success or a negative error code on failure. */ -int drm_vblank_get(struct drm_device *dev, int crtc) +int drm_vblank_get(struct drm_device *dev, unsigned int pipe) { - struct drm_vblank_crtc *vblank = &dev->vblank[crtc]; + struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; unsigned long irqflags; int ret = 0; if (!dev->num_crtcs) return -EINVAL; - if (WARN_ON(crtc >= dev->num_crtcs)) + if (WARN_ON(pipe >= dev->num_crtcs)) return -EINVAL; spin_lock_irqsave(&dev->vbl_lock, irqflags); /* Going from 0->1 means we have to enable interrupts again */ if (atomic_add_return(1, &vblank->refcount) == 1) { - ret = drm_vblank_enable(dev, crtc); + ret = drm_vblank_enable(dev, pipe); } else { if (!vblank->enabled) { atomic_dec(&vblank->refcount); @@ -1088,7 +1093,7 @@ EXPORT_SYMBOL(drm_vblank_get); * This is the native kms version of drm_vblank_get(). * * Returns: - * Zero on success, nonzero on failure. + * Zero on success or a negative error code on failure. */ int drm_crtc_vblank_get(struct drm_crtc *crtc) { @@ -1097,23 +1102,23 @@ int drm_crtc_vblank_get(struct drm_crtc *crtc) EXPORT_SYMBOL(drm_crtc_vblank_get); /** - * drm_vblank_put - give up ownership of vblank events + * drm_vblank_put - release ownership of vblank events * @dev: DRM device - * @crtc: which counter to give up + * @pipe: index of CRTC to release * * Release ownership of a given vblank counter, turning off interrupts * if possible. Disable interrupts after drm_vblank_offdelay milliseconds. * * This is the legacy version of drm_crtc_vblank_put(). */ -void drm_vblank_put(struct drm_device *dev, int crtc) +void drm_vblank_put(struct drm_device *dev, unsigned int pipe) { - struct drm_vblank_crtc *vblank = &dev->vblank[crtc]; + struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; - if (WARN_ON(atomic_read(&vblank->refcount) == 0)) + if (WARN_ON(pipe >= dev->num_crtcs)) return; - if (WARN_ON(crtc >= dev->num_crtcs)) + if (WARN_ON(atomic_read(&vblank->refcount) == 0)) return; /* Last user schedules interrupt disable */ @@ -1147,30 +1152,34 @@ EXPORT_SYMBOL(drm_crtc_vblank_put); /** * drm_wait_one_vblank - wait for one vblank * @dev: DRM device - * @crtc: crtc index + * @pipe: CRTC index * * This waits for one vblank to pass on @crtc, using the irq driver interfaces. * It is a failure to call this when the vblank irq for @crtc is disabled, e.g. * due to lack of driver support or because the crtc is off. */ -void drm_wait_one_vblank(struct drm_device *dev, int crtc) +void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe) { + struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; int ret; u32 last; - ret = drm_vblank_get(dev, crtc); - if (WARN(ret, "vblank not available on crtc %i, ret=%i\n", crtc, ret)) + if (WARN_ON(pipe >= dev->num_crtcs)) return; - last = drm_vblank_count(dev, crtc); + ret = drm_vblank_get(dev, pipe); + if (WARN(ret, "vblank not available on crtc %i, ret=%i\n", pipe, ret)) + return; - ret = wait_event_timeout(dev->vblank[crtc].queue, - last != drm_vblank_count(dev, crtc), + last = drm_vblank_count(dev, pipe); + + ret = wait_event_timeout(vblank->queue, + last != drm_vblank_count(dev, pipe), msecs_to_jiffies(100)); - WARN(ret == 0, "vblank wait timed out on crtc %i\n", crtc); + WARN(ret == 0, "vblank wait timed out on crtc %i\n", pipe); - drm_vblank_put(dev, crtc); + drm_vblank_put(dev, pipe); } EXPORT_SYMBOL(drm_wait_one_vblank); @@ -1191,7 +1200,7 @@ EXPORT_SYMBOL(drm_crtc_wait_one_vblank); /** * drm_vblank_off - disable vblank events on a CRTC * @dev: DRM device - * @crtc: CRTC in question + * @pipe: CRTC index * * Drivers can use this function to shut down the vblank interrupt handling when * disabling a crtc. This function ensures that the latest vblank frame count is @@ -1202,21 +1211,21 @@ EXPORT_SYMBOL(drm_crtc_wait_one_vblank); * * This is the legacy version of drm_crtc_vblank_off(). */ -void drm_vblank_off(struct drm_device *dev, int crtc) +void drm_vblank_off(struct drm_device *dev, unsigned int pipe) { - struct drm_vblank_crtc *vblank = &dev->vblank[crtc]; + struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; struct drm_pending_vblank_event *e, *t; struct timeval now; unsigned long irqflags; unsigned int seq; - if (WARN_ON(crtc >= dev->num_crtcs)) + if (WARN_ON(pipe >= dev->num_crtcs)) return; spin_lock_irqsave(&dev->event_lock, irqflags); spin_lock(&dev->vbl_lock); - vblank_disable_and_save(dev, crtc); + vblank_disable_and_save(dev, pipe); wake_up(&vblank->queue); /* @@ -1230,16 +1239,16 @@ void drm_vblank_off(struct drm_device *dev, int crtc) spin_unlock(&dev->vbl_lock); /* Send any queued vblank events, lest the natives grow disquiet */ - seq = drm_vblank_count_and_time(dev, crtc, &now); + seq = drm_vblank_count_and_time(dev, pipe, &now); list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) { - if (e->pipe != crtc) + if (e->pipe != pipe) continue; DRM_DEBUG("Sending premature vblank event on disable: \ wanted %d, current %d\n", e->event.sequence, seq); list_del(&e->base.link); - drm_vblank_put(dev, e->pipe); + drm_vblank_put(dev, pipe); send_vblank_event(dev, e, seq, &now); } spin_unlock_irqrestore(&dev->event_lock, irqflags); @@ -1267,7 +1276,7 @@ EXPORT_SYMBOL(drm_crtc_vblank_off); /** * drm_crtc_vblank_reset - reset vblank state to off on a CRTC - * @crtc: CRTC in question + * @drm_crtc: CRTC in question * * Drivers can use this function to reset the vblank state to off at load time. * Drivers should use this together with the drm_crtc_vblank_off() and @@ -1300,7 +1309,7 @@ EXPORT_SYMBOL(drm_crtc_vblank_reset); /** * drm_vblank_on - enable vblank events on a CRTC * @dev: DRM device - * @crtc: CRTC in question + * @pipe: CRTC index * * This functions restores the vblank interrupt state captured with * drm_vblank_off() again. Note that calls to drm_vblank_on() and @@ -1309,12 +1318,12 @@ EXPORT_SYMBOL(drm_crtc_vblank_reset); * * This is the legacy version of drm_crtc_vblank_on(). */ -void drm_vblank_on(struct drm_device *dev, int crtc) +void drm_vblank_on(struct drm_device *dev, unsigned int pipe) { - struct drm_vblank_crtc *vblank = &dev->vblank[crtc]; + struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; unsigned long irqflags; - if (WARN_ON(crtc >= dev->num_crtcs)) + if (WARN_ON(pipe >= dev->num_crtcs)) return; spin_lock_irqsave(&dev->vbl_lock, irqflags); @@ -1332,7 +1341,7 @@ void drm_vblank_on(struct drm_device *dev, int crtc) * vblank counter value before and after a modeset */ vblank->last = - (dev->driver->get_vblank_counter(dev, crtc) - 1) & + (dev->driver->get_vblank_counter(dev, pipe) - 1) & dev->max_vblank_count; /* * re-enable interrupts if there are users left, or the @@ -1340,7 +1349,7 @@ void drm_vblank_on(struct drm_device *dev, int crtc) */ if (atomic_read(&vblank->refcount) != 0 || (!dev->vblank_disable_immediate && drm_vblank_offdelay == 0)) - WARN_ON(drm_vblank_enable(dev, crtc)); + WARN_ON(drm_vblank_enable(dev, pipe)); spin_unlock_irqrestore(&dev->vbl_lock, irqflags); } EXPORT_SYMBOL(drm_vblank_on); @@ -1365,7 +1374,7 @@ EXPORT_SYMBOL(drm_crtc_vblank_on); /** * drm_vblank_pre_modeset - account for vblanks across mode sets * @dev: DRM device - * @crtc: CRTC in question + * @pipe: CRTC index * * Account for vblank events across mode setting events, which will likely * reset the hardware frame counter. @@ -1385,15 +1394,15 @@ EXPORT_SYMBOL(drm_crtc_vblank_on); * Drivers must call drm_vblank_post_modeset() when re-enabling the same crtc * again. */ -void drm_vblank_pre_modeset(struct drm_device *dev, int crtc) +void drm_vblank_pre_modeset(struct drm_device *dev, unsigned int pipe) { - struct drm_vblank_crtc *vblank = &dev->vblank[crtc]; + struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; /* vblank is not initialized (IRQ not installed ?), or has been freed */ if (!dev->num_crtcs) return; - if (WARN_ON(crtc >= dev->num_crtcs)) + if (WARN_ON(pipe >= dev->num_crtcs)) return; /* @@ -1405,7 +1414,7 @@ void drm_vblank_pre_modeset(struct drm_device *dev, int crtc) */ if (!vblank->inmodeset) { vblank->inmodeset = 0x1; - if (drm_vblank_get(dev, crtc) == 0) + if (drm_vblank_get(dev, pipe) == 0) vblank->inmodeset |= 0x2; } } @@ -1414,27 +1423,30 @@ EXPORT_SYMBOL(drm_vblank_pre_modeset); /** * drm_vblank_post_modeset - undo drm_vblank_pre_modeset changes * @dev: DRM device - * @crtc: CRTC in question + * @pipe: CRTC index * * This function again drops the temporary vblank reference acquired in * drm_vblank_pre_modeset. */ -void drm_vblank_post_modeset(struct drm_device *dev, int crtc) +void drm_vblank_post_modeset(struct drm_device *dev, unsigned int pipe) { - struct drm_vblank_crtc *vblank = &dev->vblank[crtc]; + struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; unsigned long irqflags; /* vblank is not initialized (IRQ not installed ?), or has been freed */ if (!dev->num_crtcs) return; + if (WARN_ON(pipe >= dev->num_crtcs)) + return; + if (vblank->inmodeset) { spin_lock_irqsave(&dev->vbl_lock, irqflags); dev->vblank_disable_allowed = true; spin_unlock_irqrestore(&dev->vbl_lock, irqflags); if (vblank->inmodeset & 0x2) - drm_vblank_put(dev, crtc); + drm_vblank_put(dev, pipe); vblank->inmodeset = 0; } @@ -1456,7 +1468,7 @@ int drm_modeset_ctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct drm_modeset_ctl *modeset = data; - unsigned int crtc; + unsigned int pipe; /* If drm_vblank_init() hasn't been called yet, just no-op */ if (!dev->num_crtcs) @@ -1466,16 +1478,16 @@ int drm_modeset_ctl(struct drm_device *dev, void *data, if (drm_core_check_feature(dev, DRIVER_MODESET)) return 0; - crtc = modeset->crtc; - if (crtc >= dev->num_crtcs) + pipe = modeset->crtc; + if (pipe >= dev->num_crtcs) return -EINVAL; switch (modeset->cmd) { case _DRM_PRE_MODESET: - drm_vblank_pre_modeset(dev, crtc); + drm_vblank_pre_modeset(dev, pipe); break; case _DRM_POST_MODESET: - drm_vblank_post_modeset(dev, crtc); + drm_vblank_post_modeset(dev, pipe); break; default: return -EINVAL; @@ -1484,7 +1496,7 @@ int drm_modeset_ctl(struct drm_device *dev, void *data, return 0; } -static int drm_queue_vblank_event(struct drm_device *dev, int pipe, +static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe, union drm_wait_vblank *vblwait, struct drm_file *file_priv) { @@ -1538,7 +1550,7 @@ static int drm_queue_vblank_event(struct drm_device *dev, int pipe, vblwait->reply.sequence = vblwait->request.sequence; } - DRM_DEBUG("event on vblank count %d, current %d, crtc %d\n", + DRM_DEBUG("event on vblank count %d, current %d, crtc %u\n", vblwait->request.sequence, seq, pipe); trace_drm_vblank_event_queued(current->pid, pipe, @@ -1587,7 +1599,7 @@ int drm_wait_vblank(struct drm_device *dev, void *data, struct drm_vblank_crtc *vblank; union drm_wait_vblank *vblwait = data; int ret; - unsigned int flags, seq, crtc, high_crtc; + unsigned int flags, seq, pipe, high_pipe; if (!dev->irq_enabled) return -EINVAL; @@ -1606,22 +1618,22 @@ int drm_wait_vblank(struct drm_device *dev, void *data, } flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK; - high_crtc = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK); - if (high_crtc) - crtc = high_crtc >> _DRM_VBLANK_HIGH_CRTC_SHIFT; + high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK); + if (high_pipe) + pipe = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT; else - crtc = flags & _DRM_VBLANK_SECONDARY ? 1 : 0; - if (crtc >= dev->num_crtcs) + pipe = flags & _DRM_VBLANK_SECONDARY ? 1 : 0; + if (pipe >= dev->num_crtcs) return -EINVAL; - vblank = &dev->vblank[crtc]; + vblank = &dev->vblank[pipe]; - ret = drm_vblank_get(dev, crtc); + ret = drm_vblank_get(dev, pipe); if (ret) { DRM_DEBUG("failed to acquire vblank counter, %d\n", ret); return ret; } - seq = drm_vblank_count(dev, crtc); + seq = drm_vblank_count(dev, pipe); switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) { case _DRM_VBLANK_RELATIVE: @@ -1638,7 +1650,7 @@ int drm_wait_vblank(struct drm_device *dev, void *data, /* must hold on to the vblank ref until the event fires * drm_vblank_put will be called asynchronously */ - return drm_queue_vblank_event(dev, crtc, vblwait, file_priv); + return drm_queue_vblank_event(dev, pipe, vblwait, file_priv); } if ((flags & _DRM_VBLANK_NEXTONMISS) && @@ -1646,11 +1658,11 @@ int drm_wait_vblank(struct drm_device *dev, void *data, vblwait->request.sequence = seq + 1; } - DRM_DEBUG("waiting on vblank count %d, crtc %d\n", - vblwait->request.sequence, crtc); + DRM_DEBUG("waiting on vblank count %d, crtc %u\n", + vblwait->request.sequence, pipe); vblank->last_wait = vblwait->request.sequence; DRM_WAIT_ON(ret, vblank->queue, 3 * HZ, - (((drm_vblank_count(dev, crtc) - + (((drm_vblank_count(dev, pipe) - vblwait->request.sequence) <= (1 << 23)) || !vblank->enabled || !dev->irq_enabled)); @@ -1658,7 +1670,7 @@ int drm_wait_vblank(struct drm_device *dev, void *data, if (ret != -EINTR) { struct timeval now; - vblwait->reply.sequence = drm_vblank_count_and_time(dev, crtc, &now); + vblwait->reply.sequence = drm_vblank_count_and_time(dev, pipe, &now); vblwait->reply.tval_sec = now.tv_sec; vblwait->reply.tval_usec = now.tv_usec; @@ -1669,11 +1681,11 @@ int drm_wait_vblank(struct drm_device *dev, void *data, } done: - drm_vblank_put(dev, crtc); + drm_vblank_put(dev, pipe); return ret; } -static void drm_handle_vblank_events(struct drm_device *dev, int crtc) +static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe) { struct drm_pending_vblank_event *e, *t; struct timeval now; @@ -1681,10 +1693,10 @@ static void drm_handle_vblank_events(struct drm_device *dev, int crtc) assert_spin_locked(&dev->event_lock); - seq = drm_vblank_count_and_time(dev, crtc, &now); + seq = drm_vblank_count_and_time(dev, pipe, &now); list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) { - if (e->pipe != crtc) + if (e->pipe != pipe) continue; if ((seq - e->event.sequence) > (1<<23)) continue; @@ -1693,26 +1705,26 @@ static void drm_handle_vblank_events(struct drm_device *dev, int crtc) e->event.sequence, seq); list_del(&e->base.link); - drm_vblank_put(dev, e->pipe); + drm_vblank_put(dev, pipe); send_vblank_event(dev, e, seq, &now); } - trace_drm_vblank_event(crtc, seq); + trace_drm_vblank_event(pipe, seq); } /** * drm_handle_vblank - handle a vblank event * @dev: DRM device - * @crtc: where this event occurred + * @pipe: index of CRTC where this event occurred * * Drivers should call this routine in their vblank interrupt handlers to * update the vblank counter and send any signals that may be pending. * * This is the legacy version of drm_crtc_handle_vblank(). */ -bool drm_handle_vblank(struct drm_device *dev, int crtc) +bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe) { - struct drm_vblank_crtc *vblank = &dev->vblank[crtc]; + struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; u32 vblcount; s64 diff_ns; struct timeval tvblank; @@ -1721,7 +1733,7 @@ bool drm_handle_vblank(struct drm_device *dev, int crtc) if (WARN_ON_ONCE(!dev->num_crtcs)) return false; - if (WARN_ON(crtc >= dev->num_crtcs)) + if (WARN_ON(pipe >= dev->num_crtcs)) return false; spin_lock_irqsave(&dev->event_lock, irqflags); @@ -1745,11 +1757,11 @@ bool drm_handle_vblank(struct drm_device *dev, int crtc) /* Get current timestamp and count. */ vblcount = vblank->count; - drm_get_last_vbltimestamp(dev, crtc, &tvblank, DRM_CALLED_FROM_VBLIRQ); + drm_get_last_vbltimestamp(dev, pipe, &tvblank, DRM_CALLED_FROM_VBLIRQ); /* Compute time difference to timestamp of last vblank */ diff_ns = timeval_to_ns(&tvblank) - - timeval_to_ns(&vblanktimestamp(dev, crtc, vblcount)); + timeval_to_ns(&vblanktimestamp(dev, pipe, vblcount)); /* Update vblank timestamp and count if at least * DRM_REDUNDANT_VBLIRQ_THRESH_NS nanoseconds @@ -1761,15 +1773,15 @@ bool drm_handle_vblank(struct drm_device *dev, int crtc) * ignore those for accounting. */ if (abs64(diff_ns) > DRM_REDUNDANT_VBLIRQ_THRESH_NS) - store_vblank(dev, crtc, 1, &tvblank); + store_vblank(dev, pipe, 1, &tvblank); else - DRM_DEBUG("crtc %d: Redundant vblirq ignored. diff_ns = %d\n", - crtc, (int) diff_ns); + DRM_DEBUG("crtc %u: Redundant vblirq ignored. diff_ns = %d\n", + pipe, (int) diff_ns); spin_unlock(&dev->vblank_time_lock); wake_up(&vblank->queue); - drm_handle_vblank_events(dev, crtc); + drm_handle_vblank_events(dev, pipe); spin_unlock_irqrestore(&dev->event_lock, irqflags); diff --git a/drivers/gpu/drm/drm_legacy.h b/drivers/gpu/drm/drm_legacy.h index c1dc61473db5..9b731786e4db 100644 --- a/drivers/gpu/drm/drm_legacy.h +++ b/drivers/gpu/drm/drm_legacy.h @@ -42,7 +42,7 @@ struct drm_file; #define DRM_KERNEL_CONTEXT 0 #define DRM_RESERVED_CONTEXTS 1 -int drm_legacy_ctxbitmap_init(struct drm_device *dev); +void drm_legacy_ctxbitmap_init(struct drm_device *dev); void drm_legacy_ctxbitmap_cleanup(struct drm_device *dev); void drm_legacy_ctxbitmap_free(struct drm_device *dev, int ctx_handle); void drm_legacy_ctxbitmap_flush(struct drm_device *dev, struct drm_file *file); diff --git a/drivers/gpu/drm/drm_lock.c b/drivers/gpu/drm/drm_lock.c index f861361a635e..4924d381b664 100644 --- a/drivers/gpu/drm/drm_lock.c +++ b/drivers/gpu/drm/drm_lock.c @@ -61,6 +61,9 @@ int drm_legacy_lock(struct drm_device *dev, void *data, struct drm_master *master = file_priv->master; int ret = 0; + if (drm_core_check_feature(dev, DRIVER_MODESET)) + return -EINVAL; + ++file_priv->lock_count; if (lock->context == DRM_KERNEL_CONTEXT) { @@ -153,6 +156,9 @@ int drm_legacy_unlock(struct drm_device *dev, void *data, struct drm_file *file_ struct drm_lock *lock = data; struct drm_master *master = file_priv->master; + if (drm_core_check_feature(dev, DRIVER_MODESET)) + return -EINVAL; + if (lock->context == DRM_KERNEL_CONTEXT) { DRM_ERROR("Process %d using kernel context %d\n", task_pid_nr(current), lock->context); diff --git a/drivers/gpu/drm/drm_modeset_lock.c b/drivers/gpu/drm/drm_modeset_lock.c index c0a5cd8c5262..fba321ca4344 100644 --- a/drivers/gpu/drm/drm_modeset_lock.c +++ b/drivers/gpu/drm/drm_modeset_lock.c @@ -55,41 +55,27 @@ * drm_modeset_acquire_fini(&ctx); */ - /** - * __drm_modeset_lock_all - internal helper to grab all modeset locks - * @dev: DRM device - * @trylock: trylock mode for atomic contexts + * drm_modeset_lock_all - take all modeset locks + * @dev: drm device * - * This is a special version of drm_modeset_lock_all() which can also be used in - * atomic contexts. Then @trylock must be set to true. - * - * Returns: - * 0 on success or negative error code on failure. + * This function takes all modeset locks, suitable where a more fine-grained + * scheme isn't (yet) implemented. Locks must be dropped with + * drm_modeset_unlock_all. */ -int __drm_modeset_lock_all(struct drm_device *dev, - bool trylock) +void drm_modeset_lock_all(struct drm_device *dev) { struct drm_mode_config *config = &dev->mode_config; struct drm_modeset_acquire_ctx *ctx; int ret; - ctx = kzalloc(sizeof(*ctx), - trylock ? GFP_ATOMIC : GFP_KERNEL); - if (!ctx) - return -ENOMEM; + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (WARN_ON(!ctx)) + return; - if (trylock) { - if (!mutex_trylock(&config->mutex)) { - ret = -EBUSY; - goto out; - } - } else { - mutex_lock(&config->mutex); - } + mutex_lock(&config->mutex); drm_modeset_acquire_init(ctx, 0); - ctx->trylock_only = trylock; retry: ret = drm_modeset_lock(&config->connection_mutex, ctx); @@ -108,7 +94,7 @@ int __drm_modeset_lock_all(struct drm_device *dev, drm_warn_on_modeset_not_all_locked(dev); - return 0; + return; fail: if (ret == -EDEADLK) { @@ -116,23 +102,7 @@ int __drm_modeset_lock_all(struct drm_device *dev, goto retry; } -out: kfree(ctx); - return ret; -} -EXPORT_SYMBOL(__drm_modeset_lock_all); - -/** - * drm_modeset_lock_all - take all modeset locks - * @dev: drm device - * - * This function takes all modeset locks, suitable where a more fine-grained - * scheme isn't (yet) implemented. Locks must be dropped with - * drm_modeset_unlock_all. - */ -void drm_modeset_lock_all(struct drm_device *dev) -{ - WARN_ON(__drm_modeset_lock_all(dev, false) != 0); } EXPORT_SYMBOL(drm_modeset_lock_all); @@ -276,7 +246,7 @@ void drm_warn_on_modeset_not_all_locked(struct drm_device *dev) if (oops_in_progress) return; - list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) + drm_for_each_crtc(crtc, dev) WARN_ON(!drm_modeset_is_locked(&crtc->mutex)); WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex)); @@ -464,18 +434,17 @@ EXPORT_SYMBOL(drm_modeset_unlock); int drm_modeset_lock_all_crtcs(struct drm_device *dev, struct drm_modeset_acquire_ctx *ctx) { - struct drm_mode_config *config = &dev->mode_config; struct drm_crtc *crtc; struct drm_plane *plane; int ret = 0; - list_for_each_entry(crtc, &config->crtc_list, head) { + drm_for_each_crtc(crtc, dev) { ret = drm_modeset_lock(&crtc->mutex, ctx); if (ret) return ret; } - list_for_each_entry(plane, &config->plane_list, head) { + drm_for_each_plane(plane, dev) { ret = drm_modeset_lock(&plane->mutex, ctx); if (ret) return ret; diff --git a/drivers/gpu/drm/drm_of.c b/drivers/gpu/drm/drm_of.c index aaa130736bf8..be3884073ea4 100644 --- a/drivers/gpu/drm/drm_of.c +++ b/drivers/gpu/drm/drm_of.c @@ -19,7 +19,7 @@ static uint32_t drm_crtc_port_mask(struct drm_device *dev, unsigned int index = 0; struct drm_crtc *tmp; - list_for_each_entry(tmp, &dev->mode_config.crtc_list, head) { + drm_for_each_crtc(tmp, dev) { if (tmp->port == port) return 1 << index; diff --git a/drivers/gpu/drm/drm_plane_helper.c b/drivers/gpu/drm/drm_plane_helper.c index 2f0ed11024eb..5e5a07af02c8 100644 --- a/drivers/gpu/drm/drm_plane_helper.c +++ b/drivers/gpu/drm/drm_plane_helper.c @@ -91,13 +91,14 @@ static int get_connectors_for_crtc(struct drm_crtc *crtc, */ WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex)); - list_for_each_entry(connector, &dev->mode_config.connector_list, head) + drm_for_each_connector(connector, dev) { if (connector->encoder && connector->encoder->crtc == crtc) { if (connector_list != NULL && count < num_connectors) *(connector_list++) = connector; count++; } + } return count; } @@ -436,7 +437,7 @@ int drm_plane_helper_commit(struct drm_plane *plane, for (i = 0; i < 2; i++) { if (crtc_funcs[i] && crtc_funcs[i]->atomic_begin) - crtc_funcs[i]->atomic_begin(crtc[i]); + crtc_funcs[i]->atomic_begin(crtc[i], crtc[i]->state); } /* @@ -451,7 +452,7 @@ int drm_plane_helper_commit(struct drm_plane *plane, for (i = 0; i < 2; i++) { if (crtc_funcs[i] && crtc_funcs[i]->atomic_flush) - crtc_funcs[i]->atomic_flush(crtc[i]); + crtc_funcs[i]->atomic_flush(crtc[i], crtc[i]->state); } /* @@ -525,10 +526,12 @@ int drm_plane_helper_update(struct drm_plane *plane, struct drm_crtc *crtc, if (plane->funcs->atomic_duplicate_state) plane_state = plane->funcs->atomic_duplicate_state(plane); - else if (plane->state) + else { + if (!plane->state) + drm_atomic_helper_plane_reset(plane); + plane_state = drm_atomic_helper_plane_duplicate_state(plane); - else - plane_state = kzalloc(sizeof(*plane_state), GFP_KERNEL); + } if (!plane_state) return -ENOMEM; plane_state->plane = plane; @@ -572,10 +575,12 @@ int drm_plane_helper_disable(struct drm_plane *plane) if (plane->funcs->atomic_duplicate_state) plane_state = plane->funcs->atomic_duplicate_state(plane); - else if (plane->state) + else { + if (!plane->state) + drm_atomic_helper_plane_reset(plane); + plane_state = drm_atomic_helper_plane_duplicate_state(plane); - else - plane_state = kzalloc(sizeof(*plane_state), GFP_KERNEL); + } if (!plane_state) return -ENOMEM; plane_state->plane = plane; diff --git a/drivers/gpu/drm/drm_probe_helper.c b/drivers/gpu/drm/drm_probe_helper.c index 04203c0d2ecb..d734780b31c0 100644 --- a/drivers/gpu/drm/drm_probe_helper.c +++ b/drivers/gpu/drm/drm_probe_helper.c @@ -93,6 +93,27 @@ static int drm_helper_probe_add_cmdline_mode(struct drm_connector *connector) return 1; } +#define DRM_OUTPUT_POLL_PERIOD (10*HZ) +static void __drm_kms_helper_poll_enable(struct drm_device *dev) +{ + bool poll = false; + struct drm_connector *connector; + + WARN_ON(!mutex_is_locked(&dev->mode_config.mutex)); + + if (!dev->mode_config.poll_enabled || !drm_kms_helper_poll) + return; + + drm_for_each_connector(connector, dev) { + if (connector->polled & (DRM_CONNECTOR_POLL_CONNECT | + DRM_CONNECTOR_POLL_DISCONNECT)) + poll = true; + } + + if (poll) + schedule_delayed_work(&dev->mode_config.output_poll_work, DRM_OUTPUT_POLL_PERIOD); +} + static int drm_helper_probe_single_connector_modes_merge_bits(struct drm_connector *connector, uint32_t maxX, uint32_t maxY, bool merge_type_bits) { @@ -153,7 +174,7 @@ static int drm_helper_probe_single_connector_modes_merge_bits(struct drm_connect /* Re-enable polling in case the global poll config changed. */ if (drm_kms_helper_poll != dev->mode_config.poll_running) - drm_kms_helper_poll_enable(dev); + __drm_kms_helper_poll_enable(dev); dev->mode_config.poll_running = drm_kms_helper_poll; @@ -295,7 +316,6 @@ void drm_kms_helper_hotplug_event(struct drm_device *dev) } EXPORT_SYMBOL(drm_kms_helper_hotplug_event); -#define DRM_OUTPUT_POLL_PERIOD (10*HZ) static void output_poll_execute(struct work_struct *work) { struct delayed_work *delayed_work = to_delayed_work(work); @@ -312,7 +332,7 @@ static void output_poll_execute(struct work_struct *work) goto out; mutex_lock(&dev->mode_config.mutex); - list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + drm_for_each_connector(connector, dev) { /* Ignore forced connectors. */ if (connector->force) @@ -407,20 +427,9 @@ EXPORT_SYMBOL(drm_kms_helper_poll_disable); */ void drm_kms_helper_poll_enable(struct drm_device *dev) { - bool poll = false; - struct drm_connector *connector; - - if (!dev->mode_config.poll_enabled || !drm_kms_helper_poll) - return; - - list_for_each_entry(connector, &dev->mode_config.connector_list, head) { - if (connector->polled & (DRM_CONNECTOR_POLL_CONNECT | - DRM_CONNECTOR_POLL_DISCONNECT)) - poll = true; - } - - if (poll) - schedule_delayed_work(&dev->mode_config.output_poll_work, DRM_OUTPUT_POLL_PERIOD); + mutex_lock(&dev->mode_config.mutex); + __drm_kms_helper_poll_enable(dev); + mutex_unlock(&dev->mode_config.mutex); } EXPORT_SYMBOL(drm_kms_helper_poll_enable); @@ -495,7 +504,7 @@ bool drm_helper_hpd_irq_event(struct drm_device *dev) return false; mutex_lock(&dev->mode_config.mutex); - list_for_each_entry(connector, &dev->mode_config.connector_list, head) { + drm_for_each_connector(connector, dev) { /* Only handle HPD capable connectors. */ if (!(connector->polled & DRM_CONNECTOR_POLL_HPD)) diff --git a/drivers/gpu/drm/exynos/Kconfig b/drivers/gpu/drm/exynos/Kconfig index 43003c4ad80b..bd1a4156f647 100644 --- a/drivers/gpu/drm/exynos/Kconfig +++ b/drivers/gpu/drm/exynos/Kconfig @@ -56,7 +56,7 @@ config DRM_EXYNOS_DSI config DRM_EXYNOS_DP bool "EXYNOS DRM DP driver support" - depends on DRM_EXYNOS && (DRM_EXYNOS_FIMD || DRM_EXYNOS7_DECON) && (DRM_PTN3460=n || DRM_PTN3460=y || DRM_PTN3460=DRM_EXYNOS) + depends on DRM_EXYNOS && (DRM_EXYNOS_FIMD || DRM_EXYNOS7_DECON) default DRM_EXYNOS select DRM_PANEL help @@ -77,6 +77,7 @@ config DRM_EXYNOS_VIDI config DRM_EXYNOS_G2D bool "Exynos DRM G2D" depends on DRM_EXYNOS && !VIDEO_SAMSUNG_S5P_G2D + select FRAME_VECTOR help Choose this option if you want to use Exynos G2D for DRM. diff --git a/drivers/gpu/drm/exynos/Makefile b/drivers/gpu/drm/exynos/Makefile index 7de0b1084fcd..02aecfed6354 100644 --- a/drivers/gpu/drm/exynos/Makefile +++ b/drivers/gpu/drm/exynos/Makefile @@ -3,10 +3,9 @@ # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher. ccflags-y := -Iinclude/drm -Idrivers/gpu/drm/exynos -exynosdrm-y := exynos_drm_drv.o exynos_drm_encoder.o \ - exynos_drm_crtc.o exynos_drm_fbdev.o exynos_drm_fb.o \ - exynos_drm_buf.o exynos_drm_gem.o exynos_drm_core.o \ - exynos_drm_plane.o exynos_drm_dmabuf.o +exynosdrm-y := exynos_drm_drv.o exynos_drm_crtc.o exynos_drm_fbdev.o \ + exynos_drm_fb.o exynos_drm_gem.o exynos_drm_core.o \ + exynos_drm_plane.o exynosdrm-$(CONFIG_DRM_EXYNOS_IOMMU) += exynos_drm_iommu.o exynosdrm-$(CONFIG_DRM_EXYNOS_FIMD) += exynos_drm_fimd.o diff --git a/drivers/gpu/drm/exynos/exynos5433_drm_decon.c b/drivers/gpu/drm/exynos/exynos5433_drm_decon.c index 8b1225f245fc..b3c730770b0f 100644 --- a/drivers/gpu/drm/exynos/exynos5433_drm_decon.c +++ b/drivers/gpu/drm/exynos/exynos5433_drm_decon.c @@ -54,6 +54,13 @@ static const char * const decon_clks_name[] = { "sclk_decon_eclk", }; +static const uint32_t decon_formats[] = { + DRM_FORMAT_XRGB1555, + DRM_FORMAT_RGB565, + DRM_FORMAT_XRGB8888, + DRM_FORMAT_ARGB8888, +}; + static int decon_enable_vblank(struct exynos_drm_crtc *crtc) { struct decon_context *ctx = crtc->ctx; @@ -152,15 +159,15 @@ static void decon_commit(struct exynos_drm_crtc *crtc) #define OFFSIZE(x) (((x) & 0x3fff) << 14) #define PAGEWIDTH(x) ((x) & 0x3fff) -static void decon_win_set_pixfmt(struct decon_context *ctx, unsigned int win) +static void decon_win_set_pixfmt(struct decon_context *ctx, unsigned int win, + struct drm_framebuffer *fb) { - struct exynos_drm_plane *plane = &ctx->planes[win]; unsigned long val; val = readl(ctx->addr + DECON_WINCONx(win)); val &= ~WINCONx_BPPMODE_MASK; - switch (plane->pixel_format) { + switch (fb->pixel_format) { case DRM_FORMAT_XRGB1555: val |= WINCONx_BPPMODE_16BPP_I1555; val |= WINCONx_HAWSWP_F; @@ -186,7 +193,7 @@ static void decon_win_set_pixfmt(struct decon_context *ctx, unsigned int win) return; } - DRM_DEBUG_KMS("bpp = %u\n", plane->bpp); + DRM_DEBUG_KMS("bpp = %u\n", fb->bits_per_pixel); /* * In case of exynos, setting dma-burst to 16Word causes permanent @@ -196,7 +203,7 @@ static void decon_win_set_pixfmt(struct decon_context *ctx, unsigned int win) * movement causes unstable DMA which results into iommu crash/tear. */ - if (plane->fb_width < MIN_FB_WIDTH_FOR_16WORD_BURST) { + if (fb->width < MIN_FB_WIDTH_FOR_16WORD_BURST) { val &= ~WINCONx_BURSTLEN_MASK; val |= WINCONx_BURSTLEN_8WORD; } @@ -219,27 +226,35 @@ static void decon_shadow_protect_win(struct decon_context *ctx, int win, writel(val, ctx->addr + DECON_SHADOWCON); } -static void decon_win_commit(struct exynos_drm_crtc *crtc, unsigned int win) +static void decon_atomic_begin(struct exynos_drm_crtc *crtc, + struct exynos_drm_plane *plane) { struct decon_context *ctx = crtc->ctx; - struct exynos_drm_plane *plane; - u32 val; - - if (win < 0 || win >= WINDOWS_NR) - return; - - plane = &ctx->planes[win]; if (ctx->suspended) return; - decon_shadow_protect_win(ctx, win, true); + decon_shadow_protect_win(ctx, plane->zpos, true); +} + +static void decon_update_plane(struct exynos_drm_crtc *crtc, + struct exynos_drm_plane *plane) +{ + struct decon_context *ctx = crtc->ctx; + struct drm_plane_state *state = plane->base.state; + unsigned int win = plane->zpos; + unsigned int bpp = state->fb->bits_per_pixel >> 3; + unsigned int pitch = state->fb->pitches[0]; + u32 val; + + if (ctx->suspended) + return; val = COORDINATE_X(plane->crtc_x) | COORDINATE_Y(plane->crtc_y); writel(val, ctx->addr + DECON_VIDOSDxA(win)); - val = COORDINATE_X(plane->crtc_x + plane->crtc_width - 1) | - COORDINATE_Y(plane->crtc_y + plane->crtc_height - 1); + val = COORDINATE_X(plane->crtc_x + plane->crtc_w - 1) | + COORDINATE_Y(plane->crtc_y + plane->crtc_h - 1); writel(val, ctx->addr + DECON_VIDOSDxB(win)); val = VIDOSD_Wx_ALPHA_R_F(0x0) | VIDOSD_Wx_ALPHA_G_F(0x0) | @@ -252,42 +267,33 @@ static void decon_win_commit(struct exynos_drm_crtc *crtc, unsigned int win) writel(plane->dma_addr[0], ctx->addr + DECON_VIDW0xADD0B0(win)); - val = plane->dma_addr[0] + plane->pitch * plane->crtc_height; + val = plane->dma_addr[0] + pitch * plane->crtc_h; writel(val, ctx->addr + DECON_VIDW0xADD1B0(win)); - val = OFFSIZE(plane->pitch - plane->crtc_width * (plane->bpp >> 3)) - | PAGEWIDTH(plane->crtc_width * (plane->bpp >> 3)); + val = OFFSIZE(pitch - plane->crtc_w * bpp) + | PAGEWIDTH(plane->crtc_w * bpp); writel(val, ctx->addr + DECON_VIDW0xADD2(win)); - decon_win_set_pixfmt(ctx, win); + decon_win_set_pixfmt(ctx, win, state->fb); /* window enable */ val = readl(ctx->addr + DECON_WINCONx(win)); val |= WINCONx_ENWIN_F; writel(val, ctx->addr + DECON_WINCONx(win)); - decon_shadow_protect_win(ctx, win, false); - /* standalone update */ val = readl(ctx->addr + DECON_UPDATE); val |= STANDALONE_UPDATE_F; writel(val, ctx->addr + DECON_UPDATE); - - if (ctx->i80_if) - atomic_set(&ctx->win_updated, 1); } -static void decon_win_disable(struct exynos_drm_crtc *crtc, unsigned int win) +static void decon_disable_plane(struct exynos_drm_crtc *crtc, + struct exynos_drm_plane *plane) { struct decon_context *ctx = crtc->ctx; - struct exynos_drm_plane *plane; + unsigned int win = plane->zpos; u32 val; - if (win < 0 || win >= WINDOWS_NR) - return; - - plane = &ctx->planes[win]; - if (ctx->suspended) return; @@ -306,6 +312,20 @@ static void decon_win_disable(struct exynos_drm_crtc *crtc, unsigned int win) writel(val, ctx->addr + DECON_UPDATE); } +static void decon_atomic_flush(struct exynos_drm_crtc *crtc, + struct exynos_drm_plane *plane) +{ + struct decon_context *ctx = crtc->ctx; + + if (ctx->suspended) + return; + + decon_shadow_protect_win(ctx, plane->zpos, false); + + if (ctx->i80_if) + atomic_set(&ctx->win_updated, 1); +} + static void decon_swreset(struct decon_context *ctx) { unsigned int tries; @@ -378,7 +398,7 @@ static void decon_disable(struct exynos_drm_crtc *crtc) * a destroyed buffer later. */ for (i = 0; i < WINDOWS_NR; i++) - decon_win_disable(crtc, i); + decon_disable_plane(crtc, &ctx->planes[i]); decon_swreset(ctx); @@ -407,7 +427,7 @@ void decon_te_irq_handler(struct exynos_drm_crtc *crtc) writel(val, ctx->addr + DECON_TRIGCON); } - drm_handle_vblank(ctx->drm_dev, ctx->pipe); + drm_crtc_handle_vblank(&ctx->crtc->base); } static void decon_clear_channels(struct exynos_drm_crtc *crtc) @@ -460,10 +480,11 @@ static struct exynos_drm_crtc_ops decon_crtc_ops = { .enable_vblank = decon_enable_vblank, .disable_vblank = decon_disable_vblank, .commit = decon_commit, - .win_commit = decon_win_commit, - .win_disable = decon_win_disable, + .atomic_begin = decon_atomic_begin, + .update_plane = decon_update_plane, + .disable_plane = decon_disable_plane, + .atomic_flush = decon_atomic_flush, .te_handler = decon_te_irq_handler, - .clear_channels = decon_clear_channels, }; static int decon_bind(struct device *dev, struct device *master, void *data) @@ -483,7 +504,8 @@ static int decon_bind(struct device *dev, struct device *master, void *data) type = (zpos == ctx->default_win) ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY; ret = exynos_plane_init(drm_dev, &ctx->planes[zpos], - 1 << ctx->pipe, type, zpos); + 1 << ctx->pipe, type, decon_formats, + ARRAY_SIZE(decon_formats), zpos); if (ret) return ret; } @@ -497,7 +519,9 @@ static int decon_bind(struct device *dev, struct device *master, void *data) goto err; } - ret = drm_iommu_attach_device_if_possible(ctx->crtc, drm_dev, dev); + decon_clear_channels(ctx->crtc); + + ret = drm_iommu_attach_device(drm_dev, dev); if (ret) goto err; @@ -514,8 +538,7 @@ static void decon_unbind(struct device *dev, struct device *master, void *data) decon_disable(ctx->crtc); /* detach this sub driver from iommu mapping if supported. */ - if (is_drm_iommu_supported(ctx->drm_dev)) - drm_iommu_detach_device(ctx->drm_dev, ctx->dev); + drm_iommu_detach_device(ctx->drm_dev, ctx->dev); } static const struct component_ops decon_component_ops = { @@ -533,7 +556,7 @@ static irqreturn_t decon_vsync_irq_handler(int irq, void *dev_id) val = readl(ctx->addr + DECON_VIDINTCON1); if (val & VIDINTCON1_INTFRMPEND) { - drm_handle_vblank(ctx->drm_dev, ctx->pipe); + drm_crtc_handle_vblank(&ctx->crtc->base); /* clear */ writel(VIDINTCON1_INTFRMPEND, ctx->addr + DECON_VIDINTCON1); @@ -547,13 +570,21 @@ static irqreturn_t decon_lcd_sys_irq_handler(int irq, void *dev_id) { struct decon_context *ctx = dev_id; u32 val; + int win; if (!test_bit(BIT_CLKS_ENABLED, &ctx->enabled)) goto out; val = readl(ctx->addr + DECON_VIDINTCON1); if (val & VIDINTCON1_INTFRMDONEPEND) { - exynos_drm_crtc_finish_pageflip(ctx->drm_dev, ctx->pipe); + for (win = 0 ; win < WINDOWS_NR ; win++) { + struct exynos_drm_plane *plane = &ctx->planes[win]; + + if (!plane->pending_fb) + continue; + + exynos_drm_crtc_finish_update(ctx->crtc, plane); + } /* clear */ writel(VIDINTCON1_INTFRMDONEPEND, diff --git a/drivers/gpu/drm/exynos/exynos7_drm_decon.c b/drivers/gpu/drm/exynos/exynos7_drm_decon.c index 362532afd1a5..cbdb78ef3bac 100644 --- a/drivers/gpu/drm/exynos/exynos7_drm_decon.c +++ b/drivers/gpu/drm/exynos/exynos7_drm_decon.c @@ -61,7 +61,7 @@ struct decon_context { atomic_t wait_vsync_event; struct exynos_drm_panel_info panel; - struct exynos_drm_display *display; + struct drm_encoder *encoder; }; static const struct of_device_id decon_driver_dt_match[] = { @@ -70,6 +70,18 @@ static const struct of_device_id decon_driver_dt_match[] = { }; MODULE_DEVICE_TABLE(of, decon_driver_dt_match); +static const uint32_t decon_formats[] = { + DRM_FORMAT_RGB565, + DRM_FORMAT_XRGB8888, + DRM_FORMAT_XBGR8888, + DRM_FORMAT_RGBX8888, + DRM_FORMAT_BGRX8888, + DRM_FORMAT_ARGB8888, + DRM_FORMAT_ABGR8888, + DRM_FORMAT_RGBA8888, + DRM_FORMAT_BGRA8888, +}; + static void decon_wait_for_vblank(struct exynos_drm_crtc *crtc) { struct decon_context *ctx = crtc->ctx; @@ -126,7 +138,9 @@ static int decon_ctx_initialize(struct decon_context *ctx, ctx->drm_dev = drm_dev; ctx->pipe = priv->pipe++; - ret = drm_iommu_attach_device_if_possible(ctx->crtc, drm_dev, ctx->dev); + decon_clear_channels(ctx->crtc); + + ret = drm_iommu_attach_device(drm_dev, ctx->dev); if (ret) priv->pipe--; @@ -136,8 +150,7 @@ static int decon_ctx_initialize(struct decon_context *ctx, static void decon_ctx_remove(struct decon_context *ctx) { /* detach this sub driver from iommu mapping if supported. */ - if (is_drm_iommu_supported(ctx->drm_dev)) - drm_iommu_detach_device(ctx->drm_dev, ctx->dev); + drm_iommu_detach_device(ctx->drm_dev, ctx->dev); } static u32 decon_calc_clkdiv(struct decon_context *ctx, @@ -271,16 +284,16 @@ static void decon_disable_vblank(struct exynos_drm_crtc *crtc) } } -static void decon_win_set_pixfmt(struct decon_context *ctx, unsigned int win) +static void decon_win_set_pixfmt(struct decon_context *ctx, unsigned int win, + struct drm_framebuffer *fb) { - struct exynos_drm_plane *plane = &ctx->planes[win]; unsigned long val; int padding; val = readl(ctx->regs + WINCON(win)); val &= ~WINCONx_BPPMODE_MASK; - switch (plane->pixel_format) { + switch (fb->pixel_format) { case DRM_FORMAT_RGB565: val |= WINCONx_BPPMODE_16BPP_565; val |= WINCONx_BURSTLEN_16WORD; @@ -329,7 +342,7 @@ static void decon_win_set_pixfmt(struct decon_context *ctx, unsigned int win) break; } - DRM_DEBUG_KMS("bpp = %d\n", plane->bpp); + DRM_DEBUG_KMS("bpp = %d\n", fb->bits_per_pixel); /* * In case of exynos, setting dma-burst to 16Word causes permanent @@ -339,8 +352,8 @@ static void decon_win_set_pixfmt(struct decon_context *ctx, unsigned int win) * movement causes unstable DMA which results into iommu crash/tear. */ - padding = (plane->pitch / (plane->bpp >> 3)) - plane->fb_width; - if (plane->fb_width + padding < MIN_FB_WIDTH_FOR_16WORD_BURST) { + padding = (fb->pitches[0] / (fb->bits_per_pixel >> 3)) - fb->width; + if (fb->width + padding < MIN_FB_WIDTH_FOR_16WORD_BURST) { val &= ~WINCONx_BURSTLEN_MASK; val |= WINCONx_BURSTLEN_8WORD; } @@ -382,23 +395,30 @@ static void decon_shadow_protect_win(struct decon_context *ctx, writel(val, ctx->regs + SHADOWCON); } -static void decon_win_commit(struct exynos_drm_crtc *crtc, unsigned int win) +static void decon_atomic_begin(struct exynos_drm_crtc *crtc, + struct exynos_drm_plane *plane) { struct decon_context *ctx = crtc->ctx; - struct drm_display_mode *mode = &crtc->base.state->adjusted_mode; - struct exynos_drm_plane *plane; - int padding; - unsigned long val, alpha; - unsigned int last_x; - unsigned int last_y; if (ctx->suspended) return; - if (win < 0 || win >= WINDOWS_NR) - return; + decon_shadow_protect_win(ctx, plane->zpos, true); +} - plane = &ctx->planes[win]; +static void decon_update_plane(struct exynos_drm_crtc *crtc, + struct exynos_drm_plane *plane) +{ + struct decon_context *ctx = crtc->ctx; + struct drm_display_mode *mode = &crtc->base.state->adjusted_mode; + struct drm_plane_state *state = plane->base.state; + int padding; + unsigned long val, alpha; + unsigned int last_x; + unsigned int last_y; + unsigned int win = plane->zpos; + unsigned int bpp = state->fb->bits_per_pixel >> 3; + unsigned int pitch = state->fb->pitches[0]; if (ctx->suspended) return; @@ -413,18 +433,15 @@ static void decon_win_commit(struct exynos_drm_crtc *crtc, unsigned int win) * is set. */ - /* protect windows */ - decon_shadow_protect_win(ctx, win, true); - /* buffer start address */ val = (unsigned long)plane->dma_addr[0]; writel(val, ctx->regs + VIDW_BUF_START(win)); - padding = (plane->pitch / (plane->bpp >> 3)) - plane->fb_width; + padding = (pitch / bpp) - state->fb->width; /* buffer size */ - writel(plane->fb_width + padding, ctx->regs + VIDW_WHOLE_X(win)); - writel(plane->fb_height, ctx->regs + VIDW_WHOLE_Y(win)); + writel(state->fb->width + padding, ctx->regs + VIDW_WHOLE_X(win)); + writel(state->fb->height, ctx->regs + VIDW_WHOLE_Y(win)); /* offset from the start of the buffer to read */ writel(plane->src_x, ctx->regs + VIDW_OFFSET_X(win)); @@ -433,25 +450,25 @@ static void decon_win_commit(struct exynos_drm_crtc *crtc, unsigned int win) DRM_DEBUG_KMS("start addr = 0x%lx\n", (unsigned long)val); DRM_DEBUG_KMS("ovl_width = %d, ovl_height = %d\n", - plane->crtc_width, plane->crtc_height); + plane->crtc_w, plane->crtc_h); /* * OSD position. * In case the window layout goes of LCD layout, DECON fails. */ - if ((plane->crtc_x + plane->crtc_width) > mode->hdisplay) - plane->crtc_x = mode->hdisplay - plane->crtc_width; - if ((plane->crtc_y + plane->crtc_height) > mode->vdisplay) - plane->crtc_y = mode->vdisplay - plane->crtc_height; + if ((plane->crtc_x + plane->crtc_w) > mode->hdisplay) + plane->crtc_x = mode->hdisplay - plane->crtc_w; + if ((plane->crtc_y + plane->crtc_h) > mode->vdisplay) + plane->crtc_y = mode->vdisplay - plane->crtc_h; val = VIDOSDxA_TOPLEFT_X(plane->crtc_x) | VIDOSDxA_TOPLEFT_Y(plane->crtc_y); writel(val, ctx->regs + VIDOSD_A(win)); - last_x = plane->crtc_x + plane->crtc_width; + last_x = plane->crtc_x + plane->crtc_w; if (last_x) last_x--; - last_y = plane->crtc_y + plane->crtc_height; + last_y = plane->crtc_y + plane->crtc_h; if (last_y) last_y--; @@ -475,7 +492,7 @@ static void decon_win_commit(struct exynos_drm_crtc *crtc, unsigned int win) writel(alpha, ctx->regs + VIDOSD_D(win)); - decon_win_set_pixfmt(ctx, win); + decon_win_set_pixfmt(ctx, win, state->fb); /* hardware window 0 doesn't support color key. */ if (win != 0) @@ -495,17 +512,13 @@ static void decon_win_commit(struct exynos_drm_crtc *crtc, unsigned int win) writel(val, ctx->regs + DECON_UPDATE); } -static void decon_win_disable(struct exynos_drm_crtc *crtc, unsigned int win) +static void decon_disable_plane(struct exynos_drm_crtc *crtc, + struct exynos_drm_plane *plane) { struct decon_context *ctx = crtc->ctx; - struct exynos_drm_plane *plane; + unsigned int win = plane->zpos; u32 val; - if (win < 0 || win >= WINDOWS_NR) - return; - - plane = &ctx->planes[win]; - if (ctx->suspended) return; @@ -517,14 +530,22 @@ static void decon_win_disable(struct exynos_drm_crtc *crtc, unsigned int win) val &= ~WINCONx_ENWIN; writel(val, ctx->regs + WINCON(win)); - /* unprotect windows */ - decon_shadow_protect_win(ctx, win, false); - val = readl(ctx->regs + DECON_UPDATE); val |= DECON_UPDATE_STANDALONE_F; writel(val, ctx->regs + DECON_UPDATE); } +static void decon_atomic_flush(struct exynos_drm_crtc *crtc, + struct exynos_drm_plane *plane) +{ + struct decon_context *ctx = crtc->ctx; + + if (ctx->suspended) + return; + + decon_shadow_protect_win(ctx, plane->zpos, false); +} + static void decon_init(struct decon_context *ctx) { u32 val; @@ -601,7 +622,7 @@ static void decon_disable(struct exynos_drm_crtc *crtc) * a destroyed buffer later. */ for (i = 0; i < WINDOWS_NR; i++) - decon_win_disable(crtc, i); + decon_disable_plane(crtc, &ctx->planes[i]); clk_disable_unprepare(ctx->vclk); clk_disable_unprepare(ctx->eclk); @@ -621,9 +642,10 @@ static const struct exynos_drm_crtc_ops decon_crtc_ops = { .enable_vblank = decon_enable_vblank, .disable_vblank = decon_disable_vblank, .wait_for_vblank = decon_wait_for_vblank, - .win_commit = decon_win_commit, - .win_disable = decon_win_disable, - .clear_channels = decon_clear_channels, + .atomic_begin = decon_atomic_begin, + .update_plane = decon_update_plane, + .disable_plane = decon_disable_plane, + .atomic_flush = decon_atomic_flush, }; @@ -631,6 +653,7 @@ static irqreturn_t decon_irq_handler(int irq, void *dev_id) { struct decon_context *ctx = (struct decon_context *)dev_id; u32 val, clear_bit; + int win; val = readl(ctx->regs + VIDINTCON1); @@ -643,8 +666,15 @@ static irqreturn_t decon_irq_handler(int irq, void *dev_id) goto out; if (!ctx->i80_if) { - drm_handle_vblank(ctx->drm_dev, ctx->pipe); - exynos_drm_crtc_finish_pageflip(ctx->drm_dev, ctx->pipe); + drm_crtc_handle_vblank(&ctx->crtc->base); + for (win = 0 ; win < WINDOWS_NR ; win++) { + struct exynos_drm_plane *plane = &ctx->planes[win]; + + if (!plane->pending_fb) + continue; + + exynos_drm_crtc_finish_update(ctx->crtc, plane); + } /* set wait vsync event to zero and wake up queue. */ if (atomic_read(&ctx->wait_vsync_event)) { @@ -675,7 +705,8 @@ static int decon_bind(struct device *dev, struct device *master, void *data) type = (zpos == ctx->default_win) ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY; ret = exynos_plane_init(drm_dev, &ctx->planes[zpos], - 1 << ctx->pipe, type, zpos); + 1 << ctx->pipe, type, decon_formats, + ARRAY_SIZE(decon_formats), zpos); if (ret) return ret; } @@ -689,8 +720,8 @@ static int decon_bind(struct device *dev, struct device *master, void *data) return PTR_ERR(ctx->crtc); } - if (ctx->display) - exynos_drm_create_enc_conn(drm_dev, ctx->display); + if (ctx->encoder) + exynos_dpi_bind(drm_dev, ctx->encoder); return 0; @@ -703,8 +734,8 @@ static void decon_unbind(struct device *dev, struct device *master, decon_disable(ctx->crtc); - if (ctx->display) - exynos_dpi_remove(ctx->display); + if (ctx->encoder) + exynos_dpi_remove(ctx->encoder); decon_ctx_remove(ctx); } @@ -789,9 +820,9 @@ static int decon_probe(struct platform_device *pdev) platform_set_drvdata(pdev, ctx); - ctx->display = exynos_dpi_probe(dev); - if (IS_ERR(ctx->display)) { - ret = PTR_ERR(ctx->display); + ctx->encoder = exynos_dpi_probe(dev); + if (IS_ERR(ctx->encoder)) { + ret = PTR_ERR(ctx->encoder); goto err_iounmap; } diff --git a/drivers/gpu/drm/exynos/exynos_dp_core.c b/drivers/gpu/drm/exynos/exynos_dp_core.c index 172b8002a2c8..d66ade0efac8 100644 --- a/drivers/gpu/drm/exynos/exynos_dp_core.c +++ b/drivers/gpu/drm/exynos/exynos_dp_core.c @@ -32,19 +32,20 @@ #include #include "exynos_dp_core.h" +#include "exynos_drm_crtc.h" #define ctx_from_connector(c) container_of(c, struct exynos_dp_device, \ connector) static inline struct exynos_drm_crtc *dp_to_crtc(struct exynos_dp_device *dp) { - return to_exynos_crtc(dp->encoder->crtc); + return to_exynos_crtc(dp->encoder.crtc); } -static inline struct exynos_dp_device * -display_to_dp(struct exynos_drm_display *d) +static inline struct exynos_dp_device *encoder_to_dp( + struct drm_encoder *e) { - return container_of(d, struct exynos_dp_device, display); + return container_of(e, struct exynos_dp_device, encoder); } struct bridge_init { @@ -795,9 +796,6 @@ static int exynos_dp_config_video(struct exynos_dp_device *dp) /* Configure video slave mode */ exynos_dp_enable_video_master(dp, 0); - /* Enable video */ - exynos_dp_start_video(dp); - timeout_loop = 0; for (;;) { @@ -891,9 +889,9 @@ static void exynos_dp_hotplug(struct work_struct *work) drm_helper_hpd_irq_event(dp->drm_dev); } -static void exynos_dp_commit(struct exynos_drm_display *display) +static void exynos_dp_commit(struct drm_encoder *encoder) { - struct exynos_dp_device *dp = display_to_dp(display); + struct exynos_dp_device *dp = encoder_to_dp(encoder); int ret; /* Keep the panel disabled while we configure video */ @@ -938,6 +936,9 @@ static void exynos_dp_commit(struct exynos_drm_display *display) if (drm_panel_enable(dp->panel)) DRM_ERROR("failed to enable the panel\n"); } + + /* Enable video */ + exynos_dp_start_video(dp); } static enum drm_connector_status exynos_dp_detect( @@ -994,7 +995,7 @@ static struct drm_encoder *exynos_dp_best_encoder( { struct exynos_dp_device *dp = ctx_from_connector(connector); - return dp->encoder; + return &dp->encoder; } static struct drm_connector_helper_funcs exynos_dp_connector_helper_funcs = { @@ -1019,15 +1020,12 @@ static int exynos_drm_attach_lcd_bridge(struct exynos_dp_device *dp, return 0; } -static int exynos_dp_create_connector(struct exynos_drm_display *display, - struct drm_encoder *encoder) +static int exynos_dp_create_connector(struct drm_encoder *encoder) { - struct exynos_dp_device *dp = display_to_dp(display); + struct exynos_dp_device *dp = encoder_to_dp(encoder); struct drm_connector *connector = &dp->connector; int ret; - dp->encoder = encoder; - /* Pre-empt DP connector creation if there's a bridge */ if (dp->bridge) { ret = exynos_drm_attach_lcd_bridge(dp, encoder); @@ -1054,20 +1052,22 @@ static int exynos_dp_create_connector(struct exynos_drm_display *display, return ret; } -static void exynos_dp_phy_init(struct exynos_dp_device *dp) +static bool exynos_dp_mode_fixup(struct drm_encoder *encoder, + const struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode) { - if (dp->phy) - phy_power_on(dp->phy); + return true; } -static void exynos_dp_phy_exit(struct exynos_dp_device *dp) +static void exynos_dp_mode_set(struct drm_encoder *encoder, + struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode) { - if (dp->phy) - phy_power_off(dp->phy); } -static void exynos_dp_poweron(struct exynos_dp_device *dp) +static void exynos_dp_enable(struct drm_encoder *encoder) { + struct exynos_dp_device *dp = encoder_to_dp(encoder); struct exynos_drm_crtc *crtc = dp_to_crtc(dp); if (dp->dpms_mode == DRM_MODE_DPMS_ON) @@ -1084,14 +1084,17 @@ static void exynos_dp_poweron(struct exynos_dp_device *dp) crtc->ops->clock_enable(dp_to_crtc(dp), true); clk_prepare_enable(dp->clock); - exynos_dp_phy_init(dp); + phy_power_on(dp->phy); exynos_dp_init_dp(dp); enable_irq(dp->irq); - exynos_dp_commit(&dp->display); + exynos_dp_commit(&dp->encoder); + + dp->dpms_mode = DRM_MODE_DPMS_ON; } -static void exynos_dp_poweroff(struct exynos_dp_device *dp) +static void exynos_dp_disable(struct drm_encoder *encoder) { + struct exynos_dp_device *dp = encoder_to_dp(encoder); struct exynos_drm_crtc *crtc = dp_to_crtc(dp); if (dp->dpms_mode != DRM_MODE_DPMS_ON) @@ -1106,7 +1109,7 @@ static void exynos_dp_poweroff(struct exynos_dp_device *dp) disable_irq(dp->irq); flush_work(&dp->hotplug_work); - exynos_dp_phy_exit(dp); + phy_power_off(dp->phy); clk_disable_unprepare(dp->clock); if (crtc->ops->clock_enable) @@ -1116,31 +1119,19 @@ static void exynos_dp_poweroff(struct exynos_dp_device *dp) if (drm_panel_unprepare(dp->panel)) DRM_ERROR("failed to turnoff the panel\n"); } + + dp->dpms_mode = DRM_MODE_DPMS_OFF; } -static void exynos_dp_dpms(struct exynos_drm_display *display, int mode) -{ - struct exynos_dp_device *dp = display_to_dp(display); +static struct drm_encoder_helper_funcs exynos_dp_encoder_helper_funcs = { + .mode_fixup = exynos_dp_mode_fixup, + .mode_set = exynos_dp_mode_set, + .enable = exynos_dp_enable, + .disable = exynos_dp_disable, +}; - switch (mode) { - case DRM_MODE_DPMS_ON: - exynos_dp_poweron(dp); - break; - case DRM_MODE_DPMS_STANDBY: - case DRM_MODE_DPMS_SUSPEND: - case DRM_MODE_DPMS_OFF: - exynos_dp_poweroff(dp); - break; - default: - break; - } - dp->dpms_mode = mode; -} - -static struct exynos_drm_display_ops exynos_dp_display_ops = { - .create_connector = exynos_dp_create_connector, - .dpms = exynos_dp_dpms, - .commit = exynos_dp_commit, +static struct drm_encoder_funcs exynos_dp_encoder_funcs = { + .destroy = drm_encoder_cleanup, }; static struct video_info *exynos_dp_dt_parse_pdata(struct device *dev) @@ -1219,9 +1210,10 @@ static int exynos_dp_bind(struct device *dev, struct device *master, void *data) struct exynos_dp_device *dp = dev_get_drvdata(dev); struct platform_device *pdev = to_platform_device(dev); struct drm_device *drm_dev = data; + struct drm_encoder *encoder = &dp->encoder; struct resource *res; unsigned int irq_flags; - int ret = 0; + int pipe, ret = 0; dp->dev = &pdev->dev; dp->dpms_mode = DRM_MODE_DPMS_OFF; @@ -1297,7 +1289,7 @@ static int exynos_dp_bind(struct device *dev, struct device *master, void *data) INIT_WORK(&dp->hotplug_work, exynos_dp_hotplug); - exynos_dp_phy_init(dp); + phy_power_on(dp->phy); exynos_dp_init_dp(dp); @@ -1311,7 +1303,28 @@ static int exynos_dp_bind(struct device *dev, struct device *master, void *data) dp->drm_dev = drm_dev; - return exynos_drm_create_enc_conn(drm_dev, &dp->display); + pipe = exynos_drm_crtc_get_pipe_from_type(drm_dev, + EXYNOS_DISPLAY_TYPE_LCD); + if (pipe < 0) + return pipe; + + encoder->possible_crtcs = 1 << pipe; + + DRM_DEBUG_KMS("possible_crtcs = 0x%x\n", encoder->possible_crtcs); + + drm_encoder_init(drm_dev, encoder, &exynos_dp_encoder_funcs, + DRM_MODE_ENCODER_TMDS); + + drm_encoder_helper_add(encoder, &exynos_dp_encoder_helper_funcs); + + ret = exynos_dp_create_connector(encoder); + if (ret) { + DRM_ERROR("failed to create connector ret = %d\n", ret); + drm_encoder_cleanup(encoder); + return ret; + } + + return 0; } static void exynos_dp_unbind(struct device *dev, struct device *master, @@ -1319,7 +1332,7 @@ static void exynos_dp_unbind(struct device *dev, struct device *master, { struct exynos_dp_device *dp = dev_get_drvdata(dev); - exynos_dp_dpms(&dp->display, DRM_MODE_DPMS_OFF); + exynos_dp_disable(&dp->encoder); } static const struct component_ops exynos_dp_ops = { @@ -1338,8 +1351,6 @@ static int exynos_dp_probe(struct platform_device *pdev) if (!dp) return -ENOMEM; - dp->display.type = EXYNOS_DISPLAY_TYPE_LCD; - dp->display.ops = &exynos_dp_display_ops; platform_set_drvdata(pdev, dp); panel_node = of_parse_phandle(dev->of_node, "panel", 0); @@ -1377,7 +1388,7 @@ static int exynos_dp_suspend(struct device *dev) { struct exynos_dp_device *dp = dev_get_drvdata(dev); - exynos_dp_dpms(&dp->display, DRM_MODE_DPMS_OFF); + exynos_dp_disable(&dp->encoder); return 0; } @@ -1385,7 +1396,7 @@ static int exynos_dp_resume(struct device *dev) { struct exynos_dp_device *dp = dev_get_drvdata(dev); - exynos_dp_dpms(&dp->display, DRM_MODE_DPMS_ON); + exynos_dp_enable(&dp->encoder); return 0; } #endif diff --git a/drivers/gpu/drm/exynos/exynos_dp_core.h b/drivers/gpu/drm/exynos/exynos_dp_core.h index a4e799679669..e413b6f7b0e7 100644 --- a/drivers/gpu/drm/exynos/exynos_dp_core.h +++ b/drivers/gpu/drm/exynos/exynos_dp_core.h @@ -147,11 +147,10 @@ struct link_train { }; struct exynos_dp_device { - struct exynos_drm_display display; + struct drm_encoder encoder; struct device *dev; struct drm_device *drm_dev; struct drm_connector connector; - struct drm_encoder *encoder; struct drm_panel *panel; struct drm_bridge *bridge; struct clk *clock; diff --git a/drivers/gpu/drm/exynos/exynos_drm_buf.c b/drivers/gpu/drm/exynos/exynos_drm_buf.c deleted file mode 100644 index 24994ba10e28..000000000000 --- a/drivers/gpu/drm/exynos/exynos_drm_buf.c +++ /dev/null @@ -1,186 +0,0 @@ -/* exynos_drm_buf.c - * - * Copyright (c) 2011 Samsung Electronics Co., Ltd. - * Author: Inki Dae - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - */ - -#include -#include - -#include "exynos_drm_drv.h" -#include "exynos_drm_gem.h" -#include "exynos_drm_buf.h" -#include "exynos_drm_iommu.h" - -static int lowlevel_buffer_allocate(struct drm_device *dev, - unsigned int flags, struct exynos_drm_gem_buf *buf) -{ - int ret = 0; - enum dma_attr attr; - unsigned int nr_pages; - - if (buf->dma_addr) { - DRM_DEBUG_KMS("already allocated.\n"); - return 0; - } - - init_dma_attrs(&buf->dma_attrs); - - /* - * if EXYNOS_BO_CONTIG, fully physically contiguous memory - * region will be allocated else physically contiguous - * as possible. - */ - if (!(flags & EXYNOS_BO_NONCONTIG)) - dma_set_attr(DMA_ATTR_FORCE_CONTIGUOUS, &buf->dma_attrs); - - /* - * if EXYNOS_BO_WC or EXYNOS_BO_NONCACHABLE, writecombine mapping - * else cachable mapping. - */ - if (flags & EXYNOS_BO_WC || !(flags & EXYNOS_BO_CACHABLE)) - attr = DMA_ATTR_WRITE_COMBINE; - else - attr = DMA_ATTR_NON_CONSISTENT; - - dma_set_attr(attr, &buf->dma_attrs); - dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &buf->dma_attrs); - - nr_pages = buf->size >> PAGE_SHIFT; - - if (!is_drm_iommu_supported(dev)) { - dma_addr_t start_addr; - unsigned int i = 0; - - buf->pages = drm_calloc_large(nr_pages, sizeof(struct page *)); - if (!buf->pages) { - DRM_ERROR("failed to allocate pages.\n"); - return -ENOMEM; - } - - buf->cookie = dma_alloc_attrs(dev->dev, - buf->size, - &buf->dma_addr, GFP_KERNEL, - &buf->dma_attrs); - if (!buf->cookie) { - DRM_ERROR("failed to allocate buffer.\n"); - ret = -ENOMEM; - goto err_free; - } - - start_addr = buf->dma_addr; - while (i < nr_pages) { - buf->pages[i] = phys_to_page(start_addr); - start_addr += PAGE_SIZE; - i++; - } - } else { - - buf->pages = dma_alloc_attrs(dev->dev, buf->size, - &buf->dma_addr, GFP_KERNEL, - &buf->dma_attrs); - if (!buf->pages) { - DRM_ERROR("failed to allocate buffer.\n"); - return -ENOMEM; - } - } - - buf->sgt = drm_prime_pages_to_sg(buf->pages, nr_pages); - if (IS_ERR(buf->sgt)) { - DRM_ERROR("failed to get sg table.\n"); - ret = PTR_ERR(buf->sgt); - goto err_free_attrs; - } - - DRM_DEBUG_KMS("dma_addr(0x%lx), size(0x%lx)\n", - (unsigned long)buf->dma_addr, - buf->size); - - return ret; - -err_free_attrs: - dma_free_attrs(dev->dev, buf->size, buf->pages, - (dma_addr_t)buf->dma_addr, &buf->dma_attrs); - buf->dma_addr = (dma_addr_t)NULL; -err_free: - if (!is_drm_iommu_supported(dev)) - drm_free_large(buf->pages); - - return ret; -} - -static void lowlevel_buffer_deallocate(struct drm_device *dev, - unsigned int flags, struct exynos_drm_gem_buf *buf) -{ - if (!buf->dma_addr) { - DRM_DEBUG_KMS("dma_addr is invalid.\n"); - return; - } - - DRM_DEBUG_KMS("dma_addr(0x%lx), size(0x%lx)\n", - (unsigned long)buf->dma_addr, - buf->size); - - sg_free_table(buf->sgt); - - kfree(buf->sgt); - buf->sgt = NULL; - - if (!is_drm_iommu_supported(dev)) { - dma_free_attrs(dev->dev, buf->size, buf->cookie, - (dma_addr_t)buf->dma_addr, &buf->dma_attrs); - drm_free_large(buf->pages); - } else - dma_free_attrs(dev->dev, buf->size, buf->pages, - (dma_addr_t)buf->dma_addr, &buf->dma_attrs); - - buf->dma_addr = (dma_addr_t)NULL; -} - -struct exynos_drm_gem_buf *exynos_drm_init_buf(struct drm_device *dev, - unsigned int size) -{ - struct exynos_drm_gem_buf *buffer; - - DRM_DEBUG_KMS("desired size = 0x%x\n", size); - - buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); - if (!buffer) - return NULL; - - buffer->size = size; - return buffer; -} - -void exynos_drm_fini_buf(struct drm_device *dev, - struct exynos_drm_gem_buf *buffer) -{ - kfree(buffer); - buffer = NULL; -} - -int exynos_drm_alloc_buf(struct drm_device *dev, - struct exynos_drm_gem_buf *buf, unsigned int flags) -{ - - /* - * allocate memory region and set the memory information - * to vaddr and dma_addr of a buffer object. - */ - if (lowlevel_buffer_allocate(dev, flags, buf) < 0) - return -ENOMEM; - - return 0; -} - -void exynos_drm_free_buf(struct drm_device *dev, - unsigned int flags, struct exynos_drm_gem_buf *buffer) -{ - - lowlevel_buffer_deallocate(dev, flags, buffer); -} diff --git a/drivers/gpu/drm/exynos/exynos_drm_buf.h b/drivers/gpu/drm/exynos/exynos_drm_buf.h deleted file mode 100644 index a6412f19673c..000000000000 --- a/drivers/gpu/drm/exynos/exynos_drm_buf.h +++ /dev/null @@ -1,33 +0,0 @@ -/* exynos_drm_buf.h - * - * Copyright (c) 2011 Samsung Electronics Co., Ltd. - * Author: Inki Dae - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - */ - -#ifndef _EXYNOS_DRM_BUF_H_ -#define _EXYNOS_DRM_BUF_H_ - -/* create and initialize buffer object. */ -struct exynos_drm_gem_buf *exynos_drm_init_buf(struct drm_device *dev, - unsigned int size); - -/* destroy buffer object. */ -void exynos_drm_fini_buf(struct drm_device *dev, - struct exynos_drm_gem_buf *buffer); - -/* allocate physical memory region and setup sgt. */ -int exynos_drm_alloc_buf(struct drm_device *dev, - struct exynos_drm_gem_buf *buf, - unsigned int flags); - -/* release physical memory region, and sgt. */ -void exynos_drm_free_buf(struct drm_device *dev, - unsigned int flags, - struct exynos_drm_gem_buf *buffer); - -#endif diff --git a/drivers/gpu/drm/exynos/exynos_drm_core.c b/drivers/gpu/drm/exynos/exynos_drm_core.c index 4c9f972eaa07..c68a6a2a9b57 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_core.c +++ b/drivers/gpu/drm/exynos/exynos_drm_core.c @@ -15,46 +15,10 @@ #include #include "exynos_drm_drv.h" #include "exynos_drm_crtc.h" -#include "exynos_drm_encoder.h" #include "exynos_drm_fbdev.h" static LIST_HEAD(exynos_drm_subdrv_list); -int exynos_drm_create_enc_conn(struct drm_device *dev, - struct exynos_drm_display *display) -{ - struct drm_encoder *encoder; - int ret; - unsigned long possible_crtcs = 0; - - ret = exynos_drm_crtc_get_pipe_from_type(dev, display->type); - if (ret < 0) - return ret; - - possible_crtcs |= 1 << ret; - - /* create and initialize a encoder for this sub driver. */ - encoder = exynos_drm_encoder_create(dev, display, possible_crtcs); - if (!encoder) { - DRM_ERROR("failed to create encoder\n"); - return -EFAULT; - } - - display->encoder = encoder; - - ret = display->ops->create_connector(display, encoder); - if (ret) { - DRM_ERROR("failed to create connector ret = %d\n", ret); - goto err_destroy_encoder; - } - - return 0; - -err_destroy_encoder: - encoder->funcs->destroy(encoder); - return ret; -} - int exynos_drm_subdrv_register(struct exynos_drm_subdrv *subdrv) { if (!subdrv) diff --git a/drivers/gpu/drm/exynos/exynos_drm_crtc.c b/drivers/gpu/drm/exynos/exynos_drm_crtc.c index 644b4b76e071..0872aa2f450f 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_crtc.c +++ b/drivers/gpu/drm/exynos/exynos_drm_crtc.c @@ -19,21 +19,15 @@ #include "exynos_drm_crtc.h" #include "exynos_drm_drv.h" -#include "exynos_drm_encoder.h" #include "exynos_drm_plane.h" static void exynos_drm_crtc_enable(struct drm_crtc *crtc) { struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc); - if (exynos_crtc->enabled) - return; - if (exynos_crtc->ops->enable) exynos_crtc->ops->enable(exynos_crtc); - exynos_crtc->enabled = true; - drm_crtc_vblank_on(crtc); } @@ -41,20 +35,10 @@ static void exynos_drm_crtc_disable(struct drm_crtc *crtc) { struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc); - if (!exynos_crtc->enabled) - return; - - /* wait for the completion of page flip. */ - if (!wait_event_timeout(exynos_crtc->pending_flip_queue, - (exynos_crtc->event == NULL), HZ/20)) - exynos_crtc->event = NULL; - drm_crtc_vblank_off(crtc); if (exynos_crtc->ops->disable) exynos_crtc->ops->disable(exynos_crtc); - - exynos_crtc->enabled = false; } static bool @@ -80,18 +64,36 @@ exynos_drm_crtc_mode_set_nofb(struct drm_crtc *crtc) exynos_crtc->ops->commit(exynos_crtc); } -static void exynos_crtc_atomic_begin(struct drm_crtc *crtc) +static void exynos_crtc_atomic_begin(struct drm_crtc *crtc, + struct drm_crtc_state *old_crtc_state) { struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc); + struct drm_plane *plane; - if (crtc->state->event) { - WARN_ON(drm_crtc_vblank_get(crtc) != 0); - exynos_crtc->event = crtc->state->event; + exynos_crtc->event = crtc->state->event; + + drm_atomic_crtc_for_each_plane(plane, crtc) { + struct exynos_drm_plane *exynos_plane = to_exynos_plane(plane); + + if (exynos_crtc->ops->atomic_begin) + exynos_crtc->ops->atomic_begin(exynos_crtc, + exynos_plane); } } -static void exynos_crtc_atomic_flush(struct drm_crtc *crtc) +static void exynos_crtc_atomic_flush(struct drm_crtc *crtc, + struct drm_crtc_state *old_crtc_state) { + struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc); + struct drm_plane *plane; + + drm_atomic_crtc_for_each_plane(plane, crtc) { + struct exynos_drm_plane *exynos_plane = to_exynos_plane(plane); + + if (exynos_crtc->ops->atomic_flush) + exynos_crtc->ops->atomic_flush(exynos_crtc, + exynos_plane); + } } static struct drm_crtc_helper_funcs exynos_crtc_helper_funcs = { @@ -139,13 +141,13 @@ struct exynos_drm_crtc *exynos_drm_crtc_create(struct drm_device *drm_dev, if (!exynos_crtc) return ERR_PTR(-ENOMEM); - init_waitqueue_head(&exynos_crtc->pending_flip_queue); - exynos_crtc->pipe = pipe; exynos_crtc->type = type; exynos_crtc->ops = ops; exynos_crtc->ctx = ctx; + init_waitqueue_head(&exynos_crtc->wait_update); + crtc = &exynos_crtc->base; private->crtc[pipe] = crtc; @@ -171,11 +173,8 @@ int exynos_drm_crtc_enable_vblank(struct drm_device *dev, int pipe) struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(private->crtc[pipe]); - if (!exynos_crtc->enabled) - return -EPERM; - if (exynos_crtc->ops->enable_vblank) - exynos_crtc->ops->enable_vblank(exynos_crtc); + return exynos_crtc->ops->enable_vblank(exynos_crtc); return 0; } @@ -186,31 +185,34 @@ void exynos_drm_crtc_disable_vblank(struct drm_device *dev, int pipe) struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(private->crtc[pipe]); - if (!exynos_crtc->enabled) - return; - if (exynos_crtc->ops->disable_vblank) exynos_crtc->ops->disable_vblank(exynos_crtc); } -void exynos_drm_crtc_finish_pageflip(struct drm_device *dev, int pipe) +void exynos_drm_crtc_wait_pending_update(struct exynos_drm_crtc *exynos_crtc) { - struct exynos_drm_private *dev_priv = dev->dev_private; - struct drm_crtc *drm_crtc = dev_priv->crtc[pipe]; - struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(drm_crtc); + wait_event_timeout(exynos_crtc->wait_update, + (atomic_read(&exynos_crtc->pending_update) == 0), + msecs_to_jiffies(50)); +} + +void exynos_drm_crtc_finish_update(struct exynos_drm_crtc *exynos_crtc, + struct exynos_drm_plane *exynos_plane) +{ + struct drm_crtc *crtc = &exynos_crtc->base; unsigned long flags; - spin_lock_irqsave(&dev->event_lock, flags); - if (exynos_crtc->event) { + exynos_plane->pending_fb = NULL; - drm_send_vblank_event(dev, -1, exynos_crtc->event); - drm_vblank_put(dev, pipe); - wake_up(&exynos_crtc->pending_flip_queue); + if (atomic_dec_and_test(&exynos_crtc->pending_update)) + wake_up(&exynos_crtc->wait_update); - } + spin_lock_irqsave(&crtc->dev->event_lock, flags); + if (exynos_crtc->event) + drm_crtc_send_vblank_event(crtc, exynos_crtc->event); exynos_crtc->event = NULL; - spin_unlock_irqrestore(&dev->event_lock, flags); + spin_unlock_irqrestore(&crtc->dev->event_lock, flags); } void exynos_drm_crtc_complete_scanout(struct drm_framebuffer *fb) @@ -237,7 +239,7 @@ void exynos_drm_crtc_complete_scanout(struct drm_framebuffer *fb) } int exynos_drm_crtc_get_pipe_from_type(struct drm_device *drm_dev, - unsigned int out_type) + enum exynos_drm_output_type out_type) { struct drm_crtc *crtc; diff --git a/drivers/gpu/drm/exynos/exynos_drm_crtc.h b/drivers/gpu/drm/exynos/exynos_drm_crtc.h index 0f3aa70818e3..f87d4abda6f7 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_crtc.h +++ b/drivers/gpu/drm/exynos/exynos_drm_crtc.h @@ -25,12 +25,14 @@ struct exynos_drm_crtc *exynos_drm_crtc_create(struct drm_device *drm_dev, void *context); int exynos_drm_crtc_enable_vblank(struct drm_device *dev, int pipe); void exynos_drm_crtc_disable_vblank(struct drm_device *dev, int pipe); -void exynos_drm_crtc_finish_pageflip(struct drm_device *dev, int pipe); +void exynos_drm_crtc_wait_pending_update(struct exynos_drm_crtc *exynos_crtc); +void exynos_drm_crtc_finish_update(struct exynos_drm_crtc *exynos_crtc, + struct exynos_drm_plane *exynos_plane); void exynos_drm_crtc_complete_scanout(struct drm_framebuffer *fb); /* This function gets pipe value to crtc device matched with out_type. */ int exynos_drm_crtc_get_pipe_from_type(struct drm_device *drm_dev, - unsigned int out_type); + enum exynos_drm_output_type out_type); /* * This function calls the crtc device(manager)'s te_handler() callback diff --git a/drivers/gpu/drm/exynos/exynos_drm_dmabuf.c b/drivers/gpu/drm/exynos/exynos_drm_dmabuf.c deleted file mode 100644 index cd485c091b30..000000000000 --- a/drivers/gpu/drm/exynos/exynos_drm_dmabuf.c +++ /dev/null @@ -1,286 +0,0 @@ -/* exynos_drm_dmabuf.c - * - * Copyright (c) 2012 Samsung Electronics Co., Ltd. - * Author: Inki Dae - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - */ - -#include -#include -#include "exynos_drm_dmabuf.h" -#include "exynos_drm_drv.h" -#include "exynos_drm_gem.h" - -#include - -struct exynos_drm_dmabuf_attachment { - struct sg_table sgt; - enum dma_data_direction dir; - bool is_mapped; -}; - -static struct exynos_drm_gem_obj *dma_buf_to_obj(struct dma_buf *buf) -{ - return to_exynos_gem_obj(buf->priv); -} - -static int exynos_gem_attach_dma_buf(struct dma_buf *dmabuf, - struct device *dev, - struct dma_buf_attachment *attach) -{ - struct exynos_drm_dmabuf_attachment *exynos_attach; - - exynos_attach = kzalloc(sizeof(*exynos_attach), GFP_KERNEL); - if (!exynos_attach) - return -ENOMEM; - - exynos_attach->dir = DMA_NONE; - attach->priv = exynos_attach; - - return 0; -} - -static void exynos_gem_detach_dma_buf(struct dma_buf *dmabuf, - struct dma_buf_attachment *attach) -{ - struct exynos_drm_dmabuf_attachment *exynos_attach = attach->priv; - struct sg_table *sgt; - - if (!exynos_attach) - return; - - sgt = &exynos_attach->sgt; - - if (exynos_attach->dir != DMA_NONE) - dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, - exynos_attach->dir); - - sg_free_table(sgt); - kfree(exynos_attach); - attach->priv = NULL; -} - -static struct sg_table * - exynos_gem_map_dma_buf(struct dma_buf_attachment *attach, - enum dma_data_direction dir) -{ - struct exynos_drm_dmabuf_attachment *exynos_attach = attach->priv; - struct exynos_drm_gem_obj *gem_obj = dma_buf_to_obj(attach->dmabuf); - struct drm_device *dev = gem_obj->base.dev; - struct exynos_drm_gem_buf *buf; - struct scatterlist *rd, *wr; - struct sg_table *sgt = NULL; - unsigned int i; - int nents, ret; - - /* just return current sgt if already requested. */ - if (exynos_attach->dir == dir && exynos_attach->is_mapped) - return &exynos_attach->sgt; - - buf = gem_obj->buffer; - if (!buf) { - DRM_ERROR("buffer is null.\n"); - return ERR_PTR(-ENOMEM); - } - - sgt = &exynos_attach->sgt; - - ret = sg_alloc_table(sgt, buf->sgt->orig_nents, GFP_KERNEL); - if (ret) { - DRM_ERROR("failed to alloc sgt.\n"); - return ERR_PTR(-ENOMEM); - } - - mutex_lock(&dev->struct_mutex); - - rd = buf->sgt->sgl; - wr = sgt->sgl; - for (i = 0; i < sgt->orig_nents; ++i) { - sg_set_page(wr, sg_page(rd), rd->length, rd->offset); - rd = sg_next(rd); - wr = sg_next(wr); - } - - if (dir != DMA_NONE) { - nents = dma_map_sg(attach->dev, sgt->sgl, sgt->orig_nents, dir); - if (!nents) { - DRM_ERROR("failed to map sgl with iommu.\n"); - sg_free_table(sgt); - sgt = ERR_PTR(-EIO); - goto err_unlock; - } - } - - exynos_attach->is_mapped = true; - exynos_attach->dir = dir; - attach->priv = exynos_attach; - - DRM_DEBUG_PRIME("buffer size = 0x%lx\n", buf->size); - -err_unlock: - mutex_unlock(&dev->struct_mutex); - return sgt; -} - -static void exynos_gem_unmap_dma_buf(struct dma_buf_attachment *attach, - struct sg_table *sgt, - enum dma_data_direction dir) -{ - /* Nothing to do. */ -} - -static void *exynos_gem_dmabuf_kmap_atomic(struct dma_buf *dma_buf, - unsigned long page_num) -{ - /* TODO */ - - return NULL; -} - -static void exynos_gem_dmabuf_kunmap_atomic(struct dma_buf *dma_buf, - unsigned long page_num, - void *addr) -{ - /* TODO */ -} - -static void *exynos_gem_dmabuf_kmap(struct dma_buf *dma_buf, - unsigned long page_num) -{ - /* TODO */ - - return NULL; -} - -static void exynos_gem_dmabuf_kunmap(struct dma_buf *dma_buf, - unsigned long page_num, void *addr) -{ - /* TODO */ -} - -static int exynos_gem_dmabuf_mmap(struct dma_buf *dma_buf, - struct vm_area_struct *vma) -{ - return -ENOTTY; -} - -static struct dma_buf_ops exynos_dmabuf_ops = { - .attach = exynos_gem_attach_dma_buf, - .detach = exynos_gem_detach_dma_buf, - .map_dma_buf = exynos_gem_map_dma_buf, - .unmap_dma_buf = exynos_gem_unmap_dma_buf, - .kmap = exynos_gem_dmabuf_kmap, - .kmap_atomic = exynos_gem_dmabuf_kmap_atomic, - .kunmap = exynos_gem_dmabuf_kunmap, - .kunmap_atomic = exynos_gem_dmabuf_kunmap_atomic, - .mmap = exynos_gem_dmabuf_mmap, - .release = drm_gem_dmabuf_release, -}; - -struct dma_buf *exynos_dmabuf_prime_export(struct drm_device *drm_dev, - struct drm_gem_object *obj, int flags) -{ - struct exynos_drm_gem_obj *exynos_gem_obj = to_exynos_gem_obj(obj); - DEFINE_DMA_BUF_EXPORT_INFO(exp_info); - - exp_info.ops = &exynos_dmabuf_ops; - exp_info.size = exynos_gem_obj->base.size; - exp_info.flags = flags; - exp_info.priv = obj; - - return dma_buf_export(&exp_info); -} - -struct drm_gem_object *exynos_dmabuf_prime_import(struct drm_device *drm_dev, - struct dma_buf *dma_buf) -{ - struct dma_buf_attachment *attach; - struct sg_table *sgt; - struct scatterlist *sgl; - struct exynos_drm_gem_obj *exynos_gem_obj; - struct exynos_drm_gem_buf *buffer; - int ret; - - /* is this one of own objects? */ - if (dma_buf->ops == &exynos_dmabuf_ops) { - struct drm_gem_object *obj; - - obj = dma_buf->priv; - - /* is it from our device? */ - if (obj->dev == drm_dev) { - /* - * Importing dmabuf exported from out own gem increases - * refcount on gem itself instead of f_count of dmabuf. - */ - drm_gem_object_reference(obj); - return obj; - } - } - - attach = dma_buf_attach(dma_buf, drm_dev->dev); - if (IS_ERR(attach)) - return ERR_PTR(-EINVAL); - - get_dma_buf(dma_buf); - - sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL); - if (IS_ERR(sgt)) { - ret = PTR_ERR(sgt); - goto err_buf_detach; - } - - buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); - if (!buffer) { - ret = -ENOMEM; - goto err_unmap_attach; - } - - exynos_gem_obj = exynos_drm_gem_init(drm_dev, dma_buf->size); - if (!exynos_gem_obj) { - ret = -ENOMEM; - goto err_free_buffer; - } - - sgl = sgt->sgl; - - buffer->size = dma_buf->size; - buffer->dma_addr = sg_dma_address(sgl); - - if (sgt->nents == 1) { - /* always physically continuous memory if sgt->nents is 1. */ - exynos_gem_obj->flags |= EXYNOS_BO_CONTIG; - } else { - /* - * this case could be CONTIG or NONCONTIG type but for now - * sets NONCONTIG. - * TODO. we have to find a way that exporter can notify - * the type of its own buffer to importer. - */ - exynos_gem_obj->flags |= EXYNOS_BO_NONCONTIG; - } - - exynos_gem_obj->buffer = buffer; - buffer->sgt = sgt; - exynos_gem_obj->base.import_attach = attach; - - DRM_DEBUG_PRIME("dma_addr = %pad, size = 0x%lx\n", &buffer->dma_addr, - buffer->size); - - return &exynos_gem_obj->base; - -err_free_buffer: - kfree(buffer); - buffer = NULL; -err_unmap_attach: - dma_buf_unmap_attachment(attach, sgt, DMA_BIDIRECTIONAL); -err_buf_detach: - dma_buf_detach(dma_buf, attach); - dma_buf_put(dma_buf); - - return ERR_PTR(ret); -} diff --git a/drivers/gpu/drm/exynos/exynos_drm_dmabuf.h b/drivers/gpu/drm/exynos/exynos_drm_dmabuf.h deleted file mode 100644 index 886de9ff484d..000000000000 --- a/drivers/gpu/drm/exynos/exynos_drm_dmabuf.h +++ /dev/null @@ -1,20 +0,0 @@ -/* exynos_drm_dmabuf.h - * - * Copyright (c) 2012 Samsung Electronics Co., Ltd. - * Author: Inki Dae - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - */ - -#ifndef _EXYNOS_DRM_DMABUF_H_ -#define _EXYNOS_DRM_DMABUF_H_ - -struct dma_buf *exynos_dmabuf_prime_export(struct drm_device *drm_dev, - struct drm_gem_object *obj, int flags); - -struct drm_gem_object *exynos_dmabuf_prime_import(struct drm_device *drm_dev, - struct dma_buf *dma_buf); -#endif diff --git a/drivers/gpu/drm/exynos/exynos_drm_dpi.c b/drivers/gpu/drm/exynos/exynos_drm_dpi.c index 7cb6595c1894..c748b8790de3 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_dpi.c +++ b/drivers/gpu/drm/exynos/exynos_drm_dpi.c @@ -20,26 +20,24 @@ #include