diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig
index 957c3e065b83..8782514bb89b 100644
--- a/drivers/spi/Kconfig
+++ b/drivers/spi/Kconfig
@@ -321,8 +321,7 @@ config SPI_CADENCE_QUADSPI
 
 config SPI_CADENCE_XSPI
 	tristate "Cadence XSPI controller"
-	depends on HAS_IOMEM || COMPILE_TEST
-	depends on OF
+	depends on OF && HAS_IOMEM && 64BIT
 	depends on SPI_MEM
 	help
 	  Enable support for the Cadence XSPI Flash controller.
diff --git a/drivers/spi/spi-cadence-xspi.c b/drivers/spi/spi-cadence-xspi.c
index e2bfb0c78b82..e1b337789fce 100644
--- a/drivers/spi/spi-cadence-xspi.c
+++ b/drivers/spi/spi-cadence-xspi.c
@@ -22,10 +22,6 @@
 #include <linux/bitrev.h>
 #include <linux/util_macros.h>
 
-#ifndef CONFIG_64BIT
-#include <linux/io-64-nonatomic-lo-hi.h>
-#endif
-
 #define CDNS_XSPI_MAGIC_NUM_VALUE	0x6522
 #define CDNS_XSPI_MAX_BANKS		8
 #define CDNS_XSPI_NAME			"cadence-xspi"