diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig
index 6555142a41e..a916b711ba8 100644
--- a/drivers/spi/Kconfig
+++ b/drivers/spi/Kconfig
@@ -85,7 +85,7 @@ config ATH79_SPI
 
 config ATMEL_QSPI
 	bool "Atmel Quad SPI Controller"
-	depends on ARCH_AT91
+	depends on ARCH_AT91 && SPI_MEM
 	help
 	  Enable the Atmel Quad SPI controller in master mode. This driver
 	  does not support generic SPI. The implementation supports only the